src/freedreno/vulkan/tu_cmd_buffer.c

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * based in part on anv driver which is:
   6  * Copyright © 2015 Intel Corporation
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25  * DEALINGS IN THE SOFTWARE.
  26  */
  27
  28 #include "tu_private.h"
  29
  30 #include "registers/adreno_pm4.xml.h"
  31 #include "registers/adreno_common.xml.h"
  32
  33 #include "vk_format.h"
  34
  35 #include "tu_cs.h"
  36 #include "tu_blit.h"
  37
  38 #define OVERFLOW_FLAG_REG REG_A6XX_CP_SCRATCH_REG(0)
  39
  40 void
  41 tu_bo_list_init(struct tu_bo_list *list)
  42 {
  43    list->count = list->capacity = 0;
  44    list->bo_infos = NULL;
  45 }
  46
  47 void
  48 tu_bo_list_destroy(struct tu_bo_list *list)
  49 {
  50    free(list->bo_infos);
  51 }
  52
  53 void
  54 tu_bo_list_reset(struct tu_bo_list *list)
  55 {
  56    list->count = 0;
  57 }
  58
  59 /**
  60  * \a flags consists of MSM_SUBMIT_BO_FLAGS.
  61  */
  62 static uint32_t
  63 tu_bo_list_add_info(struct tu_bo_list *list,
  64                     const struct drm_msm_gem_submit_bo *bo_info)
  65 {
  66    assert(bo_info->handle != 0);
  67
  68    for (uint32_t i = 0; i < list->count; ++i) {
  69       if (list->bo_infos[i].handle == bo_info->handle) {
  70          assert(list->bo_infos[i].presumed == bo_info->presumed);
  71          list->bo_infos[i].flags |= bo_info->flags;
  72          return i;
  73       }
  74    }
  75
  76    /* grow list->bo_infos if needed */
  77    if (list->count == list->capacity) {
  78       uint32_t new_capacity = MAX2(2 * list->count, 16);
  79       struct drm_msm_gem_submit_bo *new_bo_infos = realloc(
  80          list->bo_infos, new_capacity * sizeof(struct drm_msm_gem_submit_bo));
  81       if (!new_bo_infos)
  82          return TU_BO_LIST_FAILED;
  83       list->bo_infos = new_bo_infos;
  84       list->capacity = new_capacity;
  85    }
  86
  87    list->bo_infos[list->count] = *bo_info;
  88    return list->count++;
  89 }
  90
  91 uint32_t
  92 tu_bo_list_add(struct tu_bo_list *list,
  93                const struct tu_bo *bo,
  94                uint32_t flags)
  95 {
  96    return tu_bo_list_add_info(list, &(struct drm_msm_gem_submit_bo) {
  97                                        .flags = flags,
  98                                        .handle = bo->gem_handle,
  99                                        .presumed = bo->iova,
 100                                     });
 101 }
 102
 103 VkResult
 104 tu_bo_list_merge(struct tu_bo_list *list, const struct tu_bo_list *other)
 105 {
 106    for (uint32_t i = 0; i < other->count; i++) {
 107       if (tu_bo_list_add_info(list, other->bo_infos + i) == TU_BO_LIST_FAILED)
 108          return VK_ERROR_OUT_OF_HOST_MEMORY;
 109    }
 110
 111    return VK_SUCCESS;
 112 }
 113
 114 static void
 115 tu_tiling_config_update_tile_layout(struct tu_tiling_config *tiling,
 116                                     const struct tu_device *dev,
 117                                     uint32_t pixels)
 118 {
 119    const uint32_t tile_align_w = dev->physical_device->tile_align_w;
 120    const uint32_t tile_align_h = dev->physical_device->tile_align_h;
 121    const uint32_t max_tile_width = 1024; /* A6xx */
 122
 123    tiling->tile0.offset = (VkOffset2D) {
 124       .x = tiling->render_area.offset.x & ~(tile_align_w - 1),
 125       .y = tiling->render_area.offset.y & ~(tile_align_h - 1),
 126    };
 127
 128    const uint32_t ra_width =
 129       tiling->render_area.extent.width +
 130       (tiling->render_area.offset.x - tiling->tile0.offset.x);
 131    const uint32_t ra_height =
 132       tiling->render_area.extent.height +
 133       (tiling->render_area.offset.y - tiling->tile0.offset.y);
 134
 135    /* start from 1 tile */
 136    tiling->tile_count = (VkExtent2D) {
 137       .width = 1,
 138       .height = 1,
 139    };
 140    tiling->tile0.extent = (VkExtent2D) {
 141       .width = align(ra_width, tile_align_w),
 142       .height = align(ra_height, tile_align_h),
 143    };
 144
 145    /* do not exceed max tile width */
 146    while (tiling->tile0.extent.width > max_tile_width) {
 147       tiling->tile_count.width++;
 148       tiling->tile0.extent.width =
 149          align(ra_width / tiling->tile_count.width, tile_align_w);
 150    }
 151
 152    /* do not exceed gmem size */
 153    while (tiling->tile0.extent.width * tiling->tile0.extent.height > pixels) {
 154       if (tiling->tile0.extent.width > MAX2(tile_align_w, tiling->tile0.extent.height)) {
 155          tiling->tile_count.width++;
 156          tiling->tile0.extent.width =
 157             align(DIV_ROUND_UP(ra_width, tiling->tile_count.width), tile_align_w);
 158       } else {
 159          /* if this assert fails then layout is impossible.. */
 160          assert(tiling->tile0.extent.height > tile_align_h);
 161          tiling->tile_count.height++;
 162          tiling->tile0.extent.height =
 163             align(DIV_ROUND_UP(ra_height, tiling->tile_count.height), tile_align_h);
 164       }
 165    }
 166 }
 167
 168 static void
 169 tu_tiling_config_update_pipe_layout(struct tu_tiling_config *tiling,
 170                                     const struct tu_device *dev)
 171 {
 172    const uint32_t max_pipe_count = 32; /* A6xx */
 173
 174    /* start from 1 tile per pipe */
 175    tiling->pipe0 = (VkExtent2D) {
 176       .width = 1,
 177       .height = 1,
 178    };
 179    tiling->pipe_count = tiling->tile_count;
 180
 181    /* do not exceed max pipe count vertically */
 182    while (tiling->pipe_count.height > max_pipe_count) {
 183       tiling->pipe0.height += 2;
 184       tiling->pipe_count.height =
 185          (tiling->tile_count.height + tiling->pipe0.height - 1) /
 186          tiling->pipe0.height;
 187    }
 188
 189    /* do not exceed max pipe count */
 190    while (tiling->pipe_count.width * tiling->pipe_count.height >
 191           max_pipe_count) {
 192       tiling->pipe0.width += 1;
 193       tiling->pipe_count.width =
 194          (tiling->tile_count.width + tiling->pipe0.width - 1) /
 195          tiling->pipe0.width;
 196    }
 197 }
 198
 199 static void
 200 tu_tiling_config_update_pipes(struct tu_tiling_config *tiling,
 201                               const struct tu_device *dev)
 202 {
 203    const uint32_t max_pipe_count = 32; /* A6xx */
 204    const uint32_t used_pipe_count =
 205       tiling->pipe_count.width * tiling->pipe_count.height;
 206    const VkExtent2D last_pipe = {
 207       .width = (tiling->tile_count.width - 1) % tiling->pipe0.width + 1,
 208       .height = (tiling->tile_count.height - 1) % tiling->pipe0.height + 1,
 209    };
 210
 211    assert(used_pipe_count <= max_pipe_count);
 212    assert(max_pipe_count <= ARRAY_SIZE(tiling->pipe_config));
 213
 214    for (uint32_t y = 0; y < tiling->pipe_count.height; y++) {
 215       for (uint32_t x = 0; x < tiling->pipe_count.width; x++) {
 216          const uint32_t pipe_x = tiling->pipe0.width * x;
 217          const uint32_t pipe_y = tiling->pipe0.height * y;
 218          const uint32_t pipe_w = (x == tiling->pipe_count.width - 1)
 219                                     ? last_pipe.width
 220                                     : tiling->pipe0.width;
 221          const uint32_t pipe_h = (y == tiling->pipe_count.height - 1)
 222                                     ? last_pipe.height
 223                                     : tiling->pipe0.height;
 224          const uint32_t n = tiling->pipe_count.width * y + x;
 225
 226          tiling->pipe_config[n] = A6XX_VSC_PIPE_CONFIG_REG_X(pipe_x) |
 227                                   A6XX_VSC_PIPE_CONFIG_REG_Y(pipe_y) |
 228                                   A6XX_VSC_PIPE_CONFIG_REG_W(pipe_w) |
 229                                   A6XX_VSC_PIPE_CONFIG_REG_H(pipe_h);
 230          tiling->pipe_sizes[n] = CP_SET_BIN_DATA5_0_VSC_SIZE(pipe_w * pipe_h);
 231       }
 232    }
 233
 234    memset(tiling->pipe_config + used_pipe_count, 0,
 235           sizeof(uint32_t) * (max_pipe_count - used_pipe_count));
 236 }
 237
 238 static void
 239 tu_tiling_config_get_tile(const struct tu_tiling_config *tiling,
 240                           const struct tu_device *dev,
 241                           uint32_t tx,
 242                           uint32_t ty,
 243                           struct tu_tile *tile)
 244 {
 245    /* find the pipe and the slot for tile (tx, ty) */
 246    const uint32_t px = tx / tiling->pipe0.width;
 247    const uint32_t py = ty / tiling->pipe0.height;
 248    const uint32_t sx = tx - tiling->pipe0.width * px;
 249    const uint32_t sy = ty - tiling->pipe0.height * py;
 250
 251    assert(tx < tiling->tile_count.width && ty < tiling->tile_count.height);
 252    assert(px < tiling->pipe_count.width && py < tiling->pipe_count.height);
 253    assert(sx < tiling->pipe0.width && sy < tiling->pipe0.height);
 254
 255    /* convert to 1D indices */
 256    tile->pipe = tiling->pipe_count.width * py + px;
 257    tile->slot = tiling->pipe0.width * sy + sx;
 258
 259    /* get the blit area for the tile */
 260    tile->begin = (VkOffset2D) {
 261       .x = tiling->tile0.offset.x + tiling->tile0.extent.width * tx,
 262       .y = tiling->tile0.offset.y + tiling->tile0.extent.height * ty,
 263    };
 264    tile->end.x =
 265       (tx == tiling->tile_count.width - 1)
 266          ? tiling->render_area.offset.x + tiling->render_area.extent.width
 267          : tile->begin.x + tiling->tile0.extent.width;
 268    tile->end.y =
 269       (ty == tiling->tile_count.height - 1)
 270          ? tiling->render_area.offset.y + tiling->render_area.extent.height
 271          : tile->begin.y + tiling->tile0.extent.height;
 272 }
 273
 274 enum a3xx_msaa_samples
 275 tu_msaa_samples(uint32_t samples)
 276 {
 277    switch (samples) {
 278    case 1:
 279       return MSAA_ONE;
 280    case 2:
 281       return MSAA_TWO;
 282    case 4:
 283       return MSAA_FOUR;
 284    case 8:
 285       return MSAA_EIGHT;
 286    default:
 287       assert(!"invalid sample count");
 288       return MSAA_ONE;
 289    }
 290 }
 291
 292 static enum a4xx_index_size
 293 tu6_index_size(VkIndexType type)
 294 {
 295    switch (type) {
 296    case VK_INDEX_TYPE_UINT16:
 297       return INDEX4_SIZE_16_BIT;
 298    case VK_INDEX_TYPE_UINT32:
 299       return INDEX4_SIZE_32_BIT;
 300    default:
 301       unreachable("invalid VkIndexType");
 302       return INDEX4_SIZE_8_BIT;
 303    }
 304 }
 305
 306 static void
 307 tu6_emit_marker(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 308 {
 309    tu_cs_emit_write_reg(cs, cmd->marker_reg, ++cmd->marker_seqno);
 310 }
 311
 312 unsigned
 313 tu6_emit_event_write(struct tu_cmd_buffer *cmd,
 314                      struct tu_cs *cs,
 315                      enum vgt_event_type event,
 316                      bool need_seqno)
 317 {
 318    unsigned seqno = 0;
 319
 320    tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, need_seqno ? 4 : 1);
 321    tu_cs_emit(cs, CP_EVENT_WRITE_0_EVENT(event));
 322    if (need_seqno) {
 323       tu_cs_emit_qw(cs, cmd->scratch_bo.iova);
 324       seqno = ++cmd->scratch_seqno;
 325       tu_cs_emit(cs, seqno);
 326    }
 327
 328    return seqno;
 329 }
 330
 331 static void
 332 tu6_emit_cache_flush(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 333 {
 334    tu6_emit_event_write(cmd, cs, 0x31, false);
 335 }
 336
 337 static void
 338 tu6_emit_lrz_flush(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 339 {
 340    tu6_emit_event_write(cmd, cs, LRZ_FLUSH, false);
 341 }
 342
 343 static void
 344 tu6_emit_wfi(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 345 {
 346    if (cmd->wait_for_idle) {
 347       tu_cs_emit_wfi(cs);
 348       cmd->wait_for_idle = false;
 349    }
 350 }
 351
 352 #define tu_image_view_ubwc_pitches(iview)                                \
 353    .pitch = tu_image_ubwc_pitch(iview->image, iview->base_mip),          \
 354    .array_pitch = tu_image_ubwc_size(iview->image, iview->base_mip) >> 2
 355
 356 static void
 357 tu6_emit_zs(struct tu_cmd_buffer *cmd,
 358             const struct tu_subpass *subpass,
 359             struct tu_cs *cs)
 360 {
 361    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 362
 363    const uint32_t a = subpass->depth_stencil_attachment.attachment;
 364    if (a == VK_ATTACHMENT_UNUSED) {
 365       tu_cs_emit_regs(cs,
 366                       A6XX_RB_DEPTH_BUFFER_INFO(.depth_format = DEPTH6_NONE),
 367                       A6XX_RB_DEPTH_BUFFER_PITCH(0),
 368                       A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH(0),
 369                       A6XX_RB_DEPTH_BUFFER_BASE(0),
 370                       A6XX_RB_DEPTH_BUFFER_BASE_GMEM(0));
 371
 372       tu_cs_emit_regs(cs,
 373                       A6XX_GRAS_SU_DEPTH_BUFFER_INFO(.depth_format = DEPTH6_NONE));
 374
 375       tu_cs_emit_regs(cs,
 376                       A6XX_GRAS_LRZ_BUFFER_BASE(0),
 377                       A6XX_GRAS_LRZ_BUFFER_PITCH(0),
 378                       A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE(0));
 379
 380       tu_cs_emit_regs(cs, A6XX_RB_STENCIL_INFO(0));
 381
 382       return;
 383    }
 384
 385    const struct tu_image_view *iview = fb->attachments[a].attachment;
 386    enum a6xx_depth_format fmt = tu6_pipe2depth(iview->vk_format);
 387
 388    tu_cs_emit_regs(cs,
 389                    A6XX_RB_DEPTH_BUFFER_INFO(.depth_format = fmt),
 390                    A6XX_RB_DEPTH_BUFFER_PITCH(tu_image_stride(iview->image, iview->base_mip)),
 391                    A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH(iview->image->layout.layer_size),
 392                    A6XX_RB_DEPTH_BUFFER_BASE(tu_image_view_base_ref(iview)),
 393                    A6XX_RB_DEPTH_BUFFER_BASE_GMEM(cmd->state.pass->attachments[a].gmem_offset));
 394
 395    tu_cs_emit_regs(cs,
 396                    A6XX_GRAS_SU_DEPTH_BUFFER_INFO(.depth_format = fmt));
 397
 398    tu_cs_emit_regs(cs,
 399                    A6XX_RB_DEPTH_FLAG_BUFFER_BASE(tu_image_view_ubwc_base_ref(iview)),
 400                    A6XX_RB_DEPTH_FLAG_BUFFER_PITCH(tu_image_view_ubwc_pitches(iview)));
 401
 402    tu_cs_emit_regs(cs,
 403                    A6XX_GRAS_LRZ_BUFFER_BASE(0),
 404                    A6XX_GRAS_LRZ_BUFFER_PITCH(0),
 405                    A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE(0));
 406
 407    tu_cs_emit_regs(cs,
 408                    A6XX_RB_STENCIL_INFO(0));
 409
 410    /* enable zs? */
 411 }
 412
 413 static void
 414 tu6_emit_mrt(struct tu_cmd_buffer *cmd,
 415              const struct tu_subpass *subpass,
 416              struct tu_cs *cs)
 417 {
 418    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 419    unsigned char mrt_comp[MAX_RTS] = { 0 };
 420    unsigned srgb_cntl = 0;
 421
 422    for (uint32_t i = 0; i < subpass->color_count; ++i) {
 423       uint32_t a = subpass->color_attachments[i].attachment;
 424       if (a == VK_ATTACHMENT_UNUSED)
 425          continue;
 426
 427       const struct tu_image_view *iview = fb->attachments[a].attachment;
 428       const enum a6xx_tile_mode tile_mode =
 429          tu6_get_image_tile_mode(iview->image, iview->base_mip);
 430
 431       mrt_comp[i] = 0xf;
 432
 433       if (vk_format_is_srgb(iview->vk_format))
 434          srgb_cntl |= (1 << i);
 435
 436       const struct tu_native_format *format =
 437          tu6_get_native_format(iview->vk_format);
 438       assert(format && format->rb >= 0);
 439
 440       tu_cs_emit_regs(cs,
 441                       A6XX_RB_MRT_BUF_INFO(i,
 442                                            .color_tile_mode = tile_mode,
 443                                            .color_format = format->rb,
 444                                            .color_swap = format->swap),
 445                       A6XX_RB_MRT_PITCH(i, tu_image_stride(iview->image, iview->base_mip)),
 446                       A6XX_RB_MRT_ARRAY_PITCH(i, iview->image->layout.layer_size),
 447                       A6XX_RB_MRT_BASE(i, tu_image_view_base_ref(iview)),
 448                       A6XX_RB_MRT_BASE_GMEM(i, cmd->state.pass->attachments[a].gmem_offset));
 449
 450       tu_cs_emit_regs(cs,
 451                       A6XX_SP_FS_MRT_REG(i,
 452                                          .color_format = format->rb,
 453                                          .color_sint = vk_format_is_sint(iview->vk_format),
 454                                          .color_uint = vk_format_is_uint(iview->vk_format)));
 455
 456       tu_cs_emit_regs(cs,
 457                       A6XX_RB_MRT_FLAG_BUFFER_ADDR(i, tu_image_view_ubwc_base_ref(iview)),
 458                       A6XX_RB_MRT_FLAG_BUFFER_PITCH(i, tu_image_view_ubwc_pitches(iview)));
 459    }
 460
 461    tu_cs_emit_regs(cs,
 462                    A6XX_RB_SRGB_CNTL(srgb_cntl));
 463
 464    tu_cs_emit_regs(cs,
 465                    A6XX_SP_SRGB_CNTL(srgb_cntl));
 466
 467    tu_cs_emit_regs(cs,
 468                    A6XX_RB_RENDER_COMPONENTS(
 469                       .rt0 = mrt_comp[0],
 470                       .rt1 = mrt_comp[1],
 471                       .rt2 = mrt_comp[2],
 472                       .rt3 = mrt_comp[3],
 473                       .rt4 = mrt_comp[4],
 474                       .rt5 = mrt_comp[5],
 475                       .rt6 = mrt_comp[6],
 476                       .rt7 = mrt_comp[7]));
 477
 478    tu_cs_emit_regs(cs,
 479                    A6XX_SP_FS_RENDER_COMPONENTS(
 480                       .rt0 = mrt_comp[0],
 481                       .rt1 = mrt_comp[1],
 482                       .rt2 = mrt_comp[2],
 483                       .rt3 = mrt_comp[3],
 484                       .rt4 = mrt_comp[4],
 485                       .rt5 = mrt_comp[5],
 486                       .rt6 = mrt_comp[6],
 487                       .rt7 = mrt_comp[7]));
 488 }
 489
 490 static void
 491 tu6_emit_msaa(struct tu_cmd_buffer *cmd,
 492               const struct tu_subpass *subpass,
 493               struct tu_cs *cs)
 494 {
 495    const enum a3xx_msaa_samples samples = tu_msaa_samples(subpass->samples);
 496    bool msaa_disable = samples == MSAA_ONE;
 497
 498    tu_cs_emit_regs(cs,
 499                    A6XX_SP_TP_RAS_MSAA_CNTL(samples),
 500                    A6XX_SP_TP_DEST_MSAA_CNTL(.samples = samples,
 501                                              .msaa_disable = msaa_disable));
 502
 503    tu_cs_emit_regs(cs,
 504                    A6XX_GRAS_RAS_MSAA_CNTL(samples),
 505                    A6XX_GRAS_DEST_MSAA_CNTL(.samples = samples,
 506                                             .msaa_disable = msaa_disable));
 507
 508    tu_cs_emit_regs(cs,
 509                    A6XX_RB_RAS_MSAA_CNTL(samples),
 510                    A6XX_RB_DEST_MSAA_CNTL(.samples = samples,
 511                                           .msaa_disable = msaa_disable));
 512
 513    tu_cs_emit_regs(cs,
 514                    A6XX_RB_MSAA_CNTL(samples));
 515 }
 516
 517 static void
 518 tu6_emit_bin_size(struct tu_cs *cs,
 519                   uint32_t bin_w, uint32_t bin_h, uint32_t flags)
 520 {
 521    tu_cs_emit_regs(cs,
 522                    A6XX_GRAS_BIN_CONTROL(.binw = bin_w,
 523                                          .binh = bin_h,
 524                                          .dword = flags));
 525
 526    tu_cs_emit_regs(cs,
 527                    A6XX_RB_BIN_CONTROL(.binw = bin_w,
 528                                        .binh = bin_h,
 529                                        .dword = flags));
 530
 531    /* no flag for RB_BIN_CONTROL2... */
 532    tu_cs_emit_regs(cs,
 533                    A6XX_RB_BIN_CONTROL2(.binw = bin_w,
 534                                         .binh = bin_h));
 535 }
 536
 537 static void
 538 tu6_emit_render_cntl(struct tu_cmd_buffer *cmd,
 539                      const struct tu_subpass *subpass,
 540                      struct tu_cs *cs,
 541                      bool binning)
 542 {
 543    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 544    uint32_t cntl = 0;
 545    cntl |= A6XX_RB_RENDER_CNTL_UNK4;
 546    if (binning) {
 547       cntl |= A6XX_RB_RENDER_CNTL_BINNING;
 548    } else {
 549       uint32_t mrts_ubwc_enable = 0;
 550       for (uint32_t i = 0; i < subpass->color_count; ++i) {
 551          uint32_t a = subpass->color_attachments[i].attachment;
 552          if (a == VK_ATTACHMENT_UNUSED)
 553             continue;
 554
 555          const struct tu_image_view *iview = fb->attachments[a].attachment;
 556          if (iview->image->layout.ubwc_layer_size != 0)
 557             mrts_ubwc_enable |= 1 << i;
 558       }
 559
 560       cntl |= A6XX_RB_RENDER_CNTL_FLAG_MRTS(mrts_ubwc_enable);
 561
 562       const uint32_t a = subpass->depth_stencil_attachment.attachment;
 563       if (a != VK_ATTACHMENT_UNUSED) {
 564          const struct tu_image_view *iview = fb->attachments[a].attachment;
 565          if (iview->image->layout.ubwc_layer_size != 0)
 566             cntl |= A6XX_RB_RENDER_CNTL_FLAG_DEPTH;
 567       }
 568
 569       /* In the !binning case, we need to set RB_RENDER_CNTL in the draw_cs
 570        * in order to set it correctly for the different subpasses. However,
 571        * that means the packets we're emitting also happen during binning. So
 572        * we need to guard the write on !BINNING at CP execution time.
 573        */
 574       tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
 575       tu_cs_emit(cs, CP_COND_REG_EXEC_0_MODE(RENDER_MODE) |
 576                      CP_COND_REG_EXEC_0_GMEM | CP_COND_REG_EXEC_0_SYSMEM);
 577       tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(4));
 578    }
 579
 580    tu_cs_emit_pkt7(cs, CP_REG_WRITE, 3);
 581    tu_cs_emit(cs, CP_REG_WRITE_0_TRACKER(TRACK_RENDER_CNTL));
 582    tu_cs_emit(cs, REG_A6XX_RB_RENDER_CNTL);
 583    tu_cs_emit(cs, cntl);
 584 }
 585
 586 static void
 587 tu6_emit_blit_scissor(struct tu_cmd_buffer *cmd, struct tu_cs *cs, bool align)
 588 {
 589    const VkRect2D *render_area = &cmd->state.tiling_config.render_area;
 590    uint32_t x1 = render_area->offset.x;
 591    uint32_t y1 = render_area->offset.y;
 592    uint32_t x2 = x1 + render_area->extent.width - 1;
 593    uint32_t y2 = y1 + render_area->extent.height - 1;
 594
 595    /* TODO: alignment requirement seems to be less than tile_align_w/h */
 596    if (align) {
 597       x1 = x1 & ~cmd->device->physical_device->tile_align_w;
 598       y1 = y1 & ~cmd->device->physical_device->tile_align_h;
 599       x2 = ALIGN_POT(x2 + 1, cmd->device->physical_device->tile_align_w) - 1;
 600       y2 = ALIGN_POT(y2 + 1, cmd->device->physical_device->tile_align_h) - 1;
 601    }
 602
 603    tu_cs_emit_regs(cs,
 604                    A6XX_RB_BLIT_SCISSOR_TL(.x = x1, .y = y1),
 605                    A6XX_RB_BLIT_SCISSOR_BR(.x = x2, .y = y2));
 606 }
 607
 608 static void
 609 tu6_emit_blit_info(struct tu_cmd_buffer *cmd,
 610                    struct tu_cs *cs,
 611                    const struct tu_image_view *iview,
 612                    uint32_t gmem_offset,
 613                    bool resolve)
 614 {
 615    tu_cs_emit_regs(cs,
 616                    A6XX_RB_BLIT_INFO(.unk0 = !resolve, .gmem = !resolve));
 617
 618    const struct tu_native_format *format =
 619       tu6_get_native_format(iview->vk_format);
 620    assert(format && format->rb >= 0);
 621
 622    enum a6xx_tile_mode tile_mode =
 623       tu6_get_image_tile_mode(iview->image, iview->base_mip);
 624    tu_cs_emit_regs(cs,
 625                    A6XX_RB_BLIT_DST_INFO(
 626                       .tile_mode = tile_mode,
 627                       .samples = tu_msaa_samples(iview->image->samples),
 628                       .color_format = format->rb,
 629                       .color_swap = format->swap,
 630                       .flags = iview->image->layout.ubwc_layer_size != 0),
 631                    A6XX_RB_BLIT_DST(tu_image_view_base_ref(iview)),
 632                    A6XX_RB_BLIT_DST_PITCH(tu_image_stride(iview->image, iview->base_mip)),
 633                    A6XX_RB_BLIT_DST_ARRAY_PITCH(iview->image->layout.layer_size));
 634
 635    if (iview->image->layout.ubwc_layer_size) {
 636       tu_cs_emit_regs(cs,
 637                       A6XX_RB_BLIT_FLAG_DST(tu_image_view_ubwc_base_ref(iview)),
 638                       A6XX_RB_BLIT_FLAG_DST_PITCH(tu_image_view_ubwc_pitches(iview)));
 639    }
 640
 641    tu_cs_emit_regs(cs,
 642                    A6XX_RB_BLIT_BASE_GMEM(gmem_offset));
 643 }
 644
 645 static void
 646 tu6_emit_blit(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 647 {
 648    tu6_emit_marker(cmd, cs);
 649    tu6_emit_event_write(cmd, cs, BLIT, false);
 650    tu6_emit_marker(cmd, cs);
 651 }
 652
 653 static void
 654 tu6_emit_window_scissor(struct tu_cmd_buffer *cmd,
 655                         struct tu_cs *cs,
 656                         uint32_t x1,
 657                         uint32_t y1,
 658                         uint32_t x2,
 659                         uint32_t y2)
 660 {
 661    tu_cs_emit_regs(cs,
 662                    A6XX_GRAS_SC_WINDOW_SCISSOR_TL(.x = x1, .y = y1),
 663                    A6XX_GRAS_SC_WINDOW_SCISSOR_BR(.x = x2, .y = y2));
 664
 665    tu_cs_emit_regs(cs,
 666                    A6XX_GRAS_RESOLVE_CNTL_1(.x = x1, .y = y1),
 667                    A6XX_GRAS_RESOLVE_CNTL_2(.x = x2, .y = y2));
 668 }
 669
 670 static void
 671 tu6_emit_window_offset(struct tu_cmd_buffer *cmd,
 672                        struct tu_cs *cs,
 673                        uint32_t x1,
 674                        uint32_t y1)
 675 {
 676    tu_cs_emit_regs(cs,
 677                    A6XX_RB_WINDOW_OFFSET(.x = x1, .y = y1));
 678
 679    tu_cs_emit_regs(cs,
 680                    A6XX_RB_WINDOW_OFFSET2(.x = x1, .y = y1));
 681
 682    tu_cs_emit_regs(cs,
 683                    A6XX_SP_WINDOW_OFFSET(.x = x1, .y = y1));
 684
 685    tu_cs_emit_regs(cs,
 686                    A6XX_SP_TP_WINDOW_OFFSET(.x = x1, .y = y1));
 687 }
 688
 689 static bool
 690 use_hw_binning(struct tu_cmd_buffer *cmd)
 691 {
 692    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
 693
 694    if (unlikely(cmd->device->physical_device->instance->debug_flags & TU_DEBUG_NOBIN))
 695       return false;
 696
 697    return (tiling->tile_count.width * tiling->tile_count.height) > 2;
 698 }
 699
 700 static bool
 701 use_sysmem_rendering(struct tu_cmd_buffer *cmd)
 702 {
 703    if (unlikely(cmd->device->physical_device->instance->debug_flags & TU_DEBUG_SYSMEM))
 704       return true;
 705
 706    return false;
 707 }
 708
 709 static void
 710 tu6_emit_tile_select(struct tu_cmd_buffer *cmd,
 711                      struct tu_cs *cs,
 712                      const struct tu_tile *tile)
 713 {
 714    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
 715    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(0x7));
 716
 717    tu6_emit_marker(cmd, cs);
 718    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
 719    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_GMEM) | 0x10);
 720    tu6_emit_marker(cmd, cs);
 721
 722    const uint32_t x1 = tile->begin.x;
 723    const uint32_t y1 = tile->begin.y;
 724    const uint32_t x2 = tile->end.x - 1;
 725    const uint32_t y2 = tile->end.y - 1;
 726    tu6_emit_window_scissor(cmd, cs, x1, y1, x2, y2);
 727    tu6_emit_window_offset(cmd, cs, x1, y1);
 728
 729    tu_cs_emit_regs(cs,
 730                    A6XX_VPC_SO_OVERRIDE(.so_disable = true));
 731
 732    if (use_hw_binning(cmd)) {
 733       tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
 734
 735       tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
 736       tu_cs_emit(cs, 0x0);
 737
 738       tu_cs_emit_pkt7(cs, CP_REG_TEST, 1);
 739       tu_cs_emit(cs, A6XX_CP_REG_TEST_0_REG(OVERFLOW_FLAG_REG) |
 740                      A6XX_CP_REG_TEST_0_BIT(0) |
 741                      A6XX_CP_REG_TEST_0_WAIT_FOR_ME);
 742
 743       tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
 744       tu_cs_emit(cs, CP_COND_REG_EXEC_0_MODE(PRED_TEST));
 745       tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(11));
 746
 747       /* if (no overflow) */ {
 748          tu_cs_emit_pkt7(cs, CP_SET_BIN_DATA5, 7);
 749          tu_cs_emit(cs, cmd->state.tiling_config.pipe_sizes[tile->pipe] |
 750                         CP_SET_BIN_DATA5_0_VSC_N(tile->slot));
 751          tu_cs_emit_qw(cs, cmd->vsc_data.iova + tile->pipe * cmd->vsc_data_pitch);
 752          tu_cs_emit_qw(cs, cmd->vsc_data.iova + (tile->pipe * 4) + (32 * cmd->vsc_data_pitch));
 753          tu_cs_emit_qw(cs, cmd->vsc_data2.iova + (tile->pipe * cmd->vsc_data2_pitch));
 754
 755          tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
 756          tu_cs_emit(cs, 0x0);
 757
 758          /* use a NOP packet to skip over the 'else' side: */
 759          tu_cs_emit_pkt7(cs, CP_NOP, 2);
 760       } /* else */ {
 761          tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
 762          tu_cs_emit(cs, 0x1);
 763       }
 764
 765       tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
 766       tu_cs_emit(cs, 0x0);
 767
 768       tu_cs_emit_regs(cs,
 769                       A6XX_RB_UNKNOWN_8804(0));
 770
 771       tu_cs_emit_regs(cs,
 772                       A6XX_SP_TP_UNKNOWN_B304(0));
 773
 774       tu_cs_emit_regs(cs,
 775                       A6XX_GRAS_UNKNOWN_80A4(0));
 776    } else {
 777       tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
 778       tu_cs_emit(cs, 0x1);
 779
 780       tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
 781       tu_cs_emit(cs, 0x0);
 782    }
 783 }
 784
 785 static void
 786 tu6_emit_load_attachment(struct tu_cmd_buffer *cmd, struct tu_cs *cs, uint32_t a)
 787 {
 788    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
 789    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 790    const struct tu_image_view *iview = fb->attachments[a].attachment;
 791    const struct tu_render_pass_attachment *attachment =
 792       &cmd->state.pass->attachments[a];
 793
 794    if (attachment->gmem_offset < 0)
 795       return;
 796
 797    const uint32_t x1 = tiling->render_area.offset.x;
 798    const uint32_t y1 = tiling->render_area.offset.y;
 799    const uint32_t x2 = x1 + tiling->render_area.extent.width;
 800    const uint32_t y2 = y1 + tiling->render_area.extent.height;
 801    const uint32_t tile_x2 =
 802       tiling->tile0.offset.x + tiling->tile0.extent.width * tiling->tile_count.width;
 803    const uint32_t tile_y2 =
 804       tiling->tile0.offset.y + tiling->tile0.extent.height * tiling->tile_count.height;
 805    bool need_load =
 806       x1 != tiling->tile0.offset.x || x2 != MIN2(fb->width, tile_x2) ||
 807       y1 != tiling->tile0.offset.y || y2 != MIN2(fb->height, tile_y2);
 808
 809    if (need_load)
 810       tu_finishme("improve handling of unaligned render area");
 811
 812    if (attachment->load_op == VK_ATTACHMENT_LOAD_OP_LOAD)
 813       need_load = true;
 814
 815    if (vk_format_has_stencil(iview->vk_format) &&
 816        attachment->stencil_load_op == VK_ATTACHMENT_LOAD_OP_LOAD)
 817       need_load = true;
 818
 819    if (need_load) {
 820       tu6_emit_blit_info(cmd, cs, iview, attachment->gmem_offset, false);
 821       tu6_emit_blit(cmd, cs);
 822    }
 823 }
 824
 825 static void
 826 tu6_emit_clear_attachment(struct tu_cmd_buffer *cmd, struct tu_cs *cs,
 827                           uint32_t a,
 828                           const VkRenderPassBeginInfo *info)
 829 {
 830    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 831    const struct tu_image_view *iview = fb->attachments[a].attachment;
 832    const struct tu_render_pass_attachment *attachment =
 833       &cmd->state.pass->attachments[a];
 834    unsigned clear_mask = 0;
 835
 836    /* note: this means it isn't used by any subpass and shouldn't be cleared anyway */
 837    if (attachment->gmem_offset < 0)
 838       return;
 839
 840    if (attachment->load_op == VK_ATTACHMENT_LOAD_OP_CLEAR)
 841       clear_mask = 0xf;
 842
 843    if (vk_format_has_stencil(iview->vk_format)) {
 844       clear_mask &= 0x1;
 845       if (attachment->stencil_load_op == VK_ATTACHMENT_LOAD_OP_CLEAR)
 846          clear_mask |= 0x2;
 847    }
 848    if (!clear_mask)
 849       return;
 850
 851    tu_clear_gmem_attachment(cmd, cs, a, clear_mask,
 852                             &info->pClearValues[a]);
 853 }
 854
 855 static void
 856 tu6_emit_store_attachment(struct tu_cmd_buffer *cmd,
 857                           struct tu_cs *cs,
 858                           uint32_t a,
 859                           uint32_t gmem_a)
 860 {
 861    if (cmd->state.pass->attachments[a].store_op == VK_ATTACHMENT_STORE_OP_DONT_CARE)
 862       return;
 863
 864    tu6_emit_blit_info(cmd, cs,
 865                       cmd->state.framebuffer->attachments[a].attachment,
 866                       cmd->state.pass->attachments[gmem_a].gmem_offset, true);
 867    tu6_emit_blit(cmd, cs);
 868 }
 869
 870 static void
 871 tu6_emit_tile_store(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 872 {
 873    const struct tu_render_pass *pass = cmd->state.pass;
 874    const struct tu_subpass *subpass = &pass->subpasses[pass->subpass_count-1];
 875
 876    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
 877    tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
 878                      CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
 879                      CP_SET_DRAW_STATE__0_GROUP_ID(0));
 880    tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
 881    tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
 882
 883    tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
 884    tu_cs_emit(cs, 0x0);
 885
 886    tu6_emit_marker(cmd, cs);
 887    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
 888    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_RESOLVE) | 0x10);
 889    tu6_emit_marker(cmd, cs);
 890
 891    tu6_emit_blit_scissor(cmd, cs, true);
 892
 893    for (uint32_t a = 0; a < pass->attachment_count; ++a) {
 894       if (pass->attachments[a].gmem_offset >= 0)
 895          tu6_emit_store_attachment(cmd, cs, a, a);
 896    }
 897
 898    if (subpass->resolve_attachments) {
 899       for (unsigned i = 0; i < subpass->color_count; i++) {
 900          uint32_t a = subpass->resolve_attachments[i].attachment;
 901          if (a != VK_ATTACHMENT_UNUSED)
 902             tu6_emit_store_attachment(cmd, cs, a,
 903                                       subpass->color_attachments[i].attachment);
 904       }
 905    }
 906 }
 907
 908 static void
 909 tu6_emit_restart_index(struct tu_cs *cs, uint32_t restart_index)
 910 {
 911    tu_cs_emit_regs(cs,
 912                    A6XX_PC_RESTART_INDEX(restart_index));
 913 }
 914
 915 static void
 916 tu6_init_hw(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 917 {
 918    VkResult result = tu_cs_reserve_space(cmd->device, cs, 256);
 919    if (result != VK_SUCCESS) {
 920       cmd->record_result = result;
 921       return;
 922    }
 923
 924    tu6_emit_cache_flush(cmd, cs);
 925
 926    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UPDATE_CNTL, 0xfffff);
 927
 928    tu_cs_emit_write_reg(cs, REG_A6XX_RB_CCU_CNTL, 0x10000000);
 929    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8E04, 0x00100000);
 930    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE04, 0x8);
 931    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE00, 0);
 932    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE0F, 0x3f);
 933    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B605, 0x44);
 934    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B600, 0x100000);
 935    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE00, 0x80);
 936    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE01, 0);
 937
 938    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9600, 0);
 939    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8600, 0x880);
 940    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE04, 0);
 941    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE03, 0x00000410);
 942    tu_cs_emit_write_reg(cs, REG_A6XX_SP_IBO_COUNT, 0);
 943    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B182, 0);
 944    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BB11, 0);
 945    tu_cs_emit_write_reg(cs, REG_A6XX_UCHE_UNKNOWN_0E12, 0x3200000);
 946    tu_cs_emit_write_reg(cs, REG_A6XX_UCHE_CLIENT_PF, 4);
 947    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8E01, 0x0);
 948    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AB00, 0x5);
 949    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_ADD_OFFSET, A6XX_VFD_ADD_OFFSET_VERTEX);
 950    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8811, 0x00000010);
 951    tu_cs_emit_write_reg(cs, REG_A6XX_PC_MODE_CNTL, 0x1f);
 952
 953    tu_cs_emit_write_reg(cs, REG_A6XX_RB_SRGB_CNTL, 0);
 954
 955    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8101, 0);
 956    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_SAMPLE_CNTL, 0);
 957    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8110, 0);
 958
 959    tu_cs_emit_write_reg(cs, REG_A6XX_RB_RENDER_CONTROL0, 0x401);
 960    tu_cs_emit_write_reg(cs, REG_A6XX_RB_RENDER_CONTROL1, 0);
 961    tu_cs_emit_write_reg(cs, REG_A6XX_RB_FS_OUTPUT_CNTL0, 0);
 962    tu_cs_emit_write_reg(cs, REG_A6XX_RB_SAMPLE_CNTL, 0);
 963    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8818, 0);
 964    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8819, 0);
 965    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881A, 0);
 966    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881B, 0);
 967    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881C, 0);
 968    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881D, 0);
 969    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881E, 0);
 970    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_88F0, 0);
 971
 972    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9101, 0xffff00);
 973    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9107, 0);
 974
 975    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9236, 1);
 976    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9300, 0);
 977
 978    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_SO_OVERRIDE,
 979                         A6XX_VPC_SO_OVERRIDE_SO_DISABLE);
 980
 981    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9801, 0);
 982    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9806, 0);
 983    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9980, 0);
 984
 985    tu_cs_emit_write_reg(cs, REG_A6XX_PC_PRIMITIVE_CNTL_6, 0);
 986    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9B07, 0);
 987
 988    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_A81B, 0);
 989
 990    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B183, 0);
 991
 992    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8099, 0);
 993    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_809B, 0);
 994    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A0, 2);
 995    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80AF, 0);
 996    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9210, 0);
 997    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9211, 0);
 998    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9602, 0);
 999    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9981, 0x3);
1000    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9E72, 0);
1001    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9108, 0x3);
1002    tu_cs_emit_write_reg(cs, REG_A6XX_SP_TP_UNKNOWN_B304, 0);
1003    tu_cs_emit_write_reg(cs, REG_A6XX_SP_TP_UNKNOWN_B309, 0x000000a2);
1004    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8804, 0);
1005    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A4, 0);
1006    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A5, 0);
1007    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A6, 0);
1008    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8805, 0);
1009    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8806, 0);
1010    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8878, 0);
1011    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8879, 0);
1012    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_CONTROL_5_REG, 0xfc);
1013
1014    tu6_emit_marker(cmd, cs);
1015
1016    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_MODE_CNTL, 0x00000000);
1017
1018    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_UNKNOWN_A008, 0);
1019
1020    tu_cs_emit_write_reg(cs, REG_A6XX_PC_MODE_CNTL, 0x0000001f);
1021
1022    /* we don't use this yet.. probably best to disable.. */
1023    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
1024    tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
1025                      CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
1026                      CP_SET_DRAW_STATE__0_GROUP_ID(0));
1027    tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
1028    tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
1029
1030    tu_cs_emit_regs(cs,
1031                    A6XX_VPC_SO_BUFFER_BASE(0),
1032                    A6XX_VPC_SO_BUFFER_SIZE(0));
1033
1034    tu_cs_emit_regs(cs,
1035                    A6XX_VPC_SO_FLUSH_BASE(0));
1036
1037    tu_cs_emit_regs(cs,
1038                    A6XX_VPC_SO_BUF_CNTL(0));
1039
1040    tu_cs_emit_regs(cs,
1041                    A6XX_VPC_SO_BUFFER_OFFSET(0, 0));
1042
1043    tu_cs_emit_regs(cs,
1044                    A6XX_VPC_SO_BUFFER_BASE(1, 0),
1045                    A6XX_VPC_SO_BUFFER_SIZE(1, 0));
1046
1047    tu_cs_emit_regs(cs,
1048                    A6XX_VPC_SO_BUFFER_OFFSET(1, 0),
1049                    A6XX_VPC_SO_FLUSH_BASE(1, 0),
1050                    A6XX_VPC_SO_BUFFER_BASE(2, 0),
1051                    A6XX_VPC_SO_BUFFER_SIZE(2, 0));
1052
1053    tu_cs_emit_regs(cs,
1054                    A6XX_VPC_SO_BUFFER_OFFSET(2, 0),
1055                    A6XX_VPC_SO_FLUSH_BASE(2, 0),
1056                    A6XX_VPC_SO_BUFFER_BASE(3, 0),
1057                    A6XX_VPC_SO_BUFFER_SIZE(3, 0));
1058
1059    tu_cs_emit_regs(cs,
1060                    A6XX_VPC_SO_BUFFER_OFFSET(3, 0),
1061                    A6XX_VPC_SO_FLUSH_BASE(3, 0));
1062
1063    tu_cs_emit_regs(cs,
1064                    A6XX_SP_HS_CTRL_REG0(0));
1065
1066    tu_cs_emit_regs(cs,
1067                    A6XX_SP_GS_CTRL_REG0(0));
1068
1069    tu_cs_emit_regs(cs,
1070                    A6XX_GRAS_LRZ_CNTL(0));
1071
1072    tu_cs_emit_regs(cs,
1073                    A6XX_RB_LRZ_CNTL(0));
1074
1075    tu_cs_sanity_check(cs);
1076 }
1077
1078 static void
1079 tu6_cache_flush(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1080 {
1081    unsigned seqno;
1082
1083    seqno = tu6_emit_event_write(cmd, cs, CACHE_FLUSH_AND_INV_EVENT, true);
1084
1085    tu_cs_emit_pkt7(cs, CP_WAIT_REG_MEM, 6);
1086    tu_cs_emit(cs, CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ) |
1087                   CP_WAIT_REG_MEM_0_POLL_MEMORY);
1088    tu_cs_emit_qw(cs, cmd->scratch_bo.iova);
1089    tu_cs_emit(cs, CP_WAIT_REG_MEM_3_REF(seqno));
1090    tu_cs_emit(cs, CP_WAIT_REG_MEM_4_MASK(~0));
1091    tu_cs_emit(cs, CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(16));
1092
1093    seqno = tu6_emit_event_write(cmd, cs, CACHE_FLUSH_TS, true);
1094
1095    tu_cs_emit_pkt7(cs, CP_WAIT_MEM_GTE, 4);
1096    tu_cs_emit(cs, CP_WAIT_MEM_GTE_0_RESERVED(0));
1097    tu_cs_emit_qw(cs, cmd->scratch_bo.iova);
1098    tu_cs_emit(cs, CP_WAIT_MEM_GTE_3_REF(seqno));
1099 }
1100
1101 static void
1102 update_vsc_pipe(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1103 {
1104    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1105
1106    tu_cs_emit_regs(cs,
1107                    A6XX_VSC_BIN_SIZE(.width = tiling->tile0.extent.width,
1108                                      .height = tiling->tile0.extent.height),
1109                    A6XX_VSC_SIZE_ADDRESS(.bo = &cmd->vsc_data,
1110                                          .bo_offset = 32 * cmd->vsc_data_pitch));
1111
1112    tu_cs_emit_regs(cs,
1113                    A6XX_VSC_BIN_COUNT(.nx = tiling->tile_count.width,
1114                                       .ny = tiling->tile_count.height));
1115
1116    tu_cs_emit_pkt4(cs, REG_A6XX_VSC_PIPE_CONFIG_REG(0), 32);
1117    for (unsigned i = 0; i < 32; i++)
1118       tu_cs_emit(cs, tiling->pipe_config[i]);
1119
1120    tu_cs_emit_regs(cs,
1121                    A6XX_VSC_PIPE_DATA2_ADDRESS(.bo = &cmd->vsc_data2),
1122                    A6XX_VSC_PIPE_DATA2_PITCH(cmd->vsc_data2_pitch),
1123                    A6XX_VSC_PIPE_DATA2_ARRAY_PITCH(cmd->vsc_data2.size));
1124
1125    tu_cs_emit_regs(cs,
1126                    A6XX_VSC_PIPE_DATA_ADDRESS(.bo = &cmd->vsc_data),
1127                    A6XX_VSC_PIPE_DATA_PITCH(cmd->vsc_data_pitch),
1128                    A6XX_VSC_PIPE_DATA_ARRAY_PITCH(cmd->vsc_data.size));
1129 }
1130
1131 static void
1132 emit_vsc_overflow_test(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1133 {
1134    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1135    const uint32_t used_pipe_count =
1136       tiling->pipe_count.width * tiling->pipe_count.height;
1137
1138    /* Clear vsc_scratch: */
1139    tu_cs_emit_pkt7(cs, CP_MEM_WRITE, 3);
1140    tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1141    tu_cs_emit(cs, 0x0);
1142
1143    /* Check for overflow, write vsc_scratch if detected: */
1144    for (int i = 0; i < used_pipe_count; i++) {
1145       tu_cs_emit_pkt7(cs, CP_COND_WRITE5, 8);
1146       tu_cs_emit(cs, CP_COND_WRITE5_0_FUNCTION(WRITE_GE) |
1147             CP_COND_WRITE5_0_WRITE_MEMORY);
1148       tu_cs_emit(cs, CP_COND_WRITE5_1_POLL_ADDR_LO(REG_A6XX_VSC_SIZE_REG(i)));
1149       tu_cs_emit(cs, CP_COND_WRITE5_2_POLL_ADDR_HI(0));
1150       tu_cs_emit(cs, CP_COND_WRITE5_3_REF(cmd->vsc_data_pitch));
1151       tu_cs_emit(cs, CP_COND_WRITE5_4_MASK(~0));
1152       tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1153       tu_cs_emit(cs, CP_COND_WRITE5_7_WRITE_DATA(1 + cmd->vsc_data_pitch));
1154
1155       tu_cs_emit_pkt7(cs, CP_COND_WRITE5, 8);
1156       tu_cs_emit(cs, CP_COND_WRITE5_0_FUNCTION(WRITE_GE) |
1157             CP_COND_WRITE5_0_WRITE_MEMORY);
1158       tu_cs_emit(cs, CP_COND_WRITE5_1_POLL_ADDR_LO(REG_A6XX_VSC_SIZE2_REG(i)));
1159       tu_cs_emit(cs, CP_COND_WRITE5_2_POLL_ADDR_HI(0));
1160       tu_cs_emit(cs, CP_COND_WRITE5_3_REF(cmd->vsc_data2_pitch));
1161       tu_cs_emit(cs, CP_COND_WRITE5_4_MASK(~0));
1162       tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1163       tu_cs_emit(cs, CP_COND_WRITE5_7_WRITE_DATA(3 + cmd->vsc_data2_pitch));
1164    }
1165
1166    tu_cs_emit_pkt7(cs, CP_WAIT_MEM_WRITES, 0);
1167
1168    tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
1169
1170    tu_cs_emit_pkt7(cs, CP_MEM_TO_REG, 3);
1171    tu_cs_emit(cs, CP_MEM_TO_REG_0_REG(OVERFLOW_FLAG_REG) |
1172          CP_MEM_TO_REG_0_CNT(1 - 1));
1173    tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1174
1175    /*
1176     * This is a bit awkward, we really want a way to invert the
1177     * CP_REG_TEST/CP_COND_REG_EXEC logic, so that we can conditionally
1178     * execute cmds to use hwbinning when a bit is *not* set.  This
1179     * dance is to invert OVERFLOW_FLAG_REG
1180     *
1181     * A CP_NOP packet is used to skip executing the 'else' clause
1182     * if (b0 set)..
1183     */
1184
1185    /* b0 will be set if VSC_DATA or VSC_DATA2 overflow: */
1186    tu_cs_emit_pkt7(cs, CP_REG_TEST, 1);
1187    tu_cs_emit(cs, A6XX_CP_REG_TEST_0_REG(OVERFLOW_FLAG_REG) |
1188          A6XX_CP_REG_TEST_0_BIT(0) |
1189          A6XX_CP_REG_TEST_0_WAIT_FOR_ME);
1190
1191    tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
1192    tu_cs_emit(cs, CP_COND_REG_EXEC_0_MODE(PRED_TEST));
1193    tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(7));
1194
1195    /* if (b0 set) */ {
1196       /*
1197        * On overflow, mirror the value to control->vsc_overflow
1198        * which CPU is checking to detect overflow (see
1199        * check_vsc_overflow())
1200        */
1201       tu_cs_emit_pkt7(cs, CP_REG_TO_MEM, 3);
1202       tu_cs_emit(cs, CP_REG_TO_MEM_0_REG(OVERFLOW_FLAG_REG) |
1203             CP_REG_TO_MEM_0_CNT(0));
1204       tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_OVERFLOW);
1205
1206       tu_cs_emit_pkt4(cs, OVERFLOW_FLAG_REG, 1);
1207       tu_cs_emit(cs, 0x0);
1208
1209       tu_cs_emit_pkt7(cs, CP_NOP, 2);  /* skip 'else' when 'if' is taken */
1210    } /* else */ {
1211       tu_cs_emit_pkt4(cs, OVERFLOW_FLAG_REG, 1);
1212       tu_cs_emit(cs, 0x1);
1213    }
1214 }
1215
1216 static void
1217 tu6_emit_binning_pass(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1218 {
1219    struct tu_physical_device *phys_dev = cmd->device->physical_device;
1220    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1221
1222    uint32_t x1 = tiling->tile0.offset.x;
1223    uint32_t y1 = tiling->tile0.offset.y;
1224    uint32_t x2 = tiling->render_area.offset.x + tiling->render_area.extent.width - 1;
1225    uint32_t y2 = tiling->render_area.offset.y + tiling->render_area.extent.height - 1;
1226
1227    tu6_emit_window_scissor(cmd, cs, x1, y1, x2, y2);
1228
1229    tu6_emit_marker(cmd, cs);
1230    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1231    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_BINNING));
1232    tu6_emit_marker(cmd, cs);
1233
1234    tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
1235    tu_cs_emit(cs, 0x1);
1236
1237    tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
1238    tu_cs_emit(cs, 0x1);
1239
1240    tu_cs_emit_wfi(cs);
1241
1242    tu_cs_emit_regs(cs,
1243                    A6XX_VFD_MODE_CNTL(.binning_pass = true));
1244
1245    update_vsc_pipe(cmd, cs);
1246
1247    tu_cs_emit_regs(cs,
1248                    A6XX_PC_UNKNOWN_9805(.unknown = phys_dev->magic.PC_UNKNOWN_9805));
1249
1250    tu_cs_emit_regs(cs,
1251                    A6XX_SP_UNKNOWN_A0F8(.unknown = phys_dev->magic.SP_UNKNOWN_A0F8));
1252
1253    tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, 1);
1254    tu_cs_emit(cs, UNK_2C);
1255
1256    tu_cs_emit_regs(cs,
1257                    A6XX_RB_WINDOW_OFFSET(.x = 0, .y = 0));
1258
1259    tu_cs_emit_regs(cs,
1260                    A6XX_SP_TP_WINDOW_OFFSET(.x = 0, .y = 0));
1261
1262    /* emit IB to binning drawcmds: */
1263    tu_cs_emit_call(cs, &cmd->draw_cs);
1264
1265    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
1266    tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
1267                   CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
1268                   CP_SET_DRAW_STATE__0_GROUP_ID(0));
1269    tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
1270    tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
1271
1272    tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, 1);
1273    tu_cs_emit(cs, UNK_2D);
1274
1275    tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE, false);
1276    tu6_cache_flush(cmd, cs);
1277
1278    tu_cs_emit_wfi(cs);
1279
1280    tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
1281
1282    emit_vsc_overflow_test(cmd, cs);
1283
1284    tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
1285    tu_cs_emit(cs, 0x0);
1286
1287    tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
1288    tu_cs_emit(cs, 0x0);
1289
1290    tu_cs_emit_wfi(cs);
1291
1292    tu_cs_emit_regs(cs,
1293                    A6XX_RB_CCU_CNTL(.unknown = phys_dev->magic.RB_CCU_CNTL_gmem));
1294
1295    cmd->wait_for_idle = false;
1296 }
1297
1298 static void
1299 tu_emit_sysmem_clear_attachment(struct tu_cmd_buffer *cmd, struct tu_cs *cs,
1300                                 uint32_t a,
1301                                 const VkRenderPassBeginInfo *info)
1302 {
1303    const struct tu_framebuffer *fb = cmd->state.framebuffer;
1304    const struct tu_image_view *iview = fb->attachments[a].attachment;
1305    const struct tu_render_pass_attachment *attachment =
1306       &cmd->state.pass->attachments[a];
1307    unsigned clear_mask = 0;
1308
1309    /* note: this means it isn't used by any subpass and shouldn't be cleared anyway */
1310    if (attachment->gmem_offset < 0)
1311       return;
1312
1313    if (attachment->load_op == VK_ATTACHMENT_LOAD_OP_CLEAR) {
1314       clear_mask = 0xf;
1315    }
1316
1317    if (vk_format_has_stencil(iview->vk_format)) {
1318       clear_mask &= 0x1;
1319       if (attachment->stencil_load_op == VK_ATTACHMENT_LOAD_OP_CLEAR)
1320          clear_mask |= 0x2;
1321       if (clear_mask != 0x3)
1322          tu_finishme("depth/stencil only load op");
1323    }
1324
1325    if (!clear_mask)
1326       return;
1327
1328    tu_clear_sysmem_attachment(cmd, cs, a,
1329                               &info->pClearValues[a], &(struct VkClearRect) {
1330       .rect = info->renderArea,
1331       .baseArrayLayer = iview->base_layer,
1332       .layerCount = iview->layer_count,
1333    });
1334 }
1335
1336 static void
1337 tu_cmd_prepare_sysmem_clear_ib(struct tu_cmd_buffer *cmd,
1338                                const VkRenderPassBeginInfo *info)
1339 {
1340    const struct tu_framebuffer *fb = cmd->state.framebuffer;
1341    const uint32_t blit_cmd_space = 25 + 66 * fb->layers + 17;
1342    const uint32_t clear_space =
1343        blit_cmd_space * cmd->state.pass->attachment_count + 5;
1344
1345    struct tu_cs sub_cs;
1346
1347    VkResult result = tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs,
1348                                             clear_space, &sub_cs);
1349    if (result != VK_SUCCESS) {
1350       cmd->record_result = result;
1351       return;
1352    }
1353
1354    for (uint32_t i = 0; i < cmd->state.pass->attachment_count; ++i)
1355       tu_emit_sysmem_clear_attachment(cmd, &sub_cs, i, info);
1356
1357    /* TODO: We shouldn't need this flush, but without it we'd have an empty IB
1358     * when nothing clears which we currently can't handle.
1359     */
1360    tu_cs_reserve_space(cmd->device, &sub_cs, 5);
1361    tu6_emit_event_write(cmd, &sub_cs, UNK_1D, true);
1362
1363    cmd->state.sysmem_clear_ib = tu_cs_end_sub_stream(&cmd->sub_cs, &sub_cs);
1364 }
1365
1366 static void
1367 tu6_sysmem_render_begin(struct tu_cmd_buffer *cmd, struct tu_cs *cs,
1368                         const struct VkRect2D *renderArea)
1369 {
1370    VkResult result = tu_cs_reserve_space(cmd->device, cs, 1024);
1371    if (result != VK_SUCCESS) {
1372       cmd->record_result = result;
1373       return;
1374    }
1375
1376    const struct tu_framebuffer *fb = cmd->state.framebuffer;
1377    if (fb->width > 0 && fb->height > 0) {
1378       tu6_emit_window_scissor(cmd, cs,
1379                               0, 0, fb->width - 1, fb->height - 1);
1380    } else {
1381       tu6_emit_window_scissor(cmd, cs, 0, 0, 0, 0);
1382    }
1383
1384    tu6_emit_window_offset(cmd, cs, 0, 0);
1385
1386    tu6_emit_bin_size(cs, 0, 0, 0xc00000); /* 0xc00000 = BYPASS? */
1387
1388    tu_cs_emit_ib(cs, &cmd->state.sysmem_clear_ib);
1389
1390    tu6_emit_lrz_flush(cmd, cs);
1391
1392    tu6_emit_marker(cmd, cs);
1393    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1394    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_BYPASS) | 0x10);
1395    tu6_emit_marker(cmd, cs);
1396
1397    tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1398    tu_cs_emit(cs, 0x0);
1399
1400    tu6_emit_event_write(cmd, cs, PC_CCU_INVALIDATE_COLOR, false);
1401    tu6_emit_event_write(cmd, cs, PC_CCU_INVALIDATE_DEPTH, false);
1402    tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE, false);
1403
1404    tu6_emit_wfi(cmd, cs);
1405    tu_cs_emit_regs(cs,
1406                    A6XX_RB_CCU_CNTL(0x10000000));
1407
1408    /* enable stream-out, with sysmem there is only one pass: */
1409    tu_cs_emit_regs(cs,
1410                    A6XX_VPC_SO_OVERRIDE(.so_disable = false));
1411
1412    tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
1413    tu_cs_emit(cs, 0x1);
1414
1415    tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
1416    tu_cs_emit(cs, 0x0);
1417
1418    tu_cs_sanity_check(cs);
1419 }
1420
1421 static void
1422 tu6_sysmem_render_end(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1423 {
1424    const uint32_t space = 14 + tu_cs_get_call_size(&cmd->draw_epilogue_cs);
1425    VkResult result = tu_cs_reserve_space(cmd->device, cs, space);
1426    if (result != VK_SUCCESS) {
1427       cmd->record_result = result;
1428       return;
1429    }
1430
1431    tu_cs_emit_call(cs, &cmd->draw_epilogue_cs);
1432
1433    tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1434    tu_cs_emit(cs, 0x0);
1435
1436    tu6_emit_lrz_flush(cmd, cs);
1437
1438    tu6_emit_event_write(cmd, cs, UNK_1C, true);
1439    tu6_emit_event_write(cmd, cs, UNK_1D, true);
1440
1441    tu_cs_sanity_check(cs);
1442 }
1443
1444
1445 static void
1446 tu6_tile_render_begin(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1447 {
1448    struct tu_physical_device *phys_dev = cmd->device->physical_device;
1449
1450    VkResult result = tu_cs_reserve_space(cmd->device, cs, 1024);
1451    if (result != VK_SUCCESS) {
1452       cmd->record_result = result;
1453       return;
1454    }
1455
1456    tu6_emit_lrz_flush(cmd, cs);
1457
1458    /* lrz clear? */
1459
1460    tu6_emit_cache_flush(cmd, cs);
1461
1462    tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1463    tu_cs_emit(cs, 0x0);
1464
1465    /* 0x10000000 for BYPASS.. 0x7c13c080 for GMEM: */
1466    tu6_emit_wfi(cmd, cs);
1467    tu_cs_emit_regs(cs,
1468                    A6XX_RB_CCU_CNTL(phys_dev->magic.RB_CCU_CNTL_gmem));
1469
1470    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1471    if (use_hw_binning(cmd)) {
1472       tu6_emit_bin_size(cs,
1473                         tiling->tile0.extent.width,
1474                         tiling->tile0.extent.height,
1475                         A6XX_RB_BIN_CONTROL_BINNING_PASS | 0x6000000);
1476
1477       tu6_emit_render_cntl(cmd, cmd->state.subpass, cs, true);
1478
1479       tu6_emit_binning_pass(cmd, cs);
1480
1481       tu6_emit_bin_size(cs,
1482                         tiling->tile0.extent.width,
1483                         tiling->tile0.extent.height,
1484                         A6XX_RB_BIN_CONTROL_USE_VIZ | 0x6000000);
1485
1486       tu_cs_emit_regs(cs,
1487                       A6XX_VFD_MODE_CNTL(0));
1488
1489       tu_cs_emit_regs(cs, A6XX_PC_UNKNOWN_9805(.unknown = phys_dev->magic.PC_UNKNOWN_9805));
1490
1491       tu_cs_emit_regs(cs, A6XX_SP_UNKNOWN_A0F8(.unknown = phys_dev->magic.SP_UNKNOWN_A0F8));
1492
1493       tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1494       tu_cs_emit(cs, 0x1);
1495    } else {
1496       tu6_emit_bin_size(cs,
1497                         tiling->tile0.extent.width,
1498                         tiling->tile0.extent.height,
1499                         0x6000000);
1500    }
1501
1502    tu_cs_sanity_check(cs);
1503 }
1504
1505 static void
1506 tu6_render_tile(struct tu_cmd_buffer *cmd,
1507                 struct tu_cs *cs,
1508                 const struct tu_tile *tile)
1509 {
1510    const uint32_t render_tile_space = 256 + tu_cs_get_call_size(&cmd->draw_cs);
1511    VkResult result = tu_cs_reserve_space(cmd->device, cs, render_tile_space);
1512    if (result != VK_SUCCESS) {
1513       cmd->record_result = result;
1514       return;
1515    }
1516
1517    tu6_emit_tile_select(cmd, cs, tile);
1518    tu_cs_emit_ib(cs, &cmd->state.tile_load_ib);
1519
1520    tu_cs_emit_call(cs, &cmd->draw_cs);
1521    cmd->wait_for_idle = true;
1522
1523    if (use_hw_binning(cmd)) {
1524       tu_cs_emit_pkt7(cs, CP_REG_TEST, 1);
1525       tu_cs_emit(cs, A6XX_CP_REG_TEST_0_REG(OVERFLOW_FLAG_REG) |
1526                      A6XX_CP_REG_TEST_0_BIT(0) |
1527                      A6XX_CP_REG_TEST_0_WAIT_FOR_ME);
1528
1529       tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
1530       tu_cs_emit(cs, 0x10000000);
1531       tu_cs_emit(cs, 2);  /* conditionally execute next 2 dwords */
1532
1533       /* if (no overflow) */ {
1534          tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1535          tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(0x5) | 0x10);
1536       }
1537    }
1538
1539    tu_cs_emit_ib(cs, &cmd->state.tile_store_ib);
1540
1541    tu_cs_sanity_check(cs);
1542 }
1543
1544 static void
1545 tu6_tile_render_end(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1546 {
1547    const uint32_t space = 16 + tu_cs_get_call_size(&cmd->draw_epilogue_cs);
1548    VkResult result = tu_cs_reserve_space(cmd->device, cs, space);
1549    if (result != VK_SUCCESS) {
1550       cmd->record_result = result;
1551       return;
1552    }
1553
1554    tu_cs_emit_call(cs, &cmd->draw_epilogue_cs);
1555
1556    tu_cs_emit_regs(cs,
1557                    A6XX_GRAS_LRZ_CNTL(0));
1558
1559    tu6_emit_lrz_flush(cmd, cs);
1560
1561    tu6_emit_event_write(cmd, cs, CACHE_FLUSH_TS, true);
1562
1563    tu_cs_sanity_check(cs);
1564 }
1565
1566 static void
1567 tu_cmd_render_tiles(struct tu_cmd_buffer *cmd)
1568 {
1569    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1570
1571    tu6_tile_render_begin(cmd, &cmd->cs);
1572
1573    for (uint32_t y = 0; y < tiling->tile_count.height; y++) {
1574       for (uint32_t x = 0; x < tiling->tile_count.width; x++) {
1575          struct tu_tile tile;
1576          tu_tiling_config_get_tile(tiling, cmd->device, x, y, &tile);
1577          tu6_render_tile(cmd, &cmd->cs, &tile);
1578       }
1579    }
1580
1581    tu6_tile_render_end(cmd, &cmd->cs);
1582 }
1583
1584 static void
1585 tu_cmd_render_sysmem(struct tu_cmd_buffer *cmd)
1586 {
1587    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1588
1589    tu6_sysmem_render_begin(cmd, &cmd->cs, &tiling->render_area);
1590
1591    const uint32_t space = tu_cs_get_call_size(&cmd->draw_cs);
1592    VkResult result = tu_cs_reserve_space(cmd->device, &cmd->cs, space);
1593    if (result != VK_SUCCESS) {
1594       cmd->record_result = result;
1595       return;
1596    }
1597
1598    tu_cs_emit_call(&cmd->cs, &cmd->draw_cs);
1599    cmd->wait_for_idle = true;
1600
1601    tu6_sysmem_render_end(cmd, &cmd->cs);
1602 }
1603
1604 static void
1605 tu_cmd_prepare_tile_load_ib(struct tu_cmd_buffer *cmd,
1606                             const VkRenderPassBeginInfo *info)
1607 {
1608    const uint32_t tile_load_space =
1609       2 * 3 /* blit_scissor */ +
1610       (20 /* load */ + 19 /* clear */) * cmd->state.pass->attachment_count +
1611       2 /* cache invalidate */;
1612
1613    struct tu_cs sub_cs;
1614
1615    VkResult result = tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs,
1616                                             tile_load_space, &sub_cs);
1617    if (result != VK_SUCCESS) {
1618       cmd->record_result = result;
1619       return;
1620    }
1621
1622    tu6_emit_blit_scissor(cmd, &sub_cs, true);
1623
1624    for (uint32_t i = 0; i < cmd->state.pass->attachment_count; ++i)
1625       tu6_emit_load_attachment(cmd, &sub_cs, i);
1626
1627    tu6_emit_blit_scissor(cmd, &sub_cs, false);
1628
1629    for (uint32_t i = 0; i < cmd->state.pass->attachment_count; ++i)
1630       tu6_emit_clear_attachment(cmd, &sub_cs, i, info);
1631
1632    /* invalidate because reading input attachments will cache GMEM and
1633     * the cache isn''t updated when GMEM is written
1634     * TODO: is there a no-cache bit for textures?
1635     */
1636    if (cmd->state.subpass->input_count)
1637       tu6_emit_event_write(cmd, &sub_cs, CACHE_INVALIDATE, false);
1638
1639    cmd->state.tile_load_ib = tu_cs_end_sub_stream(&cmd->sub_cs, &sub_cs);
1640 }
1641
1642 static void
1643 tu_cmd_prepare_tile_store_ib(struct tu_cmd_buffer *cmd)
1644 {
1645    const uint32_t tile_store_space = 32 + 23 * cmd->state.pass->attachment_count;
1646    struct tu_cs sub_cs;
1647
1648    VkResult result = tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs,
1649                                             tile_store_space, &sub_cs);
1650    if (result != VK_SUCCESS) {
1651       cmd->record_result = result;
1652       return;
1653    }
1654
1655    /* emit to tile-store sub_cs */
1656    tu6_emit_tile_store(cmd, &sub_cs);
1657
1658    cmd->state.tile_store_ib = tu_cs_end_sub_stream(&cmd->sub_cs, &sub_cs);
1659 }
1660
1661 static void
1662 tu_cmd_update_tiling_config(struct tu_cmd_buffer *cmd,
1663                             const VkRect2D *render_area)
1664 {
1665    const struct tu_device *dev = cmd->device;
1666    struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1667
1668    tiling->render_area = *render_area;
1669
1670    tu_tiling_config_update_tile_layout(tiling, dev, cmd->state.pass->gmem_pixels);
1671    tu_tiling_config_update_pipe_layout(tiling, dev);
1672    tu_tiling_config_update_pipes(tiling, dev);
1673 }
1674
1675 const struct tu_dynamic_state default_dynamic_state = {
1676    .viewport =
1677      {
1678        .count = 0,
1679      },
1680    .scissor =
1681      {
1682        .count = 0,
1683      },
1684    .line_width = 1.0f,
1685    .depth_bias =
1686      {
1687        .bias = 0.0f,
1688        .clamp = 0.0f,
1689        .slope = 0.0f,
1690      },
1691    .blend_constants = { 0.0f, 0.0f, 0.0f, 0.0f },
1692    .depth_bounds =
1693      {
1694        .min = 0.0f,
1695        .max = 1.0f,
1696      },
1697    .stencil_compare_mask =
1698      {
1699        .front = ~0u,
1700        .back = ~0u,
1701      },
1702    .stencil_write_mask =
1703      {
1704        .front = ~0u,
1705        .back = ~0u,
1706      },
1707    .stencil_reference =
1708      {
1709        .front = 0u,
1710        .back = 0u,
1711      },
1712 };
1713
1714 static void UNUSED /* FINISHME */
1715 tu_bind_dynamic_state(struct tu_cmd_buffer *cmd_buffer,
1716                       const struct tu_dynamic_state *src)
1717 {
1718    struct tu_dynamic_state *dest = &cmd_buffer->state.dynamic;
1719    uint32_t copy_mask = src->mask;
1720    uint32_t dest_mask = 0;
1721
1722    tu_use_args(cmd_buffer); /* FINISHME */
1723
1724    /* Make sure to copy the number of viewports/scissors because they can
1725     * only be specified at pipeline creation time.
1726     */
1727    dest->viewport.count = src->viewport.count;
1728    dest->scissor.count = src->scissor.count;
1729    dest->discard_rectangle.count = src->discard_rectangle.count;
1730
1731    if (copy_mask & TU_DYNAMIC_VIEWPORT) {
1732       if (memcmp(&dest->viewport.viewports, &src->viewport.viewports,
1733                  src->viewport.count * sizeof(VkViewport))) {
1734          typed_memcpy(dest->viewport.viewports, src->viewport.viewports,
1735                       src->viewport.count);
1736          dest_mask |= TU_DYNAMIC_VIEWPORT;
1737       }
1738    }
1739
1740    if (copy_mask & TU_DYNAMIC_SCISSOR) {
1741       if (memcmp(&dest->scissor.scissors, &src->scissor.scissors,
1742                  src->scissor.count * sizeof(VkRect2D))) {
1743          typed_memcpy(dest->scissor.scissors, src->scissor.scissors,
1744                       src->scissor.count);
1745          dest_mask |= TU_DYNAMIC_SCISSOR;
1746       }
1747    }
1748
1749    if (copy_mask & TU_DYNAMIC_LINE_WIDTH) {
1750       if (dest->line_width != src->line_width) {
1751          dest->line_width = src->line_width;
1752          dest_mask |= TU_DYNAMIC_LINE_WIDTH;
1753       }
1754    }
1755
1756    if (copy_mask & TU_DYNAMIC_DEPTH_BIAS) {
1757       if (memcmp(&dest->depth_bias, &src->depth_bias,
1758                  sizeof(src->depth_bias))) {
1759          dest->depth_bias = src->depth_bias;
1760          dest_mask |= TU_DYNAMIC_DEPTH_BIAS;
1761       }
1762    }
1763
1764    if (copy_mask & TU_DYNAMIC_BLEND_CONSTANTS) {
1765       if (memcmp(&dest->blend_constants, &src->blend_constants,
1766                  sizeof(src->blend_constants))) {
1767          typed_memcpy(dest->blend_constants, src->blend_constants, 4);
1768          dest_mask |= TU_DYNAMIC_BLEND_CONSTANTS;
1769       }
1770    }
1771
1772    if (copy_mask & TU_DYNAMIC_DEPTH_BOUNDS) {
1773       if (memcmp(&dest->depth_bounds, &src->depth_bounds,
1774                  sizeof(src->depth_bounds))) {
1775          dest->depth_bounds = src->depth_bounds;
1776          dest_mask |= TU_DYNAMIC_DEPTH_BOUNDS;
1777       }
1778    }
1779
1780    if (copy_mask & TU_DYNAMIC_STENCIL_COMPARE_MASK) {
1781       if (memcmp(&dest->stencil_compare_mask, &src->stencil_compare_mask,
1782                  sizeof(src->stencil_compare_mask))) {
1783          dest->stencil_compare_mask = src->stencil_compare_mask;
1784          dest_mask |= TU_DYNAMIC_STENCIL_COMPARE_MASK;
1785       }
1786    }
1787
1788    if (copy_mask & TU_DYNAMIC_STENCIL_WRITE_MASK) {
1789       if (memcmp(&dest->stencil_write_mask, &src->stencil_write_mask,
1790                  sizeof(src->stencil_write_mask))) {
1791          dest->stencil_write_mask = src->stencil_write_mask;
1792          dest_mask |= TU_DYNAMIC_STENCIL_WRITE_MASK;
1793       }
1794    }
1795
1796    if (copy_mask & TU_DYNAMIC_STENCIL_REFERENCE) {
1797       if (memcmp(&dest->stencil_reference, &src->stencil_reference,
1798                  sizeof(src->stencil_reference))) {
1799          dest->stencil_reference = src->stencil_reference;
1800          dest_mask |= TU_DYNAMIC_STENCIL_REFERENCE;
1801       }
1802    }
1803
1804    if (copy_mask & TU_DYNAMIC_DISCARD_RECTANGLE) {
1805       if (memcmp(&dest->discard_rectangle.rectangles,
1806                  &src->discard_rectangle.rectangles,
1807                  src->discard_rectangle.count * sizeof(VkRect2D))) {
1808          typed_memcpy(dest->discard_rectangle.rectangles,
1809                       src->discard_rectangle.rectangles,
1810                       src->discard_rectangle.count);
1811          dest_mask |= TU_DYNAMIC_DISCARD_RECTANGLE;
1812       }
1813    }
1814 }
1815
1816 static VkResult
1817 tu_create_cmd_buffer(struct tu_device *device,
1818                      struct tu_cmd_pool *pool,
1819                      VkCommandBufferLevel level,
1820                      VkCommandBuffer *pCommandBuffer)
1821 {
1822    struct tu_cmd_buffer *cmd_buffer;
1823    cmd_buffer = vk_zalloc(&pool->alloc, sizeof(*cmd_buffer), 8,
1824                           VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1825    if (cmd_buffer == NULL)
1826       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
1827
1828    cmd_buffer->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
1829    cmd_buffer->device = device;
1830    cmd_buffer->pool = pool;
1831    cmd_buffer->level = level;
1832
1833    if (pool) {
1834       list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers);
1835       cmd_buffer->queue_family_index = pool->queue_family_index;
1836
1837    } else {
1838       /* Init the pool_link so we can safely call list_del when we destroy
1839        * the command buffer
1840        */
1841       list_inithead(&cmd_buffer->pool_link);
1842       cmd_buffer->queue_family_index = TU_QUEUE_GENERAL;
1843    }
1844
1845    tu_bo_list_init(&cmd_buffer->bo_list);
1846    tu_cs_init(&cmd_buffer->cs, TU_CS_MODE_GROW, 4096);
1847    tu_cs_init(&cmd_buffer->draw_cs, TU_CS_MODE_GROW, 4096);
1848    tu_cs_init(&cmd_buffer->draw_epilogue_cs, TU_CS_MODE_GROW, 4096);
1849    tu_cs_init(&cmd_buffer->sub_cs, TU_CS_MODE_SUB_STREAM, 2048);
1850
1851    *pCommandBuffer = tu_cmd_buffer_to_handle(cmd_buffer);
1852
1853    list_inithead(&cmd_buffer->upload.list);
1854
1855    cmd_buffer->marker_reg = REG_A6XX_CP_SCRATCH_REG(
1856       cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY ? 7 : 6);
1857
1858    VkResult result = tu_bo_init_new(device, &cmd_buffer->scratch_bo, 0x1000);
1859    if (result != VK_SUCCESS)
1860       goto fail_scratch_bo;
1861
1862    /* TODO: resize on overflow */
1863    cmd_buffer->vsc_data_pitch = device->vsc_data_pitch;
1864    cmd_buffer->vsc_data2_pitch = device->vsc_data2_pitch;
1865    cmd_buffer->vsc_data = device->vsc_data;
1866    cmd_buffer->vsc_data2 = device->vsc_data2;
1867
1868    return VK_SUCCESS;
1869
1870 fail_scratch_bo:
1871    list_del(&cmd_buffer->pool_link);
1872    return result;
1873 }
1874
1875 static void
1876 tu_cmd_buffer_destroy(struct tu_cmd_buffer *cmd_buffer)
1877 {
1878    tu_bo_finish(cmd_buffer->device, &cmd_buffer->scratch_bo);
1879
1880    list_del(&cmd_buffer->pool_link);
1881
1882    for (unsigned i = 0; i < VK_PIPELINE_BIND_POINT_RANGE_SIZE; i++)
1883       free(cmd_buffer->descriptors[i].push_set.set.mapped_ptr);
1884
1885    tu_cs_finish(cmd_buffer->device, &cmd_buffer->cs);
1886    tu_cs_finish(cmd_buffer->device, &cmd_buffer->draw_cs);
1887    tu_cs_finish(cmd_buffer->device, &cmd_buffer->draw_epilogue_cs);
1888    tu_cs_finish(cmd_buffer->device, &cmd_buffer->sub_cs);
1889
1890    tu_bo_list_destroy(&cmd_buffer->bo_list);
1891    vk_free(&cmd_buffer->pool->alloc, cmd_buffer);
1892 }
1893
1894 static VkResult
1895 tu_reset_cmd_buffer(struct tu_cmd_buffer *cmd_buffer)
1896 {
1897    cmd_buffer->wait_for_idle = true;
1898
1899    cmd_buffer->record_result = VK_SUCCESS;
1900
1901    tu_bo_list_reset(&cmd_buffer->bo_list);
1902    tu_cs_reset(cmd_buffer->device, &cmd_buffer->cs);
1903    tu_cs_reset(cmd_buffer->device, &cmd_buffer->draw_cs);
1904    tu_cs_reset(cmd_buffer->device, &cmd_buffer->draw_epilogue_cs);
1905    tu_cs_reset(cmd_buffer->device, &cmd_buffer->sub_cs);
1906
1907    for (unsigned i = 0; i < VK_PIPELINE_BIND_POINT_RANGE_SIZE; i++) {
1908       cmd_buffer->descriptors[i].valid = 0;
1909       cmd_buffer->descriptors[i].push_dirty = false;
1910    }
1911
1912    cmd_buffer->status = TU_CMD_BUFFER_STATUS_INITIAL;
1913
1914    return cmd_buffer->record_result;
1915 }
1916
1917 VkResult
1918 tu_AllocateCommandBuffers(VkDevice _device,
1919                           const VkCommandBufferAllocateInfo *pAllocateInfo,
1920                           VkCommandBuffer *pCommandBuffers)
1921 {
1922    TU_FROM_HANDLE(tu_device, device, _device);
1923    TU_FROM_HANDLE(tu_cmd_pool, pool, pAllocateInfo->commandPool);
1924
1925    VkResult result = VK_SUCCESS;
1926    uint32_t i;
1927
1928    for (i = 0; i < pAllocateInfo->commandBufferCount; i++) {
1929
1930       if (!list_is_empty(&pool->free_cmd_buffers)) {
1931          struct tu_cmd_buffer *cmd_buffer = list_first_entry(
1932             &pool->free_cmd_buffers, struct tu_cmd_buffer, pool_link);
1933
1934          list_del(&cmd_buffer->pool_link);
1935          list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers);
1936
1937          result = tu_reset_cmd_buffer(cmd_buffer);
1938          cmd_buffer->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
1939          cmd_buffer->level = pAllocateInfo->level;
1940
1941          pCommandBuffers[i] = tu_cmd_buffer_to_handle(cmd_buffer);
1942       } else {
1943          result = tu_create_cmd_buffer(device, pool, pAllocateInfo->level,
1944                                        &pCommandBuffers[i]);
1945       }
1946       if (result != VK_SUCCESS)
1947          break;
1948    }
1949
1950    if (result != VK_SUCCESS) {
1951       tu_FreeCommandBuffers(_device, pAllocateInfo->commandPool, i,
1952                             pCommandBuffers);
1953
1954       /* From the Vulkan 1.0.66 spec:
1955        *
1956        * "vkAllocateCommandBuffers can be used to create multiple
1957        *  command buffers. If the creation of any of those command
1958        *  buffers fails, the implementation must destroy all
1959        *  successfully created command buffer objects from this
1960        *  command, set all entries of the pCommandBuffers array to
1961        *  NULL and return the error."
1962        */
1963       memset(pCommandBuffers, 0,
1964              sizeof(*pCommandBuffers) * pAllocateInfo->commandBufferCount);
1965    }
1966
1967    return result;
1968 }
1969
1970 void
1971 tu_FreeCommandBuffers(VkDevice device,
1972                       VkCommandPool commandPool,
1973                       uint32_t commandBufferCount,
1974                       const VkCommandBuffer *pCommandBuffers)
1975 {
1976    for (uint32_t i = 0; i < commandBufferCount; i++) {
1977       TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, pCommandBuffers[i]);
1978
1979       if (cmd_buffer) {
1980          if (cmd_buffer->pool) {
1981             list_del(&cmd_buffer->pool_link);
1982             list_addtail(&cmd_buffer->pool_link,
1983                          &cmd_buffer->pool->free_cmd_buffers);
1984          } else
1985             tu_cmd_buffer_destroy(cmd_buffer);
1986       }
1987    }
1988 }
1989
1990 VkResult
1991 tu_ResetCommandBuffer(VkCommandBuffer commandBuffer,
1992                       VkCommandBufferResetFlags flags)
1993 {
1994    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
1995    return tu_reset_cmd_buffer(cmd_buffer);
1996 }
1997
1998 VkResult
1999 tu_BeginCommandBuffer(VkCommandBuffer commandBuffer,
2000                       const VkCommandBufferBeginInfo *pBeginInfo)
2001 {
2002    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
2003    VkResult result = VK_SUCCESS;
2004
2005    if (cmd_buffer->status != TU_CMD_BUFFER_STATUS_INITIAL) {
2006       /* If the command buffer has already been resetted with
2007        * vkResetCommandBuffer, no need to do it again.
2008        */
2009       result = tu_reset_cmd_buffer(cmd_buffer);
2010       if (result != VK_SUCCESS)
2011          return result;
2012    }
2013
2014    memset(&cmd_buffer->state, 0, sizeof(cmd_buffer->state));
2015    cmd_buffer->usage_flags = pBeginInfo->flags;
2016
2017    tu_cs_begin(&cmd_buffer->cs);
2018    tu_cs_begin(&cmd_buffer->draw_cs);
2019    tu_cs_begin(&cmd_buffer->draw_epilogue_cs);
2020
2021    cmd_buffer->marker_seqno = 0;
2022    cmd_buffer->scratch_seqno = 0;
2023
2024    /* setup initial configuration into command buffer */
2025    if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY) {
2026       switch (cmd_buffer->queue_family_index) {
2027       case TU_QUEUE_GENERAL:
2028          tu6_init_hw(cmd_buffer, &cmd_buffer->cs);
2029          break;
2030       default:
2031          break;
2032       }
2033    } else if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY &&
2034               (pBeginInfo->flags & VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT)) {
2035       assert(pBeginInfo->pInheritanceInfo);
2036       cmd_buffer->state.pass = tu_render_pass_from_handle(pBeginInfo->pInheritanceInfo->renderPass);
2037       cmd_buffer->state.subpass = &cmd_buffer->state.pass->subpasses[pBeginInfo->pInheritanceInfo->subpass];
2038    }
2039
2040    cmd_buffer->status = TU_CMD_BUFFER_STATUS_RECORDING;
2041
2042    return VK_SUCCESS;
2043 }
2044
2045 void
2046 tu_CmdBindVertexBuffers(VkCommandBuffer commandBuffer,
2047                         uint32_t firstBinding,
2048                         uint32_t bindingCount,
2049                         const VkBuffer *pBuffers,
2050                         const VkDeviceSize *pOffsets)
2051 {
2052    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2053
2054    assert(firstBinding + bindingCount <= MAX_VBS);
2055
2056    for (uint32_t i = 0; i < bindingCount; i++) {
2057       cmd->state.vb.buffers[firstBinding + i] =
2058          tu_buffer_from_handle(pBuffers[i]);
2059       cmd->state.vb.offsets[firstBinding + i] = pOffsets[i];
2060    }
2061
2062    /* VB states depend on VkPipelineVertexInputStateCreateInfo */
2063    cmd->state.dirty |= TU_CMD_DIRTY_VERTEX_BUFFERS;
2064 }
2065
2066 void
2067 tu_CmdBindIndexBuffer(VkCommandBuffer commandBuffer,
2068                       VkBuffer buffer,
2069                       VkDeviceSize offset,
2070                       VkIndexType indexType)
2071 {
2072    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2073    TU_FROM_HANDLE(tu_buffer, buf, buffer);
2074
2075    /* initialize/update the restart index */
2076    if (!cmd->state.index_buffer || cmd->state.index_type != indexType) {
2077       struct tu_cs *draw_cs = &cmd->draw_cs;
2078       VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 2);
2079       if (result != VK_SUCCESS) {
2080          cmd->record_result = result;
2081          return;
2082       }
2083
2084       tu6_emit_restart_index(
2085          draw_cs, indexType == VK_INDEX_TYPE_UINT32 ? 0xffffffff : 0xffff);
2086
2087       tu_cs_sanity_check(draw_cs);
2088    }
2089
2090    /* track the BO */
2091    if (cmd->state.index_buffer != buf)
2092       tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
2093
2094    cmd->state.index_buffer = buf;
2095    cmd->state.index_offset = offset;
2096    cmd->state.index_type = indexType;
2097 }
2098
2099 void
2100 tu_CmdBindDescriptorSets(VkCommandBuffer commandBuffer,
2101                          VkPipelineBindPoint pipelineBindPoint,
2102                          VkPipelineLayout _layout,
2103                          uint32_t firstSet,
2104                          uint32_t descriptorSetCount,
2105                          const VkDescriptorSet *pDescriptorSets,
2106                          uint32_t dynamicOffsetCount,
2107                          const uint32_t *pDynamicOffsets)
2108 {
2109    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
2110    TU_FROM_HANDLE(tu_pipeline_layout, layout, _layout);
2111    unsigned dyn_idx = 0;
2112
2113    struct tu_descriptor_state *descriptors_state =
2114       tu_get_descriptors_state(cmd_buffer, pipelineBindPoint);
2115
2116    for (unsigned i = 0; i < descriptorSetCount; ++i) {
2117       unsigned idx = i + firstSet;
2118       TU_FROM_HANDLE(tu_descriptor_set, set, pDescriptorSets[i]);
2119
2120       descriptors_state->sets[idx] = set;
2121       descriptors_state->valid |= (1u << idx);
2122
2123       for(unsigned j = 0; j < set->layout->dynamic_offset_count; ++j, ++dyn_idx) {
2124          unsigned idx = j + layout->set[i + firstSet].dynamic_offset_start;
2125          assert(dyn_idx < dynamicOffsetCount);
2126
2127          descriptors_state->dynamic_buffers[idx] =
2128          set->dynamic_descriptors[j].va + pDynamicOffsets[dyn_idx];
2129       }
2130    }
2131
2132    cmd_buffer->state.dirty |= TU_CMD_DIRTY_DESCRIPTOR_SETS;
2133 }
2134
2135 void
2136 tu_CmdPushConstants(VkCommandBuffer commandBuffer,
2137                     VkPipelineLayout layout,
2138                     VkShaderStageFlags stageFlags,
2139                     uint32_t offset,
2140                     uint32_t size,
2141                     const void *pValues)
2142 {
2143    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2144    memcpy((void*) cmd->push_constants + offset, pValues, size);
2145    cmd->state.dirty |= TU_CMD_DIRTY_PUSH_CONSTANTS;
2146 }
2147
2148 VkResult
2149 tu_EndCommandBuffer(VkCommandBuffer commandBuffer)
2150 {
2151    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
2152
2153    if (cmd_buffer->scratch_seqno) {
2154       tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->scratch_bo,
2155                      MSM_SUBMIT_BO_WRITE);
2156    }
2157
2158    if (cmd_buffer->use_vsc_data) {
2159       tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->vsc_data,
2160                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
2161       tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->vsc_data2,
2162                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
2163    }
2164
2165    for (uint32_t i = 0; i < cmd_buffer->draw_cs.bo_count; i++) {
2166       tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->draw_cs.bos[i],
2167                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2168    }
2169
2170    for (uint32_t i = 0; i < cmd_buffer->draw_epilogue_cs.bo_count; i++) {
2171       tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->draw_epilogue_cs.bos[i],
2172                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2173    }
2174
2175    for (uint32_t i = 0; i < cmd_buffer->sub_cs.bo_count; i++) {
2176       tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->sub_cs.bos[i],
2177                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2178    }
2179
2180    tu_cs_end(&cmd_buffer->cs);
2181    tu_cs_end(&cmd_buffer->draw_cs);
2182    tu_cs_end(&cmd_buffer->draw_epilogue_cs);
2183
2184    cmd_buffer->status = TU_CMD_BUFFER_STATUS_EXECUTABLE;
2185
2186    return cmd_buffer->record_result;
2187 }
2188
2189 void
2190 tu_CmdBindPipeline(VkCommandBuffer commandBuffer,
2191                    VkPipelineBindPoint pipelineBindPoint,
2192                    VkPipeline _pipeline)
2193 {
2194    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2195    TU_FROM_HANDLE(tu_pipeline, pipeline, _pipeline);
2196
2197    switch (pipelineBindPoint) {
2198    case VK_PIPELINE_BIND_POINT_GRAPHICS:
2199       cmd->state.pipeline = pipeline;
2200       cmd->state.dirty |= TU_CMD_DIRTY_PIPELINE;
2201       break;
2202    case VK_PIPELINE_BIND_POINT_COMPUTE:
2203       cmd->state.compute_pipeline = pipeline;
2204       cmd->state.dirty |= TU_CMD_DIRTY_COMPUTE_PIPELINE;
2205       break;
2206    default:
2207       unreachable("unrecognized pipeline bind point");
2208       break;
2209    }
2210
2211    tu_bo_list_add(&cmd->bo_list, &pipeline->program.binary_bo,
2212                   MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2213    for (uint32_t i = 0; i < pipeline->cs.bo_count; i++) {
2214       tu_bo_list_add(&cmd->bo_list, pipeline->cs.bos[i],
2215                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2216    }
2217 }
2218
2219 void
2220 tu_CmdSetViewport(VkCommandBuffer commandBuffer,
2221                   uint32_t firstViewport,
2222                   uint32_t viewportCount,
2223                   const VkViewport *pViewports)
2224 {
2225    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2226    struct tu_cs *draw_cs = &cmd->draw_cs;
2227
2228    VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 12);
2229    if (result != VK_SUCCESS) {
2230       cmd->record_result = result;
2231       return;
2232    }
2233
2234    assert(firstViewport == 0 && viewportCount == 1);
2235    tu6_emit_viewport(draw_cs, pViewports);
2236
2237    tu_cs_sanity_check(draw_cs);
2238 }
2239
2240 void
2241 tu_CmdSetScissor(VkCommandBuffer commandBuffer,
2242                  uint32_t firstScissor,
2243                  uint32_t scissorCount,
2244                  const VkRect2D *pScissors)
2245 {
2246    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2247    struct tu_cs *draw_cs = &cmd->draw_cs;
2248
2249    VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 3);
2250    if (result != VK_SUCCESS) {
2251       cmd->record_result = result;
2252       return;
2253    }
2254
2255    assert(firstScissor == 0 && scissorCount == 1);
2256    tu6_emit_scissor(draw_cs, pScissors);
2257
2258    tu_cs_sanity_check(draw_cs);
2259 }
2260
2261 void
2262 tu_CmdSetLineWidth(VkCommandBuffer commandBuffer, float lineWidth)
2263 {
2264    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2265
2266    cmd->state.dynamic.line_width = lineWidth;
2267
2268    /* line width depends on VkPipelineRasterizationStateCreateInfo */
2269    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_LINE_WIDTH;
2270 }
2271
2272 void
2273 tu_CmdSetDepthBias(VkCommandBuffer commandBuffer,
2274                    float depthBiasConstantFactor,
2275                    float depthBiasClamp,
2276                    float depthBiasSlopeFactor)
2277 {
2278    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2279    struct tu_cs *draw_cs = &cmd->draw_cs;
2280
2281    VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 4);
2282    if (result != VK_SUCCESS) {
2283       cmd->record_result = result;
2284       return;
2285    }
2286
2287    tu6_emit_depth_bias(draw_cs, depthBiasConstantFactor, depthBiasClamp,
2288                        depthBiasSlopeFactor);
2289
2290    tu_cs_sanity_check(draw_cs);
2291 }
2292
2293 void
2294 tu_CmdSetBlendConstants(VkCommandBuffer commandBuffer,
2295                         const float blendConstants[4])
2296 {
2297    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2298    struct tu_cs *draw_cs = &cmd->draw_cs;
2299
2300    VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 5);
2301    if (result != VK_SUCCESS) {
2302       cmd->record_result = result;
2303       return;
2304    }
2305
2306    tu6_emit_blend_constants(draw_cs, blendConstants);
2307
2308    tu_cs_sanity_check(draw_cs);
2309 }
2310
2311 void
2312 tu_CmdSetDepthBounds(VkCommandBuffer commandBuffer,
2313                      float minDepthBounds,
2314                      float maxDepthBounds)
2315 {
2316 }
2317
2318 void
2319 tu_CmdSetStencilCompareMask(VkCommandBuffer commandBuffer,
2320                             VkStencilFaceFlags faceMask,
2321                             uint32_t compareMask)
2322 {
2323    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2324
2325    if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
2326       cmd->state.dynamic.stencil_compare_mask.front = compareMask;
2327    if (faceMask & VK_STENCIL_FACE_BACK_BIT)
2328       cmd->state.dynamic.stencil_compare_mask.back = compareMask;
2329
2330    /* the front/back compare masks must be updated together */
2331    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK;
2332 }
2333
2334 void
2335 tu_CmdSetStencilWriteMask(VkCommandBuffer commandBuffer,
2336                           VkStencilFaceFlags faceMask,
2337                           uint32_t writeMask)
2338 {
2339    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2340
2341    if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
2342       cmd->state.dynamic.stencil_write_mask.front = writeMask;
2343    if (faceMask & VK_STENCIL_FACE_BACK_BIT)
2344       cmd->state.dynamic.stencil_write_mask.back = writeMask;
2345
2346    /* the front/back write masks must be updated together */
2347    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK;
2348 }
2349
2350 void
2351 tu_CmdSetStencilReference(VkCommandBuffer commandBuffer,
2352                           VkStencilFaceFlags faceMask,
2353                           uint32_t reference)
2354 {
2355    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2356
2357    if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
2358       cmd->state.dynamic.stencil_reference.front = reference;
2359    if (faceMask & VK_STENCIL_FACE_BACK_BIT)
2360       cmd->state.dynamic.stencil_reference.back = reference;
2361
2362    /* the front/back references must be updated together */
2363    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE;
2364 }
2365
2366 void
2367 tu_CmdExecuteCommands(VkCommandBuffer commandBuffer,
2368                       uint32_t commandBufferCount,
2369                       const VkCommandBuffer *pCmdBuffers)
2370 {
2371    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2372    VkResult result;
2373
2374    assert(commandBufferCount > 0);
2375
2376    for (uint32_t i = 0; i < commandBufferCount; i++) {
2377       TU_FROM_HANDLE(tu_cmd_buffer, secondary, pCmdBuffers[i]);
2378
2379       result = tu_bo_list_merge(&cmd->bo_list, &secondary->bo_list);
2380       if (result != VK_SUCCESS) {
2381          cmd->record_result = result;
2382          break;
2383       }
2384
2385       result = tu_cs_add_entries(&cmd->draw_cs, &secondary->draw_cs);
2386       if (result != VK_SUCCESS) {
2387          cmd->record_result = result;
2388          break;
2389       }
2390
2391       result = tu_cs_add_entries(&cmd->draw_epilogue_cs,
2392             &secondary->draw_epilogue_cs);
2393       if (result != VK_SUCCESS) {
2394          cmd->record_result = result;
2395          break;
2396       }
2397    }
2398    cmd->state.dirty = ~0u; /* TODO: set dirty only what needs to be */
2399 }
2400
2401 VkResult
2402 tu_CreateCommandPool(VkDevice _device,
2403                      const VkCommandPoolCreateInfo *pCreateInfo,
2404                      const VkAllocationCallbacks *pAllocator,
2405                      VkCommandPool *pCmdPool)
2406 {
2407    TU_FROM_HANDLE(tu_device, device, _device);
2408    struct tu_cmd_pool *pool;
2409
2410    pool = vk_alloc2(&device->alloc, pAllocator, sizeof(*pool), 8,
2411                     VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2412    if (pool == NULL)
2413       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
2414
2415    if (pAllocator)
2416       pool->alloc = *pAllocator;
2417    else
2418       pool->alloc = device->alloc;
2419
2420    list_inithead(&pool->cmd_buffers);
2421    list_inithead(&pool->free_cmd_buffers);
2422
2423    pool->queue_family_index = pCreateInfo->queueFamilyIndex;
2424
2425    *pCmdPool = tu_cmd_pool_to_handle(pool);
2426
2427    return VK_SUCCESS;
2428 }
2429
2430 void
2431 tu_DestroyCommandPool(VkDevice _device,
2432                       VkCommandPool commandPool,
2433                       const VkAllocationCallbacks *pAllocator)
2434 {
2435    TU_FROM_HANDLE(tu_device, device, _device);
2436    TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2437
2438    if (!pool)
2439       return;
2440
2441    list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2442                             &pool->cmd_buffers, pool_link)
2443    {
2444       tu_cmd_buffer_destroy(cmd_buffer);
2445    }
2446
2447    list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2448                             &pool->free_cmd_buffers, pool_link)
2449    {
2450       tu_cmd_buffer_destroy(cmd_buffer);
2451    }
2452
2453    vk_free2(&device->alloc, pAllocator, pool);
2454 }
2455
2456 VkResult
2457 tu_ResetCommandPool(VkDevice device,
2458                     VkCommandPool commandPool,
2459                     VkCommandPoolResetFlags flags)
2460 {
2461    TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2462    VkResult result;
2463
2464    list_for_each_entry(struct tu_cmd_buffer, cmd_buffer, &pool->cmd_buffers,
2465                        pool_link)
2466    {
2467       result = tu_reset_cmd_buffer(cmd_buffer);
2468       if (result != VK_SUCCESS)
2469          return result;
2470    }
2471
2472    return VK_SUCCESS;
2473 }
2474
2475 void
2476 tu_TrimCommandPool(VkDevice device,
2477                    VkCommandPool commandPool,
2478                    VkCommandPoolTrimFlags flags)
2479 {
2480    TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2481
2482    if (!pool)
2483       return;
2484
2485    list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2486                             &pool->free_cmd_buffers, pool_link)
2487    {
2488       tu_cmd_buffer_destroy(cmd_buffer);
2489    }
2490 }
2491
2492 void
2493 tu_CmdBeginRenderPass(VkCommandBuffer commandBuffer,
2494                       const VkRenderPassBeginInfo *pRenderPassBegin,
2495                       VkSubpassContents contents)
2496 {
2497    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2498    TU_FROM_HANDLE(tu_render_pass, pass, pRenderPassBegin->renderPass);
2499    TU_FROM_HANDLE(tu_framebuffer, fb, pRenderPassBegin->framebuffer);
2500
2501    cmd->state.pass = pass;
2502    cmd->state.subpass = pass->subpasses;
2503    cmd->state.framebuffer = fb;
2504
2505    tu_cmd_update_tiling_config(cmd, &pRenderPassBegin->renderArea);
2506    tu_cmd_prepare_sysmem_clear_ib(cmd, pRenderPassBegin);
2507    tu_cmd_prepare_tile_load_ib(cmd, pRenderPassBegin);
2508    tu_cmd_prepare_tile_store_ib(cmd);
2509
2510    VkResult result = tu_cs_reserve_space(cmd->device, &cmd->draw_cs, 1024);
2511    if (result != VK_SUCCESS) {
2512       cmd->record_result = result;
2513       return;
2514    }
2515
2516    tu6_emit_zs(cmd, cmd->state.subpass, &cmd->draw_cs);
2517    tu6_emit_mrt(cmd, cmd->state.subpass, &cmd->draw_cs);
2518    tu6_emit_msaa(cmd, cmd->state.subpass, &cmd->draw_cs);
2519    tu6_emit_render_cntl(cmd, cmd->state.subpass, &cmd->draw_cs, false);
2520
2521    /* note: use_hw_binning only checks tiling config */
2522    if (use_hw_binning(cmd))
2523       cmd->use_vsc_data = true;
2524
2525    for (uint32_t i = 0; i < fb->attachment_count; ++i) {
2526       const struct tu_image_view *iview = fb->attachments[i].attachment;
2527       tu_bo_list_add(&cmd->bo_list, iview->image->bo,
2528                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
2529    }
2530 }
2531
2532 void
2533 tu_CmdBeginRenderPass2(VkCommandBuffer commandBuffer,
2534                        const VkRenderPassBeginInfo *pRenderPassBeginInfo,
2535                        const VkSubpassBeginInfoKHR *pSubpassBeginInfo)
2536 {
2537    tu_CmdBeginRenderPass(commandBuffer, pRenderPassBeginInfo,
2538                          pSubpassBeginInfo->contents);
2539 }
2540
2541 void
2542 tu_CmdNextSubpass(VkCommandBuffer commandBuffer, VkSubpassContents contents)
2543 {
2544    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2545    const struct tu_render_pass *pass = cmd->state.pass;
2546    struct tu_cs *cs = &cmd->draw_cs;
2547
2548    VkResult result = tu_cs_reserve_space(cmd->device, cs, 1024);
2549     if (result != VK_SUCCESS) {
2550        cmd->record_result = result;
2551        return;
2552    }
2553
2554    const struct tu_subpass *subpass = cmd->state.subpass++;
2555    /* TODO:
2556     * if msaa samples change between subpasses,
2557     * attachment store is broken for some attachments
2558     */
2559    if (subpass->resolve_attachments) {
2560       tu6_emit_blit_scissor(cmd, cs, true);
2561       for (unsigned i = 0; i < subpass->color_count; i++) {
2562          uint32_t a = subpass->resolve_attachments[i].attachment;
2563          if (a != VK_ATTACHMENT_UNUSED) {
2564                tu6_emit_store_attachment(cmd, cs, a,
2565                                          subpass->color_attachments[i].attachment);
2566          }
2567       }
2568    }
2569
2570    /* invalidate because reading input attachments will cache GMEM and
2571     * the cache isn''t updated when GMEM is written
2572     * TODO: is there a no-cache bit for textures?
2573     */
2574    if (cmd->state.subpass->input_count)
2575       tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE, false);
2576
2577    /* emit mrt/zs/msaa/ubwc state for the subpass that is starting */
2578    tu6_emit_zs(cmd, cmd->state.subpass, cs);
2579    tu6_emit_mrt(cmd, cmd->state.subpass, cs);
2580    tu6_emit_msaa(cmd, cmd->state.subpass, cs);
2581    tu6_emit_render_cntl(cmd, cmd->state.subpass, cs, false);
2582
2583    /* Emit flushes so that input attachments will read the correct value. This
2584     * is for sysmem only, although it shouldn't do much harm on gmem.
2585     */
2586    tu6_emit_event_write(cmd, cs, UNK_1C, true);
2587    tu6_emit_event_write(cmd, cs, UNK_1D, true);
2588
2589    /* TODO:
2590     * since we don't know how to do GMEM->GMEM resolve,
2591     * resolve attachments are resolved to memory then loaded to GMEM again if needed
2592     */
2593    if (subpass->resolve_attachments) {
2594       for (unsigned i = 0; i < subpass->color_count; i++) {
2595          uint32_t a = subpass->resolve_attachments[i].attachment;
2596          const struct tu_image_view *iview =
2597             cmd->state.framebuffer->attachments[a].attachment;
2598          if (a != VK_ATTACHMENT_UNUSED && pass->attachments[a].gmem_offset >= 0) {
2599                tu_finishme("missing GMEM->GMEM resolve, performance will suffer\n");
2600                tu6_emit_blit_info(cmd, cs, iview, pass->attachments[a].gmem_offset, false);
2601                tu6_emit_blit(cmd, cs);
2602          }
2603       }
2604    }
2605 }
2606
2607 void
2608 tu_CmdNextSubpass2(VkCommandBuffer commandBuffer,
2609                    const VkSubpassBeginInfoKHR *pSubpassBeginInfo,
2610                    const VkSubpassEndInfoKHR *pSubpassEndInfo)
2611 {
2612    tu_CmdNextSubpass(commandBuffer, pSubpassBeginInfo->contents);
2613 }
2614
2615 struct tu_draw_info
2616 {
2617    /**
2618     * Number of vertices.
2619     */
2620    uint32_t count;
2621
2622    /**
2623     * Index of the first vertex.
2624     */
2625    int32_t vertex_offset;
2626
2627    /**
2628     * First instance id.
2629     */
2630    uint32_t first_instance;
2631
2632    /**
2633     * Number of instances.
2634     */
2635    uint32_t instance_count;
2636
2637    /**
2638     * First index (indexed draws only).
2639     */
2640    uint32_t first_index;
2641
2642    /**
2643     * Whether it's an indexed draw.
2644     */
2645    bool indexed;
2646
2647    /**
2648     * Indirect draw parameters resource.
2649     */
2650    struct tu_buffer *indirect;
2651    uint64_t indirect_offset;
2652    uint32_t stride;
2653
2654    /**
2655     * Draw count parameters resource.
2656     */
2657    struct tu_buffer *count_buffer;
2658    uint64_t count_buffer_offset;
2659 };
2660
2661 #define ENABLE_ALL (CP_SET_DRAW_STATE__0_BINNING | CP_SET_DRAW_STATE__0_GMEM | CP_SET_DRAW_STATE__0_SYSMEM)
2662 #define ENABLE_DRAW (CP_SET_DRAW_STATE__0_GMEM | CP_SET_DRAW_STATE__0_SYSMEM)
2663
2664 enum tu_draw_state_group_id
2665 {
2666    TU_DRAW_STATE_PROGRAM,
2667    TU_DRAW_STATE_PROGRAM_BINNING,
2668    TU_DRAW_STATE_VI,
2669    TU_DRAW_STATE_VI_BINNING,
2670    TU_DRAW_STATE_VP,
2671    TU_DRAW_STATE_RAST,
2672    TU_DRAW_STATE_DS,
2673    TU_DRAW_STATE_BLEND,
2674    TU_DRAW_STATE_VS_CONST,
2675    TU_DRAW_STATE_FS_CONST,
2676    TU_DRAW_STATE_VS_TEX,
2677    TU_DRAW_STATE_FS_TEX,
2678    TU_DRAW_STATE_FS_IBO,
2679    TU_DRAW_STATE_VS_PARAMS,
2680
2681    TU_DRAW_STATE_COUNT,
2682 };
2683
2684 struct tu_draw_state_group
2685 {
2686    enum tu_draw_state_group_id id;
2687    uint32_t enable_mask;
2688    struct tu_cs_entry ib;
2689 };
2690
2691 const static struct tu_sampler*
2692 sampler_ptr(struct tu_descriptor_state *descriptors_state,
2693             const struct tu_descriptor_map *map, unsigned i,
2694             unsigned array_index)
2695 {
2696    assert(descriptors_state->valid & (1 << map->set[i]));
2697
2698    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2699    assert(map->binding[i] < set->layout->binding_count);
2700
2701    const struct tu_descriptor_set_binding_layout *layout =
2702       &set->layout->binding[map->binding[i]];
2703
2704    if (layout->immutable_samplers_offset) {
2705       const struct tu_sampler *immutable_samplers =
2706          tu_immutable_samplers(set->layout, layout);
2707
2708       return &immutable_samplers[array_index];
2709    }
2710
2711    switch (layout->type) {
2712    case VK_DESCRIPTOR_TYPE_SAMPLER:
2713       return (struct tu_sampler*) &set->mapped_ptr[layout->offset / 4];
2714    case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
2715       return (struct tu_sampler*) &set->mapped_ptr[layout->offset / 4 + A6XX_TEX_CONST_DWORDS +
2716                                                    array_index *
2717                                                    (A6XX_TEX_CONST_DWORDS +
2718                                                     sizeof(struct tu_sampler) / 4)];
2719    default:
2720       unreachable("unimplemented descriptor type");
2721       break;
2722    }
2723 }
2724
2725 static void
2726 write_tex_const(struct tu_cmd_buffer *cmd,
2727                 uint32_t *dst,
2728                 struct tu_descriptor_state *descriptors_state,
2729                 const struct tu_descriptor_map *map,
2730                 unsigned i, unsigned array_index)
2731 {
2732    assert(descriptors_state->valid & (1 << map->set[i]));
2733
2734    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2735    assert(map->binding[i] < set->layout->binding_count);
2736
2737    const struct tu_descriptor_set_binding_layout *layout =
2738       &set->layout->binding[map->binding[i]];
2739
2740    switch (layout->type) {
2741    case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
2742    case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
2743    case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
2744    case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
2745       memcpy(dst, &set->mapped_ptr[layout->offset / 4 +
2746                                    array_index * A6XX_TEX_CONST_DWORDS],
2747              A6XX_TEX_CONST_DWORDS * 4);
2748       break;
2749    case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
2750       memcpy(dst, &set->mapped_ptr[layout->offset / 4 +
2751                                    array_index *
2752                                    (A6XX_TEX_CONST_DWORDS +
2753                                     sizeof(struct tu_sampler) / 4)],
2754              A6XX_TEX_CONST_DWORDS * 4);
2755       break;
2756    default:
2757       unreachable("unimplemented descriptor type");
2758       break;
2759    }
2760
2761    if (layout->type == VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT) {
2762       const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
2763       uint32_t a = cmd->state.subpass->input_attachments[map->value[i] +
2764                                                          array_index].attachment;
2765       const struct tu_render_pass_attachment *att = &cmd->state.pass->attachments[a];
2766
2767       assert(att->gmem_offset >= 0);
2768
2769       dst[0] &= ~(A6XX_TEX_CONST_0_SWAP__MASK | A6XX_TEX_CONST_0_TILE_MODE__MASK);
2770       dst[0] |= A6XX_TEX_CONST_0_TILE_MODE(TILE6_2);
2771       dst[2] &= ~(A6XX_TEX_CONST_2_TYPE__MASK | A6XX_TEX_CONST_2_PITCH__MASK);
2772       dst[2] |=
2773          A6XX_TEX_CONST_2_TYPE(A6XX_TEX_2D) |
2774          A6XX_TEX_CONST_2_PITCH(tiling->tile0.extent.width * att->cpp);
2775       dst[3] = 0;
2776       dst[4] = 0x100000 + att->gmem_offset;
2777       dst[5] = A6XX_TEX_CONST_5_DEPTH(1);
2778       for (unsigned i = 6; i < A6XX_TEX_CONST_DWORDS; i++)
2779          dst[i] = 0;
2780
2781       if (cmd->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY)
2782          tu_finishme("patch input attachment pitch for secondary cmd buffer");
2783    }
2784 }
2785
2786 static void
2787 write_image_ibo(struct tu_cmd_buffer *cmd,
2788                 uint32_t *dst,
2789                 struct tu_descriptor_state *descriptors_state,
2790                 const struct tu_descriptor_map *map,
2791                 unsigned i, unsigned array_index)
2792 {
2793    assert(descriptors_state->valid & (1 << map->set[i]));
2794
2795    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2796    assert(map->binding[i] < set->layout->binding_count);
2797
2798    const struct tu_descriptor_set_binding_layout *layout =
2799       &set->layout->binding[map->binding[i]];
2800
2801    assert(layout->type == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE);
2802
2803    memcpy(dst, &set->mapped_ptr[layout->offset / 4 +
2804                                 (array_index * 2 + 1) * A6XX_TEX_CONST_DWORDS],
2805           A6XX_TEX_CONST_DWORDS * 4);
2806 }
2807
2808 static uint64_t
2809 buffer_ptr(struct tu_descriptor_state *descriptors_state,
2810            const struct tu_descriptor_map *map,
2811            unsigned i, unsigned array_index)
2812 {
2813    assert(descriptors_state->valid & (1 << map->set[i]));
2814
2815    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2816    assert(map->binding[i] < set->layout->binding_count);
2817
2818    const struct tu_descriptor_set_binding_layout *layout =
2819       &set->layout->binding[map->binding[i]];
2820
2821    switch (layout->type) {
2822    case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
2823    case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
2824       return descriptors_state->dynamic_buffers[layout->dynamic_offset_offset +
2825                                                 array_index];
2826    case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
2827    case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
2828       return (uint64_t) set->mapped_ptr[layout->offset / 4 + array_index * 2 + 1] << 32 |
2829                         set->mapped_ptr[layout->offset / 4 + array_index * 2];
2830    default:
2831       unreachable("unimplemented descriptor type");
2832       break;
2833    }
2834 }
2835
2836 static inline uint32_t
2837 tu6_stage2opcode(gl_shader_stage type)
2838 {
2839    switch (type) {
2840    case MESA_SHADER_VERTEX:
2841    case MESA_SHADER_TESS_CTRL:
2842    case MESA_SHADER_TESS_EVAL:
2843    case MESA_SHADER_GEOMETRY:
2844       return CP_LOAD_STATE6_GEOM;
2845    case MESA_SHADER_FRAGMENT:
2846    case MESA_SHADER_COMPUTE:
2847    case MESA_SHADER_KERNEL:
2848       return CP_LOAD_STATE6_FRAG;
2849    default:
2850       unreachable("bad shader type");
2851    }
2852 }
2853
2854 static inline enum a6xx_state_block
2855 tu6_stage2shadersb(gl_shader_stage type)
2856 {
2857    switch (type) {
2858    case MESA_SHADER_VERTEX:
2859       return SB6_VS_SHADER;
2860    case MESA_SHADER_FRAGMENT:
2861       return SB6_FS_SHADER;
2862    case MESA_SHADER_COMPUTE:
2863    case MESA_SHADER_KERNEL:
2864       return SB6_CS_SHADER;
2865    default:
2866       unreachable("bad shader type");
2867       return ~0;
2868    }
2869 }
2870
2871 static void
2872 tu6_emit_user_consts(struct tu_cs *cs, const struct tu_pipeline *pipeline,
2873                      struct tu_descriptor_state *descriptors_state,
2874                      gl_shader_stage type,
2875                      uint32_t *push_constants)
2876 {
2877    const struct tu_program_descriptor_linkage *link =
2878       &pipeline->program.link[type];
2879    const struct ir3_ubo_analysis_state *state = &link->ubo_state;
2880
2881    for (uint32_t i = 0; i < ARRAY_SIZE(state->range); i++) {
2882       if (state->range[i].start < state->range[i].end) {
2883          uint32_t size = state->range[i].end - state->range[i].start;
2884          uint32_t offset = state->range[i].start;
2885
2886          /* and even if the start of the const buffer is before
2887           * first_immediate, the end may not be:
2888           */
2889          size = MIN2(size, (16 * link->constlen) - state->range[i].offset);
2890
2891          if (size == 0)
2892             continue;
2893
2894          /* things should be aligned to vec4: */
2895          debug_assert((state->range[i].offset % 16) == 0);
2896          debug_assert((size % 16) == 0);
2897          debug_assert((offset % 16) == 0);
2898
2899          if (i == 0) {
2900             /* push constants */
2901             tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + (size / 4));
2902             tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(state->range[i].offset / 16) |
2903                   CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2904                   CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
2905                   CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2906                   CP_LOAD_STATE6_0_NUM_UNIT(size / 16));
2907             tu_cs_emit(cs, 0);
2908             tu_cs_emit(cs, 0);
2909             for (unsigned i = 0; i < size / 4; i++)
2910                tu_cs_emit(cs, push_constants[i + offset / 4]);
2911             continue;
2912          }
2913
2914          /* Look through the UBO map to find our UBO index, and get the VA for
2915           * that UBO.
2916           */
2917          uint64_t va = 0;
2918          uint32_t ubo_idx = i - 1;
2919          uint32_t ubo_map_base = 0;
2920          for (int j = 0; j < link->ubo_map.num; j++) {
2921             if (ubo_idx >= ubo_map_base &&
2922                 ubo_idx < ubo_map_base + link->ubo_map.array_size[j]) {
2923                va = buffer_ptr(descriptors_state, &link->ubo_map, j,
2924                                ubo_idx - ubo_map_base);
2925                break;
2926             }
2927             ubo_map_base += link->ubo_map.array_size[j];
2928          }
2929          assert(va);
2930
2931          tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3);
2932          tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(state->range[i].offset / 16) |
2933                CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2934                CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
2935                CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2936                CP_LOAD_STATE6_0_NUM_UNIT(size / 16));
2937          tu_cs_emit_qw(cs, va + offset);
2938       }
2939    }
2940 }
2941
2942 static void
2943 tu6_emit_ubos(struct tu_cs *cs, const struct tu_pipeline *pipeline,
2944               struct tu_descriptor_state *descriptors_state,
2945               gl_shader_stage type)
2946 {
2947    const struct tu_program_descriptor_linkage *link =
2948       &pipeline->program.link[type];
2949
2950    uint32_t num = MIN2(link->ubo_map.num_desc, link->const_state.num_ubos);
2951    uint32_t anum = align(num, 2);
2952
2953    if (!num)
2954       return;
2955
2956    tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + (2 * anum));
2957    tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(link->const_state.offsets.ubo) |
2958          CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2959          CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
2960          CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2961          CP_LOAD_STATE6_0_NUM_UNIT(anum/2));
2962    tu_cs_emit(cs, CP_LOAD_STATE6_1_EXT_SRC_ADDR(0));
2963    tu_cs_emit(cs, CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI(0));
2964
2965    unsigned emitted = 0;
2966    for (unsigned i = 0; emitted < num && i < link->ubo_map.num; i++) {
2967       for (unsigned j = 0; emitted < num && j < link->ubo_map.array_size[i]; j++) {
2968          tu_cs_emit_qw(cs, buffer_ptr(descriptors_state, &link->ubo_map, i, j));
2969          emitted++;
2970       }
2971    }
2972
2973    for (; emitted < anum; emitted++) {
2974       tu_cs_emit(cs, 0xffffffff);
2975       tu_cs_emit(cs, 0xffffffff);
2976    }
2977 }
2978
2979 static struct tu_cs_entry
2980 tu6_emit_consts(struct tu_cmd_buffer *cmd,
2981                 const struct tu_pipeline *pipeline,
2982                 struct tu_descriptor_state *descriptors_state,
2983                 gl_shader_stage type)
2984 {
2985    struct tu_cs cs;
2986    tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs, 512, &cs); /* TODO: maximum size? */
2987
2988    tu6_emit_user_consts(&cs, pipeline, descriptors_state, type, cmd->push_constants);
2989    tu6_emit_ubos(&cs, pipeline, descriptors_state, type);
2990
2991    return tu_cs_end_sub_stream(&cmd->sub_cs, &cs);
2992 }
2993
2994 static VkResult
2995 tu6_emit_vs_params(struct tu_cmd_buffer *cmd,
2996                    const struct tu_draw_info *draw,
2997                    struct tu_cs_entry *entry)
2998 {
2999    /* TODO: fill out more than just base instance */
3000    const struct tu_program_descriptor_linkage *link =
3001       &cmd->state.pipeline->program.link[MESA_SHADER_VERTEX];
3002    const struct ir3_const_state *const_state = &link->const_state;
3003    struct tu_cs cs;
3004
3005    if (const_state->offsets.driver_param >= link->constlen) {
3006       *entry = (struct tu_cs_entry) {};
3007       return VK_SUCCESS;
3008    }
3009
3010    VkResult result = tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs, 8, &cs);
3011    if (result != VK_SUCCESS)
3012       return result;
3013
3014    tu_cs_emit_pkt7(&cs, CP_LOAD_STATE6_GEOM, 3 + 4);
3015    tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(const_state->offsets.driver_param) |
3016          CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
3017          CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
3018          CP_LOAD_STATE6_0_STATE_BLOCK(SB6_VS_SHADER) |
3019          CP_LOAD_STATE6_0_NUM_UNIT(1));
3020    tu_cs_emit(&cs, 0);
3021    tu_cs_emit(&cs, 0);
3022
3023    STATIC_ASSERT(IR3_DP_INSTID_BASE == 2);
3024
3025    tu_cs_emit(&cs, 0);
3026    tu_cs_emit(&cs, 0);
3027    tu_cs_emit(&cs, draw->first_instance);
3028    tu_cs_emit(&cs, 0);
3029
3030    *entry = tu_cs_end_sub_stream(&cmd->sub_cs, &cs);
3031    return VK_SUCCESS;
3032 }
3033
3034 static VkResult
3035 tu6_emit_textures(struct tu_cmd_buffer *cmd,
3036                   const struct tu_pipeline *pipeline,
3037                   struct tu_descriptor_state *descriptors_state,
3038                   gl_shader_stage type,
3039                   struct tu_cs_entry *entry,
3040                   bool *needs_border)
3041 {
3042    struct tu_device *device = cmd->device;
3043    struct tu_cs *draw_state = &cmd->sub_cs;
3044    const struct tu_program_descriptor_linkage *link =
3045       &pipeline->program.link[type];
3046    VkResult result;
3047
3048    if (link->texture_map.num_desc == 0 && link->sampler_map.num_desc == 0) {
3049       *entry = (struct tu_cs_entry) {};
3050       return VK_SUCCESS;
3051    }
3052
3053    /* allocate and fill texture state */
3054    struct ts_cs_memory tex_const;
3055    result = tu_cs_alloc(device, draw_state, link->texture_map.num_desc,
3056                         A6XX_TEX_CONST_DWORDS, &tex_const);
3057    if (result != VK_SUCCESS)
3058       return result;
3059
3060    int tex_index = 0;
3061    for (unsigned i = 0; i < link->texture_map.num; i++) {
3062       for (int j = 0; j < link->texture_map.array_size[i]; j++) {
3063          write_tex_const(cmd,
3064                          &tex_const.map[A6XX_TEX_CONST_DWORDS * tex_index++],
3065                          descriptors_state, &link->texture_map, i, j);
3066       }
3067    }
3068
3069    /* allocate and fill sampler state */
3070    struct ts_cs_memory tex_samp = { 0 };
3071    if (link->sampler_map.num_desc) {
3072       result = tu_cs_alloc(device, draw_state, link->sampler_map.num_desc,
3073                            A6XX_TEX_SAMP_DWORDS, &tex_samp);
3074       if (result != VK_SUCCESS)
3075          return result;
3076
3077       int sampler_index = 0;
3078       for (unsigned i = 0; i < link->sampler_map.num; i++) {
3079          for (int j = 0; j < link->sampler_map.array_size[i]; j++) {
3080             const struct tu_sampler *sampler = sampler_ptr(descriptors_state,
3081                                                            &link->sampler_map,
3082                                                            i, j);
3083             memcpy(&tex_samp.map[A6XX_TEX_SAMP_DWORDS * sampler_index++],
3084                    sampler->state, sizeof(sampler->state));
3085             *needs_border |= sampler->needs_border;
3086          }
3087       }
3088    }
3089
3090    unsigned tex_samp_reg, tex_const_reg, tex_count_reg;
3091    enum a6xx_state_block sb;
3092
3093    switch (type) {
3094    case MESA_SHADER_VERTEX:
3095       sb = SB6_VS_TEX;
3096       tex_samp_reg = REG_A6XX_SP_VS_TEX_SAMP_LO;
3097       tex_const_reg = REG_A6XX_SP_VS_TEX_CONST_LO;
3098       tex_count_reg = REG_A6XX_SP_VS_TEX_COUNT;
3099       break;
3100    case MESA_SHADER_FRAGMENT:
3101       sb = SB6_FS_TEX;
3102       tex_samp_reg = REG_A6XX_SP_FS_TEX_SAMP_LO;
3103       tex_const_reg = REG_A6XX_SP_FS_TEX_CONST_LO;
3104       tex_count_reg = REG_A6XX_SP_FS_TEX_COUNT;
3105       break;
3106    case MESA_SHADER_COMPUTE:
3107       sb = SB6_CS_TEX;
3108       tex_samp_reg = REG_A6XX_SP_CS_TEX_SAMP_LO;
3109       tex_const_reg = REG_A6XX_SP_CS_TEX_CONST_LO;
3110       tex_count_reg = REG_A6XX_SP_CS_TEX_COUNT;
3111       break;
3112    default:
3113       unreachable("bad state block");
3114    }
3115
3116    struct tu_cs cs;
3117    result = tu_cs_begin_sub_stream(device, draw_state, 16, &cs);
3118    if (result != VK_SUCCESS)
3119       return result;
3120
3121    if (link->sampler_map.num_desc) {
3122       /* output sampler state: */
3123       tu_cs_emit_pkt7(&cs, tu6_stage2opcode(type), 3);
3124       tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
3125                  CP_LOAD_STATE6_0_STATE_TYPE(ST6_SHADER) |
3126                  CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
3127                  CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
3128                  CP_LOAD_STATE6_0_NUM_UNIT(link->sampler_map.num_desc));
3129       tu_cs_emit_qw(&cs, tex_samp.iova); /* SRC_ADDR_LO/HI */
3130
3131       tu_cs_emit_pkt4(&cs, tex_samp_reg, 2);
3132       tu_cs_emit_qw(&cs, tex_samp.iova); /* SRC_ADDR_LO/HI */
3133    }
3134
3135    /* emit texture state: */
3136    tu_cs_emit_pkt7(&cs, tu6_stage2opcode(type), 3);
3137    tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
3138       CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
3139       CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
3140       CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
3141       CP_LOAD_STATE6_0_NUM_UNIT(link->texture_map.num_desc));
3142    tu_cs_emit_qw(&cs, tex_const.iova); /* SRC_ADDR_LO/HI */
3143
3144    tu_cs_emit_pkt4(&cs, tex_const_reg, 2);
3145    tu_cs_emit_qw(&cs, tex_const.iova); /* SRC_ADDR_LO/HI */
3146
3147    tu_cs_emit_pkt4(&cs, tex_count_reg, 1);
3148    tu_cs_emit(&cs, link->texture_map.num_desc);
3149
3150    *entry = tu_cs_end_sub_stream(draw_state, &cs);
3151    return VK_SUCCESS;
3152 }
3153
3154 static VkResult
3155 tu6_emit_ibo(struct tu_cmd_buffer *cmd,
3156              const struct tu_pipeline *pipeline,
3157              struct tu_descriptor_state *descriptors_state,
3158              gl_shader_stage type,
3159              struct tu_cs_entry *entry)
3160 {
3161    struct tu_device *device = cmd->device;
3162    struct tu_cs *draw_state = &cmd->sub_cs;
3163    const struct tu_program_descriptor_linkage *link =
3164       &pipeline->program.link[type];
3165    VkResult result;
3166
3167    unsigned num_desc = link->ssbo_map.num_desc + link->image_map.num_desc;
3168
3169    if (num_desc == 0) {
3170       *entry = (struct tu_cs_entry) {};
3171       return VK_SUCCESS;
3172    }
3173
3174    struct ts_cs_memory ibo_const;
3175    result = tu_cs_alloc(device, draw_state, num_desc,
3176                         A6XX_TEX_CONST_DWORDS, &ibo_const);
3177    if (result != VK_SUCCESS)
3178       return result;
3179
3180    int ssbo_index = 0;
3181    for (unsigned i = 0; i < link->ssbo_map.num; i++) {
3182       for (int j = 0; j < link->ssbo_map.array_size[i]; j++) {
3183          uint32_t *dst = &ibo_const.map[A6XX_TEX_CONST_DWORDS * ssbo_index];
3184
3185          uint64_t va = buffer_ptr(descriptors_state, &link->ssbo_map, i, j);
3186          /* We don't expose robustBufferAccess, so leave the size unlimited. */
3187          uint32_t sz = MAX_STORAGE_BUFFER_RANGE / 4;
3188
3189          dst[0] = A6XX_IBO_0_FMT(FMT6_32_UINT);
3190          dst[1] = A6XX_IBO_1_WIDTH(sz & MASK(15)) |
3191                   A6XX_IBO_1_HEIGHT(sz >> 15);
3192          dst[2] = A6XX_IBO_2_UNK4 |
3193                   A6XX_IBO_2_UNK31 |
3194                   A6XX_IBO_2_TYPE(A6XX_TEX_1D);
3195          dst[3] = 0;
3196          dst[4] = va;
3197          dst[5] = va >> 32;
3198          for (int i = 6; i < A6XX_TEX_CONST_DWORDS; i++)
3199             dst[i] = 0;
3200
3201          ssbo_index++;
3202       }
3203    }
3204
3205    for (unsigned i = 0; i < link->image_map.num; i++) {
3206       for (int j = 0; j < link->image_map.array_size[i]; j++) {
3207          uint32_t *dst = &ibo_const.map[A6XX_TEX_CONST_DWORDS * ssbo_index];
3208
3209          write_image_ibo(cmd, dst,
3210                          descriptors_state, &link->image_map, i, j);
3211
3212          ssbo_index++;
3213       }
3214    }
3215
3216    assert(ssbo_index == num_desc);
3217
3218    struct tu_cs cs;
3219    result = tu_cs_begin_sub_stream(device, draw_state, 7, &cs);
3220    if (result != VK_SUCCESS)
3221       return result;
3222
3223    uint32_t opcode, ibo_addr_reg;
3224    enum a6xx_state_block sb;
3225    enum a6xx_state_type st;
3226
3227    switch (type) {
3228    case MESA_SHADER_FRAGMENT:
3229       opcode = CP_LOAD_STATE6;
3230       st = ST6_SHADER;
3231       sb = SB6_IBO;
3232       ibo_addr_reg = REG_A6XX_SP_IBO_LO;
3233       break;
3234    case MESA_SHADER_COMPUTE:
3235       opcode = CP_LOAD_STATE6_FRAG;
3236       st = ST6_IBO;
3237       sb = SB6_CS_SHADER;
3238       ibo_addr_reg = REG_A6XX_SP_CS_IBO_LO;
3239       break;
3240    default:
3241       unreachable("unsupported stage for ibos");
3242    }
3243
3244    /* emit texture state: */
3245    tu_cs_emit_pkt7(&cs, opcode, 3);
3246    tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
3247               CP_LOAD_STATE6_0_STATE_TYPE(st) |
3248               CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
3249               CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
3250               CP_LOAD_STATE6_0_NUM_UNIT(num_desc));
3251    tu_cs_emit_qw(&cs, ibo_const.iova); /* SRC_ADDR_LO/HI */
3252
3253    tu_cs_emit_pkt4(&cs, ibo_addr_reg, 2);
3254    tu_cs_emit_qw(&cs, ibo_const.iova); /* SRC_ADDR_LO/HI */
3255
3256    *entry = tu_cs_end_sub_stream(draw_state, &cs);
3257    return VK_SUCCESS;
3258 }
3259
3260 struct PACKED bcolor_entry {
3261    uint32_t fp32[4];
3262    uint16_t ui16[4];
3263    int16_t  si16[4];
3264    uint16_t fp16[4];
3265    uint16_t rgb565;
3266    uint16_t rgb5a1;
3267    uint16_t rgba4;
3268    uint8_t __pad0[2];
3269    uint8_t  ui8[4];
3270    int8_t   si8[4];
3271    uint32_t rgb10a2;
3272    uint32_t z24; /* also s8? */
3273    uint16_t srgb[4];      /* appears to duplicate fp16[], but clamped, used for srgb */
3274    uint8_t  __pad1[56];
3275 } border_color[] = {
3276    [VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK] = {},
3277    [VK_BORDER_COLOR_INT_TRANSPARENT_BLACK] = {},
3278    [VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK] = {
3279       .fp32[3] = 0x3f800000,
3280       .ui16[3] = 0xffff,
3281       .si16[3] = 0x7fff,
3282       .fp16[3] = 0x3c00,
3283       .rgb5a1 = 0x8000,
3284       .rgba4 = 0xf000,
3285       .ui8[3] = 0xff,
3286       .si8[3] = 0x7f,
3287       .rgb10a2 = 0xc0000000,
3288       .srgb[3] = 0x3c00,
3289    },
3290    [VK_BORDER_COLOR_INT_OPAQUE_BLACK] = {
3291       .fp32[3] = 1,
3292       .fp16[3] = 1,
3293    },
3294    [VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE] = {
3295       .fp32[0 ... 3] = 0x3f800000,
3296       .ui16[0 ... 3] = 0xffff,
3297       .si16[0 ... 3] = 0x7fff,
3298       .fp16[0 ... 3] = 0x3c00,
3299       .rgb565 = 0xffff,
3300       .rgb5a1 = 0xffff,
3301       .rgba4 = 0xffff,
3302       .ui8[0 ... 3] = 0xff,
3303       .si8[0 ... 3] = 0x7f,
3304       .rgb10a2 = 0xffffffff,
3305       .z24 = 0xffffff,
3306       .srgb[0 ... 3] = 0x3c00,
3307    },
3308    [VK_BORDER_COLOR_INT_OPAQUE_WHITE] = {
3309       .fp32[0 ... 3] = 1,
3310       .fp16[0 ... 3] = 1,
3311    },
3312 };
3313
3314 static VkResult
3315 tu6_emit_border_color(struct tu_cmd_buffer *cmd,
3316                       struct tu_cs *cs)
3317 {
3318    STATIC_ASSERT(sizeof(struct bcolor_entry) == 128);
3319
3320    const struct tu_pipeline *pipeline = cmd->state.pipeline;
3321    struct tu_descriptor_state *descriptors_state =
3322       &cmd->descriptors[VK_PIPELINE_BIND_POINT_GRAPHICS];
3323    const struct tu_descriptor_map *vs_sampler =
3324       &pipeline->program.link[MESA_SHADER_VERTEX].sampler_map;
3325    const struct tu_descriptor_map *fs_sampler =
3326       &pipeline->program.link[MESA_SHADER_FRAGMENT].sampler_map;
3327    struct ts_cs_memory ptr;
3328
3329    VkResult result = tu_cs_alloc(cmd->device, &cmd->sub_cs,
3330                                  vs_sampler->num_desc + fs_sampler->num_desc,
3331                                  128 / 4,
3332                                  &ptr);
3333    if (result != VK_SUCCESS)
3334       return result;
3335
3336    for (unsigned i = 0; i < vs_sampler->num; i++) {
3337       for (unsigned j = 0; j < vs_sampler->array_size[i]; j++) {
3338          const struct tu_sampler *sampler = sampler_ptr(descriptors_state,
3339                                                         vs_sampler, i, j);
3340          memcpy(ptr.map, &border_color[sampler->border], 128);
3341          ptr.map += 128 / 4;
3342       }
3343    }
3344
3345    for (unsigned i = 0; i < fs_sampler->num; i++) {
3346       for (unsigned j = 0; j < fs_sampler->array_size[i]; j++) {
3347          const struct tu_sampler *sampler = sampler_ptr(descriptors_state,
3348                                                         fs_sampler, i, j);
3349          memcpy(ptr.map, &border_color[sampler->border], 128);
3350          ptr.map += 128 / 4;
3351       }
3352    }
3353
3354    tu_cs_emit_pkt4(cs, REG_A6XX_SP_TP_BORDER_COLOR_BASE_ADDR_LO, 2);
3355    tu_cs_emit_qw(cs, ptr.iova);
3356    return VK_SUCCESS;
3357 }
3358
3359 static VkResult
3360 tu6_bind_draw_states(struct tu_cmd_buffer *cmd,
3361                      struct tu_cs *cs,
3362                      const struct tu_draw_info *draw)
3363 {
3364    const struct tu_pipeline *pipeline = cmd->state.pipeline;
3365    const struct tu_dynamic_state *dynamic = &cmd->state.dynamic;
3366    struct tu_draw_state_group draw_state_groups[TU_DRAW_STATE_COUNT];
3367    uint32_t draw_state_group_count = 0;
3368
3369    struct tu_descriptor_state *descriptors_state =
3370       &cmd->descriptors[VK_PIPELINE_BIND_POINT_GRAPHICS];
3371
3372    VkResult result = tu_cs_reserve_space(cmd->device, cs, 256);
3373    if (result != VK_SUCCESS)
3374       return result;
3375
3376    /* TODO lrz */
3377
3378    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9806, 0);
3379    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9990, 0);
3380    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_UNKNOWN_A008, 0);
3381
3382    tu_cs_emit_regs(cs,
3383                    A6XX_PC_PRIMITIVE_CNTL_0(.primitive_restart =
3384                                             pipeline->ia.primitive_restart && draw->indexed));
3385
3386    if (cmd->state.dirty &
3387           (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_DYNAMIC_LINE_WIDTH) &&
3388        (pipeline->dynamic_state.mask & TU_DYNAMIC_LINE_WIDTH)) {
3389       tu6_emit_gras_su_cntl(cs, pipeline->rast.gras_su_cntl,
3390                             dynamic->line_width);
3391    }
3392
3393    if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK) &&
3394        (pipeline->dynamic_state.mask & TU_DYNAMIC_STENCIL_COMPARE_MASK)) {
3395       tu6_emit_stencil_compare_mask(cs, dynamic->stencil_compare_mask.front,
3396                                     dynamic->stencil_compare_mask.back);
3397    }
3398
3399    if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK) &&
3400        (pipeline->dynamic_state.mask & TU_DYNAMIC_STENCIL_WRITE_MASK)) {
3401       tu6_emit_stencil_write_mask(cs, dynamic->stencil_write_mask.front,
3402                                   dynamic->stencil_write_mask.back);
3403    }
3404
3405    if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE) &&
3406        (pipeline->dynamic_state.mask & TU_DYNAMIC_STENCIL_REFERENCE)) {
3407       tu6_emit_stencil_reference(cs, dynamic->stencil_reference.front,
3408                                  dynamic->stencil_reference.back);
3409    }
3410
3411    if (cmd->state.dirty &
3412        (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_VERTEX_BUFFERS)) {
3413       for (uint32_t i = 0; i < pipeline->vi.count; i++) {
3414          const uint32_t binding = pipeline->vi.bindings[i];
3415          const uint32_t stride = pipeline->vi.strides[i];
3416          const struct tu_buffer *buf = cmd->state.vb.buffers[binding];
3417          const VkDeviceSize offset = buf->bo_offset +
3418                                      cmd->state.vb.offsets[binding] +
3419                                      pipeline->vi.offsets[i];
3420          const VkDeviceSize size =
3421             offset < buf->bo->size ? buf->bo->size - offset : 0;
3422
3423          tu_cs_emit_regs(cs,
3424                          A6XX_VFD_FETCH_BASE(i, .bo = buf->bo, .bo_offset = offset),
3425                          A6XX_VFD_FETCH_SIZE(i, size),
3426                          A6XX_VFD_FETCH_STRIDE(i, stride));
3427       }
3428    }
3429
3430    if (cmd->state.dirty & TU_CMD_DIRTY_PIPELINE) {
3431       draw_state_groups[draw_state_group_count++] =
3432          (struct tu_draw_state_group) {
3433             .id = TU_DRAW_STATE_PROGRAM,
3434             .enable_mask = ENABLE_DRAW,
3435             .ib = pipeline->program.state_ib,
3436          };
3437       draw_state_groups[draw_state_group_count++] =
3438          (struct tu_draw_state_group) {
3439             .id = TU_DRAW_STATE_PROGRAM_BINNING,
3440             .enable_mask = CP_SET_DRAW_STATE__0_BINNING,
3441             .ib = pipeline->program.binning_state_ib,
3442          };
3443       draw_state_groups[draw_state_group_count++] =
3444          (struct tu_draw_state_group) {
3445             .id = TU_DRAW_STATE_VI,
3446             .enable_mask = ENABLE_DRAW,
3447             .ib = pipeline->vi.state_ib,
3448          };
3449       draw_state_groups[draw_state_group_count++] =
3450          (struct tu_draw_state_group) {
3451             .id = TU_DRAW_STATE_VI_BINNING,
3452             .enable_mask = CP_SET_DRAW_STATE__0_BINNING,
3453             .ib = pipeline->vi.binning_state_ib,
3454          };
3455       draw_state_groups[draw_state_group_count++] =
3456          (struct tu_draw_state_group) {
3457             .id = TU_DRAW_STATE_VP,
3458             .enable_mask = ENABLE_ALL,
3459             .ib = pipeline->vp.state_ib,
3460          };
3461       draw_state_groups[draw_state_group_count++] =
3462          (struct tu_draw_state_group) {
3463             .id = TU_DRAW_STATE_RAST,
3464             .enable_mask = ENABLE_ALL,
3465             .ib = pipeline->rast.state_ib,
3466          };
3467       draw_state_groups[draw_state_group_count++] =
3468          (struct tu_draw_state_group) {
3469             .id = TU_DRAW_STATE_DS,
3470             .enable_mask = ENABLE_ALL,
3471             .ib = pipeline->ds.state_ib,
3472          };
3473       draw_state_groups[draw_state_group_count++] =
3474          (struct tu_draw_state_group) {
3475             .id = TU_DRAW_STATE_BLEND,
3476             .enable_mask = ENABLE_ALL,
3477             .ib = pipeline->blend.state_ib,
3478          };
3479    }
3480
3481    if (cmd->state.dirty &
3482          (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_DESCRIPTOR_SETS | TU_CMD_DIRTY_PUSH_CONSTANTS)) {
3483       draw_state_groups[draw_state_group_count++] =
3484          (struct tu_draw_state_group) {
3485             .id = TU_DRAW_STATE_VS_CONST,
3486             .enable_mask = ENABLE_ALL,
3487             .ib = tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_VERTEX)
3488          };
3489       draw_state_groups[draw_state_group_count++] =
3490          (struct tu_draw_state_group) {
3491             .id = TU_DRAW_STATE_FS_CONST,
3492             .enable_mask = ENABLE_DRAW,
3493             .ib = tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_FRAGMENT)
3494          };
3495    }
3496
3497    if (cmd->state.dirty &
3498          (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_DESCRIPTOR_SETS)) {
3499       bool needs_border = false;
3500       struct tu_cs_entry vs_tex, fs_tex, fs_ibo;
3501
3502       result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3503                                  MESA_SHADER_VERTEX, &vs_tex, &needs_border);
3504       if (result != VK_SUCCESS)
3505          return result;
3506
3507       result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3508                                  MESA_SHADER_FRAGMENT, &fs_tex, &needs_border);
3509       if (result != VK_SUCCESS)
3510          return result;
3511
3512       result = tu6_emit_ibo(cmd, pipeline, descriptors_state,
3513                             MESA_SHADER_FRAGMENT, &fs_ibo);
3514       if (result != VK_SUCCESS)
3515          return result;
3516
3517       draw_state_groups[draw_state_group_count++] =
3518          (struct tu_draw_state_group) {
3519             .id = TU_DRAW_STATE_VS_TEX,
3520             .enable_mask = ENABLE_ALL,
3521             .ib = vs_tex,
3522          };
3523       draw_state_groups[draw_state_group_count++] =
3524          (struct tu_draw_state_group) {
3525             .id = TU_DRAW_STATE_FS_TEX,
3526             .enable_mask = ENABLE_DRAW,
3527             .ib = fs_tex,
3528          };
3529       draw_state_groups[draw_state_group_count++] =
3530          (struct tu_draw_state_group) {
3531             .id = TU_DRAW_STATE_FS_IBO,
3532             .enable_mask = ENABLE_DRAW,
3533             .ib = fs_ibo,
3534          };
3535
3536       if (needs_border) {
3537          result = tu6_emit_border_color(cmd, cs);
3538          if (result != VK_SUCCESS)
3539             return result;
3540       }
3541    }
3542
3543    struct tu_cs_entry vs_params;
3544    result = tu6_emit_vs_params(cmd, draw, &vs_params);
3545    if (result != VK_SUCCESS)
3546       return result;
3547
3548    draw_state_groups[draw_state_group_count++] =
3549       (struct tu_draw_state_group) {
3550          .id = TU_DRAW_STATE_VS_PARAMS,
3551          .enable_mask = ENABLE_ALL,
3552          .ib = vs_params,
3553       };
3554
3555    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3 * draw_state_group_count);
3556    for (uint32_t i = 0; i < draw_state_group_count; i++) {
3557       const struct tu_draw_state_group *group = &draw_state_groups[i];
3558       debug_assert((group->enable_mask & ~ENABLE_ALL) == 0);
3559       uint32_t cp_set_draw_state =
3560          CP_SET_DRAW_STATE__0_COUNT(group->ib.size / 4) |
3561          group->enable_mask |
3562          CP_SET_DRAW_STATE__0_GROUP_ID(group->id);
3563       uint64_t iova;
3564       if (group->ib.size) {
3565          iova = group->ib.bo->iova + group->ib.offset;
3566       } else {
3567          cp_set_draw_state |= CP_SET_DRAW_STATE__0_DISABLE;
3568          iova = 0;
3569       }
3570
3571       tu_cs_emit(cs, cp_set_draw_state);
3572       tu_cs_emit_qw(cs, iova);
3573    }
3574
3575    tu_cs_sanity_check(cs);
3576
3577    /* track BOs */
3578    if (cmd->state.dirty & TU_CMD_DIRTY_VERTEX_BUFFERS) {
3579       for (uint32_t i = 0; i < MAX_VBS; i++) {
3580          const struct tu_buffer *buf = cmd->state.vb.buffers[i];
3581          if (buf)
3582             tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
3583       }
3584    }
3585    if (cmd->state.dirty & TU_CMD_DIRTY_DESCRIPTOR_SETS) {
3586       unsigned i;
3587       for_each_bit(i, descriptors_state->valid) {
3588          struct tu_descriptor_set *set = descriptors_state->sets[i];
3589          for (unsigned j = 0; j < set->layout->buffer_count; ++j)
3590             if (set->descriptors[j]) {
3591                tu_bo_list_add(&cmd->bo_list, set->descriptors[j],
3592                               MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3593             }
3594       }
3595    }
3596
3597    /* Fragment shader state overwrites compute shader state, so flag the
3598     * compute pipeline for re-emit.
3599     */
3600    cmd->state.dirty = TU_CMD_DIRTY_COMPUTE_PIPELINE;
3601    return VK_SUCCESS;
3602 }
3603
3604 static void
3605 tu6_emit_draw_direct(struct tu_cmd_buffer *cmd,
3606                      struct tu_cs *cs,
3607                      const struct tu_draw_info *draw)
3608 {
3609
3610    const enum pc_di_primtype primtype = cmd->state.pipeline->ia.primtype;
3611
3612    tu_cs_emit_regs(cs,
3613                    A6XX_VFD_INDEX_OFFSET(draw->vertex_offset),
3614                    A6XX_VFD_INSTANCE_START_OFFSET(draw->first_instance));
3615
3616    /* TODO hw binning */
3617    if (draw->indexed) {
3618       const enum a4xx_index_size index_size =
3619          tu6_index_size(cmd->state.index_type);
3620       const uint32_t index_bytes =
3621          (cmd->state.index_type == VK_INDEX_TYPE_UINT32) ? 4 : 2;
3622       const struct tu_buffer *buf = cmd->state.index_buffer;
3623       const VkDeviceSize offset = buf->bo_offset + cmd->state.index_offset +
3624                                   index_bytes * draw->first_index;
3625       const uint32_t size = index_bytes * draw->count;
3626
3627       const uint32_t cp_draw_indx =
3628          CP_DRAW_INDX_OFFSET_0_PRIM_TYPE(primtype) |
3629          CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT(DI_SRC_SEL_DMA) |
3630          CP_DRAW_INDX_OFFSET_0_INDEX_SIZE(index_size) |
3631          CP_DRAW_INDX_OFFSET_0_VIS_CULL(USE_VISIBILITY) | 0x2000;
3632
3633       tu_cs_emit_pkt7(cs, CP_DRAW_INDX_OFFSET, 7);
3634       tu_cs_emit(cs, cp_draw_indx);
3635       tu_cs_emit(cs, draw->instance_count);
3636       tu_cs_emit(cs, draw->count);
3637       tu_cs_emit(cs, 0x0); /* XXX */
3638       tu_cs_emit_qw(cs, buf->bo->iova + offset);
3639       tu_cs_emit(cs, size);
3640    } else {
3641       const uint32_t cp_draw_indx =
3642          CP_DRAW_INDX_OFFSET_0_PRIM_TYPE(primtype) |
3643          CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT(DI_SRC_SEL_AUTO_INDEX) |
3644          CP_DRAW_INDX_OFFSET_0_VIS_CULL(USE_VISIBILITY) | 0x2000;
3645
3646       tu_cs_emit_pkt7(cs, CP_DRAW_INDX_OFFSET, 3);
3647       tu_cs_emit(cs, cp_draw_indx);
3648       tu_cs_emit(cs, draw->instance_count);
3649       tu_cs_emit(cs, draw->count);
3650    }
3651 }
3652
3653 static void
3654 tu_draw(struct tu_cmd_buffer *cmd, const struct tu_draw_info *draw)
3655 {
3656    struct tu_cs *cs = &cmd->draw_cs;
3657    VkResult result;
3658
3659    result = tu6_bind_draw_states(cmd, cs, draw);
3660    if (result != VK_SUCCESS) {
3661       cmd->record_result = result;
3662       return;
3663    }
3664
3665    result = tu_cs_reserve_space(cmd->device, cs, 32);
3666    if (result != VK_SUCCESS) {
3667       cmd->record_result = result;
3668       return;
3669    }
3670
3671    if (draw->indirect) {
3672       tu_finishme("indirect draw");
3673       return;
3674    }
3675
3676    /* TODO tu6_emit_marker should pick different regs depending on cs */
3677
3678    tu6_emit_marker(cmd, cs);
3679    tu6_emit_draw_direct(cmd, cs, draw);
3680    tu6_emit_marker(cmd, cs);
3681
3682    cmd->wait_for_idle = true;
3683
3684    tu_cs_sanity_check(cs);
3685 }
3686
3687 void
3688 tu_CmdDraw(VkCommandBuffer commandBuffer,
3689            uint32_t vertexCount,
3690            uint32_t instanceCount,
3691            uint32_t firstVertex,
3692            uint32_t firstInstance)
3693 {
3694    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3695    struct tu_draw_info info = {};
3696
3697    info.count = vertexCount;
3698    info.instance_count = instanceCount;
3699    info.first_instance = firstInstance;
3700    info.vertex_offset = firstVertex;
3701
3702    tu_draw(cmd_buffer, &info);
3703 }
3704
3705 void
3706 tu_CmdDrawIndexed(VkCommandBuffer commandBuffer,
3707                   uint32_t indexCount,
3708                   uint32_t instanceCount,
3709                   uint32_t firstIndex,
3710                   int32_t vertexOffset,
3711                   uint32_t firstInstance)
3712 {
3713    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3714    struct tu_draw_info info = {};
3715
3716    info.indexed = true;
3717    info.count = indexCount;
3718    info.instance_count = instanceCount;
3719    info.first_index = firstIndex;
3720    info.vertex_offset = vertexOffset;
3721    info.first_instance = firstInstance;
3722
3723    tu_draw(cmd_buffer, &info);
3724 }
3725
3726 void
3727 tu_CmdDrawIndirect(VkCommandBuffer commandBuffer,
3728                    VkBuffer _buffer,
3729                    VkDeviceSize offset,
3730                    uint32_t drawCount,
3731                    uint32_t stride)
3732 {
3733    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3734    TU_FROM_HANDLE(tu_buffer, buffer, _buffer);
3735    struct tu_draw_info info = {};
3736
3737    info.count = drawCount;
3738    info.indirect = buffer;
3739    info.indirect_offset = offset;
3740    info.stride = stride;
3741
3742    tu_draw(cmd_buffer, &info);
3743 }
3744
3745 void
3746 tu_CmdDrawIndexedIndirect(VkCommandBuffer commandBuffer,
3747                           VkBuffer _buffer,
3748                           VkDeviceSize offset,
3749                           uint32_t drawCount,
3750                           uint32_t stride)
3751 {
3752    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3753    TU_FROM_HANDLE(tu_buffer, buffer, _buffer);
3754    struct tu_draw_info info = {};
3755
3756    info.indexed = true;
3757    info.count = drawCount;
3758    info.indirect = buffer;
3759    info.indirect_offset = offset;
3760    info.stride = stride;
3761
3762    tu_draw(cmd_buffer, &info);
3763 }
3764
3765 struct tu_dispatch_info
3766 {
3767    /**
3768     * Determine the layout of the grid (in block units) to be used.
3769     */
3770    uint32_t blocks[3];
3771
3772    /**
3773     * A starting offset for the grid. If unaligned is set, the offset
3774     * must still be aligned.
3775     */
3776    uint32_t offsets[3];
3777    /**
3778     * Whether it's an unaligned compute dispatch.
3779     */
3780    bool unaligned;
3781
3782    /**
3783     * Indirect compute parameters resource.
3784     */
3785    struct tu_buffer *indirect;
3786    uint64_t indirect_offset;
3787 };
3788
3789 static void
3790 tu_emit_compute_driver_params(struct tu_cs *cs, struct tu_pipeline *pipeline,
3791                               const struct tu_dispatch_info *info)
3792 {
3793    gl_shader_stage type = MESA_SHADER_COMPUTE;
3794    const struct tu_program_descriptor_linkage *link =
3795       &pipeline->program.link[type];
3796    const struct ir3_const_state *const_state = &link->const_state;
3797    uint32_t offset = const_state->offsets.driver_param;
3798
3799    if (link->constlen <= offset)
3800       return;
3801
3802    if (!info->indirect) {
3803       uint32_t driver_params[IR3_DP_CS_COUNT] = {
3804          [IR3_DP_NUM_WORK_GROUPS_X] = info->blocks[0],
3805          [IR3_DP_NUM_WORK_GROUPS_Y] = info->blocks[1],
3806          [IR3_DP_NUM_WORK_GROUPS_Z] = info->blocks[2],
3807          [IR3_DP_LOCAL_GROUP_SIZE_X] = pipeline->compute.local_size[0],
3808          [IR3_DP_LOCAL_GROUP_SIZE_Y] = pipeline->compute.local_size[1],
3809          [IR3_DP_LOCAL_GROUP_SIZE_Z] = pipeline->compute.local_size[2],
3810       };
3811
3812       uint32_t num_consts = MIN2(const_state->num_driver_params,
3813                                  (link->constlen - offset) * 4);
3814       /* push constants */
3815       tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + num_consts);
3816       tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(offset) |
3817                  CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
3818                  CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
3819                  CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
3820                  CP_LOAD_STATE6_0_NUM_UNIT(num_consts / 4));
3821       tu_cs_emit(cs, 0);
3822       tu_cs_emit(cs, 0);
3823       uint32_t i;
3824       for (i = 0; i < num_consts; i++)
3825          tu_cs_emit(cs, driver_params[i]);
3826    } else {
3827       tu_finishme("Indirect driver params");
3828    }
3829 }
3830
3831 static void
3832 tu_dispatch(struct tu_cmd_buffer *cmd,
3833             const struct tu_dispatch_info *info)
3834 {
3835    struct tu_cs *cs = &cmd->cs;
3836    struct tu_pipeline *pipeline = cmd->state.compute_pipeline;
3837    struct tu_descriptor_state *descriptors_state =
3838       &cmd->descriptors[VK_PIPELINE_BIND_POINT_COMPUTE];
3839
3840    VkResult result = tu_cs_reserve_space(cmd->device, cs, 256);
3841    if (result != VK_SUCCESS) {
3842       cmd->record_result = result;
3843       return;
3844    }
3845
3846    if (cmd->state.dirty & TU_CMD_DIRTY_COMPUTE_PIPELINE)
3847       tu_cs_emit_ib(cs, &pipeline->program.state_ib);
3848
3849    struct tu_cs_entry ib;
3850
3851    ib = tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_COMPUTE);
3852    if (ib.size)
3853       tu_cs_emit_ib(cs, &ib);
3854
3855    tu_emit_compute_driver_params(cs, pipeline, info);
3856
3857    bool needs_border;
3858    result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3859                               MESA_SHADER_COMPUTE, &ib, &needs_border);
3860    if (result != VK_SUCCESS) {
3861       cmd->record_result = result;
3862       return;
3863    }
3864
3865    if (ib.size)
3866       tu_cs_emit_ib(cs, &ib);
3867
3868    if (needs_border)
3869       tu_finishme("compute border color");
3870
3871    result = tu6_emit_ibo(cmd, pipeline, descriptors_state, MESA_SHADER_COMPUTE, &ib);
3872    if (result != VK_SUCCESS) {
3873       cmd->record_result = result;
3874       return;
3875    }
3876
3877    if (ib.size)
3878       tu_cs_emit_ib(cs, &ib);
3879
3880    /* track BOs */
3881    if (cmd->state.dirty & TU_CMD_DIRTY_DESCRIPTOR_SETS) {
3882       unsigned i;
3883       for_each_bit(i, descriptors_state->valid) {
3884          struct tu_descriptor_set *set = descriptors_state->sets[i];
3885          for (unsigned j = 0; j < set->layout->buffer_count; ++j)
3886             if (set->descriptors[j]) {
3887                tu_bo_list_add(&cmd->bo_list, set->descriptors[j],
3888                               MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3889             }
3890       }
3891    }
3892
3893    /* Compute shader state overwrites fragment shader state, so we flag the
3894     * graphics pipeline for re-emit.
3895     */
3896    cmd->state.dirty = TU_CMD_DIRTY_PIPELINE;
3897
3898    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
3899    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(0x8));
3900
3901    const uint32_t *local_size = pipeline->compute.local_size;
3902    const uint32_t *num_groups = info->blocks;
3903    tu_cs_emit_regs(cs,
3904                    A6XX_HLSQ_CS_NDRANGE_0(.kerneldim = 3,
3905                                           .localsizex = local_size[0] - 1,
3906                                           .localsizey = local_size[1] - 1,
3907                                           .localsizez = local_size[2] - 1),
3908                    A6XX_HLSQ_CS_NDRANGE_1(.globalsize_x = local_size[0] * num_groups[0]),
3909                    A6XX_HLSQ_CS_NDRANGE_2(.globaloff_x = 0),
3910                    A6XX_HLSQ_CS_NDRANGE_3(.globalsize_y = local_size[1] * num_groups[1]),
3911                    A6XX_HLSQ_CS_NDRANGE_4(.globaloff_y = 0),
3912                    A6XX_HLSQ_CS_NDRANGE_5(.globalsize_z = local_size[2] * num_groups[2]),
3913                    A6XX_HLSQ_CS_NDRANGE_6(.globaloff_z = 0));
3914
3915    tu_cs_emit_regs(cs,
3916                    A6XX_HLSQ_CS_KERNEL_GROUP_X(1),
3917                    A6XX_HLSQ_CS_KERNEL_GROUP_Y(1),
3918                    A6XX_HLSQ_CS_KERNEL_GROUP_Z(1));
3919
3920    if (info->indirect) {
3921       uint64_t iova = tu_buffer_iova(info->indirect) + info->indirect_offset;
3922
3923       tu_bo_list_add(&cmd->bo_list, info->indirect->bo,
3924                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3925
3926       tu_cs_emit_pkt7(cs, CP_EXEC_CS_INDIRECT, 4);
3927       tu_cs_emit(cs, 0x00000000);
3928       tu_cs_emit_qw(cs, iova);
3929       tu_cs_emit(cs,
3930                  A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX(local_size[0] - 1) |
3931                  A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY(local_size[1] - 1) |
3932                  A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ(local_size[2] - 1));
3933    } else {
3934       tu_cs_emit_pkt7(cs, CP_EXEC_CS, 4);
3935       tu_cs_emit(cs, 0x00000000);
3936       tu_cs_emit(cs, CP_EXEC_CS_1_NGROUPS_X(info->blocks[0]));
3937       tu_cs_emit(cs, CP_EXEC_CS_2_NGROUPS_Y(info->blocks[1]));
3938       tu_cs_emit(cs, CP_EXEC_CS_3_NGROUPS_Z(info->blocks[2]));
3939    }
3940
3941    tu_cs_emit_wfi(cs);
3942
3943    tu6_emit_cache_flush(cmd, cs);
3944 }
3945
3946 void
3947 tu_CmdDispatchBase(VkCommandBuffer commandBuffer,
3948                    uint32_t base_x,
3949                    uint32_t base_y,
3950                    uint32_t base_z,
3951                    uint32_t x,
3952                    uint32_t y,
3953                    uint32_t z)
3954 {
3955    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3956    struct tu_dispatch_info info = {};
3957
3958    info.blocks[0] = x;
3959    info.blocks[1] = y;
3960    info.blocks[2] = z;
3961
3962    info.offsets[0] = base_x;
3963    info.offsets[1] = base_y;
3964    info.offsets[2] = base_z;
3965    tu_dispatch(cmd_buffer, &info);
3966 }
3967
3968 void
3969 tu_CmdDispatch(VkCommandBuffer commandBuffer,
3970                uint32_t x,
3971                uint32_t y,
3972                uint32_t z)
3973 {
3974    tu_CmdDispatchBase(commandBuffer, 0, 0, 0, x, y, z);
3975 }
3976
3977 void
3978 tu_CmdDispatchIndirect(VkCommandBuffer commandBuffer,
3979                        VkBuffer _buffer,
3980                        VkDeviceSize offset)
3981 {
3982    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3983    TU_FROM_HANDLE(tu_buffer, buffer, _buffer);
3984    struct tu_dispatch_info info = {};
3985
3986    info.indirect = buffer;
3987    info.indirect_offset = offset;
3988
3989    tu_dispatch(cmd_buffer, &info);
3990 }
3991
3992 void
3993 tu_CmdEndRenderPass(VkCommandBuffer commandBuffer)
3994 {
3995    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3996
3997    tu_cs_end(&cmd_buffer->draw_cs);
3998    tu_cs_end(&cmd_buffer->draw_epilogue_cs);
3999
4000    if (use_sysmem_rendering(cmd_buffer))
4001       tu_cmd_render_sysmem(cmd_buffer);
4002    else
4003       tu_cmd_render_tiles(cmd_buffer);
4004
4005    /* discard draw_cs and draw_epilogue_cs entries now that the tiles are
4006       rendered */
4007    tu_cs_discard_entries(&cmd_buffer->draw_cs);
4008    tu_cs_begin(&cmd_buffer->draw_cs);
4009    tu_cs_discard_entries(&cmd_buffer->draw_epilogue_cs);
4010    tu_cs_begin(&cmd_buffer->draw_epilogue_cs);
4011
4012    cmd_buffer->state.pass = NULL;
4013    cmd_buffer->state.subpass = NULL;
4014    cmd_buffer->state.framebuffer = NULL;
4015 }
4016
4017 void
4018 tu_CmdEndRenderPass2(VkCommandBuffer commandBuffer,
4019                      const VkSubpassEndInfoKHR *pSubpassEndInfo)
4020 {
4021    tu_CmdEndRenderPass(commandBuffer);
4022 }
4023
4024 struct tu_barrier_info
4025 {
4026    uint32_t eventCount;
4027    const VkEvent *pEvents;
4028    VkPipelineStageFlags srcStageMask;
4029 };
4030
4031 static void
4032 tu_barrier(struct tu_cmd_buffer *cmd_buffer,
4033            uint32_t memoryBarrierCount,
4034            const VkMemoryBarrier *pMemoryBarriers,
4035            uint32_t bufferMemoryBarrierCount,
4036            const VkBufferMemoryBarrier *pBufferMemoryBarriers,
4037            uint32_t imageMemoryBarrierCount,
4038            const VkImageMemoryBarrier *pImageMemoryBarriers,
4039            const struct tu_barrier_info *info)
4040 {
4041 }
4042
4043 void
4044 tu_CmdPipelineBarrier(VkCommandBuffer commandBuffer,
4045                       VkPipelineStageFlags srcStageMask,
4046                       VkPipelineStageFlags destStageMask,
4047                       VkBool32 byRegion,
4048                       uint32_t memoryBarrierCount,
4049                       const VkMemoryBarrier *pMemoryBarriers,
4050                       uint32_t bufferMemoryBarrierCount,
4051                       const VkBufferMemoryBarrier *pBufferMemoryBarriers,
4052                       uint32_t imageMemoryBarrierCount,
4053                       const VkImageMemoryBarrier *pImageMemoryBarriers)
4054 {
4055    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
4056    struct tu_barrier_info info;
4057
4058    info.eventCount = 0;
4059    info.pEvents = NULL;
4060    info.srcStageMask = srcStageMask;
4061
4062    tu_barrier(cmd_buffer, memoryBarrierCount, pMemoryBarriers,
4063               bufferMemoryBarrierCount, pBufferMemoryBarriers,
4064               imageMemoryBarrierCount, pImageMemoryBarriers, &info);
4065 }
4066
4067 static void
4068 write_event(struct tu_cmd_buffer *cmd, struct tu_event *event, unsigned value)
4069 {
4070    struct tu_cs *cs = &cmd->cs;
4071
4072    VkResult result = tu_cs_reserve_space(cmd->device, cs, 4);
4073    if (result != VK_SUCCESS) {
4074       cmd->record_result = result;
4075       return;
4076    }
4077
4078    tu_bo_list_add(&cmd->bo_list, &event->bo, MSM_SUBMIT_BO_WRITE);
4079
4080    /* TODO: any flush required before/after ? */
4081
4082    tu_cs_emit_pkt7(cs, CP_MEM_WRITE, 3);
4083    tu_cs_emit_qw(cs, event->bo.iova); /* ADDR_LO/HI */
4084    tu_cs_emit(cs, value);
4085 }
4086
4087 void
4088 tu_CmdSetEvent(VkCommandBuffer commandBuffer,
4089                VkEvent _event,
4090                VkPipelineStageFlags stageMask)
4091 {
4092    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
4093    TU_FROM_HANDLE(tu_event, event, _event);
4094
4095    write_event(cmd, event, 1);
4096 }
4097
4098 void
4099 tu_CmdResetEvent(VkCommandBuffer commandBuffer,
4100                  VkEvent _event,
4101                  VkPipelineStageFlags stageMask)
4102 {
4103    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
4104    TU_FROM_HANDLE(tu_event, event, _event);
4105
4106    write_event(cmd, event, 0);
4107 }
4108
4109 void
4110 tu_CmdWaitEvents(VkCommandBuffer commandBuffer,
4111                  uint32_t eventCount,
4112                  const VkEvent *pEvents,
4113                  VkPipelineStageFlags srcStageMask,
4114                  VkPipelineStageFlags dstStageMask,
4115                  uint32_t memoryBarrierCount,
4116                  const VkMemoryBarrier *pMemoryBarriers,
4117                  uint32_t bufferMemoryBarrierCount,
4118                  const VkBufferMemoryBarrier *pBufferMemoryBarriers,
4119                  uint32_t imageMemoryBarrierCount,
4120                  const VkImageMemoryBarrier *pImageMemoryBarriers)
4121 {
4122    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
4123    struct tu_cs *cs = &cmd->cs;
4124
4125    VkResult result = tu_cs_reserve_space(cmd->device, cs, eventCount * 7);
4126    if (result != VK_SUCCESS) {
4127       cmd->record_result = result;
4128       return;
4129    }
4130
4131    /* TODO: any flush required before/after? (CP_WAIT_FOR_ME?) */
4132
4133    for (uint32_t i = 0; i < eventCount; i++) {
4134       const struct tu_event *event = (const struct tu_event*) pEvents[i];
4135
4136       tu_bo_list_add(&cmd->bo_list, &event->bo, MSM_SUBMIT_BO_READ);
4137
4138       tu_cs_emit_pkt7(cs, CP_WAIT_REG_MEM, 6);
4139       tu_cs_emit(cs, CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ) |
4140                      CP_WAIT_REG_MEM_0_POLL_MEMORY);
4141       tu_cs_emit_qw(cs, event->bo.iova); /* POLL_ADDR_LO/HI */
4142       tu_cs_emit(cs, CP_WAIT_REG_MEM_3_REF(1));
4143       tu_cs_emit(cs, CP_WAIT_REG_MEM_4_MASK(~0u));
4144       tu_cs_emit(cs, CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(20));
4145    }
4146 }
4147
4148 void
4149 tu_CmdSetDeviceMask(VkCommandBuffer commandBuffer, uint32_t deviceMask)
4150 {
4151    /* No-op */
4152 }