src/freedreno/vulkan/tu_cmd_buffer.c

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * based in part on anv driver which is:
   6  * Copyright © 2015 Intel Corporation
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25  * DEALINGS IN THE SOFTWARE.
  26  */
  27
  28 #include "tu_private.h"
  29
  30 #include "registers/adreno_pm4.xml.h"
  31 #include "registers/adreno_common.xml.h"
  32
  33 #include "vk_format.h"
  34
  35 #include "tu_cs.h"
  36 #include "tu_blit.h"
  37
  38 #define OVERFLOW_FLAG_REG REG_A6XX_CP_SCRATCH_REG(0)
  39
  40 void
  41 tu_bo_list_init(struct tu_bo_list *list)
  42 {
  43    list->count = list->capacity = 0;
  44    list->bo_infos = NULL;
  45 }
  46
  47 void
  48 tu_bo_list_destroy(struct tu_bo_list *list)
  49 {
  50    free(list->bo_infos);
  51 }
  52
  53 void
  54 tu_bo_list_reset(struct tu_bo_list *list)
  55 {
  56    list->count = 0;
  57 }
  58
  59 /**
  60  * \a flags consists of MSM_SUBMIT_BO_FLAGS.
  61  */
  62 static uint32_t
  63 tu_bo_list_add_info(struct tu_bo_list *list,
  64                     const struct drm_msm_gem_submit_bo *bo_info)
  65 {
  66    assert(bo_info->handle != 0);
  67
  68    for (uint32_t i = 0; i < list->count; ++i) {
  69       if (list->bo_infos[i].handle == bo_info->handle) {
  70          assert(list->bo_infos[i].presumed == bo_info->presumed);
  71          list->bo_infos[i].flags |= bo_info->flags;
  72          return i;
  73       }
  74    }
  75
  76    /* grow list->bo_infos if needed */
  77    if (list->count == list->capacity) {
  78       uint32_t new_capacity = MAX2(2 * list->count, 16);
  79       struct drm_msm_gem_submit_bo *new_bo_infos = realloc(
  80          list->bo_infos, new_capacity * sizeof(struct drm_msm_gem_submit_bo));
  81       if (!new_bo_infos)
  82          return TU_BO_LIST_FAILED;
  83       list->bo_infos = new_bo_infos;
  84       list->capacity = new_capacity;
  85    }
  86
  87    list->bo_infos[list->count] = *bo_info;
  88    return list->count++;
  89 }
  90
  91 uint32_t
  92 tu_bo_list_add(struct tu_bo_list *list,
  93                const struct tu_bo *bo,
  94                uint32_t flags)
  95 {
  96    return tu_bo_list_add_info(list, &(struct drm_msm_gem_submit_bo) {
  97                                        .flags = flags,
  98                                        .handle = bo->gem_handle,
  99                                        .presumed = bo->iova,
 100                                     });
 101 }
 102
 103 VkResult
 104 tu_bo_list_merge(struct tu_bo_list *list, const struct tu_bo_list *other)
 105 {
 106    for (uint32_t i = 0; i < other->count; i++) {
 107       if (tu_bo_list_add_info(list, other->bo_infos + i) == TU_BO_LIST_FAILED)
 108          return VK_ERROR_OUT_OF_HOST_MEMORY;
 109    }
 110
 111    return VK_SUCCESS;
 112 }
 113
 114 static bool
 115 is_linear_mipmapped(const struct tu_image_view *iview)
 116 {
 117    return iview->image->layout.tile_mode == TILE6_LINEAR &&
 118           iview->base_mip != iview->image->level_count - 1;
 119 }
 120
 121 static bool
 122 force_sysmem(const struct tu_cmd_buffer *cmd,
 123              const struct VkRect2D *render_area)
 124 {
 125    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 126    const struct tu_physical_device *device = cmd->device->physical_device;
 127    bool has_linear_mipmapped_store = false;
 128    const struct tu_render_pass *pass = cmd->state.pass;
 129
 130    /* Iterate over all the places we call tu6_emit_store_attachment() */
 131    for (unsigned i = 0; i < pass->subpass_count; i++) {
 132       const struct tu_subpass *subpass = &pass->subpasses[i];
 133       if (subpass->resolve_attachments) {
 134          for (unsigned i = 0; i < subpass->color_count; i++) {
 135             uint32_t a = subpass->resolve_attachments[i].attachment;
 136             if (a != VK_ATTACHMENT_UNUSED &&
 137                 cmd->state.pass->attachments[a].store_op == VK_ATTACHMENT_STORE_OP_STORE) {
 138                const struct tu_image_view *iview = fb->attachments[a].attachment;
 139                if (is_linear_mipmapped(iview)) {
 140                   has_linear_mipmapped_store = true;
 141                   break;
 142                }
 143             }
 144          }
 145       }
 146    }
 147
 148    for (unsigned i = 0; i < pass->attachment_count; i++) {
 149       if (pass->attachments[i].gmem_offset >= 0 &&
 150           cmd->state.pass->attachments[i].store_op == VK_ATTACHMENT_STORE_OP_STORE) {
 151          const struct tu_image_view *iview = fb->attachments[i].attachment;
 152          if (is_linear_mipmapped(iview)) {
 153             has_linear_mipmapped_store = true;
 154             break;
 155          }
 156       }
 157    }
 158
 159    /* Linear textures cannot have any padding between mipmap levels and their
 160     * height isn't padded, while at the same time the GMEM->MEM resolve does
 161     * not have per-pixel granularity, so if the image height isn't aligned to
 162     * the resolve granularity and the render area is tall enough, we may wind
 163     * up writing past the bottom of the image into the next miplevel or even
 164     * past the end of the image. For the last miplevel, the layout code should
 165     * insert enough padding so that the overdraw writes to the padding.  To
 166     * work around this, we force-enable sysmem rendering.
 167     */
 168    const uint32_t y2 = render_area->offset.y + render_area->extent.height;
 169    const uint32_t aligned_y2 = ALIGN_POT(y2, device->tile_align_h);
 170
 171    return has_linear_mipmapped_store && aligned_y2 > fb->height;
 172 }
 173
 174 static void
 175 tu_tiling_config_update_tile_layout(struct tu_tiling_config *tiling,
 176                                     const struct tu_device *dev,
 177                                     uint32_t pixels)
 178 {
 179    const uint32_t tile_align_w = dev->physical_device->tile_align_w;
 180    const uint32_t tile_align_h = dev->physical_device->tile_align_h;
 181    const uint32_t max_tile_width = 1024; /* A6xx */
 182
 183    tiling->tile0.offset = (VkOffset2D) {
 184       .x = tiling->render_area.offset.x & ~(tile_align_w - 1),
 185       .y = tiling->render_area.offset.y & ~(tile_align_h - 1),
 186    };
 187
 188    const uint32_t ra_width =
 189       tiling->render_area.extent.width +
 190       (tiling->render_area.offset.x - tiling->tile0.offset.x);
 191    const uint32_t ra_height =
 192       tiling->render_area.extent.height +
 193       (tiling->render_area.offset.y - tiling->tile0.offset.y);
 194
 195    /* start from 1 tile */
 196    tiling->tile_count = (VkExtent2D) {
 197       .width = 1,
 198       .height = 1,
 199    };
 200    tiling->tile0.extent = (VkExtent2D) {
 201       .width = align(ra_width, tile_align_w),
 202       .height = align(ra_height, tile_align_h),
 203    };
 204
 205    if (unlikely(dev->physical_device->instance->debug_flags & TU_DEBUG_FORCEBIN)) {
 206       /* start with 2x2 tiles */
 207       tiling->tile_count.width = 2;
 208       tiling->tile_count.height = 2;
 209       tiling->tile0.extent.width = align(DIV_ROUND_UP(ra_width, 2), tile_align_w);
 210       tiling->tile0.extent.height = align(DIV_ROUND_UP(ra_height, 2), tile_align_h);
 211    }
 212
 213    /* do not exceed max tile width */
 214    while (tiling->tile0.extent.width > max_tile_width) {
 215       tiling->tile_count.width++;
 216       tiling->tile0.extent.width =
 217          align(DIV_ROUND_UP(ra_width, tiling->tile_count.width), tile_align_w);
 218    }
 219
 220    /* do not exceed gmem size */
 221    while (tiling->tile0.extent.width * tiling->tile0.extent.height > pixels) {
 222       if (tiling->tile0.extent.width > MAX2(tile_align_w, tiling->tile0.extent.height)) {
 223          tiling->tile_count.width++;
 224          tiling->tile0.extent.width =
 225             align(DIV_ROUND_UP(ra_width, tiling->tile_count.width), tile_align_w);
 226       } else {
 227          /* if this assert fails then layout is impossible.. */
 228          assert(tiling->tile0.extent.height > tile_align_h);
 229          tiling->tile_count.height++;
 230          tiling->tile0.extent.height =
 231             align(DIV_ROUND_UP(ra_height, tiling->tile_count.height), tile_align_h);
 232       }
 233    }
 234 }
 235
 236 static void
 237 tu_tiling_config_update_pipe_layout(struct tu_tiling_config *tiling,
 238                                     const struct tu_device *dev)
 239 {
 240    const uint32_t max_pipe_count = 32; /* A6xx */
 241
 242    /* start from 1 tile per pipe */
 243    tiling->pipe0 = (VkExtent2D) {
 244       .width = 1,
 245       .height = 1,
 246    };
 247    tiling->pipe_count = tiling->tile_count;
 248
 249    /* do not exceed max pipe count vertically */
 250    while (tiling->pipe_count.height > max_pipe_count) {
 251       tiling->pipe0.height += 2;
 252       tiling->pipe_count.height =
 253          (tiling->tile_count.height + tiling->pipe0.height - 1) /
 254          tiling->pipe0.height;
 255    }
 256
 257    /* do not exceed max pipe count */
 258    while (tiling->pipe_count.width * tiling->pipe_count.height >
 259           max_pipe_count) {
 260       tiling->pipe0.width += 1;
 261       tiling->pipe_count.width =
 262          (tiling->tile_count.width + tiling->pipe0.width - 1) /
 263          tiling->pipe0.width;
 264    }
 265 }
 266
 267 static void
 268 tu_tiling_config_update_pipes(struct tu_tiling_config *tiling,
 269                               const struct tu_device *dev)
 270 {
 271    const uint32_t max_pipe_count = 32; /* A6xx */
 272    const uint32_t used_pipe_count =
 273       tiling->pipe_count.width * tiling->pipe_count.height;
 274    const VkExtent2D last_pipe = {
 275       .width = (tiling->tile_count.width - 1) % tiling->pipe0.width + 1,
 276       .height = (tiling->tile_count.height - 1) % tiling->pipe0.height + 1,
 277    };
 278
 279    assert(used_pipe_count <= max_pipe_count);
 280    assert(max_pipe_count <= ARRAY_SIZE(tiling->pipe_config));
 281
 282    for (uint32_t y = 0; y < tiling->pipe_count.height; y++) {
 283       for (uint32_t x = 0; x < tiling->pipe_count.width; x++) {
 284          const uint32_t pipe_x = tiling->pipe0.width * x;
 285          const uint32_t pipe_y = tiling->pipe0.height * y;
 286          const uint32_t pipe_w = (x == tiling->pipe_count.width - 1)
 287                                     ? last_pipe.width
 288                                     : tiling->pipe0.width;
 289          const uint32_t pipe_h = (y == tiling->pipe_count.height - 1)
 290                                     ? last_pipe.height
 291                                     : tiling->pipe0.height;
 292          const uint32_t n = tiling->pipe_count.width * y + x;
 293
 294          tiling->pipe_config[n] = A6XX_VSC_PIPE_CONFIG_REG_X(pipe_x) |
 295                                   A6XX_VSC_PIPE_CONFIG_REG_Y(pipe_y) |
 296                                   A6XX_VSC_PIPE_CONFIG_REG_W(pipe_w) |
 297                                   A6XX_VSC_PIPE_CONFIG_REG_H(pipe_h);
 298          tiling->pipe_sizes[n] = CP_SET_BIN_DATA5_0_VSC_SIZE(pipe_w * pipe_h);
 299       }
 300    }
 301
 302    memset(tiling->pipe_config + used_pipe_count, 0,
 303           sizeof(uint32_t) * (max_pipe_count - used_pipe_count));
 304 }
 305
 306 static void
 307 tu_tiling_config_get_tile(const struct tu_tiling_config *tiling,
 308                           const struct tu_device *dev,
 309                           uint32_t tx,
 310                           uint32_t ty,
 311                           struct tu_tile *tile)
 312 {
 313    /* find the pipe and the slot for tile (tx, ty) */
 314    const uint32_t px = tx / tiling->pipe0.width;
 315    const uint32_t py = ty / tiling->pipe0.height;
 316    const uint32_t sx = tx - tiling->pipe0.width * px;
 317    const uint32_t sy = ty - tiling->pipe0.height * py;
 318
 319    assert(tx < tiling->tile_count.width && ty < tiling->tile_count.height);
 320    assert(px < tiling->pipe_count.width && py < tiling->pipe_count.height);
 321    assert(sx < tiling->pipe0.width && sy < tiling->pipe0.height);
 322
 323    /* convert to 1D indices */
 324    tile->pipe = tiling->pipe_count.width * py + px;
 325    tile->slot = tiling->pipe0.width * sy + sx;
 326
 327    /* get the blit area for the tile */
 328    tile->begin = (VkOffset2D) {
 329       .x = tiling->tile0.offset.x + tiling->tile0.extent.width * tx,
 330       .y = tiling->tile0.offset.y + tiling->tile0.extent.height * ty,
 331    };
 332    tile->end.x =
 333       (tx == tiling->tile_count.width - 1)
 334          ? tiling->render_area.offset.x + tiling->render_area.extent.width
 335          : tile->begin.x + tiling->tile0.extent.width;
 336    tile->end.y =
 337       (ty == tiling->tile_count.height - 1)
 338          ? tiling->render_area.offset.y + tiling->render_area.extent.height
 339          : tile->begin.y + tiling->tile0.extent.height;
 340 }
 341
 342 enum a3xx_msaa_samples
 343 tu_msaa_samples(uint32_t samples)
 344 {
 345    switch (samples) {
 346    case 1:
 347       return MSAA_ONE;
 348    case 2:
 349       return MSAA_TWO;
 350    case 4:
 351       return MSAA_FOUR;
 352    case 8:
 353       return MSAA_EIGHT;
 354    default:
 355       assert(!"invalid sample count");
 356       return MSAA_ONE;
 357    }
 358 }
 359
 360 static enum a4xx_index_size
 361 tu6_index_size(VkIndexType type)
 362 {
 363    switch (type) {
 364    case VK_INDEX_TYPE_UINT16:
 365       return INDEX4_SIZE_16_BIT;
 366    case VK_INDEX_TYPE_UINT32:
 367       return INDEX4_SIZE_32_BIT;
 368    default:
 369       unreachable("invalid VkIndexType");
 370       return INDEX4_SIZE_8_BIT;
 371    }
 372 }
 373
 374 unsigned
 375 tu6_emit_event_write(struct tu_cmd_buffer *cmd,
 376                      struct tu_cs *cs,
 377                      enum vgt_event_type event,
 378                      bool need_seqno)
 379 {
 380    unsigned seqno = 0;
 381
 382    tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, need_seqno ? 4 : 1);
 383    tu_cs_emit(cs, CP_EVENT_WRITE_0_EVENT(event));
 384    if (need_seqno) {
 385       tu_cs_emit_qw(cs, cmd->scratch_bo.iova);
 386       seqno = ++cmd->scratch_seqno;
 387       tu_cs_emit(cs, seqno);
 388    }
 389
 390    return seqno;
 391 }
 392
 393 static void
 394 tu6_emit_cache_flush(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 395 {
 396    tu6_emit_event_write(cmd, cs, 0x31, false);
 397 }
 398
 399 static void
 400 tu6_emit_lrz_flush(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 401 {
 402    tu6_emit_event_write(cmd, cs, LRZ_FLUSH, false);
 403 }
 404
 405 static void
 406 tu6_emit_wfi(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 407 {
 408    if (cmd->wait_for_idle) {
 409       tu_cs_emit_wfi(cs);
 410       cmd->wait_for_idle = false;
 411    }
 412 }
 413
 414 #define tu_image_view_ubwc_pitches(iview)                                \
 415    .pitch = tu_image_ubwc_pitch(iview->image, iview->base_mip),          \
 416    .array_pitch = tu_image_ubwc_size(iview->image, iview->base_mip) >> 2
 417
 418 static void
 419 tu6_emit_zs(struct tu_cmd_buffer *cmd,
 420             const struct tu_subpass *subpass,
 421             struct tu_cs *cs)
 422 {
 423    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 424
 425    const uint32_t a = subpass->depth_stencil_attachment.attachment;
 426    if (a == VK_ATTACHMENT_UNUSED) {
 427       tu_cs_emit_regs(cs,
 428                       A6XX_RB_DEPTH_BUFFER_INFO(.depth_format = DEPTH6_NONE),
 429                       A6XX_RB_DEPTH_BUFFER_PITCH(0),
 430                       A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH(0),
 431                       A6XX_RB_DEPTH_BUFFER_BASE(0),
 432                       A6XX_RB_DEPTH_BUFFER_BASE_GMEM(0));
 433
 434       tu_cs_emit_regs(cs,
 435                       A6XX_GRAS_SU_DEPTH_BUFFER_INFO(.depth_format = DEPTH6_NONE));
 436
 437       tu_cs_emit_regs(cs,
 438                       A6XX_GRAS_LRZ_BUFFER_BASE(0),
 439                       A6XX_GRAS_LRZ_BUFFER_PITCH(0),
 440                       A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE(0));
 441
 442       tu_cs_emit_regs(cs, A6XX_RB_STENCIL_INFO(0));
 443
 444       return;
 445    }
 446
 447    const struct tu_image_view *iview = fb->attachments[a].attachment;
 448    enum a6xx_depth_format fmt = tu6_pipe2depth(iview->vk_format);
 449
 450    tu_cs_emit_regs(cs,
 451                    A6XX_RB_DEPTH_BUFFER_INFO(.depth_format = fmt),
 452                    A6XX_RB_DEPTH_BUFFER_PITCH(tu_image_stride(iview->image, iview->base_mip)),
 453                    A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH(iview->image->layout.layer_size),
 454                    A6XX_RB_DEPTH_BUFFER_BASE(tu_image_view_base_ref(iview)),
 455                    A6XX_RB_DEPTH_BUFFER_BASE_GMEM(cmd->state.pass->attachments[a].gmem_offset));
 456
 457    tu_cs_emit_regs(cs,
 458                    A6XX_GRAS_SU_DEPTH_BUFFER_INFO(.depth_format = fmt));
 459
 460    tu_cs_emit_regs(cs,
 461                    A6XX_RB_DEPTH_FLAG_BUFFER_BASE(tu_image_view_ubwc_base_ref(iview)),
 462                    A6XX_RB_DEPTH_FLAG_BUFFER_PITCH(tu_image_view_ubwc_pitches(iview)));
 463
 464    tu_cs_emit_regs(cs,
 465                    A6XX_GRAS_LRZ_BUFFER_BASE(0),
 466                    A6XX_GRAS_LRZ_BUFFER_PITCH(0),
 467                    A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE(0));
 468
 469    tu_cs_emit_regs(cs,
 470                    A6XX_RB_STENCIL_INFO(0));
 471
 472    /* enable zs? */
 473 }
 474
 475 static void
 476 tu6_emit_mrt(struct tu_cmd_buffer *cmd,
 477              const struct tu_subpass *subpass,
 478              struct tu_cs *cs)
 479 {
 480    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 481    unsigned char mrt_comp[MAX_RTS] = { 0 };
 482    unsigned srgb_cntl = 0;
 483
 484    for (uint32_t i = 0; i < subpass->color_count; ++i) {
 485       uint32_t a = subpass->color_attachments[i].attachment;
 486       if (a == VK_ATTACHMENT_UNUSED)
 487          continue;
 488
 489       const struct tu_image_view *iview = fb->attachments[a].attachment;
 490       const enum a6xx_tile_mode tile_mode =
 491          tu6_get_image_tile_mode(iview->image, iview->base_mip);
 492
 493       mrt_comp[i] = 0xf;
 494
 495       if (vk_format_is_srgb(iview->vk_format))
 496          srgb_cntl |= (1 << i);
 497
 498       const struct tu_native_format *format =
 499          tu6_get_native_format(iview->vk_format);
 500       assert(format && format->rb >= 0);
 501
 502       tu_cs_emit_regs(cs,
 503                       A6XX_RB_MRT_BUF_INFO(i,
 504                                            .color_tile_mode = tile_mode,
 505                                            .color_format = format->rb,
 506                                            .color_swap = format->swap),
 507                       A6XX_RB_MRT_PITCH(i, tu_image_stride(iview->image, iview->base_mip)),
 508                       A6XX_RB_MRT_ARRAY_PITCH(i, iview->image->layout.layer_size),
 509                       A6XX_RB_MRT_BASE(i, tu_image_view_base_ref(iview)),
 510                       A6XX_RB_MRT_BASE_GMEM(i, cmd->state.pass->attachments[a].gmem_offset));
 511
 512       tu_cs_emit_regs(cs,
 513                       A6XX_SP_FS_MRT_REG(i,
 514                                          .color_format = format->rb,
 515                                          .color_sint = vk_format_is_sint(iview->vk_format),
 516                                          .color_uint = vk_format_is_uint(iview->vk_format)));
 517
 518       tu_cs_emit_regs(cs,
 519                       A6XX_RB_MRT_FLAG_BUFFER_ADDR(i, tu_image_view_ubwc_base_ref(iview)),
 520                       A6XX_RB_MRT_FLAG_BUFFER_PITCH(i, tu_image_view_ubwc_pitches(iview)));
 521    }
 522
 523    tu_cs_emit_regs(cs,
 524                    A6XX_RB_SRGB_CNTL(srgb_cntl));
 525
 526    tu_cs_emit_regs(cs,
 527                    A6XX_SP_SRGB_CNTL(srgb_cntl));
 528
 529    tu_cs_emit_regs(cs,
 530                    A6XX_RB_RENDER_COMPONENTS(
 531                       .rt0 = mrt_comp[0],
 532                       .rt1 = mrt_comp[1],
 533                       .rt2 = mrt_comp[2],
 534                       .rt3 = mrt_comp[3],
 535                       .rt4 = mrt_comp[4],
 536                       .rt5 = mrt_comp[5],
 537                       .rt6 = mrt_comp[6],
 538                       .rt7 = mrt_comp[7]));
 539
 540    tu_cs_emit_regs(cs,
 541                    A6XX_SP_FS_RENDER_COMPONENTS(
 542                       .rt0 = mrt_comp[0],
 543                       .rt1 = mrt_comp[1],
 544                       .rt2 = mrt_comp[2],
 545                       .rt3 = mrt_comp[3],
 546                       .rt4 = mrt_comp[4],
 547                       .rt5 = mrt_comp[5],
 548                       .rt6 = mrt_comp[6],
 549                       .rt7 = mrt_comp[7]));
 550 }
 551
 552 static void
 553 tu6_emit_msaa(struct tu_cmd_buffer *cmd,
 554               const struct tu_subpass *subpass,
 555               struct tu_cs *cs)
 556 {
 557    const enum a3xx_msaa_samples samples = tu_msaa_samples(subpass->samples);
 558    bool msaa_disable = samples == MSAA_ONE;
 559
 560    tu_cs_emit_regs(cs,
 561                    A6XX_SP_TP_RAS_MSAA_CNTL(samples),
 562                    A6XX_SP_TP_DEST_MSAA_CNTL(.samples = samples,
 563                                              .msaa_disable = msaa_disable));
 564
 565    tu_cs_emit_regs(cs,
 566                    A6XX_GRAS_RAS_MSAA_CNTL(samples),
 567                    A6XX_GRAS_DEST_MSAA_CNTL(.samples = samples,
 568                                             .msaa_disable = msaa_disable));
 569
 570    tu_cs_emit_regs(cs,
 571                    A6XX_RB_RAS_MSAA_CNTL(samples),
 572                    A6XX_RB_DEST_MSAA_CNTL(.samples = samples,
 573                                           .msaa_disable = msaa_disable));
 574
 575    tu_cs_emit_regs(cs,
 576                    A6XX_RB_MSAA_CNTL(samples));
 577 }
 578
 579 static void
 580 tu6_emit_bin_size(struct tu_cs *cs,
 581                   uint32_t bin_w, uint32_t bin_h, uint32_t flags)
 582 {
 583    tu_cs_emit_regs(cs,
 584                    A6XX_GRAS_BIN_CONTROL(.binw = bin_w,
 585                                          .binh = bin_h,
 586                                          .dword = flags));
 587
 588    tu_cs_emit_regs(cs,
 589                    A6XX_RB_BIN_CONTROL(.binw = bin_w,
 590                                        .binh = bin_h,
 591                                        .dword = flags));
 592
 593    /* no flag for RB_BIN_CONTROL2... */
 594    tu_cs_emit_regs(cs,
 595                    A6XX_RB_BIN_CONTROL2(.binw = bin_w,
 596                                         .binh = bin_h));
 597 }
 598
 599 static void
 600 tu6_emit_render_cntl(struct tu_cmd_buffer *cmd,
 601                      const struct tu_subpass *subpass,
 602                      struct tu_cs *cs,
 603                      bool binning)
 604 {
 605    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 606    uint32_t cntl = 0;
 607    cntl |= A6XX_RB_RENDER_CNTL_UNK4;
 608    if (binning) {
 609       cntl |= A6XX_RB_RENDER_CNTL_BINNING;
 610    } else {
 611       uint32_t mrts_ubwc_enable = 0;
 612       for (uint32_t i = 0; i < subpass->color_count; ++i) {
 613          uint32_t a = subpass->color_attachments[i].attachment;
 614          if (a == VK_ATTACHMENT_UNUSED)
 615             continue;
 616
 617          const struct tu_image_view *iview = fb->attachments[a].attachment;
 618          if (iview->image->layout.ubwc_layer_size != 0)
 619             mrts_ubwc_enable |= 1 << i;
 620       }
 621
 622       cntl |= A6XX_RB_RENDER_CNTL_FLAG_MRTS(mrts_ubwc_enable);
 623
 624       const uint32_t a = subpass->depth_stencil_attachment.attachment;
 625       if (a != VK_ATTACHMENT_UNUSED) {
 626          const struct tu_image_view *iview = fb->attachments[a].attachment;
 627          if (iview->image->layout.ubwc_layer_size != 0)
 628             cntl |= A6XX_RB_RENDER_CNTL_FLAG_DEPTH;
 629       }
 630
 631       /* In the !binning case, we need to set RB_RENDER_CNTL in the draw_cs
 632        * in order to set it correctly for the different subpasses. However,
 633        * that means the packets we're emitting also happen during binning. So
 634        * we need to guard the write on !BINNING at CP execution time.
 635        */
 636       tu_cs_reserve(cs, 3 + 4);
 637       tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
 638       tu_cs_emit(cs, CP_COND_REG_EXEC_0_MODE(RENDER_MODE) |
 639                      CP_COND_REG_EXEC_0_GMEM | CP_COND_REG_EXEC_0_SYSMEM);
 640       tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(4));
 641    }
 642
 643    tu_cs_emit_pkt7(cs, CP_REG_WRITE, 3);
 644    tu_cs_emit(cs, CP_REG_WRITE_0_TRACKER(TRACK_RENDER_CNTL));
 645    tu_cs_emit(cs, REG_A6XX_RB_RENDER_CNTL);
 646    tu_cs_emit(cs, cntl);
 647 }
 648
 649 static void
 650 tu6_emit_blit_scissor(struct tu_cmd_buffer *cmd, struct tu_cs *cs, bool align)
 651 {
 652    const VkRect2D *render_area = &cmd->state.tiling_config.render_area;
 653    uint32_t x1 = render_area->offset.x;
 654    uint32_t y1 = render_area->offset.y;
 655    uint32_t x2 = x1 + render_area->extent.width - 1;
 656    uint32_t y2 = y1 + render_area->extent.height - 1;
 657
 658    /* TODO: alignment requirement seems to be less than tile_align_w/h */
 659    if (align) {
 660       x1 = x1 & ~cmd->device->physical_device->tile_align_w;
 661       y1 = y1 & ~cmd->device->physical_device->tile_align_h;
 662       x2 = ALIGN_POT(x2 + 1, cmd->device->physical_device->tile_align_w) - 1;
 663       y2 = ALIGN_POT(y2 + 1, cmd->device->physical_device->tile_align_h) - 1;
 664    }
 665
 666    tu_cs_emit_regs(cs,
 667                    A6XX_RB_BLIT_SCISSOR_TL(.x = x1, .y = y1),
 668                    A6XX_RB_BLIT_SCISSOR_BR(.x = x2, .y = y2));
 669 }
 670
 671 static void
 672 tu6_emit_blit_info(struct tu_cmd_buffer *cmd,
 673                    struct tu_cs *cs,
 674                    const struct tu_image_view *iview,
 675                    uint32_t gmem_offset,
 676                    bool resolve)
 677 {
 678    tu_cs_emit_regs(cs,
 679                    A6XX_RB_BLIT_INFO(.unk0 = !resolve, .gmem = !resolve));
 680
 681    const struct tu_native_format *format =
 682       tu6_get_native_format(iview->vk_format);
 683    assert(format && format->rb >= 0);
 684
 685    enum a6xx_tile_mode tile_mode =
 686       tu6_get_image_tile_mode(iview->image, iview->base_mip);
 687    tu_cs_emit_regs(cs,
 688                    A6XX_RB_BLIT_DST_INFO(
 689                       .tile_mode = tile_mode,
 690                       .samples = tu_msaa_samples(iview->image->samples),
 691                       .color_format = format->rb,
 692                       .color_swap = format->swap,
 693                       .flags = iview->image->layout.ubwc_layer_size != 0),
 694                    A6XX_RB_BLIT_DST(tu_image_view_base_ref(iview)),
 695                    A6XX_RB_BLIT_DST_PITCH(tu_image_stride(iview->image, iview->base_mip)),
 696                    A6XX_RB_BLIT_DST_ARRAY_PITCH(iview->image->layout.layer_size));
 697
 698    if (iview->image->layout.ubwc_layer_size) {
 699       tu_cs_emit_regs(cs,
 700                       A6XX_RB_BLIT_FLAG_DST(tu_image_view_ubwc_base_ref(iview)),
 701                       A6XX_RB_BLIT_FLAG_DST_PITCH(tu_image_view_ubwc_pitches(iview)));
 702    }
 703
 704    tu_cs_emit_regs(cs,
 705                    A6XX_RB_BLIT_BASE_GMEM(gmem_offset));
 706 }
 707
 708 static void
 709 tu6_emit_blit(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 710 {
 711    tu6_emit_event_write(cmd, cs, BLIT, false);
 712 }
 713
 714 static void
 715 tu6_emit_window_scissor(struct tu_cmd_buffer *cmd,
 716                         struct tu_cs *cs,
 717                         uint32_t x1,
 718                         uint32_t y1,
 719                         uint32_t x2,
 720                         uint32_t y2)
 721 {
 722    tu_cs_emit_regs(cs,
 723                    A6XX_GRAS_SC_WINDOW_SCISSOR_TL(.x = x1, .y = y1),
 724                    A6XX_GRAS_SC_WINDOW_SCISSOR_BR(.x = x2, .y = y2));
 725
 726    tu_cs_emit_regs(cs,
 727                    A6XX_GRAS_RESOLVE_CNTL_1(.x = x1, .y = y1),
 728                    A6XX_GRAS_RESOLVE_CNTL_2(.x = x2, .y = y2));
 729 }
 730
 731 static void
 732 tu6_emit_window_offset(struct tu_cmd_buffer *cmd,
 733                        struct tu_cs *cs,
 734                        uint32_t x1,
 735                        uint32_t y1)
 736 {
 737    tu_cs_emit_regs(cs,
 738                    A6XX_RB_WINDOW_OFFSET(.x = x1, .y = y1));
 739
 740    tu_cs_emit_regs(cs,
 741                    A6XX_RB_WINDOW_OFFSET2(.x = x1, .y = y1));
 742
 743    tu_cs_emit_regs(cs,
 744                    A6XX_SP_WINDOW_OFFSET(.x = x1, .y = y1));
 745
 746    tu_cs_emit_regs(cs,
 747                    A6XX_SP_TP_WINDOW_OFFSET(.x = x1, .y = y1));
 748 }
 749
 750 static bool
 751 use_hw_binning(struct tu_cmd_buffer *cmd)
 752 {
 753    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
 754
 755    if (unlikely(cmd->device->physical_device->instance->debug_flags & TU_DEBUG_NOBIN))
 756       return false;
 757
 758    if (unlikely(cmd->device->physical_device->instance->debug_flags & TU_DEBUG_FORCEBIN))
 759       return true;
 760
 761    return (tiling->tile_count.width * tiling->tile_count.height) > 2;
 762 }
 763
 764 static bool
 765 use_sysmem_rendering(struct tu_cmd_buffer *cmd)
 766 {
 767    if (unlikely(cmd->device->physical_device->instance->debug_flags & TU_DEBUG_SYSMEM))
 768       return true;
 769
 770    return cmd->state.tiling_config.force_sysmem;
 771 }
 772
 773 static void
 774 tu6_emit_tile_select(struct tu_cmd_buffer *cmd,
 775                      struct tu_cs *cs,
 776                      const struct tu_tile *tile)
 777 {
 778    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
 779    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_YIELD));
 780
 781    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
 782    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_GMEM));
 783
 784    const uint32_t x1 = tile->begin.x;
 785    const uint32_t y1 = tile->begin.y;
 786    const uint32_t x2 = tile->end.x - 1;
 787    const uint32_t y2 = tile->end.y - 1;
 788    tu6_emit_window_scissor(cmd, cs, x1, y1, x2, y2);
 789    tu6_emit_window_offset(cmd, cs, x1, y1);
 790
 791    tu_cs_emit_regs(cs,
 792                    A6XX_VPC_SO_OVERRIDE(.so_disable = true));
 793
 794    if (use_hw_binning(cmd)) {
 795       tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
 796
 797       tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
 798       tu_cs_emit(cs, 0x0);
 799
 800       tu_cs_emit_pkt7(cs, CP_REG_TEST, 1);
 801       tu_cs_emit(cs, A6XX_CP_REG_TEST_0_REG(OVERFLOW_FLAG_REG) |
 802                      A6XX_CP_REG_TEST_0_BIT(0) |
 803                      A6XX_CP_REG_TEST_0_WAIT_FOR_ME);
 804
 805       tu_cs_reserve(cs, 3 + 11);
 806       tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
 807       tu_cs_emit(cs, CP_COND_REG_EXEC_0_MODE(PRED_TEST));
 808       tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(11));
 809
 810       /* if (no overflow) */ {
 811          tu_cs_emit_pkt7(cs, CP_SET_BIN_DATA5, 7);
 812          tu_cs_emit(cs, cmd->state.tiling_config.pipe_sizes[tile->pipe] |
 813                         CP_SET_BIN_DATA5_0_VSC_N(tile->slot));
 814          tu_cs_emit_qw(cs, cmd->vsc_data.iova + tile->pipe * cmd->vsc_data_pitch);
 815          tu_cs_emit_qw(cs, cmd->vsc_data.iova + (tile->pipe * 4) + (32 * cmd->vsc_data_pitch));
 816          tu_cs_emit_qw(cs, cmd->vsc_data2.iova + (tile->pipe * cmd->vsc_data2_pitch));
 817
 818          tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
 819          tu_cs_emit(cs, 0x0);
 820
 821          /* use a NOP packet to skip over the 'else' side: */
 822          tu_cs_emit_pkt7(cs, CP_NOP, 2);
 823       } /* else */ {
 824          tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
 825          tu_cs_emit(cs, 0x1);
 826       }
 827
 828       tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
 829       tu_cs_emit(cs, 0x0);
 830
 831       tu_cs_emit_regs(cs,
 832                       A6XX_RB_UNKNOWN_8804(0));
 833
 834       tu_cs_emit_regs(cs,
 835                       A6XX_SP_TP_UNKNOWN_B304(0));
 836
 837       tu_cs_emit_regs(cs,
 838                       A6XX_GRAS_UNKNOWN_80A4(0));
 839    } else {
 840       tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
 841       tu_cs_emit(cs, 0x1);
 842
 843       tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
 844       tu_cs_emit(cs, 0x0);
 845    }
 846 }
 847
 848 static void
 849 tu6_emit_load_attachment(struct tu_cmd_buffer *cmd, struct tu_cs *cs, uint32_t a)
 850 {
 851    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
 852    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 853    const struct tu_image_view *iview = fb->attachments[a].attachment;
 854    const struct tu_render_pass_attachment *attachment =
 855       &cmd->state.pass->attachments[a];
 856
 857    if (attachment->gmem_offset < 0)
 858       return;
 859
 860    const uint32_t x1 = tiling->render_area.offset.x;
 861    const uint32_t y1 = tiling->render_area.offset.y;
 862    const uint32_t x2 = x1 + tiling->render_area.extent.width;
 863    const uint32_t y2 = y1 + tiling->render_area.extent.height;
 864    const uint32_t tile_x2 =
 865       tiling->tile0.offset.x + tiling->tile0.extent.width * tiling->tile_count.width;
 866    const uint32_t tile_y2 =
 867       tiling->tile0.offset.y + tiling->tile0.extent.height * tiling->tile_count.height;
 868    bool need_load =
 869       x1 != tiling->tile0.offset.x || x2 != MIN2(fb->width, tile_x2) ||
 870       y1 != tiling->tile0.offset.y || y2 != MIN2(fb->height, tile_y2);
 871
 872    if (need_load)
 873       tu_finishme("improve handling of unaligned render area");
 874
 875    if (attachment->load_op == VK_ATTACHMENT_LOAD_OP_LOAD)
 876       need_load = true;
 877
 878    if (vk_format_has_stencil(iview->vk_format) &&
 879        attachment->stencil_load_op == VK_ATTACHMENT_LOAD_OP_LOAD)
 880       need_load = true;
 881
 882    if (need_load) {
 883       tu6_emit_blit_info(cmd, cs, iview, attachment->gmem_offset, false);
 884       tu6_emit_blit(cmd, cs);
 885    }
 886 }
 887
 888 static void
 889 tu6_emit_clear_attachment(struct tu_cmd_buffer *cmd, struct tu_cs *cs,
 890                           uint32_t a,
 891                           const VkRenderPassBeginInfo *info)
 892 {
 893    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 894    const struct tu_image_view *iview = fb->attachments[a].attachment;
 895    const struct tu_render_pass_attachment *attachment =
 896       &cmd->state.pass->attachments[a];
 897    unsigned clear_mask = 0;
 898
 899    /* note: this means it isn't used by any subpass and shouldn't be cleared anyway */
 900    if (attachment->gmem_offset < 0)
 901       return;
 902
 903    if (attachment->load_op == VK_ATTACHMENT_LOAD_OP_CLEAR)
 904       clear_mask = 0xf;
 905
 906    if (vk_format_has_stencil(iview->vk_format)) {
 907       clear_mask &= 0x1;
 908       if (attachment->stencil_load_op == VK_ATTACHMENT_LOAD_OP_CLEAR)
 909          clear_mask |= 0x2;
 910    }
 911    if (!clear_mask)
 912       return;
 913
 914    tu_clear_gmem_attachment(cmd, cs, a, clear_mask,
 915                             &info->pClearValues[a]);
 916 }
 917
 918 static void
 919 tu6_emit_predicated_blit(struct tu_cmd_buffer *cmd,
 920                          struct tu_cs *cs,
 921                          uint32_t a,
 922                          uint32_t gmem_a,
 923                          bool resolve)
 924 {
 925    tu_cond_exec_start(cs, CP_COND_EXEC_0_RENDER_MODE_GMEM);
 926
 927    tu6_emit_blit_info(cmd, cs,
 928                       cmd->state.framebuffer->attachments[a].attachment,
 929                       cmd->state.pass->attachments[gmem_a].gmem_offset, resolve);
 930    tu6_emit_blit(cmd, cs);
 931
 932    tu_cond_exec_end(cs);
 933 }
 934
 935 static void
 936 tu6_emit_sysmem_resolve(struct tu_cmd_buffer *cmd,
 937                         struct tu_cs *cs,
 938                         uint32_t a,
 939                         uint32_t gmem_a)
 940 {
 941    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 942    const struct tu_image_view *dst = fb->attachments[a].attachment;
 943    const struct tu_image_view *src = fb->attachments[gmem_a].attachment;
 944
 945    tu_blit(cmd, cs, &(struct tu_blit) {
 946       .dst = sysmem_attachment_surf(dst, dst->base_layer,
 947                                     &cmd->state.tiling_config.render_area),
 948       .src = sysmem_attachment_surf(src, src->base_layer,
 949                                     &cmd->state.tiling_config.render_area),
 950       .layers = fb->layers,
 951    });
 952 }
 953
 954
 955 /* Emit a MSAA resolve operation, with both gmem and sysmem paths. */
 956 static void tu6_emit_resolve(struct tu_cmd_buffer *cmd,
 957                              struct tu_cs *cs,
 958                              uint32_t a,
 959                              uint32_t gmem_a)
 960 {
 961    if (cmd->state.pass->attachments[a].store_op == VK_ATTACHMENT_STORE_OP_DONT_CARE)
 962       return;
 963
 964    tu6_emit_predicated_blit(cmd, cs, a, gmem_a, true);
 965
 966    tu_cond_exec_start(cs, CP_COND_EXEC_0_RENDER_MODE_SYSMEM);
 967    tu6_emit_sysmem_resolve(cmd, cs, a, gmem_a);
 968    tu_cond_exec_end(cs);
 969 }
 970
 971 static void
 972 tu6_emit_store_attachment(struct tu_cmd_buffer *cmd,
 973                           struct tu_cs *cs,
 974                           uint32_t a,
 975                           uint32_t gmem_a)
 976 {
 977    if (cmd->state.pass->attachments[a].store_op == VK_ATTACHMENT_STORE_OP_DONT_CARE)
 978       return;
 979
 980    tu6_emit_blit_info(cmd, cs,
 981                       cmd->state.framebuffer->attachments[a].attachment,
 982                       cmd->state.pass->attachments[gmem_a].gmem_offset, true);
 983    tu6_emit_blit(cmd, cs);
 984 }
 985
 986 static void
 987 tu6_emit_tile_store(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 988 {
 989    const struct tu_render_pass *pass = cmd->state.pass;
 990    const struct tu_subpass *subpass = &pass->subpasses[pass->subpass_count-1];
 991
 992    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
 993    tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
 994                      CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
 995                      CP_SET_DRAW_STATE__0_GROUP_ID(0));
 996    tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
 997    tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
 998
 999    tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1000    tu_cs_emit(cs, 0x0);
1001
1002    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1003    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_RESOLVE));
1004
1005    tu6_emit_blit_scissor(cmd, cs, true);
1006
1007    for (uint32_t a = 0; a < pass->attachment_count; ++a) {
1008       if (pass->attachments[a].gmem_offset >= 0)
1009          tu6_emit_store_attachment(cmd, cs, a, a);
1010    }
1011
1012    if (subpass->resolve_attachments) {
1013       for (unsigned i = 0; i < subpass->color_count; i++) {
1014          uint32_t a = subpass->resolve_attachments[i].attachment;
1015          if (a != VK_ATTACHMENT_UNUSED)
1016             tu6_emit_store_attachment(cmd, cs, a,
1017                                       subpass->color_attachments[i].attachment);
1018       }
1019    }
1020 }
1021
1022 static void
1023 tu6_emit_restart_index(struct tu_cs *cs, uint32_t restart_index)
1024 {
1025    tu_cs_emit_regs(cs,
1026                    A6XX_PC_RESTART_INDEX(restart_index));
1027 }
1028
1029 static void
1030 tu6_init_hw(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1031 {
1032    tu6_emit_cache_flush(cmd, cs);
1033
1034    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UPDATE_CNTL, 0xfffff);
1035
1036    tu_cs_emit_write_reg(cs, REG_A6XX_RB_CCU_CNTL, 0x10000000);
1037    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8E04, 0x00100000);
1038    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE04, 0x8);
1039    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE00, 0);
1040    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE0F, 0x3f);
1041    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B605, 0x44);
1042    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B600, 0x100000);
1043    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE00, 0x80);
1044    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE01, 0);
1045
1046    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9600, 0);
1047    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8600, 0x880);
1048    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE04, 0);
1049    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE03, 0x00000410);
1050    tu_cs_emit_write_reg(cs, REG_A6XX_SP_IBO_COUNT, 0);
1051    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B182, 0);
1052    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BB11, 0);
1053    tu_cs_emit_write_reg(cs, REG_A6XX_UCHE_UNKNOWN_0E12, 0x3200000);
1054    tu_cs_emit_write_reg(cs, REG_A6XX_UCHE_CLIENT_PF, 4);
1055    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8E01, 0x0);
1056    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AB00, 0x5);
1057    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_ADD_OFFSET, A6XX_VFD_ADD_OFFSET_VERTEX);
1058    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8811, 0x00000010);
1059    tu_cs_emit_write_reg(cs, REG_A6XX_PC_MODE_CNTL, 0x1f);
1060
1061    tu_cs_emit_write_reg(cs, REG_A6XX_RB_SRGB_CNTL, 0);
1062
1063    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8110, 0);
1064
1065    tu_cs_emit_write_reg(cs, REG_A6XX_RB_RENDER_CONTROL0, 0x401);
1066    tu_cs_emit_write_reg(cs, REG_A6XX_RB_RENDER_CONTROL1, 0);
1067    tu_cs_emit_write_reg(cs, REG_A6XX_RB_FS_OUTPUT_CNTL0, 0);
1068    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8818, 0);
1069    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8819, 0);
1070    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881A, 0);
1071    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881B, 0);
1072    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881C, 0);
1073    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881D, 0);
1074    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881E, 0);
1075    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_88F0, 0);
1076
1077    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9101, 0xffff00);
1078    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9107, 0);
1079
1080    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9236, 1);
1081    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9300, 0);
1082
1083    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_SO_OVERRIDE,
1084                         A6XX_VPC_SO_OVERRIDE_SO_DISABLE);
1085
1086    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9801, 0);
1087    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9806, 0);
1088    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9980, 0);
1089
1090    tu_cs_emit_write_reg(cs, REG_A6XX_PC_PRIMITIVE_CNTL_6, 0);
1091    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9B07, 0);
1092
1093    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_A81B, 0);
1094
1095    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B183, 0);
1096
1097    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8099, 0);
1098    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_809B, 0);
1099    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A0, 2);
1100    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80AF, 0);
1101    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9210, 0);
1102    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9211, 0);
1103    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9602, 0);
1104    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9981, 0x3);
1105    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9E72, 0);
1106    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9108, 0x3);
1107    tu_cs_emit_write_reg(cs, REG_A6XX_SP_TP_UNKNOWN_B304, 0);
1108    tu_cs_emit_write_reg(cs, REG_A6XX_SP_TP_UNKNOWN_B309, 0x000000a2);
1109    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8804, 0);
1110    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A4, 0);
1111    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A5, 0);
1112    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A6, 0);
1113    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8805, 0);
1114    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8806, 0);
1115    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8878, 0);
1116    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8879, 0);
1117    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_CONTROL_5_REG, 0xfc);
1118
1119    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_MODE_CNTL, 0x00000000);
1120
1121    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_UNKNOWN_A008, 0);
1122
1123    tu_cs_emit_write_reg(cs, REG_A6XX_PC_MODE_CNTL, 0x0000001f);
1124
1125    /* we don't use this yet.. probably best to disable.. */
1126    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
1127    tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
1128                      CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
1129                      CP_SET_DRAW_STATE__0_GROUP_ID(0));
1130    tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
1131    tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
1132
1133    tu_cs_emit_regs(cs,
1134                    A6XX_VPC_SO_BUFFER_BASE(0),
1135                    A6XX_VPC_SO_BUFFER_SIZE(0));
1136
1137    tu_cs_emit_regs(cs,
1138                    A6XX_VPC_SO_FLUSH_BASE(0));
1139
1140    tu_cs_emit_regs(cs,
1141                    A6XX_VPC_SO_BUF_CNTL(0));
1142
1143    tu_cs_emit_regs(cs,
1144                    A6XX_VPC_SO_BUFFER_OFFSET(0, 0));
1145
1146    tu_cs_emit_regs(cs,
1147                    A6XX_VPC_SO_BUFFER_BASE(1, 0),
1148                    A6XX_VPC_SO_BUFFER_SIZE(1, 0));
1149
1150    tu_cs_emit_regs(cs,
1151                    A6XX_VPC_SO_BUFFER_OFFSET(1, 0),
1152                    A6XX_VPC_SO_FLUSH_BASE(1, 0),
1153                    A6XX_VPC_SO_BUFFER_BASE(2, 0),
1154                    A6XX_VPC_SO_BUFFER_SIZE(2, 0));
1155
1156    tu_cs_emit_regs(cs,
1157                    A6XX_VPC_SO_BUFFER_OFFSET(2, 0),
1158                    A6XX_VPC_SO_FLUSH_BASE(2, 0),
1159                    A6XX_VPC_SO_BUFFER_BASE(3, 0),
1160                    A6XX_VPC_SO_BUFFER_SIZE(3, 0));
1161
1162    tu_cs_emit_regs(cs,
1163                    A6XX_VPC_SO_BUFFER_OFFSET(3, 0),
1164                    A6XX_VPC_SO_FLUSH_BASE(3, 0));
1165
1166    tu_cs_emit_regs(cs,
1167                    A6XX_SP_HS_CTRL_REG0(0));
1168
1169    tu_cs_emit_regs(cs,
1170                    A6XX_SP_GS_CTRL_REG0(0));
1171
1172    tu_cs_emit_regs(cs,
1173                    A6XX_GRAS_LRZ_CNTL(0));
1174
1175    tu_cs_emit_regs(cs,
1176                    A6XX_RB_LRZ_CNTL(0));
1177
1178    tu_cs_sanity_check(cs);
1179 }
1180
1181 static void
1182 tu6_cache_flush(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1183 {
1184    unsigned seqno;
1185
1186    seqno = tu6_emit_event_write(cmd, cs, CACHE_FLUSH_AND_INV_EVENT, true);
1187
1188    tu_cs_emit_pkt7(cs, CP_WAIT_REG_MEM, 6);
1189    tu_cs_emit(cs, CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ) |
1190                   CP_WAIT_REG_MEM_0_POLL_MEMORY);
1191    tu_cs_emit_qw(cs, cmd->scratch_bo.iova);
1192    tu_cs_emit(cs, CP_WAIT_REG_MEM_3_REF(seqno));
1193    tu_cs_emit(cs, CP_WAIT_REG_MEM_4_MASK(~0));
1194    tu_cs_emit(cs, CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(16));
1195
1196    seqno = tu6_emit_event_write(cmd, cs, CACHE_FLUSH_TS, true);
1197
1198    tu_cs_emit_pkt7(cs, CP_WAIT_MEM_GTE, 4);
1199    tu_cs_emit(cs, CP_WAIT_MEM_GTE_0_RESERVED(0));
1200    tu_cs_emit_qw(cs, cmd->scratch_bo.iova);
1201    tu_cs_emit(cs, CP_WAIT_MEM_GTE_3_REF(seqno));
1202 }
1203
1204 static void
1205 update_vsc_pipe(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1206 {
1207    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1208
1209    tu_cs_emit_regs(cs,
1210                    A6XX_VSC_BIN_SIZE(.width = tiling->tile0.extent.width,
1211                                      .height = tiling->tile0.extent.height),
1212                    A6XX_VSC_SIZE_ADDRESS(.bo = &cmd->vsc_data,
1213                                          .bo_offset = 32 * cmd->vsc_data_pitch));
1214
1215    tu_cs_emit_regs(cs,
1216                    A6XX_VSC_BIN_COUNT(.nx = tiling->tile_count.width,
1217                                       .ny = tiling->tile_count.height));
1218
1219    tu_cs_emit_pkt4(cs, REG_A6XX_VSC_PIPE_CONFIG_REG(0), 32);
1220    for (unsigned i = 0; i < 32; i++)
1221       tu_cs_emit(cs, tiling->pipe_config[i]);
1222
1223    tu_cs_emit_regs(cs,
1224                    A6XX_VSC_PIPE_DATA2_ADDRESS(.bo = &cmd->vsc_data2),
1225                    A6XX_VSC_PIPE_DATA2_PITCH(cmd->vsc_data2_pitch),
1226                    A6XX_VSC_PIPE_DATA2_ARRAY_PITCH(cmd->vsc_data2.size));
1227
1228    tu_cs_emit_regs(cs,
1229                    A6XX_VSC_PIPE_DATA_ADDRESS(.bo = &cmd->vsc_data),
1230                    A6XX_VSC_PIPE_DATA_PITCH(cmd->vsc_data_pitch),
1231                    A6XX_VSC_PIPE_DATA_ARRAY_PITCH(cmd->vsc_data.size));
1232 }
1233
1234 static void
1235 emit_vsc_overflow_test(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1236 {
1237    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1238    const uint32_t used_pipe_count =
1239       tiling->pipe_count.width * tiling->pipe_count.height;
1240
1241    /* Clear vsc_scratch: */
1242    tu_cs_emit_pkt7(cs, CP_MEM_WRITE, 3);
1243    tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1244    tu_cs_emit(cs, 0x0);
1245
1246    /* Check for overflow, write vsc_scratch if detected: */
1247    for (int i = 0; i < used_pipe_count; i++) {
1248       tu_cs_emit_pkt7(cs, CP_COND_WRITE5, 8);
1249       tu_cs_emit(cs, CP_COND_WRITE5_0_FUNCTION(WRITE_GE) |
1250             CP_COND_WRITE5_0_WRITE_MEMORY);
1251       tu_cs_emit(cs, CP_COND_WRITE5_1_POLL_ADDR_LO(REG_A6XX_VSC_SIZE_REG(i)));
1252       tu_cs_emit(cs, CP_COND_WRITE5_2_POLL_ADDR_HI(0));
1253       tu_cs_emit(cs, CP_COND_WRITE5_3_REF(cmd->vsc_data_pitch));
1254       tu_cs_emit(cs, CP_COND_WRITE5_4_MASK(~0));
1255       tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1256       tu_cs_emit(cs, CP_COND_WRITE5_7_WRITE_DATA(1 + cmd->vsc_data_pitch));
1257
1258       tu_cs_emit_pkt7(cs, CP_COND_WRITE5, 8);
1259       tu_cs_emit(cs, CP_COND_WRITE5_0_FUNCTION(WRITE_GE) |
1260             CP_COND_WRITE5_0_WRITE_MEMORY);
1261       tu_cs_emit(cs, CP_COND_WRITE5_1_POLL_ADDR_LO(REG_A6XX_VSC_SIZE2_REG(i)));
1262       tu_cs_emit(cs, CP_COND_WRITE5_2_POLL_ADDR_HI(0));
1263       tu_cs_emit(cs, CP_COND_WRITE5_3_REF(cmd->vsc_data2_pitch));
1264       tu_cs_emit(cs, CP_COND_WRITE5_4_MASK(~0));
1265       tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1266       tu_cs_emit(cs, CP_COND_WRITE5_7_WRITE_DATA(3 + cmd->vsc_data2_pitch));
1267    }
1268
1269    tu_cs_emit_pkt7(cs, CP_WAIT_MEM_WRITES, 0);
1270
1271    tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
1272
1273    tu_cs_emit_pkt7(cs, CP_MEM_TO_REG, 3);
1274    tu_cs_emit(cs, CP_MEM_TO_REG_0_REG(OVERFLOW_FLAG_REG) |
1275          CP_MEM_TO_REG_0_CNT(1 - 1));
1276    tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1277
1278    /*
1279     * This is a bit awkward, we really want a way to invert the
1280     * CP_REG_TEST/CP_COND_REG_EXEC logic, so that we can conditionally
1281     * execute cmds to use hwbinning when a bit is *not* set.  This
1282     * dance is to invert OVERFLOW_FLAG_REG
1283     *
1284     * A CP_NOP packet is used to skip executing the 'else' clause
1285     * if (b0 set)..
1286     */
1287
1288    /* b0 will be set if VSC_DATA or VSC_DATA2 overflow: */
1289    tu_cs_emit_pkt7(cs, CP_REG_TEST, 1);
1290    tu_cs_emit(cs, A6XX_CP_REG_TEST_0_REG(OVERFLOW_FLAG_REG) |
1291          A6XX_CP_REG_TEST_0_BIT(0) |
1292          A6XX_CP_REG_TEST_0_WAIT_FOR_ME);
1293
1294    tu_cs_reserve(cs, 3 + 7);
1295    tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
1296    tu_cs_emit(cs, CP_COND_REG_EXEC_0_MODE(PRED_TEST));
1297    tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(7));
1298
1299    /* if (b0 set) */ {
1300       /*
1301        * On overflow, mirror the value to control->vsc_overflow
1302        * which CPU is checking to detect overflow (see
1303        * check_vsc_overflow())
1304        */
1305       tu_cs_emit_pkt7(cs, CP_REG_TO_MEM, 3);
1306       tu_cs_emit(cs, CP_REG_TO_MEM_0_REG(OVERFLOW_FLAG_REG) |
1307             CP_REG_TO_MEM_0_CNT(0));
1308       tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_OVERFLOW);
1309
1310       tu_cs_emit_pkt4(cs, OVERFLOW_FLAG_REG, 1);
1311       tu_cs_emit(cs, 0x0);
1312
1313       tu_cs_emit_pkt7(cs, CP_NOP, 2);  /* skip 'else' when 'if' is taken */
1314    } /* else */ {
1315       tu_cs_emit_pkt4(cs, OVERFLOW_FLAG_REG, 1);
1316       tu_cs_emit(cs, 0x1);
1317    }
1318 }
1319
1320 static void
1321 tu6_emit_binning_pass(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1322 {
1323    struct tu_physical_device *phys_dev = cmd->device->physical_device;
1324    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1325
1326    uint32_t x1 = tiling->tile0.offset.x;
1327    uint32_t y1 = tiling->tile0.offset.y;
1328    uint32_t x2 = tiling->render_area.offset.x + tiling->render_area.extent.width - 1;
1329    uint32_t y2 = tiling->render_area.offset.y + tiling->render_area.extent.height - 1;
1330
1331    tu6_emit_window_scissor(cmd, cs, x1, y1, x2, y2);
1332
1333    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1334    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_BINNING));
1335
1336    tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
1337    tu_cs_emit(cs, 0x1);
1338
1339    tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
1340    tu_cs_emit(cs, 0x1);
1341
1342    tu_cs_emit_wfi(cs);
1343
1344    tu_cs_emit_regs(cs,
1345                    A6XX_VFD_MODE_CNTL(.binning_pass = true));
1346
1347    update_vsc_pipe(cmd, cs);
1348
1349    tu_cs_emit_regs(cs,
1350                    A6XX_PC_UNKNOWN_9805(.unknown = phys_dev->magic.PC_UNKNOWN_9805));
1351
1352    tu_cs_emit_regs(cs,
1353                    A6XX_SP_UNKNOWN_A0F8(.unknown = phys_dev->magic.SP_UNKNOWN_A0F8));
1354
1355    tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, 1);
1356    tu_cs_emit(cs, UNK_2C);
1357
1358    tu_cs_emit_regs(cs,
1359                    A6XX_RB_WINDOW_OFFSET(.x = 0, .y = 0));
1360
1361    tu_cs_emit_regs(cs,
1362                    A6XX_SP_TP_WINDOW_OFFSET(.x = 0, .y = 0));
1363
1364    /* emit IB to binning drawcmds: */
1365    tu_cs_emit_call(cs, &cmd->draw_cs);
1366
1367    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
1368    tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
1369                   CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
1370                   CP_SET_DRAW_STATE__0_GROUP_ID(0));
1371    tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
1372    tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
1373
1374    tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, 1);
1375    tu_cs_emit(cs, UNK_2D);
1376
1377    tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE, false);
1378    tu6_cache_flush(cmd, cs);
1379
1380    tu_cs_emit_wfi(cs);
1381
1382    tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
1383
1384    emit_vsc_overflow_test(cmd, cs);
1385
1386    tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
1387    tu_cs_emit(cs, 0x0);
1388
1389    tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
1390    tu_cs_emit(cs, 0x0);
1391
1392    tu_cs_emit_wfi(cs);
1393
1394    tu_cs_emit_regs(cs,
1395                    A6XX_RB_CCU_CNTL(.unknown = phys_dev->magic.RB_CCU_CNTL_gmem));
1396
1397    cmd->wait_for_idle = false;
1398 }
1399
1400 static void
1401 tu_emit_sysmem_clear_attachment(struct tu_cmd_buffer *cmd, struct tu_cs *cs,
1402                                 uint32_t a,
1403                                 const VkRenderPassBeginInfo *info)
1404 {
1405    const struct tu_framebuffer *fb = cmd->state.framebuffer;
1406    const struct tu_image_view *iview = fb->attachments[a].attachment;
1407    const struct tu_render_pass_attachment *attachment =
1408       &cmd->state.pass->attachments[a];
1409    unsigned clear_mask = 0;
1410
1411    /* note: this means it isn't used by any subpass and shouldn't be cleared anyway */
1412    if (attachment->gmem_offset < 0)
1413       return;
1414
1415    if (attachment->load_op == VK_ATTACHMENT_LOAD_OP_CLEAR) {
1416       clear_mask = 0xf;
1417    }
1418
1419    if (vk_format_has_stencil(iview->vk_format)) {
1420       clear_mask &= 0x1;
1421       if (attachment->stencil_load_op == VK_ATTACHMENT_LOAD_OP_CLEAR)
1422          clear_mask |= 0x2;
1423       if (clear_mask != 0x3)
1424          tu_finishme("depth/stencil only load op");
1425    }
1426
1427    if (!clear_mask)
1428       return;
1429
1430    tu_clear_sysmem_attachment(cmd, cs, a,
1431                               &info->pClearValues[a], &(struct VkClearRect) {
1432       .rect = info->renderArea,
1433       .baseArrayLayer = iview->base_layer,
1434       .layerCount = iview->layer_count,
1435    });
1436 }
1437
1438 static void
1439 tu_emit_load_clear(struct tu_cmd_buffer *cmd,
1440                    const VkRenderPassBeginInfo *info)
1441 {
1442    struct tu_cs *cs = &cmd->draw_cs;
1443
1444    tu_cond_exec_start(cs, CP_COND_EXEC_0_RENDER_MODE_GMEM);
1445
1446    tu6_emit_blit_scissor(cmd, cs, true);
1447
1448    for (uint32_t i = 0; i < cmd->state.pass->attachment_count; ++i)
1449       tu6_emit_load_attachment(cmd, cs, i);
1450
1451    tu6_emit_blit_scissor(cmd, cs, false);
1452
1453    for (uint32_t i = 0; i < cmd->state.pass->attachment_count; ++i)
1454       tu6_emit_clear_attachment(cmd, cs, i, info);
1455
1456    tu_cond_exec_end(cs);
1457
1458    /* invalidate because reading input attachments will cache GMEM and
1459     * the cache isn''t updated when GMEM is written
1460     * TODO: is there a no-cache bit for textures?
1461     */
1462    if (cmd->state.subpass->input_count)
1463       tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE, false);
1464
1465    tu_cond_exec_start(cs, CP_COND_EXEC_0_RENDER_MODE_SYSMEM);
1466
1467    for (uint32_t i = 0; i < cmd->state.pass->attachment_count; ++i)
1468       tu_emit_sysmem_clear_attachment(cmd, cs, i, info);
1469
1470    tu_cond_exec_end(cs);
1471 }
1472
1473 static void
1474 tu6_sysmem_render_begin(struct tu_cmd_buffer *cmd, struct tu_cs *cs,
1475                         const struct VkRect2D *renderArea)
1476 {
1477    const struct tu_framebuffer *fb = cmd->state.framebuffer;
1478    if (fb->width > 0 && fb->height > 0) {
1479       tu6_emit_window_scissor(cmd, cs,
1480                               0, 0, fb->width - 1, fb->height - 1);
1481    } else {
1482       tu6_emit_window_scissor(cmd, cs, 0, 0, 0, 0);
1483    }
1484
1485    tu6_emit_window_offset(cmd, cs, 0, 0);
1486
1487    tu6_emit_bin_size(cs, 0, 0, 0xc00000); /* 0xc00000 = BYPASS? */
1488
1489    tu6_emit_lrz_flush(cmd, cs);
1490
1491    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1492    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_BYPASS));
1493
1494    tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1495    tu_cs_emit(cs, 0x0);
1496
1497    tu6_emit_event_write(cmd, cs, PC_CCU_INVALIDATE_COLOR, false);
1498    tu6_emit_event_write(cmd, cs, PC_CCU_INVALIDATE_DEPTH, false);
1499    tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE, false);
1500
1501    tu6_emit_wfi(cmd, cs);
1502    tu_cs_emit_regs(cs,
1503                    A6XX_RB_CCU_CNTL(0x10000000));
1504
1505    /* enable stream-out, with sysmem there is only one pass: */
1506    tu_cs_emit_regs(cs,
1507                    A6XX_VPC_SO_OVERRIDE(.so_disable = false));
1508
1509    tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
1510    tu_cs_emit(cs, 0x1);
1511
1512    tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
1513    tu_cs_emit(cs, 0x0);
1514
1515    tu_cs_sanity_check(cs);
1516 }
1517
1518 static void
1519 tu6_sysmem_render_end(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1520 {
1521    /* Do any resolves of the last subpass. These are handled in the
1522     * tile_store_ib in the gmem path.
1523     */
1524
1525    const struct tu_subpass *subpass = cmd->state.subpass;
1526    if (subpass->resolve_attachments) {
1527       for (unsigned i = 0; i < subpass->color_count; i++) {
1528          uint32_t a = subpass->resolve_attachments[i].attachment;
1529          if (a != VK_ATTACHMENT_UNUSED)
1530             tu6_emit_sysmem_resolve(cmd, cs, a,
1531                                     subpass->color_attachments[i].attachment);
1532       }
1533    }
1534
1535    tu_cs_emit_call(cs, &cmd->draw_epilogue_cs);
1536
1537    tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1538    tu_cs_emit(cs, 0x0);
1539
1540    tu6_emit_lrz_flush(cmd, cs);
1541
1542    tu6_emit_event_write(cmd, cs, PC_CCU_FLUSH_COLOR_TS, true);
1543    tu6_emit_event_write(cmd, cs, PC_CCU_FLUSH_DEPTH_TS, true);
1544
1545    tu_cs_sanity_check(cs);
1546 }
1547
1548
1549 static void
1550 tu6_tile_render_begin(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1551 {
1552    struct tu_physical_device *phys_dev = cmd->device->physical_device;
1553
1554    tu6_emit_lrz_flush(cmd, cs);
1555
1556    /* lrz clear? */
1557
1558    tu6_emit_cache_flush(cmd, cs);
1559
1560    tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1561    tu_cs_emit(cs, 0x0);
1562
1563    /* 0x10000000 for BYPASS.. 0x7c13c080 for GMEM: */
1564    tu6_emit_wfi(cmd, cs);
1565    tu_cs_emit_regs(cs,
1566                    A6XX_RB_CCU_CNTL(phys_dev->magic.RB_CCU_CNTL_gmem));
1567
1568    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1569    if (use_hw_binning(cmd)) {
1570       tu6_emit_bin_size(cs,
1571                         tiling->tile0.extent.width,
1572                         tiling->tile0.extent.height,
1573                         A6XX_RB_BIN_CONTROL_BINNING_PASS | 0x6000000);
1574
1575       tu6_emit_render_cntl(cmd, cmd->state.subpass, cs, true);
1576
1577       tu6_emit_binning_pass(cmd, cs);
1578
1579       tu6_emit_bin_size(cs,
1580                         tiling->tile0.extent.width,
1581                         tiling->tile0.extent.height,
1582                         A6XX_RB_BIN_CONTROL_USE_VIZ | 0x6000000);
1583
1584       tu_cs_emit_regs(cs,
1585                       A6XX_VFD_MODE_CNTL(0));
1586
1587       tu_cs_emit_regs(cs, A6XX_PC_UNKNOWN_9805(.unknown = phys_dev->magic.PC_UNKNOWN_9805));
1588
1589       tu_cs_emit_regs(cs, A6XX_SP_UNKNOWN_A0F8(.unknown = phys_dev->magic.SP_UNKNOWN_A0F8));
1590
1591       tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1592       tu_cs_emit(cs, 0x1);
1593    } else {
1594       tu6_emit_bin_size(cs,
1595                         tiling->tile0.extent.width,
1596                         tiling->tile0.extent.height,
1597                         0x6000000);
1598    }
1599
1600    tu_cs_sanity_check(cs);
1601 }
1602
1603 static void
1604 tu6_render_tile(struct tu_cmd_buffer *cmd,
1605                 struct tu_cs *cs,
1606                 const struct tu_tile *tile)
1607 {
1608    tu6_emit_tile_select(cmd, cs, tile);
1609
1610    tu_cs_emit_call(cs, &cmd->draw_cs);
1611    cmd->wait_for_idle = true;
1612
1613    if (use_hw_binning(cmd)) {
1614       tu_cs_emit_pkt7(cs, CP_REG_TEST, 1);
1615       tu_cs_emit(cs, A6XX_CP_REG_TEST_0_REG(OVERFLOW_FLAG_REG) |
1616                      A6XX_CP_REG_TEST_0_BIT(0) |
1617                      A6XX_CP_REG_TEST_0_WAIT_FOR_ME);
1618
1619       tu_cs_reserve(cs, 3 + 2);
1620       tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
1621       tu_cs_emit(cs, CP_COND_REG_EXEC_0_MODE(PRED_TEST));
1622       tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(2));
1623
1624       /* if (no overflow) */ {
1625          tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1626          tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_ENDVIS));
1627       }
1628    }
1629
1630    tu_cs_emit_ib(cs, &cmd->state.tile_store_ib);
1631
1632    tu_cs_sanity_check(cs);
1633 }
1634
1635 static void
1636 tu6_tile_render_end(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1637 {
1638    tu_cs_emit_call(cs, &cmd->draw_epilogue_cs);
1639
1640    tu_cs_emit_regs(cs,
1641                    A6XX_GRAS_LRZ_CNTL(0));
1642
1643    tu6_emit_lrz_flush(cmd, cs);
1644
1645    tu6_emit_event_write(cmd, cs, CACHE_FLUSH_TS, true);
1646
1647    tu_cs_sanity_check(cs);
1648 }
1649
1650 static void
1651 tu_cmd_render_tiles(struct tu_cmd_buffer *cmd)
1652 {
1653    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1654
1655    tu6_tile_render_begin(cmd, &cmd->cs);
1656
1657    for (uint32_t y = 0; y < tiling->tile_count.height; y++) {
1658       for (uint32_t x = 0; x < tiling->tile_count.width; x++) {
1659          struct tu_tile tile;
1660          tu_tiling_config_get_tile(tiling, cmd->device, x, y, &tile);
1661          tu6_render_tile(cmd, &cmd->cs, &tile);
1662       }
1663    }
1664
1665    tu6_tile_render_end(cmd, &cmd->cs);
1666 }
1667
1668 static void
1669 tu_cmd_render_sysmem(struct tu_cmd_buffer *cmd)
1670 {
1671    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1672
1673    tu6_sysmem_render_begin(cmd, &cmd->cs, &tiling->render_area);
1674
1675    tu_cs_emit_call(&cmd->cs, &cmd->draw_cs);
1676    cmd->wait_for_idle = true;
1677
1678    tu6_sysmem_render_end(cmd, &cmd->cs);
1679 }
1680
1681 static void
1682 tu_cmd_prepare_tile_store_ib(struct tu_cmd_buffer *cmd)
1683 {
1684    const uint32_t tile_store_space = 32 + 23 * cmd->state.pass->attachment_count;
1685    struct tu_cs sub_cs;
1686
1687    VkResult result =
1688       tu_cs_begin_sub_stream(&cmd->sub_cs, tile_store_space, &sub_cs);
1689    if (result != VK_SUCCESS) {
1690       cmd->record_result = result;
1691       return;
1692    }
1693
1694    /* emit to tile-store sub_cs */
1695    tu6_emit_tile_store(cmd, &sub_cs);
1696
1697    cmd->state.tile_store_ib = tu_cs_end_sub_stream(&cmd->sub_cs, &sub_cs);
1698 }
1699
1700 static void
1701 tu_cmd_update_tiling_config(struct tu_cmd_buffer *cmd,
1702                             const VkRect2D *render_area)
1703 {
1704    const struct tu_device *dev = cmd->device;
1705    struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1706
1707    tiling->render_area = *render_area;
1708    tiling->force_sysmem = force_sysmem(cmd, render_area);
1709
1710    tu_tiling_config_update_tile_layout(tiling, dev, cmd->state.pass->gmem_pixels);
1711    tu_tiling_config_update_pipe_layout(tiling, dev);
1712    tu_tiling_config_update_pipes(tiling, dev);
1713 }
1714
1715 const struct tu_dynamic_state default_dynamic_state = {
1716    .viewport =
1717      {
1718        .count = 0,
1719      },
1720    .scissor =
1721      {
1722        .count = 0,
1723      },
1724    .line_width = 1.0f,
1725    .depth_bias =
1726      {
1727        .bias = 0.0f,
1728        .clamp = 0.0f,
1729        .slope = 0.0f,
1730      },
1731    .blend_constants = { 0.0f, 0.0f, 0.0f, 0.0f },
1732    .depth_bounds =
1733      {
1734        .min = 0.0f,
1735        .max = 1.0f,
1736      },
1737    .stencil_compare_mask =
1738      {
1739        .front = ~0u,
1740        .back = ~0u,
1741      },
1742    .stencil_write_mask =
1743      {
1744        .front = ~0u,
1745        .back = ~0u,
1746      },
1747    .stencil_reference =
1748      {
1749        .front = 0u,
1750        .back = 0u,
1751      },
1752 };
1753
1754 static void UNUSED /* FINISHME */
1755 tu_bind_dynamic_state(struct tu_cmd_buffer *cmd_buffer,
1756                       const struct tu_dynamic_state *src)
1757 {
1758    struct tu_dynamic_state *dest = &cmd_buffer->state.dynamic;
1759    uint32_t copy_mask = src->mask;
1760    uint32_t dest_mask = 0;
1761
1762    tu_use_args(cmd_buffer); /* FINISHME */
1763
1764    /* Make sure to copy the number of viewports/scissors because they can
1765     * only be specified at pipeline creation time.
1766     */
1767    dest->viewport.count = src->viewport.count;
1768    dest->scissor.count = src->scissor.count;
1769    dest->discard_rectangle.count = src->discard_rectangle.count;
1770
1771    if (copy_mask & TU_DYNAMIC_VIEWPORT) {
1772       if (memcmp(&dest->viewport.viewports, &src->viewport.viewports,
1773                  src->viewport.count * sizeof(VkViewport))) {
1774          typed_memcpy(dest->viewport.viewports, src->viewport.viewports,
1775                       src->viewport.count);
1776          dest_mask |= TU_DYNAMIC_VIEWPORT;
1777       }
1778    }
1779
1780    if (copy_mask & TU_DYNAMIC_SCISSOR) {
1781       if (memcmp(&dest->scissor.scissors, &src->scissor.scissors,
1782                  src->scissor.count * sizeof(VkRect2D))) {
1783          typed_memcpy(dest->scissor.scissors, src->scissor.scissors,
1784                       src->scissor.count);
1785          dest_mask |= TU_DYNAMIC_SCISSOR;
1786       }
1787    }
1788
1789    if (copy_mask & TU_DYNAMIC_LINE_WIDTH) {
1790       if (dest->line_width != src->line_width) {
1791          dest->line_width = src->line_width;
1792          dest_mask |= TU_DYNAMIC_LINE_WIDTH;
1793       }
1794    }
1795
1796    if (copy_mask & TU_DYNAMIC_DEPTH_BIAS) {
1797       if (memcmp(&dest->depth_bias, &src->depth_bias,
1798                  sizeof(src->depth_bias))) {
1799          dest->depth_bias = src->depth_bias;
1800          dest_mask |= TU_DYNAMIC_DEPTH_BIAS;
1801       }
1802    }
1803
1804    if (copy_mask & TU_DYNAMIC_BLEND_CONSTANTS) {
1805       if (memcmp(&dest->blend_constants, &src->blend_constants,
1806                  sizeof(src->blend_constants))) {
1807          typed_memcpy(dest->blend_constants, src->blend_constants, 4);
1808          dest_mask |= TU_DYNAMIC_BLEND_CONSTANTS;
1809       }
1810    }
1811
1812    if (copy_mask & TU_DYNAMIC_DEPTH_BOUNDS) {
1813       if (memcmp(&dest->depth_bounds, &src->depth_bounds,
1814                  sizeof(src->depth_bounds))) {
1815          dest->depth_bounds = src->depth_bounds;
1816          dest_mask |= TU_DYNAMIC_DEPTH_BOUNDS;
1817       }
1818    }
1819
1820    if (copy_mask & TU_DYNAMIC_STENCIL_COMPARE_MASK) {
1821       if (memcmp(&dest->stencil_compare_mask, &src->stencil_compare_mask,
1822                  sizeof(src->stencil_compare_mask))) {
1823          dest->stencil_compare_mask = src->stencil_compare_mask;
1824          dest_mask |= TU_DYNAMIC_STENCIL_COMPARE_MASK;
1825       }
1826    }
1827
1828    if (copy_mask & TU_DYNAMIC_STENCIL_WRITE_MASK) {
1829       if (memcmp(&dest->stencil_write_mask, &src->stencil_write_mask,
1830                  sizeof(src->stencil_write_mask))) {
1831          dest->stencil_write_mask = src->stencil_write_mask;
1832          dest_mask |= TU_DYNAMIC_STENCIL_WRITE_MASK;
1833       }
1834    }
1835
1836    if (copy_mask & TU_DYNAMIC_STENCIL_REFERENCE) {
1837       if (memcmp(&dest->stencil_reference, &src->stencil_reference,
1838                  sizeof(src->stencil_reference))) {
1839          dest->stencil_reference = src->stencil_reference;
1840          dest_mask |= TU_DYNAMIC_STENCIL_REFERENCE;
1841       }
1842    }
1843
1844    if (copy_mask & TU_DYNAMIC_DISCARD_RECTANGLE) {
1845       if (memcmp(&dest->discard_rectangle.rectangles,
1846                  &src->discard_rectangle.rectangles,
1847                  src->discard_rectangle.count * sizeof(VkRect2D))) {
1848          typed_memcpy(dest->discard_rectangle.rectangles,
1849                       src->discard_rectangle.rectangles,
1850                       src->discard_rectangle.count);
1851          dest_mask |= TU_DYNAMIC_DISCARD_RECTANGLE;
1852       }
1853    }
1854 }
1855
1856 static VkResult
1857 tu_create_cmd_buffer(struct tu_device *device,
1858                      struct tu_cmd_pool *pool,
1859                      VkCommandBufferLevel level,
1860                      VkCommandBuffer *pCommandBuffer)
1861 {
1862    struct tu_cmd_buffer *cmd_buffer;
1863    cmd_buffer = vk_zalloc(&pool->alloc, sizeof(*cmd_buffer), 8,
1864                           VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1865    if (cmd_buffer == NULL)
1866       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
1867
1868    cmd_buffer->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
1869    cmd_buffer->device = device;
1870    cmd_buffer->pool = pool;
1871    cmd_buffer->level = level;
1872
1873    if (pool) {
1874       list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers);
1875       cmd_buffer->queue_family_index = pool->queue_family_index;
1876
1877    } else {
1878       /* Init the pool_link so we can safely call list_del when we destroy
1879        * the command buffer
1880        */
1881       list_inithead(&cmd_buffer->pool_link);
1882       cmd_buffer->queue_family_index = TU_QUEUE_GENERAL;
1883    }
1884
1885    tu_bo_list_init(&cmd_buffer->bo_list);
1886    tu_cs_init(&cmd_buffer->cs, device, TU_CS_MODE_GROW, 4096);
1887    tu_cs_init(&cmd_buffer->draw_cs, device, TU_CS_MODE_GROW, 4096);
1888    tu_cs_init(&cmd_buffer->draw_epilogue_cs, device, TU_CS_MODE_GROW, 4096);
1889    tu_cs_init(&cmd_buffer->sub_cs, device, TU_CS_MODE_SUB_STREAM, 2048);
1890
1891    *pCommandBuffer = tu_cmd_buffer_to_handle(cmd_buffer);
1892
1893    list_inithead(&cmd_buffer->upload.list);
1894
1895    VkResult result = tu_bo_init_new(device, &cmd_buffer->scratch_bo, 0x1000);
1896    if (result != VK_SUCCESS)
1897       goto fail_scratch_bo;
1898
1899    /* TODO: resize on overflow */
1900    cmd_buffer->vsc_data_pitch = device->vsc_data_pitch;
1901    cmd_buffer->vsc_data2_pitch = device->vsc_data2_pitch;
1902    cmd_buffer->vsc_data = device->vsc_data;
1903    cmd_buffer->vsc_data2 = device->vsc_data2;
1904
1905    return VK_SUCCESS;
1906
1907 fail_scratch_bo:
1908    list_del(&cmd_buffer->pool_link);
1909    return result;
1910 }
1911
1912 static void
1913 tu_cmd_buffer_destroy(struct tu_cmd_buffer *cmd_buffer)
1914 {
1915    tu_bo_finish(cmd_buffer->device, &cmd_buffer->scratch_bo);
1916
1917    list_del(&cmd_buffer->pool_link);
1918
1919    for (unsigned i = 0; i < VK_PIPELINE_BIND_POINT_RANGE_SIZE; i++)
1920       free(cmd_buffer->descriptors[i].push_set.set.mapped_ptr);
1921
1922    tu_cs_finish(&cmd_buffer->cs);
1923    tu_cs_finish(&cmd_buffer->draw_cs);
1924    tu_cs_finish(&cmd_buffer->draw_epilogue_cs);
1925    tu_cs_finish(&cmd_buffer->sub_cs);
1926
1927    tu_bo_list_destroy(&cmd_buffer->bo_list);
1928    vk_free(&cmd_buffer->pool->alloc, cmd_buffer);
1929 }
1930
1931 static VkResult
1932 tu_reset_cmd_buffer(struct tu_cmd_buffer *cmd_buffer)
1933 {
1934    cmd_buffer->wait_for_idle = true;
1935
1936    cmd_buffer->record_result = VK_SUCCESS;
1937
1938    tu_bo_list_reset(&cmd_buffer->bo_list);
1939    tu_cs_reset(&cmd_buffer->cs);
1940    tu_cs_reset(&cmd_buffer->draw_cs);
1941    tu_cs_reset(&cmd_buffer->draw_epilogue_cs);
1942    tu_cs_reset(&cmd_buffer->sub_cs);
1943
1944    for (unsigned i = 0; i < VK_PIPELINE_BIND_POINT_RANGE_SIZE; i++) {
1945       cmd_buffer->descriptors[i].valid = 0;
1946       cmd_buffer->descriptors[i].push_dirty = false;
1947    }
1948
1949    cmd_buffer->status = TU_CMD_BUFFER_STATUS_INITIAL;
1950
1951    return cmd_buffer->record_result;
1952 }
1953
1954 VkResult
1955 tu_AllocateCommandBuffers(VkDevice _device,
1956                           const VkCommandBufferAllocateInfo *pAllocateInfo,
1957                           VkCommandBuffer *pCommandBuffers)
1958 {
1959    TU_FROM_HANDLE(tu_device, device, _device);
1960    TU_FROM_HANDLE(tu_cmd_pool, pool, pAllocateInfo->commandPool);
1961
1962    VkResult result = VK_SUCCESS;
1963    uint32_t i;
1964
1965    for (i = 0; i < pAllocateInfo->commandBufferCount; i++) {
1966
1967       if (!list_is_empty(&pool->free_cmd_buffers)) {
1968          struct tu_cmd_buffer *cmd_buffer = list_first_entry(
1969             &pool->free_cmd_buffers, struct tu_cmd_buffer, pool_link);
1970
1971          list_del(&cmd_buffer->pool_link);
1972          list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers);
1973
1974          result = tu_reset_cmd_buffer(cmd_buffer);
1975          cmd_buffer->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
1976          cmd_buffer->level = pAllocateInfo->level;
1977
1978          pCommandBuffers[i] = tu_cmd_buffer_to_handle(cmd_buffer);
1979       } else {
1980          result = tu_create_cmd_buffer(device, pool, pAllocateInfo->level,
1981                                        &pCommandBuffers[i]);
1982       }
1983       if (result != VK_SUCCESS)
1984          break;
1985    }
1986
1987    if (result != VK_SUCCESS) {
1988       tu_FreeCommandBuffers(_device, pAllocateInfo->commandPool, i,
1989                             pCommandBuffers);
1990
1991       /* From the Vulkan 1.0.66 spec:
1992        *
1993        * "vkAllocateCommandBuffers can be used to create multiple
1994        *  command buffers. If the creation of any of those command
1995        *  buffers fails, the implementation must destroy all
1996        *  successfully created command buffer objects from this
1997        *  command, set all entries of the pCommandBuffers array to
1998        *  NULL and return the error."
1999        */
2000       memset(pCommandBuffers, 0,
2001              sizeof(*pCommandBuffers) * pAllocateInfo->commandBufferCount);
2002    }
2003
2004    return result;
2005 }
2006
2007 void
2008 tu_FreeCommandBuffers(VkDevice device,
2009                       VkCommandPool commandPool,
2010                       uint32_t commandBufferCount,
2011                       const VkCommandBuffer *pCommandBuffers)
2012 {
2013    for (uint32_t i = 0; i < commandBufferCount; i++) {
2014       TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, pCommandBuffers[i]);
2015
2016       if (cmd_buffer) {
2017          if (cmd_buffer->pool) {
2018             list_del(&cmd_buffer->pool_link);
2019             list_addtail(&cmd_buffer->pool_link,
2020                          &cmd_buffer->pool->free_cmd_buffers);
2021          } else
2022             tu_cmd_buffer_destroy(cmd_buffer);
2023       }
2024    }
2025 }
2026
2027 VkResult
2028 tu_ResetCommandBuffer(VkCommandBuffer commandBuffer,
2029                       VkCommandBufferResetFlags flags)
2030 {
2031    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
2032    return tu_reset_cmd_buffer(cmd_buffer);
2033 }
2034
2035 VkResult
2036 tu_BeginCommandBuffer(VkCommandBuffer commandBuffer,
2037                       const VkCommandBufferBeginInfo *pBeginInfo)
2038 {
2039    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
2040    VkResult result = VK_SUCCESS;
2041
2042    if (cmd_buffer->status != TU_CMD_BUFFER_STATUS_INITIAL) {
2043       /* If the command buffer has already been resetted with
2044        * vkResetCommandBuffer, no need to do it again.
2045        */
2046       result = tu_reset_cmd_buffer(cmd_buffer);
2047       if (result != VK_SUCCESS)
2048          return result;
2049    }
2050
2051    memset(&cmd_buffer->state, 0, sizeof(cmd_buffer->state));
2052    cmd_buffer->usage_flags = pBeginInfo->flags;
2053
2054    tu_cs_begin(&cmd_buffer->cs);
2055    tu_cs_begin(&cmd_buffer->draw_cs);
2056    tu_cs_begin(&cmd_buffer->draw_epilogue_cs);
2057
2058    cmd_buffer->scratch_seqno = 0;
2059
2060    /* setup initial configuration into command buffer */
2061    if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY) {
2062       switch (cmd_buffer->queue_family_index) {
2063       case TU_QUEUE_GENERAL:
2064          tu6_init_hw(cmd_buffer, &cmd_buffer->cs);
2065          break;
2066       default:
2067          break;
2068       }
2069    } else if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY &&
2070               (pBeginInfo->flags & VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT)) {
2071       assert(pBeginInfo->pInheritanceInfo);
2072       cmd_buffer->state.pass = tu_render_pass_from_handle(pBeginInfo->pInheritanceInfo->renderPass);
2073       cmd_buffer->state.subpass = &cmd_buffer->state.pass->subpasses[pBeginInfo->pInheritanceInfo->subpass];
2074    }
2075
2076    cmd_buffer->status = TU_CMD_BUFFER_STATUS_RECORDING;
2077
2078    return VK_SUCCESS;
2079 }
2080
2081 void
2082 tu_CmdBindVertexBuffers(VkCommandBuffer commandBuffer,
2083                         uint32_t firstBinding,
2084                         uint32_t bindingCount,
2085                         const VkBuffer *pBuffers,
2086                         const VkDeviceSize *pOffsets)
2087 {
2088    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2089
2090    assert(firstBinding + bindingCount <= MAX_VBS);
2091
2092    for (uint32_t i = 0; i < bindingCount; i++) {
2093       cmd->state.vb.buffers[firstBinding + i] =
2094          tu_buffer_from_handle(pBuffers[i]);
2095       cmd->state.vb.offsets[firstBinding + i] = pOffsets[i];
2096    }
2097
2098    /* VB states depend on VkPipelineVertexInputStateCreateInfo */
2099    cmd->state.dirty |= TU_CMD_DIRTY_VERTEX_BUFFERS;
2100 }
2101
2102 void
2103 tu_CmdBindIndexBuffer(VkCommandBuffer commandBuffer,
2104                       VkBuffer buffer,
2105                       VkDeviceSize offset,
2106                       VkIndexType indexType)
2107 {
2108    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2109    TU_FROM_HANDLE(tu_buffer, buf, buffer);
2110
2111    /* initialize/update the restart index */
2112    if (!cmd->state.index_buffer || cmd->state.index_type != indexType) {
2113       struct tu_cs *draw_cs = &cmd->draw_cs;
2114
2115       tu6_emit_restart_index(
2116          draw_cs, indexType == VK_INDEX_TYPE_UINT32 ? 0xffffffff : 0xffff);
2117
2118       tu_cs_sanity_check(draw_cs);
2119    }
2120
2121    /* track the BO */
2122    if (cmd->state.index_buffer != buf)
2123       tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
2124
2125    cmd->state.index_buffer = buf;
2126    cmd->state.index_offset = offset;
2127    cmd->state.index_type = indexType;
2128 }
2129
2130 void
2131 tu_CmdBindDescriptorSets(VkCommandBuffer commandBuffer,
2132                          VkPipelineBindPoint pipelineBindPoint,
2133                          VkPipelineLayout _layout,
2134                          uint32_t firstSet,
2135                          uint32_t descriptorSetCount,
2136                          const VkDescriptorSet *pDescriptorSets,
2137                          uint32_t dynamicOffsetCount,
2138                          const uint32_t *pDynamicOffsets)
2139 {
2140    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
2141    TU_FROM_HANDLE(tu_pipeline_layout, layout, _layout);
2142    unsigned dyn_idx = 0;
2143
2144    struct tu_descriptor_state *descriptors_state =
2145       tu_get_descriptors_state(cmd_buffer, pipelineBindPoint);
2146
2147    for (unsigned i = 0; i < descriptorSetCount; ++i) {
2148       unsigned idx = i + firstSet;
2149       TU_FROM_HANDLE(tu_descriptor_set, set, pDescriptorSets[i]);
2150
2151       descriptors_state->sets[idx] = set;
2152       descriptors_state->valid |= (1u << idx);
2153
2154       for(unsigned j = 0; j < set->layout->dynamic_offset_count; ++j, ++dyn_idx) {
2155          unsigned idx = j + layout->set[i + firstSet].dynamic_offset_start;
2156          assert(dyn_idx < dynamicOffsetCount);
2157
2158          descriptors_state->dynamic_buffers[idx] =
2159          set->dynamic_descriptors[j].va + pDynamicOffsets[dyn_idx];
2160       }
2161    }
2162
2163    cmd_buffer->state.dirty |= TU_CMD_DIRTY_DESCRIPTOR_SETS;
2164 }
2165
2166 void
2167 tu_CmdPushConstants(VkCommandBuffer commandBuffer,
2168                     VkPipelineLayout layout,
2169                     VkShaderStageFlags stageFlags,
2170                     uint32_t offset,
2171                     uint32_t size,
2172                     const void *pValues)
2173 {
2174    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2175    memcpy((void*) cmd->push_constants + offset, pValues, size);
2176    cmd->state.dirty |= TU_CMD_DIRTY_PUSH_CONSTANTS;
2177 }
2178
2179 VkResult
2180 tu_EndCommandBuffer(VkCommandBuffer commandBuffer)
2181 {
2182    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
2183
2184    if (cmd_buffer->scratch_seqno) {
2185       tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->scratch_bo,
2186                      MSM_SUBMIT_BO_WRITE);
2187    }
2188
2189    if (cmd_buffer->use_vsc_data) {
2190       tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->vsc_data,
2191                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
2192       tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->vsc_data2,
2193                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
2194    }
2195
2196    for (uint32_t i = 0; i < cmd_buffer->draw_cs.bo_count; i++) {
2197       tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->draw_cs.bos[i],
2198                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2199    }
2200
2201    for (uint32_t i = 0; i < cmd_buffer->draw_epilogue_cs.bo_count; i++) {
2202       tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->draw_epilogue_cs.bos[i],
2203                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2204    }
2205
2206    for (uint32_t i = 0; i < cmd_buffer->sub_cs.bo_count; i++) {
2207       tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->sub_cs.bos[i],
2208                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2209    }
2210
2211    tu_cs_end(&cmd_buffer->cs);
2212    tu_cs_end(&cmd_buffer->draw_cs);
2213    tu_cs_end(&cmd_buffer->draw_epilogue_cs);
2214
2215    cmd_buffer->status = TU_CMD_BUFFER_STATUS_EXECUTABLE;
2216
2217    return cmd_buffer->record_result;
2218 }
2219
2220 void
2221 tu_CmdBindPipeline(VkCommandBuffer commandBuffer,
2222                    VkPipelineBindPoint pipelineBindPoint,
2223                    VkPipeline _pipeline)
2224 {
2225    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2226    TU_FROM_HANDLE(tu_pipeline, pipeline, _pipeline);
2227
2228    switch (pipelineBindPoint) {
2229    case VK_PIPELINE_BIND_POINT_GRAPHICS:
2230       cmd->state.pipeline = pipeline;
2231       cmd->state.dirty |= TU_CMD_DIRTY_PIPELINE;
2232       break;
2233    case VK_PIPELINE_BIND_POINT_COMPUTE:
2234       cmd->state.compute_pipeline = pipeline;
2235       cmd->state.dirty |= TU_CMD_DIRTY_COMPUTE_PIPELINE;
2236       break;
2237    default:
2238       unreachable("unrecognized pipeline bind point");
2239       break;
2240    }
2241
2242    tu_bo_list_add(&cmd->bo_list, &pipeline->program.binary_bo,
2243                   MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2244    for (uint32_t i = 0; i < pipeline->cs.bo_count; i++) {
2245       tu_bo_list_add(&cmd->bo_list, pipeline->cs.bos[i],
2246                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2247    }
2248 }
2249
2250 void
2251 tu_CmdSetViewport(VkCommandBuffer commandBuffer,
2252                   uint32_t firstViewport,
2253                   uint32_t viewportCount,
2254                   const VkViewport *pViewports)
2255 {
2256    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2257    struct tu_cs *draw_cs = &cmd->draw_cs;
2258
2259    assert(firstViewport == 0 && viewportCount == 1);
2260    tu6_emit_viewport(draw_cs, pViewports);
2261
2262    tu_cs_sanity_check(draw_cs);
2263 }
2264
2265 void
2266 tu_CmdSetScissor(VkCommandBuffer commandBuffer,
2267                  uint32_t firstScissor,
2268                  uint32_t scissorCount,
2269                  const VkRect2D *pScissors)
2270 {
2271    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2272    struct tu_cs *draw_cs = &cmd->draw_cs;
2273
2274    assert(firstScissor == 0 && scissorCount == 1);
2275    tu6_emit_scissor(draw_cs, pScissors);
2276
2277    tu_cs_sanity_check(draw_cs);
2278 }
2279
2280 void
2281 tu_CmdSetLineWidth(VkCommandBuffer commandBuffer, float lineWidth)
2282 {
2283    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2284
2285    cmd->state.dynamic.line_width = lineWidth;
2286
2287    /* line width depends on VkPipelineRasterizationStateCreateInfo */
2288    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_LINE_WIDTH;
2289 }
2290
2291 void
2292 tu_CmdSetDepthBias(VkCommandBuffer commandBuffer,
2293                    float depthBiasConstantFactor,
2294                    float depthBiasClamp,
2295                    float depthBiasSlopeFactor)
2296 {
2297    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2298    struct tu_cs *draw_cs = &cmd->draw_cs;
2299
2300    tu6_emit_depth_bias(draw_cs, depthBiasConstantFactor, depthBiasClamp,
2301                        depthBiasSlopeFactor);
2302
2303    tu_cs_sanity_check(draw_cs);
2304 }
2305
2306 void
2307 tu_CmdSetBlendConstants(VkCommandBuffer commandBuffer,
2308                         const float blendConstants[4])
2309 {
2310    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2311    struct tu_cs *draw_cs = &cmd->draw_cs;
2312
2313    tu6_emit_blend_constants(draw_cs, blendConstants);
2314
2315    tu_cs_sanity_check(draw_cs);
2316 }
2317
2318 void
2319 tu_CmdSetDepthBounds(VkCommandBuffer commandBuffer,
2320                      float minDepthBounds,
2321                      float maxDepthBounds)
2322 {
2323 }
2324
2325 void
2326 tu_CmdSetStencilCompareMask(VkCommandBuffer commandBuffer,
2327                             VkStencilFaceFlags faceMask,
2328                             uint32_t compareMask)
2329 {
2330    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2331
2332    if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
2333       cmd->state.dynamic.stencil_compare_mask.front = compareMask;
2334    if (faceMask & VK_STENCIL_FACE_BACK_BIT)
2335       cmd->state.dynamic.stencil_compare_mask.back = compareMask;
2336
2337    /* the front/back compare masks must be updated together */
2338    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK;
2339 }
2340
2341 void
2342 tu_CmdSetStencilWriteMask(VkCommandBuffer commandBuffer,
2343                           VkStencilFaceFlags faceMask,
2344                           uint32_t writeMask)
2345 {
2346    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2347
2348    if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
2349       cmd->state.dynamic.stencil_write_mask.front = writeMask;
2350    if (faceMask & VK_STENCIL_FACE_BACK_BIT)
2351       cmd->state.dynamic.stencil_write_mask.back = writeMask;
2352
2353    /* the front/back write masks must be updated together */
2354    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK;
2355 }
2356
2357 void
2358 tu_CmdSetStencilReference(VkCommandBuffer commandBuffer,
2359                           VkStencilFaceFlags faceMask,
2360                           uint32_t reference)
2361 {
2362    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2363
2364    if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
2365       cmd->state.dynamic.stencil_reference.front = reference;
2366    if (faceMask & VK_STENCIL_FACE_BACK_BIT)
2367       cmd->state.dynamic.stencil_reference.back = reference;
2368
2369    /* the front/back references must be updated together */
2370    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE;
2371 }
2372
2373 void
2374 tu_CmdExecuteCommands(VkCommandBuffer commandBuffer,
2375                       uint32_t commandBufferCount,
2376                       const VkCommandBuffer *pCmdBuffers)
2377 {
2378    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2379    VkResult result;
2380
2381    assert(commandBufferCount > 0);
2382
2383    for (uint32_t i = 0; i < commandBufferCount; i++) {
2384       TU_FROM_HANDLE(tu_cmd_buffer, secondary, pCmdBuffers[i]);
2385
2386       result = tu_bo_list_merge(&cmd->bo_list, &secondary->bo_list);
2387       if (result != VK_SUCCESS) {
2388          cmd->record_result = result;
2389          break;
2390       }
2391
2392       result = tu_cs_add_entries(&cmd->draw_cs, &secondary->draw_cs);
2393       if (result != VK_SUCCESS) {
2394          cmd->record_result = result;
2395          break;
2396       }
2397
2398       result = tu_cs_add_entries(&cmd->draw_epilogue_cs,
2399             &secondary->draw_epilogue_cs);
2400       if (result != VK_SUCCESS) {
2401          cmd->record_result = result;
2402          break;
2403       }
2404    }
2405    cmd->state.dirty = ~0u; /* TODO: set dirty only what needs to be */
2406 }
2407
2408 VkResult
2409 tu_CreateCommandPool(VkDevice _device,
2410                      const VkCommandPoolCreateInfo *pCreateInfo,
2411                      const VkAllocationCallbacks *pAllocator,
2412                      VkCommandPool *pCmdPool)
2413 {
2414    TU_FROM_HANDLE(tu_device, device, _device);
2415    struct tu_cmd_pool *pool;
2416
2417    pool = vk_alloc2(&device->alloc, pAllocator, sizeof(*pool), 8,
2418                     VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2419    if (pool == NULL)
2420       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
2421
2422    if (pAllocator)
2423       pool->alloc = *pAllocator;
2424    else
2425       pool->alloc = device->alloc;
2426
2427    list_inithead(&pool->cmd_buffers);
2428    list_inithead(&pool->free_cmd_buffers);
2429
2430    pool->queue_family_index = pCreateInfo->queueFamilyIndex;
2431
2432    *pCmdPool = tu_cmd_pool_to_handle(pool);
2433
2434    return VK_SUCCESS;
2435 }
2436
2437 void
2438 tu_DestroyCommandPool(VkDevice _device,
2439                       VkCommandPool commandPool,
2440                       const VkAllocationCallbacks *pAllocator)
2441 {
2442    TU_FROM_HANDLE(tu_device, device, _device);
2443    TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2444
2445    if (!pool)
2446       return;
2447
2448    list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2449                             &pool->cmd_buffers, pool_link)
2450    {
2451       tu_cmd_buffer_destroy(cmd_buffer);
2452    }
2453
2454    list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2455                             &pool->free_cmd_buffers, pool_link)
2456    {
2457       tu_cmd_buffer_destroy(cmd_buffer);
2458    }
2459
2460    vk_free2(&device->alloc, pAllocator, pool);
2461 }
2462
2463 VkResult
2464 tu_ResetCommandPool(VkDevice device,
2465                     VkCommandPool commandPool,
2466                     VkCommandPoolResetFlags flags)
2467 {
2468    TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2469    VkResult result;
2470
2471    list_for_each_entry(struct tu_cmd_buffer, cmd_buffer, &pool->cmd_buffers,
2472                        pool_link)
2473    {
2474       result = tu_reset_cmd_buffer(cmd_buffer);
2475       if (result != VK_SUCCESS)
2476          return result;
2477    }
2478
2479    return VK_SUCCESS;
2480 }
2481
2482 void
2483 tu_TrimCommandPool(VkDevice device,
2484                    VkCommandPool commandPool,
2485                    VkCommandPoolTrimFlags flags)
2486 {
2487    TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2488
2489    if (!pool)
2490       return;
2491
2492    list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2493                             &pool->free_cmd_buffers, pool_link)
2494    {
2495       tu_cmd_buffer_destroy(cmd_buffer);
2496    }
2497 }
2498
2499 void
2500 tu_CmdBeginRenderPass(VkCommandBuffer commandBuffer,
2501                       const VkRenderPassBeginInfo *pRenderPassBegin,
2502                       VkSubpassContents contents)
2503 {
2504    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2505    TU_FROM_HANDLE(tu_render_pass, pass, pRenderPassBegin->renderPass);
2506    TU_FROM_HANDLE(tu_framebuffer, fb, pRenderPassBegin->framebuffer);
2507
2508    cmd->state.pass = pass;
2509    cmd->state.subpass = pass->subpasses;
2510    cmd->state.framebuffer = fb;
2511
2512    tu_cmd_update_tiling_config(cmd, &pRenderPassBegin->renderArea);
2513    tu_cmd_prepare_tile_store_ib(cmd);
2514
2515    tu_emit_load_clear(cmd, pRenderPassBegin);
2516
2517    tu6_emit_zs(cmd, cmd->state.subpass, &cmd->draw_cs);
2518    tu6_emit_mrt(cmd, cmd->state.subpass, &cmd->draw_cs);
2519    tu6_emit_msaa(cmd, cmd->state.subpass, &cmd->draw_cs);
2520    tu6_emit_render_cntl(cmd, cmd->state.subpass, &cmd->draw_cs, false);
2521
2522    /* note: use_hw_binning only checks tiling config */
2523    if (use_hw_binning(cmd))
2524       cmd->use_vsc_data = true;
2525
2526    for (uint32_t i = 0; i < fb->attachment_count; ++i) {
2527       const struct tu_image_view *iview = fb->attachments[i].attachment;
2528       tu_bo_list_add(&cmd->bo_list, iview->image->bo,
2529                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
2530    }
2531 }
2532
2533 void
2534 tu_CmdBeginRenderPass2(VkCommandBuffer commandBuffer,
2535                        const VkRenderPassBeginInfo *pRenderPassBeginInfo,
2536                        const VkSubpassBeginInfoKHR *pSubpassBeginInfo)
2537 {
2538    tu_CmdBeginRenderPass(commandBuffer, pRenderPassBeginInfo,
2539                          pSubpassBeginInfo->contents);
2540 }
2541
2542 void
2543 tu_CmdNextSubpass(VkCommandBuffer commandBuffer, VkSubpassContents contents)
2544 {
2545    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2546    const struct tu_render_pass *pass = cmd->state.pass;
2547    struct tu_cs *cs = &cmd->draw_cs;
2548
2549    const struct tu_subpass *subpass = cmd->state.subpass++;
2550    /* TODO:
2551     * if msaa samples change between subpasses,
2552     * attachment store is broken for some attachments
2553     */
2554    if (subpass->resolve_attachments) {
2555       tu6_emit_blit_scissor(cmd, cs, true);
2556       for (unsigned i = 0; i < subpass->color_count; i++) {
2557          uint32_t a = subpass->resolve_attachments[i].attachment;
2558          if (a != VK_ATTACHMENT_UNUSED) {
2559             tu6_emit_resolve(cmd, cs, a,
2560                              subpass->color_attachments[i].attachment);
2561          }
2562       }
2563    }
2564
2565    /* invalidate because reading input attachments will cache GMEM and
2566     * the cache isn''t updated when GMEM is written
2567     * TODO: is there a no-cache bit for textures?
2568     */
2569    if (cmd->state.subpass->input_count)
2570       tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE, false);
2571
2572    /* emit mrt/zs/msaa/ubwc state for the subpass that is starting */
2573    tu6_emit_zs(cmd, cmd->state.subpass, cs);
2574    tu6_emit_mrt(cmd, cmd->state.subpass, cs);
2575    tu6_emit_msaa(cmd, cmd->state.subpass, cs);
2576    tu6_emit_render_cntl(cmd, cmd->state.subpass, cs, false);
2577
2578    /* Emit flushes so that input attachments will read the correct value. This
2579     * is for sysmem only, although it shouldn't do much harm on gmem.
2580     */
2581    tu6_emit_event_write(cmd, cs, PC_CCU_FLUSH_COLOR_TS, true);
2582    tu6_emit_event_write(cmd, cs, PC_CCU_FLUSH_DEPTH_TS, true);
2583
2584    /* TODO:
2585     * since we don't know how to do GMEM->GMEM resolve,
2586     * resolve attachments are resolved to memory then loaded to GMEM again if needed
2587     */
2588    if (subpass->resolve_attachments) {
2589       for (unsigned i = 0; i < subpass->color_count; i++) {
2590          uint32_t a = subpass->resolve_attachments[i].attachment;
2591          if (a != VK_ATTACHMENT_UNUSED && pass->attachments[a].gmem_offset >= 0) {
2592                tu_finishme("missing GMEM->GMEM resolve, performance will suffer\n");
2593                tu6_emit_predicated_blit(cmd, cs, a, a, false);
2594          }
2595       }
2596    }
2597 }
2598
2599 void
2600 tu_CmdNextSubpass2(VkCommandBuffer commandBuffer,
2601                    const VkSubpassBeginInfoKHR *pSubpassBeginInfo,
2602                    const VkSubpassEndInfoKHR *pSubpassEndInfo)
2603 {
2604    tu_CmdNextSubpass(commandBuffer, pSubpassBeginInfo->contents);
2605 }
2606
2607 struct tu_draw_info
2608 {
2609    /**
2610     * Number of vertices.
2611     */
2612    uint32_t count;
2613
2614    /**
2615     * Index of the first vertex.
2616     */
2617    int32_t vertex_offset;
2618
2619    /**
2620     * First instance id.
2621     */
2622    uint32_t first_instance;
2623
2624    /**
2625     * Number of instances.
2626     */
2627    uint32_t instance_count;
2628
2629    /**
2630     * First index (indexed draws only).
2631     */
2632    uint32_t first_index;
2633
2634    /**
2635     * Whether it's an indexed draw.
2636     */
2637    bool indexed;
2638
2639    /**
2640     * Indirect draw parameters resource.
2641     */
2642    struct tu_buffer *indirect;
2643    uint64_t indirect_offset;
2644    uint32_t stride;
2645
2646    /**
2647     * Draw count parameters resource.
2648     */
2649    struct tu_buffer *count_buffer;
2650    uint64_t count_buffer_offset;
2651 };
2652
2653 #define ENABLE_ALL (CP_SET_DRAW_STATE__0_BINNING | CP_SET_DRAW_STATE__0_GMEM | CP_SET_DRAW_STATE__0_SYSMEM)
2654 #define ENABLE_DRAW (CP_SET_DRAW_STATE__0_GMEM | CP_SET_DRAW_STATE__0_SYSMEM)
2655
2656 enum tu_draw_state_group_id
2657 {
2658    TU_DRAW_STATE_PROGRAM,
2659    TU_DRAW_STATE_PROGRAM_BINNING,
2660    TU_DRAW_STATE_VI,
2661    TU_DRAW_STATE_VI_BINNING,
2662    TU_DRAW_STATE_VP,
2663    TU_DRAW_STATE_RAST,
2664    TU_DRAW_STATE_DS,
2665    TU_DRAW_STATE_BLEND,
2666    TU_DRAW_STATE_VS_CONST,
2667    TU_DRAW_STATE_FS_CONST,
2668    TU_DRAW_STATE_VS_TEX,
2669    TU_DRAW_STATE_FS_TEX_SYSMEM,
2670    TU_DRAW_STATE_FS_TEX_GMEM,
2671    TU_DRAW_STATE_FS_IBO,
2672    TU_DRAW_STATE_VS_PARAMS,
2673
2674    TU_DRAW_STATE_COUNT,
2675 };
2676
2677 struct tu_draw_state_group
2678 {
2679    enum tu_draw_state_group_id id;
2680    uint32_t enable_mask;
2681    struct tu_cs_entry ib;
2682 };
2683
2684 const static struct tu_sampler*
2685 sampler_ptr(struct tu_descriptor_state *descriptors_state,
2686             const struct tu_descriptor_map *map, unsigned i,
2687             unsigned array_index)
2688 {
2689    assert(descriptors_state->valid & (1 << map->set[i]));
2690
2691    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2692    assert(map->binding[i] < set->layout->binding_count);
2693
2694    const struct tu_descriptor_set_binding_layout *layout =
2695       &set->layout->binding[map->binding[i]];
2696
2697    if (layout->immutable_samplers_offset) {
2698       const struct tu_sampler *immutable_samplers =
2699          tu_immutable_samplers(set->layout, layout);
2700
2701       return &immutable_samplers[array_index];
2702    }
2703
2704    switch (layout->type) {
2705    case VK_DESCRIPTOR_TYPE_SAMPLER:
2706       return (struct tu_sampler*) &set->mapped_ptr[layout->offset / 4];
2707    case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
2708       return (struct tu_sampler*) &set->mapped_ptr[layout->offset / 4 + A6XX_TEX_CONST_DWORDS +
2709                                                    array_index *
2710                                                    (A6XX_TEX_CONST_DWORDS +
2711                                                     sizeof(struct tu_sampler) / 4)];
2712    default:
2713       unreachable("unimplemented descriptor type");
2714       break;
2715    }
2716 }
2717
2718 static void
2719 write_tex_const(struct tu_cmd_buffer *cmd,
2720                 uint32_t *dst,
2721                 struct tu_descriptor_state *descriptors_state,
2722                 const struct tu_descriptor_map *map,
2723                 unsigned i, unsigned array_index, bool is_sysmem)
2724 {
2725    assert(descriptors_state->valid & (1 << map->set[i]));
2726
2727    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2728    assert(map->binding[i] < set->layout->binding_count);
2729
2730    const struct tu_descriptor_set_binding_layout *layout =
2731       &set->layout->binding[map->binding[i]];
2732
2733    switch (layout->type) {
2734    case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
2735    case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
2736    case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
2737    case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
2738       memcpy(dst, &set->mapped_ptr[layout->offset / 4 +
2739                                    array_index * A6XX_TEX_CONST_DWORDS],
2740              A6XX_TEX_CONST_DWORDS * 4);
2741       break;
2742    case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
2743       memcpy(dst, &set->mapped_ptr[layout->offset / 4 +
2744                                    array_index *
2745                                    (A6XX_TEX_CONST_DWORDS +
2746                                     sizeof(struct tu_sampler) / 4)],
2747              A6XX_TEX_CONST_DWORDS * 4);
2748       break;
2749    default:
2750       unreachable("unimplemented descriptor type");
2751       break;
2752    }
2753
2754    if (layout->type == VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT && !is_sysmem) {
2755       const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
2756       uint32_t a = cmd->state.subpass->input_attachments[map->value[i] +
2757                                                          array_index].attachment;
2758       const struct tu_render_pass_attachment *att = &cmd->state.pass->attachments[a];
2759
2760       assert(att->gmem_offset >= 0);
2761
2762       dst[0] &= ~(A6XX_TEX_CONST_0_SWAP__MASK | A6XX_TEX_CONST_0_TILE_MODE__MASK);
2763       dst[0] |= A6XX_TEX_CONST_0_TILE_MODE(TILE6_2);
2764       dst[2] &= ~(A6XX_TEX_CONST_2_TYPE__MASK | A6XX_TEX_CONST_2_PITCH__MASK);
2765       dst[2] |=
2766          A6XX_TEX_CONST_2_TYPE(A6XX_TEX_2D) |
2767          A6XX_TEX_CONST_2_PITCH(tiling->tile0.extent.width * att->cpp);
2768       dst[3] = 0;
2769       dst[4] = 0x100000 + att->gmem_offset;
2770       dst[5] = A6XX_TEX_CONST_5_DEPTH(1);
2771       for (unsigned i = 6; i < A6XX_TEX_CONST_DWORDS; i++)
2772          dst[i] = 0;
2773
2774       if (cmd->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY)
2775          tu_finishme("patch input attachment pitch for secondary cmd buffer");
2776    }
2777 }
2778
2779 static void
2780 write_image_ibo(struct tu_cmd_buffer *cmd,
2781                 uint32_t *dst,
2782                 struct tu_descriptor_state *descriptors_state,
2783                 const struct tu_descriptor_map *map,
2784                 unsigned i, unsigned array_index)
2785 {
2786    assert(descriptors_state->valid & (1 << map->set[i]));
2787
2788    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2789    assert(map->binding[i] < set->layout->binding_count);
2790
2791    const struct tu_descriptor_set_binding_layout *layout =
2792       &set->layout->binding[map->binding[i]];
2793
2794    assert(layout->type == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE);
2795
2796    memcpy(dst, &set->mapped_ptr[layout->offset / 4 +
2797                                 (array_index * 2 + 1) * A6XX_TEX_CONST_DWORDS],
2798           A6XX_TEX_CONST_DWORDS * 4);
2799 }
2800
2801 static uint64_t
2802 buffer_ptr(struct tu_descriptor_state *descriptors_state,
2803            const struct tu_descriptor_map *map,
2804            unsigned i, unsigned array_index)
2805 {
2806    assert(descriptors_state->valid & (1 << map->set[i]));
2807
2808    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2809    assert(map->binding[i] < set->layout->binding_count);
2810
2811    const struct tu_descriptor_set_binding_layout *layout =
2812       &set->layout->binding[map->binding[i]];
2813
2814    switch (layout->type) {
2815    case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
2816    case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
2817       return descriptors_state->dynamic_buffers[layout->dynamic_offset_offset +
2818                                                 array_index];
2819    case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
2820    case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
2821       return (uint64_t) set->mapped_ptr[layout->offset / 4 + array_index * 2 + 1] << 32 |
2822                         set->mapped_ptr[layout->offset / 4 + array_index * 2];
2823    default:
2824       unreachable("unimplemented descriptor type");
2825       break;
2826    }
2827 }
2828
2829 static inline uint32_t
2830 tu6_stage2opcode(gl_shader_stage type)
2831 {
2832    switch (type) {
2833    case MESA_SHADER_VERTEX:
2834    case MESA_SHADER_TESS_CTRL:
2835    case MESA_SHADER_TESS_EVAL:
2836    case MESA_SHADER_GEOMETRY:
2837       return CP_LOAD_STATE6_GEOM;
2838    case MESA_SHADER_FRAGMENT:
2839    case MESA_SHADER_COMPUTE:
2840    case MESA_SHADER_KERNEL:
2841       return CP_LOAD_STATE6_FRAG;
2842    default:
2843       unreachable("bad shader type");
2844    }
2845 }
2846
2847 static inline enum a6xx_state_block
2848 tu6_stage2shadersb(gl_shader_stage type)
2849 {
2850    switch (type) {
2851    case MESA_SHADER_VERTEX:
2852       return SB6_VS_SHADER;
2853    case MESA_SHADER_FRAGMENT:
2854       return SB6_FS_SHADER;
2855    case MESA_SHADER_COMPUTE:
2856    case MESA_SHADER_KERNEL:
2857       return SB6_CS_SHADER;
2858    default:
2859       unreachable("bad shader type");
2860       return ~0;
2861    }
2862 }
2863
2864 static void
2865 tu6_emit_user_consts(struct tu_cs *cs, const struct tu_pipeline *pipeline,
2866                      struct tu_descriptor_state *descriptors_state,
2867                      gl_shader_stage type,
2868                      uint32_t *push_constants)
2869 {
2870    const struct tu_program_descriptor_linkage *link =
2871       &pipeline->program.link[type];
2872    const struct ir3_ubo_analysis_state *state = &link->ubo_state;
2873
2874    for (uint32_t i = 0; i < ARRAY_SIZE(state->range); i++) {
2875       if (state->range[i].start < state->range[i].end) {
2876          uint32_t size = state->range[i].end - state->range[i].start;
2877          uint32_t offset = state->range[i].start;
2878
2879          /* and even if the start of the const buffer is before
2880           * first_immediate, the end may not be:
2881           */
2882          size = MIN2(size, (16 * link->constlen) - state->range[i].offset);
2883
2884          if (size == 0)
2885             continue;
2886
2887          /* things should be aligned to vec4: */
2888          debug_assert((state->range[i].offset % 16) == 0);
2889          debug_assert((size % 16) == 0);
2890          debug_assert((offset % 16) == 0);
2891
2892          if (i == 0) {
2893             /* push constants */
2894             tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + (size / 4));
2895             tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(state->range[i].offset / 16) |
2896                   CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2897                   CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
2898                   CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2899                   CP_LOAD_STATE6_0_NUM_UNIT(size / 16));
2900             tu_cs_emit(cs, 0);
2901             tu_cs_emit(cs, 0);
2902             for (unsigned i = 0; i < size / 4; i++)
2903                tu_cs_emit(cs, push_constants[i + offset / 4]);
2904             continue;
2905          }
2906
2907          /* Look through the UBO map to find our UBO index, and get the VA for
2908           * that UBO.
2909           */
2910          uint64_t va = 0;
2911          uint32_t ubo_idx = i - 1;
2912          uint32_t ubo_map_base = 0;
2913          for (int j = 0; j < link->ubo_map.num; j++) {
2914             if (ubo_idx >= ubo_map_base &&
2915                 ubo_idx < ubo_map_base + link->ubo_map.array_size[j]) {
2916                va = buffer_ptr(descriptors_state, &link->ubo_map, j,
2917                                ubo_idx - ubo_map_base);
2918                break;
2919             }
2920             ubo_map_base += link->ubo_map.array_size[j];
2921          }
2922          assert(va);
2923
2924          tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3);
2925          tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(state->range[i].offset / 16) |
2926                CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2927                CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
2928                CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2929                CP_LOAD_STATE6_0_NUM_UNIT(size / 16));
2930          tu_cs_emit_qw(cs, va + offset);
2931       }
2932    }
2933 }
2934
2935 static void
2936 tu6_emit_ubos(struct tu_cs *cs, const struct tu_pipeline *pipeline,
2937               struct tu_descriptor_state *descriptors_state,
2938               gl_shader_stage type)
2939 {
2940    const struct tu_program_descriptor_linkage *link =
2941       &pipeline->program.link[type];
2942
2943    uint32_t num = MIN2(link->ubo_map.num_desc, link->const_state.num_ubos);
2944    uint32_t anum = align(num, 2);
2945
2946    if (!num)
2947       return;
2948
2949    tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + (2 * anum));
2950    tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(link->const_state.offsets.ubo) |
2951          CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2952          CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
2953          CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2954          CP_LOAD_STATE6_0_NUM_UNIT(anum/2));
2955    tu_cs_emit(cs, CP_LOAD_STATE6_1_EXT_SRC_ADDR(0));
2956    tu_cs_emit(cs, CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI(0));
2957
2958    unsigned emitted = 0;
2959    for (unsigned i = 0; emitted < num && i < link->ubo_map.num; i++) {
2960       for (unsigned j = 0; emitted < num && j < link->ubo_map.array_size[i]; j++) {
2961          tu_cs_emit_qw(cs, buffer_ptr(descriptors_state, &link->ubo_map, i, j));
2962          emitted++;
2963       }
2964    }
2965
2966    for (; emitted < anum; emitted++) {
2967       tu_cs_emit(cs, 0xffffffff);
2968       tu_cs_emit(cs, 0xffffffff);
2969    }
2970 }
2971
2972 static struct tu_cs_entry
2973 tu6_emit_consts(struct tu_cmd_buffer *cmd,
2974                 const struct tu_pipeline *pipeline,
2975                 struct tu_descriptor_state *descriptors_state,
2976                 gl_shader_stage type)
2977 {
2978    struct tu_cs cs;
2979    tu_cs_begin_sub_stream(&cmd->sub_cs, 512, &cs); /* TODO: maximum size? */
2980
2981    tu6_emit_user_consts(&cs, pipeline, descriptors_state, type, cmd->push_constants);
2982    tu6_emit_ubos(&cs, pipeline, descriptors_state, type);
2983
2984    return tu_cs_end_sub_stream(&cmd->sub_cs, &cs);
2985 }
2986
2987 static VkResult
2988 tu6_emit_vs_params(struct tu_cmd_buffer *cmd,
2989                    const struct tu_draw_info *draw,
2990                    struct tu_cs_entry *entry)
2991 {
2992    /* TODO: fill out more than just base instance */
2993    const struct tu_program_descriptor_linkage *link =
2994       &cmd->state.pipeline->program.link[MESA_SHADER_VERTEX];
2995    const struct ir3_const_state *const_state = &link->const_state;
2996    struct tu_cs cs;
2997
2998    if (const_state->offsets.driver_param >= link->constlen) {
2999       *entry = (struct tu_cs_entry) {};
3000       return VK_SUCCESS;
3001    }
3002
3003    VkResult result = tu_cs_begin_sub_stream(&cmd->sub_cs, 8, &cs);
3004    if (result != VK_SUCCESS)
3005       return result;
3006
3007    tu_cs_emit_pkt7(&cs, CP_LOAD_STATE6_GEOM, 3 + 4);
3008    tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(const_state->offsets.driver_param) |
3009          CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
3010          CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
3011          CP_LOAD_STATE6_0_STATE_BLOCK(SB6_VS_SHADER) |
3012          CP_LOAD_STATE6_0_NUM_UNIT(1));
3013    tu_cs_emit(&cs, 0);
3014    tu_cs_emit(&cs, 0);
3015
3016    STATIC_ASSERT(IR3_DP_INSTID_BASE == 2);
3017
3018    tu_cs_emit(&cs, 0);
3019    tu_cs_emit(&cs, 0);
3020    tu_cs_emit(&cs, draw->first_instance);
3021    tu_cs_emit(&cs, 0);
3022
3023    *entry = tu_cs_end_sub_stream(&cmd->sub_cs, &cs);
3024    return VK_SUCCESS;
3025 }
3026
3027 static VkResult
3028 tu6_emit_textures(struct tu_cmd_buffer *cmd,
3029                   const struct tu_pipeline *pipeline,
3030                   struct tu_descriptor_state *descriptors_state,
3031                   gl_shader_stage type,
3032                   struct tu_cs_entry *entry,
3033                   bool *needs_border,
3034                   bool is_sysmem)
3035 {
3036    struct tu_cs *draw_state = &cmd->sub_cs;
3037    const struct tu_program_descriptor_linkage *link =
3038       &pipeline->program.link[type];
3039    VkResult result;
3040
3041    if (link->texture_map.num_desc == 0 && link->sampler_map.num_desc == 0) {
3042       *entry = (struct tu_cs_entry) {};
3043       return VK_SUCCESS;
3044    }
3045
3046    /* allocate and fill texture state */
3047    struct ts_cs_memory tex_const;
3048    result = tu_cs_alloc(draw_state, link->texture_map.num_desc,
3049                         A6XX_TEX_CONST_DWORDS, &tex_const);
3050    if (result != VK_SUCCESS)
3051       return result;
3052
3053    int tex_index = 0;
3054    for (unsigned i = 0; i < link->texture_map.num; i++) {
3055       for (int j = 0; j < link->texture_map.array_size[i]; j++) {
3056          write_tex_const(cmd,
3057                          &tex_const.map[A6XX_TEX_CONST_DWORDS * tex_index++],
3058                          descriptors_state, &link->texture_map, i, j,
3059                          is_sysmem);
3060       }
3061    }
3062
3063    /* allocate and fill sampler state */
3064    struct ts_cs_memory tex_samp = { 0 };
3065    if (link->sampler_map.num_desc) {
3066       result = tu_cs_alloc(draw_state, link->sampler_map.num_desc,
3067                            A6XX_TEX_SAMP_DWORDS, &tex_samp);
3068       if (result != VK_SUCCESS)
3069          return result;
3070
3071       int sampler_index = 0;
3072       for (unsigned i = 0; i < link->sampler_map.num; i++) {
3073          for (int j = 0; j < link->sampler_map.array_size[i]; j++) {
3074             const struct tu_sampler *sampler = sampler_ptr(descriptors_state,
3075                                                            &link->sampler_map,
3076                                                            i, j);
3077             memcpy(&tex_samp.map[A6XX_TEX_SAMP_DWORDS * sampler_index++],
3078                    sampler->state, sizeof(sampler->state));
3079             *needs_border |= sampler->needs_border;
3080          }
3081       }
3082    }
3083
3084    unsigned tex_samp_reg, tex_const_reg, tex_count_reg;
3085    enum a6xx_state_block sb;
3086
3087    switch (type) {
3088    case MESA_SHADER_VERTEX:
3089       sb = SB6_VS_TEX;
3090       tex_samp_reg = REG_A6XX_SP_VS_TEX_SAMP_LO;
3091       tex_const_reg = REG_A6XX_SP_VS_TEX_CONST_LO;
3092       tex_count_reg = REG_A6XX_SP_VS_TEX_COUNT;
3093       break;
3094    case MESA_SHADER_FRAGMENT:
3095       sb = SB6_FS_TEX;
3096       tex_samp_reg = REG_A6XX_SP_FS_TEX_SAMP_LO;
3097       tex_const_reg = REG_A6XX_SP_FS_TEX_CONST_LO;
3098       tex_count_reg = REG_A6XX_SP_FS_TEX_COUNT;
3099       break;
3100    case MESA_SHADER_COMPUTE:
3101       sb = SB6_CS_TEX;
3102       tex_samp_reg = REG_A6XX_SP_CS_TEX_SAMP_LO;
3103       tex_const_reg = REG_A6XX_SP_CS_TEX_CONST_LO;
3104       tex_count_reg = REG_A6XX_SP_CS_TEX_COUNT;
3105       break;
3106    default:
3107       unreachable("bad state block");
3108    }
3109
3110    struct tu_cs cs;
3111    result = tu_cs_begin_sub_stream(draw_state, 16, &cs);
3112    if (result != VK_SUCCESS)
3113       return result;
3114
3115    if (link->sampler_map.num_desc) {
3116       /* output sampler state: */
3117       tu_cs_emit_pkt7(&cs, tu6_stage2opcode(type), 3);
3118       tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
3119                  CP_LOAD_STATE6_0_STATE_TYPE(ST6_SHADER) |
3120                  CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
3121                  CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
3122                  CP_LOAD_STATE6_0_NUM_UNIT(link->sampler_map.num_desc));
3123       tu_cs_emit_qw(&cs, tex_samp.iova); /* SRC_ADDR_LO/HI */
3124
3125       tu_cs_emit_pkt4(&cs, tex_samp_reg, 2);
3126       tu_cs_emit_qw(&cs, tex_samp.iova); /* SRC_ADDR_LO/HI */
3127    }
3128
3129    /* emit texture state: */
3130    tu_cs_emit_pkt7(&cs, tu6_stage2opcode(type), 3);
3131    tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
3132       CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
3133       CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
3134       CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
3135       CP_LOAD_STATE6_0_NUM_UNIT(link->texture_map.num_desc));
3136    tu_cs_emit_qw(&cs, tex_const.iova); /* SRC_ADDR_LO/HI */
3137
3138    tu_cs_emit_pkt4(&cs, tex_const_reg, 2);
3139    tu_cs_emit_qw(&cs, tex_const.iova); /* SRC_ADDR_LO/HI */
3140
3141    tu_cs_emit_pkt4(&cs, tex_count_reg, 1);
3142    tu_cs_emit(&cs, link->texture_map.num_desc);
3143
3144    *entry = tu_cs_end_sub_stream(draw_state, &cs);
3145    return VK_SUCCESS;
3146 }
3147
3148 static VkResult
3149 tu6_emit_ibo(struct tu_cmd_buffer *cmd,
3150              const struct tu_pipeline *pipeline,
3151              struct tu_descriptor_state *descriptors_state,
3152              gl_shader_stage type,
3153              struct tu_cs_entry *entry)
3154 {
3155    struct tu_cs *draw_state = &cmd->sub_cs;
3156    const struct tu_program_descriptor_linkage *link =
3157       &pipeline->program.link[type];
3158    VkResult result;
3159
3160    unsigned num_desc = link->ssbo_map.num_desc + link->image_map.num_desc;
3161
3162    if (num_desc == 0) {
3163       *entry = (struct tu_cs_entry) {};
3164       return VK_SUCCESS;
3165    }
3166
3167    struct ts_cs_memory ibo_const;
3168    result = tu_cs_alloc(draw_state, num_desc,
3169                         A6XX_TEX_CONST_DWORDS, &ibo_const);
3170    if (result != VK_SUCCESS)
3171       return result;
3172
3173    int ssbo_index = 0;
3174    for (unsigned i = 0; i < link->ssbo_map.num; i++) {
3175       for (int j = 0; j < link->ssbo_map.array_size[i]; j++) {
3176          uint32_t *dst = &ibo_const.map[A6XX_TEX_CONST_DWORDS * ssbo_index];
3177
3178          uint64_t va = buffer_ptr(descriptors_state, &link->ssbo_map, i, j);
3179          /* We don't expose robustBufferAccess, so leave the size unlimited. */
3180          uint32_t sz = MAX_STORAGE_BUFFER_RANGE / 4;
3181
3182          dst[0] = A6XX_IBO_0_FMT(FMT6_32_UINT);
3183          dst[1] = A6XX_IBO_1_WIDTH(sz & MASK(15)) |
3184                   A6XX_IBO_1_HEIGHT(sz >> 15);
3185          dst[2] = A6XX_IBO_2_UNK4 |
3186                   A6XX_IBO_2_UNK31 |
3187                   A6XX_IBO_2_TYPE(A6XX_TEX_1D);
3188          dst[3] = 0;
3189          dst[4] = va;
3190          dst[5] = va >> 32;
3191          for (int i = 6; i < A6XX_TEX_CONST_DWORDS; i++)
3192             dst[i] = 0;
3193
3194          ssbo_index++;
3195       }
3196    }
3197
3198    for (unsigned i = 0; i < link->image_map.num; i++) {
3199       for (int j = 0; j < link->image_map.array_size[i]; j++) {
3200          uint32_t *dst = &ibo_const.map[A6XX_TEX_CONST_DWORDS * ssbo_index];
3201
3202          write_image_ibo(cmd, dst,
3203                          descriptors_state, &link->image_map, i, j);
3204
3205          ssbo_index++;
3206       }
3207    }
3208
3209    assert(ssbo_index == num_desc);
3210
3211    struct tu_cs cs;
3212    result = tu_cs_begin_sub_stream(draw_state, 7, &cs);
3213    if (result != VK_SUCCESS)
3214       return result;
3215
3216    uint32_t opcode, ibo_addr_reg;
3217    enum a6xx_state_block sb;
3218    enum a6xx_state_type st;
3219
3220    switch (type) {
3221    case MESA_SHADER_FRAGMENT:
3222       opcode = CP_LOAD_STATE6;
3223       st = ST6_SHADER;
3224       sb = SB6_IBO;
3225       ibo_addr_reg = REG_A6XX_SP_IBO_LO;
3226       break;
3227    case MESA_SHADER_COMPUTE:
3228       opcode = CP_LOAD_STATE6_FRAG;
3229       st = ST6_IBO;
3230       sb = SB6_CS_SHADER;
3231       ibo_addr_reg = REG_A6XX_SP_CS_IBO_LO;
3232       break;
3233    default:
3234       unreachable("unsupported stage for ibos");
3235    }
3236
3237    /* emit texture state: */
3238    tu_cs_emit_pkt7(&cs, opcode, 3);
3239    tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
3240               CP_LOAD_STATE6_0_STATE_TYPE(st) |
3241               CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
3242               CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
3243               CP_LOAD_STATE6_0_NUM_UNIT(num_desc));
3244    tu_cs_emit_qw(&cs, ibo_const.iova); /* SRC_ADDR_LO/HI */
3245
3246    tu_cs_emit_pkt4(&cs, ibo_addr_reg, 2);
3247    tu_cs_emit_qw(&cs, ibo_const.iova); /* SRC_ADDR_LO/HI */
3248
3249    *entry = tu_cs_end_sub_stream(draw_state, &cs);
3250    return VK_SUCCESS;
3251 }
3252
3253 struct PACKED bcolor_entry {
3254    uint32_t fp32[4];
3255    uint16_t ui16[4];
3256    int16_t  si16[4];
3257    uint16_t fp16[4];
3258    uint16_t rgb565;
3259    uint16_t rgb5a1;
3260    uint16_t rgba4;
3261    uint8_t __pad0[2];
3262    uint8_t  ui8[4];
3263    int8_t   si8[4];
3264    uint32_t rgb10a2;
3265    uint32_t z24; /* also s8? */
3266    uint16_t srgb[4];      /* appears to duplicate fp16[], but clamped, used for srgb */
3267    uint8_t  __pad1[56];
3268 } border_color[] = {
3269    [VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK] = {},
3270    [VK_BORDER_COLOR_INT_TRANSPARENT_BLACK] = {},
3271    [VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK] = {
3272       .fp32[3] = 0x3f800000,
3273       .ui16[3] = 0xffff,
3274       .si16[3] = 0x7fff,
3275       .fp16[3] = 0x3c00,
3276       .rgb5a1 = 0x8000,
3277       .rgba4 = 0xf000,
3278       .ui8[3] = 0xff,
3279       .si8[3] = 0x7f,
3280       .rgb10a2 = 0xc0000000,
3281       .srgb[3] = 0x3c00,
3282    },
3283    [VK_BORDER_COLOR_INT_OPAQUE_BLACK] = {
3284       .fp32[3] = 1,
3285       .fp16[3] = 1,
3286    },
3287    [VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE] = {
3288       .fp32[0 ... 3] = 0x3f800000,
3289       .ui16[0 ... 3] = 0xffff,
3290       .si16[0 ... 3] = 0x7fff,
3291       .fp16[0 ... 3] = 0x3c00,
3292       .rgb565 = 0xffff,
3293       .rgb5a1 = 0xffff,
3294       .rgba4 = 0xffff,
3295       .ui8[0 ... 3] = 0xff,
3296       .si8[0 ... 3] = 0x7f,
3297       .rgb10a2 = 0xffffffff,
3298       .z24 = 0xffffff,
3299       .srgb[0 ... 3] = 0x3c00,
3300    },
3301    [VK_BORDER_COLOR_INT_OPAQUE_WHITE] = {
3302       .fp32[0 ... 3] = 1,
3303       .fp16[0 ... 3] = 1,
3304    },
3305 };
3306
3307 static VkResult
3308 tu6_emit_border_color(struct tu_cmd_buffer *cmd,
3309                       struct tu_cs *cs)
3310 {
3311    STATIC_ASSERT(sizeof(struct bcolor_entry) == 128);
3312
3313    const struct tu_pipeline *pipeline = cmd->state.pipeline;
3314    struct tu_descriptor_state *descriptors_state =
3315       &cmd->descriptors[VK_PIPELINE_BIND_POINT_GRAPHICS];
3316    const struct tu_descriptor_map *vs_sampler =
3317       &pipeline->program.link[MESA_SHADER_VERTEX].sampler_map;
3318    const struct tu_descriptor_map *fs_sampler =
3319       &pipeline->program.link[MESA_SHADER_FRAGMENT].sampler_map;
3320    struct ts_cs_memory ptr;
3321
3322    VkResult result = tu_cs_alloc(&cmd->sub_cs,
3323                                  vs_sampler->num_desc + fs_sampler->num_desc,
3324                                  128 / 4,
3325                                  &ptr);
3326    if (result != VK_SUCCESS)
3327       return result;
3328
3329    for (unsigned i = 0; i < vs_sampler->num; i++) {
3330       for (unsigned j = 0; j < vs_sampler->array_size[i]; j++) {
3331          const struct tu_sampler *sampler = sampler_ptr(descriptors_state,
3332                                                         vs_sampler, i, j);
3333          memcpy(ptr.map, &border_color[sampler->border], 128);
3334          ptr.map += 128 / 4;
3335       }
3336    }
3337
3338    for (unsigned i = 0; i < fs_sampler->num; i++) {
3339       for (unsigned j = 0; j < fs_sampler->array_size[i]; j++) {
3340          const struct tu_sampler *sampler = sampler_ptr(descriptors_state,
3341                                                         fs_sampler, i, j);
3342          memcpy(ptr.map, &border_color[sampler->border], 128);
3343          ptr.map += 128 / 4;
3344       }
3345    }
3346
3347    tu_cs_emit_pkt4(cs, REG_A6XX_SP_TP_BORDER_COLOR_BASE_ADDR_LO, 2);
3348    tu_cs_emit_qw(cs, ptr.iova);
3349    return VK_SUCCESS;
3350 }
3351
3352 static VkResult
3353 tu6_bind_draw_states(struct tu_cmd_buffer *cmd,
3354                      struct tu_cs *cs,
3355                      const struct tu_draw_info *draw)
3356 {
3357    const struct tu_pipeline *pipeline = cmd->state.pipeline;
3358    const struct tu_dynamic_state *dynamic = &cmd->state.dynamic;
3359    struct tu_draw_state_group draw_state_groups[TU_DRAW_STATE_COUNT];
3360    uint32_t draw_state_group_count = 0;
3361    VkResult result;
3362
3363    struct tu_descriptor_state *descriptors_state =
3364       &cmd->descriptors[VK_PIPELINE_BIND_POINT_GRAPHICS];
3365
3366    /* TODO lrz */
3367
3368    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9806, 0);
3369    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9990, 0);
3370    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_UNKNOWN_A008, 0);
3371
3372    tu_cs_emit_regs(cs,
3373                    A6XX_PC_PRIMITIVE_CNTL_0(.primitive_restart =
3374                                             pipeline->ia.primitive_restart && draw->indexed));
3375
3376    if (cmd->state.dirty &
3377           (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_DYNAMIC_LINE_WIDTH) &&
3378        (pipeline->dynamic_state.mask & TU_DYNAMIC_LINE_WIDTH)) {
3379       tu6_emit_gras_su_cntl(cs, pipeline->rast.gras_su_cntl,
3380                             dynamic->line_width);
3381    }
3382
3383    if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK) &&
3384        (pipeline->dynamic_state.mask & TU_DYNAMIC_STENCIL_COMPARE_MASK)) {
3385       tu6_emit_stencil_compare_mask(cs, dynamic->stencil_compare_mask.front,
3386                                     dynamic->stencil_compare_mask.back);
3387    }
3388
3389    if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK) &&
3390        (pipeline->dynamic_state.mask & TU_DYNAMIC_STENCIL_WRITE_MASK)) {
3391       tu6_emit_stencil_write_mask(cs, dynamic->stencil_write_mask.front,
3392                                   dynamic->stencil_write_mask.back);
3393    }
3394
3395    if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE) &&
3396        (pipeline->dynamic_state.mask & TU_DYNAMIC_STENCIL_REFERENCE)) {
3397       tu6_emit_stencil_reference(cs, dynamic->stencil_reference.front,
3398                                  dynamic->stencil_reference.back);
3399    }
3400
3401    if (cmd->state.dirty &
3402        (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_VERTEX_BUFFERS)) {
3403       for (uint32_t i = 0; i < pipeline->vi.count; i++) {
3404          const uint32_t binding = pipeline->vi.bindings[i];
3405          const uint32_t stride = pipeline->vi.strides[i];
3406          const struct tu_buffer *buf = cmd->state.vb.buffers[binding];
3407          const VkDeviceSize offset = buf->bo_offset +
3408                                      cmd->state.vb.offsets[binding] +
3409                                      pipeline->vi.offsets[i];
3410          const VkDeviceSize size =
3411             offset < buf->bo->size ? buf->bo->size - offset : 0;
3412
3413          tu_cs_emit_regs(cs,
3414                          A6XX_VFD_FETCH_BASE(i, .bo = buf->bo, .bo_offset = offset),
3415                          A6XX_VFD_FETCH_SIZE(i, size),
3416                          A6XX_VFD_FETCH_STRIDE(i, stride));
3417       }
3418    }
3419
3420    if (cmd->state.dirty & TU_CMD_DIRTY_PIPELINE) {
3421       draw_state_groups[draw_state_group_count++] =
3422          (struct tu_draw_state_group) {
3423             .id = TU_DRAW_STATE_PROGRAM,
3424             .enable_mask = ENABLE_DRAW,
3425             .ib = pipeline->program.state_ib,
3426          };
3427       draw_state_groups[draw_state_group_count++] =
3428          (struct tu_draw_state_group) {
3429             .id = TU_DRAW_STATE_PROGRAM_BINNING,
3430             .enable_mask = CP_SET_DRAW_STATE__0_BINNING,
3431             .ib = pipeline->program.binning_state_ib,
3432          };
3433       draw_state_groups[draw_state_group_count++] =
3434          (struct tu_draw_state_group) {
3435             .id = TU_DRAW_STATE_VI,
3436             .enable_mask = ENABLE_DRAW,
3437             .ib = pipeline->vi.state_ib,
3438          };
3439       draw_state_groups[draw_state_group_count++] =
3440          (struct tu_draw_state_group) {
3441             .id = TU_DRAW_STATE_VI_BINNING,
3442             .enable_mask = CP_SET_DRAW_STATE__0_BINNING,
3443             .ib = pipeline->vi.binning_state_ib,
3444          };
3445       draw_state_groups[draw_state_group_count++] =
3446          (struct tu_draw_state_group) {
3447             .id = TU_DRAW_STATE_VP,
3448             .enable_mask = ENABLE_ALL,
3449             .ib = pipeline->vp.state_ib,
3450          };
3451       draw_state_groups[draw_state_group_count++] =
3452          (struct tu_draw_state_group) {
3453             .id = TU_DRAW_STATE_RAST,
3454             .enable_mask = ENABLE_ALL,
3455             .ib = pipeline->rast.state_ib,
3456          };
3457       draw_state_groups[draw_state_group_count++] =
3458          (struct tu_draw_state_group) {
3459             .id = TU_DRAW_STATE_DS,
3460             .enable_mask = ENABLE_ALL,
3461             .ib = pipeline->ds.state_ib,
3462          };
3463       draw_state_groups[draw_state_group_count++] =
3464          (struct tu_draw_state_group) {
3465             .id = TU_DRAW_STATE_BLEND,
3466             .enable_mask = ENABLE_ALL,
3467             .ib = pipeline->blend.state_ib,
3468          };
3469    }
3470
3471    if (cmd->state.dirty &
3472          (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_DESCRIPTOR_SETS | TU_CMD_DIRTY_PUSH_CONSTANTS)) {
3473       draw_state_groups[draw_state_group_count++] =
3474          (struct tu_draw_state_group) {
3475             .id = TU_DRAW_STATE_VS_CONST,
3476             .enable_mask = ENABLE_ALL,
3477             .ib = tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_VERTEX)
3478          };
3479       draw_state_groups[draw_state_group_count++] =
3480          (struct tu_draw_state_group) {
3481             .id = TU_DRAW_STATE_FS_CONST,
3482             .enable_mask = ENABLE_DRAW,
3483             .ib = tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_FRAGMENT)
3484          };
3485    }
3486
3487    if (cmd->state.dirty &
3488          (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_DESCRIPTOR_SETS)) {
3489       bool needs_border = false;
3490       struct tu_cs_entry vs_tex, fs_tex_sysmem, fs_tex_gmem, fs_ibo;
3491
3492       result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3493                                  MESA_SHADER_VERTEX, &vs_tex, &needs_border,
3494                                  false);
3495       if (result != VK_SUCCESS)
3496          return result;
3497
3498       /* TODO: we could emit just one texture descriptor draw state when there
3499        * are no input attachments, which is the most common case. We could
3500        * also split out the sampler state, which doesn't change even for input
3501        * attachments.
3502        */
3503       result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3504                                  MESA_SHADER_FRAGMENT, &fs_tex_sysmem,
3505                                  &needs_border, true);
3506       if (result != VK_SUCCESS)
3507          return result;
3508
3509       result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3510                                  MESA_SHADER_FRAGMENT, &fs_tex_gmem,
3511                                  &needs_border, false);
3512       if (result != VK_SUCCESS)
3513          return result;
3514
3515       result = tu6_emit_ibo(cmd, pipeline, descriptors_state,
3516                             MESA_SHADER_FRAGMENT, &fs_ibo);
3517       if (result != VK_SUCCESS)
3518          return result;
3519
3520       draw_state_groups[draw_state_group_count++] =
3521          (struct tu_draw_state_group) {
3522             .id = TU_DRAW_STATE_VS_TEX,
3523             .enable_mask = ENABLE_ALL,
3524             .ib = vs_tex,
3525          };
3526       draw_state_groups[draw_state_group_count++] =
3527          (struct tu_draw_state_group) {
3528             .id = TU_DRAW_STATE_FS_TEX_GMEM,
3529             .enable_mask = CP_SET_DRAW_STATE__0_GMEM,
3530             .ib = fs_tex_gmem,
3531          };
3532       draw_state_groups[draw_state_group_count++] =
3533          (struct tu_draw_state_group) {
3534             .id = TU_DRAW_STATE_FS_TEX_SYSMEM,
3535             .enable_mask = CP_SET_DRAW_STATE__0_SYSMEM,
3536             .ib = fs_tex_sysmem,
3537          };
3538       draw_state_groups[draw_state_group_count++] =
3539          (struct tu_draw_state_group) {
3540             .id = TU_DRAW_STATE_FS_IBO,
3541             .enable_mask = ENABLE_DRAW,
3542             .ib = fs_ibo,
3543          };
3544
3545       if (needs_border) {
3546          result = tu6_emit_border_color(cmd, cs);
3547          if (result != VK_SUCCESS)
3548             return result;
3549       }
3550    }
3551
3552    struct tu_cs_entry vs_params;
3553    result = tu6_emit_vs_params(cmd, draw, &vs_params);
3554    if (result != VK_SUCCESS)
3555       return result;
3556
3557    draw_state_groups[draw_state_group_count++] =
3558       (struct tu_draw_state_group) {
3559          .id = TU_DRAW_STATE_VS_PARAMS,
3560          .enable_mask = ENABLE_ALL,
3561          .ib = vs_params,
3562       };
3563
3564    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3 * draw_state_group_count);
3565    for (uint32_t i = 0; i < draw_state_group_count; i++) {
3566       const struct tu_draw_state_group *group = &draw_state_groups[i];
3567       debug_assert((group->enable_mask & ~ENABLE_ALL) == 0);
3568       uint32_t cp_set_draw_state =
3569          CP_SET_DRAW_STATE__0_COUNT(group->ib.size / 4) |
3570          group->enable_mask |
3571          CP_SET_DRAW_STATE__0_GROUP_ID(group->id);
3572       uint64_t iova;
3573       if (group->ib.size) {
3574          iova = group->ib.bo->iova + group->ib.offset;
3575       } else {
3576          cp_set_draw_state |= CP_SET_DRAW_STATE__0_DISABLE;
3577          iova = 0;
3578       }
3579
3580       tu_cs_emit(cs, cp_set_draw_state);
3581       tu_cs_emit_qw(cs, iova);
3582    }
3583
3584    tu_cs_sanity_check(cs);
3585
3586    /* track BOs */
3587    if (cmd->state.dirty & TU_CMD_DIRTY_VERTEX_BUFFERS) {
3588       for (uint32_t i = 0; i < MAX_VBS; i++) {
3589          const struct tu_buffer *buf = cmd->state.vb.buffers[i];
3590          if (buf)
3591             tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
3592       }
3593    }
3594    if (cmd->state.dirty & TU_CMD_DIRTY_DESCRIPTOR_SETS) {
3595       unsigned i;
3596       for_each_bit(i, descriptors_state->valid) {
3597          struct tu_descriptor_set *set = descriptors_state->sets[i];
3598          for (unsigned j = 0; j < set->layout->buffer_count; ++j)
3599             if (set->descriptors[j]) {
3600                tu_bo_list_add(&cmd->bo_list, set->descriptors[j],
3601                               MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3602             }
3603       }
3604    }
3605
3606    /* Fragment shader state overwrites compute shader state, so flag the
3607     * compute pipeline for re-emit.
3608     */
3609    cmd->state.dirty = TU_CMD_DIRTY_COMPUTE_PIPELINE;
3610    return VK_SUCCESS;
3611 }
3612
3613 static void
3614 tu6_emit_draw_direct(struct tu_cmd_buffer *cmd,
3615                      struct tu_cs *cs,
3616                      const struct tu_draw_info *draw)
3617 {
3618
3619    const enum pc_di_primtype primtype = cmd->state.pipeline->ia.primtype;
3620
3621    tu_cs_emit_regs(cs,
3622                    A6XX_VFD_INDEX_OFFSET(draw->vertex_offset),
3623                    A6XX_VFD_INSTANCE_START_OFFSET(draw->first_instance));
3624
3625    /* TODO hw binning */
3626    if (draw->indexed) {
3627       const enum a4xx_index_size index_size =
3628          tu6_index_size(cmd->state.index_type);
3629       const uint32_t index_bytes =
3630          (cmd->state.index_type == VK_INDEX_TYPE_UINT32) ? 4 : 2;
3631       const struct tu_buffer *buf = cmd->state.index_buffer;
3632       const VkDeviceSize offset = buf->bo_offset + cmd->state.index_offset +
3633                                   index_bytes * draw->first_index;
3634       const uint32_t size = index_bytes * draw->count;
3635
3636       const uint32_t cp_draw_indx =
3637          CP_DRAW_INDX_OFFSET_0_PRIM_TYPE(primtype) |
3638          CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT(DI_SRC_SEL_DMA) |
3639          CP_DRAW_INDX_OFFSET_0_INDEX_SIZE(index_size) |
3640          CP_DRAW_INDX_OFFSET_0_VIS_CULL(USE_VISIBILITY) | 0x2000;
3641
3642       tu_cs_emit_pkt7(cs, CP_DRAW_INDX_OFFSET, 7);
3643       tu_cs_emit(cs, cp_draw_indx);
3644       tu_cs_emit(cs, draw->instance_count);
3645       tu_cs_emit(cs, draw->count);
3646       tu_cs_emit(cs, 0x0); /* XXX */
3647       tu_cs_emit_qw(cs, buf->bo->iova + offset);
3648       tu_cs_emit(cs, size);
3649    } else {
3650       const uint32_t cp_draw_indx =
3651          CP_DRAW_INDX_OFFSET_0_PRIM_TYPE(primtype) |
3652          CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT(DI_SRC_SEL_AUTO_INDEX) |
3653          CP_DRAW_INDX_OFFSET_0_VIS_CULL(USE_VISIBILITY) | 0x2000;
3654
3655       tu_cs_emit_pkt7(cs, CP_DRAW_INDX_OFFSET, 3);
3656       tu_cs_emit(cs, cp_draw_indx);
3657       tu_cs_emit(cs, draw->instance_count);
3658       tu_cs_emit(cs, draw->count);
3659    }
3660 }
3661
3662 static void
3663 tu_draw(struct tu_cmd_buffer *cmd, const struct tu_draw_info *draw)
3664 {
3665    struct tu_cs *cs = &cmd->draw_cs;
3666    VkResult result;
3667
3668    result = tu6_bind_draw_states(cmd, cs, draw);
3669    if (result != VK_SUCCESS) {
3670       cmd->record_result = result;
3671       return;
3672    }
3673
3674    if (draw->indirect) {
3675       tu_finishme("indirect draw");
3676       return;
3677    }
3678
3679    tu6_emit_draw_direct(cmd, cs, draw);
3680
3681    cmd->wait_for_idle = true;
3682
3683    tu_cs_sanity_check(cs);
3684 }
3685
3686 void
3687 tu_CmdDraw(VkCommandBuffer commandBuffer,
3688            uint32_t vertexCount,
3689            uint32_t instanceCount,
3690            uint32_t firstVertex,
3691            uint32_t firstInstance)
3692 {
3693    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3694    struct tu_draw_info info = {};
3695
3696    info.count = vertexCount;
3697    info.instance_count = instanceCount;
3698    info.first_instance = firstInstance;
3699    info.vertex_offset = firstVertex;
3700
3701    tu_draw(cmd_buffer, &info);
3702 }
3703
3704 void
3705 tu_CmdDrawIndexed(VkCommandBuffer commandBuffer,
3706                   uint32_t indexCount,
3707                   uint32_t instanceCount,
3708                   uint32_t firstIndex,
3709                   int32_t vertexOffset,
3710                   uint32_t firstInstance)
3711 {
3712    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3713    struct tu_draw_info info = {};
3714
3715    info.indexed = true;
3716    info.count = indexCount;
3717    info.instance_count = instanceCount;
3718    info.first_index = firstIndex;
3719    info.vertex_offset = vertexOffset;
3720    info.first_instance = firstInstance;
3721
3722    tu_draw(cmd_buffer, &info);
3723 }
3724
3725 void
3726 tu_CmdDrawIndirect(VkCommandBuffer commandBuffer,
3727                    VkBuffer _buffer,
3728                    VkDeviceSize offset,
3729                    uint32_t drawCount,
3730                    uint32_t stride)
3731 {
3732    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3733    TU_FROM_HANDLE(tu_buffer, buffer, _buffer);
3734    struct tu_draw_info info = {};
3735
3736    info.count = drawCount;
3737    info.indirect = buffer;
3738    info.indirect_offset = offset;
3739    info.stride = stride;
3740
3741    tu_draw(cmd_buffer, &info);
3742 }
3743
3744 void
3745 tu_CmdDrawIndexedIndirect(VkCommandBuffer commandBuffer,
3746                           VkBuffer _buffer,
3747                           VkDeviceSize offset,
3748                           uint32_t drawCount,
3749                           uint32_t stride)
3750 {
3751    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3752    TU_FROM_HANDLE(tu_buffer, buffer, _buffer);
3753    struct tu_draw_info info = {};
3754
3755    info.indexed = true;
3756    info.count = drawCount;
3757    info.indirect = buffer;
3758    info.indirect_offset = offset;
3759    info.stride = stride;
3760
3761    tu_draw(cmd_buffer, &info);
3762 }
3763
3764 struct tu_dispatch_info
3765 {
3766    /**
3767     * Determine the layout of the grid (in block units) to be used.
3768     */
3769    uint32_t blocks[3];
3770
3771    /**
3772     * A starting offset for the grid. If unaligned is set, the offset
3773     * must still be aligned.
3774     */
3775    uint32_t offsets[3];
3776    /**
3777     * Whether it's an unaligned compute dispatch.
3778     */
3779    bool unaligned;
3780
3781    /**
3782     * Indirect compute parameters resource.
3783     */
3784    struct tu_buffer *indirect;
3785    uint64_t indirect_offset;
3786 };
3787
3788 static void
3789 tu_emit_compute_driver_params(struct tu_cs *cs, struct tu_pipeline *pipeline,
3790                               const struct tu_dispatch_info *info)
3791 {
3792    gl_shader_stage type = MESA_SHADER_COMPUTE;
3793    const struct tu_program_descriptor_linkage *link =
3794       &pipeline->program.link[type];
3795    const struct ir3_const_state *const_state = &link->const_state;
3796    uint32_t offset = const_state->offsets.driver_param;
3797
3798    if (link->constlen <= offset)
3799       return;
3800
3801    if (!info->indirect) {
3802       uint32_t driver_params[IR3_DP_CS_COUNT] = {
3803          [IR3_DP_NUM_WORK_GROUPS_X] = info->blocks[0],
3804          [IR3_DP_NUM_WORK_GROUPS_Y] = info->blocks[1],
3805          [IR3_DP_NUM_WORK_GROUPS_Z] = info->blocks[2],
3806          [IR3_DP_LOCAL_GROUP_SIZE_X] = pipeline->compute.local_size[0],
3807          [IR3_DP_LOCAL_GROUP_SIZE_Y] = pipeline->compute.local_size[1],
3808          [IR3_DP_LOCAL_GROUP_SIZE_Z] = pipeline->compute.local_size[2],
3809       };
3810
3811       uint32_t num_consts = MIN2(const_state->num_driver_params,
3812                                  (link->constlen - offset) * 4);
3813       /* push constants */
3814       tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + num_consts);
3815       tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(offset) |
3816                  CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
3817                  CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
3818                  CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
3819                  CP_LOAD_STATE6_0_NUM_UNIT(num_consts / 4));
3820       tu_cs_emit(cs, 0);
3821       tu_cs_emit(cs, 0);
3822       uint32_t i;
3823       for (i = 0; i < num_consts; i++)
3824          tu_cs_emit(cs, driver_params[i]);
3825    } else {
3826       tu_finishme("Indirect driver params");
3827    }
3828 }
3829
3830 static void
3831 tu_dispatch(struct tu_cmd_buffer *cmd,
3832             const struct tu_dispatch_info *info)
3833 {
3834    struct tu_cs *cs = &cmd->cs;
3835    struct tu_pipeline *pipeline = cmd->state.compute_pipeline;
3836    struct tu_descriptor_state *descriptors_state =
3837       &cmd->descriptors[VK_PIPELINE_BIND_POINT_COMPUTE];
3838    VkResult result;
3839
3840    if (cmd->state.dirty & TU_CMD_DIRTY_COMPUTE_PIPELINE)
3841       tu_cs_emit_ib(cs, &pipeline->program.state_ib);
3842
3843    struct tu_cs_entry ib;
3844
3845    ib = tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_COMPUTE);
3846    if (ib.size)
3847       tu_cs_emit_ib(cs, &ib);
3848
3849    tu_emit_compute_driver_params(cs, pipeline, info);
3850
3851    bool needs_border;
3852    result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3853                               MESA_SHADER_COMPUTE, &ib, &needs_border, false);
3854    if (result != VK_SUCCESS) {
3855       cmd->record_result = result;
3856       return;
3857    }
3858
3859    if (ib.size)
3860       tu_cs_emit_ib(cs, &ib);
3861
3862    if (needs_border)
3863       tu_finishme("compute border color");
3864
3865    result = tu6_emit_ibo(cmd, pipeline, descriptors_state, MESA_SHADER_COMPUTE, &ib);
3866    if (result != VK_SUCCESS) {
3867       cmd->record_result = result;
3868       return;
3869    }
3870
3871    if (ib.size)
3872       tu_cs_emit_ib(cs, &ib);
3873
3874    /* track BOs */
3875    if (cmd->state.dirty & TU_CMD_DIRTY_DESCRIPTOR_SETS) {
3876       unsigned i;
3877       for_each_bit(i, descriptors_state->valid) {
3878          struct tu_descriptor_set *set = descriptors_state->sets[i];
3879          for (unsigned j = 0; j < set->layout->buffer_count; ++j)
3880             if (set->descriptors[j]) {
3881                tu_bo_list_add(&cmd->bo_list, set->descriptors[j],
3882                               MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3883             }
3884       }
3885    }
3886
3887    /* Compute shader state overwrites fragment shader state, so we flag the
3888     * graphics pipeline for re-emit.
3889     */
3890    cmd->state.dirty = TU_CMD_DIRTY_PIPELINE;
3891
3892    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
3893    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_COMPUTE));
3894
3895    const uint32_t *local_size = pipeline->compute.local_size;
3896    const uint32_t *num_groups = info->blocks;
3897    tu_cs_emit_regs(cs,
3898                    A6XX_HLSQ_CS_NDRANGE_0(.kerneldim = 3,
3899                                           .localsizex = local_size[0] - 1,
3900                                           .localsizey = local_size[1] - 1,
3901                                           .localsizez = local_size[2] - 1),
3902                    A6XX_HLSQ_CS_NDRANGE_1(.globalsize_x = local_size[0] * num_groups[0]),
3903                    A6XX_HLSQ_CS_NDRANGE_2(.globaloff_x = 0),
3904                    A6XX_HLSQ_CS_NDRANGE_3(.globalsize_y = local_size[1] * num_groups[1]),
3905                    A6XX_HLSQ_CS_NDRANGE_4(.globaloff_y = 0),
3906                    A6XX_HLSQ_CS_NDRANGE_5(.globalsize_z = local_size[2] * num_groups[2]),
3907                    A6XX_HLSQ_CS_NDRANGE_6(.globaloff_z = 0));
3908
3909    tu_cs_emit_regs(cs,
3910                    A6XX_HLSQ_CS_KERNEL_GROUP_X(1),
3911                    A6XX_HLSQ_CS_KERNEL_GROUP_Y(1),
3912                    A6XX_HLSQ_CS_KERNEL_GROUP_Z(1));
3913
3914    if (info->indirect) {
3915       uint64_t iova = tu_buffer_iova(info->indirect) + info->indirect_offset;
3916
3917       tu_bo_list_add(&cmd->bo_list, info->indirect->bo,
3918                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3919
3920       tu_cs_emit_pkt7(cs, CP_EXEC_CS_INDIRECT, 4);
3921       tu_cs_emit(cs, 0x00000000);
3922       tu_cs_emit_qw(cs, iova);
3923       tu_cs_emit(cs,
3924                  A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX(local_size[0] - 1) |
3925                  A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY(local_size[1] - 1) |
3926                  A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ(local_size[2] - 1));
3927    } else {
3928       tu_cs_emit_pkt7(cs, CP_EXEC_CS, 4);
3929       tu_cs_emit(cs, 0x00000000);
3930       tu_cs_emit(cs, CP_EXEC_CS_1_NGROUPS_X(info->blocks[0]));
3931       tu_cs_emit(cs, CP_EXEC_CS_2_NGROUPS_Y(info->blocks[1]));
3932       tu_cs_emit(cs, CP_EXEC_CS_3_NGROUPS_Z(info->blocks[2]));
3933    }
3934
3935    tu_cs_emit_wfi(cs);
3936
3937    tu6_emit_cache_flush(cmd, cs);
3938 }
3939
3940 void
3941 tu_CmdDispatchBase(VkCommandBuffer commandBuffer,
3942                    uint32_t base_x,
3943                    uint32_t base_y,
3944                    uint32_t base_z,
3945                    uint32_t x,
3946                    uint32_t y,
3947                    uint32_t z)
3948 {
3949    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3950    struct tu_dispatch_info info = {};
3951
3952    info.blocks[0] = x;
3953    info.blocks[1] = y;
3954    info.blocks[2] = z;
3955
3956    info.offsets[0] = base_x;
3957    info.offsets[1] = base_y;
3958    info.offsets[2] = base_z;
3959    tu_dispatch(cmd_buffer, &info);
3960 }
3961
3962 void
3963 tu_CmdDispatch(VkCommandBuffer commandBuffer,
3964                uint32_t x,
3965                uint32_t y,
3966                uint32_t z)
3967 {
3968    tu_CmdDispatchBase(commandBuffer, 0, 0, 0, x, y, z);
3969 }
3970
3971 void
3972 tu_CmdDispatchIndirect(VkCommandBuffer commandBuffer,
3973                        VkBuffer _buffer,
3974                        VkDeviceSize offset)
3975 {
3976    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3977    TU_FROM_HANDLE(tu_buffer, buffer, _buffer);
3978    struct tu_dispatch_info info = {};
3979
3980    info.indirect = buffer;
3981    info.indirect_offset = offset;
3982
3983    tu_dispatch(cmd_buffer, &info);
3984 }
3985
3986 void
3987 tu_CmdEndRenderPass(VkCommandBuffer commandBuffer)
3988 {
3989    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3990
3991    tu_cs_end(&cmd_buffer->draw_cs);
3992    tu_cs_end(&cmd_buffer->draw_epilogue_cs);
3993
3994    if (use_sysmem_rendering(cmd_buffer))
3995       tu_cmd_render_sysmem(cmd_buffer);
3996    else
3997       tu_cmd_render_tiles(cmd_buffer);
3998
3999    /* discard draw_cs and draw_epilogue_cs entries now that the tiles are
4000       rendered */
4001    tu_cs_discard_entries(&cmd_buffer->draw_cs);
4002    tu_cs_begin(&cmd_buffer->draw_cs);
4003    tu_cs_discard_entries(&cmd_buffer->draw_epilogue_cs);
4004    tu_cs_begin(&cmd_buffer->draw_epilogue_cs);
4005
4006    cmd_buffer->state.pass = NULL;
4007    cmd_buffer->state.subpass = NULL;
4008    cmd_buffer->state.framebuffer = NULL;
4009 }
4010
4011 void
4012 tu_CmdEndRenderPass2(VkCommandBuffer commandBuffer,
4013                      const VkSubpassEndInfoKHR *pSubpassEndInfo)
4014 {
4015    tu_CmdEndRenderPass(commandBuffer);
4016 }
4017
4018 struct tu_barrier_info
4019 {
4020    uint32_t eventCount;
4021    const VkEvent *pEvents;
4022    VkPipelineStageFlags srcStageMask;
4023 };
4024
4025 static void
4026 tu_barrier(struct tu_cmd_buffer *cmd_buffer,
4027            uint32_t memoryBarrierCount,
4028            const VkMemoryBarrier *pMemoryBarriers,
4029            uint32_t bufferMemoryBarrierCount,
4030            const VkBufferMemoryBarrier *pBufferMemoryBarriers,
4031            uint32_t imageMemoryBarrierCount,
4032            const VkImageMemoryBarrier *pImageMemoryBarriers,
4033            const struct tu_barrier_info *info)
4034 {
4035 }
4036
4037 void
4038 tu_CmdPipelineBarrier(VkCommandBuffer commandBuffer,
4039                       VkPipelineStageFlags srcStageMask,
4040                       VkPipelineStageFlags destStageMask,
4041                       VkBool32 byRegion,
4042                       uint32_t memoryBarrierCount,
4043                       const VkMemoryBarrier *pMemoryBarriers,
4044                       uint32_t bufferMemoryBarrierCount,
4045                       const VkBufferMemoryBarrier *pBufferMemoryBarriers,
4046                       uint32_t imageMemoryBarrierCount,
4047                       const VkImageMemoryBarrier *pImageMemoryBarriers)
4048 {
4049    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
4050    struct tu_barrier_info info;
4051
4052    info.eventCount = 0;
4053    info.pEvents = NULL;
4054    info.srcStageMask = srcStageMask;
4055
4056    tu_barrier(cmd_buffer, memoryBarrierCount, pMemoryBarriers,
4057               bufferMemoryBarrierCount, pBufferMemoryBarriers,
4058               imageMemoryBarrierCount, pImageMemoryBarriers, &info);
4059 }
4060
4061 static void
4062 write_event(struct tu_cmd_buffer *cmd, struct tu_event *event, unsigned value)
4063 {
4064    struct tu_cs *cs = &cmd->cs;
4065
4066    tu_bo_list_add(&cmd->bo_list, &event->bo, MSM_SUBMIT_BO_WRITE);
4067
4068    /* TODO: any flush required before/after ? */
4069
4070    tu_cs_emit_pkt7(cs, CP_MEM_WRITE, 3);
4071    tu_cs_emit_qw(cs, event->bo.iova); /* ADDR_LO/HI */
4072    tu_cs_emit(cs, value);
4073 }
4074
4075 void
4076 tu_CmdSetEvent(VkCommandBuffer commandBuffer,
4077                VkEvent _event,
4078                VkPipelineStageFlags stageMask)
4079 {
4080    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
4081    TU_FROM_HANDLE(tu_event, event, _event);
4082
4083    write_event(cmd, event, 1);
4084 }
4085
4086 void
4087 tu_CmdResetEvent(VkCommandBuffer commandBuffer,
4088                  VkEvent _event,
4089                  VkPipelineStageFlags stageMask)
4090 {
4091    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
4092    TU_FROM_HANDLE(tu_event, event, _event);
4093
4094    write_event(cmd, event, 0);
4095 }
4096
4097 void
4098 tu_CmdWaitEvents(VkCommandBuffer commandBuffer,
4099                  uint32_t eventCount,
4100                  const VkEvent *pEvents,
4101                  VkPipelineStageFlags srcStageMask,
4102                  VkPipelineStageFlags dstStageMask,
4103                  uint32_t memoryBarrierCount,
4104                  const VkMemoryBarrier *pMemoryBarriers,
4105                  uint32_t bufferMemoryBarrierCount,
4106                  const VkBufferMemoryBarrier *pBufferMemoryBarriers,
4107                  uint32_t imageMemoryBarrierCount,
4108                  const VkImageMemoryBarrier *pImageMemoryBarriers)
4109 {
4110    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
4111    struct tu_cs *cs = &cmd->cs;
4112
4113    /* TODO: any flush required before/after? (CP_WAIT_FOR_ME?) */
4114
4115    for (uint32_t i = 0; i < eventCount; i++) {
4116       TU_FROM_HANDLE(tu_event, event, pEvents[i]);
4117
4118       tu_bo_list_add(&cmd->bo_list, &event->bo, MSM_SUBMIT_BO_READ);
4119
4120       tu_cs_emit_pkt7(cs, CP_WAIT_REG_MEM, 6);
4121       tu_cs_emit(cs, CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ) |
4122                      CP_WAIT_REG_MEM_0_POLL_MEMORY);
4123       tu_cs_emit_qw(cs, event->bo.iova); /* POLL_ADDR_LO/HI */
4124       tu_cs_emit(cs, CP_WAIT_REG_MEM_3_REF(1));
4125       tu_cs_emit(cs, CP_WAIT_REG_MEM_4_MASK(~0u));
4126       tu_cs_emit(cs, CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(20));
4127    }
4128 }
4129
4130 void
4131 tu_CmdSetDeviceMask(VkCommandBuffer commandBuffer, uint32_t deviceMask)
4132 {
4133    /* No-op */
4134 }