src/freedreno/vulkan/tu_cmd_buffer.c

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * based in part on anv driver which is:
   6  * Copyright © 2015 Intel Corporation
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25  * DEALINGS IN THE SOFTWARE.
  26  */
  27
  28 #include "tu_private.h"
  29
  30 #include "registers/adreno_pm4.xml.h"
  31 #include "registers/adreno_common.xml.h"
  32
  33 #include "vk_format.h"
  34
  35 #include "tu_cs.h"
  36 #include "tu_blit.h"
  37
  38 #define OVERFLOW_FLAG_REG REG_A6XX_CP_SCRATCH_REG(0)
  39
  40 void
  41 tu_bo_list_init(struct tu_bo_list *list)
  42 {
  43    list->count = list->capacity = 0;
  44    list->bo_infos = NULL;
  45 }
  46
  47 void
  48 tu_bo_list_destroy(struct tu_bo_list *list)
  49 {
  50    free(list->bo_infos);
  51 }
  52
  53 void
  54 tu_bo_list_reset(struct tu_bo_list *list)
  55 {
  56    list->count = 0;
  57 }
  58
  59 /**
  60  * \a flags consists of MSM_SUBMIT_BO_FLAGS.
  61  */
  62 static uint32_t
  63 tu_bo_list_add_info(struct tu_bo_list *list,
  64                     const struct drm_msm_gem_submit_bo *bo_info)
  65 {
  66    assert(bo_info->handle != 0);
  67
  68    for (uint32_t i = 0; i < list->count; ++i) {
  69       if (list->bo_infos[i].handle == bo_info->handle) {
  70          assert(list->bo_infos[i].presumed == bo_info->presumed);
  71          list->bo_infos[i].flags |= bo_info->flags;
  72          return i;
  73       }
  74    }
  75
  76    /* grow list->bo_infos if needed */
  77    if (list->count == list->capacity) {
  78       uint32_t new_capacity = MAX2(2 * list->count, 16);
  79       struct drm_msm_gem_submit_bo *new_bo_infos = realloc(
  80          list->bo_infos, new_capacity * sizeof(struct drm_msm_gem_submit_bo));
  81       if (!new_bo_infos)
  82          return TU_BO_LIST_FAILED;
  83       list->bo_infos = new_bo_infos;
  84       list->capacity = new_capacity;
  85    }
  86
  87    list->bo_infos[list->count] = *bo_info;
  88    return list->count++;
  89 }
  90
  91 uint32_t
  92 tu_bo_list_add(struct tu_bo_list *list,
  93                const struct tu_bo *bo,
  94                uint32_t flags)
  95 {
  96    return tu_bo_list_add_info(list, &(struct drm_msm_gem_submit_bo) {
  97                                        .flags = flags,
  98                                        .handle = bo->gem_handle,
  99                                        .presumed = bo->iova,
 100                                     });
 101 }
 102
 103 VkResult
 104 tu_bo_list_merge(struct tu_bo_list *list, const struct tu_bo_list *other)
 105 {
 106    for (uint32_t i = 0; i < other->count; i++) {
 107       if (tu_bo_list_add_info(list, other->bo_infos + i) == TU_BO_LIST_FAILED)
 108          return VK_ERROR_OUT_OF_HOST_MEMORY;
 109    }
 110
 111    return VK_SUCCESS;
 112 }
 113
 114 static void
 115 tu_tiling_config_update_tile_layout(struct tu_tiling_config *tiling,
 116                                     const struct tu_device *dev,
 117                                     uint32_t pixels)
 118 {
 119    const uint32_t tile_align_w = dev->physical_device->tile_align_w;
 120    const uint32_t tile_align_h = dev->physical_device->tile_align_h;
 121    const uint32_t max_tile_width = 1024; /* A6xx */
 122
 123    tiling->tile0.offset = (VkOffset2D) {
 124       .x = tiling->render_area.offset.x & ~(tile_align_w - 1),
 125       .y = tiling->render_area.offset.y & ~(tile_align_h - 1),
 126    };
 127
 128    const uint32_t ra_width =
 129       tiling->render_area.extent.width +
 130       (tiling->render_area.offset.x - tiling->tile0.offset.x);
 131    const uint32_t ra_height =
 132       tiling->render_area.extent.height +
 133       (tiling->render_area.offset.y - tiling->tile0.offset.y);
 134
 135    /* start from 1 tile */
 136    tiling->tile_count = (VkExtent2D) {
 137       .width = 1,
 138       .height = 1,
 139    };
 140    tiling->tile0.extent = (VkExtent2D) {
 141       .width = align(ra_width, tile_align_w),
 142       .height = align(ra_height, tile_align_h),
 143    };
 144
 145    /* do not exceed max tile width */
 146    while (tiling->tile0.extent.width > max_tile_width) {
 147       tiling->tile_count.width++;
 148       tiling->tile0.extent.width =
 149          align(ra_width / tiling->tile_count.width, tile_align_w);
 150    }
 151
 152    /* do not exceed gmem size */
 153    while (tiling->tile0.extent.width * tiling->tile0.extent.height > pixels) {
 154       if (tiling->tile0.extent.width > MAX2(tile_align_w, tiling->tile0.extent.height)) {
 155          tiling->tile_count.width++;
 156          tiling->tile0.extent.width =
 157             align(DIV_ROUND_UP(ra_width, tiling->tile_count.width), tile_align_w);
 158       } else {
 159          /* if this assert fails then layout is impossible.. */
 160          assert(tiling->tile0.extent.height > tile_align_h);
 161          tiling->tile_count.height++;
 162          tiling->tile0.extent.height =
 163             align(DIV_ROUND_UP(ra_height, tiling->tile_count.height), tile_align_h);
 164       }
 165    }
 166 }
 167
 168 static void
 169 tu_tiling_config_update_pipe_layout(struct tu_tiling_config *tiling,
 170                                     const struct tu_device *dev)
 171 {
 172    const uint32_t max_pipe_count = 32; /* A6xx */
 173
 174    /* start from 1 tile per pipe */
 175    tiling->pipe0 = (VkExtent2D) {
 176       .width = 1,
 177       .height = 1,
 178    };
 179    tiling->pipe_count = tiling->tile_count;
 180
 181    /* do not exceed max pipe count vertically */
 182    while (tiling->pipe_count.height > max_pipe_count) {
 183       tiling->pipe0.height += 2;
 184       tiling->pipe_count.height =
 185          (tiling->tile_count.height + tiling->pipe0.height - 1) /
 186          tiling->pipe0.height;
 187    }
 188
 189    /* do not exceed max pipe count */
 190    while (tiling->pipe_count.width * tiling->pipe_count.height >
 191           max_pipe_count) {
 192       tiling->pipe0.width += 1;
 193       tiling->pipe_count.width =
 194          (tiling->tile_count.width + tiling->pipe0.width - 1) /
 195          tiling->pipe0.width;
 196    }
 197 }
 198
 199 static void
 200 tu_tiling_config_update_pipes(struct tu_tiling_config *tiling,
 201                               const struct tu_device *dev)
 202 {
 203    const uint32_t max_pipe_count = 32; /* A6xx */
 204    const uint32_t used_pipe_count =
 205       tiling->pipe_count.width * tiling->pipe_count.height;
 206    const VkExtent2D last_pipe = {
 207       .width = (tiling->tile_count.width - 1) % tiling->pipe0.width + 1,
 208       .height = (tiling->tile_count.height - 1) % tiling->pipe0.height + 1,
 209    };
 210
 211    assert(used_pipe_count <= max_pipe_count);
 212    assert(max_pipe_count <= ARRAY_SIZE(tiling->pipe_config));
 213
 214    for (uint32_t y = 0; y < tiling->pipe_count.height; y++) {
 215       for (uint32_t x = 0; x < tiling->pipe_count.width; x++) {
 216          const uint32_t pipe_x = tiling->pipe0.width * x;
 217          const uint32_t pipe_y = tiling->pipe0.height * y;
 218          const uint32_t pipe_w = (x == tiling->pipe_count.width - 1)
 219                                     ? last_pipe.width
 220                                     : tiling->pipe0.width;
 221          const uint32_t pipe_h = (y == tiling->pipe_count.height - 1)
 222                                     ? last_pipe.height
 223                                     : tiling->pipe0.height;
 224          const uint32_t n = tiling->pipe_count.width * y + x;
 225
 226          tiling->pipe_config[n] = A6XX_VSC_PIPE_CONFIG_REG_X(pipe_x) |
 227                                   A6XX_VSC_PIPE_CONFIG_REG_Y(pipe_y) |
 228                                   A6XX_VSC_PIPE_CONFIG_REG_W(pipe_w) |
 229                                   A6XX_VSC_PIPE_CONFIG_REG_H(pipe_h);
 230          tiling->pipe_sizes[n] = CP_SET_BIN_DATA5_0_VSC_SIZE(pipe_w * pipe_h);
 231       }
 232    }
 233
 234    memset(tiling->pipe_config + used_pipe_count, 0,
 235           sizeof(uint32_t) * (max_pipe_count - used_pipe_count));
 236 }
 237
 238 static void
 239 tu_tiling_config_get_tile(const struct tu_tiling_config *tiling,
 240                           const struct tu_device *dev,
 241                           uint32_t tx,
 242                           uint32_t ty,
 243                           struct tu_tile *tile)
 244 {
 245    /* find the pipe and the slot for tile (tx, ty) */
 246    const uint32_t px = tx / tiling->pipe0.width;
 247    const uint32_t py = ty / tiling->pipe0.height;
 248    const uint32_t sx = tx - tiling->pipe0.width * px;
 249    const uint32_t sy = ty - tiling->pipe0.height * py;
 250
 251    assert(tx < tiling->tile_count.width && ty < tiling->tile_count.height);
 252    assert(px < tiling->pipe_count.width && py < tiling->pipe_count.height);
 253    assert(sx < tiling->pipe0.width && sy < tiling->pipe0.height);
 254
 255    /* convert to 1D indices */
 256    tile->pipe = tiling->pipe_count.width * py + px;
 257    tile->slot = tiling->pipe0.width * sy + sx;
 258
 259    /* get the blit area for the tile */
 260    tile->begin = (VkOffset2D) {
 261       .x = tiling->tile0.offset.x + tiling->tile0.extent.width * tx,
 262       .y = tiling->tile0.offset.y + tiling->tile0.extent.height * ty,
 263    };
 264    tile->end.x =
 265       (tx == tiling->tile_count.width - 1)
 266          ? tiling->render_area.offset.x + tiling->render_area.extent.width
 267          : tile->begin.x + tiling->tile0.extent.width;
 268    tile->end.y =
 269       (ty == tiling->tile_count.height - 1)
 270          ? tiling->render_area.offset.y + tiling->render_area.extent.height
 271          : tile->begin.y + tiling->tile0.extent.height;
 272 }
 273
 274 enum a3xx_msaa_samples
 275 tu_msaa_samples(uint32_t samples)
 276 {
 277    switch (samples) {
 278    case 1:
 279       return MSAA_ONE;
 280    case 2:
 281       return MSAA_TWO;
 282    case 4:
 283       return MSAA_FOUR;
 284    case 8:
 285       return MSAA_EIGHT;
 286    default:
 287       assert(!"invalid sample count");
 288       return MSAA_ONE;
 289    }
 290 }
 291
 292 static enum a4xx_index_size
 293 tu6_index_size(VkIndexType type)
 294 {
 295    switch (type) {
 296    case VK_INDEX_TYPE_UINT16:
 297       return INDEX4_SIZE_16_BIT;
 298    case VK_INDEX_TYPE_UINT32:
 299       return INDEX4_SIZE_32_BIT;
 300    default:
 301       unreachable("invalid VkIndexType");
 302       return INDEX4_SIZE_8_BIT;
 303    }
 304 }
 305
 306 static void
 307 tu6_emit_marker(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 308 {
 309    tu_cs_emit_write_reg(cs, cmd->marker_reg, ++cmd->marker_seqno);
 310 }
 311
 312 unsigned
 313 tu6_emit_event_write(struct tu_cmd_buffer *cmd,
 314                      struct tu_cs *cs,
 315                      enum vgt_event_type event,
 316                      bool need_seqno)
 317 {
 318    unsigned seqno = 0;
 319
 320    tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, need_seqno ? 4 : 1);
 321    tu_cs_emit(cs, CP_EVENT_WRITE_0_EVENT(event));
 322    if (need_seqno) {
 323       tu_cs_emit_qw(cs, cmd->scratch_bo.iova);
 324       seqno = ++cmd->scratch_seqno;
 325       tu_cs_emit(cs, seqno);
 326    }
 327
 328    return seqno;
 329 }
 330
 331 static void
 332 tu6_emit_cache_flush(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 333 {
 334    tu6_emit_event_write(cmd, cs, 0x31, false);
 335 }
 336
 337 static void
 338 tu6_emit_lrz_flush(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 339 {
 340    tu6_emit_event_write(cmd, cs, LRZ_FLUSH, false);
 341 }
 342
 343 static void
 344 tu6_emit_wfi(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 345 {
 346    if (cmd->wait_for_idle) {
 347       tu_cs_emit_wfi(cs);
 348       cmd->wait_for_idle = false;
 349    }
 350 }
 351
 352 #define tu_image_view_ubwc_pitches(iview)                                \
 353    .pitch = tu_image_ubwc_pitch(iview->image, iview->base_mip),          \
 354    .array_pitch = tu_image_ubwc_size(iview->image, iview->base_mip) >> 2
 355
 356 static void
 357 tu6_emit_zs(struct tu_cmd_buffer *cmd,
 358             const struct tu_subpass *subpass,
 359             struct tu_cs *cs)
 360 {
 361    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 362
 363    const uint32_t a = subpass->depth_stencil_attachment.attachment;
 364    if (a == VK_ATTACHMENT_UNUSED) {
 365       tu_cs_emit_regs(cs,
 366                       A6XX_RB_DEPTH_BUFFER_INFO(.depth_format = DEPTH6_NONE),
 367                       A6XX_RB_DEPTH_BUFFER_PITCH(0),
 368                       A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH(0),
 369                       A6XX_RB_DEPTH_BUFFER_BASE(0),
 370                       A6XX_RB_DEPTH_BUFFER_BASE_GMEM(0));
 371
 372       tu_cs_emit_regs(cs,
 373                       A6XX_GRAS_SU_DEPTH_BUFFER_INFO(.depth_format = DEPTH6_NONE));
 374
 375       tu_cs_emit_regs(cs,
 376                       A6XX_GRAS_LRZ_BUFFER_BASE(0),
 377                       A6XX_GRAS_LRZ_BUFFER_PITCH(0),
 378                       A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE(0));
 379
 380       tu_cs_emit_regs(cs, A6XX_RB_STENCIL_INFO(0));
 381
 382       return;
 383    }
 384
 385    const struct tu_image_view *iview = fb->attachments[a].attachment;
 386    enum a6xx_depth_format fmt = tu6_pipe2depth(iview->vk_format);
 387
 388    tu_cs_emit_regs(cs,
 389                    A6XX_RB_DEPTH_BUFFER_INFO(.depth_format = fmt),
 390                    A6XX_RB_DEPTH_BUFFER_PITCH(tu_image_stride(iview->image, iview->base_mip)),
 391                    A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH(iview->image->layout.layer_size),
 392                    A6XX_RB_DEPTH_BUFFER_BASE(tu_image_view_base_ref(iview)),
 393                    A6XX_RB_DEPTH_BUFFER_BASE_GMEM(cmd->state.pass->attachments[a].gmem_offset));
 394
 395    tu_cs_emit_regs(cs,
 396                    A6XX_GRAS_SU_DEPTH_BUFFER_INFO(.depth_format = fmt));
 397
 398    tu_cs_emit_regs(cs,
 399                    A6XX_RB_DEPTH_FLAG_BUFFER_BASE(tu_image_view_ubwc_base_ref(iview)),
 400                    A6XX_RB_DEPTH_FLAG_BUFFER_PITCH(tu_image_view_ubwc_pitches(iview)));
 401
 402    tu_cs_emit_regs(cs,
 403                    A6XX_GRAS_LRZ_BUFFER_BASE(0),
 404                    A6XX_GRAS_LRZ_BUFFER_PITCH(0),
 405                    A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE(0));
 406
 407    tu_cs_emit_regs(cs,
 408                    A6XX_RB_STENCIL_INFO(0));
 409
 410    /* enable zs? */
 411 }
 412
 413 static void
 414 tu6_emit_mrt(struct tu_cmd_buffer *cmd,
 415              const struct tu_subpass *subpass,
 416              struct tu_cs *cs)
 417 {
 418    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 419    unsigned char mrt_comp[MAX_RTS] = { 0 };
 420    unsigned srgb_cntl = 0;
 421
 422    for (uint32_t i = 0; i < subpass->color_count; ++i) {
 423       uint32_t a = subpass->color_attachments[i].attachment;
 424       if (a == VK_ATTACHMENT_UNUSED)
 425          continue;
 426
 427       const struct tu_image_view *iview = fb->attachments[a].attachment;
 428       const enum a6xx_tile_mode tile_mode =
 429          tu6_get_image_tile_mode(iview->image, iview->base_mip);
 430
 431       mrt_comp[i] = 0xf;
 432
 433       if (vk_format_is_srgb(iview->vk_format))
 434          srgb_cntl |= (1 << i);
 435
 436       const struct tu_native_format *format =
 437          tu6_get_native_format(iview->vk_format);
 438       assert(format && format->rb >= 0);
 439
 440       tu_cs_emit_regs(cs,
 441                       A6XX_RB_MRT_BUF_INFO(i,
 442                                            .color_tile_mode = tile_mode,
 443                                            .color_format = format->rb,
 444                                            .color_swap = format->swap),
 445                       A6XX_RB_MRT_PITCH(i, tu_image_stride(iview->image, iview->base_mip)),
 446                       A6XX_RB_MRT_ARRAY_PITCH(i, iview->image->layout.layer_size),
 447                       A6XX_RB_MRT_BASE(i, tu_image_view_base_ref(iview)),
 448                       A6XX_RB_MRT_BASE_GMEM(i, cmd->state.pass->attachments[a].gmem_offset));
 449
 450       tu_cs_emit_regs(cs,
 451                       A6XX_SP_FS_MRT_REG(i,
 452                                          .color_format = format->rb,
 453                                          .color_sint = vk_format_is_sint(iview->vk_format),
 454                                          .color_uint = vk_format_is_uint(iview->vk_format)));
 455
 456       tu_cs_emit_regs(cs,
 457                       A6XX_RB_MRT_FLAG_BUFFER_ADDR(i, tu_image_view_ubwc_base_ref(iview)),
 458                       A6XX_RB_MRT_FLAG_BUFFER_PITCH(i, tu_image_view_ubwc_pitches(iview)));
 459    }
 460
 461    tu_cs_emit_regs(cs,
 462                    A6XX_RB_SRGB_CNTL(srgb_cntl));
 463
 464    tu_cs_emit_regs(cs,
 465                    A6XX_SP_SRGB_CNTL(srgb_cntl));
 466
 467    tu_cs_emit_regs(cs,
 468                    A6XX_RB_RENDER_COMPONENTS(
 469                       .rt0 = mrt_comp[0],
 470                       .rt1 = mrt_comp[1],
 471                       .rt2 = mrt_comp[2],
 472                       .rt3 = mrt_comp[3],
 473                       .rt4 = mrt_comp[4],
 474                       .rt5 = mrt_comp[5],
 475                       .rt6 = mrt_comp[6],
 476                       .rt7 = mrt_comp[7]));
 477
 478    tu_cs_emit_regs(cs,
 479                    A6XX_SP_FS_RENDER_COMPONENTS(
 480                       .rt0 = mrt_comp[0],
 481                       .rt1 = mrt_comp[1],
 482                       .rt2 = mrt_comp[2],
 483                       .rt3 = mrt_comp[3],
 484                       .rt4 = mrt_comp[4],
 485                       .rt5 = mrt_comp[5],
 486                       .rt6 = mrt_comp[6],
 487                       .rt7 = mrt_comp[7]));
 488 }
 489
 490 static void
 491 tu6_emit_msaa(struct tu_cmd_buffer *cmd,
 492               const struct tu_subpass *subpass,
 493               struct tu_cs *cs)
 494 {
 495    const enum a3xx_msaa_samples samples = tu_msaa_samples(subpass->samples);
 496    bool msaa_disable = samples == MSAA_ONE;
 497
 498    tu_cs_emit_regs(cs,
 499                    A6XX_SP_TP_RAS_MSAA_CNTL(samples),
 500                    A6XX_SP_TP_DEST_MSAA_CNTL(.samples = samples,
 501                                              .msaa_disable = msaa_disable));
 502
 503    tu_cs_emit_regs(cs,
 504                    A6XX_GRAS_RAS_MSAA_CNTL(samples),
 505                    A6XX_GRAS_DEST_MSAA_CNTL(.samples = samples,
 506                                             .msaa_disable = msaa_disable));
 507
 508    tu_cs_emit_regs(cs,
 509                    A6XX_RB_RAS_MSAA_CNTL(samples),
 510                    A6XX_RB_DEST_MSAA_CNTL(.samples = samples,
 511                                           .msaa_disable = msaa_disable));
 512
 513    tu_cs_emit_regs(cs,
 514                    A6XX_RB_MSAA_CNTL(samples));
 515 }
 516
 517 static void
 518 tu6_emit_bin_size(struct tu_cs *cs,
 519                   uint32_t bin_w, uint32_t bin_h, uint32_t flags)
 520 {
 521    tu_cs_emit_regs(cs,
 522                    A6XX_GRAS_BIN_CONTROL(.binw = bin_w,
 523                                          .binh = bin_h,
 524                                          .dword = flags));
 525
 526    tu_cs_emit_regs(cs,
 527                    A6XX_RB_BIN_CONTROL(.binw = bin_w,
 528                                        .binh = bin_h,
 529                                        .dword = flags));
 530
 531    /* no flag for RB_BIN_CONTROL2... */
 532    tu_cs_emit_regs(cs,
 533                    A6XX_RB_BIN_CONTROL2(.binw = bin_w,
 534                                         .binh = bin_h));
 535 }
 536
 537 static void
 538 tu6_emit_render_cntl(struct tu_cmd_buffer *cmd,
 539                      const struct tu_subpass *subpass,
 540                      struct tu_cs *cs,
 541                      bool binning)
 542 {
 543    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 544    uint32_t cntl = 0;
 545    cntl |= A6XX_RB_RENDER_CNTL_UNK4;
 546    if (binning) {
 547       cntl |= A6XX_RB_RENDER_CNTL_BINNING;
 548    } else {
 549       uint32_t mrts_ubwc_enable = 0;
 550       for (uint32_t i = 0; i < subpass->color_count; ++i) {
 551          uint32_t a = subpass->color_attachments[i].attachment;
 552          if (a == VK_ATTACHMENT_UNUSED)
 553             continue;
 554
 555          const struct tu_image_view *iview = fb->attachments[a].attachment;
 556          if (iview->image->layout.ubwc_layer_size != 0)
 557             mrts_ubwc_enable |= 1 << i;
 558       }
 559
 560       cntl |= A6XX_RB_RENDER_CNTL_FLAG_MRTS(mrts_ubwc_enable);
 561
 562       const uint32_t a = subpass->depth_stencil_attachment.attachment;
 563       if (a != VK_ATTACHMENT_UNUSED) {
 564          const struct tu_image_view *iview = fb->attachments[a].attachment;
 565          if (iview->image->layout.ubwc_layer_size != 0)
 566             cntl |= A6XX_RB_RENDER_CNTL_FLAG_DEPTH;
 567       }
 568
 569       /* In the !binning case, we need to set RB_RENDER_CNTL in the draw_cs
 570        * in order to set it correctly for the different subpasses. However,
 571        * that means the packets we're emitting also happen during binning. So
 572        * we need to guard the write on !BINNING at CP execution time.
 573        */
 574       tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
 575       tu_cs_emit(cs, CP_COND_REG_EXEC_0_MODE(RENDER_MODE) |
 576                      CP_COND_REG_EXEC_0_GMEM | CP_COND_REG_EXEC_0_SYSMEM);
 577       tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(4));
 578    }
 579
 580    tu_cs_emit_pkt7(cs, CP_REG_WRITE, 3);
 581    tu_cs_emit(cs, CP_REG_WRITE_0_TRACKER(TRACK_RENDER_CNTL));
 582    tu_cs_emit(cs, REG_A6XX_RB_RENDER_CNTL);
 583    tu_cs_emit(cs, cntl);
 584 }
 585
 586 static void
 587 tu6_emit_blit_scissor(struct tu_cmd_buffer *cmd, struct tu_cs *cs, bool align)
 588 {
 589    const VkRect2D *render_area = &cmd->state.tiling_config.render_area;
 590    uint32_t x1 = render_area->offset.x;
 591    uint32_t y1 = render_area->offset.y;
 592    uint32_t x2 = x1 + render_area->extent.width - 1;
 593    uint32_t y2 = y1 + render_area->extent.height - 1;
 594
 595    /* TODO: alignment requirement seems to be less than tile_align_w/h */
 596    if (align) {
 597       x1 = x1 & ~cmd->device->physical_device->tile_align_w;
 598       y1 = y1 & ~cmd->device->physical_device->tile_align_h;
 599       x2 = ALIGN_POT(x2 + 1, cmd->device->physical_device->tile_align_w) - 1;
 600       y2 = ALIGN_POT(y2 + 1, cmd->device->physical_device->tile_align_h) - 1;
 601    }
 602
 603    tu_cs_emit_regs(cs,
 604                    A6XX_RB_BLIT_SCISSOR_TL(.x = x1, .y = y1),
 605                    A6XX_RB_BLIT_SCISSOR_BR(.x = x2, .y = y2));
 606 }
 607
 608 static void
 609 tu6_emit_blit_info(struct tu_cmd_buffer *cmd,
 610                    struct tu_cs *cs,
 611                    const struct tu_image_view *iview,
 612                    uint32_t gmem_offset,
 613                    bool resolve)
 614 {
 615    tu_cs_emit_regs(cs,
 616                    A6XX_RB_BLIT_INFO(.unk0 = !resolve, .gmem = !resolve));
 617
 618    const struct tu_native_format *format =
 619       tu6_get_native_format(iview->vk_format);
 620    assert(format && format->rb >= 0);
 621
 622    enum a6xx_tile_mode tile_mode =
 623       tu6_get_image_tile_mode(iview->image, iview->base_mip);
 624    tu_cs_emit_regs(cs,
 625                    A6XX_RB_BLIT_DST_INFO(
 626                       .tile_mode = tile_mode,
 627                       .samples = tu_msaa_samples(iview->image->samples),
 628                       .color_format = format->rb,
 629                       .color_swap = format->swap,
 630                       .flags = iview->image->layout.ubwc_layer_size != 0),
 631                    A6XX_RB_BLIT_DST(tu_image_view_base_ref(iview)),
 632                    A6XX_RB_BLIT_DST_PITCH(tu_image_stride(iview->image, iview->base_mip)),
 633                    A6XX_RB_BLIT_DST_ARRAY_PITCH(iview->image->layout.layer_size));
 634
 635    if (iview->image->layout.ubwc_layer_size) {
 636       tu_cs_emit_regs(cs,
 637                       A6XX_RB_BLIT_FLAG_DST(tu_image_view_ubwc_base_ref(iview)),
 638                       A6XX_RB_BLIT_FLAG_DST_PITCH(tu_image_view_ubwc_pitches(iview)));
 639    }
 640
 641    tu_cs_emit_regs(cs,
 642                    A6XX_RB_BLIT_BASE_GMEM(gmem_offset));
 643 }
 644
 645 static void
 646 tu6_emit_blit(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 647 {
 648    tu6_emit_marker(cmd, cs);
 649    tu6_emit_event_write(cmd, cs, BLIT, false);
 650    tu6_emit_marker(cmd, cs);
 651 }
 652
 653 static void
 654 tu6_emit_window_scissor(struct tu_cmd_buffer *cmd,
 655                         struct tu_cs *cs,
 656                         uint32_t x1,
 657                         uint32_t y1,
 658                         uint32_t x2,
 659                         uint32_t y2)
 660 {
 661    tu_cs_emit_regs(cs,
 662                    A6XX_GRAS_SC_WINDOW_SCISSOR_TL(.x = x1, .y = y1),
 663                    A6XX_GRAS_SC_WINDOW_SCISSOR_BR(.x = x2, .y = y2));
 664
 665    tu_cs_emit_regs(cs,
 666                    A6XX_GRAS_RESOLVE_CNTL_1(.x = x1, .y = y1),
 667                    A6XX_GRAS_RESOLVE_CNTL_2(.x = x2, .y = y2));
 668 }
 669
 670 static void
 671 tu6_emit_window_offset(struct tu_cmd_buffer *cmd,
 672                        struct tu_cs *cs,
 673                        uint32_t x1,
 674                        uint32_t y1)
 675 {
 676    tu_cs_emit_regs(cs,
 677                    A6XX_RB_WINDOW_OFFSET(.x = x1, .y = y1));
 678
 679    tu_cs_emit_regs(cs,
 680                    A6XX_RB_WINDOW_OFFSET2(.x = x1, .y = y1));
 681
 682    tu_cs_emit_regs(cs,
 683                    A6XX_SP_WINDOW_OFFSET(.x = x1, .y = y1));
 684
 685    tu_cs_emit_regs(cs,
 686                    A6XX_SP_TP_WINDOW_OFFSET(.x = x1, .y = y1));
 687 }
 688
 689 static bool
 690 use_hw_binning(struct tu_cmd_buffer *cmd)
 691 {
 692    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
 693
 694    if (unlikely(cmd->device->physical_device->instance->debug_flags & TU_DEBUG_NOBIN))
 695       return false;
 696
 697    return (tiling->tile_count.width * tiling->tile_count.height) > 2;
 698 }
 699
 700 static bool
 701 use_sysmem_rendering(struct tu_cmd_buffer *cmd)
 702 {
 703    if (unlikely(cmd->device->physical_device->instance->debug_flags & TU_DEBUG_SYSMEM))
 704       return true;
 705
 706    return false;
 707 }
 708
 709 static void
 710 tu6_emit_tile_select(struct tu_cmd_buffer *cmd,
 711                      struct tu_cs *cs,
 712                      const struct tu_tile *tile)
 713 {
 714    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
 715    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(0x7));
 716
 717    tu6_emit_marker(cmd, cs);
 718    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
 719    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_GMEM) | 0x10);
 720    tu6_emit_marker(cmd, cs);
 721
 722    const uint32_t x1 = tile->begin.x;
 723    const uint32_t y1 = tile->begin.y;
 724    const uint32_t x2 = tile->end.x - 1;
 725    const uint32_t y2 = tile->end.y - 1;
 726    tu6_emit_window_scissor(cmd, cs, x1, y1, x2, y2);
 727    tu6_emit_window_offset(cmd, cs, x1, y1);
 728
 729    tu_cs_emit_regs(cs,
 730                    A6XX_VPC_SO_OVERRIDE(.so_disable = true));
 731
 732    if (use_hw_binning(cmd)) {
 733       tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
 734
 735       tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
 736       tu_cs_emit(cs, 0x0);
 737
 738       tu_cs_emit_pkt7(cs, CP_REG_TEST, 1);
 739       tu_cs_emit(cs, A6XX_CP_REG_TEST_0_REG(OVERFLOW_FLAG_REG) |
 740                      A6XX_CP_REG_TEST_0_BIT(0) |
 741                      A6XX_CP_REG_TEST_0_WAIT_FOR_ME);
 742
 743       tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
 744       tu_cs_emit(cs, CP_COND_REG_EXEC_0_MODE(PRED_TEST));
 745       tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(11));
 746
 747       /* if (no overflow) */ {
 748          tu_cs_emit_pkt7(cs, CP_SET_BIN_DATA5, 7);
 749          tu_cs_emit(cs, cmd->state.tiling_config.pipe_sizes[tile->pipe] |
 750                         CP_SET_BIN_DATA5_0_VSC_N(tile->slot));
 751          tu_cs_emit_qw(cs, cmd->vsc_data.iova + tile->pipe * cmd->vsc_data_pitch);
 752          tu_cs_emit_qw(cs, cmd->vsc_data.iova + (tile->pipe * 4) + (32 * cmd->vsc_data_pitch));
 753          tu_cs_emit_qw(cs, cmd->vsc_data2.iova + (tile->pipe * cmd->vsc_data2_pitch));
 754
 755          tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
 756          tu_cs_emit(cs, 0x0);
 757
 758          /* use a NOP packet to skip over the 'else' side: */
 759          tu_cs_emit_pkt7(cs, CP_NOP, 2);
 760       } /* else */ {
 761          tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
 762          tu_cs_emit(cs, 0x1);
 763       }
 764
 765       tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
 766       tu_cs_emit(cs, 0x0);
 767
 768       tu_cs_emit_regs(cs,
 769                       A6XX_RB_UNKNOWN_8804(0));
 770
 771       tu_cs_emit_regs(cs,
 772                       A6XX_SP_TP_UNKNOWN_B304(0));
 773
 774       tu_cs_emit_regs(cs,
 775                       A6XX_GRAS_UNKNOWN_80A4(0));
 776    } else {
 777       tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
 778       tu_cs_emit(cs, 0x1);
 779
 780       tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
 781       tu_cs_emit(cs, 0x0);
 782    }
 783 }
 784
 785 static void
 786 tu6_emit_load_attachment(struct tu_cmd_buffer *cmd, struct tu_cs *cs, uint32_t a)
 787 {
 788    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
 789    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 790    const struct tu_image_view *iview = fb->attachments[a].attachment;
 791    const struct tu_render_pass_attachment *attachment =
 792       &cmd->state.pass->attachments[a];
 793
 794    if (attachment->gmem_offset < 0)
 795       return;
 796
 797    const uint32_t x1 = tiling->render_area.offset.x;
 798    const uint32_t y1 = tiling->render_area.offset.y;
 799    const uint32_t x2 = x1 + tiling->render_area.extent.width;
 800    const uint32_t y2 = y1 + tiling->render_area.extent.height;
 801    const uint32_t tile_x2 =
 802       tiling->tile0.offset.x + tiling->tile0.extent.width * tiling->tile_count.width;
 803    const uint32_t tile_y2 =
 804       tiling->tile0.offset.y + tiling->tile0.extent.height * tiling->tile_count.height;
 805    bool need_load =
 806       x1 != tiling->tile0.offset.x || x2 != MIN2(fb->width, tile_x2) ||
 807       y1 != tiling->tile0.offset.y || y2 != MIN2(fb->height, tile_y2);
 808
 809    if (need_load)
 810       tu_finishme("improve handling of unaligned render area");
 811
 812    if (attachment->load_op == VK_ATTACHMENT_LOAD_OP_LOAD)
 813       need_load = true;
 814
 815    if (vk_format_has_stencil(iview->vk_format) &&
 816        attachment->stencil_load_op == VK_ATTACHMENT_LOAD_OP_LOAD)
 817       need_load = true;
 818
 819    if (need_load) {
 820       tu6_emit_blit_info(cmd, cs, iview, attachment->gmem_offset, false);
 821       tu6_emit_blit(cmd, cs);
 822    }
 823 }
 824
 825 static void
 826 tu6_emit_clear_attachment(struct tu_cmd_buffer *cmd, struct tu_cs *cs,
 827                           uint32_t a,
 828                           const VkRenderPassBeginInfo *info)
 829 {
 830    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 831    const struct tu_image_view *iview = fb->attachments[a].attachment;
 832    const struct tu_render_pass_attachment *attachment =
 833       &cmd->state.pass->attachments[a];
 834    unsigned clear_mask = 0;
 835
 836    /* note: this means it isn't used by any subpass and shouldn't be cleared anyway */
 837    if (attachment->gmem_offset < 0)
 838       return;
 839
 840    if (attachment->load_op == VK_ATTACHMENT_LOAD_OP_CLEAR)
 841       clear_mask = 0xf;
 842
 843    if (vk_format_has_stencil(iview->vk_format)) {
 844       clear_mask &= 0x1;
 845       if (attachment->stencil_load_op == VK_ATTACHMENT_LOAD_OP_CLEAR)
 846          clear_mask |= 0x2;
 847    }
 848    if (!clear_mask)
 849       return;
 850
 851    const struct tu_native_format *format =
 852       tu6_get_native_format(iview->vk_format);
 853    assert(format && format->rb >= 0);
 854
 855    tu_cs_emit_regs(cs,
 856                    A6XX_RB_BLIT_DST_INFO(.color_format = format->rb));
 857
 858    tu_cs_emit_regs(cs,
 859                    A6XX_RB_BLIT_INFO(.gmem = true,
 860                                      .clear_mask = clear_mask));
 861
 862    tu_cs_emit_regs(cs,
 863                    A6XX_RB_BLIT_BASE_GMEM(attachment->gmem_offset));
 864
 865    tu_cs_emit_regs(cs,
 866                    A6XX_RB_UNKNOWN_88D0(0));
 867
 868    uint32_t clear_vals[4] = { 0 };
 869    tu_pack_clear_value(&info->pClearValues[a], iview->vk_format, clear_vals);
 870
 871    tu_cs_emit_regs(cs,
 872                    A6XX_RB_BLIT_CLEAR_COLOR_DW0(clear_vals[0]),
 873                    A6XX_RB_BLIT_CLEAR_COLOR_DW1(clear_vals[1]),
 874                    A6XX_RB_BLIT_CLEAR_COLOR_DW2(clear_vals[2]),
 875                    A6XX_RB_BLIT_CLEAR_COLOR_DW3(clear_vals[3]));
 876
 877    tu6_emit_blit(cmd, cs);
 878 }
 879
 880 static void
 881 tu6_emit_store_attachment(struct tu_cmd_buffer *cmd,
 882                           struct tu_cs *cs,
 883                           uint32_t a,
 884                           uint32_t gmem_a)
 885 {
 886    if (cmd->state.pass->attachments[a].store_op == VK_ATTACHMENT_STORE_OP_DONT_CARE)
 887       return;
 888
 889    tu6_emit_blit_info(cmd, cs,
 890                       cmd->state.framebuffer->attachments[a].attachment,
 891                       cmd->state.pass->attachments[gmem_a].gmem_offset, true);
 892    tu6_emit_blit(cmd, cs);
 893 }
 894
 895 static void
 896 tu6_emit_tile_store(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 897 {
 898    const struct tu_render_pass *pass = cmd->state.pass;
 899    const struct tu_subpass *subpass = &pass->subpasses[pass->subpass_count-1];
 900
 901    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
 902    tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
 903                      CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
 904                      CP_SET_DRAW_STATE__0_GROUP_ID(0));
 905    tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
 906    tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
 907
 908    tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
 909    tu_cs_emit(cs, 0x0);
 910
 911    tu6_emit_marker(cmd, cs);
 912    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
 913    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_RESOLVE) | 0x10);
 914    tu6_emit_marker(cmd, cs);
 915
 916    tu6_emit_blit_scissor(cmd, cs, true);
 917
 918    for (uint32_t a = 0; a < pass->attachment_count; ++a) {
 919       if (pass->attachments[a].gmem_offset >= 0)
 920          tu6_emit_store_attachment(cmd, cs, a, a);
 921    }
 922
 923    if (subpass->resolve_attachments) {
 924       for (unsigned i = 0; i < subpass->color_count; i++) {
 925          uint32_t a = subpass->resolve_attachments[i].attachment;
 926          if (a != VK_ATTACHMENT_UNUSED)
 927             tu6_emit_store_attachment(cmd, cs, a,
 928                                       subpass->color_attachments[i].attachment);
 929       }
 930    }
 931 }
 932
 933 static void
 934 tu6_emit_restart_index(struct tu_cs *cs, uint32_t restart_index)
 935 {
 936    tu_cs_emit_regs(cs,
 937                    A6XX_PC_RESTART_INDEX(restart_index));
 938 }
 939
 940 static void
 941 tu6_init_hw(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 942 {
 943    VkResult result = tu_cs_reserve_space(cmd->device, cs, 256);
 944    if (result != VK_SUCCESS) {
 945       cmd->record_result = result;
 946       return;
 947    }
 948
 949    tu6_emit_cache_flush(cmd, cs);
 950
 951    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UPDATE_CNTL, 0xfffff);
 952
 953    tu_cs_emit_write_reg(cs, REG_A6XX_RB_CCU_CNTL, 0x10000000);
 954    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8E04, 0x00100000);
 955    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE04, 0x8);
 956    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE00, 0);
 957    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE0F, 0x3f);
 958    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B605, 0x44);
 959    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B600, 0x100000);
 960    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE00, 0x80);
 961    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE01, 0);
 962
 963    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9600, 0);
 964    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8600, 0x880);
 965    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE04, 0);
 966    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE03, 0x00000410);
 967    tu_cs_emit_write_reg(cs, REG_A6XX_SP_IBO_COUNT, 0);
 968    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B182, 0);
 969    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BB11, 0);
 970    tu_cs_emit_write_reg(cs, REG_A6XX_UCHE_UNKNOWN_0E12, 0x3200000);
 971    tu_cs_emit_write_reg(cs, REG_A6XX_UCHE_CLIENT_PF, 4);
 972    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8E01, 0x0);
 973    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AB00, 0x5);
 974    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_ADD_OFFSET, A6XX_VFD_ADD_OFFSET_VERTEX);
 975    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8811, 0x00000010);
 976    tu_cs_emit_write_reg(cs, REG_A6XX_PC_MODE_CNTL, 0x1f);
 977
 978    tu_cs_emit_write_reg(cs, REG_A6XX_RB_SRGB_CNTL, 0);
 979
 980    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8101, 0);
 981    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_SAMPLE_CNTL, 0);
 982    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8110, 0);
 983
 984    tu_cs_emit_write_reg(cs, REG_A6XX_RB_RENDER_CONTROL0, 0x401);
 985    tu_cs_emit_write_reg(cs, REG_A6XX_RB_RENDER_CONTROL1, 0);
 986    tu_cs_emit_write_reg(cs, REG_A6XX_RB_FS_OUTPUT_CNTL0, 0);
 987    tu_cs_emit_write_reg(cs, REG_A6XX_RB_SAMPLE_CNTL, 0);
 988    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8818, 0);
 989    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8819, 0);
 990    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881A, 0);
 991    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881B, 0);
 992    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881C, 0);
 993    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881D, 0);
 994    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881E, 0);
 995    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_88F0, 0);
 996
 997    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9101, 0xffff00);
 998    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9107, 0);
 999
1000    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9236, 1);
1001    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9300, 0);
1002
1003    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_SO_OVERRIDE,
1004                         A6XX_VPC_SO_OVERRIDE_SO_DISABLE);
1005
1006    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9801, 0);
1007    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9806, 0);
1008    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9980, 0);
1009
1010    tu_cs_emit_write_reg(cs, REG_A6XX_PC_PRIMITIVE_CNTL_6, 0);
1011    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9B07, 0);
1012
1013    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_A81B, 0);
1014
1015    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B183, 0);
1016
1017    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8099, 0);
1018    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_809B, 0);
1019    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A0, 2);
1020    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80AF, 0);
1021    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9210, 0);
1022    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9211, 0);
1023    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9602, 0);
1024    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9981, 0x3);
1025    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9E72, 0);
1026    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9108, 0x3);
1027    tu_cs_emit_write_reg(cs, REG_A6XX_SP_TP_UNKNOWN_B304, 0);
1028    tu_cs_emit_write_reg(cs, REG_A6XX_SP_TP_UNKNOWN_B309, 0x000000a2);
1029    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8804, 0);
1030    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A4, 0);
1031    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A5, 0);
1032    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A6, 0);
1033    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8805, 0);
1034    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8806, 0);
1035    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8878, 0);
1036    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8879, 0);
1037    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_CONTROL_5_REG, 0xfc);
1038
1039    tu6_emit_marker(cmd, cs);
1040
1041    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_MODE_CNTL, 0x00000000);
1042
1043    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_UNKNOWN_A008, 0);
1044
1045    tu_cs_emit_write_reg(cs, REG_A6XX_PC_MODE_CNTL, 0x0000001f);
1046
1047    /* we don't use this yet.. probably best to disable.. */
1048    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
1049    tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
1050                      CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
1051                      CP_SET_DRAW_STATE__0_GROUP_ID(0));
1052    tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
1053    tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
1054
1055    tu_cs_emit_regs(cs,
1056                    A6XX_VPC_SO_BUFFER_BASE(0),
1057                    A6XX_VPC_SO_BUFFER_SIZE(0));
1058
1059    tu_cs_emit_regs(cs,
1060                    A6XX_VPC_SO_FLUSH_BASE(0));
1061
1062    tu_cs_emit_regs(cs,
1063                    A6XX_VPC_SO_BUF_CNTL(0));
1064
1065    tu_cs_emit_regs(cs,
1066                    A6XX_VPC_SO_BUFFER_OFFSET(0, 0));
1067
1068    tu_cs_emit_regs(cs,
1069                    A6XX_VPC_SO_BUFFER_BASE(1, 0),
1070                    A6XX_VPC_SO_BUFFER_SIZE(1, 0));
1071
1072    tu_cs_emit_regs(cs,
1073                    A6XX_VPC_SO_BUFFER_OFFSET(1, 0),
1074                    A6XX_VPC_SO_FLUSH_BASE(1, 0),
1075                    A6XX_VPC_SO_BUFFER_BASE(2, 0),
1076                    A6XX_VPC_SO_BUFFER_SIZE(2, 0));
1077
1078    tu_cs_emit_regs(cs,
1079                    A6XX_VPC_SO_BUFFER_OFFSET(2, 0),
1080                    A6XX_VPC_SO_FLUSH_BASE(2, 0),
1081                    A6XX_VPC_SO_BUFFER_BASE(3, 0),
1082                    A6XX_VPC_SO_BUFFER_SIZE(3, 0));
1083
1084    tu_cs_emit_regs(cs,
1085                    A6XX_VPC_SO_BUFFER_OFFSET(3, 0),
1086                    A6XX_VPC_SO_FLUSH_BASE(3, 0));
1087
1088    tu_cs_emit_regs(cs,
1089                    A6XX_SP_HS_CTRL_REG0(0));
1090
1091    tu_cs_emit_regs(cs,
1092                    A6XX_SP_GS_CTRL_REG0(0));
1093
1094    tu_cs_emit_regs(cs,
1095                    A6XX_GRAS_LRZ_CNTL(0));
1096
1097    tu_cs_emit_regs(cs,
1098                    A6XX_RB_LRZ_CNTL(0));
1099
1100    tu_cs_sanity_check(cs);
1101 }
1102
1103 static void
1104 tu6_cache_flush(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1105 {
1106    unsigned seqno;
1107
1108    seqno = tu6_emit_event_write(cmd, cs, CACHE_FLUSH_AND_INV_EVENT, true);
1109
1110    tu_cs_emit_pkt7(cs, CP_WAIT_REG_MEM, 6);
1111    tu_cs_emit(cs, CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ) |
1112                   CP_WAIT_REG_MEM_0_POLL_MEMORY);
1113    tu_cs_emit_qw(cs, cmd->scratch_bo.iova);
1114    tu_cs_emit(cs, CP_WAIT_REG_MEM_3_REF(seqno));
1115    tu_cs_emit(cs, CP_WAIT_REG_MEM_4_MASK(~0));
1116    tu_cs_emit(cs, CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(16));
1117
1118    seqno = tu6_emit_event_write(cmd, cs, CACHE_FLUSH_TS, true);
1119
1120    tu_cs_emit_pkt7(cs, CP_WAIT_MEM_GTE, 4);
1121    tu_cs_emit(cs, CP_WAIT_MEM_GTE_0_RESERVED(0));
1122    tu_cs_emit_qw(cs, cmd->scratch_bo.iova);
1123    tu_cs_emit(cs, CP_WAIT_MEM_GTE_3_REF(seqno));
1124 }
1125
1126 static void
1127 update_vsc_pipe(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1128 {
1129    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1130
1131    tu_cs_emit_regs(cs,
1132                    A6XX_VSC_BIN_SIZE(.width = tiling->tile0.extent.width,
1133                                      .height = tiling->tile0.extent.height),
1134                    A6XX_VSC_SIZE_ADDRESS(.bo = &cmd->vsc_data,
1135                                          .bo_offset = 32 * cmd->vsc_data_pitch));
1136
1137    tu_cs_emit_regs(cs,
1138                    A6XX_VSC_BIN_COUNT(.nx = tiling->tile_count.width,
1139                                       .ny = tiling->tile_count.height));
1140
1141    tu_cs_emit_pkt4(cs, REG_A6XX_VSC_PIPE_CONFIG_REG(0), 32);
1142    for (unsigned i = 0; i < 32; i++)
1143       tu_cs_emit(cs, tiling->pipe_config[i]);
1144
1145    tu_cs_emit_regs(cs,
1146                    A6XX_VSC_PIPE_DATA2_ADDRESS(.bo = &cmd->vsc_data2),
1147                    A6XX_VSC_PIPE_DATA2_PITCH(cmd->vsc_data2_pitch),
1148                    A6XX_VSC_PIPE_DATA2_ARRAY_PITCH(cmd->vsc_data2.size));
1149
1150    tu_cs_emit_regs(cs,
1151                    A6XX_VSC_PIPE_DATA_ADDRESS(.bo = &cmd->vsc_data),
1152                    A6XX_VSC_PIPE_DATA_PITCH(cmd->vsc_data_pitch),
1153                    A6XX_VSC_PIPE_DATA_ARRAY_PITCH(cmd->vsc_data.size));
1154 }
1155
1156 static void
1157 emit_vsc_overflow_test(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1158 {
1159    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1160    const uint32_t used_pipe_count =
1161       tiling->pipe_count.width * tiling->pipe_count.height;
1162
1163    /* Clear vsc_scratch: */
1164    tu_cs_emit_pkt7(cs, CP_MEM_WRITE, 3);
1165    tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1166    tu_cs_emit(cs, 0x0);
1167
1168    /* Check for overflow, write vsc_scratch if detected: */
1169    for (int i = 0; i < used_pipe_count; i++) {
1170       tu_cs_emit_pkt7(cs, CP_COND_WRITE5, 8);
1171       tu_cs_emit(cs, CP_COND_WRITE5_0_FUNCTION(WRITE_GE) |
1172             CP_COND_WRITE5_0_WRITE_MEMORY);
1173       tu_cs_emit(cs, CP_COND_WRITE5_1_POLL_ADDR_LO(REG_A6XX_VSC_SIZE_REG(i)));
1174       tu_cs_emit(cs, CP_COND_WRITE5_2_POLL_ADDR_HI(0));
1175       tu_cs_emit(cs, CP_COND_WRITE5_3_REF(cmd->vsc_data_pitch));
1176       tu_cs_emit(cs, CP_COND_WRITE5_4_MASK(~0));
1177       tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1178       tu_cs_emit(cs, CP_COND_WRITE5_7_WRITE_DATA(1 + cmd->vsc_data_pitch));
1179
1180       tu_cs_emit_pkt7(cs, CP_COND_WRITE5, 8);
1181       tu_cs_emit(cs, CP_COND_WRITE5_0_FUNCTION(WRITE_GE) |
1182             CP_COND_WRITE5_0_WRITE_MEMORY);
1183       tu_cs_emit(cs, CP_COND_WRITE5_1_POLL_ADDR_LO(REG_A6XX_VSC_SIZE2_REG(i)));
1184       tu_cs_emit(cs, CP_COND_WRITE5_2_POLL_ADDR_HI(0));
1185       tu_cs_emit(cs, CP_COND_WRITE5_3_REF(cmd->vsc_data2_pitch));
1186       tu_cs_emit(cs, CP_COND_WRITE5_4_MASK(~0));
1187       tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1188       tu_cs_emit(cs, CP_COND_WRITE5_7_WRITE_DATA(3 + cmd->vsc_data2_pitch));
1189    }
1190
1191    tu_cs_emit_pkt7(cs, CP_WAIT_MEM_WRITES, 0);
1192
1193    tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
1194
1195    tu_cs_emit_pkt7(cs, CP_MEM_TO_REG, 3);
1196    tu_cs_emit(cs, CP_MEM_TO_REG_0_REG(OVERFLOW_FLAG_REG) |
1197          CP_MEM_TO_REG_0_CNT(1 - 1));
1198    tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1199
1200    /*
1201     * This is a bit awkward, we really want a way to invert the
1202     * CP_REG_TEST/CP_COND_REG_EXEC logic, so that we can conditionally
1203     * execute cmds to use hwbinning when a bit is *not* set.  This
1204     * dance is to invert OVERFLOW_FLAG_REG
1205     *
1206     * A CP_NOP packet is used to skip executing the 'else' clause
1207     * if (b0 set)..
1208     */
1209
1210    /* b0 will be set if VSC_DATA or VSC_DATA2 overflow: */
1211    tu_cs_emit_pkt7(cs, CP_REG_TEST, 1);
1212    tu_cs_emit(cs, A6XX_CP_REG_TEST_0_REG(OVERFLOW_FLAG_REG) |
1213          A6XX_CP_REG_TEST_0_BIT(0) |
1214          A6XX_CP_REG_TEST_0_WAIT_FOR_ME);
1215
1216    tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
1217    tu_cs_emit(cs, CP_COND_REG_EXEC_0_MODE(PRED_TEST));
1218    tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(7));
1219
1220    /* if (b0 set) */ {
1221       /*
1222        * On overflow, mirror the value to control->vsc_overflow
1223        * which CPU is checking to detect overflow (see
1224        * check_vsc_overflow())
1225        */
1226       tu_cs_emit_pkt7(cs, CP_REG_TO_MEM, 3);
1227       tu_cs_emit(cs, CP_REG_TO_MEM_0_REG(OVERFLOW_FLAG_REG) |
1228             CP_REG_TO_MEM_0_CNT(0));
1229       tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_OVERFLOW);
1230
1231       tu_cs_emit_pkt4(cs, OVERFLOW_FLAG_REG, 1);
1232       tu_cs_emit(cs, 0x0);
1233
1234       tu_cs_emit_pkt7(cs, CP_NOP, 2);  /* skip 'else' when 'if' is taken */
1235    } /* else */ {
1236       tu_cs_emit_pkt4(cs, OVERFLOW_FLAG_REG, 1);
1237       tu_cs_emit(cs, 0x1);
1238    }
1239 }
1240
1241 static void
1242 tu6_emit_binning_pass(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1243 {
1244    struct tu_physical_device *phys_dev = cmd->device->physical_device;
1245    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1246
1247    uint32_t x1 = tiling->tile0.offset.x;
1248    uint32_t y1 = tiling->tile0.offset.y;
1249    uint32_t x2 = tiling->render_area.offset.x + tiling->render_area.extent.width - 1;
1250    uint32_t y2 = tiling->render_area.offset.y + tiling->render_area.extent.height - 1;
1251
1252    tu6_emit_window_scissor(cmd, cs, x1, y1, x2, y2);
1253
1254    tu6_emit_marker(cmd, cs);
1255    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1256    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_BINNING));
1257    tu6_emit_marker(cmd, cs);
1258
1259    tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
1260    tu_cs_emit(cs, 0x1);
1261
1262    tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
1263    tu_cs_emit(cs, 0x1);
1264
1265    tu_cs_emit_wfi(cs);
1266
1267    tu_cs_emit_regs(cs,
1268                    A6XX_VFD_MODE_CNTL(.binning_pass = true));
1269
1270    update_vsc_pipe(cmd, cs);
1271
1272    tu_cs_emit_regs(cs,
1273                    A6XX_PC_UNKNOWN_9805(.unknown = phys_dev->magic.PC_UNKNOWN_9805));
1274
1275    tu_cs_emit_regs(cs,
1276                    A6XX_SP_UNKNOWN_A0F8(.unknown = phys_dev->magic.SP_UNKNOWN_A0F8));
1277
1278    tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, 1);
1279    tu_cs_emit(cs, UNK_2C);
1280
1281    tu_cs_emit_regs(cs,
1282                    A6XX_RB_WINDOW_OFFSET(.x = 0, .y = 0));
1283
1284    tu_cs_emit_regs(cs,
1285                    A6XX_SP_TP_WINDOW_OFFSET(.x = 0, .y = 0));
1286
1287    /* emit IB to binning drawcmds: */
1288    tu_cs_emit_call(cs, &cmd->draw_cs);
1289
1290    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
1291    tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
1292                   CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
1293                   CP_SET_DRAW_STATE__0_GROUP_ID(0));
1294    tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
1295    tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
1296
1297    tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, 1);
1298    tu_cs_emit(cs, UNK_2D);
1299
1300    tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE, false);
1301    tu6_cache_flush(cmd, cs);
1302
1303    tu_cs_emit_wfi(cs);
1304
1305    tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
1306
1307    emit_vsc_overflow_test(cmd, cs);
1308
1309    tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
1310    tu_cs_emit(cs, 0x0);
1311
1312    tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
1313    tu_cs_emit(cs, 0x0);
1314
1315    tu_cs_emit_wfi(cs);
1316
1317    tu_cs_emit_regs(cs,
1318                    A6XX_RB_CCU_CNTL(.unknown = phys_dev->magic.RB_CCU_CNTL_gmem));
1319
1320    cmd->wait_for_idle = false;
1321 }
1322
1323 static inline struct tu_blit_surf
1324 sysmem_clear_surf(const struct tu_image_view *view, const VkRect2D *render_area)
1325 {
1326    return tu_blit_surf_ext(view->image, (VkImageSubresourceLayers) {
1327       .mipLevel = view->base_mip,
1328       .baseArrayLayer = view->base_layer,
1329    }, (VkOffset3D) {
1330       .x = render_area->offset.x,
1331       .y = render_area->offset.y,
1332       .z = 0,
1333    }, (VkExtent3D) {
1334       .width = render_area->extent.width,
1335       .height = render_area->extent.height,
1336       .depth = 1,
1337    });
1338 }
1339
1340 static void
1341 tu6_emit_sysmem_clear_attachment(struct tu_cmd_buffer *cmd, struct tu_cs *cs,
1342                                  uint32_t a,
1343                                  const VkRenderPassBeginInfo *info)
1344 {
1345    const struct tu_framebuffer *fb = cmd->state.framebuffer;
1346    const struct tu_image_view *iview = fb->attachments[a].attachment;
1347    const struct tu_render_pass_attachment *attachment =
1348       &cmd->state.pass->attachments[a];
1349    unsigned clear_mask = 0;
1350
1351    /* note: this means it isn't used by any subpass and shouldn't be cleared anyway */
1352    if (attachment->gmem_offset < 0)
1353       return;
1354
1355    uint32_t clear_vals[4] = { 0 };
1356
1357    if (attachment->load_op == VK_ATTACHMENT_LOAD_OP_CLEAR) {
1358       clear_mask = 0xf;
1359    }
1360
1361    if (vk_format_has_stencil(iview->vk_format)) {
1362       clear_mask &= 0x1;
1363       if (attachment->stencil_load_op == VK_ATTACHMENT_LOAD_OP_CLEAR)
1364          clear_mask |= 0x2;
1365       if (clear_mask != 0x3)
1366          tu_finishme("depth/stencil only load op");
1367    }
1368
1369    if (!clear_mask)
1370       return;
1371
1372    if (iview->aspect_mask & (VK_IMAGE_ASPECT_DEPTH_BIT |
1373                              VK_IMAGE_ASPECT_STENCIL_BIT)) {
1374       tu_2d_clear_zs(&info->pClearValues[a].depthStencil, iview->vk_format,
1375                      clear_vals);
1376    } else {
1377       tu_2d_clear_color(&info->pClearValues[a].color, iview->vk_format,
1378                         clear_vals);
1379    }
1380
1381    tu_blit(cmd, cs, &(struct tu_blit) {
1382       .dst = sysmem_clear_surf(iview, &info->renderArea),
1383       .layers = iview->layer_count,
1384       .clear_value = { clear_vals[0], clear_vals[1], clear_vals[2], clear_vals[3] },
1385       .type = TU_BLIT_CLEAR,
1386    });
1387 }
1388
1389 static void
1390 tu_cmd_prepare_sysmem_clear_ib(struct tu_cmd_buffer *cmd,
1391                                const VkRenderPassBeginInfo *info)
1392 {
1393    const struct tu_framebuffer *fb = cmd->state.framebuffer;
1394    const uint32_t blit_cmd_space = 25 + 66 * fb->layers + 17;
1395    const uint32_t clear_space =
1396        blit_cmd_space * cmd->state.pass->attachment_count + 5;
1397
1398    struct tu_cs sub_cs;
1399
1400    VkResult result = tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs,
1401                                             clear_space, &sub_cs);
1402    if (result != VK_SUCCESS) {
1403       cmd->record_result = result;
1404       return;
1405    }
1406
1407    for (uint32_t i = 0; i < cmd->state.pass->attachment_count; ++i)
1408       tu6_emit_sysmem_clear_attachment(cmd, &sub_cs, i, info);
1409
1410    /* TODO: We shouldn't need this flush, but without it we'd have an empty IB
1411     * when nothing clears which we currently can't handle.
1412     */
1413    tu_cs_reserve_space(cmd->device, &sub_cs, 5);
1414    tu6_emit_event_write(cmd, &sub_cs, UNK_1D, true);
1415
1416    cmd->state.sysmem_clear_ib = tu_cs_end_sub_stream(&cmd->sub_cs, &sub_cs);
1417 }
1418
1419 static void
1420 tu6_sysmem_render_begin(struct tu_cmd_buffer *cmd, struct tu_cs *cs,
1421                         const struct VkRect2D *renderArea)
1422 {
1423    VkResult result = tu_cs_reserve_space(cmd->device, cs, 1024);
1424    if (result != VK_SUCCESS) {
1425       cmd->record_result = result;
1426       return;
1427    }
1428
1429    const struct tu_framebuffer *fb = cmd->state.framebuffer;
1430    if (fb->width > 0 && fb->height > 0) {
1431       tu6_emit_window_scissor(cmd, cs,
1432                               0, 0, fb->width - 1, fb->height - 1);
1433    } else {
1434       tu6_emit_window_scissor(cmd, cs, 0, 0, 0, 0);
1435    }
1436
1437    tu6_emit_window_offset(cmd, cs, 0, 0);
1438
1439    tu6_emit_bin_size(cs, 0, 0, 0xc00000); /* 0xc00000 = BYPASS? */
1440
1441    tu_cs_emit_ib(cs, &cmd->state.sysmem_clear_ib);
1442
1443    tu6_emit_lrz_flush(cmd, cs);
1444
1445    tu6_emit_marker(cmd, cs);
1446    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1447    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_BYPASS) | 0x10);
1448    tu6_emit_marker(cmd, cs);
1449
1450    tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1451    tu_cs_emit(cs, 0x0);
1452
1453    tu6_emit_event_write(cmd, cs, PC_CCU_INVALIDATE_COLOR, false);
1454    tu6_emit_event_write(cmd, cs, PC_CCU_INVALIDATE_DEPTH, false);
1455    tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE, false);
1456
1457    tu6_emit_wfi(cmd, cs);
1458    tu_cs_emit_regs(cs,
1459                    A6XX_RB_CCU_CNTL(0x10000000));
1460
1461    /* enable stream-out, with sysmem there is only one pass: */
1462    tu_cs_emit_regs(cs,
1463                    A6XX_VPC_SO_OVERRIDE(.so_disable = false));
1464
1465    tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
1466    tu_cs_emit(cs, 0x1);
1467
1468    tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
1469    tu_cs_emit(cs, 0x0);
1470
1471    tu_cs_sanity_check(cs);
1472 }
1473
1474 static void
1475 tu6_sysmem_render_end(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1476 {
1477    const uint32_t space = 14 + tu_cs_get_call_size(&cmd->draw_epilogue_cs);
1478    VkResult result = tu_cs_reserve_space(cmd->device, cs, space);
1479    if (result != VK_SUCCESS) {
1480       cmd->record_result = result;
1481       return;
1482    }
1483
1484    tu_cs_emit_call(cs, &cmd->draw_epilogue_cs);
1485
1486    tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1487    tu_cs_emit(cs, 0x0);
1488
1489    tu6_emit_lrz_flush(cmd, cs);
1490
1491    tu6_emit_event_write(cmd, cs, UNK_1C, true);
1492    tu6_emit_event_write(cmd, cs, UNK_1D, true);
1493
1494    tu_cs_sanity_check(cs);
1495 }
1496
1497
1498 static void
1499 tu6_tile_render_begin(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1500 {
1501    struct tu_physical_device *phys_dev = cmd->device->physical_device;
1502
1503    VkResult result = tu_cs_reserve_space(cmd->device, cs, 1024);
1504    if (result != VK_SUCCESS) {
1505       cmd->record_result = result;
1506       return;
1507    }
1508
1509    tu6_emit_lrz_flush(cmd, cs);
1510
1511    /* lrz clear? */
1512
1513    tu6_emit_cache_flush(cmd, cs);
1514
1515    tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1516    tu_cs_emit(cs, 0x0);
1517
1518    /* 0x10000000 for BYPASS.. 0x7c13c080 for GMEM: */
1519    tu6_emit_wfi(cmd, cs);
1520    tu_cs_emit_regs(cs,
1521                    A6XX_RB_CCU_CNTL(phys_dev->magic.RB_CCU_CNTL_gmem));
1522
1523    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1524    if (use_hw_binning(cmd)) {
1525       tu6_emit_bin_size(cs,
1526                         tiling->tile0.extent.width,
1527                         tiling->tile0.extent.height,
1528                         A6XX_RB_BIN_CONTROL_BINNING_PASS | 0x6000000);
1529
1530       tu6_emit_render_cntl(cmd, cmd->state.subpass, cs, true);
1531
1532       tu6_emit_binning_pass(cmd, cs);
1533
1534       tu6_emit_bin_size(cs,
1535                         tiling->tile0.extent.width,
1536                         tiling->tile0.extent.height,
1537                         A6XX_RB_BIN_CONTROL_USE_VIZ | 0x6000000);
1538
1539       tu_cs_emit_regs(cs,
1540                       A6XX_VFD_MODE_CNTL(0));
1541
1542       tu_cs_emit_regs(cs, A6XX_PC_UNKNOWN_9805(.unknown = phys_dev->magic.PC_UNKNOWN_9805));
1543
1544       tu_cs_emit_regs(cs, A6XX_SP_UNKNOWN_A0F8(.unknown = phys_dev->magic.SP_UNKNOWN_A0F8));
1545
1546       tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1547       tu_cs_emit(cs, 0x1);
1548    } else {
1549       tu6_emit_bin_size(cs,
1550                         tiling->tile0.extent.width,
1551                         tiling->tile0.extent.height,
1552                         0x6000000);
1553    }
1554
1555    tu_cs_sanity_check(cs);
1556 }
1557
1558 static void
1559 tu6_render_tile(struct tu_cmd_buffer *cmd,
1560                 struct tu_cs *cs,
1561                 const struct tu_tile *tile)
1562 {
1563    const uint32_t render_tile_space = 256 + tu_cs_get_call_size(&cmd->draw_cs);
1564    VkResult result = tu_cs_reserve_space(cmd->device, cs, render_tile_space);
1565    if (result != VK_SUCCESS) {
1566       cmd->record_result = result;
1567       return;
1568    }
1569
1570    tu6_emit_tile_select(cmd, cs, tile);
1571    tu_cs_emit_ib(cs, &cmd->state.tile_load_ib);
1572
1573    tu_cs_emit_call(cs, &cmd->draw_cs);
1574    cmd->wait_for_idle = true;
1575
1576    if (use_hw_binning(cmd)) {
1577       tu_cs_emit_pkt7(cs, CP_REG_TEST, 1);
1578       tu_cs_emit(cs, A6XX_CP_REG_TEST_0_REG(OVERFLOW_FLAG_REG) |
1579                      A6XX_CP_REG_TEST_0_BIT(0) |
1580                      A6XX_CP_REG_TEST_0_WAIT_FOR_ME);
1581
1582       tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
1583       tu_cs_emit(cs, 0x10000000);
1584       tu_cs_emit(cs, 2);  /* conditionally execute next 2 dwords */
1585
1586       /* if (no overflow) */ {
1587          tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1588          tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(0x5) | 0x10);
1589       }
1590    }
1591
1592    tu_cs_emit_ib(cs, &cmd->state.tile_store_ib);
1593
1594    tu_cs_sanity_check(cs);
1595 }
1596
1597 static void
1598 tu6_tile_render_end(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1599 {
1600    const uint32_t space = 16 + tu_cs_get_call_size(&cmd->draw_epilogue_cs);
1601    VkResult result = tu_cs_reserve_space(cmd->device, cs, space);
1602    if (result != VK_SUCCESS) {
1603       cmd->record_result = result;
1604       return;
1605    }
1606
1607    tu_cs_emit_call(cs, &cmd->draw_epilogue_cs);
1608
1609    tu_cs_emit_regs(cs,
1610                    A6XX_GRAS_LRZ_CNTL(0));
1611
1612    tu6_emit_lrz_flush(cmd, cs);
1613
1614    tu6_emit_event_write(cmd, cs, CACHE_FLUSH_TS, true);
1615
1616    tu_cs_sanity_check(cs);
1617 }
1618
1619 static void
1620 tu_cmd_render_tiles(struct tu_cmd_buffer *cmd)
1621 {
1622    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1623
1624    tu6_tile_render_begin(cmd, &cmd->cs);
1625
1626    for (uint32_t y = 0; y < tiling->tile_count.height; y++) {
1627       for (uint32_t x = 0; x < tiling->tile_count.width; x++) {
1628          struct tu_tile tile;
1629          tu_tiling_config_get_tile(tiling, cmd->device, x, y, &tile);
1630          tu6_render_tile(cmd, &cmd->cs, &tile);
1631       }
1632    }
1633
1634    tu6_tile_render_end(cmd, &cmd->cs);
1635 }
1636
1637 static void
1638 tu_cmd_render_sysmem(struct tu_cmd_buffer *cmd)
1639 {
1640    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1641
1642    tu6_sysmem_render_begin(cmd, &cmd->cs, &tiling->render_area);
1643
1644    const uint32_t space = tu_cs_get_call_size(&cmd->draw_cs);
1645    VkResult result = tu_cs_reserve_space(cmd->device, &cmd->cs, space);
1646    if (result != VK_SUCCESS) {
1647       cmd->record_result = result;
1648       return;
1649    }
1650
1651    tu_cs_emit_call(&cmd->cs, &cmd->draw_cs);
1652    cmd->wait_for_idle = true;
1653
1654    tu6_sysmem_render_end(cmd, &cmd->cs);
1655 }
1656
1657 static void
1658 tu_cmd_prepare_tile_load_ib(struct tu_cmd_buffer *cmd,
1659                             const VkRenderPassBeginInfo *info)
1660 {
1661    const uint32_t tile_load_space =
1662       2 * 3 /* blit_scissor */ +
1663       (20 /* load */ + 19 /* clear */) * cmd->state.pass->attachment_count +
1664       2 /* cache invalidate */;
1665
1666    struct tu_cs sub_cs;
1667
1668    VkResult result = tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs,
1669                                             tile_load_space, &sub_cs);
1670    if (result != VK_SUCCESS) {
1671       cmd->record_result = result;
1672       return;
1673    }
1674
1675    tu6_emit_blit_scissor(cmd, &sub_cs, true);
1676
1677    for (uint32_t i = 0; i < cmd->state.pass->attachment_count; ++i)
1678       tu6_emit_load_attachment(cmd, &sub_cs, i);
1679
1680    tu6_emit_blit_scissor(cmd, &sub_cs, false);
1681
1682    for (uint32_t i = 0; i < cmd->state.pass->attachment_count; ++i)
1683       tu6_emit_clear_attachment(cmd, &sub_cs, i, info);
1684
1685    /* invalidate because reading input attachments will cache GMEM and
1686     * the cache isn''t updated when GMEM is written
1687     * TODO: is there a no-cache bit for textures?
1688     */
1689    if (cmd->state.subpass->input_count)
1690       tu6_emit_event_write(cmd, &sub_cs, CACHE_INVALIDATE, false);
1691
1692    cmd->state.tile_load_ib = tu_cs_end_sub_stream(&cmd->sub_cs, &sub_cs);
1693 }
1694
1695 static void
1696 tu_cmd_prepare_tile_store_ib(struct tu_cmd_buffer *cmd)
1697 {
1698    const uint32_t tile_store_space = 32 + 23 * cmd->state.pass->attachment_count;
1699    struct tu_cs sub_cs;
1700
1701    VkResult result = tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs,
1702                                             tile_store_space, &sub_cs);
1703    if (result != VK_SUCCESS) {
1704       cmd->record_result = result;
1705       return;
1706    }
1707
1708    /* emit to tile-store sub_cs */
1709    tu6_emit_tile_store(cmd, &sub_cs);
1710
1711    cmd->state.tile_store_ib = tu_cs_end_sub_stream(&cmd->sub_cs, &sub_cs);
1712 }
1713
1714 static void
1715 tu_cmd_update_tiling_config(struct tu_cmd_buffer *cmd,
1716                             const VkRect2D *render_area)
1717 {
1718    const struct tu_device *dev = cmd->device;
1719    struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1720
1721    tiling->render_area = *render_area;
1722
1723    tu_tiling_config_update_tile_layout(tiling, dev, cmd->state.pass->gmem_pixels);
1724    tu_tiling_config_update_pipe_layout(tiling, dev);
1725    tu_tiling_config_update_pipes(tiling, dev);
1726 }
1727
1728 const struct tu_dynamic_state default_dynamic_state = {
1729    .viewport =
1730      {
1731        .count = 0,
1732      },
1733    .scissor =
1734      {
1735        .count = 0,
1736      },
1737    .line_width = 1.0f,
1738    .depth_bias =
1739      {
1740        .bias = 0.0f,
1741        .clamp = 0.0f,
1742        .slope = 0.0f,
1743      },
1744    .blend_constants = { 0.0f, 0.0f, 0.0f, 0.0f },
1745    .depth_bounds =
1746      {
1747        .min = 0.0f,
1748        .max = 1.0f,
1749      },
1750    .stencil_compare_mask =
1751      {
1752        .front = ~0u,
1753        .back = ~0u,
1754      },
1755    .stencil_write_mask =
1756      {
1757        .front = ~0u,
1758        .back = ~0u,
1759      },
1760    .stencil_reference =
1761      {
1762        .front = 0u,
1763        .back = 0u,
1764      },
1765 };
1766
1767 static void UNUSED /* FINISHME */
1768 tu_bind_dynamic_state(struct tu_cmd_buffer *cmd_buffer,
1769                       const struct tu_dynamic_state *src)
1770 {
1771    struct tu_dynamic_state *dest = &cmd_buffer->state.dynamic;
1772    uint32_t copy_mask = src->mask;
1773    uint32_t dest_mask = 0;
1774
1775    tu_use_args(cmd_buffer); /* FINISHME */
1776
1777    /* Make sure to copy the number of viewports/scissors because they can
1778     * only be specified at pipeline creation time.
1779     */
1780    dest->viewport.count = src->viewport.count;
1781    dest->scissor.count = src->scissor.count;
1782    dest->discard_rectangle.count = src->discard_rectangle.count;
1783
1784    if (copy_mask & TU_DYNAMIC_VIEWPORT) {
1785       if (memcmp(&dest->viewport.viewports, &src->viewport.viewports,
1786                  src->viewport.count * sizeof(VkViewport))) {
1787          typed_memcpy(dest->viewport.viewports, src->viewport.viewports,
1788                       src->viewport.count);
1789          dest_mask |= TU_DYNAMIC_VIEWPORT;
1790       }
1791    }
1792
1793    if (copy_mask & TU_DYNAMIC_SCISSOR) {
1794       if (memcmp(&dest->scissor.scissors, &src->scissor.scissors,
1795                  src->scissor.count * sizeof(VkRect2D))) {
1796          typed_memcpy(dest->scissor.scissors, src->scissor.scissors,
1797                       src->scissor.count);
1798          dest_mask |= TU_DYNAMIC_SCISSOR;
1799       }
1800    }
1801
1802    if (copy_mask & TU_DYNAMIC_LINE_WIDTH) {
1803       if (dest->line_width != src->line_width) {
1804          dest->line_width = src->line_width;
1805          dest_mask |= TU_DYNAMIC_LINE_WIDTH;
1806       }
1807    }
1808
1809    if (copy_mask & TU_DYNAMIC_DEPTH_BIAS) {
1810       if (memcmp(&dest->depth_bias, &src->depth_bias,
1811                  sizeof(src->depth_bias))) {
1812          dest->depth_bias = src->depth_bias;
1813          dest_mask |= TU_DYNAMIC_DEPTH_BIAS;
1814       }
1815    }
1816
1817    if (copy_mask & TU_DYNAMIC_BLEND_CONSTANTS) {
1818       if (memcmp(&dest->blend_constants, &src->blend_constants,
1819                  sizeof(src->blend_constants))) {
1820          typed_memcpy(dest->blend_constants, src->blend_constants, 4);
1821          dest_mask |= TU_DYNAMIC_BLEND_CONSTANTS;
1822       }
1823    }
1824
1825    if (copy_mask & TU_DYNAMIC_DEPTH_BOUNDS) {
1826       if (memcmp(&dest->depth_bounds, &src->depth_bounds,
1827                  sizeof(src->depth_bounds))) {
1828          dest->depth_bounds = src->depth_bounds;
1829          dest_mask |= TU_DYNAMIC_DEPTH_BOUNDS;
1830       }
1831    }
1832
1833    if (copy_mask & TU_DYNAMIC_STENCIL_COMPARE_MASK) {
1834       if (memcmp(&dest->stencil_compare_mask, &src->stencil_compare_mask,
1835                  sizeof(src->stencil_compare_mask))) {
1836          dest->stencil_compare_mask = src->stencil_compare_mask;
1837          dest_mask |= TU_DYNAMIC_STENCIL_COMPARE_MASK;
1838       }
1839    }
1840
1841    if (copy_mask & TU_DYNAMIC_STENCIL_WRITE_MASK) {
1842       if (memcmp(&dest->stencil_write_mask, &src->stencil_write_mask,
1843                  sizeof(src->stencil_write_mask))) {
1844          dest->stencil_write_mask = src->stencil_write_mask;
1845          dest_mask |= TU_DYNAMIC_STENCIL_WRITE_MASK;
1846       }
1847    }
1848
1849    if (copy_mask & TU_DYNAMIC_STENCIL_REFERENCE) {
1850       if (memcmp(&dest->stencil_reference, &src->stencil_reference,
1851                  sizeof(src->stencil_reference))) {
1852          dest->stencil_reference = src->stencil_reference;
1853          dest_mask |= TU_DYNAMIC_STENCIL_REFERENCE;
1854       }
1855    }
1856
1857    if (copy_mask & TU_DYNAMIC_DISCARD_RECTANGLE) {
1858       if (memcmp(&dest->discard_rectangle.rectangles,
1859                  &src->discard_rectangle.rectangles,
1860                  src->discard_rectangle.count * sizeof(VkRect2D))) {
1861          typed_memcpy(dest->discard_rectangle.rectangles,
1862                       src->discard_rectangle.rectangles,
1863                       src->discard_rectangle.count);
1864          dest_mask |= TU_DYNAMIC_DISCARD_RECTANGLE;
1865       }
1866    }
1867 }
1868
1869 static VkResult
1870 tu_create_cmd_buffer(struct tu_device *device,
1871                      struct tu_cmd_pool *pool,
1872                      VkCommandBufferLevel level,
1873                      VkCommandBuffer *pCommandBuffer)
1874 {
1875    struct tu_cmd_buffer *cmd_buffer;
1876    cmd_buffer = vk_zalloc(&pool->alloc, sizeof(*cmd_buffer), 8,
1877                           VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1878    if (cmd_buffer == NULL)
1879       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
1880
1881    cmd_buffer->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
1882    cmd_buffer->device = device;
1883    cmd_buffer->pool = pool;
1884    cmd_buffer->level = level;
1885
1886    if (pool) {
1887       list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers);
1888       cmd_buffer->queue_family_index = pool->queue_family_index;
1889
1890    } else {
1891       /* Init the pool_link so we can safely call list_del when we destroy
1892        * the command buffer
1893        */
1894       list_inithead(&cmd_buffer->pool_link);
1895       cmd_buffer->queue_family_index = TU_QUEUE_GENERAL;
1896    }
1897
1898    tu_bo_list_init(&cmd_buffer->bo_list);
1899    tu_cs_init(&cmd_buffer->cs, TU_CS_MODE_GROW, 4096);
1900    tu_cs_init(&cmd_buffer->draw_cs, TU_CS_MODE_GROW, 4096);
1901    tu_cs_init(&cmd_buffer->draw_epilogue_cs, TU_CS_MODE_GROW, 4096);
1902    tu_cs_init(&cmd_buffer->sub_cs, TU_CS_MODE_SUB_STREAM, 2048);
1903
1904    *pCommandBuffer = tu_cmd_buffer_to_handle(cmd_buffer);
1905
1906    list_inithead(&cmd_buffer->upload.list);
1907
1908    cmd_buffer->marker_reg = REG_A6XX_CP_SCRATCH_REG(
1909       cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY ? 7 : 6);
1910
1911    VkResult result = tu_bo_init_new(device, &cmd_buffer->scratch_bo, 0x1000);
1912    if (result != VK_SUCCESS)
1913       goto fail_scratch_bo;
1914
1915    /* TODO: resize on overflow */
1916    cmd_buffer->vsc_data_pitch = device->vsc_data_pitch;
1917    cmd_buffer->vsc_data2_pitch = device->vsc_data2_pitch;
1918    cmd_buffer->vsc_data = device->vsc_data;
1919    cmd_buffer->vsc_data2 = device->vsc_data2;
1920
1921    return VK_SUCCESS;
1922
1923 fail_scratch_bo:
1924    list_del(&cmd_buffer->pool_link);
1925    return result;
1926 }
1927
1928 static void
1929 tu_cmd_buffer_destroy(struct tu_cmd_buffer *cmd_buffer)
1930 {
1931    tu_bo_finish(cmd_buffer->device, &cmd_buffer->scratch_bo);
1932
1933    list_del(&cmd_buffer->pool_link);
1934
1935    for (unsigned i = 0; i < VK_PIPELINE_BIND_POINT_RANGE_SIZE; i++)
1936       free(cmd_buffer->descriptors[i].push_set.set.mapped_ptr);
1937
1938    tu_cs_finish(cmd_buffer->device, &cmd_buffer->cs);
1939    tu_cs_finish(cmd_buffer->device, &cmd_buffer->draw_cs);
1940    tu_cs_finish(cmd_buffer->device, &cmd_buffer->draw_epilogue_cs);
1941    tu_cs_finish(cmd_buffer->device, &cmd_buffer->sub_cs);
1942
1943    tu_bo_list_destroy(&cmd_buffer->bo_list);
1944    vk_free(&cmd_buffer->pool->alloc, cmd_buffer);
1945 }
1946
1947 static VkResult
1948 tu_reset_cmd_buffer(struct tu_cmd_buffer *cmd_buffer)
1949 {
1950    cmd_buffer->wait_for_idle = true;
1951
1952    cmd_buffer->record_result = VK_SUCCESS;
1953
1954    tu_bo_list_reset(&cmd_buffer->bo_list);
1955    tu_cs_reset(cmd_buffer->device, &cmd_buffer->cs);
1956    tu_cs_reset(cmd_buffer->device, &cmd_buffer->draw_cs);
1957    tu_cs_reset(cmd_buffer->device, &cmd_buffer->draw_epilogue_cs);
1958    tu_cs_reset(cmd_buffer->device, &cmd_buffer->sub_cs);
1959
1960    for (unsigned i = 0; i < VK_PIPELINE_BIND_POINT_RANGE_SIZE; i++) {
1961       cmd_buffer->descriptors[i].valid = 0;
1962       cmd_buffer->descriptors[i].push_dirty = false;
1963    }
1964
1965    cmd_buffer->status = TU_CMD_BUFFER_STATUS_INITIAL;
1966
1967    return cmd_buffer->record_result;
1968 }
1969
1970 VkResult
1971 tu_AllocateCommandBuffers(VkDevice _device,
1972                           const VkCommandBufferAllocateInfo *pAllocateInfo,
1973                           VkCommandBuffer *pCommandBuffers)
1974 {
1975    TU_FROM_HANDLE(tu_device, device, _device);
1976    TU_FROM_HANDLE(tu_cmd_pool, pool, pAllocateInfo->commandPool);
1977
1978    VkResult result = VK_SUCCESS;
1979    uint32_t i;
1980
1981    for (i = 0; i < pAllocateInfo->commandBufferCount; i++) {
1982
1983       if (!list_is_empty(&pool->free_cmd_buffers)) {
1984          struct tu_cmd_buffer *cmd_buffer = list_first_entry(
1985             &pool->free_cmd_buffers, struct tu_cmd_buffer, pool_link);
1986
1987          list_del(&cmd_buffer->pool_link);
1988          list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers);
1989
1990          result = tu_reset_cmd_buffer(cmd_buffer);
1991          cmd_buffer->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
1992          cmd_buffer->level = pAllocateInfo->level;
1993
1994          pCommandBuffers[i] = tu_cmd_buffer_to_handle(cmd_buffer);
1995       } else {
1996          result = tu_create_cmd_buffer(device, pool, pAllocateInfo->level,
1997                                        &pCommandBuffers[i]);
1998       }
1999       if (result != VK_SUCCESS)
2000          break;
2001    }
2002
2003    if (result != VK_SUCCESS) {
2004       tu_FreeCommandBuffers(_device, pAllocateInfo->commandPool, i,
2005                             pCommandBuffers);
2006
2007       /* From the Vulkan 1.0.66 spec:
2008        *
2009        * "vkAllocateCommandBuffers can be used to create multiple
2010        *  command buffers. If the creation of any of those command
2011        *  buffers fails, the implementation must destroy all
2012        *  successfully created command buffer objects from this
2013        *  command, set all entries of the pCommandBuffers array to
2014        *  NULL and return the error."
2015        */
2016       memset(pCommandBuffers, 0,
2017              sizeof(*pCommandBuffers) * pAllocateInfo->commandBufferCount);
2018    }
2019
2020    return result;
2021 }
2022
2023 void
2024 tu_FreeCommandBuffers(VkDevice device,
2025                       VkCommandPool commandPool,
2026                       uint32_t commandBufferCount,
2027                       const VkCommandBuffer *pCommandBuffers)
2028 {
2029    for (uint32_t i = 0; i < commandBufferCount; i++) {
2030       TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, pCommandBuffers[i]);
2031
2032       if (cmd_buffer) {
2033          if (cmd_buffer->pool) {
2034             list_del(&cmd_buffer->pool_link);
2035             list_addtail(&cmd_buffer->pool_link,
2036                          &cmd_buffer->pool->free_cmd_buffers);
2037          } else
2038             tu_cmd_buffer_destroy(cmd_buffer);
2039       }
2040    }
2041 }
2042
2043 VkResult
2044 tu_ResetCommandBuffer(VkCommandBuffer commandBuffer,
2045                       VkCommandBufferResetFlags flags)
2046 {
2047    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
2048    return tu_reset_cmd_buffer(cmd_buffer);
2049 }
2050
2051 VkResult
2052 tu_BeginCommandBuffer(VkCommandBuffer commandBuffer,
2053                       const VkCommandBufferBeginInfo *pBeginInfo)
2054 {
2055    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
2056    VkResult result = VK_SUCCESS;
2057
2058    if (cmd_buffer->status != TU_CMD_BUFFER_STATUS_INITIAL) {
2059       /* If the command buffer has already been resetted with
2060        * vkResetCommandBuffer, no need to do it again.
2061        */
2062       result = tu_reset_cmd_buffer(cmd_buffer);
2063       if (result != VK_SUCCESS)
2064          return result;
2065    }
2066
2067    memset(&cmd_buffer->state, 0, sizeof(cmd_buffer->state));
2068    cmd_buffer->usage_flags = pBeginInfo->flags;
2069
2070    tu_cs_begin(&cmd_buffer->cs);
2071    tu_cs_begin(&cmd_buffer->draw_cs);
2072    tu_cs_begin(&cmd_buffer->draw_epilogue_cs);
2073
2074    cmd_buffer->marker_seqno = 0;
2075    cmd_buffer->scratch_seqno = 0;
2076
2077    /* setup initial configuration into command buffer */
2078    if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY) {
2079       switch (cmd_buffer->queue_family_index) {
2080       case TU_QUEUE_GENERAL:
2081          tu6_init_hw(cmd_buffer, &cmd_buffer->cs);
2082          break;
2083       default:
2084          break;
2085       }
2086    } else if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY &&
2087               (pBeginInfo->flags & VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT)) {
2088       assert(pBeginInfo->pInheritanceInfo);
2089       cmd_buffer->state.pass = tu_render_pass_from_handle(pBeginInfo->pInheritanceInfo->renderPass);
2090       cmd_buffer->state.subpass = &cmd_buffer->state.pass->subpasses[pBeginInfo->pInheritanceInfo->subpass];
2091    }
2092
2093    cmd_buffer->status = TU_CMD_BUFFER_STATUS_RECORDING;
2094
2095    return VK_SUCCESS;
2096 }
2097
2098 void
2099 tu_CmdBindVertexBuffers(VkCommandBuffer commandBuffer,
2100                         uint32_t firstBinding,
2101                         uint32_t bindingCount,
2102                         const VkBuffer *pBuffers,
2103                         const VkDeviceSize *pOffsets)
2104 {
2105    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2106
2107    assert(firstBinding + bindingCount <= MAX_VBS);
2108
2109    for (uint32_t i = 0; i < bindingCount; i++) {
2110       cmd->state.vb.buffers[firstBinding + i] =
2111          tu_buffer_from_handle(pBuffers[i]);
2112       cmd->state.vb.offsets[firstBinding + i] = pOffsets[i];
2113    }
2114
2115    /* VB states depend on VkPipelineVertexInputStateCreateInfo */
2116    cmd->state.dirty |= TU_CMD_DIRTY_VERTEX_BUFFERS;
2117 }
2118
2119 void
2120 tu_CmdBindIndexBuffer(VkCommandBuffer commandBuffer,
2121                       VkBuffer buffer,
2122                       VkDeviceSize offset,
2123                       VkIndexType indexType)
2124 {
2125    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2126    TU_FROM_HANDLE(tu_buffer, buf, buffer);
2127
2128    /* initialize/update the restart index */
2129    if (!cmd->state.index_buffer || cmd->state.index_type != indexType) {
2130       struct tu_cs *draw_cs = &cmd->draw_cs;
2131       VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 2);
2132       if (result != VK_SUCCESS) {
2133          cmd->record_result = result;
2134          return;
2135       }
2136
2137       tu6_emit_restart_index(
2138          draw_cs, indexType == VK_INDEX_TYPE_UINT32 ? 0xffffffff : 0xffff);
2139
2140       tu_cs_sanity_check(draw_cs);
2141    }
2142
2143    /* track the BO */
2144    if (cmd->state.index_buffer != buf)
2145       tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
2146
2147    cmd->state.index_buffer = buf;
2148    cmd->state.index_offset = offset;
2149    cmd->state.index_type = indexType;
2150 }
2151
2152 void
2153 tu_CmdBindDescriptorSets(VkCommandBuffer commandBuffer,
2154                          VkPipelineBindPoint pipelineBindPoint,
2155                          VkPipelineLayout _layout,
2156                          uint32_t firstSet,
2157                          uint32_t descriptorSetCount,
2158                          const VkDescriptorSet *pDescriptorSets,
2159                          uint32_t dynamicOffsetCount,
2160                          const uint32_t *pDynamicOffsets)
2161 {
2162    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
2163    TU_FROM_HANDLE(tu_pipeline_layout, layout, _layout);
2164    unsigned dyn_idx = 0;
2165
2166    struct tu_descriptor_state *descriptors_state =
2167       tu_get_descriptors_state(cmd_buffer, pipelineBindPoint);
2168
2169    for (unsigned i = 0; i < descriptorSetCount; ++i) {
2170       unsigned idx = i + firstSet;
2171       TU_FROM_HANDLE(tu_descriptor_set, set, pDescriptorSets[i]);
2172
2173       descriptors_state->sets[idx] = set;
2174       descriptors_state->valid |= (1u << idx);
2175
2176       for(unsigned j = 0; j < set->layout->dynamic_offset_count; ++j, ++dyn_idx) {
2177          unsigned idx = j + layout->set[i + firstSet].dynamic_offset_start;
2178          assert(dyn_idx < dynamicOffsetCount);
2179
2180          descriptors_state->dynamic_buffers[idx] =
2181          set->dynamic_descriptors[j].va + pDynamicOffsets[dyn_idx];
2182       }
2183    }
2184
2185    cmd_buffer->state.dirty |= TU_CMD_DIRTY_DESCRIPTOR_SETS;
2186 }
2187
2188 void
2189 tu_CmdPushConstants(VkCommandBuffer commandBuffer,
2190                     VkPipelineLayout layout,
2191                     VkShaderStageFlags stageFlags,
2192                     uint32_t offset,
2193                     uint32_t size,
2194                     const void *pValues)
2195 {
2196    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2197    memcpy((void*) cmd->push_constants + offset, pValues, size);
2198    cmd->state.dirty |= TU_CMD_DIRTY_PUSH_CONSTANTS;
2199 }
2200
2201 VkResult
2202 tu_EndCommandBuffer(VkCommandBuffer commandBuffer)
2203 {
2204    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
2205
2206    if (cmd_buffer->scratch_seqno) {
2207       tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->scratch_bo,
2208                      MSM_SUBMIT_BO_WRITE);
2209    }
2210
2211    if (cmd_buffer->use_vsc_data) {
2212       tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->vsc_data,
2213                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
2214       tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->vsc_data2,
2215                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
2216    }
2217
2218    for (uint32_t i = 0; i < cmd_buffer->draw_cs.bo_count; i++) {
2219       tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->draw_cs.bos[i],
2220                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2221    }
2222
2223    for (uint32_t i = 0; i < cmd_buffer->draw_epilogue_cs.bo_count; i++) {
2224       tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->draw_epilogue_cs.bos[i],
2225                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2226    }
2227
2228    for (uint32_t i = 0; i < cmd_buffer->sub_cs.bo_count; i++) {
2229       tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->sub_cs.bos[i],
2230                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2231    }
2232
2233    tu_cs_end(&cmd_buffer->cs);
2234    tu_cs_end(&cmd_buffer->draw_cs);
2235    tu_cs_end(&cmd_buffer->draw_epilogue_cs);
2236
2237    cmd_buffer->status = TU_CMD_BUFFER_STATUS_EXECUTABLE;
2238
2239    return cmd_buffer->record_result;
2240 }
2241
2242 void
2243 tu_CmdBindPipeline(VkCommandBuffer commandBuffer,
2244                    VkPipelineBindPoint pipelineBindPoint,
2245                    VkPipeline _pipeline)
2246 {
2247    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2248    TU_FROM_HANDLE(tu_pipeline, pipeline, _pipeline);
2249
2250    switch (pipelineBindPoint) {
2251    case VK_PIPELINE_BIND_POINT_GRAPHICS:
2252       cmd->state.pipeline = pipeline;
2253       cmd->state.dirty |= TU_CMD_DIRTY_PIPELINE;
2254       break;
2255    case VK_PIPELINE_BIND_POINT_COMPUTE:
2256       cmd->state.compute_pipeline = pipeline;
2257       cmd->state.dirty |= TU_CMD_DIRTY_COMPUTE_PIPELINE;
2258       break;
2259    default:
2260       unreachable("unrecognized pipeline bind point");
2261       break;
2262    }
2263
2264    tu_bo_list_add(&cmd->bo_list, &pipeline->program.binary_bo,
2265                   MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2266    for (uint32_t i = 0; i < pipeline->cs.bo_count; i++) {
2267       tu_bo_list_add(&cmd->bo_list, pipeline->cs.bos[i],
2268                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2269    }
2270 }
2271
2272 void
2273 tu_CmdSetViewport(VkCommandBuffer commandBuffer,
2274                   uint32_t firstViewport,
2275                   uint32_t viewportCount,
2276                   const VkViewport *pViewports)
2277 {
2278    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2279    struct tu_cs *draw_cs = &cmd->draw_cs;
2280
2281    VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 12);
2282    if (result != VK_SUCCESS) {
2283       cmd->record_result = result;
2284       return;
2285    }
2286
2287    assert(firstViewport == 0 && viewportCount == 1);
2288    tu6_emit_viewport(draw_cs, pViewports);
2289
2290    tu_cs_sanity_check(draw_cs);
2291 }
2292
2293 void
2294 tu_CmdSetScissor(VkCommandBuffer commandBuffer,
2295                  uint32_t firstScissor,
2296                  uint32_t scissorCount,
2297                  const VkRect2D *pScissors)
2298 {
2299    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2300    struct tu_cs *draw_cs = &cmd->draw_cs;
2301
2302    VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 3);
2303    if (result != VK_SUCCESS) {
2304       cmd->record_result = result;
2305       return;
2306    }
2307
2308    assert(firstScissor == 0 && scissorCount == 1);
2309    tu6_emit_scissor(draw_cs, pScissors);
2310
2311    tu_cs_sanity_check(draw_cs);
2312 }
2313
2314 void
2315 tu_CmdSetLineWidth(VkCommandBuffer commandBuffer, float lineWidth)
2316 {
2317    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2318
2319    cmd->state.dynamic.line_width = lineWidth;
2320
2321    /* line width depends on VkPipelineRasterizationStateCreateInfo */
2322    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_LINE_WIDTH;
2323 }
2324
2325 void
2326 tu_CmdSetDepthBias(VkCommandBuffer commandBuffer,
2327                    float depthBiasConstantFactor,
2328                    float depthBiasClamp,
2329                    float depthBiasSlopeFactor)
2330 {
2331    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2332    struct tu_cs *draw_cs = &cmd->draw_cs;
2333
2334    VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 4);
2335    if (result != VK_SUCCESS) {
2336       cmd->record_result = result;
2337       return;
2338    }
2339
2340    tu6_emit_depth_bias(draw_cs, depthBiasConstantFactor, depthBiasClamp,
2341                        depthBiasSlopeFactor);
2342
2343    tu_cs_sanity_check(draw_cs);
2344 }
2345
2346 void
2347 tu_CmdSetBlendConstants(VkCommandBuffer commandBuffer,
2348                         const float blendConstants[4])
2349 {
2350    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2351    struct tu_cs *draw_cs = &cmd->draw_cs;
2352
2353    VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 5);
2354    if (result != VK_SUCCESS) {
2355       cmd->record_result = result;
2356       return;
2357    }
2358
2359    tu6_emit_blend_constants(draw_cs, blendConstants);
2360
2361    tu_cs_sanity_check(draw_cs);
2362 }
2363
2364 void
2365 tu_CmdSetDepthBounds(VkCommandBuffer commandBuffer,
2366                      float minDepthBounds,
2367                      float maxDepthBounds)
2368 {
2369 }
2370
2371 void
2372 tu_CmdSetStencilCompareMask(VkCommandBuffer commandBuffer,
2373                             VkStencilFaceFlags faceMask,
2374                             uint32_t compareMask)
2375 {
2376    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2377
2378    if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
2379       cmd->state.dynamic.stencil_compare_mask.front = compareMask;
2380    if (faceMask & VK_STENCIL_FACE_BACK_BIT)
2381       cmd->state.dynamic.stencil_compare_mask.back = compareMask;
2382
2383    /* the front/back compare masks must be updated together */
2384    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK;
2385 }
2386
2387 void
2388 tu_CmdSetStencilWriteMask(VkCommandBuffer commandBuffer,
2389                           VkStencilFaceFlags faceMask,
2390                           uint32_t writeMask)
2391 {
2392    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2393
2394    if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
2395       cmd->state.dynamic.stencil_write_mask.front = writeMask;
2396    if (faceMask & VK_STENCIL_FACE_BACK_BIT)
2397       cmd->state.dynamic.stencil_write_mask.back = writeMask;
2398
2399    /* the front/back write masks must be updated together */
2400    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK;
2401 }
2402
2403 void
2404 tu_CmdSetStencilReference(VkCommandBuffer commandBuffer,
2405                           VkStencilFaceFlags faceMask,
2406                           uint32_t reference)
2407 {
2408    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2409
2410    if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
2411       cmd->state.dynamic.stencil_reference.front = reference;
2412    if (faceMask & VK_STENCIL_FACE_BACK_BIT)
2413       cmd->state.dynamic.stencil_reference.back = reference;
2414
2415    /* the front/back references must be updated together */
2416    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE;
2417 }
2418
2419 void
2420 tu_CmdExecuteCommands(VkCommandBuffer commandBuffer,
2421                       uint32_t commandBufferCount,
2422                       const VkCommandBuffer *pCmdBuffers)
2423 {
2424    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2425    VkResult result;
2426
2427    assert(commandBufferCount > 0);
2428
2429    for (uint32_t i = 0; i < commandBufferCount; i++) {
2430       TU_FROM_HANDLE(tu_cmd_buffer, secondary, pCmdBuffers[i]);
2431
2432       result = tu_bo_list_merge(&cmd->bo_list, &secondary->bo_list);
2433       if (result != VK_SUCCESS) {
2434          cmd->record_result = result;
2435          break;
2436       }
2437
2438       result = tu_cs_add_entries(&cmd->draw_cs, &secondary->draw_cs);
2439       if (result != VK_SUCCESS) {
2440          cmd->record_result = result;
2441          break;
2442       }
2443
2444       result = tu_cs_add_entries(&cmd->draw_epilogue_cs,
2445             &secondary->draw_epilogue_cs);
2446       if (result != VK_SUCCESS) {
2447          cmd->record_result = result;
2448          break;
2449       }
2450    }
2451    cmd->state.dirty = ~0u; /* TODO: set dirty only what needs to be */
2452 }
2453
2454 VkResult
2455 tu_CreateCommandPool(VkDevice _device,
2456                      const VkCommandPoolCreateInfo *pCreateInfo,
2457                      const VkAllocationCallbacks *pAllocator,
2458                      VkCommandPool *pCmdPool)
2459 {
2460    TU_FROM_HANDLE(tu_device, device, _device);
2461    struct tu_cmd_pool *pool;
2462
2463    pool = vk_alloc2(&device->alloc, pAllocator, sizeof(*pool), 8,
2464                     VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2465    if (pool == NULL)
2466       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
2467
2468    if (pAllocator)
2469       pool->alloc = *pAllocator;
2470    else
2471       pool->alloc = device->alloc;
2472
2473    list_inithead(&pool->cmd_buffers);
2474    list_inithead(&pool->free_cmd_buffers);
2475
2476    pool->queue_family_index = pCreateInfo->queueFamilyIndex;
2477
2478    *pCmdPool = tu_cmd_pool_to_handle(pool);
2479
2480    return VK_SUCCESS;
2481 }
2482
2483 void
2484 tu_DestroyCommandPool(VkDevice _device,
2485                       VkCommandPool commandPool,
2486                       const VkAllocationCallbacks *pAllocator)
2487 {
2488    TU_FROM_HANDLE(tu_device, device, _device);
2489    TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2490
2491    if (!pool)
2492       return;
2493
2494    list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2495                             &pool->cmd_buffers, pool_link)
2496    {
2497       tu_cmd_buffer_destroy(cmd_buffer);
2498    }
2499
2500    list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2501                             &pool->free_cmd_buffers, pool_link)
2502    {
2503       tu_cmd_buffer_destroy(cmd_buffer);
2504    }
2505
2506    vk_free2(&device->alloc, pAllocator, pool);
2507 }
2508
2509 VkResult
2510 tu_ResetCommandPool(VkDevice device,
2511                     VkCommandPool commandPool,
2512                     VkCommandPoolResetFlags flags)
2513 {
2514    TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2515    VkResult result;
2516
2517    list_for_each_entry(struct tu_cmd_buffer, cmd_buffer, &pool->cmd_buffers,
2518                        pool_link)
2519    {
2520       result = tu_reset_cmd_buffer(cmd_buffer);
2521       if (result != VK_SUCCESS)
2522          return result;
2523    }
2524
2525    return VK_SUCCESS;
2526 }
2527
2528 void
2529 tu_TrimCommandPool(VkDevice device,
2530                    VkCommandPool commandPool,
2531                    VkCommandPoolTrimFlags flags)
2532 {
2533    TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2534
2535    if (!pool)
2536       return;
2537
2538    list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2539                             &pool->free_cmd_buffers, pool_link)
2540    {
2541       tu_cmd_buffer_destroy(cmd_buffer);
2542    }
2543 }
2544
2545 void
2546 tu_CmdBeginRenderPass(VkCommandBuffer commandBuffer,
2547                       const VkRenderPassBeginInfo *pRenderPassBegin,
2548                       VkSubpassContents contents)
2549 {
2550    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2551    TU_FROM_HANDLE(tu_render_pass, pass, pRenderPassBegin->renderPass);
2552    TU_FROM_HANDLE(tu_framebuffer, fb, pRenderPassBegin->framebuffer);
2553
2554    cmd->state.pass = pass;
2555    cmd->state.subpass = pass->subpasses;
2556    cmd->state.framebuffer = fb;
2557
2558    tu_cmd_update_tiling_config(cmd, &pRenderPassBegin->renderArea);
2559    tu_cmd_prepare_sysmem_clear_ib(cmd, pRenderPassBegin);
2560    tu_cmd_prepare_tile_load_ib(cmd, pRenderPassBegin);
2561    tu_cmd_prepare_tile_store_ib(cmd);
2562
2563    VkResult result = tu_cs_reserve_space(cmd->device, &cmd->draw_cs, 1024);
2564    if (result != VK_SUCCESS) {
2565       cmd->record_result = result;
2566       return;
2567    }
2568
2569    tu6_emit_zs(cmd, cmd->state.subpass, &cmd->draw_cs);
2570    tu6_emit_mrt(cmd, cmd->state.subpass, &cmd->draw_cs);
2571    tu6_emit_msaa(cmd, cmd->state.subpass, &cmd->draw_cs);
2572    tu6_emit_render_cntl(cmd, cmd->state.subpass, &cmd->draw_cs, false);
2573
2574    /* note: use_hw_binning only checks tiling config */
2575    if (use_hw_binning(cmd))
2576       cmd->use_vsc_data = true;
2577
2578    for (uint32_t i = 0; i < fb->attachment_count; ++i) {
2579       const struct tu_image_view *iview = fb->attachments[i].attachment;
2580       tu_bo_list_add(&cmd->bo_list, iview->image->bo,
2581                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
2582    }
2583 }
2584
2585 void
2586 tu_CmdBeginRenderPass2(VkCommandBuffer commandBuffer,
2587                        const VkRenderPassBeginInfo *pRenderPassBeginInfo,
2588                        const VkSubpassBeginInfoKHR *pSubpassBeginInfo)
2589 {
2590    tu_CmdBeginRenderPass(commandBuffer, pRenderPassBeginInfo,
2591                          pSubpassBeginInfo->contents);
2592 }
2593
2594 void
2595 tu_CmdNextSubpass(VkCommandBuffer commandBuffer, VkSubpassContents contents)
2596 {
2597    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2598    const struct tu_render_pass *pass = cmd->state.pass;
2599    struct tu_cs *cs = &cmd->draw_cs;
2600
2601    VkResult result = tu_cs_reserve_space(cmd->device, cs, 1024);
2602    if (result != VK_SUCCESS) {
2603       cmd->record_result = result;
2604       return;
2605    }
2606
2607    const struct tu_subpass *subpass = cmd->state.subpass++;
2608    /* TODO:
2609     * if msaa samples change between subpasses,
2610     * attachment store is broken for some attachments
2611     */
2612    if (subpass->resolve_attachments) {
2613       tu6_emit_blit_scissor(cmd, cs, true);
2614       for (unsigned i = 0; i < subpass->color_count; i++) {
2615          uint32_t a = subpass->resolve_attachments[i].attachment;
2616          if (a != VK_ATTACHMENT_UNUSED) {
2617                tu6_emit_store_attachment(cmd, cs, a,
2618                                          subpass->color_attachments[i].attachment);
2619          }
2620       }
2621    }
2622
2623    /* invalidate because reading input attachments will cache GMEM and
2624     * the cache isn''t updated when GMEM is written
2625     * TODO: is there a no-cache bit for textures?
2626     */
2627    if (cmd->state.subpass->input_count)
2628       tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE, false);
2629
2630    /* emit mrt/zs/msaa/ubwc state for the subpass that is starting */
2631    tu6_emit_zs(cmd, cmd->state.subpass, cs);
2632    tu6_emit_mrt(cmd, cmd->state.subpass, cs);
2633    tu6_emit_msaa(cmd, cmd->state.subpass, cs);
2634    tu6_emit_render_cntl(cmd, cmd->state.subpass, cs, false);
2635
2636    /* Emit flushes so that input attachments will read the correct value. This
2637     * is for sysmem only, although it shouldn't do much harm on gmem.
2638     */
2639    tu6_emit_event_write(cmd, cs, UNK_1C, true);
2640    tu6_emit_event_write(cmd, cs, UNK_1D, true);
2641
2642    /* TODO:
2643     * since we don't know how to do GMEM->GMEM resolve,
2644     * resolve attachments are resolved to memory then loaded to GMEM again if needed
2645     */
2646    if (subpass->resolve_attachments) {
2647       for (unsigned i = 0; i < subpass->color_count; i++) {
2648          uint32_t a = subpass->resolve_attachments[i].attachment;
2649          const struct tu_image_view *iview =
2650             cmd->state.framebuffer->attachments[a].attachment;
2651          if (a != VK_ATTACHMENT_UNUSED && pass->attachments[a].gmem_offset >= 0) {
2652                tu_finishme("missing GMEM->GMEM resolve, performance will suffer\n");
2653                tu6_emit_blit_info(cmd, cs, iview, pass->attachments[a].gmem_offset, false);
2654                tu6_emit_blit(cmd, cs);
2655          }
2656       }
2657    }
2658 }
2659
2660 void
2661 tu_CmdNextSubpass2(VkCommandBuffer commandBuffer,
2662                    const VkSubpassBeginInfoKHR *pSubpassBeginInfo,
2663                    const VkSubpassEndInfoKHR *pSubpassEndInfo)
2664 {
2665    tu_CmdNextSubpass(commandBuffer, pSubpassBeginInfo->contents);
2666 }
2667
2668 struct tu_draw_info
2669 {
2670    /**
2671     * Number of vertices.
2672     */
2673    uint32_t count;
2674
2675    /**
2676     * Index of the first vertex.
2677     */
2678    int32_t vertex_offset;
2679
2680    /**
2681     * First instance id.
2682     */
2683    uint32_t first_instance;
2684
2685    /**
2686     * Number of instances.
2687     */
2688    uint32_t instance_count;
2689
2690    /**
2691     * First index (indexed draws only).
2692     */
2693    uint32_t first_index;
2694
2695    /**
2696     * Whether it's an indexed draw.
2697     */
2698    bool indexed;
2699
2700    /**
2701     * Indirect draw parameters resource.
2702     */
2703    struct tu_buffer *indirect;
2704    uint64_t indirect_offset;
2705    uint32_t stride;
2706
2707    /**
2708     * Draw count parameters resource.
2709     */
2710    struct tu_buffer *count_buffer;
2711    uint64_t count_buffer_offset;
2712 };
2713
2714 #define ENABLE_ALL (CP_SET_DRAW_STATE__0_BINNING | CP_SET_DRAW_STATE__0_GMEM | CP_SET_DRAW_STATE__0_SYSMEM)
2715 #define ENABLE_DRAW (CP_SET_DRAW_STATE__0_GMEM | CP_SET_DRAW_STATE__0_SYSMEM)
2716
2717 enum tu_draw_state_group_id
2718 {
2719    TU_DRAW_STATE_PROGRAM,
2720    TU_DRAW_STATE_PROGRAM_BINNING,
2721    TU_DRAW_STATE_VI,
2722    TU_DRAW_STATE_VI_BINNING,
2723    TU_DRAW_STATE_VP,
2724    TU_DRAW_STATE_RAST,
2725    TU_DRAW_STATE_DS,
2726    TU_DRAW_STATE_BLEND,
2727    TU_DRAW_STATE_VS_CONST,
2728    TU_DRAW_STATE_FS_CONST,
2729    TU_DRAW_STATE_VS_TEX,
2730    TU_DRAW_STATE_FS_TEX,
2731    TU_DRAW_STATE_FS_IBO,
2732    TU_DRAW_STATE_VS_PARAMS,
2733
2734    TU_DRAW_STATE_COUNT,
2735 };
2736
2737 struct tu_draw_state_group
2738 {
2739    enum tu_draw_state_group_id id;
2740    uint32_t enable_mask;
2741    struct tu_cs_entry ib;
2742 };
2743
2744 const static struct tu_sampler*
2745 sampler_ptr(struct tu_descriptor_state *descriptors_state,
2746             const struct tu_descriptor_map *map, unsigned i,
2747             unsigned array_index)
2748 {
2749    assert(descriptors_state->valid & (1 << map->set[i]));
2750
2751    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2752    assert(map->binding[i] < set->layout->binding_count);
2753
2754    const struct tu_descriptor_set_binding_layout *layout =
2755       &set->layout->binding[map->binding[i]];
2756
2757    if (layout->immutable_samplers_offset) {
2758       const struct tu_sampler *immutable_samplers =
2759          tu_immutable_samplers(set->layout, layout);
2760
2761       return &immutable_samplers[array_index];
2762    }
2763
2764    switch (layout->type) {
2765    case VK_DESCRIPTOR_TYPE_SAMPLER:
2766       return (struct tu_sampler*) &set->mapped_ptr[layout->offset / 4];
2767    case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
2768       return (struct tu_sampler*) &set->mapped_ptr[layout->offset / 4 + A6XX_TEX_CONST_DWORDS +
2769                                                    array_index *
2770                                                    (A6XX_TEX_CONST_DWORDS +
2771                                                     sizeof(struct tu_sampler) / 4)];
2772    default:
2773       unreachable("unimplemented descriptor type");
2774       break;
2775    }
2776 }
2777
2778 static void
2779 write_tex_const(struct tu_cmd_buffer *cmd,
2780                 uint32_t *dst,
2781                 struct tu_descriptor_state *descriptors_state,
2782                 const struct tu_descriptor_map *map,
2783                 unsigned i, unsigned array_index)
2784 {
2785    assert(descriptors_state->valid & (1 << map->set[i]));
2786
2787    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2788    assert(map->binding[i] < set->layout->binding_count);
2789
2790    const struct tu_descriptor_set_binding_layout *layout =
2791       &set->layout->binding[map->binding[i]];
2792
2793    switch (layout->type) {
2794    case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
2795    case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
2796    case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
2797    case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
2798       memcpy(dst, &set->mapped_ptr[layout->offset / 4 +
2799                                    array_index * A6XX_TEX_CONST_DWORDS],
2800              A6XX_TEX_CONST_DWORDS * 4);
2801       break;
2802    case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
2803       memcpy(dst, &set->mapped_ptr[layout->offset / 4 +
2804                                    array_index *
2805                                    (A6XX_TEX_CONST_DWORDS +
2806                                     sizeof(struct tu_sampler) / 4)],
2807              A6XX_TEX_CONST_DWORDS * 4);
2808       break;
2809    default:
2810       unreachable("unimplemented descriptor type");
2811       break;
2812    }
2813
2814    if (layout->type == VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT) {
2815       const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
2816       uint32_t a = cmd->state.subpass->input_attachments[map->value[i] +
2817                                                          array_index].attachment;
2818       const struct tu_render_pass_attachment *att = &cmd->state.pass->attachments[a];
2819
2820       assert(att->gmem_offset >= 0);
2821
2822       dst[0] &= ~(A6XX_TEX_CONST_0_SWAP__MASK | A6XX_TEX_CONST_0_TILE_MODE__MASK);
2823       dst[0] |= A6XX_TEX_CONST_0_TILE_MODE(TILE6_2);
2824       dst[2] &= ~(A6XX_TEX_CONST_2_TYPE__MASK | A6XX_TEX_CONST_2_PITCH__MASK);
2825       dst[2] |=
2826          A6XX_TEX_CONST_2_TYPE(A6XX_TEX_2D) |
2827          A6XX_TEX_CONST_2_PITCH(tiling->tile0.extent.width * att->cpp);
2828       dst[3] = 0;
2829       dst[4] = 0x100000 + att->gmem_offset;
2830       dst[5] = A6XX_TEX_CONST_5_DEPTH(1);
2831       for (unsigned i = 6; i < A6XX_TEX_CONST_DWORDS; i++)
2832          dst[i] = 0;
2833
2834       if (cmd->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY)
2835          tu_finishme("patch input attachment pitch for secondary cmd buffer");
2836    }
2837 }
2838
2839 static void
2840 write_image_ibo(struct tu_cmd_buffer *cmd,
2841                 uint32_t *dst,
2842                 struct tu_descriptor_state *descriptors_state,
2843                 const struct tu_descriptor_map *map,
2844                 unsigned i, unsigned array_index)
2845 {
2846    assert(descriptors_state->valid & (1 << map->set[i]));
2847
2848    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2849    assert(map->binding[i] < set->layout->binding_count);
2850
2851    const struct tu_descriptor_set_binding_layout *layout =
2852       &set->layout->binding[map->binding[i]];
2853
2854    assert(layout->type == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE);
2855
2856    memcpy(dst, &set->mapped_ptr[layout->offset / 4 +
2857                                 (array_index * 2 + 1) * A6XX_TEX_CONST_DWORDS],
2858           A6XX_TEX_CONST_DWORDS * 4);
2859 }
2860
2861 static uint64_t
2862 buffer_ptr(struct tu_descriptor_state *descriptors_state,
2863            const struct tu_descriptor_map *map,
2864            unsigned i, unsigned array_index)
2865 {
2866    assert(descriptors_state->valid & (1 << map->set[i]));
2867
2868    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2869    assert(map->binding[i] < set->layout->binding_count);
2870
2871    const struct tu_descriptor_set_binding_layout *layout =
2872       &set->layout->binding[map->binding[i]];
2873
2874    switch (layout->type) {
2875    case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
2876    case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
2877       return descriptors_state->dynamic_buffers[layout->dynamic_offset_offset +
2878                                                 array_index];
2879    case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
2880    case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
2881       return (uint64_t) set->mapped_ptr[layout->offset / 4 + array_index * 2 + 1] << 32 |
2882                         set->mapped_ptr[layout->offset / 4 + array_index * 2];
2883    default:
2884       unreachable("unimplemented descriptor type");
2885       break;
2886    }
2887 }
2888
2889 static inline uint32_t
2890 tu6_stage2opcode(gl_shader_stage type)
2891 {
2892    switch (type) {
2893    case MESA_SHADER_VERTEX:
2894    case MESA_SHADER_TESS_CTRL:
2895    case MESA_SHADER_TESS_EVAL:
2896    case MESA_SHADER_GEOMETRY:
2897       return CP_LOAD_STATE6_GEOM;
2898    case MESA_SHADER_FRAGMENT:
2899    case MESA_SHADER_COMPUTE:
2900    case MESA_SHADER_KERNEL:
2901       return CP_LOAD_STATE6_FRAG;
2902    default:
2903       unreachable("bad shader type");
2904    }
2905 }
2906
2907 static inline enum a6xx_state_block
2908 tu6_stage2shadersb(gl_shader_stage type)
2909 {
2910    switch (type) {
2911    case MESA_SHADER_VERTEX:
2912       return SB6_VS_SHADER;
2913    case MESA_SHADER_FRAGMENT:
2914       return SB6_FS_SHADER;
2915    case MESA_SHADER_COMPUTE:
2916    case MESA_SHADER_KERNEL:
2917       return SB6_CS_SHADER;
2918    default:
2919       unreachable("bad shader type");
2920       return ~0;
2921    }
2922 }
2923
2924 static void
2925 tu6_emit_user_consts(struct tu_cs *cs, const struct tu_pipeline *pipeline,
2926                      struct tu_descriptor_state *descriptors_state,
2927                      gl_shader_stage type,
2928                      uint32_t *push_constants)
2929 {
2930    const struct tu_program_descriptor_linkage *link =
2931       &pipeline->program.link[type];
2932    const struct ir3_ubo_analysis_state *state = &link->ubo_state;
2933
2934    for (uint32_t i = 0; i < ARRAY_SIZE(state->range); i++) {
2935       if (state->range[i].start < state->range[i].end) {
2936          uint32_t size = state->range[i].end - state->range[i].start;
2937          uint32_t offset = state->range[i].start;
2938
2939          /* and even if the start of the const buffer is before
2940           * first_immediate, the end may not be:
2941           */
2942          size = MIN2(size, (16 * link->constlen) - state->range[i].offset);
2943
2944          if (size == 0)
2945             continue;
2946
2947          /* things should be aligned to vec4: */
2948          debug_assert((state->range[i].offset % 16) == 0);
2949          debug_assert((size % 16) == 0);
2950          debug_assert((offset % 16) == 0);
2951
2952          if (i == 0) {
2953             /* push constants */
2954             tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + (size / 4));
2955             tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(state->range[i].offset / 16) |
2956                   CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2957                   CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
2958                   CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2959                   CP_LOAD_STATE6_0_NUM_UNIT(size / 16));
2960             tu_cs_emit(cs, 0);
2961             tu_cs_emit(cs, 0);
2962             for (unsigned i = 0; i < size / 4; i++)
2963                tu_cs_emit(cs, push_constants[i + offset / 4]);
2964             continue;
2965          }
2966
2967          /* Look through the UBO map to find our UBO index, and get the VA for
2968           * that UBO.
2969           */
2970          uint64_t va = 0;
2971          uint32_t ubo_idx = i - 1;
2972          uint32_t ubo_map_base = 0;
2973          for (int j = 0; j < link->ubo_map.num; j++) {
2974             if (ubo_idx >= ubo_map_base &&
2975                 ubo_idx < ubo_map_base + link->ubo_map.array_size[j]) {
2976                va = buffer_ptr(descriptors_state, &link->ubo_map, j,
2977                                ubo_idx - ubo_map_base);
2978                break;
2979             }
2980             ubo_map_base += link->ubo_map.array_size[j];
2981          }
2982          assert(va);
2983
2984          tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3);
2985          tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(state->range[i].offset / 16) |
2986                CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2987                CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
2988                CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2989                CP_LOAD_STATE6_0_NUM_UNIT(size / 16));
2990          tu_cs_emit_qw(cs, va + offset);
2991       }
2992    }
2993 }
2994
2995 static void
2996 tu6_emit_ubos(struct tu_cs *cs, const struct tu_pipeline *pipeline,
2997               struct tu_descriptor_state *descriptors_state,
2998               gl_shader_stage type)
2999 {
3000    const struct tu_program_descriptor_linkage *link =
3001       &pipeline->program.link[type];
3002
3003    uint32_t num = MIN2(link->ubo_map.num_desc, link->const_state.num_ubos);
3004    uint32_t anum = align(num, 2);
3005
3006    if (!num)
3007       return;
3008
3009    tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + (2 * anum));
3010    tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(link->const_state.offsets.ubo) |
3011          CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
3012          CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
3013          CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
3014          CP_LOAD_STATE6_0_NUM_UNIT(anum/2));
3015    tu_cs_emit(cs, CP_LOAD_STATE6_1_EXT_SRC_ADDR(0));
3016    tu_cs_emit(cs, CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI(0));
3017
3018    unsigned emitted = 0;
3019    for (unsigned i = 0; emitted < num && i < link->ubo_map.num; i++) {
3020       for (unsigned j = 0; emitted < num && j < link->ubo_map.array_size[i]; j++) {
3021          tu_cs_emit_qw(cs, buffer_ptr(descriptors_state, &link->ubo_map, i, j));
3022          emitted++;
3023       }
3024    }
3025
3026    for (; emitted < anum; emitted++) {
3027       tu_cs_emit(cs, 0xffffffff);
3028       tu_cs_emit(cs, 0xffffffff);
3029    }
3030 }
3031
3032 static struct tu_cs_entry
3033 tu6_emit_consts(struct tu_cmd_buffer *cmd,
3034                 const struct tu_pipeline *pipeline,
3035                 struct tu_descriptor_state *descriptors_state,
3036                 gl_shader_stage type)
3037 {
3038    struct tu_cs cs;
3039    tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs, 512, &cs); /* TODO: maximum size? */
3040
3041    tu6_emit_user_consts(&cs, pipeline, descriptors_state, type, cmd->push_constants);
3042    tu6_emit_ubos(&cs, pipeline, descriptors_state, type);
3043
3044    return tu_cs_end_sub_stream(&cmd->sub_cs, &cs);
3045 }
3046
3047 static VkResult
3048 tu6_emit_vs_params(struct tu_cmd_buffer *cmd,
3049                    const struct tu_draw_info *draw,
3050                    struct tu_cs_entry *entry)
3051 {
3052    /* TODO: fill out more than just base instance */
3053    const struct tu_program_descriptor_linkage *link =
3054       &cmd->state.pipeline->program.link[MESA_SHADER_VERTEX];
3055    const struct ir3_const_state *const_state = &link->const_state;
3056    struct tu_cs cs;
3057
3058    if (const_state->offsets.driver_param >= link->constlen) {
3059       *entry = (struct tu_cs_entry) {};
3060       return VK_SUCCESS;
3061    }
3062
3063    VkResult result = tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs, 8, &cs);
3064    if (result != VK_SUCCESS)
3065       return result;
3066
3067    tu_cs_emit_pkt7(&cs, CP_LOAD_STATE6_GEOM, 3 + 4);
3068    tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(const_state->offsets.driver_param) |
3069          CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
3070          CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
3071          CP_LOAD_STATE6_0_STATE_BLOCK(SB6_VS_SHADER) |
3072          CP_LOAD_STATE6_0_NUM_UNIT(1));
3073    tu_cs_emit(&cs, 0);
3074    tu_cs_emit(&cs, 0);
3075
3076    STATIC_ASSERT(IR3_DP_INSTID_BASE == 2);
3077
3078    tu_cs_emit(&cs, 0);
3079    tu_cs_emit(&cs, 0);
3080    tu_cs_emit(&cs, draw->first_instance);
3081    tu_cs_emit(&cs, 0);
3082
3083    *entry = tu_cs_end_sub_stream(&cmd->sub_cs, &cs);
3084    return VK_SUCCESS;
3085 }
3086
3087 static VkResult
3088 tu6_emit_textures(struct tu_cmd_buffer *cmd,
3089                   const struct tu_pipeline *pipeline,
3090                   struct tu_descriptor_state *descriptors_state,
3091                   gl_shader_stage type,
3092                   struct tu_cs_entry *entry,
3093                   bool *needs_border)
3094 {
3095    struct tu_device *device = cmd->device;
3096    struct tu_cs *draw_state = &cmd->sub_cs;
3097    const struct tu_program_descriptor_linkage *link =
3098       &pipeline->program.link[type];
3099    VkResult result;
3100
3101    if (link->texture_map.num_desc == 0 && link->sampler_map.num_desc == 0) {
3102       *entry = (struct tu_cs_entry) {};
3103       return VK_SUCCESS;
3104    }
3105
3106    /* allocate and fill texture state */
3107    struct ts_cs_memory tex_const;
3108    result = tu_cs_alloc(device, draw_state, link->texture_map.num_desc,
3109                         A6XX_TEX_CONST_DWORDS, &tex_const);
3110    if (result != VK_SUCCESS)
3111       return result;
3112
3113    int tex_index = 0;
3114    for (unsigned i = 0; i < link->texture_map.num; i++) {
3115       for (int j = 0; j < link->texture_map.array_size[i]; j++) {
3116          write_tex_const(cmd,
3117                          &tex_const.map[A6XX_TEX_CONST_DWORDS * tex_index++],
3118                          descriptors_state, &link->texture_map, i, j);
3119       }
3120    }
3121
3122    /* allocate and fill sampler state */
3123    struct ts_cs_memory tex_samp = { 0 };
3124    if (link->sampler_map.num_desc) {
3125       result = tu_cs_alloc(device, draw_state, link->sampler_map.num_desc,
3126                            A6XX_TEX_SAMP_DWORDS, &tex_samp);
3127       if (result != VK_SUCCESS)
3128          return result;
3129
3130       int sampler_index = 0;
3131       for (unsigned i = 0; i < link->sampler_map.num; i++) {
3132          for (int j = 0; j < link->sampler_map.array_size[i]; j++) {
3133             const struct tu_sampler *sampler = sampler_ptr(descriptors_state,
3134                                                            &link->sampler_map,
3135                                                            i, j);
3136             memcpy(&tex_samp.map[A6XX_TEX_SAMP_DWORDS * sampler_index++],
3137                    sampler->state, sizeof(sampler->state));
3138             *needs_border |= sampler->needs_border;
3139          }
3140       }
3141    }
3142
3143    unsigned tex_samp_reg, tex_const_reg, tex_count_reg;
3144    enum a6xx_state_block sb;
3145
3146    switch (type) {
3147    case MESA_SHADER_VERTEX:
3148       sb = SB6_VS_TEX;
3149       tex_samp_reg = REG_A6XX_SP_VS_TEX_SAMP_LO;
3150       tex_const_reg = REG_A6XX_SP_VS_TEX_CONST_LO;
3151       tex_count_reg = REG_A6XX_SP_VS_TEX_COUNT;
3152       break;
3153    case MESA_SHADER_FRAGMENT:
3154       sb = SB6_FS_TEX;
3155       tex_samp_reg = REG_A6XX_SP_FS_TEX_SAMP_LO;
3156       tex_const_reg = REG_A6XX_SP_FS_TEX_CONST_LO;
3157       tex_count_reg = REG_A6XX_SP_FS_TEX_COUNT;
3158       break;
3159    case MESA_SHADER_COMPUTE:
3160       sb = SB6_CS_TEX;
3161       tex_samp_reg = REG_A6XX_SP_CS_TEX_SAMP_LO;
3162       tex_const_reg = REG_A6XX_SP_CS_TEX_CONST_LO;
3163       tex_count_reg = REG_A6XX_SP_CS_TEX_COUNT;
3164       break;
3165    default:
3166       unreachable("bad state block");
3167    }
3168
3169    struct tu_cs cs;
3170    result = tu_cs_begin_sub_stream(device, draw_state, 16, &cs);
3171    if (result != VK_SUCCESS)
3172       return result;
3173
3174    if (link->sampler_map.num_desc) {
3175       /* output sampler state: */
3176       tu_cs_emit_pkt7(&cs, tu6_stage2opcode(type), 3);
3177       tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
3178                  CP_LOAD_STATE6_0_STATE_TYPE(ST6_SHADER) |
3179                  CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
3180                  CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
3181                  CP_LOAD_STATE6_0_NUM_UNIT(link->sampler_map.num_desc));
3182       tu_cs_emit_qw(&cs, tex_samp.iova); /* SRC_ADDR_LO/HI */
3183
3184       tu_cs_emit_pkt4(&cs, tex_samp_reg, 2);
3185       tu_cs_emit_qw(&cs, tex_samp.iova); /* SRC_ADDR_LO/HI */
3186    }
3187
3188    /* emit texture state: */
3189    tu_cs_emit_pkt7(&cs, tu6_stage2opcode(type), 3);
3190    tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
3191       CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
3192       CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
3193       CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
3194       CP_LOAD_STATE6_0_NUM_UNIT(link->texture_map.num_desc));
3195    tu_cs_emit_qw(&cs, tex_const.iova); /* SRC_ADDR_LO/HI */
3196
3197    tu_cs_emit_pkt4(&cs, tex_const_reg, 2);
3198    tu_cs_emit_qw(&cs, tex_const.iova); /* SRC_ADDR_LO/HI */
3199
3200    tu_cs_emit_pkt4(&cs, tex_count_reg, 1);
3201    tu_cs_emit(&cs, link->texture_map.num_desc);
3202
3203    *entry = tu_cs_end_sub_stream(draw_state, &cs);
3204    return VK_SUCCESS;
3205 }
3206
3207 static VkResult
3208 tu6_emit_ibo(struct tu_cmd_buffer *cmd,
3209              const struct tu_pipeline *pipeline,
3210              struct tu_descriptor_state *descriptors_state,
3211              gl_shader_stage type,
3212              struct tu_cs_entry *entry)
3213 {
3214    struct tu_device *device = cmd->device;
3215    struct tu_cs *draw_state = &cmd->sub_cs;
3216    const struct tu_program_descriptor_linkage *link =
3217       &pipeline->program.link[type];
3218    VkResult result;
3219
3220    unsigned num_desc = link->ssbo_map.num_desc + link->image_map.num_desc;
3221
3222    if (num_desc == 0) {
3223       *entry = (struct tu_cs_entry) {};
3224       return VK_SUCCESS;
3225    }
3226
3227    struct ts_cs_memory ibo_const;
3228    result = tu_cs_alloc(device, draw_state, num_desc,
3229                         A6XX_TEX_CONST_DWORDS, &ibo_const);
3230    if (result != VK_SUCCESS)
3231       return result;
3232
3233    int ssbo_index = 0;
3234    for (unsigned i = 0; i < link->ssbo_map.num; i++) {
3235       for (int j = 0; j < link->ssbo_map.array_size[i]; j++) {
3236          uint32_t *dst = &ibo_const.map[A6XX_TEX_CONST_DWORDS * ssbo_index];
3237
3238          uint64_t va = buffer_ptr(descriptors_state, &link->ssbo_map, i, j);
3239          /* We don't expose robustBufferAccess, so leave the size unlimited. */
3240          uint32_t sz = MAX_STORAGE_BUFFER_RANGE / 4;
3241
3242          dst[0] = A6XX_IBO_0_FMT(FMT6_32_UINT);
3243          dst[1] = A6XX_IBO_1_WIDTH(sz & MASK(15)) |
3244                   A6XX_IBO_1_HEIGHT(sz >> 15);
3245          dst[2] = A6XX_IBO_2_UNK4 |
3246                   A6XX_IBO_2_UNK31 |
3247                   A6XX_IBO_2_TYPE(A6XX_TEX_1D);
3248          dst[3] = 0;
3249          dst[4] = va;
3250          dst[5] = va >> 32;
3251          for (int i = 6; i < A6XX_TEX_CONST_DWORDS; i++)
3252             dst[i] = 0;
3253
3254          ssbo_index++;
3255       }
3256    }
3257
3258    for (unsigned i = 0; i < link->image_map.num; i++) {
3259       for (int j = 0; j < link->image_map.array_size[i]; j++) {
3260          uint32_t *dst = &ibo_const.map[A6XX_TEX_CONST_DWORDS * ssbo_index];
3261
3262          write_image_ibo(cmd, dst,
3263                          descriptors_state, &link->image_map, i, j);
3264
3265          ssbo_index++;
3266       }
3267    }
3268
3269    assert(ssbo_index == num_desc);
3270
3271    struct tu_cs cs;
3272    result = tu_cs_begin_sub_stream(device, draw_state, 7, &cs);
3273    if (result != VK_SUCCESS)
3274       return result;
3275
3276    uint32_t opcode, ibo_addr_reg;
3277    enum a6xx_state_block sb;
3278    enum a6xx_state_type st;
3279
3280    switch (type) {
3281    case MESA_SHADER_FRAGMENT:
3282       opcode = CP_LOAD_STATE6;
3283       st = ST6_SHADER;
3284       sb = SB6_IBO;
3285       ibo_addr_reg = REG_A6XX_SP_IBO_LO;
3286       break;
3287    case MESA_SHADER_COMPUTE:
3288       opcode = CP_LOAD_STATE6_FRAG;
3289       st = ST6_IBO;
3290       sb = SB6_CS_SHADER;
3291       ibo_addr_reg = REG_A6XX_SP_CS_IBO_LO;
3292       break;
3293    default:
3294       unreachable("unsupported stage for ibos");
3295    }
3296
3297    /* emit texture state: */
3298    tu_cs_emit_pkt7(&cs, opcode, 3);
3299    tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
3300               CP_LOAD_STATE6_0_STATE_TYPE(st) |
3301               CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
3302               CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
3303               CP_LOAD_STATE6_0_NUM_UNIT(num_desc));
3304    tu_cs_emit_qw(&cs, ibo_const.iova); /* SRC_ADDR_LO/HI */
3305
3306    tu_cs_emit_pkt4(&cs, ibo_addr_reg, 2);
3307    tu_cs_emit_qw(&cs, ibo_const.iova); /* SRC_ADDR_LO/HI */
3308
3309    *entry = tu_cs_end_sub_stream(draw_state, &cs);
3310    return VK_SUCCESS;
3311 }
3312
3313 struct PACKED bcolor_entry {
3314    uint32_t fp32[4];
3315    uint16_t ui16[4];
3316    int16_t  si16[4];
3317    uint16_t fp16[4];
3318    uint16_t rgb565;
3319    uint16_t rgb5a1;
3320    uint16_t rgba4;
3321    uint8_t __pad0[2];
3322    uint8_t  ui8[4];
3323    int8_t   si8[4];
3324    uint32_t rgb10a2;
3325    uint32_t z24; /* also s8? */
3326    uint16_t srgb[4];      /* appears to duplicate fp16[], but clamped, used for srgb */
3327    uint8_t  __pad1[56];
3328 } border_color[] = {
3329    [VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK] = {},
3330    [VK_BORDER_COLOR_INT_TRANSPARENT_BLACK] = {},
3331    [VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK] = {
3332       .fp32[3] = 0x3f800000,
3333       .ui16[3] = 0xffff,
3334       .si16[3] = 0x7fff,
3335       .fp16[3] = 0x3c00,
3336       .rgb5a1 = 0x8000,
3337       .rgba4 = 0xf000,
3338       .ui8[3] = 0xff,
3339       .si8[3] = 0x7f,
3340       .rgb10a2 = 0xc0000000,
3341       .srgb[3] = 0x3c00,
3342    },
3343    [VK_BORDER_COLOR_INT_OPAQUE_BLACK] = {
3344       .fp32[3] = 1,
3345       .fp16[3] = 1,
3346    },
3347    [VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE] = {
3348       .fp32[0 ... 3] = 0x3f800000,
3349       .ui16[0 ... 3] = 0xffff,
3350       .si16[0 ... 3] = 0x7fff,
3351       .fp16[0 ... 3] = 0x3c00,
3352       .rgb565 = 0xffff,
3353       .rgb5a1 = 0xffff,
3354       .rgba4 = 0xffff,
3355       .ui8[0 ... 3] = 0xff,
3356       .si8[0 ... 3] = 0x7f,
3357       .rgb10a2 = 0xffffffff,
3358       .z24 = 0xffffff,
3359       .srgb[0 ... 3] = 0x3c00,
3360    },
3361    [VK_BORDER_COLOR_INT_OPAQUE_WHITE] = {
3362       .fp32[0 ... 3] = 1,
3363       .fp16[0 ... 3] = 1,
3364    },
3365 };
3366
3367 static VkResult
3368 tu6_emit_border_color(struct tu_cmd_buffer *cmd,
3369                       struct tu_cs *cs)
3370 {
3371    STATIC_ASSERT(sizeof(struct bcolor_entry) == 128);
3372
3373    const struct tu_pipeline *pipeline = cmd->state.pipeline;
3374    struct tu_descriptor_state *descriptors_state =
3375       &cmd->descriptors[VK_PIPELINE_BIND_POINT_GRAPHICS];
3376    const struct tu_descriptor_map *vs_sampler =
3377       &pipeline->program.link[MESA_SHADER_VERTEX].sampler_map;
3378    const struct tu_descriptor_map *fs_sampler =
3379       &pipeline->program.link[MESA_SHADER_FRAGMENT].sampler_map;
3380    struct ts_cs_memory ptr;
3381
3382    VkResult result = tu_cs_alloc(cmd->device, &cmd->sub_cs,
3383                                  vs_sampler->num_desc + fs_sampler->num_desc,
3384                                  128 / 4,
3385                                  &ptr);
3386    if (result != VK_SUCCESS)
3387       return result;
3388
3389    for (unsigned i = 0; i < vs_sampler->num; i++) {
3390       for (unsigned j = 0; j < vs_sampler->array_size[i]; j++) {
3391          const struct tu_sampler *sampler = sampler_ptr(descriptors_state,
3392                                                         vs_sampler, i, j);
3393          memcpy(ptr.map, &border_color[sampler->border], 128);
3394          ptr.map += 128 / 4;
3395       }
3396    }
3397
3398    for (unsigned i = 0; i < fs_sampler->num; i++) {
3399       for (unsigned j = 0; j < fs_sampler->array_size[i]; j++) {
3400          const struct tu_sampler *sampler = sampler_ptr(descriptors_state,
3401                                                         fs_sampler, i, j);
3402          memcpy(ptr.map, &border_color[sampler->border], 128);
3403          ptr.map += 128 / 4;
3404       }
3405    }
3406
3407    tu_cs_emit_pkt4(cs, REG_A6XX_SP_TP_BORDER_COLOR_BASE_ADDR_LO, 2);
3408    tu_cs_emit_qw(cs, ptr.iova);
3409    return VK_SUCCESS;
3410 }
3411
3412 static VkResult
3413 tu6_bind_draw_states(struct tu_cmd_buffer *cmd,
3414                      struct tu_cs *cs,
3415                      const struct tu_draw_info *draw)
3416 {
3417    const struct tu_pipeline *pipeline = cmd->state.pipeline;
3418    const struct tu_dynamic_state *dynamic = &cmd->state.dynamic;
3419    struct tu_draw_state_group draw_state_groups[TU_DRAW_STATE_COUNT];
3420    uint32_t draw_state_group_count = 0;
3421
3422    struct tu_descriptor_state *descriptors_state =
3423       &cmd->descriptors[VK_PIPELINE_BIND_POINT_GRAPHICS];
3424
3425    VkResult result = tu_cs_reserve_space(cmd->device, cs, 256);
3426    if (result != VK_SUCCESS)
3427       return result;
3428
3429    /* TODO lrz */
3430
3431    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9806, 0);
3432    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9990, 0);
3433    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_UNKNOWN_A008, 0);
3434
3435    tu_cs_emit_regs(cs,
3436                    A6XX_PC_PRIMITIVE_CNTL_0(.primitive_restart =
3437                                             pipeline->ia.primitive_restart && draw->indexed));
3438
3439    if (cmd->state.dirty &
3440           (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_DYNAMIC_LINE_WIDTH) &&
3441        (pipeline->dynamic_state.mask & TU_DYNAMIC_LINE_WIDTH)) {
3442       tu6_emit_gras_su_cntl(cs, pipeline->rast.gras_su_cntl,
3443                             dynamic->line_width);
3444    }
3445
3446    if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK) &&
3447        (pipeline->dynamic_state.mask & TU_DYNAMIC_STENCIL_COMPARE_MASK)) {
3448       tu6_emit_stencil_compare_mask(cs, dynamic->stencil_compare_mask.front,
3449                                     dynamic->stencil_compare_mask.back);
3450    }
3451
3452    if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK) &&
3453        (pipeline->dynamic_state.mask & TU_DYNAMIC_STENCIL_WRITE_MASK)) {
3454       tu6_emit_stencil_write_mask(cs, dynamic->stencil_write_mask.front,
3455                                   dynamic->stencil_write_mask.back);
3456    }
3457
3458    if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE) &&
3459        (pipeline->dynamic_state.mask & TU_DYNAMIC_STENCIL_REFERENCE)) {
3460       tu6_emit_stencil_reference(cs, dynamic->stencil_reference.front,
3461                                  dynamic->stencil_reference.back);
3462    }
3463
3464    if (cmd->state.dirty &
3465        (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_VERTEX_BUFFERS)) {
3466       for (uint32_t i = 0; i < pipeline->vi.count; i++) {
3467          const uint32_t binding = pipeline->vi.bindings[i];
3468          const uint32_t stride = pipeline->vi.strides[i];
3469          const struct tu_buffer *buf = cmd->state.vb.buffers[binding];
3470          const VkDeviceSize offset = buf->bo_offset +
3471                                      cmd->state.vb.offsets[binding] +
3472                                      pipeline->vi.offsets[i];
3473          const VkDeviceSize size =
3474             offset < buf->bo->size ? buf->bo->size - offset : 0;
3475
3476          tu_cs_emit_regs(cs,
3477                          A6XX_VFD_FETCH_BASE(i, .bo = buf->bo, .bo_offset = offset),
3478                          A6XX_VFD_FETCH_SIZE(i, size),
3479                          A6XX_VFD_FETCH_STRIDE(i, stride));
3480       }
3481    }
3482
3483    if (cmd->state.dirty & TU_CMD_DIRTY_PIPELINE) {
3484       draw_state_groups[draw_state_group_count++] =
3485          (struct tu_draw_state_group) {
3486             .id = TU_DRAW_STATE_PROGRAM,
3487             .enable_mask = ENABLE_DRAW,
3488             .ib = pipeline->program.state_ib,
3489          };
3490       draw_state_groups[draw_state_group_count++] =
3491          (struct tu_draw_state_group) {
3492             .id = TU_DRAW_STATE_PROGRAM_BINNING,
3493             .enable_mask = CP_SET_DRAW_STATE__0_BINNING,
3494             .ib = pipeline->program.binning_state_ib,
3495          };
3496       draw_state_groups[draw_state_group_count++] =
3497          (struct tu_draw_state_group) {
3498             .id = TU_DRAW_STATE_VI,
3499             .enable_mask = ENABLE_DRAW,
3500             .ib = pipeline->vi.state_ib,
3501          };
3502       draw_state_groups[draw_state_group_count++] =
3503          (struct tu_draw_state_group) {
3504             .id = TU_DRAW_STATE_VI_BINNING,
3505             .enable_mask = CP_SET_DRAW_STATE__0_BINNING,
3506             .ib = pipeline->vi.binning_state_ib,
3507          };
3508       draw_state_groups[draw_state_group_count++] =
3509          (struct tu_draw_state_group) {
3510             .id = TU_DRAW_STATE_VP,
3511             .enable_mask = ENABLE_ALL,
3512             .ib = pipeline->vp.state_ib,
3513          };
3514       draw_state_groups[draw_state_group_count++] =
3515          (struct tu_draw_state_group) {
3516             .id = TU_DRAW_STATE_RAST,
3517             .enable_mask = ENABLE_ALL,
3518             .ib = pipeline->rast.state_ib,
3519          };
3520       draw_state_groups[draw_state_group_count++] =
3521          (struct tu_draw_state_group) {
3522             .id = TU_DRAW_STATE_DS,
3523             .enable_mask = ENABLE_ALL,
3524             .ib = pipeline->ds.state_ib,
3525          };
3526       draw_state_groups[draw_state_group_count++] =
3527          (struct tu_draw_state_group) {
3528             .id = TU_DRAW_STATE_BLEND,
3529             .enable_mask = ENABLE_ALL,
3530             .ib = pipeline->blend.state_ib,
3531          };
3532    }
3533
3534    if (cmd->state.dirty &
3535          (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_DESCRIPTOR_SETS | TU_CMD_DIRTY_PUSH_CONSTANTS)) {
3536       draw_state_groups[draw_state_group_count++] =
3537          (struct tu_draw_state_group) {
3538             .id = TU_DRAW_STATE_VS_CONST,
3539             .enable_mask = ENABLE_ALL,
3540             .ib = tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_VERTEX)
3541          };
3542       draw_state_groups[draw_state_group_count++] =
3543          (struct tu_draw_state_group) {
3544             .id = TU_DRAW_STATE_FS_CONST,
3545             .enable_mask = ENABLE_DRAW,
3546             .ib = tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_FRAGMENT)
3547          };
3548    }
3549
3550    if (cmd->state.dirty &
3551          (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_DESCRIPTOR_SETS)) {
3552       bool needs_border = false;
3553       struct tu_cs_entry vs_tex, fs_tex, fs_ibo;
3554
3555       result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3556                                  MESA_SHADER_VERTEX, &vs_tex, &needs_border);
3557       if (result != VK_SUCCESS)
3558          return result;
3559
3560       result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3561                                  MESA_SHADER_FRAGMENT, &fs_tex, &needs_border);
3562       if (result != VK_SUCCESS)
3563          return result;
3564
3565       result = tu6_emit_ibo(cmd, pipeline, descriptors_state,
3566                             MESA_SHADER_FRAGMENT, &fs_ibo);
3567       if (result != VK_SUCCESS)
3568          return result;
3569
3570       draw_state_groups[draw_state_group_count++] =
3571          (struct tu_draw_state_group) {
3572             .id = TU_DRAW_STATE_VS_TEX,
3573             .enable_mask = ENABLE_ALL,
3574             .ib = vs_tex,
3575          };
3576       draw_state_groups[draw_state_group_count++] =
3577          (struct tu_draw_state_group) {
3578             .id = TU_DRAW_STATE_FS_TEX,
3579             .enable_mask = ENABLE_DRAW,
3580             .ib = fs_tex,
3581          };
3582       draw_state_groups[draw_state_group_count++] =
3583          (struct tu_draw_state_group) {
3584             .id = TU_DRAW_STATE_FS_IBO,
3585             .enable_mask = ENABLE_DRAW,
3586             .ib = fs_ibo,
3587          };
3588
3589       if (needs_border) {
3590          result = tu6_emit_border_color(cmd, cs);
3591          if (result != VK_SUCCESS)
3592             return result;
3593       }
3594    }
3595
3596    struct tu_cs_entry vs_params;
3597    result = tu6_emit_vs_params(cmd, draw, &vs_params);
3598    if (result != VK_SUCCESS)
3599       return result;
3600
3601    draw_state_groups[draw_state_group_count++] =
3602       (struct tu_draw_state_group) {
3603          .id = TU_DRAW_STATE_VS_PARAMS,
3604          .enable_mask = ENABLE_ALL,
3605          .ib = vs_params,
3606       };
3607
3608    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3 * draw_state_group_count);
3609    for (uint32_t i = 0; i < draw_state_group_count; i++) {
3610       const struct tu_draw_state_group *group = &draw_state_groups[i];
3611       debug_assert((group->enable_mask & ~ENABLE_ALL) == 0);
3612       uint32_t cp_set_draw_state =
3613          CP_SET_DRAW_STATE__0_COUNT(group->ib.size / 4) |
3614          group->enable_mask |
3615          CP_SET_DRAW_STATE__0_GROUP_ID(group->id);
3616       uint64_t iova;
3617       if (group->ib.size) {
3618          iova = group->ib.bo->iova + group->ib.offset;
3619       } else {
3620          cp_set_draw_state |= CP_SET_DRAW_STATE__0_DISABLE;
3621          iova = 0;
3622       }
3623
3624       tu_cs_emit(cs, cp_set_draw_state);
3625       tu_cs_emit_qw(cs, iova);
3626    }
3627
3628    tu_cs_sanity_check(cs);
3629
3630    /* track BOs */
3631    if (cmd->state.dirty & TU_CMD_DIRTY_VERTEX_BUFFERS) {
3632       for (uint32_t i = 0; i < MAX_VBS; i++) {
3633          const struct tu_buffer *buf = cmd->state.vb.buffers[i];
3634          if (buf)
3635             tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
3636       }
3637    }
3638    if (cmd->state.dirty & TU_CMD_DIRTY_DESCRIPTOR_SETS) {
3639       unsigned i;
3640       for_each_bit(i, descriptors_state->valid) {
3641          struct tu_descriptor_set *set = descriptors_state->sets[i];
3642          for (unsigned j = 0; j < set->layout->buffer_count; ++j)
3643             if (set->descriptors[j]) {
3644                tu_bo_list_add(&cmd->bo_list, set->descriptors[j],
3645                               MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3646             }
3647       }
3648    }
3649
3650    /* Fragment shader state overwrites compute shader state, so flag the
3651     * compute pipeline for re-emit.
3652     */
3653    cmd->state.dirty = TU_CMD_DIRTY_COMPUTE_PIPELINE;
3654    return VK_SUCCESS;
3655 }
3656
3657 static void
3658 tu6_emit_draw_direct(struct tu_cmd_buffer *cmd,
3659                      struct tu_cs *cs,
3660                      const struct tu_draw_info *draw)
3661 {
3662
3663    const enum pc_di_primtype primtype = cmd->state.pipeline->ia.primtype;
3664
3665    tu_cs_emit_regs(cs,
3666                    A6XX_VFD_INDEX_OFFSET(draw->vertex_offset),
3667                    A6XX_VFD_INSTANCE_START_OFFSET(draw->first_instance));
3668
3669    /* TODO hw binning */
3670    if (draw->indexed) {
3671       const enum a4xx_index_size index_size =
3672          tu6_index_size(cmd->state.index_type);
3673       const uint32_t index_bytes =
3674          (cmd->state.index_type == VK_INDEX_TYPE_UINT32) ? 4 : 2;
3675       const struct tu_buffer *buf = cmd->state.index_buffer;
3676       const VkDeviceSize offset = buf->bo_offset + cmd->state.index_offset +
3677                                   index_bytes * draw->first_index;
3678       const uint32_t size = index_bytes * draw->count;
3679
3680       const uint32_t cp_draw_indx =
3681          CP_DRAW_INDX_OFFSET_0_PRIM_TYPE(primtype) |
3682          CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT(DI_SRC_SEL_DMA) |
3683          CP_DRAW_INDX_OFFSET_0_INDEX_SIZE(index_size) |
3684          CP_DRAW_INDX_OFFSET_0_VIS_CULL(USE_VISIBILITY) | 0x2000;
3685
3686       tu_cs_emit_pkt7(cs, CP_DRAW_INDX_OFFSET, 7);
3687       tu_cs_emit(cs, cp_draw_indx);
3688       tu_cs_emit(cs, draw->instance_count);
3689       tu_cs_emit(cs, draw->count);
3690       tu_cs_emit(cs, 0x0); /* XXX */
3691       tu_cs_emit_qw(cs, buf->bo->iova + offset);
3692       tu_cs_emit(cs, size);
3693    } else {
3694       const uint32_t cp_draw_indx =
3695          CP_DRAW_INDX_OFFSET_0_PRIM_TYPE(primtype) |
3696          CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT(DI_SRC_SEL_AUTO_INDEX) |
3697          CP_DRAW_INDX_OFFSET_0_VIS_CULL(USE_VISIBILITY) | 0x2000;
3698
3699       tu_cs_emit_pkt7(cs, CP_DRAW_INDX_OFFSET, 3);
3700       tu_cs_emit(cs, cp_draw_indx);
3701       tu_cs_emit(cs, draw->instance_count);
3702       tu_cs_emit(cs, draw->count);
3703    }
3704 }
3705
3706 static void
3707 tu_draw(struct tu_cmd_buffer *cmd, const struct tu_draw_info *draw)
3708 {
3709    struct tu_cs *cs = &cmd->draw_cs;
3710    VkResult result;
3711
3712    result = tu6_bind_draw_states(cmd, cs, draw);
3713    if (result != VK_SUCCESS) {
3714       cmd->record_result = result;
3715       return;
3716    }
3717
3718    result = tu_cs_reserve_space(cmd->device, cs, 32);
3719    if (result != VK_SUCCESS) {
3720       cmd->record_result = result;
3721       return;
3722    }
3723
3724    if (draw->indirect) {
3725       tu_finishme("indirect draw");
3726       return;
3727    }
3728
3729    /* TODO tu6_emit_marker should pick different regs depending on cs */
3730
3731    tu6_emit_marker(cmd, cs);
3732    tu6_emit_draw_direct(cmd, cs, draw);
3733    tu6_emit_marker(cmd, cs);
3734
3735    cmd->wait_for_idle = true;
3736
3737    tu_cs_sanity_check(cs);
3738 }
3739
3740 void
3741 tu_CmdDraw(VkCommandBuffer commandBuffer,
3742            uint32_t vertexCount,
3743            uint32_t instanceCount,
3744            uint32_t firstVertex,
3745            uint32_t firstInstance)
3746 {
3747    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3748    struct tu_draw_info info = {};
3749
3750    info.count = vertexCount;
3751    info.instance_count = instanceCount;
3752    info.first_instance = firstInstance;
3753    info.vertex_offset = firstVertex;
3754
3755    tu_draw(cmd_buffer, &info);
3756 }
3757
3758 void
3759 tu_CmdDrawIndexed(VkCommandBuffer commandBuffer,
3760                   uint32_t indexCount,
3761                   uint32_t instanceCount,
3762                   uint32_t firstIndex,
3763                   int32_t vertexOffset,
3764                   uint32_t firstInstance)
3765 {
3766    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3767    struct tu_draw_info info = {};
3768
3769    info.indexed = true;
3770    info.count = indexCount;
3771    info.instance_count = instanceCount;
3772    info.first_index = firstIndex;
3773    info.vertex_offset = vertexOffset;
3774    info.first_instance = firstInstance;
3775
3776    tu_draw(cmd_buffer, &info);
3777 }
3778
3779 void
3780 tu_CmdDrawIndirect(VkCommandBuffer commandBuffer,
3781                    VkBuffer _buffer,
3782                    VkDeviceSize offset,
3783                    uint32_t drawCount,
3784                    uint32_t stride)
3785 {
3786    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3787    TU_FROM_HANDLE(tu_buffer, buffer, _buffer);
3788    struct tu_draw_info info = {};
3789
3790    info.count = drawCount;
3791    info.indirect = buffer;
3792    info.indirect_offset = offset;
3793    info.stride = stride;
3794
3795    tu_draw(cmd_buffer, &info);
3796 }
3797
3798 void
3799 tu_CmdDrawIndexedIndirect(VkCommandBuffer commandBuffer,
3800                           VkBuffer _buffer,
3801                           VkDeviceSize offset,
3802                           uint32_t drawCount,
3803                           uint32_t stride)
3804 {
3805    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3806    TU_FROM_HANDLE(tu_buffer, buffer, _buffer);
3807    struct tu_draw_info info = {};
3808
3809    info.indexed = true;
3810    info.count = drawCount;
3811    info.indirect = buffer;
3812    info.indirect_offset = offset;
3813    info.stride = stride;
3814
3815    tu_draw(cmd_buffer, &info);
3816 }
3817
3818 struct tu_dispatch_info
3819 {
3820    /**
3821     * Determine the layout of the grid (in block units) to be used.
3822     */
3823    uint32_t blocks[3];
3824
3825    /**
3826     * A starting offset for the grid. If unaligned is set, the offset
3827     * must still be aligned.
3828     */
3829    uint32_t offsets[3];
3830    /**
3831     * Whether it's an unaligned compute dispatch.
3832     */
3833    bool unaligned;
3834
3835    /**
3836     * Indirect compute parameters resource.
3837     */
3838    struct tu_buffer *indirect;
3839    uint64_t indirect_offset;
3840 };
3841
3842 static void
3843 tu_emit_compute_driver_params(struct tu_cs *cs, struct tu_pipeline *pipeline,
3844                               const struct tu_dispatch_info *info)
3845 {
3846    gl_shader_stage type = MESA_SHADER_COMPUTE;
3847    const struct tu_program_descriptor_linkage *link =
3848       &pipeline->program.link[type];
3849    const struct ir3_const_state *const_state = &link->const_state;
3850    uint32_t offset = const_state->offsets.driver_param;
3851
3852    if (link->constlen <= offset)
3853       return;
3854
3855    if (!info->indirect) {
3856       uint32_t driver_params[IR3_DP_CS_COUNT] = {
3857          [IR3_DP_NUM_WORK_GROUPS_X] = info->blocks[0],
3858          [IR3_DP_NUM_WORK_GROUPS_Y] = info->blocks[1],
3859          [IR3_DP_NUM_WORK_GROUPS_Z] = info->blocks[2],
3860          [IR3_DP_LOCAL_GROUP_SIZE_X] = pipeline->compute.local_size[0],
3861          [IR3_DP_LOCAL_GROUP_SIZE_Y] = pipeline->compute.local_size[1],
3862          [IR3_DP_LOCAL_GROUP_SIZE_Z] = pipeline->compute.local_size[2],
3863       };
3864
3865       uint32_t num_consts = MIN2(const_state->num_driver_params,
3866                                  (link->constlen - offset) * 4);
3867       /* push constants */
3868       tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + num_consts);
3869       tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(offset) |
3870                  CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
3871                  CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
3872                  CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
3873                  CP_LOAD_STATE6_0_NUM_UNIT(num_consts / 4));
3874       tu_cs_emit(cs, 0);
3875       tu_cs_emit(cs, 0);
3876       uint32_t i;
3877       for (i = 0; i < num_consts; i++)
3878          tu_cs_emit(cs, driver_params[i]);
3879    } else {
3880       tu_finishme("Indirect driver params");
3881    }
3882 }
3883
3884 static void
3885 tu_dispatch(struct tu_cmd_buffer *cmd,
3886             const struct tu_dispatch_info *info)
3887 {
3888    struct tu_cs *cs = &cmd->cs;
3889    struct tu_pipeline *pipeline = cmd->state.compute_pipeline;
3890    struct tu_descriptor_state *descriptors_state =
3891       &cmd->descriptors[VK_PIPELINE_BIND_POINT_COMPUTE];
3892
3893    VkResult result = tu_cs_reserve_space(cmd->device, cs, 256);
3894    if (result != VK_SUCCESS) {
3895       cmd->record_result = result;
3896       return;
3897    }
3898
3899    if (cmd->state.dirty & TU_CMD_DIRTY_COMPUTE_PIPELINE)
3900       tu_cs_emit_ib(cs, &pipeline->program.state_ib);
3901
3902    struct tu_cs_entry ib;
3903
3904    ib = tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_COMPUTE);
3905    if (ib.size)
3906       tu_cs_emit_ib(cs, &ib);
3907
3908    tu_emit_compute_driver_params(cs, pipeline, info);
3909
3910    bool needs_border;
3911    result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3912                               MESA_SHADER_COMPUTE, &ib, &needs_border);
3913    if (result != VK_SUCCESS) {
3914       cmd->record_result = result;
3915       return;
3916    }
3917
3918    if (ib.size)
3919       tu_cs_emit_ib(cs, &ib);
3920
3921    if (needs_border)
3922       tu_finishme("compute border color");
3923
3924    result = tu6_emit_ibo(cmd, pipeline, descriptors_state, MESA_SHADER_COMPUTE, &ib);
3925    if (result != VK_SUCCESS) {
3926       cmd->record_result = result;
3927       return;
3928    }
3929
3930    if (ib.size)
3931       tu_cs_emit_ib(cs, &ib);
3932
3933    /* track BOs */
3934    if (cmd->state.dirty & TU_CMD_DIRTY_DESCRIPTOR_SETS) {
3935       unsigned i;
3936       for_each_bit(i, descriptors_state->valid) {
3937          struct tu_descriptor_set *set = descriptors_state->sets[i];
3938          for (unsigned j = 0; j < set->layout->buffer_count; ++j)
3939             if (set->descriptors[j]) {
3940                tu_bo_list_add(&cmd->bo_list, set->descriptors[j],
3941                               MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3942             }
3943       }
3944    }
3945
3946    /* Compute shader state overwrites fragment shader state, so we flag the
3947     * graphics pipeline for re-emit.
3948     */
3949    cmd->state.dirty = TU_CMD_DIRTY_PIPELINE;
3950
3951    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
3952    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(0x8));
3953
3954    const uint32_t *local_size = pipeline->compute.local_size;
3955    const uint32_t *num_groups = info->blocks;
3956    tu_cs_emit_regs(cs,
3957                    A6XX_HLSQ_CS_NDRANGE_0(.kerneldim = 3,
3958                                           .localsizex = local_size[0] - 1,
3959                                           .localsizey = local_size[1] - 1,
3960                                           .localsizez = local_size[2] - 1),
3961                    A6XX_HLSQ_CS_NDRANGE_1(.globalsize_x = local_size[0] * num_groups[0]),
3962                    A6XX_HLSQ_CS_NDRANGE_2(.globaloff_x = 0),
3963                    A6XX_HLSQ_CS_NDRANGE_3(.globalsize_y = local_size[1] * num_groups[1]),
3964                    A6XX_HLSQ_CS_NDRANGE_4(.globaloff_y = 0),
3965                    A6XX_HLSQ_CS_NDRANGE_5(.globalsize_z = local_size[2] * num_groups[2]),
3966                    A6XX_HLSQ_CS_NDRANGE_6(.globaloff_z = 0));
3967
3968    tu_cs_emit_regs(cs,
3969                    A6XX_HLSQ_CS_KERNEL_GROUP_X(1),
3970                    A6XX_HLSQ_CS_KERNEL_GROUP_Y(1),
3971                    A6XX_HLSQ_CS_KERNEL_GROUP_Z(1));
3972
3973    if (info->indirect) {
3974       uint64_t iova = tu_buffer_iova(info->indirect) + info->indirect_offset;
3975
3976       tu_bo_list_add(&cmd->bo_list, info->indirect->bo,
3977                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3978
3979       tu_cs_emit_pkt7(cs, CP_EXEC_CS_INDIRECT, 4);
3980       tu_cs_emit(cs, 0x00000000);
3981       tu_cs_emit_qw(cs, iova);
3982       tu_cs_emit(cs,
3983                  A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX(local_size[0] - 1) |
3984                  A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY(local_size[1] - 1) |
3985                  A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ(local_size[2] - 1));
3986    } else {
3987       tu_cs_emit_pkt7(cs, CP_EXEC_CS, 4);
3988       tu_cs_emit(cs, 0x00000000);
3989       tu_cs_emit(cs, CP_EXEC_CS_1_NGROUPS_X(info->blocks[0]));
3990       tu_cs_emit(cs, CP_EXEC_CS_2_NGROUPS_Y(info->blocks[1]));
3991       tu_cs_emit(cs, CP_EXEC_CS_3_NGROUPS_Z(info->blocks[2]));
3992    }
3993
3994    tu_cs_emit_wfi(cs);
3995
3996    tu6_emit_cache_flush(cmd, cs);
3997 }
3998
3999 void
4000 tu_CmdDispatchBase(VkCommandBuffer commandBuffer,
4001                    uint32_t base_x,
4002                    uint32_t base_y,
4003                    uint32_t base_z,
4004                    uint32_t x,
4005                    uint32_t y,
4006                    uint32_t z)
4007 {
4008    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
4009    struct tu_dispatch_info info = {};
4010
4011    info.blocks[0] = x;
4012    info.blocks[1] = y;
4013    info.blocks[2] = z;
4014
4015    info.offsets[0] = base_x;
4016    info.offsets[1] = base_y;
4017    info.offsets[2] = base_z;
4018    tu_dispatch(cmd_buffer, &info);
4019 }
4020
4021 void
4022 tu_CmdDispatch(VkCommandBuffer commandBuffer,
4023                uint32_t x,
4024                uint32_t y,
4025                uint32_t z)
4026 {
4027    tu_CmdDispatchBase(commandBuffer, 0, 0, 0, x, y, z);
4028 }
4029
4030 void
4031 tu_CmdDispatchIndirect(VkCommandBuffer commandBuffer,
4032                        VkBuffer _buffer,
4033                        VkDeviceSize offset)
4034 {
4035    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
4036    TU_FROM_HANDLE(tu_buffer, buffer, _buffer);
4037    struct tu_dispatch_info info = {};
4038
4039    info.indirect = buffer;
4040    info.indirect_offset = offset;
4041
4042    tu_dispatch(cmd_buffer, &info);
4043 }
4044
4045 void
4046 tu_CmdEndRenderPass(VkCommandBuffer commandBuffer)
4047 {
4048    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
4049
4050    tu_cs_end(&cmd_buffer->draw_cs);
4051    tu_cs_end(&cmd_buffer->draw_epilogue_cs);
4052
4053    if (use_sysmem_rendering(cmd_buffer))
4054       tu_cmd_render_sysmem(cmd_buffer);
4055    else
4056       tu_cmd_render_tiles(cmd_buffer);
4057
4058    /* discard draw_cs and draw_epilogue_cs entries now that the tiles are
4059       rendered */
4060    tu_cs_discard_entries(&cmd_buffer->draw_cs);
4061    tu_cs_begin(&cmd_buffer->draw_cs);
4062    tu_cs_discard_entries(&cmd_buffer->draw_epilogue_cs);
4063    tu_cs_begin(&cmd_buffer->draw_epilogue_cs);
4064
4065    cmd_buffer->state.pass = NULL;
4066    cmd_buffer->state.subpass = NULL;
4067    cmd_buffer->state.framebuffer = NULL;
4068 }
4069
4070 void
4071 tu_CmdEndRenderPass2(VkCommandBuffer commandBuffer,
4072                      const VkSubpassEndInfoKHR *pSubpassEndInfo)
4073 {
4074    tu_CmdEndRenderPass(commandBuffer);
4075 }
4076
4077 struct tu_barrier_info
4078 {
4079    uint32_t eventCount;
4080    const VkEvent *pEvents;
4081    VkPipelineStageFlags srcStageMask;
4082 };
4083
4084 static void
4085 tu_barrier(struct tu_cmd_buffer *cmd_buffer,
4086            uint32_t memoryBarrierCount,
4087            const VkMemoryBarrier *pMemoryBarriers,
4088            uint32_t bufferMemoryBarrierCount,
4089            const VkBufferMemoryBarrier *pBufferMemoryBarriers,
4090            uint32_t imageMemoryBarrierCount,
4091            const VkImageMemoryBarrier *pImageMemoryBarriers,
4092            const struct tu_barrier_info *info)
4093 {
4094 }
4095
4096 void
4097 tu_CmdPipelineBarrier(VkCommandBuffer commandBuffer,
4098                       VkPipelineStageFlags srcStageMask,
4099                       VkPipelineStageFlags destStageMask,
4100                       VkBool32 byRegion,
4101                       uint32_t memoryBarrierCount,
4102                       const VkMemoryBarrier *pMemoryBarriers,
4103                       uint32_t bufferMemoryBarrierCount,
4104                       const VkBufferMemoryBarrier *pBufferMemoryBarriers,
4105                       uint32_t imageMemoryBarrierCount,
4106                       const VkImageMemoryBarrier *pImageMemoryBarriers)
4107 {
4108    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
4109    struct tu_barrier_info info;
4110
4111    info.eventCount = 0;
4112    info.pEvents = NULL;
4113    info.srcStageMask = srcStageMask;
4114
4115    tu_barrier(cmd_buffer, memoryBarrierCount, pMemoryBarriers,
4116               bufferMemoryBarrierCount, pBufferMemoryBarriers,
4117               imageMemoryBarrierCount, pImageMemoryBarriers, &info);
4118 }
4119
4120 static void
4121 write_event(struct tu_cmd_buffer *cmd, struct tu_event *event, unsigned value)
4122 {
4123    struct tu_cs *cs = &cmd->cs;
4124
4125    VkResult result = tu_cs_reserve_space(cmd->device, cs, 4);
4126    if (result != VK_SUCCESS) {
4127       cmd->record_result = result;
4128       return;
4129    }
4130
4131    tu_bo_list_add(&cmd->bo_list, &event->bo, MSM_SUBMIT_BO_WRITE);
4132
4133    /* TODO: any flush required before/after ? */
4134
4135    tu_cs_emit_pkt7(cs, CP_MEM_WRITE, 3);
4136    tu_cs_emit_qw(cs, event->bo.iova); /* ADDR_LO/HI */
4137    tu_cs_emit(cs, value);
4138 }
4139
4140 void
4141 tu_CmdSetEvent(VkCommandBuffer commandBuffer,
4142                VkEvent _event,
4143                VkPipelineStageFlags stageMask)
4144 {
4145    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
4146    TU_FROM_HANDLE(tu_event, event, _event);
4147
4148    write_event(cmd, event, 1);
4149 }
4150
4151 void
4152 tu_CmdResetEvent(VkCommandBuffer commandBuffer,
4153                  VkEvent _event,
4154                  VkPipelineStageFlags stageMask)
4155 {
4156    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
4157    TU_FROM_HANDLE(tu_event, event, _event);
4158
4159    write_event(cmd, event, 0);
4160 }
4161
4162 void
4163 tu_CmdWaitEvents(VkCommandBuffer commandBuffer,
4164                  uint32_t eventCount,
4165                  const VkEvent *pEvents,
4166                  VkPipelineStageFlags srcStageMask,
4167                  VkPipelineStageFlags dstStageMask,
4168                  uint32_t memoryBarrierCount,
4169                  const VkMemoryBarrier *pMemoryBarriers,
4170                  uint32_t bufferMemoryBarrierCount,
4171                  const VkBufferMemoryBarrier *pBufferMemoryBarriers,
4172                  uint32_t imageMemoryBarrierCount,
4173                  const VkImageMemoryBarrier *pImageMemoryBarriers)
4174 {
4175    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
4176    struct tu_cs *cs = &cmd->cs;
4177
4178    VkResult result = tu_cs_reserve_space(cmd->device, cs, eventCount * 7);
4179    if (result != VK_SUCCESS) {
4180       cmd->record_result = result;
4181       return;
4182    }
4183
4184    /* TODO: any flush required before/after? (CP_WAIT_FOR_ME?) */
4185
4186    for (uint32_t i = 0; i < eventCount; i++) {
4187       const struct tu_event *event = (const struct tu_event*) pEvents[i];
4188
4189       tu_bo_list_add(&cmd->bo_list, &event->bo, MSM_SUBMIT_BO_READ);
4190
4191       tu_cs_emit_pkt7(cs, CP_WAIT_REG_MEM, 6);
4192       tu_cs_emit(cs, CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ) |
4193                      CP_WAIT_REG_MEM_0_POLL_MEMORY);
4194       tu_cs_emit_qw(cs, event->bo.iova); /* POLL_ADDR_LO/HI */
4195       tu_cs_emit(cs, CP_WAIT_REG_MEM_3_REF(1));
4196       tu_cs_emit(cs, CP_WAIT_REG_MEM_4_MASK(~0u));
4197       tu_cs_emit(cs, CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(20));
4198    }
4199 }
4200
4201 void
4202 tu_CmdSetDeviceMask(VkCommandBuffer commandBuffer, uint32_t deviceMask)
4203 {
4204    /* No-op */
4205 }