src/freedreno/vulkan/tu_cmd_buffer.c

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * based in part on anv driver which is:
   6  * Copyright © 2015 Intel Corporation
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25  * DEALINGS IN THE SOFTWARE.
  26  */
  27
  28 #include "tu_private.h"
  29
  30 #include "registers/adreno_pm4.xml.h"
  31 #include "registers/adreno_common.xml.h"
  32
  33 #include "vk_format.h"
  34
  35 #include "tu_cs.h"
  36 #include "tu_blit.h"
  37
  38 #define OVERFLOW_FLAG_REG REG_A6XX_CP_SCRATCH_REG(0)
  39
  40 void
  41 tu_bo_list_init(struct tu_bo_list *list)
  42 {
  43    list->count = list->capacity = 0;
  44    list->bo_infos = NULL;
  45 }
  46
  47 void
  48 tu_bo_list_destroy(struct tu_bo_list *list)
  49 {
  50    free(list->bo_infos);
  51 }
  52
  53 void
  54 tu_bo_list_reset(struct tu_bo_list *list)
  55 {
  56    list->count = 0;
  57 }
  58
  59 /**
  60  * \a flags consists of MSM_SUBMIT_BO_FLAGS.
  61  */
  62 static uint32_t
  63 tu_bo_list_add_info(struct tu_bo_list *list,
  64                     const struct drm_msm_gem_submit_bo *bo_info)
  65 {
  66    assert(bo_info->handle != 0);
  67
  68    for (uint32_t i = 0; i < list->count; ++i) {
  69       if (list->bo_infos[i].handle == bo_info->handle) {
  70          assert(list->bo_infos[i].presumed == bo_info->presumed);
  71          list->bo_infos[i].flags |= bo_info->flags;
  72          return i;
  73       }
  74    }
  75
  76    /* grow list->bo_infos if needed */
  77    if (list->count == list->capacity) {
  78       uint32_t new_capacity = MAX2(2 * list->count, 16);
  79       struct drm_msm_gem_submit_bo *new_bo_infos = realloc(
  80          list->bo_infos, new_capacity * sizeof(struct drm_msm_gem_submit_bo));
  81       if (!new_bo_infos)
  82          return TU_BO_LIST_FAILED;
  83       list->bo_infos = new_bo_infos;
  84       list->capacity = new_capacity;
  85    }
  86
  87    list->bo_infos[list->count] = *bo_info;
  88    return list->count++;
  89 }
  90
  91 uint32_t
  92 tu_bo_list_add(struct tu_bo_list *list,
  93                const struct tu_bo *bo,
  94                uint32_t flags)
  95 {
  96    return tu_bo_list_add_info(list, &(struct drm_msm_gem_submit_bo) {
  97                                        .flags = flags,
  98                                        .handle = bo->gem_handle,
  99                                        .presumed = bo->iova,
 100                                     });
 101 }
 102
 103 VkResult
 104 tu_bo_list_merge(struct tu_bo_list *list, const struct tu_bo_list *other)
 105 {
 106    for (uint32_t i = 0; i < other->count; i++) {
 107       if (tu_bo_list_add_info(list, other->bo_infos + i) == TU_BO_LIST_FAILED)
 108          return VK_ERROR_OUT_OF_HOST_MEMORY;
 109    }
 110
 111    return VK_SUCCESS;
 112 }
 113
 114 static void
 115 tu_tiling_config_update_tile_layout(struct tu_tiling_config *tiling,
 116                                     const struct tu_device *dev,
 117                                     uint32_t pixels)
 118 {
 119    const uint32_t tile_align_w = dev->physical_device->tile_align_w;
 120    const uint32_t tile_align_h = dev->physical_device->tile_align_h;
 121    const uint32_t max_tile_width = 1024; /* A6xx */
 122
 123    tiling->tile0.offset = (VkOffset2D) {
 124       .x = tiling->render_area.offset.x & ~(tile_align_w - 1),
 125       .y = tiling->render_area.offset.y & ~(tile_align_h - 1),
 126    };
 127
 128    const uint32_t ra_width =
 129       tiling->render_area.extent.width +
 130       (tiling->render_area.offset.x - tiling->tile0.offset.x);
 131    const uint32_t ra_height =
 132       tiling->render_area.extent.height +
 133       (tiling->render_area.offset.y - tiling->tile0.offset.y);
 134
 135    /* start from 1 tile */
 136    tiling->tile_count = (VkExtent2D) {
 137       .width = 1,
 138       .height = 1,
 139    };
 140    tiling->tile0.extent = (VkExtent2D) {
 141       .width = align(ra_width, tile_align_w),
 142       .height = align(ra_height, tile_align_h),
 143    };
 144
 145    /* do not exceed max tile width */
 146    while (tiling->tile0.extent.width > max_tile_width) {
 147       tiling->tile_count.width++;
 148       tiling->tile0.extent.width =
 149          align(ra_width / tiling->tile_count.width, tile_align_w);
 150    }
 151
 152    /* do not exceed gmem size */
 153    while (tiling->tile0.extent.width * tiling->tile0.extent.height > pixels) {
 154       if (tiling->tile0.extent.width > MAX2(tile_align_w, tiling->tile0.extent.height)) {
 155          tiling->tile_count.width++;
 156          tiling->tile0.extent.width =
 157             align(DIV_ROUND_UP(ra_width, tiling->tile_count.width), tile_align_w);
 158       } else {
 159          /* if this assert fails then layout is impossible.. */
 160          assert(tiling->tile0.extent.height > tile_align_h);
 161          tiling->tile_count.height++;
 162          tiling->tile0.extent.height =
 163             align(DIV_ROUND_UP(ra_height, tiling->tile_count.height), tile_align_h);
 164       }
 165    }
 166 }
 167
 168 static void
 169 tu_tiling_config_update_pipe_layout(struct tu_tiling_config *tiling,
 170                                     const struct tu_device *dev)
 171 {
 172    const uint32_t max_pipe_count = 32; /* A6xx */
 173
 174    /* start from 1 tile per pipe */
 175    tiling->pipe0 = (VkExtent2D) {
 176       .width = 1,
 177       .height = 1,
 178    };
 179    tiling->pipe_count = tiling->tile_count;
 180
 181    /* do not exceed max pipe count vertically */
 182    while (tiling->pipe_count.height > max_pipe_count) {
 183       tiling->pipe0.height += 2;
 184       tiling->pipe_count.height =
 185          (tiling->tile_count.height + tiling->pipe0.height - 1) /
 186          tiling->pipe0.height;
 187    }
 188
 189    /* do not exceed max pipe count */
 190    while (tiling->pipe_count.width * tiling->pipe_count.height >
 191           max_pipe_count) {
 192       tiling->pipe0.width += 1;
 193       tiling->pipe_count.width =
 194          (tiling->tile_count.width + tiling->pipe0.width - 1) /
 195          tiling->pipe0.width;
 196    }
 197 }
 198
 199 static void
 200 tu_tiling_config_update_pipes(struct tu_tiling_config *tiling,
 201                               const struct tu_device *dev)
 202 {
 203    const uint32_t max_pipe_count = 32; /* A6xx */
 204    const uint32_t used_pipe_count =
 205       tiling->pipe_count.width * tiling->pipe_count.height;
 206    const VkExtent2D last_pipe = {
 207       .width = (tiling->tile_count.width - 1) % tiling->pipe0.width + 1,
 208       .height = (tiling->tile_count.height - 1) % tiling->pipe0.height + 1,
 209    };
 210
 211    assert(used_pipe_count <= max_pipe_count);
 212    assert(max_pipe_count <= ARRAY_SIZE(tiling->pipe_config));
 213
 214    for (uint32_t y = 0; y < tiling->pipe_count.height; y++) {
 215       for (uint32_t x = 0; x < tiling->pipe_count.width; x++) {
 216          const uint32_t pipe_x = tiling->pipe0.width * x;
 217          const uint32_t pipe_y = tiling->pipe0.height * y;
 218          const uint32_t pipe_w = (x == tiling->pipe_count.width - 1)
 219                                     ? last_pipe.width
 220                                     : tiling->pipe0.width;
 221          const uint32_t pipe_h = (y == tiling->pipe_count.height - 1)
 222                                     ? last_pipe.height
 223                                     : tiling->pipe0.height;
 224          const uint32_t n = tiling->pipe_count.width * y + x;
 225
 226          tiling->pipe_config[n] = A6XX_VSC_PIPE_CONFIG_REG_X(pipe_x) |
 227                                   A6XX_VSC_PIPE_CONFIG_REG_Y(pipe_y) |
 228                                   A6XX_VSC_PIPE_CONFIG_REG_W(pipe_w) |
 229                                   A6XX_VSC_PIPE_CONFIG_REG_H(pipe_h);
 230          tiling->pipe_sizes[n] = CP_SET_BIN_DATA5_0_VSC_SIZE(pipe_w * pipe_h);
 231       }
 232    }
 233
 234    memset(tiling->pipe_config + used_pipe_count, 0,
 235           sizeof(uint32_t) * (max_pipe_count - used_pipe_count));
 236 }
 237
 238 static void
 239 tu_tiling_config_get_tile(const struct tu_tiling_config *tiling,
 240                           const struct tu_device *dev,
 241                           uint32_t tx,
 242                           uint32_t ty,
 243                           struct tu_tile *tile)
 244 {
 245    /* find the pipe and the slot for tile (tx, ty) */
 246    const uint32_t px = tx / tiling->pipe0.width;
 247    const uint32_t py = ty / tiling->pipe0.height;
 248    const uint32_t sx = tx - tiling->pipe0.width * px;
 249    const uint32_t sy = ty - tiling->pipe0.height * py;
 250
 251    assert(tx < tiling->tile_count.width && ty < tiling->tile_count.height);
 252    assert(px < tiling->pipe_count.width && py < tiling->pipe_count.height);
 253    assert(sx < tiling->pipe0.width && sy < tiling->pipe0.height);
 254
 255    /* convert to 1D indices */
 256    tile->pipe = tiling->pipe_count.width * py + px;
 257    tile->slot = tiling->pipe0.width * sy + sx;
 258
 259    /* get the blit area for the tile */
 260    tile->begin = (VkOffset2D) {
 261       .x = tiling->tile0.offset.x + tiling->tile0.extent.width * tx,
 262       .y = tiling->tile0.offset.y + tiling->tile0.extent.height * ty,
 263    };
 264    tile->end.x =
 265       (tx == tiling->tile_count.width - 1)
 266          ? tiling->render_area.offset.x + tiling->render_area.extent.width
 267          : tile->begin.x + tiling->tile0.extent.width;
 268    tile->end.y =
 269       (ty == tiling->tile_count.height - 1)
 270          ? tiling->render_area.offset.y + tiling->render_area.extent.height
 271          : tile->begin.y + tiling->tile0.extent.height;
 272 }
 273
 274 enum a3xx_msaa_samples
 275 tu_msaa_samples(uint32_t samples)
 276 {
 277    switch (samples) {
 278    case 1:
 279       return MSAA_ONE;
 280    case 2:
 281       return MSAA_TWO;
 282    case 4:
 283       return MSAA_FOUR;
 284    case 8:
 285       return MSAA_EIGHT;
 286    default:
 287       assert(!"invalid sample count");
 288       return MSAA_ONE;
 289    }
 290 }
 291
 292 static enum a4xx_index_size
 293 tu6_index_size(VkIndexType type)
 294 {
 295    switch (type) {
 296    case VK_INDEX_TYPE_UINT16:
 297       return INDEX4_SIZE_16_BIT;
 298    case VK_INDEX_TYPE_UINT32:
 299       return INDEX4_SIZE_32_BIT;
 300    default:
 301       unreachable("invalid VkIndexType");
 302       return INDEX4_SIZE_8_BIT;
 303    }
 304 }
 305
 306 static void
 307 tu6_emit_marker(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 308 {
 309    tu_cs_emit_write_reg(cs, cmd->marker_reg, ++cmd->marker_seqno);
 310 }
 311
 312 unsigned
 313 tu6_emit_event_write(struct tu_cmd_buffer *cmd,
 314                      struct tu_cs *cs,
 315                      enum vgt_event_type event,
 316                      bool need_seqno)
 317 {
 318    unsigned seqno = 0;
 319
 320    tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, need_seqno ? 4 : 1);
 321    tu_cs_emit(cs, CP_EVENT_WRITE_0_EVENT(event));
 322    if (need_seqno) {
 323       tu_cs_emit_qw(cs, cmd->scratch_bo.iova);
 324       seqno = ++cmd->scratch_seqno;
 325       tu_cs_emit(cs, seqno);
 326    }
 327
 328    return seqno;
 329 }
 330
 331 static void
 332 tu6_emit_cache_flush(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 333 {
 334    tu6_emit_event_write(cmd, cs, 0x31, false);
 335 }
 336
 337 static void
 338 tu6_emit_lrz_flush(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 339 {
 340    tu6_emit_event_write(cmd, cs, LRZ_FLUSH, false);
 341 }
 342
 343 static void
 344 tu6_emit_wfi(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 345 {
 346    if (cmd->wait_for_idle) {
 347       tu_cs_emit_wfi(cs);
 348       cmd->wait_for_idle = false;
 349    }
 350 }
 351
 352 #define tu_image_view_ubwc_pitches(iview)                                \
 353    .pitch = tu_image_ubwc_pitch(iview->image, iview->base_mip),          \
 354    .array_pitch = tu_image_ubwc_size(iview->image, iview->base_mip) >> 2
 355
 356 static void
 357 tu6_emit_zs(struct tu_cmd_buffer *cmd,
 358             const struct tu_subpass *subpass,
 359             struct tu_cs *cs)
 360 {
 361    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 362
 363    const uint32_t a = subpass->depth_stencil_attachment.attachment;
 364    if (a == VK_ATTACHMENT_UNUSED) {
 365       tu_cs_emit_regs(cs,
 366                       A6XX_RB_DEPTH_BUFFER_INFO(.depth_format = DEPTH6_NONE),
 367                       A6XX_RB_DEPTH_BUFFER_PITCH(0),
 368                       A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH(0),
 369                       A6XX_RB_DEPTH_BUFFER_BASE(0),
 370                       A6XX_RB_DEPTH_BUFFER_BASE_GMEM(0));
 371
 372       tu_cs_emit_regs(cs,
 373                       A6XX_GRAS_SU_DEPTH_BUFFER_INFO(.depth_format = DEPTH6_NONE));
 374
 375       tu_cs_emit_regs(cs,
 376                       A6XX_GRAS_LRZ_BUFFER_BASE(0),
 377                       A6XX_GRAS_LRZ_BUFFER_PITCH(0),
 378                       A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE(0));
 379
 380       tu_cs_emit_regs(cs, A6XX_RB_STENCIL_INFO(0));
 381
 382       return;
 383    }
 384
 385    const struct tu_image_view *iview = fb->attachments[a].attachment;
 386    enum a6xx_depth_format fmt = tu6_pipe2depth(iview->vk_format);
 387
 388    tu_cs_emit_regs(cs,
 389                    A6XX_RB_DEPTH_BUFFER_INFO(.depth_format = fmt),
 390                    A6XX_RB_DEPTH_BUFFER_PITCH(tu_image_stride(iview->image, iview->base_mip)),
 391                    A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH(iview->image->layout.layer_size),
 392                    A6XX_RB_DEPTH_BUFFER_BASE(tu_image_view_base_ref(iview)),
 393                    A6XX_RB_DEPTH_BUFFER_BASE_GMEM(cmd->state.pass->attachments[a].gmem_offset));
 394
 395    tu_cs_emit_regs(cs,
 396                    A6XX_GRAS_SU_DEPTH_BUFFER_INFO(.depth_format = fmt));
 397
 398    tu_cs_emit_regs(cs,
 399                    A6XX_RB_DEPTH_FLAG_BUFFER_BASE(tu_image_view_ubwc_base_ref(iview)),
 400                    A6XX_RB_DEPTH_FLAG_BUFFER_PITCH(tu_image_view_ubwc_pitches(iview)));
 401
 402    tu_cs_emit_regs(cs,
 403                    A6XX_GRAS_LRZ_BUFFER_BASE(0),
 404                    A6XX_GRAS_LRZ_BUFFER_PITCH(0),
 405                    A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE(0));
 406
 407    tu_cs_emit_regs(cs,
 408                    A6XX_RB_STENCIL_INFO(0));
 409
 410    /* enable zs? */
 411 }
 412
 413 static void
 414 tu6_emit_mrt(struct tu_cmd_buffer *cmd,
 415              const struct tu_subpass *subpass,
 416              struct tu_cs *cs)
 417 {
 418    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 419    unsigned char mrt_comp[MAX_RTS] = { 0 };
 420    unsigned srgb_cntl = 0;
 421
 422    for (uint32_t i = 0; i < subpass->color_count; ++i) {
 423       uint32_t a = subpass->color_attachments[i].attachment;
 424       if (a == VK_ATTACHMENT_UNUSED)
 425          continue;
 426
 427       const struct tu_image_view *iview = fb->attachments[a].attachment;
 428       const enum a6xx_tile_mode tile_mode =
 429          tu6_get_image_tile_mode(iview->image, iview->base_mip);
 430
 431       mrt_comp[i] = 0xf;
 432
 433       if (vk_format_is_srgb(iview->vk_format))
 434          srgb_cntl |= (1 << i);
 435
 436       const struct tu_native_format *format =
 437          tu6_get_native_format(iview->vk_format);
 438       assert(format && format->rb >= 0);
 439
 440       tu_cs_emit_regs(cs,
 441                       A6XX_RB_MRT_BUF_INFO(i,
 442                                            .color_tile_mode = tile_mode,
 443                                            .color_format = format->rb,
 444                                            .color_swap = format->swap),
 445                       A6XX_RB_MRT_PITCH(i, tu_image_stride(iview->image, iview->base_mip)),
 446                       A6XX_RB_MRT_ARRAY_PITCH(i, iview->image->layout.layer_size),
 447                       A6XX_RB_MRT_BASE(i, tu_image_view_base_ref(iview)),
 448                       A6XX_RB_MRT_BASE_GMEM(i, cmd->state.pass->attachments[a].gmem_offset));
 449
 450       tu_cs_emit_regs(cs,
 451                       A6XX_SP_FS_MRT_REG(i,
 452                                          .color_format = format->rb,
 453                                          .color_sint = vk_format_is_sint(iview->vk_format),
 454                                          .color_uint = vk_format_is_uint(iview->vk_format)));
 455
 456       tu_cs_emit_regs(cs,
 457                       A6XX_RB_MRT_FLAG_BUFFER_ADDR(i, tu_image_view_ubwc_base_ref(iview)),
 458                       A6XX_RB_MRT_FLAG_BUFFER_PITCH(i, tu_image_view_ubwc_pitches(iview)));
 459    }
 460
 461    tu_cs_emit_regs(cs,
 462                    A6XX_RB_SRGB_CNTL(srgb_cntl));
 463
 464    tu_cs_emit_regs(cs,
 465                    A6XX_SP_SRGB_CNTL(srgb_cntl));
 466
 467    tu_cs_emit_regs(cs,
 468                    A6XX_RB_RENDER_COMPONENTS(
 469                       .rt0 = mrt_comp[0],
 470                       .rt1 = mrt_comp[1],
 471                       .rt2 = mrt_comp[2],
 472                       .rt3 = mrt_comp[3],
 473                       .rt4 = mrt_comp[4],
 474                       .rt5 = mrt_comp[5],
 475                       .rt6 = mrt_comp[6],
 476                       .rt7 = mrt_comp[7]));
 477
 478    tu_cs_emit_regs(cs,
 479                    A6XX_SP_FS_RENDER_COMPONENTS(
 480                       .rt0 = mrt_comp[0],
 481                       .rt1 = mrt_comp[1],
 482                       .rt2 = mrt_comp[2],
 483                       .rt3 = mrt_comp[3],
 484                       .rt4 = mrt_comp[4],
 485                       .rt5 = mrt_comp[5],
 486                       .rt6 = mrt_comp[6],
 487                       .rt7 = mrt_comp[7]));
 488 }
 489
 490 static void
 491 tu6_emit_msaa(struct tu_cmd_buffer *cmd,
 492               const struct tu_subpass *subpass,
 493               struct tu_cs *cs)
 494 {
 495    const enum a3xx_msaa_samples samples = tu_msaa_samples(subpass->samples);
 496    bool msaa_disable = samples == MSAA_ONE;
 497
 498    tu_cs_emit_regs(cs,
 499                    A6XX_SP_TP_RAS_MSAA_CNTL(samples),
 500                    A6XX_SP_TP_DEST_MSAA_CNTL(.samples = samples,
 501                                              .msaa_disable = msaa_disable));
 502
 503    tu_cs_emit_regs(cs,
 504                    A6XX_GRAS_RAS_MSAA_CNTL(samples),
 505                    A6XX_GRAS_DEST_MSAA_CNTL(.samples = samples,
 506                                             .msaa_disable = msaa_disable));
 507
 508    tu_cs_emit_regs(cs,
 509                    A6XX_RB_RAS_MSAA_CNTL(samples),
 510                    A6XX_RB_DEST_MSAA_CNTL(.samples = samples,
 511                                           .msaa_disable = msaa_disable));
 512
 513    tu_cs_emit_regs(cs,
 514                    A6XX_RB_MSAA_CNTL(samples));
 515 }
 516
 517 static void
 518 tu6_emit_bin_size(struct tu_cmd_buffer *cmd, struct tu_cs *cs, uint32_t flags)
 519 {
 520    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
 521    const uint32_t bin_w = tiling->tile0.extent.width;
 522    const uint32_t bin_h = tiling->tile0.extent.height;
 523
 524    tu_cs_emit_regs(cs,
 525                    A6XX_GRAS_BIN_CONTROL(.binw = bin_w,
 526                                          .binh = bin_h,
 527                                          .dword = flags));
 528
 529    tu_cs_emit_regs(cs,
 530                    A6XX_RB_BIN_CONTROL(.binw = bin_w,
 531                                        .binh = bin_h,
 532                                        .dword = flags));
 533
 534    /* no flag for RB_BIN_CONTROL2... */
 535    tu_cs_emit_regs(cs,
 536                    A6XX_RB_BIN_CONTROL2(.binw = bin_w,
 537                                         .binh = bin_h));
 538 }
 539
 540 static void
 541 tu6_emit_render_cntl(struct tu_cmd_buffer *cmd,
 542                      struct tu_cs *cs,
 543                      bool binning)
 544 {
 545    uint32_t cntl = 0;
 546    cntl |= A6XX_RB_RENDER_CNTL_UNK4;
 547    if (binning)
 548       cntl |= A6XX_RB_RENDER_CNTL_BINNING;
 549
 550    tu_cs_emit_pkt7(cs, CP_REG_WRITE, 3);
 551    tu_cs_emit(cs, CP_REG_WRITE_0_TRACKER(TRACK_RENDER_CNTL));
 552    tu_cs_emit(cs, REG_A6XX_RB_RENDER_CNTL);
 553    tu_cs_emit(cs, cntl);
 554 }
 555
 556 static void
 557 tu6_emit_blit_scissor(struct tu_cmd_buffer *cmd, struct tu_cs *cs, bool align)
 558 {
 559    const VkRect2D *render_area = &cmd->state.tiling_config.render_area;
 560    uint32_t x1 = render_area->offset.x;
 561    uint32_t y1 = render_area->offset.y;
 562    uint32_t x2 = x1 + render_area->extent.width - 1;
 563    uint32_t y2 = y1 + render_area->extent.height - 1;
 564
 565    /* TODO: alignment requirement seems to be less than tile_align_w/h */
 566    if (align) {
 567       x1 = x1 & ~cmd->device->physical_device->tile_align_w;
 568       y1 = y1 & ~cmd->device->physical_device->tile_align_h;
 569       x2 = ALIGN_POT(x2 + 1, cmd->device->physical_device->tile_align_w) - 1;
 570       y2 = ALIGN_POT(y2 + 1, cmd->device->physical_device->tile_align_h) - 1;
 571    }
 572
 573    tu_cs_emit_regs(cs,
 574                    A6XX_RB_BLIT_SCISSOR_TL(.x = x1, .y = y1),
 575                    A6XX_RB_BLIT_SCISSOR_BR(.x = x2, .y = y2));
 576 }
 577
 578 static void
 579 tu6_emit_blit_info(struct tu_cmd_buffer *cmd,
 580                    struct tu_cs *cs,
 581                    const struct tu_image_view *iview,
 582                    uint32_t gmem_offset,
 583                    bool resolve)
 584 {
 585    tu_cs_emit_regs(cs,
 586                    A6XX_RB_BLIT_INFO(.unk0 = !resolve, .gmem = !resolve));
 587
 588    const struct tu_native_format *format =
 589       tu6_get_native_format(iview->vk_format);
 590    assert(format && format->rb >= 0);
 591
 592    enum a6xx_tile_mode tile_mode =
 593       tu6_get_image_tile_mode(iview->image, iview->base_mip);
 594    tu_cs_emit_regs(cs,
 595                    A6XX_RB_BLIT_DST_INFO(
 596                       .tile_mode = tile_mode,
 597                       .samples = tu_msaa_samples(iview->image->samples),
 598                       .color_format = format->rb,
 599                       .color_swap = format->swap,
 600                       .flags = iview->image->layout.ubwc_layer_size != 0),
 601                    A6XX_RB_BLIT_DST(tu_image_view_base_ref(iview)),
 602                    A6XX_RB_BLIT_DST_PITCH(tu_image_stride(iview->image, iview->base_mip)),
 603                    A6XX_RB_BLIT_DST_ARRAY_PITCH(iview->image->layout.layer_size));
 604
 605    if (iview->image->layout.ubwc_layer_size) {
 606       tu_cs_emit_regs(cs,
 607                       A6XX_RB_BLIT_FLAG_DST(tu_image_view_ubwc_base_ref(iview)),
 608                       A6XX_RB_BLIT_FLAG_DST_PITCH(tu_image_view_ubwc_pitches(iview)));
 609    }
 610
 611    tu_cs_emit_regs(cs,
 612                    A6XX_RB_BLIT_BASE_GMEM(gmem_offset));
 613 }
 614
 615 static void
 616 tu6_emit_blit(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 617 {
 618    tu6_emit_marker(cmd, cs);
 619    tu6_emit_event_write(cmd, cs, BLIT, false);
 620    tu6_emit_marker(cmd, cs);
 621 }
 622
 623 static void
 624 tu6_emit_window_scissor(struct tu_cmd_buffer *cmd,
 625                         struct tu_cs *cs,
 626                         uint32_t x1,
 627                         uint32_t y1,
 628                         uint32_t x2,
 629                         uint32_t y2)
 630 {
 631    tu_cs_emit_regs(cs,
 632                    A6XX_GRAS_SC_WINDOW_SCISSOR_TL(.x = x1, .y = y1),
 633                    A6XX_GRAS_SC_WINDOW_SCISSOR_BR(.x = x2, .y = y2));
 634
 635    tu_cs_emit_regs(cs,
 636                    A6XX_GRAS_RESOLVE_CNTL_1(.x = x1, .y = y1),
 637                    A6XX_GRAS_RESOLVE_CNTL_2(.x = x2, .y = y2));
 638 }
 639
 640 static void
 641 tu6_emit_window_offset(struct tu_cmd_buffer *cmd,
 642                        struct tu_cs *cs,
 643                        uint32_t x1,
 644                        uint32_t y1)
 645 {
 646    tu_cs_emit_regs(cs,
 647                    A6XX_RB_WINDOW_OFFSET(.x = x1, .y = y1));
 648
 649    tu_cs_emit_regs(cs,
 650                    A6XX_RB_WINDOW_OFFSET2(.x = x1, .y = y1));
 651
 652    tu_cs_emit_regs(cs,
 653                    A6XX_SP_WINDOW_OFFSET(.x = x1, .y = y1));
 654
 655    tu_cs_emit_regs(cs,
 656                    A6XX_SP_TP_WINDOW_OFFSET(.x = x1, .y = y1));
 657 }
 658
 659 static bool
 660 use_hw_binning(struct tu_cmd_buffer *cmd)
 661 {
 662    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
 663
 664    if (unlikely(cmd->device->physical_device->instance->debug_flags & TU_DEBUG_NOBIN))
 665       return false;
 666
 667    return (tiling->tile_count.width * tiling->tile_count.height) > 2;
 668 }
 669
 670 static void
 671 tu6_emit_tile_select(struct tu_cmd_buffer *cmd,
 672                      struct tu_cs *cs,
 673                      const struct tu_tile *tile)
 674 {
 675    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
 676    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(0x7));
 677
 678    tu6_emit_marker(cmd, cs);
 679    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
 680    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_GMEM) | 0x10);
 681    tu6_emit_marker(cmd, cs);
 682
 683    const uint32_t x1 = tile->begin.x;
 684    const uint32_t y1 = tile->begin.y;
 685    const uint32_t x2 = tile->end.x - 1;
 686    const uint32_t y2 = tile->end.y - 1;
 687    tu6_emit_window_scissor(cmd, cs, x1, y1, x2, y2);
 688    tu6_emit_window_offset(cmd, cs, x1, y1);
 689
 690    tu_cs_emit_regs(cs,
 691                    A6XX_VPC_SO_OVERRIDE(.so_disable = true));
 692
 693    if (use_hw_binning(cmd)) {
 694       tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
 695
 696       tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
 697       tu_cs_emit(cs, 0x0);
 698
 699       tu_cs_emit_pkt7(cs, CP_REG_TEST, 1);
 700       tu_cs_emit(cs, A6XX_CP_REG_TEST_0_REG(OVERFLOW_FLAG_REG) |
 701                      A6XX_CP_REG_TEST_0_BIT(0) |
 702                      A6XX_CP_REG_TEST_0_WAIT_FOR_ME);
 703
 704       tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
 705       tu_cs_emit(cs, CP_COND_REG_EXEC_0_MODE(PRED_TEST));
 706       tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(11));
 707
 708       /* if (no overflow) */ {
 709          tu_cs_emit_pkt7(cs, CP_SET_BIN_DATA5, 7);
 710          tu_cs_emit(cs, cmd->state.tiling_config.pipe_sizes[tile->pipe] |
 711                         CP_SET_BIN_DATA5_0_VSC_N(tile->slot));
 712          tu_cs_emit_qw(cs, cmd->vsc_data.iova + tile->pipe * cmd->vsc_data_pitch);
 713          tu_cs_emit_qw(cs, cmd->vsc_data.iova + (tile->pipe * 4) + (32 * cmd->vsc_data_pitch));
 714          tu_cs_emit_qw(cs, cmd->vsc_data2.iova + (tile->pipe * cmd->vsc_data2_pitch));
 715
 716          tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
 717          tu_cs_emit(cs, 0x0);
 718
 719          /* use a NOP packet to skip over the 'else' side: */
 720          tu_cs_emit_pkt7(cs, CP_NOP, 2);
 721       } /* else */ {
 722          tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
 723          tu_cs_emit(cs, 0x1);
 724       }
 725
 726       tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
 727       tu_cs_emit(cs, 0x0);
 728
 729       tu_cs_emit_regs(cs,
 730                       A6XX_RB_UNKNOWN_8804(0));
 731
 732       tu_cs_emit_regs(cs,
 733                       A6XX_SP_TP_UNKNOWN_B304(0));
 734
 735       tu_cs_emit_regs(cs,
 736                       A6XX_GRAS_UNKNOWN_80A4(0));
 737    } else {
 738       tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
 739       tu_cs_emit(cs, 0x1);
 740
 741       tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
 742       tu_cs_emit(cs, 0x0);
 743    }
 744 }
 745
 746 static void
 747 tu6_emit_load_attachment(struct tu_cmd_buffer *cmd, struct tu_cs *cs, uint32_t a)
 748 {
 749    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
 750    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 751    const struct tu_image_view *iview = fb->attachments[a].attachment;
 752    const struct tu_render_pass_attachment *attachment =
 753       &cmd->state.pass->attachments[a];
 754
 755    if (attachment->gmem_offset < 0)
 756       return;
 757
 758    const uint32_t x1 = tiling->render_area.offset.x;
 759    const uint32_t y1 = tiling->render_area.offset.y;
 760    const uint32_t x2 = x1 + tiling->render_area.extent.width;
 761    const uint32_t y2 = y1 + tiling->render_area.extent.height;
 762    const uint32_t tile_x2 =
 763       tiling->tile0.offset.x + tiling->tile0.extent.width * tiling->tile_count.width;
 764    const uint32_t tile_y2 =
 765       tiling->tile0.offset.y + tiling->tile0.extent.height * tiling->tile_count.height;
 766    bool need_load =
 767       x1 != tiling->tile0.offset.x || x2 != MIN2(fb->width, tile_x2) ||
 768       y1 != tiling->tile0.offset.y || y2 != MIN2(fb->height, tile_y2);
 769
 770    if (need_load)
 771       tu_finishme("improve handling of unaligned render area");
 772
 773    if (attachment->load_op == VK_ATTACHMENT_LOAD_OP_LOAD)
 774       need_load = true;
 775
 776    if (vk_format_has_stencil(iview->vk_format) &&
 777        attachment->stencil_load_op == VK_ATTACHMENT_LOAD_OP_LOAD)
 778       need_load = true;
 779
 780    if (need_load) {
 781       tu6_emit_blit_info(cmd, cs, iview, attachment->gmem_offset, false);
 782       tu6_emit_blit(cmd, cs);
 783    }
 784 }
 785
 786 static void
 787 tu6_emit_clear_attachment(struct tu_cmd_buffer *cmd, struct tu_cs *cs,
 788                           uint32_t a,
 789                           const VkRenderPassBeginInfo *info)
 790 {
 791    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 792    const struct tu_image_view *iview = fb->attachments[a].attachment;
 793    const struct tu_render_pass_attachment *attachment =
 794       &cmd->state.pass->attachments[a];
 795    unsigned clear_mask = 0;
 796
 797    /* note: this means it isn't used by any subpass and shouldn't be cleared anyway */
 798    if (attachment->gmem_offset < 0)
 799       return;
 800
 801    if (attachment->load_op == VK_ATTACHMENT_LOAD_OP_CLEAR)
 802       clear_mask = 0xf;
 803
 804    if (vk_format_has_stencil(iview->vk_format)) {
 805       clear_mask &= 0x1;
 806       if (attachment->stencil_load_op == VK_ATTACHMENT_LOAD_OP_CLEAR)
 807          clear_mask |= 0x2;
 808    }
 809    if (!clear_mask)
 810       return;
 811
 812    const struct tu_native_format *format =
 813       tu6_get_native_format(iview->vk_format);
 814    assert(format && format->rb >= 0);
 815
 816    tu_cs_emit_regs(cs,
 817                    A6XX_RB_BLIT_DST_INFO(.color_format = format->rb));
 818
 819    tu_cs_emit_regs(cs,
 820                    A6XX_RB_BLIT_INFO(.gmem = true,
 821                                      .clear_mask = clear_mask));
 822
 823    tu_cs_emit_regs(cs,
 824                    A6XX_RB_BLIT_BASE_GMEM(attachment->gmem_offset));
 825
 826    tu_cs_emit_regs(cs,
 827                    A6XX_RB_UNKNOWN_88D0(0));
 828
 829    uint32_t clear_vals[4] = { 0 };
 830    tu_pack_clear_value(&info->pClearValues[a], iview->vk_format, clear_vals);
 831
 832    tu_cs_emit_regs(cs,
 833                    A6XX_RB_BLIT_CLEAR_COLOR_DW0(clear_vals[0]),
 834                    A6XX_RB_BLIT_CLEAR_COLOR_DW1(clear_vals[1]),
 835                    A6XX_RB_BLIT_CLEAR_COLOR_DW2(clear_vals[2]),
 836                    A6XX_RB_BLIT_CLEAR_COLOR_DW3(clear_vals[3]));
 837
 838    tu6_emit_blit(cmd, cs);
 839 }
 840
 841 static void
 842 tu6_emit_store_attachment(struct tu_cmd_buffer *cmd,
 843                           struct tu_cs *cs,
 844                           uint32_t a,
 845                           uint32_t gmem_a)
 846 {
 847    if (cmd->state.pass->attachments[a].store_op == VK_ATTACHMENT_STORE_OP_DONT_CARE)
 848       return;
 849
 850    tu6_emit_blit_info(cmd, cs,
 851                       cmd->state.framebuffer->attachments[a].attachment,
 852                       cmd->state.pass->attachments[gmem_a].gmem_offset, true);
 853    tu6_emit_blit(cmd, cs);
 854 }
 855
 856 static void
 857 tu6_emit_tile_store(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 858 {
 859    const struct tu_render_pass *pass = cmd->state.pass;
 860    const struct tu_subpass *subpass = &pass->subpasses[pass->subpass_count-1];
 861
 862    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
 863    tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
 864                      CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
 865                      CP_SET_DRAW_STATE__0_GROUP_ID(0));
 866    tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
 867    tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
 868
 869    tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
 870    tu_cs_emit(cs, 0x0);
 871
 872    tu6_emit_marker(cmd, cs);
 873    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
 874    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_RESOLVE) | 0x10);
 875    tu6_emit_marker(cmd, cs);
 876
 877    tu6_emit_blit_scissor(cmd, cs, true);
 878
 879    for (uint32_t a = 0; a < pass->attachment_count; ++a) {
 880       if (pass->attachments[a].gmem_offset >= 0)
 881          tu6_emit_store_attachment(cmd, cs, a, a);
 882    }
 883
 884    if (subpass->resolve_attachments) {
 885       for (unsigned i = 0; i < subpass->color_count; i++) {
 886          uint32_t a = subpass->resolve_attachments[i].attachment;
 887          if (a != VK_ATTACHMENT_UNUSED)
 888             tu6_emit_store_attachment(cmd, cs, a,
 889                                       subpass->color_attachments[i].attachment);
 890       }
 891    }
 892 }
 893
 894 static void
 895 tu6_emit_restart_index(struct tu_cs *cs, uint32_t restart_index)
 896 {
 897    tu_cs_emit_regs(cs,
 898                    A6XX_PC_RESTART_INDEX(restart_index));
 899 }
 900
 901 static void
 902 tu6_init_hw(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 903 {
 904    struct tu_physical_device *phys_dev = cmd->device->physical_device;
 905
 906    VkResult result = tu_cs_reserve_space(cmd->device, cs, 256);
 907    if (result != VK_SUCCESS) {
 908       cmd->record_result = result;
 909       return;
 910    }
 911
 912    tu6_emit_cache_flush(cmd, cs);
 913
 914    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UPDATE_CNTL, 0xfffff);
 915
 916    tu_cs_emit_write_reg(cs, REG_A6XX_RB_CCU_CNTL, phys_dev->magic.RB_CCU_CNTL_gmem);
 917    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8E04, 0x00100000);
 918    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE04, 0x8);
 919    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE00, 0);
 920    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE0F, 0x3f);
 921    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B605, 0x44);
 922    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B600, 0x100000);
 923    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE00, 0x80);
 924    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE01, 0);
 925
 926    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9600, 0);
 927    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8600, 0x880);
 928    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE04, 0);
 929    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE03, 0x00000410);
 930    tu_cs_emit_write_reg(cs, REG_A6XX_SP_IBO_COUNT, 0);
 931    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B182, 0);
 932    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BB11, 0);
 933    tu_cs_emit_write_reg(cs, REG_A6XX_UCHE_UNKNOWN_0E12, 0x3200000);
 934    tu_cs_emit_write_reg(cs, REG_A6XX_UCHE_CLIENT_PF, 4);
 935    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8E01, 0x0);
 936    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AB00, 0x5);
 937    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_ADD_OFFSET, A6XX_VFD_ADD_OFFSET_VERTEX);
 938    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8811, 0x00000010);
 939    tu_cs_emit_write_reg(cs, REG_A6XX_PC_MODE_CNTL, 0x1f);
 940
 941    tu_cs_emit_write_reg(cs, REG_A6XX_RB_SRGB_CNTL, 0);
 942
 943    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8101, 0);
 944    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_SAMPLE_CNTL, 0);
 945    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8110, 0);
 946
 947    tu_cs_emit_write_reg(cs, REG_A6XX_RB_RENDER_CONTROL0, 0x401);
 948    tu_cs_emit_write_reg(cs, REG_A6XX_RB_RENDER_CONTROL1, 0);
 949    tu_cs_emit_write_reg(cs, REG_A6XX_RB_FS_OUTPUT_CNTL0, 0);
 950    tu_cs_emit_write_reg(cs, REG_A6XX_RB_SAMPLE_CNTL, 0);
 951    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8818, 0);
 952    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8819, 0);
 953    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881A, 0);
 954    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881B, 0);
 955    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881C, 0);
 956    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881D, 0);
 957    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881E, 0);
 958    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_88F0, 0);
 959
 960    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9101, 0xffff00);
 961    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9107, 0);
 962
 963    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9236, 1);
 964    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9300, 0);
 965
 966    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_SO_OVERRIDE,
 967                         A6XX_VPC_SO_OVERRIDE_SO_DISABLE);
 968
 969    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9801, 0);
 970    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9806, 0);
 971    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9980, 0);
 972
 973    tu_cs_emit_write_reg(cs, REG_A6XX_PC_PRIMITIVE_CNTL_6, 0);
 974    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9B07, 0);
 975
 976    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_A81B, 0);
 977
 978    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B183, 0);
 979
 980    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8099, 0);
 981    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_809B, 0);
 982    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A0, 2);
 983    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80AF, 0);
 984    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9210, 0);
 985    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9211, 0);
 986    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9602, 0);
 987    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9981, 0x3);
 988    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9E72, 0);
 989    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9108, 0x3);
 990    tu_cs_emit_write_reg(cs, REG_A6XX_SP_TP_UNKNOWN_B304, 0);
 991    tu_cs_emit_write_reg(cs, REG_A6XX_SP_TP_UNKNOWN_B309, 0x000000a2);
 992    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8804, 0);
 993    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A4, 0);
 994    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A5, 0);
 995    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A6, 0);
 996    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8805, 0);
 997    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8806, 0);
 998    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8878, 0);
 999    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8879, 0);
1000    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_CONTROL_5_REG, 0xfc);
1001
1002    tu6_emit_marker(cmd, cs);
1003
1004    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_MODE_CNTL, 0x00000000);
1005
1006    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_UNKNOWN_A008, 0);
1007
1008    tu_cs_emit_write_reg(cs, REG_A6XX_PC_MODE_CNTL, 0x0000001f);
1009
1010    /* we don't use this yet.. probably best to disable.. */
1011    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
1012    tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
1013                      CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
1014                      CP_SET_DRAW_STATE__0_GROUP_ID(0));
1015    tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
1016    tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
1017
1018    tu_cs_emit_regs(cs,
1019                    A6XX_VPC_SO_BUFFER_BASE(0),
1020                    A6XX_VPC_SO_BUFFER_SIZE(0));
1021
1022    tu_cs_emit_regs(cs,
1023                    A6XX_VPC_SO_FLUSH_BASE(0));
1024
1025    tu_cs_emit_regs(cs,
1026                    A6XX_VPC_SO_BUF_CNTL(0));
1027
1028    tu_cs_emit_regs(cs,
1029                    A6XX_VPC_SO_BUFFER_OFFSET(0, 0));
1030
1031    tu_cs_emit_regs(cs,
1032                    A6XX_VPC_SO_BUFFER_BASE(1, 0),
1033                    A6XX_VPC_SO_BUFFER_SIZE(1, 0));
1034
1035    tu_cs_emit_regs(cs,
1036                    A6XX_VPC_SO_BUFFER_OFFSET(1, 0),
1037                    A6XX_VPC_SO_FLUSH_BASE(1, 0),
1038                    A6XX_VPC_SO_BUFFER_BASE(2, 0),
1039                    A6XX_VPC_SO_BUFFER_SIZE(2, 0));
1040
1041    tu_cs_emit_regs(cs,
1042                    A6XX_VPC_SO_BUFFER_OFFSET(2, 0),
1043                    A6XX_VPC_SO_FLUSH_BASE(2, 0),
1044                    A6XX_VPC_SO_BUFFER_BASE(3, 0),
1045                    A6XX_VPC_SO_BUFFER_SIZE(3, 0));
1046
1047    tu_cs_emit_regs(cs,
1048                    A6XX_VPC_SO_BUFFER_OFFSET(3, 0),
1049                    A6XX_VPC_SO_FLUSH_BASE(3, 0));
1050
1051    tu_cs_emit_regs(cs,
1052                    A6XX_SP_HS_CTRL_REG0(0));
1053
1054    tu_cs_emit_regs(cs,
1055                    A6XX_SP_GS_CTRL_REG0(0));
1056
1057    tu_cs_emit_regs(cs,
1058                    A6XX_GRAS_LRZ_CNTL(0));
1059
1060    tu_cs_emit_regs(cs,
1061                    A6XX_RB_LRZ_CNTL(0));
1062
1063    tu_cs_sanity_check(cs);
1064 }
1065
1066 static void
1067 tu6_cache_flush(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1068 {
1069    unsigned seqno;
1070
1071    seqno = tu6_emit_event_write(cmd, cs, CACHE_FLUSH_AND_INV_EVENT, true);
1072
1073    tu_cs_emit_pkt7(cs, CP_WAIT_REG_MEM, 6);
1074    tu_cs_emit(cs, CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ) |
1075                   CP_WAIT_REG_MEM_0_POLL_MEMORY);
1076    tu_cs_emit_qw(cs, cmd->scratch_bo.iova);
1077    tu_cs_emit(cs, CP_WAIT_REG_MEM_3_REF(seqno));
1078    tu_cs_emit(cs, CP_WAIT_REG_MEM_4_MASK(~0));
1079    tu_cs_emit(cs, CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(16));
1080
1081    seqno = tu6_emit_event_write(cmd, cs, CACHE_FLUSH_TS, true);
1082
1083    tu_cs_emit_pkt7(cs, CP_WAIT_MEM_GTE, 4);
1084    tu_cs_emit(cs, CP_WAIT_MEM_GTE_0_RESERVED(0));
1085    tu_cs_emit_qw(cs, cmd->scratch_bo.iova);
1086    tu_cs_emit(cs, CP_WAIT_MEM_GTE_3_REF(seqno));
1087 }
1088
1089 static void
1090 update_vsc_pipe(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1091 {
1092    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1093
1094    tu_cs_emit_regs(cs,
1095                    A6XX_VSC_BIN_SIZE(.width = tiling->tile0.extent.width,
1096                                      .height = tiling->tile0.extent.height),
1097                    A6XX_VSC_SIZE_ADDRESS(.bo = &cmd->vsc_data,
1098                                          .bo_offset = 32 * cmd->vsc_data_pitch));
1099
1100    tu_cs_emit_regs(cs,
1101                    A6XX_VSC_BIN_COUNT(.nx = tiling->tile_count.width,
1102                                       .ny = tiling->tile_count.height));
1103
1104    tu_cs_emit_pkt4(cs, REG_A6XX_VSC_PIPE_CONFIG_REG(0), 32);
1105    for (unsigned i = 0; i < 32; i++)
1106       tu_cs_emit(cs, tiling->pipe_config[i]);
1107
1108    tu_cs_emit_regs(cs,
1109                    A6XX_VSC_PIPE_DATA2_ADDRESS(.bo = &cmd->vsc_data2),
1110                    A6XX_VSC_PIPE_DATA2_PITCH(cmd->vsc_data2_pitch),
1111                    A6XX_VSC_PIPE_DATA2_ARRAY_PITCH(cmd->vsc_data2.size));
1112
1113    tu_cs_emit_regs(cs,
1114                    A6XX_VSC_PIPE_DATA_ADDRESS(.bo = &cmd->vsc_data),
1115                    A6XX_VSC_PIPE_DATA_PITCH(cmd->vsc_data_pitch),
1116                    A6XX_VSC_PIPE_DATA_ARRAY_PITCH(cmd->vsc_data.size));
1117 }
1118
1119 static void
1120 emit_vsc_overflow_test(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1121 {
1122    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1123    const uint32_t used_pipe_count =
1124       tiling->pipe_count.width * tiling->pipe_count.height;
1125
1126    /* Clear vsc_scratch: */
1127    tu_cs_emit_pkt7(cs, CP_MEM_WRITE, 3);
1128    tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1129    tu_cs_emit(cs, 0x0);
1130
1131    /* Check for overflow, write vsc_scratch if detected: */
1132    for (int i = 0; i < used_pipe_count; i++) {
1133       tu_cs_emit_pkt7(cs, CP_COND_WRITE5, 8);
1134       tu_cs_emit(cs, CP_COND_WRITE5_0_FUNCTION(WRITE_GE) |
1135             CP_COND_WRITE5_0_WRITE_MEMORY);
1136       tu_cs_emit(cs, CP_COND_WRITE5_1_POLL_ADDR_LO(REG_A6XX_VSC_SIZE_REG(i)));
1137       tu_cs_emit(cs, CP_COND_WRITE5_2_POLL_ADDR_HI(0));
1138       tu_cs_emit(cs, CP_COND_WRITE5_3_REF(cmd->vsc_data_pitch));
1139       tu_cs_emit(cs, CP_COND_WRITE5_4_MASK(~0));
1140       tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1141       tu_cs_emit(cs, CP_COND_WRITE5_7_WRITE_DATA(1 + cmd->vsc_data_pitch));
1142
1143       tu_cs_emit_pkt7(cs, CP_COND_WRITE5, 8);
1144       tu_cs_emit(cs, CP_COND_WRITE5_0_FUNCTION(WRITE_GE) |
1145             CP_COND_WRITE5_0_WRITE_MEMORY);
1146       tu_cs_emit(cs, CP_COND_WRITE5_1_POLL_ADDR_LO(REG_A6XX_VSC_SIZE2_REG(i)));
1147       tu_cs_emit(cs, CP_COND_WRITE5_2_POLL_ADDR_HI(0));
1148       tu_cs_emit(cs, CP_COND_WRITE5_3_REF(cmd->vsc_data2_pitch));
1149       tu_cs_emit(cs, CP_COND_WRITE5_4_MASK(~0));
1150       tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1151       tu_cs_emit(cs, CP_COND_WRITE5_7_WRITE_DATA(3 + cmd->vsc_data2_pitch));
1152    }
1153
1154    tu_cs_emit_pkt7(cs, CP_WAIT_MEM_WRITES, 0);
1155
1156    tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
1157
1158    tu_cs_emit_pkt7(cs, CP_MEM_TO_REG, 3);
1159    tu_cs_emit(cs, CP_MEM_TO_REG_0_REG(OVERFLOW_FLAG_REG) |
1160          CP_MEM_TO_REG_0_CNT(1 - 1));
1161    tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1162
1163    /*
1164     * This is a bit awkward, we really want a way to invert the
1165     * CP_REG_TEST/CP_COND_REG_EXEC logic, so that we can conditionally
1166     * execute cmds to use hwbinning when a bit is *not* set.  This
1167     * dance is to invert OVERFLOW_FLAG_REG
1168     *
1169     * A CP_NOP packet is used to skip executing the 'else' clause
1170     * if (b0 set)..
1171     */
1172
1173    /* b0 will be set if VSC_DATA or VSC_DATA2 overflow: */
1174    tu_cs_emit_pkt7(cs, CP_REG_TEST, 1);
1175    tu_cs_emit(cs, A6XX_CP_REG_TEST_0_REG(OVERFLOW_FLAG_REG) |
1176          A6XX_CP_REG_TEST_0_BIT(0) |
1177          A6XX_CP_REG_TEST_0_WAIT_FOR_ME);
1178
1179    tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
1180    tu_cs_emit(cs, CP_COND_REG_EXEC_0_MODE(PRED_TEST));
1181    tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(7));
1182
1183    /* if (b0 set) */ {
1184       /*
1185        * On overflow, mirror the value to control->vsc_overflow
1186        * which CPU is checking to detect overflow (see
1187        * check_vsc_overflow())
1188        */
1189       tu_cs_emit_pkt7(cs, CP_REG_TO_MEM, 3);
1190       tu_cs_emit(cs, CP_REG_TO_MEM_0_REG(OVERFLOW_FLAG_REG) |
1191             CP_REG_TO_MEM_0_CNT(0));
1192       tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_OVERFLOW);
1193
1194       tu_cs_emit_pkt4(cs, OVERFLOW_FLAG_REG, 1);
1195       tu_cs_emit(cs, 0x0);
1196
1197       tu_cs_emit_pkt7(cs, CP_NOP, 2);  /* skip 'else' when 'if' is taken */
1198    } /* else */ {
1199       tu_cs_emit_pkt4(cs, OVERFLOW_FLAG_REG, 1);
1200       tu_cs_emit(cs, 0x1);
1201    }
1202 }
1203
1204 static void
1205 tu6_emit_binning_pass(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1206 {
1207    struct tu_physical_device *phys_dev = cmd->device->physical_device;
1208    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1209
1210    uint32_t x1 = tiling->tile0.offset.x;
1211    uint32_t y1 = tiling->tile0.offset.y;
1212    uint32_t x2 = tiling->render_area.offset.x + tiling->render_area.extent.width - 1;
1213    uint32_t y2 = tiling->render_area.offset.y + tiling->render_area.extent.height - 1;
1214
1215    tu6_emit_window_scissor(cmd, cs, x1, y1, x2, y2);
1216
1217    tu6_emit_marker(cmd, cs);
1218    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1219    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_BINNING));
1220    tu6_emit_marker(cmd, cs);
1221
1222    tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
1223    tu_cs_emit(cs, 0x1);
1224
1225    tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
1226    tu_cs_emit(cs, 0x1);
1227
1228    tu_cs_emit_wfi(cs);
1229
1230    tu_cs_emit_regs(cs,
1231                    A6XX_VFD_MODE_CNTL(.binning_pass = true));
1232
1233    update_vsc_pipe(cmd, cs);
1234
1235    tu_cs_emit_regs(cs,
1236                    A6XX_PC_UNKNOWN_9805(.unknown = phys_dev->magic.PC_UNKNOWN_9805));
1237
1238    tu_cs_emit_regs(cs,
1239                    A6XX_SP_UNKNOWN_A0F8(.unknown = phys_dev->magic.SP_UNKNOWN_A0F8));
1240
1241    tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, 1);
1242    tu_cs_emit(cs, UNK_2C);
1243
1244    tu_cs_emit_regs(cs,
1245                    A6XX_RB_WINDOW_OFFSET(.x = 0, .y = 0));
1246
1247    tu_cs_emit_regs(cs,
1248                    A6XX_SP_TP_WINDOW_OFFSET(.x = 0, .y = 0));
1249
1250    /* emit IB to binning drawcmds: */
1251    tu_cs_emit_call(cs, &cmd->draw_cs);
1252
1253    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
1254    tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
1255                   CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
1256                   CP_SET_DRAW_STATE__0_GROUP_ID(0));
1257    tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
1258    tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
1259
1260    tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, 1);
1261    tu_cs_emit(cs, UNK_2D);
1262
1263    tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE, false);
1264    tu6_cache_flush(cmd, cs);
1265
1266    tu_cs_emit_wfi(cs);
1267
1268    tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
1269
1270    emit_vsc_overflow_test(cmd, cs);
1271
1272    tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
1273    tu_cs_emit(cs, 0x0);
1274
1275    tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
1276    tu_cs_emit(cs, 0x0);
1277
1278    tu_cs_emit_wfi(cs);
1279
1280    tu_cs_emit_regs(cs,
1281                    A6XX_RB_CCU_CNTL(.unknown = 0x7c400004));
1282
1283    cmd->wait_for_idle = false;
1284 }
1285
1286 static void
1287 tu6_render_begin(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1288 {
1289    struct tu_physical_device *phys_dev = cmd->device->physical_device;
1290
1291    VkResult result = tu_cs_reserve_space(cmd->device, cs, 1024);
1292    if (result != VK_SUCCESS) {
1293       cmd->record_result = result;
1294       return;
1295    }
1296
1297    tu6_emit_lrz_flush(cmd, cs);
1298
1299    /* lrz clear? */
1300
1301    tu6_emit_cache_flush(cmd, cs);
1302
1303    tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1304    tu_cs_emit(cs, 0x0);
1305
1306    /* 0x10000000 for BYPASS.. 0x7c13c080 for GMEM: */
1307    tu6_emit_wfi(cmd, cs);
1308    tu_cs_emit_regs(cs,
1309                    A6XX_RB_CCU_CNTL(0x7c400004));
1310
1311    if (use_hw_binning(cmd)) {
1312       tu6_emit_bin_size(cmd, cs, A6XX_RB_BIN_CONTROL_BINNING_PASS | 0x6000000);
1313
1314       tu6_emit_render_cntl(cmd, cs, true);
1315
1316       tu6_emit_binning_pass(cmd, cs);
1317
1318       tu6_emit_bin_size(cmd, cs, A6XX_RB_BIN_CONTROL_USE_VIZ | 0x6000000);
1319
1320       tu_cs_emit_regs(cs,
1321                       A6XX_VFD_MODE_CNTL(0));
1322
1323       tu_cs_emit_regs(cs, A6XX_PC_UNKNOWN_9805(.unknown = phys_dev->magic.PC_UNKNOWN_9805));
1324
1325       tu_cs_emit_regs(cs, A6XX_SP_UNKNOWN_A0F8(.unknown = phys_dev->magic.SP_UNKNOWN_A0F8));
1326
1327       tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1328       tu_cs_emit(cs, 0x1);
1329    } else {
1330       tu6_emit_bin_size(cmd, cs, 0x6000000);
1331    }
1332
1333    tu6_emit_render_cntl(cmd, cs, false);
1334
1335    tu_cs_sanity_check(cs);
1336 }
1337
1338 static void
1339 tu6_render_tile(struct tu_cmd_buffer *cmd,
1340                 struct tu_cs *cs,
1341                 const struct tu_tile *tile)
1342 {
1343    const uint32_t render_tile_space = 256 + tu_cs_get_call_size(&cmd->draw_cs);
1344    VkResult result = tu_cs_reserve_space(cmd->device, cs, render_tile_space);
1345    if (result != VK_SUCCESS) {
1346       cmd->record_result = result;
1347       return;
1348    }
1349
1350    tu6_emit_tile_select(cmd, cs, tile);
1351    tu_cs_emit_ib(cs, &cmd->state.tile_load_ib);
1352
1353    tu_cs_emit_call(cs, &cmd->draw_cs);
1354    cmd->wait_for_idle = true;
1355
1356    if (use_hw_binning(cmd)) {
1357       tu_cs_emit_pkt7(cs, CP_REG_TEST, 1);
1358       tu_cs_emit(cs, A6XX_CP_REG_TEST_0_REG(OVERFLOW_FLAG_REG) |
1359                      A6XX_CP_REG_TEST_0_BIT(0) |
1360                      A6XX_CP_REG_TEST_0_WAIT_FOR_ME);
1361
1362       tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
1363       tu_cs_emit(cs, 0x10000000);
1364       tu_cs_emit(cs, 2);  /* conditionally execute next 2 dwords */
1365
1366       /* if (no overflow) */ {
1367          tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1368          tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(0x5) | 0x10);
1369       }
1370    }
1371
1372    tu_cs_emit_ib(cs, &cmd->state.tile_store_ib);
1373
1374    tu_cs_sanity_check(cs);
1375 }
1376
1377 static void
1378 tu6_render_end(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1379 {
1380    const uint32_t space = 16 + tu_cs_get_call_size(&cmd->draw_epilogue_cs);
1381    VkResult result = tu_cs_reserve_space(cmd->device, cs, space);
1382    if (result != VK_SUCCESS) {
1383       cmd->record_result = result;
1384       return;
1385    }
1386
1387    tu_cs_emit_call(cs, &cmd->draw_epilogue_cs);
1388
1389    tu_cs_emit_regs(cs,
1390                    A6XX_GRAS_LRZ_CNTL(0));
1391
1392    tu6_emit_lrz_flush(cmd, cs);
1393
1394    tu6_emit_event_write(cmd, cs, CACHE_FLUSH_TS, true);
1395
1396    tu_cs_sanity_check(cs);
1397 }
1398
1399 static void
1400 tu_cmd_render_tiles(struct tu_cmd_buffer *cmd)
1401 {
1402    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1403
1404    tu6_render_begin(cmd, &cmd->cs);
1405
1406    for (uint32_t y = 0; y < tiling->tile_count.height; y++) {
1407       for (uint32_t x = 0; x < tiling->tile_count.width; x++) {
1408          struct tu_tile tile;
1409          tu_tiling_config_get_tile(tiling, cmd->device, x, y, &tile);
1410          tu6_render_tile(cmd, &cmd->cs, &tile);
1411       }
1412    }
1413
1414    tu6_render_end(cmd, &cmd->cs);
1415 }
1416
1417 static void
1418 tu_cmd_prepare_tile_load_ib(struct tu_cmd_buffer *cmd,
1419                             const VkRenderPassBeginInfo *info)
1420 {
1421    const uint32_t tile_load_space =
1422       8 + (23+19) * cmd->state.pass->attachment_count +
1423       21 + (13 * cmd->state.subpass->color_count + 8) + 11;
1424
1425    struct tu_cs sub_cs;
1426
1427    VkResult result = tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs,
1428                                             tile_load_space, &sub_cs);
1429    if (result != VK_SUCCESS) {
1430       cmd->record_result = result;
1431       return;
1432    }
1433
1434    tu6_emit_blit_scissor(cmd, &sub_cs, true);
1435
1436    for (uint32_t i = 0; i < cmd->state.pass->attachment_count; ++i)
1437       tu6_emit_load_attachment(cmd, &sub_cs, i);
1438
1439    tu6_emit_blit_scissor(cmd, &sub_cs, false);
1440
1441    for (uint32_t i = 0; i < cmd->state.pass->attachment_count; ++i)
1442       tu6_emit_clear_attachment(cmd, &sub_cs, i, info);
1443
1444    /* invalidate because reading input attachments will cache GMEM and
1445     * the cache isn''t updated when GMEM is written
1446     * TODO: is there a no-cache bit for textures?
1447     */
1448    if (cmd->state.subpass->input_count)
1449       tu6_emit_event_write(cmd, &sub_cs, CACHE_INVALIDATE, false);
1450
1451    tu6_emit_zs(cmd, cmd->state.subpass, &sub_cs);
1452    tu6_emit_mrt(cmd, cmd->state.subpass, &sub_cs);
1453    tu6_emit_msaa(cmd, cmd->state.subpass, &sub_cs);
1454
1455    cmd->state.tile_load_ib = tu_cs_end_sub_stream(&cmd->sub_cs, &sub_cs);
1456 }
1457
1458 static void
1459 tu_cmd_prepare_tile_store_ib(struct tu_cmd_buffer *cmd)
1460 {
1461    const uint32_t tile_store_space = 32 + 23 * cmd->state.pass->attachment_count;
1462    struct tu_cs sub_cs;
1463
1464    VkResult result = tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs,
1465                                             tile_store_space, &sub_cs);
1466    if (result != VK_SUCCESS) {
1467       cmd->record_result = result;
1468       return;
1469    }
1470
1471    /* emit to tile-store sub_cs */
1472    tu6_emit_tile_store(cmd, &sub_cs);
1473
1474    cmd->state.tile_store_ib = tu_cs_end_sub_stream(&cmd->sub_cs, &sub_cs);
1475 }
1476
1477 static void
1478 tu_cmd_update_tiling_config(struct tu_cmd_buffer *cmd,
1479                             const VkRect2D *render_area)
1480 {
1481    const struct tu_device *dev = cmd->device;
1482    struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1483
1484    tiling->render_area = *render_area;
1485
1486    tu_tiling_config_update_tile_layout(tiling, dev, cmd->state.pass->gmem_pixels);
1487    tu_tiling_config_update_pipe_layout(tiling, dev);
1488    tu_tiling_config_update_pipes(tiling, dev);
1489 }
1490
1491 const struct tu_dynamic_state default_dynamic_state = {
1492    .viewport =
1493      {
1494        .count = 0,
1495      },
1496    .scissor =
1497      {
1498        .count = 0,
1499      },
1500    .line_width = 1.0f,
1501    .depth_bias =
1502      {
1503        .bias = 0.0f,
1504        .clamp = 0.0f,
1505        .slope = 0.0f,
1506      },
1507    .blend_constants = { 0.0f, 0.0f, 0.0f, 0.0f },
1508    .depth_bounds =
1509      {
1510        .min = 0.0f,
1511        .max = 1.0f,
1512      },
1513    .stencil_compare_mask =
1514      {
1515        .front = ~0u,
1516        .back = ~0u,
1517      },
1518    .stencil_write_mask =
1519      {
1520        .front = ~0u,
1521        .back = ~0u,
1522      },
1523    .stencil_reference =
1524      {
1525        .front = 0u,
1526        .back = 0u,
1527      },
1528 };
1529
1530 static void UNUSED /* FINISHME */
1531 tu_bind_dynamic_state(struct tu_cmd_buffer *cmd_buffer,
1532                       const struct tu_dynamic_state *src)
1533 {
1534    struct tu_dynamic_state *dest = &cmd_buffer->state.dynamic;
1535    uint32_t copy_mask = src->mask;
1536    uint32_t dest_mask = 0;
1537
1538    tu_use_args(cmd_buffer); /* FINISHME */
1539
1540    /* Make sure to copy the number of viewports/scissors because they can
1541     * only be specified at pipeline creation time.
1542     */
1543    dest->viewport.count = src->viewport.count;
1544    dest->scissor.count = src->scissor.count;
1545    dest->discard_rectangle.count = src->discard_rectangle.count;
1546
1547    if (copy_mask & TU_DYNAMIC_VIEWPORT) {
1548       if (memcmp(&dest->viewport.viewports, &src->viewport.viewports,
1549                  src->viewport.count * sizeof(VkViewport))) {
1550          typed_memcpy(dest->viewport.viewports, src->viewport.viewports,
1551                       src->viewport.count);
1552          dest_mask |= TU_DYNAMIC_VIEWPORT;
1553       }
1554    }
1555
1556    if (copy_mask & TU_DYNAMIC_SCISSOR) {
1557       if (memcmp(&dest->scissor.scissors, &src->scissor.scissors,
1558                  src->scissor.count * sizeof(VkRect2D))) {
1559          typed_memcpy(dest->scissor.scissors, src->scissor.scissors,
1560                       src->scissor.count);
1561          dest_mask |= TU_DYNAMIC_SCISSOR;
1562       }
1563    }
1564
1565    if (copy_mask & TU_DYNAMIC_LINE_WIDTH) {
1566       if (dest->line_width != src->line_width) {
1567          dest->line_width = src->line_width;
1568          dest_mask |= TU_DYNAMIC_LINE_WIDTH;
1569       }
1570    }
1571
1572    if (copy_mask & TU_DYNAMIC_DEPTH_BIAS) {
1573       if (memcmp(&dest->depth_bias, &src->depth_bias,
1574                  sizeof(src->depth_bias))) {
1575          dest->depth_bias = src->depth_bias;
1576          dest_mask |= TU_DYNAMIC_DEPTH_BIAS;
1577       }
1578    }
1579
1580    if (copy_mask & TU_DYNAMIC_BLEND_CONSTANTS) {
1581       if (memcmp(&dest->blend_constants, &src->blend_constants,
1582                  sizeof(src->blend_constants))) {
1583          typed_memcpy(dest->blend_constants, src->blend_constants, 4);
1584          dest_mask |= TU_DYNAMIC_BLEND_CONSTANTS;
1585       }
1586    }
1587
1588    if (copy_mask & TU_DYNAMIC_DEPTH_BOUNDS) {
1589       if (memcmp(&dest->depth_bounds, &src->depth_bounds,
1590                  sizeof(src->depth_bounds))) {
1591          dest->depth_bounds = src->depth_bounds;
1592          dest_mask |= TU_DYNAMIC_DEPTH_BOUNDS;
1593       }
1594    }
1595
1596    if (copy_mask & TU_DYNAMIC_STENCIL_COMPARE_MASK) {
1597       if (memcmp(&dest->stencil_compare_mask, &src->stencil_compare_mask,
1598                  sizeof(src->stencil_compare_mask))) {
1599          dest->stencil_compare_mask = src->stencil_compare_mask;
1600          dest_mask |= TU_DYNAMIC_STENCIL_COMPARE_MASK;
1601       }
1602    }
1603
1604    if (copy_mask & TU_DYNAMIC_STENCIL_WRITE_MASK) {
1605       if (memcmp(&dest->stencil_write_mask, &src->stencil_write_mask,
1606                  sizeof(src->stencil_write_mask))) {
1607          dest->stencil_write_mask = src->stencil_write_mask;
1608          dest_mask |= TU_DYNAMIC_STENCIL_WRITE_MASK;
1609       }
1610    }
1611
1612    if (copy_mask & TU_DYNAMIC_STENCIL_REFERENCE) {
1613       if (memcmp(&dest->stencil_reference, &src->stencil_reference,
1614                  sizeof(src->stencil_reference))) {
1615          dest->stencil_reference = src->stencil_reference;
1616          dest_mask |= TU_DYNAMIC_STENCIL_REFERENCE;
1617       }
1618    }
1619
1620    if (copy_mask & TU_DYNAMIC_DISCARD_RECTANGLE) {
1621       if (memcmp(&dest->discard_rectangle.rectangles,
1622                  &src->discard_rectangle.rectangles,
1623                  src->discard_rectangle.count * sizeof(VkRect2D))) {
1624          typed_memcpy(dest->discard_rectangle.rectangles,
1625                       src->discard_rectangle.rectangles,
1626                       src->discard_rectangle.count);
1627          dest_mask |= TU_DYNAMIC_DISCARD_RECTANGLE;
1628       }
1629    }
1630 }
1631
1632 static VkResult
1633 tu_create_cmd_buffer(struct tu_device *device,
1634                      struct tu_cmd_pool *pool,
1635                      VkCommandBufferLevel level,
1636                      VkCommandBuffer *pCommandBuffer)
1637 {
1638    struct tu_cmd_buffer *cmd_buffer;
1639    cmd_buffer = vk_zalloc(&pool->alloc, sizeof(*cmd_buffer), 8,
1640                           VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1641    if (cmd_buffer == NULL)
1642       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
1643
1644    cmd_buffer->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
1645    cmd_buffer->device = device;
1646    cmd_buffer->pool = pool;
1647    cmd_buffer->level = level;
1648
1649    if (pool) {
1650       list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers);
1651       cmd_buffer->queue_family_index = pool->queue_family_index;
1652
1653    } else {
1654       /* Init the pool_link so we can safely call list_del when we destroy
1655        * the command buffer
1656        */
1657       list_inithead(&cmd_buffer->pool_link);
1658       cmd_buffer->queue_family_index = TU_QUEUE_GENERAL;
1659    }
1660
1661    tu_bo_list_init(&cmd_buffer->bo_list);
1662    tu_cs_init(&cmd_buffer->cs, TU_CS_MODE_GROW, 4096);
1663    tu_cs_init(&cmd_buffer->draw_cs, TU_CS_MODE_GROW, 4096);
1664    tu_cs_init(&cmd_buffer->draw_epilogue_cs, TU_CS_MODE_GROW, 4096);
1665    tu_cs_init(&cmd_buffer->sub_cs, TU_CS_MODE_SUB_STREAM, 2048);
1666
1667    *pCommandBuffer = tu_cmd_buffer_to_handle(cmd_buffer);
1668
1669    list_inithead(&cmd_buffer->upload.list);
1670
1671    cmd_buffer->marker_reg = REG_A6XX_CP_SCRATCH_REG(
1672       cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY ? 7 : 6);
1673
1674    VkResult result = tu_bo_init_new(device, &cmd_buffer->scratch_bo, 0x1000);
1675    if (result != VK_SUCCESS)
1676       goto fail_scratch_bo;
1677
1678    /* TODO: resize on overflow */
1679    cmd_buffer->vsc_data_pitch = device->vsc_data_pitch;
1680    cmd_buffer->vsc_data2_pitch = device->vsc_data2_pitch;
1681    cmd_buffer->vsc_data = device->vsc_data;
1682    cmd_buffer->vsc_data2 = device->vsc_data2;
1683
1684    return VK_SUCCESS;
1685
1686 fail_scratch_bo:
1687    list_del(&cmd_buffer->pool_link);
1688    return result;
1689 }
1690
1691 static void
1692 tu_cmd_buffer_destroy(struct tu_cmd_buffer *cmd_buffer)
1693 {
1694    tu_bo_finish(cmd_buffer->device, &cmd_buffer->scratch_bo);
1695
1696    list_del(&cmd_buffer->pool_link);
1697
1698    for (unsigned i = 0; i < VK_PIPELINE_BIND_POINT_RANGE_SIZE; i++)
1699       free(cmd_buffer->descriptors[i].push_set.set.mapped_ptr);
1700
1701    tu_cs_finish(cmd_buffer->device, &cmd_buffer->cs);
1702    tu_cs_finish(cmd_buffer->device, &cmd_buffer->draw_cs);
1703    tu_cs_finish(cmd_buffer->device, &cmd_buffer->draw_epilogue_cs);
1704    tu_cs_finish(cmd_buffer->device, &cmd_buffer->sub_cs);
1705
1706    tu_bo_list_destroy(&cmd_buffer->bo_list);
1707    vk_free(&cmd_buffer->pool->alloc, cmd_buffer);
1708 }
1709
1710 static VkResult
1711 tu_reset_cmd_buffer(struct tu_cmd_buffer *cmd_buffer)
1712 {
1713    cmd_buffer->wait_for_idle = true;
1714
1715    cmd_buffer->record_result = VK_SUCCESS;
1716
1717    tu_bo_list_reset(&cmd_buffer->bo_list);
1718    tu_cs_reset(cmd_buffer->device, &cmd_buffer->cs);
1719    tu_cs_reset(cmd_buffer->device, &cmd_buffer->draw_cs);
1720    tu_cs_reset(cmd_buffer->device, &cmd_buffer->draw_epilogue_cs);
1721    tu_cs_reset(cmd_buffer->device, &cmd_buffer->sub_cs);
1722
1723    for (unsigned i = 0; i < VK_PIPELINE_BIND_POINT_RANGE_SIZE; i++) {
1724       cmd_buffer->descriptors[i].valid = 0;
1725       cmd_buffer->descriptors[i].push_dirty = false;
1726    }
1727
1728    cmd_buffer->status = TU_CMD_BUFFER_STATUS_INITIAL;
1729
1730    return cmd_buffer->record_result;
1731 }
1732
1733 VkResult
1734 tu_AllocateCommandBuffers(VkDevice _device,
1735                           const VkCommandBufferAllocateInfo *pAllocateInfo,
1736                           VkCommandBuffer *pCommandBuffers)
1737 {
1738    TU_FROM_HANDLE(tu_device, device, _device);
1739    TU_FROM_HANDLE(tu_cmd_pool, pool, pAllocateInfo->commandPool);
1740
1741    VkResult result = VK_SUCCESS;
1742    uint32_t i;
1743
1744    for (i = 0; i < pAllocateInfo->commandBufferCount; i++) {
1745
1746       if (!list_is_empty(&pool->free_cmd_buffers)) {
1747          struct tu_cmd_buffer *cmd_buffer = list_first_entry(
1748             &pool->free_cmd_buffers, struct tu_cmd_buffer, pool_link);
1749
1750          list_del(&cmd_buffer->pool_link);
1751          list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers);
1752
1753          result = tu_reset_cmd_buffer(cmd_buffer);
1754          cmd_buffer->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
1755          cmd_buffer->level = pAllocateInfo->level;
1756
1757          pCommandBuffers[i] = tu_cmd_buffer_to_handle(cmd_buffer);
1758       } else {
1759          result = tu_create_cmd_buffer(device, pool, pAllocateInfo->level,
1760                                        &pCommandBuffers[i]);
1761       }
1762       if (result != VK_SUCCESS)
1763          break;
1764    }
1765
1766    if (result != VK_SUCCESS) {
1767       tu_FreeCommandBuffers(_device, pAllocateInfo->commandPool, i,
1768                             pCommandBuffers);
1769
1770       /* From the Vulkan 1.0.66 spec:
1771        *
1772        * "vkAllocateCommandBuffers can be used to create multiple
1773        *  command buffers. If the creation of any of those command
1774        *  buffers fails, the implementation must destroy all
1775        *  successfully created command buffer objects from this
1776        *  command, set all entries of the pCommandBuffers array to
1777        *  NULL and return the error."
1778        */
1779       memset(pCommandBuffers, 0,
1780              sizeof(*pCommandBuffers) * pAllocateInfo->commandBufferCount);
1781    }
1782
1783    return result;
1784 }
1785
1786 void
1787 tu_FreeCommandBuffers(VkDevice device,
1788                       VkCommandPool commandPool,
1789                       uint32_t commandBufferCount,
1790                       const VkCommandBuffer *pCommandBuffers)
1791 {
1792    for (uint32_t i = 0; i < commandBufferCount; i++) {
1793       TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, pCommandBuffers[i]);
1794
1795       if (cmd_buffer) {
1796          if (cmd_buffer->pool) {
1797             list_del(&cmd_buffer->pool_link);
1798             list_addtail(&cmd_buffer->pool_link,
1799                          &cmd_buffer->pool->free_cmd_buffers);
1800          } else
1801             tu_cmd_buffer_destroy(cmd_buffer);
1802       }
1803    }
1804 }
1805
1806 VkResult
1807 tu_ResetCommandBuffer(VkCommandBuffer commandBuffer,
1808                       VkCommandBufferResetFlags flags)
1809 {
1810    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
1811    return tu_reset_cmd_buffer(cmd_buffer);
1812 }
1813
1814 VkResult
1815 tu_BeginCommandBuffer(VkCommandBuffer commandBuffer,
1816                       const VkCommandBufferBeginInfo *pBeginInfo)
1817 {
1818    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
1819    VkResult result = VK_SUCCESS;
1820
1821    if (cmd_buffer->status != TU_CMD_BUFFER_STATUS_INITIAL) {
1822       /* If the command buffer has already been resetted with
1823        * vkResetCommandBuffer, no need to do it again.
1824        */
1825       result = tu_reset_cmd_buffer(cmd_buffer);
1826       if (result != VK_SUCCESS)
1827          return result;
1828    }
1829
1830    memset(&cmd_buffer->state, 0, sizeof(cmd_buffer->state));
1831    cmd_buffer->usage_flags = pBeginInfo->flags;
1832
1833    tu_cs_begin(&cmd_buffer->cs);
1834    tu_cs_begin(&cmd_buffer->draw_cs);
1835    tu_cs_begin(&cmd_buffer->draw_epilogue_cs);
1836
1837    cmd_buffer->marker_seqno = 0;
1838    cmd_buffer->scratch_seqno = 0;
1839
1840    /* setup initial configuration into command buffer */
1841    if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY) {
1842       switch (cmd_buffer->queue_family_index) {
1843       case TU_QUEUE_GENERAL:
1844          tu6_init_hw(cmd_buffer, &cmd_buffer->cs);
1845          break;
1846       default:
1847          break;
1848       }
1849    } else if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY &&
1850               (pBeginInfo->flags & VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT)) {
1851       assert(pBeginInfo->pInheritanceInfo);
1852       cmd_buffer->state.pass = tu_render_pass_from_handle(pBeginInfo->pInheritanceInfo->renderPass);
1853       cmd_buffer->state.subpass = &cmd_buffer->state.pass->subpasses[pBeginInfo->pInheritanceInfo->subpass];
1854    }
1855
1856    cmd_buffer->status = TU_CMD_BUFFER_STATUS_RECORDING;
1857
1858    return VK_SUCCESS;
1859 }
1860
1861 void
1862 tu_CmdBindVertexBuffers(VkCommandBuffer commandBuffer,
1863                         uint32_t firstBinding,
1864                         uint32_t bindingCount,
1865                         const VkBuffer *pBuffers,
1866                         const VkDeviceSize *pOffsets)
1867 {
1868    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
1869
1870    assert(firstBinding + bindingCount <= MAX_VBS);
1871
1872    for (uint32_t i = 0; i < bindingCount; i++) {
1873       cmd->state.vb.buffers[firstBinding + i] =
1874          tu_buffer_from_handle(pBuffers[i]);
1875       cmd->state.vb.offsets[firstBinding + i] = pOffsets[i];
1876    }
1877
1878    /* VB states depend on VkPipelineVertexInputStateCreateInfo */
1879    cmd->state.dirty |= TU_CMD_DIRTY_VERTEX_BUFFERS;
1880 }
1881
1882 void
1883 tu_CmdBindIndexBuffer(VkCommandBuffer commandBuffer,
1884                       VkBuffer buffer,
1885                       VkDeviceSize offset,
1886                       VkIndexType indexType)
1887 {
1888    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
1889    TU_FROM_HANDLE(tu_buffer, buf, buffer);
1890
1891    /* initialize/update the restart index */
1892    if (!cmd->state.index_buffer || cmd->state.index_type != indexType) {
1893       struct tu_cs *draw_cs = &cmd->draw_cs;
1894       VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 2);
1895       if (result != VK_SUCCESS) {
1896          cmd->record_result = result;
1897          return;
1898       }
1899
1900       tu6_emit_restart_index(
1901          draw_cs, indexType == VK_INDEX_TYPE_UINT32 ? 0xffffffff : 0xffff);
1902
1903       tu_cs_sanity_check(draw_cs);
1904    }
1905
1906    /* track the BO */
1907    if (cmd->state.index_buffer != buf)
1908       tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
1909
1910    cmd->state.index_buffer = buf;
1911    cmd->state.index_offset = offset;
1912    cmd->state.index_type = indexType;
1913 }
1914
1915 void
1916 tu_CmdBindDescriptorSets(VkCommandBuffer commandBuffer,
1917                          VkPipelineBindPoint pipelineBindPoint,
1918                          VkPipelineLayout _layout,
1919                          uint32_t firstSet,
1920                          uint32_t descriptorSetCount,
1921                          const VkDescriptorSet *pDescriptorSets,
1922                          uint32_t dynamicOffsetCount,
1923                          const uint32_t *pDynamicOffsets)
1924 {
1925    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
1926    TU_FROM_HANDLE(tu_pipeline_layout, layout, _layout);
1927    unsigned dyn_idx = 0;
1928
1929    struct tu_descriptor_state *descriptors_state =
1930       tu_get_descriptors_state(cmd_buffer, pipelineBindPoint);
1931
1932    for (unsigned i = 0; i < descriptorSetCount; ++i) {
1933       unsigned idx = i + firstSet;
1934       TU_FROM_HANDLE(tu_descriptor_set, set, pDescriptorSets[i]);
1935
1936       descriptors_state->sets[idx] = set;
1937       descriptors_state->valid |= (1u << idx);
1938
1939       for(unsigned j = 0; j < set->layout->dynamic_offset_count; ++j, ++dyn_idx) {
1940          unsigned idx = j + layout->set[i + firstSet].dynamic_offset_start;
1941          assert(dyn_idx < dynamicOffsetCount);
1942
1943          descriptors_state->dynamic_buffers[idx] =
1944          set->dynamic_descriptors[j].va + pDynamicOffsets[dyn_idx];
1945       }
1946    }
1947
1948    cmd_buffer->state.dirty |= TU_CMD_DIRTY_DESCRIPTOR_SETS;
1949 }
1950
1951 void
1952 tu_CmdPushConstants(VkCommandBuffer commandBuffer,
1953                     VkPipelineLayout layout,
1954                     VkShaderStageFlags stageFlags,
1955                     uint32_t offset,
1956                     uint32_t size,
1957                     const void *pValues)
1958 {
1959    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
1960    memcpy((void*) cmd->push_constants + offset, pValues, size);
1961    cmd->state.dirty |= TU_CMD_DIRTY_PUSH_CONSTANTS;
1962 }
1963
1964 VkResult
1965 tu_EndCommandBuffer(VkCommandBuffer commandBuffer)
1966 {
1967    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
1968
1969    if (cmd_buffer->scratch_seqno) {
1970       tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->scratch_bo,
1971                      MSM_SUBMIT_BO_WRITE);
1972    }
1973
1974    if (cmd_buffer->use_vsc_data) {
1975       tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->vsc_data,
1976                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
1977       tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->vsc_data2,
1978                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
1979    }
1980
1981    for (uint32_t i = 0; i < cmd_buffer->draw_cs.bo_count; i++) {
1982       tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->draw_cs.bos[i],
1983                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
1984    }
1985
1986    for (uint32_t i = 0; i < cmd_buffer->draw_epilogue_cs.bo_count; i++) {
1987       tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->draw_epilogue_cs.bos[i],
1988                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
1989    }
1990
1991    for (uint32_t i = 0; i < cmd_buffer->sub_cs.bo_count; i++) {
1992       tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->sub_cs.bos[i],
1993                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
1994    }
1995
1996    tu_cs_end(&cmd_buffer->cs);
1997    tu_cs_end(&cmd_buffer->draw_cs);
1998    tu_cs_end(&cmd_buffer->draw_epilogue_cs);
1999
2000    cmd_buffer->status = TU_CMD_BUFFER_STATUS_EXECUTABLE;
2001
2002    return cmd_buffer->record_result;
2003 }
2004
2005 void
2006 tu_CmdBindPipeline(VkCommandBuffer commandBuffer,
2007                    VkPipelineBindPoint pipelineBindPoint,
2008                    VkPipeline _pipeline)
2009 {
2010    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2011    TU_FROM_HANDLE(tu_pipeline, pipeline, _pipeline);
2012
2013    switch (pipelineBindPoint) {
2014    case VK_PIPELINE_BIND_POINT_GRAPHICS:
2015       cmd->state.pipeline = pipeline;
2016       cmd->state.dirty |= TU_CMD_DIRTY_PIPELINE;
2017       break;
2018    case VK_PIPELINE_BIND_POINT_COMPUTE:
2019       cmd->state.compute_pipeline = pipeline;
2020       cmd->state.dirty |= TU_CMD_DIRTY_COMPUTE_PIPELINE;
2021       break;
2022    default:
2023       unreachable("unrecognized pipeline bind point");
2024       break;
2025    }
2026
2027    tu_bo_list_add(&cmd->bo_list, &pipeline->program.binary_bo,
2028                   MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2029    for (uint32_t i = 0; i < pipeline->cs.bo_count; i++) {
2030       tu_bo_list_add(&cmd->bo_list, pipeline->cs.bos[i],
2031                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2032    }
2033 }
2034
2035 void
2036 tu_CmdSetViewport(VkCommandBuffer commandBuffer,
2037                   uint32_t firstViewport,
2038                   uint32_t viewportCount,
2039                   const VkViewport *pViewports)
2040 {
2041    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2042    struct tu_cs *draw_cs = &cmd->draw_cs;
2043
2044    VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 12);
2045    if (result != VK_SUCCESS) {
2046       cmd->record_result = result;
2047       return;
2048    }
2049
2050    assert(firstViewport == 0 && viewportCount == 1);
2051    tu6_emit_viewport(draw_cs, pViewports);
2052
2053    tu_cs_sanity_check(draw_cs);
2054 }
2055
2056 void
2057 tu_CmdSetScissor(VkCommandBuffer commandBuffer,
2058                  uint32_t firstScissor,
2059                  uint32_t scissorCount,
2060                  const VkRect2D *pScissors)
2061 {
2062    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2063    struct tu_cs *draw_cs = &cmd->draw_cs;
2064
2065    VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 3);
2066    if (result != VK_SUCCESS) {
2067       cmd->record_result = result;
2068       return;
2069    }
2070
2071    assert(firstScissor == 0 && scissorCount == 1);
2072    tu6_emit_scissor(draw_cs, pScissors);
2073
2074    tu_cs_sanity_check(draw_cs);
2075 }
2076
2077 void
2078 tu_CmdSetLineWidth(VkCommandBuffer commandBuffer, float lineWidth)
2079 {
2080    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2081
2082    cmd->state.dynamic.line_width = lineWidth;
2083
2084    /* line width depends on VkPipelineRasterizationStateCreateInfo */
2085    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_LINE_WIDTH;
2086 }
2087
2088 void
2089 tu_CmdSetDepthBias(VkCommandBuffer commandBuffer,
2090                    float depthBiasConstantFactor,
2091                    float depthBiasClamp,
2092                    float depthBiasSlopeFactor)
2093 {
2094    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2095    struct tu_cs *draw_cs = &cmd->draw_cs;
2096
2097    VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 4);
2098    if (result != VK_SUCCESS) {
2099       cmd->record_result = result;
2100       return;
2101    }
2102
2103    tu6_emit_depth_bias(draw_cs, depthBiasConstantFactor, depthBiasClamp,
2104                        depthBiasSlopeFactor);
2105
2106    tu_cs_sanity_check(draw_cs);
2107 }
2108
2109 void
2110 tu_CmdSetBlendConstants(VkCommandBuffer commandBuffer,
2111                         const float blendConstants[4])
2112 {
2113    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2114    struct tu_cs *draw_cs = &cmd->draw_cs;
2115
2116    VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 5);
2117    if (result != VK_SUCCESS) {
2118       cmd->record_result = result;
2119       return;
2120    }
2121
2122    tu6_emit_blend_constants(draw_cs, blendConstants);
2123
2124    tu_cs_sanity_check(draw_cs);
2125 }
2126
2127 void
2128 tu_CmdSetDepthBounds(VkCommandBuffer commandBuffer,
2129                      float minDepthBounds,
2130                      float maxDepthBounds)
2131 {
2132 }
2133
2134 void
2135 tu_CmdSetStencilCompareMask(VkCommandBuffer commandBuffer,
2136                             VkStencilFaceFlags faceMask,
2137                             uint32_t compareMask)
2138 {
2139    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2140
2141    if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
2142       cmd->state.dynamic.stencil_compare_mask.front = compareMask;
2143    if (faceMask & VK_STENCIL_FACE_BACK_BIT)
2144       cmd->state.dynamic.stencil_compare_mask.back = compareMask;
2145
2146    /* the front/back compare masks must be updated together */
2147    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK;
2148 }
2149
2150 void
2151 tu_CmdSetStencilWriteMask(VkCommandBuffer commandBuffer,
2152                           VkStencilFaceFlags faceMask,
2153                           uint32_t writeMask)
2154 {
2155    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2156
2157    if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
2158       cmd->state.dynamic.stencil_write_mask.front = writeMask;
2159    if (faceMask & VK_STENCIL_FACE_BACK_BIT)
2160       cmd->state.dynamic.stencil_write_mask.back = writeMask;
2161
2162    /* the front/back write masks must be updated together */
2163    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK;
2164 }
2165
2166 void
2167 tu_CmdSetStencilReference(VkCommandBuffer commandBuffer,
2168                           VkStencilFaceFlags faceMask,
2169                           uint32_t reference)
2170 {
2171    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2172
2173    if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
2174       cmd->state.dynamic.stencil_reference.front = reference;
2175    if (faceMask & VK_STENCIL_FACE_BACK_BIT)
2176       cmd->state.dynamic.stencil_reference.back = reference;
2177
2178    /* the front/back references must be updated together */
2179    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE;
2180 }
2181
2182 void
2183 tu_CmdExecuteCommands(VkCommandBuffer commandBuffer,
2184                       uint32_t commandBufferCount,
2185                       const VkCommandBuffer *pCmdBuffers)
2186 {
2187    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2188    VkResult result;
2189
2190    assert(commandBufferCount > 0);
2191
2192    for (uint32_t i = 0; i < commandBufferCount; i++) {
2193       TU_FROM_HANDLE(tu_cmd_buffer, secondary, pCmdBuffers[i]);
2194
2195       result = tu_bo_list_merge(&cmd->bo_list, &secondary->bo_list);
2196       if (result != VK_SUCCESS) {
2197          cmd->record_result = result;
2198          break;
2199       }
2200
2201       result = tu_cs_add_entries(&cmd->draw_cs, &secondary->draw_cs);
2202       if (result != VK_SUCCESS) {
2203          cmd->record_result = result;
2204          break;
2205       }
2206
2207       result = tu_cs_add_entries(&cmd->draw_epilogue_cs,
2208             &secondary->draw_epilogue_cs);
2209       if (result != VK_SUCCESS) {
2210          cmd->record_result = result;
2211          break;
2212       }
2213    }
2214    cmd->state.dirty = ~0u; /* TODO: set dirty only what needs to be */
2215 }
2216
2217 VkResult
2218 tu_CreateCommandPool(VkDevice _device,
2219                      const VkCommandPoolCreateInfo *pCreateInfo,
2220                      const VkAllocationCallbacks *pAllocator,
2221                      VkCommandPool *pCmdPool)
2222 {
2223    TU_FROM_HANDLE(tu_device, device, _device);
2224    struct tu_cmd_pool *pool;
2225
2226    pool = vk_alloc2(&device->alloc, pAllocator, sizeof(*pool), 8,
2227                     VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2228    if (pool == NULL)
2229       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
2230
2231    if (pAllocator)
2232       pool->alloc = *pAllocator;
2233    else
2234       pool->alloc = device->alloc;
2235
2236    list_inithead(&pool->cmd_buffers);
2237    list_inithead(&pool->free_cmd_buffers);
2238
2239    pool->queue_family_index = pCreateInfo->queueFamilyIndex;
2240
2241    *pCmdPool = tu_cmd_pool_to_handle(pool);
2242
2243    return VK_SUCCESS;
2244 }
2245
2246 void
2247 tu_DestroyCommandPool(VkDevice _device,
2248                       VkCommandPool commandPool,
2249                       const VkAllocationCallbacks *pAllocator)
2250 {
2251    TU_FROM_HANDLE(tu_device, device, _device);
2252    TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2253
2254    if (!pool)
2255       return;
2256
2257    list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2258                             &pool->cmd_buffers, pool_link)
2259    {
2260       tu_cmd_buffer_destroy(cmd_buffer);
2261    }
2262
2263    list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2264                             &pool->free_cmd_buffers, pool_link)
2265    {
2266       tu_cmd_buffer_destroy(cmd_buffer);
2267    }
2268
2269    vk_free2(&device->alloc, pAllocator, pool);
2270 }
2271
2272 VkResult
2273 tu_ResetCommandPool(VkDevice device,
2274                     VkCommandPool commandPool,
2275                     VkCommandPoolResetFlags flags)
2276 {
2277    TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2278    VkResult result;
2279
2280    list_for_each_entry(struct tu_cmd_buffer, cmd_buffer, &pool->cmd_buffers,
2281                        pool_link)
2282    {
2283       result = tu_reset_cmd_buffer(cmd_buffer);
2284       if (result != VK_SUCCESS)
2285          return result;
2286    }
2287
2288    return VK_SUCCESS;
2289 }
2290
2291 void
2292 tu_TrimCommandPool(VkDevice device,
2293                    VkCommandPool commandPool,
2294                    VkCommandPoolTrimFlags flags)
2295 {
2296    TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2297
2298    if (!pool)
2299       return;
2300
2301    list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2302                             &pool->free_cmd_buffers, pool_link)
2303    {
2304       tu_cmd_buffer_destroy(cmd_buffer);
2305    }
2306 }
2307
2308 void
2309 tu_CmdBeginRenderPass(VkCommandBuffer commandBuffer,
2310                       const VkRenderPassBeginInfo *pRenderPassBegin,
2311                       VkSubpassContents contents)
2312 {
2313    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2314    TU_FROM_HANDLE(tu_render_pass, pass, pRenderPassBegin->renderPass);
2315    TU_FROM_HANDLE(tu_framebuffer, fb, pRenderPassBegin->framebuffer);
2316
2317    cmd->state.pass = pass;
2318    cmd->state.subpass = pass->subpasses;
2319    cmd->state.framebuffer = fb;
2320
2321    tu_cmd_update_tiling_config(cmd, &pRenderPassBegin->renderArea);
2322    tu_cmd_prepare_tile_load_ib(cmd, pRenderPassBegin);
2323    tu_cmd_prepare_tile_store_ib(cmd);
2324
2325    /* note: use_hw_binning only checks tiling config */
2326    if (use_hw_binning(cmd))
2327       cmd->use_vsc_data = true;
2328
2329    for (uint32_t i = 0; i < fb->attachment_count; ++i) {
2330       const struct tu_image_view *iview = fb->attachments[i].attachment;
2331       tu_bo_list_add(&cmd->bo_list, iview->image->bo,
2332                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
2333    }
2334 }
2335
2336 void
2337 tu_CmdBeginRenderPass2(VkCommandBuffer commandBuffer,
2338                        const VkRenderPassBeginInfo *pRenderPassBeginInfo,
2339                        const VkSubpassBeginInfoKHR *pSubpassBeginInfo)
2340 {
2341    tu_CmdBeginRenderPass(commandBuffer, pRenderPassBeginInfo,
2342                          pSubpassBeginInfo->contents);
2343 }
2344
2345 void
2346 tu_CmdNextSubpass(VkCommandBuffer commandBuffer, VkSubpassContents contents)
2347 {
2348    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2349    const struct tu_render_pass *pass = cmd->state.pass;
2350    struct tu_cs *cs = &cmd->draw_cs;
2351
2352    VkResult result = tu_cs_reserve_space(cmd->device, cs, 1024);
2353    if (result != VK_SUCCESS) {
2354       cmd->record_result = result;
2355       return;
2356    }
2357
2358    const struct tu_subpass *subpass = cmd->state.subpass++;
2359    /* TODO:
2360     * if msaa samples change between subpasses,
2361     * attachment store is broken for some attachments
2362     */
2363    if (subpass->resolve_attachments) {
2364       tu6_emit_blit_scissor(cmd, cs, true);
2365       for (unsigned i = 0; i < subpass->color_count; i++) {
2366          uint32_t a = subpass->resolve_attachments[i].attachment;
2367          if (a != VK_ATTACHMENT_UNUSED) {
2368                tu6_emit_store_attachment(cmd, cs, a,
2369                                          subpass->color_attachments[i].attachment);
2370          }
2371       }
2372    }
2373
2374    /* invalidate because reading input attachments will cache GMEM and
2375     * the cache isn''t updated when GMEM is written
2376     * TODO: is there a no-cache bit for textures?
2377     */
2378    if (cmd->state.subpass->input_count)
2379       tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE, false);
2380
2381    /* emit mrt/zs/msaa state for the subpass that is starting */
2382    tu6_emit_zs(cmd, cmd->state.subpass, cs);
2383    tu6_emit_mrt(cmd, cmd->state.subpass, cs);
2384    tu6_emit_msaa(cmd, cmd->state.subpass, cs);
2385
2386    /* TODO:
2387     * since we don't know how to do GMEM->GMEM resolve,
2388     * resolve attachments are resolved to memory then loaded to GMEM again if needed
2389     */
2390    if (subpass->resolve_attachments) {
2391       for (unsigned i = 0; i < subpass->color_count; i++) {
2392          uint32_t a = subpass->resolve_attachments[i].attachment;
2393          const struct tu_image_view *iview =
2394             cmd->state.framebuffer->attachments[a].attachment;
2395          if (a != VK_ATTACHMENT_UNUSED && pass->attachments[a].gmem_offset >= 0) {
2396                tu_finishme("missing GMEM->GMEM resolve, performance will suffer\n");
2397                tu6_emit_blit_info(cmd, cs, iview, pass->attachments[a].gmem_offset, false);
2398                tu6_emit_blit(cmd, cs);
2399          }
2400       }
2401    }
2402 }
2403
2404 void
2405 tu_CmdNextSubpass2(VkCommandBuffer commandBuffer,
2406                    const VkSubpassBeginInfoKHR *pSubpassBeginInfo,
2407                    const VkSubpassEndInfoKHR *pSubpassEndInfo)
2408 {
2409    tu_CmdNextSubpass(commandBuffer, pSubpassBeginInfo->contents);
2410 }
2411
2412 struct tu_draw_info
2413 {
2414    /**
2415     * Number of vertices.
2416     */
2417    uint32_t count;
2418
2419    /**
2420     * Index of the first vertex.
2421     */
2422    int32_t vertex_offset;
2423
2424    /**
2425     * First instance id.
2426     */
2427    uint32_t first_instance;
2428
2429    /**
2430     * Number of instances.
2431     */
2432    uint32_t instance_count;
2433
2434    /**
2435     * First index (indexed draws only).
2436     */
2437    uint32_t first_index;
2438
2439    /**
2440     * Whether it's an indexed draw.
2441     */
2442    bool indexed;
2443
2444    /**
2445     * Indirect draw parameters resource.
2446     */
2447    struct tu_buffer *indirect;
2448    uint64_t indirect_offset;
2449    uint32_t stride;
2450
2451    /**
2452     * Draw count parameters resource.
2453     */
2454    struct tu_buffer *count_buffer;
2455    uint64_t count_buffer_offset;
2456 };
2457
2458 #define ENABLE_ALL (CP_SET_DRAW_STATE__0_BINNING | CP_SET_DRAW_STATE__0_GMEM | CP_SET_DRAW_STATE__0_SYSMEM)
2459 #define ENABLE_DRAW (CP_SET_DRAW_STATE__0_GMEM | CP_SET_DRAW_STATE__0_SYSMEM)
2460
2461 enum tu_draw_state_group_id
2462 {
2463    TU_DRAW_STATE_PROGRAM,
2464    TU_DRAW_STATE_PROGRAM_BINNING,
2465    TU_DRAW_STATE_VI,
2466    TU_DRAW_STATE_VI_BINNING,
2467    TU_DRAW_STATE_VP,
2468    TU_DRAW_STATE_RAST,
2469    TU_DRAW_STATE_DS,
2470    TU_DRAW_STATE_BLEND,
2471    TU_DRAW_STATE_VS_CONST,
2472    TU_DRAW_STATE_FS_CONST,
2473    TU_DRAW_STATE_VS_TEX,
2474    TU_DRAW_STATE_FS_TEX,
2475    TU_DRAW_STATE_FS_IBO,
2476    TU_DRAW_STATE_VS_PARAMS,
2477
2478    TU_DRAW_STATE_COUNT,
2479 };
2480
2481 struct tu_draw_state_group
2482 {
2483    enum tu_draw_state_group_id id;
2484    uint32_t enable_mask;
2485    struct tu_cs_entry ib;
2486 };
2487
2488 const static struct tu_sampler*
2489 sampler_ptr(struct tu_descriptor_state *descriptors_state,
2490             const struct tu_descriptor_map *map, unsigned i,
2491             unsigned array_index)
2492 {
2493    assert(descriptors_state->valid & (1 << map->set[i]));
2494
2495    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2496    assert(map->binding[i] < set->layout->binding_count);
2497
2498    const struct tu_descriptor_set_binding_layout *layout =
2499       &set->layout->binding[map->binding[i]];
2500
2501    if (layout->immutable_samplers_offset) {
2502       const struct tu_sampler *immutable_samplers =
2503          tu_immutable_samplers(set->layout, layout);
2504
2505       return &immutable_samplers[array_index];
2506    }
2507
2508    switch (layout->type) {
2509    case VK_DESCRIPTOR_TYPE_SAMPLER:
2510       return (struct tu_sampler*) &set->mapped_ptr[layout->offset / 4];
2511    case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
2512       return (struct tu_sampler*) &set->mapped_ptr[layout->offset / 4 + A6XX_TEX_CONST_DWORDS +
2513                                                    array_index *
2514                                                    (A6XX_TEX_CONST_DWORDS +
2515                                                     sizeof(struct tu_sampler) / 4)];
2516    default:
2517       unreachable("unimplemented descriptor type");
2518       break;
2519    }
2520 }
2521
2522 static void
2523 write_tex_const(struct tu_cmd_buffer *cmd,
2524                 uint32_t *dst,
2525                 struct tu_descriptor_state *descriptors_state,
2526                 const struct tu_descriptor_map *map,
2527                 unsigned i, unsigned array_index)
2528 {
2529    assert(descriptors_state->valid & (1 << map->set[i]));
2530
2531    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2532    assert(map->binding[i] < set->layout->binding_count);
2533
2534    const struct tu_descriptor_set_binding_layout *layout =
2535       &set->layout->binding[map->binding[i]];
2536
2537    switch (layout->type) {
2538    case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
2539    case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
2540    case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
2541    case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
2542       memcpy(dst, &set->mapped_ptr[layout->offset / 4 +
2543                                    array_index * A6XX_TEX_CONST_DWORDS],
2544              A6XX_TEX_CONST_DWORDS * 4);
2545       break;
2546    case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
2547       memcpy(dst, &set->mapped_ptr[layout->offset / 4 +
2548                                    array_index *
2549                                    (A6XX_TEX_CONST_DWORDS +
2550                                     sizeof(struct tu_sampler) / 4)],
2551              A6XX_TEX_CONST_DWORDS * 4);
2552       break;
2553    default:
2554       unreachable("unimplemented descriptor type");
2555       break;
2556    }
2557
2558    if (layout->type == VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT) {
2559       const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
2560       uint32_t a = cmd->state.subpass->input_attachments[map->value[i] +
2561                                                          array_index].attachment;
2562       const struct tu_render_pass_attachment *att = &cmd->state.pass->attachments[a];
2563
2564       assert(att->gmem_offset >= 0);
2565
2566       dst[0] &= ~(A6XX_TEX_CONST_0_SWAP__MASK | A6XX_TEX_CONST_0_TILE_MODE__MASK);
2567       dst[0] |= A6XX_TEX_CONST_0_TILE_MODE(TILE6_2);
2568       dst[2] &= ~(A6XX_TEX_CONST_2_TYPE__MASK | A6XX_TEX_CONST_2_PITCH__MASK);
2569       dst[2] |=
2570          A6XX_TEX_CONST_2_TYPE(A6XX_TEX_2D) |
2571          A6XX_TEX_CONST_2_PITCH(tiling->tile0.extent.width * att->cpp);
2572       dst[3] = 0;
2573       dst[4] = 0x100000 + att->gmem_offset;
2574       dst[5] = A6XX_TEX_CONST_5_DEPTH(1);
2575       for (unsigned i = 6; i < A6XX_TEX_CONST_DWORDS; i++)
2576          dst[i] = 0;
2577
2578       if (cmd->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY)
2579          tu_finishme("patch input attachment pitch for secondary cmd buffer");
2580    }
2581 }
2582
2583 static void
2584 write_image_ibo(struct tu_cmd_buffer *cmd,
2585                 uint32_t *dst,
2586                 struct tu_descriptor_state *descriptors_state,
2587                 const struct tu_descriptor_map *map,
2588                 unsigned i, unsigned array_index)
2589 {
2590    assert(descriptors_state->valid & (1 << map->set[i]));
2591
2592    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2593    assert(map->binding[i] < set->layout->binding_count);
2594
2595    const struct tu_descriptor_set_binding_layout *layout =
2596       &set->layout->binding[map->binding[i]];
2597
2598    assert(layout->type == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE);
2599
2600    memcpy(dst, &set->mapped_ptr[layout->offset / 4 +
2601                                 (array_index * 2 + 1) * A6XX_TEX_CONST_DWORDS],
2602           A6XX_TEX_CONST_DWORDS * 4);
2603 }
2604
2605 static uint64_t
2606 buffer_ptr(struct tu_descriptor_state *descriptors_state,
2607            const struct tu_descriptor_map *map,
2608            unsigned i, unsigned array_index)
2609 {
2610    assert(descriptors_state->valid & (1 << map->set[i]));
2611
2612    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2613    assert(map->binding[i] < set->layout->binding_count);
2614
2615    const struct tu_descriptor_set_binding_layout *layout =
2616       &set->layout->binding[map->binding[i]];
2617
2618    switch (layout->type) {
2619    case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
2620    case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
2621       return descriptors_state->dynamic_buffers[layout->dynamic_offset_offset +
2622                                                 array_index];
2623    case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
2624    case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
2625       return (uint64_t) set->mapped_ptr[layout->offset / 4 + array_index * 2 + 1] << 32 |
2626                         set->mapped_ptr[layout->offset / 4 + array_index * 2];
2627    default:
2628       unreachable("unimplemented descriptor type");
2629       break;
2630    }
2631 }
2632
2633 static inline uint32_t
2634 tu6_stage2opcode(gl_shader_stage type)
2635 {
2636    switch (type) {
2637    case MESA_SHADER_VERTEX:
2638    case MESA_SHADER_TESS_CTRL:
2639    case MESA_SHADER_TESS_EVAL:
2640    case MESA_SHADER_GEOMETRY:
2641       return CP_LOAD_STATE6_GEOM;
2642    case MESA_SHADER_FRAGMENT:
2643    case MESA_SHADER_COMPUTE:
2644    case MESA_SHADER_KERNEL:
2645       return CP_LOAD_STATE6_FRAG;
2646    default:
2647       unreachable("bad shader type");
2648    }
2649 }
2650
2651 static inline enum a6xx_state_block
2652 tu6_stage2shadersb(gl_shader_stage type)
2653 {
2654    switch (type) {
2655    case MESA_SHADER_VERTEX:
2656       return SB6_VS_SHADER;
2657    case MESA_SHADER_FRAGMENT:
2658       return SB6_FS_SHADER;
2659    case MESA_SHADER_COMPUTE:
2660    case MESA_SHADER_KERNEL:
2661       return SB6_CS_SHADER;
2662    default:
2663       unreachable("bad shader type");
2664       return ~0;
2665    }
2666 }
2667
2668 static void
2669 tu6_emit_user_consts(struct tu_cs *cs, const struct tu_pipeline *pipeline,
2670                      struct tu_descriptor_state *descriptors_state,
2671                      gl_shader_stage type,
2672                      uint32_t *push_constants)
2673 {
2674    const struct tu_program_descriptor_linkage *link =
2675       &pipeline->program.link[type];
2676    const struct ir3_ubo_analysis_state *state = &link->ubo_state;
2677
2678    for (uint32_t i = 0; i < ARRAY_SIZE(state->range); i++) {
2679       if (state->range[i].start < state->range[i].end) {
2680          uint32_t size = state->range[i].end - state->range[i].start;
2681          uint32_t offset = state->range[i].start;
2682
2683          /* and even if the start of the const buffer is before
2684           * first_immediate, the end may not be:
2685           */
2686          size = MIN2(size, (16 * link->constlen) - state->range[i].offset);
2687
2688          if (size == 0)
2689             continue;
2690
2691          /* things should be aligned to vec4: */
2692          debug_assert((state->range[i].offset % 16) == 0);
2693          debug_assert((size % 16) == 0);
2694          debug_assert((offset % 16) == 0);
2695
2696          if (i == 0) {
2697             /* push constants */
2698             tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + (size / 4));
2699             tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(state->range[i].offset / 16) |
2700                   CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2701                   CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
2702                   CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2703                   CP_LOAD_STATE6_0_NUM_UNIT(size / 16));
2704             tu_cs_emit(cs, 0);
2705             tu_cs_emit(cs, 0);
2706             for (unsigned i = 0; i < size / 4; i++)
2707                tu_cs_emit(cs, push_constants[i + offset / 4]);
2708             continue;
2709          }
2710
2711          /* Look through the UBO map to find our UBO index, and get the VA for
2712           * that UBO.
2713           */
2714          uint64_t va = 0;
2715          uint32_t ubo_idx = i - 1;
2716          uint32_t ubo_map_base = 0;
2717          for (int j = 0; j < link->ubo_map.num; j++) {
2718             if (ubo_idx >= ubo_map_base &&
2719                 ubo_idx < ubo_map_base + link->ubo_map.array_size[j]) {
2720                va = buffer_ptr(descriptors_state, &link->ubo_map, j,
2721                                ubo_idx - ubo_map_base);
2722                break;
2723             }
2724             ubo_map_base += link->ubo_map.array_size[j];
2725          }
2726          assert(va);
2727
2728          tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3);
2729          tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(state->range[i].offset / 16) |
2730                CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2731                CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
2732                CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2733                CP_LOAD_STATE6_0_NUM_UNIT(size / 16));
2734          tu_cs_emit_qw(cs, va + offset);
2735       }
2736    }
2737 }
2738
2739 static void
2740 tu6_emit_ubos(struct tu_cs *cs, const struct tu_pipeline *pipeline,
2741               struct tu_descriptor_state *descriptors_state,
2742               gl_shader_stage type)
2743 {
2744    const struct tu_program_descriptor_linkage *link =
2745       &pipeline->program.link[type];
2746
2747    uint32_t num = MIN2(link->ubo_map.num_desc, link->const_state.num_ubos);
2748    uint32_t anum = align(num, 2);
2749
2750    if (!num)
2751       return;
2752
2753    tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + (2 * anum));
2754    tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(link->const_state.offsets.ubo) |
2755          CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2756          CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
2757          CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2758          CP_LOAD_STATE6_0_NUM_UNIT(anum/2));
2759    tu_cs_emit(cs, CP_LOAD_STATE6_1_EXT_SRC_ADDR(0));
2760    tu_cs_emit(cs, CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI(0));
2761
2762    unsigned emitted = 0;
2763    for (unsigned i = 0; emitted < num && i < link->ubo_map.num; i++) {
2764       for (unsigned j = 0; emitted < num && j < link->ubo_map.array_size[i]; j++) {
2765          tu_cs_emit_qw(cs, buffer_ptr(descriptors_state, &link->ubo_map, i, j));
2766          emitted++;
2767       }
2768    }
2769
2770    for (; emitted < anum; emitted++) {
2771       tu_cs_emit(cs, 0xffffffff);
2772       tu_cs_emit(cs, 0xffffffff);
2773    }
2774 }
2775
2776 static struct tu_cs_entry
2777 tu6_emit_consts(struct tu_cmd_buffer *cmd,
2778                 const struct tu_pipeline *pipeline,
2779                 struct tu_descriptor_state *descriptors_state,
2780                 gl_shader_stage type)
2781 {
2782    struct tu_cs cs;
2783    tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs, 512, &cs); /* TODO: maximum size? */
2784
2785    tu6_emit_user_consts(&cs, pipeline, descriptors_state, type, cmd->push_constants);
2786    tu6_emit_ubos(&cs, pipeline, descriptors_state, type);
2787
2788    return tu_cs_end_sub_stream(&cmd->sub_cs, &cs);
2789 }
2790
2791 static VkResult
2792 tu6_emit_vs_params(struct tu_cmd_buffer *cmd,
2793                    const struct tu_draw_info *draw,
2794                    struct tu_cs_entry *entry)
2795 {
2796    /* TODO: fill out more than just base instance */
2797    const struct tu_program_descriptor_linkage *link =
2798       &cmd->state.pipeline->program.link[MESA_SHADER_VERTEX];
2799    const struct ir3_const_state *const_state = &link->const_state;
2800    struct tu_cs cs;
2801
2802    if (const_state->offsets.driver_param >= link->constlen) {
2803       *entry = (struct tu_cs_entry) {};
2804       return VK_SUCCESS;
2805    }
2806
2807    VkResult result = tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs, 8, &cs);
2808    if (result != VK_SUCCESS)
2809       return result;
2810
2811    tu_cs_emit_pkt7(&cs, CP_LOAD_STATE6_GEOM, 3 + 4);
2812    tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(const_state->offsets.driver_param) |
2813          CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2814          CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
2815          CP_LOAD_STATE6_0_STATE_BLOCK(SB6_VS_SHADER) |
2816          CP_LOAD_STATE6_0_NUM_UNIT(1));
2817    tu_cs_emit(&cs, 0);
2818    tu_cs_emit(&cs, 0);
2819
2820    STATIC_ASSERT(IR3_DP_INSTID_BASE == 2);
2821
2822    tu_cs_emit(&cs, 0);
2823    tu_cs_emit(&cs, 0);
2824    tu_cs_emit(&cs, draw->first_instance);
2825    tu_cs_emit(&cs, 0);
2826
2827    *entry = tu_cs_end_sub_stream(&cmd->sub_cs, &cs);
2828    return VK_SUCCESS;
2829 }
2830
2831 static VkResult
2832 tu6_emit_textures(struct tu_cmd_buffer *cmd,
2833                   const struct tu_pipeline *pipeline,
2834                   struct tu_descriptor_state *descriptors_state,
2835                   gl_shader_stage type,
2836                   struct tu_cs_entry *entry,
2837                   bool *needs_border)
2838 {
2839    struct tu_device *device = cmd->device;
2840    struct tu_cs *draw_state = &cmd->sub_cs;
2841    const struct tu_program_descriptor_linkage *link =
2842       &pipeline->program.link[type];
2843    VkResult result;
2844
2845    if (link->texture_map.num_desc == 0 && link->sampler_map.num_desc == 0) {
2846       *entry = (struct tu_cs_entry) {};
2847       return VK_SUCCESS;
2848    }
2849
2850    /* allocate and fill texture state */
2851    struct ts_cs_memory tex_const;
2852    result = tu_cs_alloc(device, draw_state, link->texture_map.num_desc,
2853                         A6XX_TEX_CONST_DWORDS, &tex_const);
2854    if (result != VK_SUCCESS)
2855       return result;
2856
2857    int tex_index = 0;
2858    for (unsigned i = 0; i < link->texture_map.num; i++) {
2859       for (int j = 0; j < link->texture_map.array_size[i]; j++) {
2860          write_tex_const(cmd,
2861                          &tex_const.map[A6XX_TEX_CONST_DWORDS * tex_index++],
2862                          descriptors_state, &link->texture_map, i, j);
2863       }
2864    }
2865
2866    /* allocate and fill sampler state */
2867    struct ts_cs_memory tex_samp = { 0 };
2868    if (link->sampler_map.num_desc) {
2869       result = tu_cs_alloc(device, draw_state, link->sampler_map.num_desc,
2870                            A6XX_TEX_SAMP_DWORDS, &tex_samp);
2871       if (result != VK_SUCCESS)
2872          return result;
2873
2874       int sampler_index = 0;
2875       for (unsigned i = 0; i < link->sampler_map.num; i++) {
2876          for (int j = 0; j < link->sampler_map.array_size[i]; j++) {
2877             const struct tu_sampler *sampler = sampler_ptr(descriptors_state,
2878                                                            &link->sampler_map,
2879                                                            i, j);
2880             memcpy(&tex_samp.map[A6XX_TEX_SAMP_DWORDS * sampler_index++],
2881                    sampler->state, sizeof(sampler->state));
2882             *needs_border |= sampler->needs_border;
2883          }
2884       }
2885    }
2886
2887    unsigned tex_samp_reg, tex_const_reg, tex_count_reg;
2888    enum a6xx_state_block sb;
2889
2890    switch (type) {
2891    case MESA_SHADER_VERTEX:
2892       sb = SB6_VS_TEX;
2893       tex_samp_reg = REG_A6XX_SP_VS_TEX_SAMP_LO;
2894       tex_const_reg = REG_A6XX_SP_VS_TEX_CONST_LO;
2895       tex_count_reg = REG_A6XX_SP_VS_TEX_COUNT;
2896       break;
2897    case MESA_SHADER_FRAGMENT:
2898       sb = SB6_FS_TEX;
2899       tex_samp_reg = REG_A6XX_SP_FS_TEX_SAMP_LO;
2900       tex_const_reg = REG_A6XX_SP_FS_TEX_CONST_LO;
2901       tex_count_reg = REG_A6XX_SP_FS_TEX_COUNT;
2902       break;
2903    case MESA_SHADER_COMPUTE:
2904       sb = SB6_CS_TEX;
2905       tex_samp_reg = REG_A6XX_SP_CS_TEX_SAMP_LO;
2906       tex_const_reg = REG_A6XX_SP_CS_TEX_CONST_LO;
2907       tex_count_reg = REG_A6XX_SP_CS_TEX_COUNT;
2908       break;
2909    default:
2910       unreachable("bad state block");
2911    }
2912
2913    struct tu_cs cs;
2914    result = tu_cs_begin_sub_stream(device, draw_state, 16, &cs);
2915    if (result != VK_SUCCESS)
2916       return result;
2917
2918    if (link->sampler_map.num_desc) {
2919       /* output sampler state: */
2920       tu_cs_emit_pkt7(&cs, tu6_stage2opcode(type), 3);
2921       tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
2922                  CP_LOAD_STATE6_0_STATE_TYPE(ST6_SHADER) |
2923                  CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
2924                  CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
2925                  CP_LOAD_STATE6_0_NUM_UNIT(link->sampler_map.num_desc));
2926       tu_cs_emit_qw(&cs, tex_samp.iova); /* SRC_ADDR_LO/HI */
2927
2928       tu_cs_emit_pkt4(&cs, tex_samp_reg, 2);
2929       tu_cs_emit_qw(&cs, tex_samp.iova); /* SRC_ADDR_LO/HI */
2930    }
2931
2932    /* emit texture state: */
2933    tu_cs_emit_pkt7(&cs, tu6_stage2opcode(type), 3);
2934    tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
2935       CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2936       CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
2937       CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
2938       CP_LOAD_STATE6_0_NUM_UNIT(link->texture_map.num_desc));
2939    tu_cs_emit_qw(&cs, tex_const.iova); /* SRC_ADDR_LO/HI */
2940
2941    tu_cs_emit_pkt4(&cs, tex_const_reg, 2);
2942    tu_cs_emit_qw(&cs, tex_const.iova); /* SRC_ADDR_LO/HI */
2943
2944    tu_cs_emit_pkt4(&cs, tex_count_reg, 1);
2945    tu_cs_emit(&cs, link->texture_map.num_desc);
2946
2947    *entry = tu_cs_end_sub_stream(draw_state, &cs);
2948    return VK_SUCCESS;
2949 }
2950
2951 static VkResult
2952 tu6_emit_ibo(struct tu_cmd_buffer *cmd,
2953              const struct tu_pipeline *pipeline,
2954              struct tu_descriptor_state *descriptors_state,
2955              gl_shader_stage type,
2956              struct tu_cs_entry *entry)
2957 {
2958    struct tu_device *device = cmd->device;
2959    struct tu_cs *draw_state = &cmd->sub_cs;
2960    const struct tu_program_descriptor_linkage *link =
2961       &pipeline->program.link[type];
2962    VkResult result;
2963
2964    unsigned num_desc = link->ssbo_map.num_desc + link->image_map.num_desc;
2965
2966    if (num_desc == 0) {
2967       *entry = (struct tu_cs_entry) {};
2968       return VK_SUCCESS;
2969    }
2970
2971    struct ts_cs_memory ibo_const;
2972    result = tu_cs_alloc(device, draw_state, num_desc,
2973                         A6XX_TEX_CONST_DWORDS, &ibo_const);
2974    if (result != VK_SUCCESS)
2975       return result;
2976
2977    int ssbo_index = 0;
2978    for (unsigned i = 0; i < link->ssbo_map.num; i++) {
2979       for (int j = 0; j < link->ssbo_map.array_size[i]; j++) {
2980          uint32_t *dst = &ibo_const.map[A6XX_TEX_CONST_DWORDS * ssbo_index];
2981
2982          uint64_t va = buffer_ptr(descriptors_state, &link->ssbo_map, i, j);
2983          /* We don't expose robustBufferAccess, so leave the size unlimited. */
2984          uint32_t sz = MAX_STORAGE_BUFFER_RANGE / 4;
2985
2986          dst[0] = A6XX_IBO_0_FMT(FMT6_32_UINT);
2987          dst[1] = A6XX_IBO_1_WIDTH(sz & MASK(15)) |
2988                   A6XX_IBO_1_HEIGHT(sz >> 15);
2989          dst[2] = A6XX_IBO_2_UNK4 |
2990                   A6XX_IBO_2_UNK31 |
2991                   A6XX_IBO_2_TYPE(A6XX_TEX_1D);
2992          dst[3] = 0;
2993          dst[4] = va;
2994          dst[5] = va >> 32;
2995          for (int i = 6; i < A6XX_TEX_CONST_DWORDS; i++)
2996             dst[i] = 0;
2997
2998          ssbo_index++;
2999       }
3000    }
3001
3002    for (unsigned i = 0; i < link->image_map.num; i++) {
3003       for (int j = 0; j < link->image_map.array_size[i]; j++) {
3004          uint32_t *dst = &ibo_const.map[A6XX_TEX_CONST_DWORDS * ssbo_index];
3005
3006          write_image_ibo(cmd, dst,
3007                          descriptors_state, &link->image_map, i, j);
3008
3009          ssbo_index++;
3010       }
3011    }
3012
3013    assert(ssbo_index == num_desc);
3014
3015    struct tu_cs cs;
3016    result = tu_cs_begin_sub_stream(device, draw_state, 7, &cs);
3017    if (result != VK_SUCCESS)
3018       return result;
3019
3020    uint32_t opcode, ibo_addr_reg;
3021    enum a6xx_state_block sb;
3022    enum a6xx_state_type st;
3023
3024    switch (type) {
3025    case MESA_SHADER_FRAGMENT:
3026       opcode = CP_LOAD_STATE6;
3027       st = ST6_SHADER;
3028       sb = SB6_IBO;
3029       ibo_addr_reg = REG_A6XX_SP_IBO_LO;
3030       break;
3031    case MESA_SHADER_COMPUTE:
3032       opcode = CP_LOAD_STATE6_FRAG;
3033       st = ST6_IBO;
3034       sb = SB6_CS_SHADER;
3035       ibo_addr_reg = REG_A6XX_SP_CS_IBO_LO;
3036       break;
3037    default:
3038       unreachable("unsupported stage for ibos");
3039    }
3040
3041    /* emit texture state: */
3042    tu_cs_emit_pkt7(&cs, opcode, 3);
3043    tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
3044               CP_LOAD_STATE6_0_STATE_TYPE(st) |
3045               CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
3046               CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
3047               CP_LOAD_STATE6_0_NUM_UNIT(num_desc));
3048    tu_cs_emit_qw(&cs, ibo_const.iova); /* SRC_ADDR_LO/HI */
3049
3050    tu_cs_emit_pkt4(&cs, ibo_addr_reg, 2);
3051    tu_cs_emit_qw(&cs, ibo_const.iova); /* SRC_ADDR_LO/HI */
3052
3053    *entry = tu_cs_end_sub_stream(draw_state, &cs);
3054    return VK_SUCCESS;
3055 }
3056
3057 struct PACKED bcolor_entry {
3058    uint32_t fp32[4];
3059    uint16_t ui16[4];
3060    int16_t  si16[4];
3061    uint16_t fp16[4];
3062    uint16_t rgb565;
3063    uint16_t rgb5a1;
3064    uint16_t rgba4;
3065    uint8_t __pad0[2];
3066    uint8_t  ui8[4];
3067    int8_t   si8[4];
3068    uint32_t rgb10a2;
3069    uint32_t z24; /* also s8? */
3070    uint16_t srgb[4];      /* appears to duplicate fp16[], but clamped, used for srgb */
3071    uint8_t  __pad1[56];
3072 } border_color[] = {
3073    [VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK] = {},
3074    [VK_BORDER_COLOR_INT_TRANSPARENT_BLACK] = {},
3075    [VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK] = {
3076       .fp32[3] = 0x3f800000,
3077       .ui16[3] = 0xffff,
3078       .si16[3] = 0x7fff,
3079       .fp16[3] = 0x3c00,
3080       .rgb5a1 = 0x8000,
3081       .rgba4 = 0xf000,
3082       .ui8[3] = 0xff,
3083       .si8[3] = 0x7f,
3084       .rgb10a2 = 0xc0000000,
3085       .srgb[3] = 0x3c00,
3086    },
3087    [VK_BORDER_COLOR_INT_OPAQUE_BLACK] = {
3088       .fp32[3] = 1,
3089       .fp16[3] = 1,
3090    },
3091    [VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE] = {
3092       .fp32[0 ... 3] = 0x3f800000,
3093       .ui16[0 ... 3] = 0xffff,
3094       .si16[0 ... 3] = 0x7fff,
3095       .fp16[0 ... 3] = 0x3c00,
3096       .rgb565 = 0xffff,
3097       .rgb5a1 = 0xffff,
3098       .rgba4 = 0xffff,
3099       .ui8[0 ... 3] = 0xff,
3100       .si8[0 ... 3] = 0x7f,
3101       .rgb10a2 = 0xffffffff,
3102       .z24 = 0xffffff,
3103       .srgb[0 ... 3] = 0x3c00,
3104    },
3105    [VK_BORDER_COLOR_INT_OPAQUE_WHITE] = {
3106       .fp32[0 ... 3] = 1,
3107       .fp16[0 ... 3] = 1,
3108    },
3109 };
3110
3111 static VkResult
3112 tu6_emit_border_color(struct tu_cmd_buffer *cmd,
3113                       struct tu_cs *cs)
3114 {
3115    STATIC_ASSERT(sizeof(struct bcolor_entry) == 128);
3116
3117    const struct tu_pipeline *pipeline = cmd->state.pipeline;
3118    struct tu_descriptor_state *descriptors_state =
3119       &cmd->descriptors[VK_PIPELINE_BIND_POINT_GRAPHICS];
3120    const struct tu_descriptor_map *vs_sampler =
3121       &pipeline->program.link[MESA_SHADER_VERTEX].sampler_map;
3122    const struct tu_descriptor_map *fs_sampler =
3123       &pipeline->program.link[MESA_SHADER_FRAGMENT].sampler_map;
3124    struct ts_cs_memory ptr;
3125
3126    VkResult result = tu_cs_alloc(cmd->device, &cmd->sub_cs,
3127                                  vs_sampler->num_desc + fs_sampler->num_desc,
3128                                  128 / 4,
3129                                  &ptr);
3130    if (result != VK_SUCCESS)
3131       return result;
3132
3133    for (unsigned i = 0; i < vs_sampler->num; i++) {
3134       for (unsigned j = 0; j < vs_sampler->array_size[i]; j++) {
3135          const struct tu_sampler *sampler = sampler_ptr(descriptors_state,
3136                                                         vs_sampler, i, j);
3137          memcpy(ptr.map, &border_color[sampler->border], 128);
3138          ptr.map += 128 / 4;
3139       }
3140    }
3141
3142    for (unsigned i = 0; i < fs_sampler->num; i++) {
3143       for (unsigned j = 0; j < fs_sampler->array_size[i]; j++) {
3144          const struct tu_sampler *sampler = sampler_ptr(descriptors_state,
3145                                                         fs_sampler, i, j);
3146          memcpy(ptr.map, &border_color[sampler->border], 128);
3147          ptr.map += 128 / 4;
3148       }
3149    }
3150
3151    tu_cs_emit_pkt4(cs, REG_A6XX_SP_TP_BORDER_COLOR_BASE_ADDR_LO, 2);
3152    tu_cs_emit_qw(cs, ptr.iova);
3153    return VK_SUCCESS;
3154 }
3155
3156 static VkResult
3157 tu6_bind_draw_states(struct tu_cmd_buffer *cmd,
3158                      struct tu_cs *cs,
3159                      const struct tu_draw_info *draw)
3160 {
3161    const struct tu_pipeline *pipeline = cmd->state.pipeline;
3162    const struct tu_dynamic_state *dynamic = &cmd->state.dynamic;
3163    struct tu_draw_state_group draw_state_groups[TU_DRAW_STATE_COUNT];
3164    uint32_t draw_state_group_count = 0;
3165
3166    struct tu_descriptor_state *descriptors_state =
3167       &cmd->descriptors[VK_PIPELINE_BIND_POINT_GRAPHICS];
3168
3169    VkResult result = tu_cs_reserve_space(cmd->device, cs, 256);
3170    if (result != VK_SUCCESS)
3171       return result;
3172
3173    /* TODO lrz */
3174
3175    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9806, 0);
3176    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9990, 0);
3177    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_UNKNOWN_A008, 0);
3178
3179    tu_cs_emit_regs(cs,
3180                    A6XX_PC_PRIMITIVE_CNTL_0(.primitive_restart =
3181                                             pipeline->ia.primitive_restart && draw->indexed));
3182
3183    if (cmd->state.dirty &
3184           (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_DYNAMIC_LINE_WIDTH) &&
3185        (pipeline->dynamic_state.mask & TU_DYNAMIC_LINE_WIDTH)) {
3186       tu6_emit_gras_su_cntl(cs, pipeline->rast.gras_su_cntl,
3187                             dynamic->line_width);
3188    }
3189
3190    if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK) &&
3191        (pipeline->dynamic_state.mask & TU_DYNAMIC_STENCIL_COMPARE_MASK)) {
3192       tu6_emit_stencil_compare_mask(cs, dynamic->stencil_compare_mask.front,
3193                                     dynamic->stencil_compare_mask.back);
3194    }
3195
3196    if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK) &&
3197        (pipeline->dynamic_state.mask & TU_DYNAMIC_STENCIL_WRITE_MASK)) {
3198       tu6_emit_stencil_write_mask(cs, dynamic->stencil_write_mask.front,
3199                                   dynamic->stencil_write_mask.back);
3200    }
3201
3202    if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE) &&
3203        (pipeline->dynamic_state.mask & TU_DYNAMIC_STENCIL_REFERENCE)) {
3204       tu6_emit_stencil_reference(cs, dynamic->stencil_reference.front,
3205                                  dynamic->stencil_reference.back);
3206    }
3207
3208    if (cmd->state.dirty &
3209        (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_VERTEX_BUFFERS)) {
3210       for (uint32_t i = 0; i < pipeline->vi.count; i++) {
3211          const uint32_t binding = pipeline->vi.bindings[i];
3212          const uint32_t stride = pipeline->vi.strides[i];
3213          const struct tu_buffer *buf = cmd->state.vb.buffers[binding];
3214          const VkDeviceSize offset = buf->bo_offset +
3215                                      cmd->state.vb.offsets[binding] +
3216                                      pipeline->vi.offsets[i];
3217          const VkDeviceSize size =
3218             offset < buf->bo->size ? buf->bo->size - offset : 0;
3219
3220          tu_cs_emit_regs(cs,
3221                          A6XX_VFD_FETCH_BASE(i, .bo = buf->bo, .bo_offset = offset),
3222                          A6XX_VFD_FETCH_SIZE(i, size),
3223                          A6XX_VFD_FETCH_STRIDE(i, stride));
3224       }
3225    }
3226
3227    if (cmd->state.dirty & TU_CMD_DIRTY_PIPELINE) {
3228       draw_state_groups[draw_state_group_count++] =
3229          (struct tu_draw_state_group) {
3230             .id = TU_DRAW_STATE_PROGRAM,
3231             .enable_mask = ENABLE_DRAW,
3232             .ib = pipeline->program.state_ib,
3233          };
3234       draw_state_groups[draw_state_group_count++] =
3235          (struct tu_draw_state_group) {
3236             .id = TU_DRAW_STATE_PROGRAM_BINNING,
3237             .enable_mask = CP_SET_DRAW_STATE__0_BINNING,
3238             .ib = pipeline->program.binning_state_ib,
3239          };
3240       draw_state_groups[draw_state_group_count++] =
3241          (struct tu_draw_state_group) {
3242             .id = TU_DRAW_STATE_VI,
3243             .enable_mask = ENABLE_DRAW,
3244             .ib = pipeline->vi.state_ib,
3245          };
3246       draw_state_groups[draw_state_group_count++] =
3247          (struct tu_draw_state_group) {
3248             .id = TU_DRAW_STATE_VI_BINNING,
3249             .enable_mask = CP_SET_DRAW_STATE__0_BINNING,
3250             .ib = pipeline->vi.binning_state_ib,
3251          };
3252       draw_state_groups[draw_state_group_count++] =
3253          (struct tu_draw_state_group) {
3254             .id = TU_DRAW_STATE_VP,
3255             .enable_mask = ENABLE_ALL,
3256             .ib = pipeline->vp.state_ib,
3257          };
3258       draw_state_groups[draw_state_group_count++] =
3259          (struct tu_draw_state_group) {
3260             .id = TU_DRAW_STATE_RAST,
3261             .enable_mask = ENABLE_ALL,
3262             .ib = pipeline->rast.state_ib,
3263          };
3264       draw_state_groups[draw_state_group_count++] =
3265          (struct tu_draw_state_group) {
3266             .id = TU_DRAW_STATE_DS,
3267             .enable_mask = ENABLE_ALL,
3268             .ib = pipeline->ds.state_ib,
3269          };
3270       draw_state_groups[draw_state_group_count++] =
3271          (struct tu_draw_state_group) {
3272             .id = TU_DRAW_STATE_BLEND,
3273             .enable_mask = ENABLE_ALL,
3274             .ib = pipeline->blend.state_ib,
3275          };
3276    }
3277
3278    if (cmd->state.dirty &
3279          (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_DESCRIPTOR_SETS | TU_CMD_DIRTY_PUSH_CONSTANTS)) {
3280       draw_state_groups[draw_state_group_count++] =
3281          (struct tu_draw_state_group) {
3282             .id = TU_DRAW_STATE_VS_CONST,
3283             .enable_mask = ENABLE_ALL,
3284             .ib = tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_VERTEX)
3285          };
3286       draw_state_groups[draw_state_group_count++] =
3287          (struct tu_draw_state_group) {
3288             .id = TU_DRAW_STATE_FS_CONST,
3289             .enable_mask = ENABLE_DRAW,
3290             .ib = tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_FRAGMENT)
3291          };
3292    }
3293
3294    if (cmd->state.dirty &
3295          (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_DESCRIPTOR_SETS)) {
3296       bool needs_border = false;
3297       struct tu_cs_entry vs_tex, fs_tex, fs_ibo;
3298
3299       result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3300                                  MESA_SHADER_VERTEX, &vs_tex, &needs_border);
3301       if (result != VK_SUCCESS)
3302          return result;
3303
3304       result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3305                                  MESA_SHADER_FRAGMENT, &fs_tex, &needs_border);
3306       if (result != VK_SUCCESS)
3307          return result;
3308
3309       result = tu6_emit_ibo(cmd, pipeline, descriptors_state,
3310                             MESA_SHADER_FRAGMENT, &fs_ibo);
3311       if (result != VK_SUCCESS)
3312          return result;
3313
3314       draw_state_groups[draw_state_group_count++] =
3315          (struct tu_draw_state_group) {
3316             .id = TU_DRAW_STATE_VS_TEX,
3317             .enable_mask = ENABLE_ALL,
3318             .ib = vs_tex,
3319          };
3320       draw_state_groups[draw_state_group_count++] =
3321          (struct tu_draw_state_group) {
3322             .id = TU_DRAW_STATE_FS_TEX,
3323             .enable_mask = ENABLE_DRAW,
3324             .ib = fs_tex,
3325          };
3326       draw_state_groups[draw_state_group_count++] =
3327          (struct tu_draw_state_group) {
3328             .id = TU_DRAW_STATE_FS_IBO,
3329             .enable_mask = ENABLE_DRAW,
3330             .ib = fs_ibo,
3331          };
3332
3333       if (needs_border) {
3334          result = tu6_emit_border_color(cmd, cs);
3335          if (result != VK_SUCCESS)
3336             return result;
3337       }
3338    }
3339
3340    struct tu_cs_entry vs_params;
3341    result = tu6_emit_vs_params(cmd, draw, &vs_params);
3342    if (result != VK_SUCCESS)
3343       return result;
3344
3345    draw_state_groups[draw_state_group_count++] =
3346       (struct tu_draw_state_group) {
3347          .id = TU_DRAW_STATE_VS_PARAMS,
3348          .enable_mask = ENABLE_ALL,
3349          .ib = vs_params,
3350       };
3351
3352    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3 * draw_state_group_count);
3353    for (uint32_t i = 0; i < draw_state_group_count; i++) {
3354       const struct tu_draw_state_group *group = &draw_state_groups[i];
3355       debug_assert((group->enable_mask & ~ENABLE_ALL) == 0);
3356       uint32_t cp_set_draw_state =
3357          CP_SET_DRAW_STATE__0_COUNT(group->ib.size / 4) |
3358          group->enable_mask |
3359          CP_SET_DRAW_STATE__0_GROUP_ID(group->id);
3360       uint64_t iova;
3361       if (group->ib.size) {
3362          iova = group->ib.bo->iova + group->ib.offset;
3363       } else {
3364          cp_set_draw_state |= CP_SET_DRAW_STATE__0_DISABLE;
3365          iova = 0;
3366       }
3367
3368       tu_cs_emit(cs, cp_set_draw_state);
3369       tu_cs_emit_qw(cs, iova);
3370    }
3371
3372    tu_cs_sanity_check(cs);
3373
3374    /* track BOs */
3375    if (cmd->state.dirty & TU_CMD_DIRTY_VERTEX_BUFFERS) {
3376       for (uint32_t i = 0; i < MAX_VBS; i++) {
3377          const struct tu_buffer *buf = cmd->state.vb.buffers[i];
3378          if (buf)
3379             tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
3380       }
3381    }
3382    if (cmd->state.dirty & TU_CMD_DIRTY_DESCRIPTOR_SETS) {
3383       unsigned i;
3384       for_each_bit(i, descriptors_state->valid) {
3385          struct tu_descriptor_set *set = descriptors_state->sets[i];
3386          for (unsigned j = 0; j < set->layout->buffer_count; ++j)
3387             if (set->descriptors[j]) {
3388                tu_bo_list_add(&cmd->bo_list, set->descriptors[j],
3389                               MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3390             }
3391       }
3392    }
3393
3394    /* Fragment shader state overwrites compute shader state, so flag the
3395     * compute pipeline for re-emit.
3396     */
3397    cmd->state.dirty = TU_CMD_DIRTY_COMPUTE_PIPELINE;
3398    return VK_SUCCESS;
3399 }
3400
3401 static void
3402 tu6_emit_draw_direct(struct tu_cmd_buffer *cmd,
3403                      struct tu_cs *cs,
3404                      const struct tu_draw_info *draw)
3405 {
3406
3407    const enum pc_di_primtype primtype = cmd->state.pipeline->ia.primtype;
3408
3409    tu_cs_emit_regs(cs,
3410                    A6XX_VFD_INDEX_OFFSET(draw->vertex_offset),
3411                    A6XX_VFD_INSTANCE_START_OFFSET(draw->first_instance));
3412
3413    /* TODO hw binning */
3414    if (draw->indexed) {
3415       const enum a4xx_index_size index_size =
3416          tu6_index_size(cmd->state.index_type);
3417       const uint32_t index_bytes =
3418          (cmd->state.index_type == VK_INDEX_TYPE_UINT32) ? 4 : 2;
3419       const struct tu_buffer *buf = cmd->state.index_buffer;
3420       const VkDeviceSize offset = buf->bo_offset + cmd->state.index_offset +
3421                                   index_bytes * draw->first_index;
3422       const uint32_t size = index_bytes * draw->count;
3423
3424       const uint32_t cp_draw_indx =
3425          CP_DRAW_INDX_OFFSET_0_PRIM_TYPE(primtype) |
3426          CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT(DI_SRC_SEL_DMA) |
3427          CP_DRAW_INDX_OFFSET_0_INDEX_SIZE(index_size) |
3428          CP_DRAW_INDX_OFFSET_0_VIS_CULL(USE_VISIBILITY) | 0x2000;
3429
3430       tu_cs_emit_pkt7(cs, CP_DRAW_INDX_OFFSET, 7);
3431       tu_cs_emit(cs, cp_draw_indx);
3432       tu_cs_emit(cs, draw->instance_count);
3433       tu_cs_emit(cs, draw->count);
3434       tu_cs_emit(cs, 0x0); /* XXX */
3435       tu_cs_emit_qw(cs, buf->bo->iova + offset);
3436       tu_cs_emit(cs, size);
3437    } else {
3438       const uint32_t cp_draw_indx =
3439          CP_DRAW_INDX_OFFSET_0_PRIM_TYPE(primtype) |
3440          CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT(DI_SRC_SEL_AUTO_INDEX) |
3441          CP_DRAW_INDX_OFFSET_0_VIS_CULL(USE_VISIBILITY) | 0x2000;
3442
3443       tu_cs_emit_pkt7(cs, CP_DRAW_INDX_OFFSET, 3);
3444       tu_cs_emit(cs, cp_draw_indx);
3445       tu_cs_emit(cs, draw->instance_count);
3446       tu_cs_emit(cs, draw->count);
3447    }
3448 }
3449
3450 static void
3451 tu_draw(struct tu_cmd_buffer *cmd, const struct tu_draw_info *draw)
3452 {
3453    struct tu_cs *cs = &cmd->draw_cs;
3454    VkResult result;
3455
3456    result = tu6_bind_draw_states(cmd, cs, draw);
3457    if (result != VK_SUCCESS) {
3458       cmd->record_result = result;
3459       return;
3460    }
3461
3462    result = tu_cs_reserve_space(cmd->device, cs, 32);
3463    if (result != VK_SUCCESS) {
3464       cmd->record_result = result;
3465       return;
3466    }
3467
3468    if (draw->indirect) {
3469       tu_finishme("indirect draw");
3470       return;
3471    }
3472
3473    /* TODO tu6_emit_marker should pick different regs depending on cs */
3474
3475    tu6_emit_marker(cmd, cs);
3476    tu6_emit_draw_direct(cmd, cs, draw);
3477    tu6_emit_marker(cmd, cs);
3478
3479    cmd->wait_for_idle = true;
3480
3481    tu_cs_sanity_check(cs);
3482 }
3483
3484 void
3485 tu_CmdDraw(VkCommandBuffer commandBuffer,
3486            uint32_t vertexCount,
3487            uint32_t instanceCount,
3488            uint32_t firstVertex,
3489            uint32_t firstInstance)
3490 {
3491    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3492    struct tu_draw_info info = {};
3493
3494    info.count = vertexCount;
3495    info.instance_count = instanceCount;
3496    info.first_instance = firstInstance;
3497    info.vertex_offset = firstVertex;
3498
3499    tu_draw(cmd_buffer, &info);
3500 }
3501
3502 void
3503 tu_CmdDrawIndexed(VkCommandBuffer commandBuffer,
3504                   uint32_t indexCount,
3505                   uint32_t instanceCount,
3506                   uint32_t firstIndex,
3507                   int32_t vertexOffset,
3508                   uint32_t firstInstance)
3509 {
3510    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3511    struct tu_draw_info info = {};
3512
3513    info.indexed = true;
3514    info.count = indexCount;
3515    info.instance_count = instanceCount;
3516    info.first_index = firstIndex;
3517    info.vertex_offset = vertexOffset;
3518    info.first_instance = firstInstance;
3519
3520    tu_draw(cmd_buffer, &info);
3521 }
3522
3523 void
3524 tu_CmdDrawIndirect(VkCommandBuffer commandBuffer,
3525                    VkBuffer _buffer,
3526                    VkDeviceSize offset,
3527                    uint32_t drawCount,
3528                    uint32_t stride)
3529 {
3530    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3531    TU_FROM_HANDLE(tu_buffer, buffer, _buffer);
3532    struct tu_draw_info info = {};
3533
3534    info.count = drawCount;
3535    info.indirect = buffer;
3536    info.indirect_offset = offset;
3537    info.stride = stride;
3538
3539    tu_draw(cmd_buffer, &info);
3540 }
3541
3542 void
3543 tu_CmdDrawIndexedIndirect(VkCommandBuffer commandBuffer,
3544                           VkBuffer _buffer,
3545                           VkDeviceSize offset,
3546                           uint32_t drawCount,
3547                           uint32_t stride)
3548 {
3549    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3550    TU_FROM_HANDLE(tu_buffer, buffer, _buffer);
3551    struct tu_draw_info info = {};
3552
3553    info.indexed = true;
3554    info.count = drawCount;
3555    info.indirect = buffer;
3556    info.indirect_offset = offset;
3557    info.stride = stride;
3558
3559    tu_draw(cmd_buffer, &info);
3560 }
3561
3562 struct tu_dispatch_info
3563 {
3564    /**
3565     * Determine the layout of the grid (in block units) to be used.
3566     */
3567    uint32_t blocks[3];
3568
3569    /**
3570     * A starting offset for the grid. If unaligned is set, the offset
3571     * must still be aligned.
3572     */
3573    uint32_t offsets[3];
3574    /**
3575     * Whether it's an unaligned compute dispatch.
3576     */
3577    bool unaligned;
3578
3579    /**
3580     * Indirect compute parameters resource.
3581     */
3582    struct tu_buffer *indirect;
3583    uint64_t indirect_offset;
3584 };
3585
3586 static void
3587 tu_emit_compute_driver_params(struct tu_cs *cs, struct tu_pipeline *pipeline,
3588                               const struct tu_dispatch_info *info)
3589 {
3590    gl_shader_stage type = MESA_SHADER_COMPUTE;
3591    const struct tu_program_descriptor_linkage *link =
3592       &pipeline->program.link[type];
3593    const struct ir3_const_state *const_state = &link->const_state;
3594    uint32_t offset = const_state->offsets.driver_param;
3595
3596    if (link->constlen <= offset)
3597       return;
3598
3599    if (!info->indirect) {
3600       uint32_t driver_params[IR3_DP_CS_COUNT] = {
3601          [IR3_DP_NUM_WORK_GROUPS_X] = info->blocks[0],
3602          [IR3_DP_NUM_WORK_GROUPS_Y] = info->blocks[1],
3603          [IR3_DP_NUM_WORK_GROUPS_Z] = info->blocks[2],
3604          [IR3_DP_LOCAL_GROUP_SIZE_X] = pipeline->compute.local_size[0],
3605          [IR3_DP_LOCAL_GROUP_SIZE_Y] = pipeline->compute.local_size[1],
3606          [IR3_DP_LOCAL_GROUP_SIZE_Z] = pipeline->compute.local_size[2],
3607       };
3608
3609       uint32_t num_consts = MIN2(const_state->num_driver_params,
3610                                  (link->constlen - offset) * 4);
3611       /* push constants */
3612       tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + num_consts);
3613       tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(offset) |
3614                  CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
3615                  CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
3616                  CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
3617                  CP_LOAD_STATE6_0_NUM_UNIT(num_consts / 4));
3618       tu_cs_emit(cs, 0);
3619       tu_cs_emit(cs, 0);
3620       uint32_t i;
3621       for (i = 0; i < num_consts; i++)
3622          tu_cs_emit(cs, driver_params[i]);
3623    } else {
3624       tu_finishme("Indirect driver params");
3625    }
3626 }
3627
3628 static void
3629 tu_dispatch(struct tu_cmd_buffer *cmd,
3630             const struct tu_dispatch_info *info)
3631 {
3632    struct tu_cs *cs = &cmd->cs;
3633    struct tu_pipeline *pipeline = cmd->state.compute_pipeline;
3634    struct tu_descriptor_state *descriptors_state =
3635       &cmd->descriptors[VK_PIPELINE_BIND_POINT_COMPUTE];
3636
3637    VkResult result = tu_cs_reserve_space(cmd->device, cs, 256);
3638    if (result != VK_SUCCESS) {
3639       cmd->record_result = result;
3640       return;
3641    }
3642
3643    if (cmd->state.dirty & TU_CMD_DIRTY_COMPUTE_PIPELINE)
3644       tu_cs_emit_ib(cs, &pipeline->program.state_ib);
3645
3646    struct tu_cs_entry ib;
3647
3648    ib = tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_COMPUTE);
3649    if (ib.size)
3650       tu_cs_emit_ib(cs, &ib);
3651
3652    tu_emit_compute_driver_params(cs, pipeline, info);
3653
3654    bool needs_border;
3655    result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3656                               MESA_SHADER_COMPUTE, &ib, &needs_border);
3657    if (result != VK_SUCCESS) {
3658       cmd->record_result = result;
3659       return;
3660    }
3661
3662    if (ib.size)
3663       tu_cs_emit_ib(cs, &ib);
3664
3665    if (needs_border)
3666       tu_finishme("compute border color");
3667
3668    result = tu6_emit_ibo(cmd, pipeline, descriptors_state, MESA_SHADER_COMPUTE, &ib);
3669    if (result != VK_SUCCESS) {
3670       cmd->record_result = result;
3671       return;
3672    }
3673
3674    if (ib.size)
3675       tu_cs_emit_ib(cs, &ib);
3676
3677    /* track BOs */
3678    if (cmd->state.dirty & TU_CMD_DIRTY_DESCRIPTOR_SETS) {
3679       unsigned i;
3680       for_each_bit(i, descriptors_state->valid) {
3681          struct tu_descriptor_set *set = descriptors_state->sets[i];
3682          for (unsigned j = 0; j < set->layout->buffer_count; ++j)
3683             if (set->descriptors[j]) {
3684                tu_bo_list_add(&cmd->bo_list, set->descriptors[j],
3685                               MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3686             }
3687       }
3688    }
3689
3690    /* Compute shader state overwrites fragment shader state, so we flag the
3691     * graphics pipeline for re-emit.
3692     */
3693    cmd->state.dirty = TU_CMD_DIRTY_PIPELINE;
3694
3695    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
3696    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(0x8));
3697
3698    const uint32_t *local_size = pipeline->compute.local_size;
3699    const uint32_t *num_groups = info->blocks;
3700    tu_cs_emit_regs(cs,
3701                    A6XX_HLSQ_CS_NDRANGE_0(.kerneldim = 3,
3702                                           .localsizex = local_size[0] - 1,
3703                                           .localsizey = local_size[1] - 1,
3704                                           .localsizez = local_size[2] - 1),
3705                    A6XX_HLSQ_CS_NDRANGE_1(.globalsize_x = local_size[0] * num_groups[0]),
3706                    A6XX_HLSQ_CS_NDRANGE_2(.globaloff_x = 0),
3707                    A6XX_HLSQ_CS_NDRANGE_3(.globalsize_y = local_size[1] * num_groups[1]),
3708                    A6XX_HLSQ_CS_NDRANGE_4(.globaloff_y = 0),
3709                    A6XX_HLSQ_CS_NDRANGE_5(.globalsize_z = local_size[2] * num_groups[2]),
3710                    A6XX_HLSQ_CS_NDRANGE_6(.globaloff_z = 0));
3711
3712    tu_cs_emit_regs(cs,
3713                    A6XX_HLSQ_CS_KERNEL_GROUP_X(1),
3714                    A6XX_HLSQ_CS_KERNEL_GROUP_Y(1),
3715                    A6XX_HLSQ_CS_KERNEL_GROUP_Z(1));
3716
3717    if (info->indirect) {
3718       uint64_t iova = tu_buffer_iova(info->indirect) + info->indirect_offset;
3719
3720       tu_bo_list_add(&cmd->bo_list, info->indirect->bo,
3721                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3722
3723       tu_cs_emit_pkt7(cs, CP_EXEC_CS_INDIRECT, 4);
3724       tu_cs_emit(cs, 0x00000000);
3725       tu_cs_emit_qw(cs, iova);
3726       tu_cs_emit(cs,
3727                  A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX(local_size[0] - 1) |
3728                  A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY(local_size[1] - 1) |
3729                  A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ(local_size[2] - 1));
3730    } else {
3731       tu_cs_emit_pkt7(cs, CP_EXEC_CS, 4);
3732       tu_cs_emit(cs, 0x00000000);
3733       tu_cs_emit(cs, CP_EXEC_CS_1_NGROUPS_X(info->blocks[0]));
3734       tu_cs_emit(cs, CP_EXEC_CS_2_NGROUPS_Y(info->blocks[1]));
3735       tu_cs_emit(cs, CP_EXEC_CS_3_NGROUPS_Z(info->blocks[2]));
3736    }
3737
3738    tu_cs_emit_wfi(cs);
3739
3740    tu6_emit_cache_flush(cmd, cs);
3741 }
3742
3743 void
3744 tu_CmdDispatchBase(VkCommandBuffer commandBuffer,
3745                    uint32_t base_x,
3746                    uint32_t base_y,
3747                    uint32_t base_z,
3748                    uint32_t x,
3749                    uint32_t y,
3750                    uint32_t z)
3751 {
3752    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3753    struct tu_dispatch_info info = {};
3754
3755    info.blocks[0] = x;
3756    info.blocks[1] = y;
3757    info.blocks[2] = z;
3758
3759    info.offsets[0] = base_x;
3760    info.offsets[1] = base_y;
3761    info.offsets[2] = base_z;
3762    tu_dispatch(cmd_buffer, &info);
3763 }
3764
3765 void
3766 tu_CmdDispatch(VkCommandBuffer commandBuffer,
3767                uint32_t x,
3768                uint32_t y,
3769                uint32_t z)
3770 {
3771    tu_CmdDispatchBase(commandBuffer, 0, 0, 0, x, y, z);
3772 }
3773
3774 void
3775 tu_CmdDispatchIndirect(VkCommandBuffer commandBuffer,
3776                        VkBuffer _buffer,
3777                        VkDeviceSize offset)
3778 {
3779    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3780    TU_FROM_HANDLE(tu_buffer, buffer, _buffer);
3781    struct tu_dispatch_info info = {};
3782
3783    info.indirect = buffer;
3784    info.indirect_offset = offset;
3785
3786    tu_dispatch(cmd_buffer, &info);
3787 }
3788
3789 void
3790 tu_CmdEndRenderPass(VkCommandBuffer commandBuffer)
3791 {
3792    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3793
3794    tu_cs_end(&cmd_buffer->draw_cs);
3795    tu_cs_end(&cmd_buffer->draw_epilogue_cs);
3796
3797    tu_cmd_render_tiles(cmd_buffer);
3798
3799    /* discard draw_cs and draw_epilogue_cs entries now that the tiles are
3800       rendered */
3801    tu_cs_discard_entries(&cmd_buffer->draw_cs);
3802    tu_cs_begin(&cmd_buffer->draw_cs);
3803    tu_cs_discard_entries(&cmd_buffer->draw_epilogue_cs);
3804    tu_cs_begin(&cmd_buffer->draw_epilogue_cs);
3805
3806    cmd_buffer->state.pass = NULL;
3807    cmd_buffer->state.subpass = NULL;
3808    cmd_buffer->state.framebuffer = NULL;
3809 }
3810
3811 void
3812 tu_CmdEndRenderPass2(VkCommandBuffer commandBuffer,
3813                      const VkSubpassEndInfoKHR *pSubpassEndInfo)
3814 {
3815    tu_CmdEndRenderPass(commandBuffer);
3816 }
3817
3818 struct tu_barrier_info
3819 {
3820    uint32_t eventCount;
3821    const VkEvent *pEvents;
3822    VkPipelineStageFlags srcStageMask;
3823 };
3824
3825 static void
3826 tu_barrier(struct tu_cmd_buffer *cmd_buffer,
3827            uint32_t memoryBarrierCount,
3828            const VkMemoryBarrier *pMemoryBarriers,
3829            uint32_t bufferMemoryBarrierCount,
3830            const VkBufferMemoryBarrier *pBufferMemoryBarriers,
3831            uint32_t imageMemoryBarrierCount,
3832            const VkImageMemoryBarrier *pImageMemoryBarriers,
3833            const struct tu_barrier_info *info)
3834 {
3835 }
3836
3837 void
3838 tu_CmdPipelineBarrier(VkCommandBuffer commandBuffer,
3839                       VkPipelineStageFlags srcStageMask,
3840                       VkPipelineStageFlags destStageMask,
3841                       VkBool32 byRegion,
3842                       uint32_t memoryBarrierCount,
3843                       const VkMemoryBarrier *pMemoryBarriers,
3844                       uint32_t bufferMemoryBarrierCount,
3845                       const VkBufferMemoryBarrier *pBufferMemoryBarriers,
3846                       uint32_t imageMemoryBarrierCount,
3847                       const VkImageMemoryBarrier *pImageMemoryBarriers)
3848 {
3849    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3850    struct tu_barrier_info info;
3851
3852    info.eventCount = 0;
3853    info.pEvents = NULL;
3854    info.srcStageMask = srcStageMask;
3855
3856    tu_barrier(cmd_buffer, memoryBarrierCount, pMemoryBarriers,
3857               bufferMemoryBarrierCount, pBufferMemoryBarriers,
3858               imageMemoryBarrierCount, pImageMemoryBarriers, &info);
3859 }
3860
3861 static void
3862 write_event(struct tu_cmd_buffer *cmd, struct tu_event *event, unsigned value)
3863 {
3864    struct tu_cs *cs = &cmd->cs;
3865
3866    VkResult result = tu_cs_reserve_space(cmd->device, cs, 4);
3867    if (result != VK_SUCCESS) {
3868       cmd->record_result = result;
3869       return;
3870    }
3871
3872    tu_bo_list_add(&cmd->bo_list, &event->bo, MSM_SUBMIT_BO_WRITE);
3873
3874    /* TODO: any flush required before/after ? */
3875
3876    tu_cs_emit_pkt7(cs, CP_MEM_WRITE, 3);
3877    tu_cs_emit_qw(cs, event->bo.iova); /* ADDR_LO/HI */
3878    tu_cs_emit(cs, value);
3879 }
3880
3881 void
3882 tu_CmdSetEvent(VkCommandBuffer commandBuffer,
3883                VkEvent _event,
3884                VkPipelineStageFlags stageMask)
3885 {
3886    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
3887    TU_FROM_HANDLE(tu_event, event, _event);
3888
3889    write_event(cmd, event, 1);
3890 }
3891
3892 void
3893 tu_CmdResetEvent(VkCommandBuffer commandBuffer,
3894                  VkEvent _event,
3895                  VkPipelineStageFlags stageMask)
3896 {
3897    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
3898    TU_FROM_HANDLE(tu_event, event, _event);
3899
3900    write_event(cmd, event, 0);
3901 }
3902
3903 void
3904 tu_CmdWaitEvents(VkCommandBuffer commandBuffer,
3905                  uint32_t eventCount,
3906                  const VkEvent *pEvents,
3907                  VkPipelineStageFlags srcStageMask,
3908                  VkPipelineStageFlags dstStageMask,
3909                  uint32_t memoryBarrierCount,
3910                  const VkMemoryBarrier *pMemoryBarriers,
3911                  uint32_t bufferMemoryBarrierCount,
3912                  const VkBufferMemoryBarrier *pBufferMemoryBarriers,
3913                  uint32_t imageMemoryBarrierCount,
3914                  const VkImageMemoryBarrier *pImageMemoryBarriers)
3915 {
3916    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
3917    struct tu_cs *cs = &cmd->cs;
3918
3919    VkResult result = tu_cs_reserve_space(cmd->device, cs, eventCount * 7);
3920    if (result != VK_SUCCESS) {
3921       cmd->record_result = result;
3922       return;
3923    }
3924
3925    /* TODO: any flush required before/after? (CP_WAIT_FOR_ME?) */
3926
3927    for (uint32_t i = 0; i < eventCount; i++) {
3928       const struct tu_event *event = (const struct tu_event*) pEvents[i];
3929
3930       tu_bo_list_add(&cmd->bo_list, &event->bo, MSM_SUBMIT_BO_READ);
3931
3932       tu_cs_emit_pkt7(cs, CP_WAIT_REG_MEM, 6);
3933       tu_cs_emit(cs, CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ) |
3934                      CP_WAIT_REG_MEM_0_POLL_MEMORY);
3935       tu_cs_emit_qw(cs, event->bo.iova); /* POLL_ADDR_LO/HI */
3936       tu_cs_emit(cs, CP_WAIT_REG_MEM_3_REF(1));
3937       tu_cs_emit(cs, CP_WAIT_REG_MEM_4_MASK(~0u));
3938       tu_cs_emit(cs, CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(20));
3939    }
3940 }
3941
3942 void
3943 tu_CmdSetDeviceMask(VkCommandBuffer commandBuffer, uint32_t deviceMask)
3944 {
3945    /* No-op */
3946 }