src/freedreno/vulkan/tu_cmd_buffer.c

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * based in part on anv driver which is:
   6  * Copyright © 2015 Intel Corporation
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25  * DEALINGS IN THE SOFTWARE.
  26  */
  27
  28 #include "tu_private.h"
  29
  30 #include "registers/adreno_pm4.xml.h"
  31 #include "registers/adreno_common.xml.h"
  32
  33 #include "vk_format.h"
  34
  35 #include "tu_cs.h"
  36 #include "tu_blit.h"
  37
  38 #define OVERFLOW_FLAG_REG REG_A6XX_CP_SCRATCH_REG(0)
  39
  40 void
  41 tu_bo_list_init(struct tu_bo_list *list)
  42 {
  43    list->count = list->capacity = 0;
  44    list->bo_infos = NULL;
  45 }
  46
  47 void
  48 tu_bo_list_destroy(struct tu_bo_list *list)
  49 {
  50    free(list->bo_infos);
  51 }
  52
  53 void
  54 tu_bo_list_reset(struct tu_bo_list *list)
  55 {
  56    list->count = 0;
  57 }
  58
  59 /**
  60  * \a flags consists of MSM_SUBMIT_BO_FLAGS.
  61  */
  62 static uint32_t
  63 tu_bo_list_add_info(struct tu_bo_list *list,
  64                     const struct drm_msm_gem_submit_bo *bo_info)
  65 {
  66    assert(bo_info->handle != 0);
  67
  68    for (uint32_t i = 0; i < list->count; ++i) {
  69       if (list->bo_infos[i].handle == bo_info->handle) {
  70          assert(list->bo_infos[i].presumed == bo_info->presumed);
  71          list->bo_infos[i].flags |= bo_info->flags;
  72          return i;
  73       }
  74    }
  75
  76    /* grow list->bo_infos if needed */
  77    if (list->count == list->capacity) {
  78       uint32_t new_capacity = MAX2(2 * list->count, 16);
  79       struct drm_msm_gem_submit_bo *new_bo_infos = realloc(
  80          list->bo_infos, new_capacity * sizeof(struct drm_msm_gem_submit_bo));
  81       if (!new_bo_infos)
  82          return TU_BO_LIST_FAILED;
  83       list->bo_infos = new_bo_infos;
  84       list->capacity = new_capacity;
  85    }
  86
  87    list->bo_infos[list->count] = *bo_info;
  88    return list->count++;
  89 }
  90
  91 uint32_t
  92 tu_bo_list_add(struct tu_bo_list *list,
  93                const struct tu_bo *bo,
  94                uint32_t flags)
  95 {
  96    return tu_bo_list_add_info(list, &(struct drm_msm_gem_submit_bo) {
  97                                        .flags = flags,
  98                                        .handle = bo->gem_handle,
  99                                        .presumed = bo->iova,
 100                                     });
 101 }
 102
 103 VkResult
 104 tu_bo_list_merge(struct tu_bo_list *list, const struct tu_bo_list *other)
 105 {
 106    for (uint32_t i = 0; i < other->count; i++) {
 107       if (tu_bo_list_add_info(list, other->bo_infos + i) == TU_BO_LIST_FAILED)
 108          return VK_ERROR_OUT_OF_HOST_MEMORY;
 109    }
 110
 111    return VK_SUCCESS;
 112 }
 113
 114 static void
 115 tu_tiling_config_update_tile_layout(struct tu_tiling_config *tiling,
 116                                     const struct tu_device *dev,
 117                                     uint32_t pixels)
 118 {
 119    const uint32_t tile_align_w = dev->physical_device->tile_align_w;
 120    const uint32_t tile_align_h = dev->physical_device->tile_align_h;
 121    const uint32_t max_tile_width = 1024; /* A6xx */
 122
 123    tiling->tile0.offset = (VkOffset2D) {
 124       .x = tiling->render_area.offset.x & ~(tile_align_w - 1),
 125       .y = tiling->render_area.offset.y & ~(tile_align_h - 1),
 126    };
 127
 128    const uint32_t ra_width =
 129       tiling->render_area.extent.width +
 130       (tiling->render_area.offset.x - tiling->tile0.offset.x);
 131    const uint32_t ra_height =
 132       tiling->render_area.extent.height +
 133       (tiling->render_area.offset.y - tiling->tile0.offset.y);
 134
 135    /* start from 1 tile */
 136    tiling->tile_count = (VkExtent2D) {
 137       .width = 1,
 138       .height = 1,
 139    };
 140    tiling->tile0.extent = (VkExtent2D) {
 141       .width = align(ra_width, tile_align_w),
 142       .height = align(ra_height, tile_align_h),
 143    };
 144
 145    /* do not exceed max tile width */
 146    while (tiling->tile0.extent.width > max_tile_width) {
 147       tiling->tile_count.width++;
 148       tiling->tile0.extent.width =
 149          align(ra_width / tiling->tile_count.width, tile_align_w);
 150    }
 151
 152    /* do not exceed gmem size */
 153    while (tiling->tile0.extent.width * tiling->tile0.extent.height > pixels) {
 154       if (tiling->tile0.extent.width > MAX2(tile_align_w, tiling->tile0.extent.height)) {
 155          tiling->tile_count.width++;
 156          tiling->tile0.extent.width =
 157             align(DIV_ROUND_UP(ra_width, tiling->tile_count.width), tile_align_w);
 158       } else {
 159          /* if this assert fails then layout is impossible.. */
 160          assert(tiling->tile0.extent.height > tile_align_h);
 161          tiling->tile_count.height++;
 162          tiling->tile0.extent.height =
 163             align(DIV_ROUND_UP(ra_height, tiling->tile_count.height), tile_align_h);
 164       }
 165    }
 166 }
 167
 168 static void
 169 tu_tiling_config_update_pipe_layout(struct tu_tiling_config *tiling,
 170                                     const struct tu_device *dev)
 171 {
 172    const uint32_t max_pipe_count = 32; /* A6xx */
 173
 174    /* start from 1 tile per pipe */
 175    tiling->pipe0 = (VkExtent2D) {
 176       .width = 1,
 177       .height = 1,
 178    };
 179    tiling->pipe_count = tiling->tile_count;
 180
 181    /* do not exceed max pipe count vertically */
 182    while (tiling->pipe_count.height > max_pipe_count) {
 183       tiling->pipe0.height += 2;
 184       tiling->pipe_count.height =
 185          (tiling->tile_count.height + tiling->pipe0.height - 1) /
 186          tiling->pipe0.height;
 187    }
 188
 189    /* do not exceed max pipe count */
 190    while (tiling->pipe_count.width * tiling->pipe_count.height >
 191           max_pipe_count) {
 192       tiling->pipe0.width += 1;
 193       tiling->pipe_count.width =
 194          (tiling->tile_count.width + tiling->pipe0.width - 1) /
 195          tiling->pipe0.width;
 196    }
 197 }
 198
 199 static void
 200 tu_tiling_config_update_pipes(struct tu_tiling_config *tiling,
 201                               const struct tu_device *dev)
 202 {
 203    const uint32_t max_pipe_count = 32; /* A6xx */
 204    const uint32_t used_pipe_count =
 205       tiling->pipe_count.width * tiling->pipe_count.height;
 206    const VkExtent2D last_pipe = {
 207       .width = (tiling->tile_count.width - 1) % tiling->pipe0.width + 1,
 208       .height = (tiling->tile_count.height - 1) % tiling->pipe0.height + 1,
 209    };
 210
 211    assert(used_pipe_count <= max_pipe_count);
 212    assert(max_pipe_count <= ARRAY_SIZE(tiling->pipe_config));
 213
 214    for (uint32_t y = 0; y < tiling->pipe_count.height; y++) {
 215       for (uint32_t x = 0; x < tiling->pipe_count.width; x++) {
 216          const uint32_t pipe_x = tiling->pipe0.width * x;
 217          const uint32_t pipe_y = tiling->pipe0.height * y;
 218          const uint32_t pipe_w = (x == tiling->pipe_count.width - 1)
 219                                     ? last_pipe.width
 220                                     : tiling->pipe0.width;
 221          const uint32_t pipe_h = (y == tiling->pipe_count.height - 1)
 222                                     ? last_pipe.height
 223                                     : tiling->pipe0.height;
 224          const uint32_t n = tiling->pipe_count.width * y + x;
 225
 226          tiling->pipe_config[n] = A6XX_VSC_PIPE_CONFIG_REG_X(pipe_x) |
 227                                   A6XX_VSC_PIPE_CONFIG_REG_Y(pipe_y) |
 228                                   A6XX_VSC_PIPE_CONFIG_REG_W(pipe_w) |
 229                                   A6XX_VSC_PIPE_CONFIG_REG_H(pipe_h);
 230          tiling->pipe_sizes[n] = CP_SET_BIN_DATA5_0_VSC_SIZE(pipe_w * pipe_h);
 231       }
 232    }
 233
 234    memset(tiling->pipe_config + used_pipe_count, 0,
 235           sizeof(uint32_t) * (max_pipe_count - used_pipe_count));
 236 }
 237
 238 static void
 239 tu_tiling_config_get_tile(const struct tu_tiling_config *tiling,
 240                           const struct tu_device *dev,
 241                           uint32_t tx,
 242                           uint32_t ty,
 243                           struct tu_tile *tile)
 244 {
 245    /* find the pipe and the slot for tile (tx, ty) */
 246    const uint32_t px = tx / tiling->pipe0.width;
 247    const uint32_t py = ty / tiling->pipe0.height;
 248    const uint32_t sx = tx - tiling->pipe0.width * px;
 249    const uint32_t sy = ty - tiling->pipe0.height * py;
 250
 251    assert(tx < tiling->tile_count.width && ty < tiling->tile_count.height);
 252    assert(px < tiling->pipe_count.width && py < tiling->pipe_count.height);
 253    assert(sx < tiling->pipe0.width && sy < tiling->pipe0.height);
 254
 255    /* convert to 1D indices */
 256    tile->pipe = tiling->pipe_count.width * py + px;
 257    tile->slot = tiling->pipe0.width * sy + sx;
 258
 259    /* get the blit area for the tile */
 260    tile->begin = (VkOffset2D) {
 261       .x = tiling->tile0.offset.x + tiling->tile0.extent.width * tx,
 262       .y = tiling->tile0.offset.y + tiling->tile0.extent.height * ty,
 263    };
 264    tile->end.x =
 265       (tx == tiling->tile_count.width - 1)
 266          ? tiling->render_area.offset.x + tiling->render_area.extent.width
 267          : tile->begin.x + tiling->tile0.extent.width;
 268    tile->end.y =
 269       (ty == tiling->tile_count.height - 1)
 270          ? tiling->render_area.offset.y + tiling->render_area.extent.height
 271          : tile->begin.y + tiling->tile0.extent.height;
 272 }
 273
 274 enum a3xx_msaa_samples
 275 tu_msaa_samples(uint32_t samples)
 276 {
 277    switch (samples) {
 278    case 1:
 279       return MSAA_ONE;
 280    case 2:
 281       return MSAA_TWO;
 282    case 4:
 283       return MSAA_FOUR;
 284    case 8:
 285       return MSAA_EIGHT;
 286    default:
 287       assert(!"invalid sample count");
 288       return MSAA_ONE;
 289    }
 290 }
 291
 292 static enum a4xx_index_size
 293 tu6_index_size(VkIndexType type)
 294 {
 295    switch (type) {
 296    case VK_INDEX_TYPE_UINT16:
 297       return INDEX4_SIZE_16_BIT;
 298    case VK_INDEX_TYPE_UINT32:
 299       return INDEX4_SIZE_32_BIT;
 300    default:
 301       unreachable("invalid VkIndexType");
 302       return INDEX4_SIZE_8_BIT;
 303    }
 304 }
 305
 306 static void
 307 tu6_emit_marker(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 308 {
 309    tu_cs_emit_write_reg(cs, cmd->marker_reg, ++cmd->marker_seqno);
 310 }
 311
 312 unsigned
 313 tu6_emit_event_write(struct tu_cmd_buffer *cmd,
 314                      struct tu_cs *cs,
 315                      enum vgt_event_type event,
 316                      bool need_seqno)
 317 {
 318    unsigned seqno = 0;
 319
 320    tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, need_seqno ? 4 : 1);
 321    tu_cs_emit(cs, CP_EVENT_WRITE_0_EVENT(event));
 322    if (need_seqno) {
 323       tu_cs_emit_qw(cs, cmd->scratch_bo.iova);
 324       seqno = ++cmd->scratch_seqno;
 325       tu_cs_emit(cs, seqno);
 326    }
 327
 328    return seqno;
 329 }
 330
 331 static void
 332 tu6_emit_cache_flush(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 333 {
 334    tu6_emit_event_write(cmd, cs, 0x31, false);
 335 }
 336
 337 static void
 338 tu6_emit_lrz_flush(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 339 {
 340    tu6_emit_event_write(cmd, cs, LRZ_FLUSH, false);
 341 }
 342
 343 static void
 344 tu6_emit_wfi(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 345 {
 346    if (cmd->wait_for_idle) {
 347       tu_cs_emit_wfi(cs);
 348       cmd->wait_for_idle = false;
 349    }
 350 }
 351
 352 #define tu_image_view_ubwc_pitches(iview)                                \
 353    .pitch = tu_image_ubwc_pitch(iview->image, iview->base_mip),          \
 354    .array_pitch = tu_image_ubwc_size(iview->image, iview->base_mip) >> 2
 355
 356 static void
 357 tu6_emit_zs(struct tu_cmd_buffer *cmd,
 358             const struct tu_subpass *subpass,
 359             struct tu_cs *cs)
 360 {
 361    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 362
 363    const uint32_t a = subpass->depth_stencil_attachment.attachment;
 364    if (a == VK_ATTACHMENT_UNUSED) {
 365       tu_cs_emit_regs(cs,
 366                       A6XX_RB_DEPTH_BUFFER_INFO(.depth_format = DEPTH6_NONE),
 367                       A6XX_RB_DEPTH_BUFFER_PITCH(0),
 368                       A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH(0),
 369                       A6XX_RB_DEPTH_BUFFER_BASE(0),
 370                       A6XX_RB_DEPTH_BUFFER_BASE_GMEM(0));
 371
 372       tu_cs_emit_regs(cs,
 373                       A6XX_GRAS_SU_DEPTH_BUFFER_INFO(.depth_format = DEPTH6_NONE));
 374
 375       tu_cs_emit_regs(cs,
 376                       A6XX_GRAS_LRZ_BUFFER_BASE(0),
 377                       A6XX_GRAS_LRZ_BUFFER_PITCH(0),
 378                       A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE(0));
 379
 380       tu_cs_emit_regs(cs, A6XX_RB_STENCIL_INFO(0));
 381
 382       return;
 383    }
 384
 385    const struct tu_image_view *iview = fb->attachments[a].attachment;
 386    enum a6xx_depth_format fmt = tu6_pipe2depth(iview->vk_format);
 387
 388    tu_cs_emit_regs(cs,
 389                    A6XX_RB_DEPTH_BUFFER_INFO(.depth_format = fmt),
 390                    A6XX_RB_DEPTH_BUFFER_PITCH(tu_image_stride(iview->image, iview->base_mip)),
 391                    A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH(iview->image->layout.layer_size),
 392                    A6XX_RB_DEPTH_BUFFER_BASE(tu_image_view_base_ref(iview)),
 393                    A6XX_RB_DEPTH_BUFFER_BASE_GMEM(cmd->state.pass->attachments[a].gmem_offset));
 394
 395    tu_cs_emit_regs(cs,
 396                    A6XX_GRAS_SU_DEPTH_BUFFER_INFO(.depth_format = fmt));
 397
 398    tu_cs_emit_regs(cs,
 399                    A6XX_RB_DEPTH_FLAG_BUFFER_BASE(tu_image_view_ubwc_base_ref(iview)),
 400                    A6XX_RB_DEPTH_FLAG_BUFFER_PITCH(tu_image_view_ubwc_pitches(iview)));
 401
 402    tu_cs_emit_regs(cs,
 403                    A6XX_GRAS_LRZ_BUFFER_BASE(0),
 404                    A6XX_GRAS_LRZ_BUFFER_PITCH(0),
 405                    A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE(0));
 406
 407    tu_cs_emit_regs(cs,
 408                    A6XX_RB_STENCIL_INFO(0));
 409
 410    /* enable zs? */
 411 }
 412
 413 static void
 414 tu6_emit_mrt(struct tu_cmd_buffer *cmd,
 415              const struct tu_subpass *subpass,
 416              struct tu_cs *cs)
 417 {
 418    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 419    unsigned char mrt_comp[MAX_RTS] = { 0 };
 420    unsigned srgb_cntl = 0;
 421
 422    for (uint32_t i = 0; i < subpass->color_count; ++i) {
 423       uint32_t a = subpass->color_attachments[i].attachment;
 424       if (a == VK_ATTACHMENT_UNUSED)
 425          continue;
 426
 427       const struct tu_image_view *iview = fb->attachments[a].attachment;
 428       const enum a6xx_tile_mode tile_mode =
 429          tu6_get_image_tile_mode(iview->image, iview->base_mip);
 430
 431       mrt_comp[i] = 0xf;
 432
 433       if (vk_format_is_srgb(iview->vk_format))
 434          srgb_cntl |= (1 << i);
 435
 436       const struct tu_native_format *format =
 437          tu6_get_native_format(iview->vk_format);
 438       assert(format && format->rb >= 0);
 439
 440       tu_cs_emit_regs(cs,
 441                       A6XX_RB_MRT_BUF_INFO(i,
 442                                            .color_tile_mode = tile_mode,
 443                                            .color_format = format->rb,
 444                                            .color_swap = format->swap),
 445                       A6XX_RB_MRT_PITCH(i, tu_image_stride(iview->image, iview->base_mip)),
 446                       A6XX_RB_MRT_ARRAY_PITCH(i, iview->image->layout.layer_size),
 447                       A6XX_RB_MRT_BASE(i, tu_image_view_base_ref(iview)),
 448                       A6XX_RB_MRT_BASE_GMEM(i, cmd->state.pass->attachments[a].gmem_offset));
 449
 450       tu_cs_emit_regs(cs,
 451                       A6XX_SP_FS_MRT_REG(i,
 452                                          .color_format = format->rb,
 453                                          .color_sint = vk_format_is_sint(iview->vk_format),
 454                                          .color_uint = vk_format_is_uint(iview->vk_format)));
 455
 456       tu_cs_emit_regs(cs,
 457                       A6XX_RB_MRT_FLAG_BUFFER_ADDR(i, tu_image_view_ubwc_base_ref(iview)),
 458                       A6XX_RB_MRT_FLAG_BUFFER_PITCH(i, tu_image_view_ubwc_pitches(iview)));
 459    }
 460
 461    tu_cs_emit_regs(cs,
 462                    A6XX_RB_SRGB_CNTL(srgb_cntl));
 463
 464    tu_cs_emit_regs(cs,
 465                    A6XX_SP_SRGB_CNTL(srgb_cntl));
 466
 467    tu_cs_emit_regs(cs,
 468                    A6XX_RB_RENDER_COMPONENTS(
 469                       .rt0 = mrt_comp[0],
 470                       .rt1 = mrt_comp[1],
 471                       .rt2 = mrt_comp[2],
 472                       .rt3 = mrt_comp[3],
 473                       .rt4 = mrt_comp[4],
 474                       .rt5 = mrt_comp[5],
 475                       .rt6 = mrt_comp[6],
 476                       .rt7 = mrt_comp[7]));
 477
 478    tu_cs_emit_regs(cs,
 479                    A6XX_SP_FS_RENDER_COMPONENTS(
 480                       .rt0 = mrt_comp[0],
 481                       .rt1 = mrt_comp[1],
 482                       .rt2 = mrt_comp[2],
 483                       .rt3 = mrt_comp[3],
 484                       .rt4 = mrt_comp[4],
 485                       .rt5 = mrt_comp[5],
 486                       .rt6 = mrt_comp[6],
 487                       .rt7 = mrt_comp[7]));
 488 }
 489
 490 static void
 491 tu6_emit_msaa(struct tu_cmd_buffer *cmd,
 492               const struct tu_subpass *subpass,
 493               struct tu_cs *cs)
 494 {
 495    const enum a3xx_msaa_samples samples = tu_msaa_samples(subpass->samples);
 496    bool msaa_disable = samples == MSAA_ONE;
 497
 498    tu_cs_emit_regs(cs,
 499                    A6XX_SP_TP_RAS_MSAA_CNTL(samples),
 500                    A6XX_SP_TP_DEST_MSAA_CNTL(.samples = samples,
 501                                              .msaa_disable = msaa_disable));
 502
 503    tu_cs_emit_regs(cs,
 504                    A6XX_GRAS_RAS_MSAA_CNTL(samples),
 505                    A6XX_GRAS_DEST_MSAA_CNTL(.samples = samples,
 506                                             .msaa_disable = msaa_disable));
 507
 508    tu_cs_emit_regs(cs,
 509                    A6XX_RB_RAS_MSAA_CNTL(samples),
 510                    A6XX_RB_DEST_MSAA_CNTL(.samples = samples,
 511                                           .msaa_disable = msaa_disable));
 512
 513    tu_cs_emit_regs(cs,
 514                    A6XX_RB_MSAA_CNTL(samples));
 515 }
 516
 517 static void
 518 tu6_emit_bin_size(struct tu_cmd_buffer *cmd, struct tu_cs *cs, uint32_t flags)
 519 {
 520    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
 521    const uint32_t bin_w = tiling->tile0.extent.width;
 522    const uint32_t bin_h = tiling->tile0.extent.height;
 523
 524    tu_cs_emit_regs(cs,
 525                    A6XX_GRAS_BIN_CONTROL(.binw = bin_w,
 526                                          .binh = bin_h,
 527                                          .dword = flags));
 528
 529    tu_cs_emit_regs(cs,
 530                    A6XX_RB_BIN_CONTROL(.binw = bin_w,
 531                                        .binh = bin_h,
 532                                        .dword = flags));
 533
 534    /* no flag for RB_BIN_CONTROL2... */
 535    tu_cs_emit_regs(cs,
 536                    A6XX_RB_BIN_CONTROL2(.binw = bin_w,
 537                                         .binh = bin_h));
 538 }
 539
 540 static void
 541 tu6_emit_render_cntl(struct tu_cmd_buffer *cmd,
 542                      struct tu_cs *cs,
 543                      bool binning)
 544 {
 545    uint32_t cntl = 0;
 546    cntl |= A6XX_RB_RENDER_CNTL_UNK4;
 547    if (binning)
 548       cntl |= A6XX_RB_RENDER_CNTL_BINNING;
 549
 550    tu_cs_emit_pkt7(cs, CP_REG_WRITE, 3);
 551    tu_cs_emit(cs, CP_REG_WRITE_0_TRACKER(TRACK_RENDER_CNTL));
 552    tu_cs_emit(cs, REG_A6XX_RB_RENDER_CNTL);
 553    tu_cs_emit(cs, cntl);
 554 }
 555
 556 static void
 557 tu6_emit_blit_scissor(struct tu_cmd_buffer *cmd, struct tu_cs *cs, bool align)
 558 {
 559    const VkRect2D *render_area = &cmd->state.tiling_config.render_area;
 560    uint32_t x1 = render_area->offset.x;
 561    uint32_t y1 = render_area->offset.y;
 562    uint32_t x2 = x1 + render_area->extent.width - 1;
 563    uint32_t y2 = y1 + render_area->extent.height - 1;
 564
 565    /* TODO: alignment requirement seems to be less than tile_align_w/h */
 566    if (align) {
 567       x1 = x1 & ~cmd->device->physical_device->tile_align_w;
 568       y1 = y1 & ~cmd->device->physical_device->tile_align_h;
 569       x2 = ALIGN_POT(x2 + 1, cmd->device->physical_device->tile_align_w) - 1;
 570       y2 = ALIGN_POT(y2 + 1, cmd->device->physical_device->tile_align_h) - 1;
 571    }
 572
 573    tu_cs_emit_regs(cs,
 574                    A6XX_RB_BLIT_SCISSOR_TL(.x = x1, .y = y1),
 575                    A6XX_RB_BLIT_SCISSOR_BR(.x = x2, .y = y2));
 576 }
 577
 578 static void
 579 tu6_emit_blit_info(struct tu_cmd_buffer *cmd,
 580                    struct tu_cs *cs,
 581                    const struct tu_image_view *iview,
 582                    uint32_t gmem_offset,
 583                    bool resolve)
 584 {
 585    tu_cs_emit_regs(cs,
 586                    A6XX_RB_BLIT_INFO(.unk0 = !resolve, .gmem = !resolve));
 587
 588    const struct tu_native_format *format =
 589       tu6_get_native_format(iview->vk_format);
 590    assert(format && format->rb >= 0);
 591
 592    enum a6xx_tile_mode tile_mode =
 593       tu6_get_image_tile_mode(iview->image, iview->base_mip);
 594    tu_cs_emit_regs(cs,
 595                    A6XX_RB_BLIT_DST_INFO(
 596                       .tile_mode = tile_mode,
 597                       .samples = tu_msaa_samples(iview->image->samples),
 598                       .color_format = format->rb,
 599                       .color_swap = format->swap,
 600                       .flags = iview->image->layout.ubwc_layer_size != 0),
 601                    A6XX_RB_BLIT_DST(tu_image_view_base_ref(iview)),
 602                    A6XX_RB_BLIT_DST_PITCH(tu_image_stride(iview->image, iview->base_mip)),
 603                    A6XX_RB_BLIT_DST_ARRAY_PITCH(iview->image->layout.layer_size));
 604
 605    if (iview->image->layout.ubwc_layer_size) {
 606       tu_cs_emit_regs(cs,
 607                       A6XX_RB_BLIT_FLAG_DST(tu_image_view_ubwc_base_ref(iview)),
 608                       A6XX_RB_BLIT_FLAG_DST_PITCH(tu_image_view_ubwc_pitches(iview)));
 609    }
 610
 611    tu_cs_emit_regs(cs,
 612                    A6XX_RB_BLIT_BASE_GMEM(gmem_offset));
 613 }
 614
 615 static void
 616 tu6_emit_blit(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 617 {
 618    tu6_emit_marker(cmd, cs);
 619    tu6_emit_event_write(cmd, cs, BLIT, false);
 620    tu6_emit_marker(cmd, cs);
 621 }
 622
 623 static void
 624 tu6_emit_window_scissor(struct tu_cmd_buffer *cmd,
 625                         struct tu_cs *cs,
 626                         uint32_t x1,
 627                         uint32_t y1,
 628                         uint32_t x2,
 629                         uint32_t y2)
 630 {
 631    tu_cs_emit_regs(cs,
 632                    A6XX_GRAS_SC_WINDOW_SCISSOR_TL(.x = x1, .y = y1),
 633                    A6XX_GRAS_SC_WINDOW_SCISSOR_BR(.x = x2, .y = y2));
 634
 635    tu_cs_emit_regs(cs,
 636                    A6XX_GRAS_RESOLVE_CNTL_1(.x = x1, .y = y1),
 637                    A6XX_GRAS_RESOLVE_CNTL_2(.x = x2, .y = y2));
 638 }
 639
 640 static void
 641 tu6_emit_window_offset(struct tu_cmd_buffer *cmd,
 642                        struct tu_cs *cs,
 643                        uint32_t x1,
 644                        uint32_t y1)
 645 {
 646    tu_cs_emit_regs(cs,
 647                    A6XX_RB_WINDOW_OFFSET(.x = x1, .y = y1));
 648
 649    tu_cs_emit_regs(cs,
 650                    A6XX_RB_WINDOW_OFFSET2(.x = x1, .y = y1));
 651
 652    tu_cs_emit_regs(cs,
 653                    A6XX_SP_WINDOW_OFFSET(.x = x1, .y = y1));
 654
 655    tu_cs_emit_regs(cs,
 656                    A6XX_SP_TP_WINDOW_OFFSET(.x = x1, .y = y1));
 657 }
 658
 659 static bool
 660 use_hw_binning(struct tu_cmd_buffer *cmd)
 661 {
 662    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
 663
 664    if (unlikely(cmd->device->physical_device->instance->debug_flags & TU_DEBUG_NOBIN))
 665       return false;
 666
 667    return (tiling->tile_count.width * tiling->tile_count.height) > 2;
 668 }
 669
 670 static void
 671 tu6_emit_tile_select(struct tu_cmd_buffer *cmd,
 672                      struct tu_cs *cs,
 673                      const struct tu_tile *tile)
 674 {
 675    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
 676    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(0x7));
 677
 678    tu6_emit_marker(cmd, cs);
 679    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
 680    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_GMEM) | 0x10);
 681    tu6_emit_marker(cmd, cs);
 682
 683    const uint32_t x1 = tile->begin.x;
 684    const uint32_t y1 = tile->begin.y;
 685    const uint32_t x2 = tile->end.x - 1;
 686    const uint32_t y2 = tile->end.y - 1;
 687    tu6_emit_window_scissor(cmd, cs, x1, y1, x2, y2);
 688    tu6_emit_window_offset(cmd, cs, x1, y1);
 689
 690    tu_cs_emit_regs(cs,
 691                    A6XX_VPC_SO_OVERRIDE(.so_disable = true));
 692
 693    if (use_hw_binning(cmd)) {
 694       tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
 695
 696       tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
 697       tu_cs_emit(cs, 0x0);
 698
 699       tu_cs_emit_pkt7(cs, CP_REG_TEST, 1);
 700       tu_cs_emit(cs, A6XX_CP_REG_TEST_0_REG(OVERFLOW_FLAG_REG) |
 701                      A6XX_CP_REG_TEST_0_BIT(0) |
 702                      A6XX_CP_REG_TEST_0_WAIT_FOR_ME);
 703
 704       tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
 705       tu_cs_emit(cs, CP_COND_REG_EXEC_0_MODE(PRED_TEST));
 706       tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(11));
 707
 708       /* if (no overflow) */ {
 709          tu_cs_emit_pkt7(cs, CP_SET_BIN_DATA5, 7);
 710          tu_cs_emit(cs, cmd->state.tiling_config.pipe_sizes[tile->pipe] |
 711                         CP_SET_BIN_DATA5_0_VSC_N(tile->slot));
 712          tu_cs_emit_qw(cs, cmd->vsc_data.iova + tile->pipe * cmd->vsc_data_pitch);
 713          tu_cs_emit_qw(cs, cmd->vsc_data.iova + (tile->pipe * 4) + (32 * cmd->vsc_data_pitch));
 714          tu_cs_emit_qw(cs, cmd->vsc_data2.iova + (tile->pipe * cmd->vsc_data2_pitch));
 715
 716          tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
 717          tu_cs_emit(cs, 0x0);
 718
 719          /* use a NOP packet to skip over the 'else' side: */
 720          tu_cs_emit_pkt7(cs, CP_NOP, 2);
 721       } /* else */ {
 722          tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
 723          tu_cs_emit(cs, 0x1);
 724       }
 725
 726       tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
 727       tu_cs_emit(cs, 0x0);
 728
 729       tu_cs_emit_regs(cs,
 730                       A6XX_RB_UNKNOWN_8804(0));
 731
 732       tu_cs_emit_regs(cs,
 733                       A6XX_SP_TP_UNKNOWN_B304(0));
 734
 735       tu_cs_emit_regs(cs,
 736                       A6XX_GRAS_UNKNOWN_80A4(0));
 737    } else {
 738       tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
 739       tu_cs_emit(cs, 0x1);
 740
 741       tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
 742       tu_cs_emit(cs, 0x0);
 743    }
 744 }
 745
 746 static void
 747 tu6_emit_load_attachment(struct tu_cmd_buffer *cmd, struct tu_cs *cs, uint32_t a)
 748 {
 749    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
 750    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 751    const struct tu_image_view *iview = fb->attachments[a].attachment;
 752    const struct tu_render_pass_attachment *attachment =
 753       &cmd->state.pass->attachments[a];
 754
 755    if (attachment->gmem_offset < 0)
 756       return;
 757
 758    const uint32_t x1 = tiling->render_area.offset.x;
 759    const uint32_t y1 = tiling->render_area.offset.y;
 760    const uint32_t x2 = x1 + tiling->render_area.extent.width;
 761    const uint32_t y2 = y1 + tiling->render_area.extent.height;
 762    const uint32_t tile_x2 =
 763       tiling->tile0.offset.x + tiling->tile0.extent.width * tiling->tile_count.width;
 764    const uint32_t tile_y2 =
 765       tiling->tile0.offset.y + tiling->tile0.extent.height * tiling->tile_count.height;
 766    bool need_load =
 767       x1 != tiling->tile0.offset.x || x2 != MIN2(fb->width, tile_x2) ||
 768       y1 != tiling->tile0.offset.y || y2 != MIN2(fb->height, tile_y2);
 769
 770    if (need_load)
 771       tu_finishme("improve handling of unaligned render area");
 772
 773    if (attachment->load_op == VK_ATTACHMENT_LOAD_OP_LOAD)
 774       need_load = true;
 775
 776    if (vk_format_has_stencil(iview->vk_format) &&
 777        attachment->stencil_load_op == VK_ATTACHMENT_LOAD_OP_LOAD)
 778       need_load = true;
 779
 780    if (need_load) {
 781       tu6_emit_blit_info(cmd, cs, iview, attachment->gmem_offset, false);
 782       tu6_emit_blit(cmd, cs);
 783    }
 784 }
 785
 786 static void
 787 tu6_emit_clear_attachment(struct tu_cmd_buffer *cmd, struct tu_cs *cs,
 788                           uint32_t a,
 789                           const VkRenderPassBeginInfo *info)
 790 {
 791    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 792    const struct tu_image_view *iview = fb->attachments[a].attachment;
 793    const struct tu_render_pass_attachment *attachment =
 794       &cmd->state.pass->attachments[a];
 795    unsigned clear_mask = 0;
 796
 797    /* note: this means it isn't used by any subpass and shouldn't be cleared anyway */
 798    if (attachment->gmem_offset < 0)
 799       return;
 800
 801    if (attachment->load_op == VK_ATTACHMENT_LOAD_OP_CLEAR)
 802       clear_mask = 0xf;
 803
 804    if (vk_format_has_stencil(iview->vk_format)) {
 805       clear_mask &= 0x1;
 806       if (attachment->stencil_load_op == VK_ATTACHMENT_LOAD_OP_CLEAR)
 807          clear_mask |= 0x2;
 808    }
 809    if (!clear_mask)
 810       return;
 811
 812    const struct tu_native_format *format =
 813       tu6_get_native_format(iview->vk_format);
 814    assert(format && format->rb >= 0);
 815
 816    tu_cs_emit_regs(cs,
 817                    A6XX_RB_BLIT_DST_INFO(.color_format = format->rb));
 818
 819    tu_cs_emit_regs(cs,
 820                    A6XX_RB_BLIT_INFO(.gmem = true,
 821                                      .clear_mask = clear_mask));
 822
 823    tu_cs_emit_regs(cs,
 824                    A6XX_RB_BLIT_BASE_GMEM(attachment->gmem_offset));
 825
 826    tu_cs_emit_regs(cs,
 827                    A6XX_RB_UNKNOWN_88D0(0));
 828
 829    uint32_t clear_vals[4] = { 0 };
 830    tu_pack_clear_value(&info->pClearValues[a], iview->vk_format, clear_vals);
 831
 832    tu_cs_emit_regs(cs,
 833                    A6XX_RB_BLIT_CLEAR_COLOR_DW0(clear_vals[0]),
 834                    A6XX_RB_BLIT_CLEAR_COLOR_DW1(clear_vals[1]),
 835                    A6XX_RB_BLIT_CLEAR_COLOR_DW2(clear_vals[2]),
 836                    A6XX_RB_BLIT_CLEAR_COLOR_DW3(clear_vals[3]));
 837
 838    tu6_emit_blit(cmd, cs);
 839 }
 840
 841 static void
 842 tu6_emit_store_attachment(struct tu_cmd_buffer *cmd,
 843                           struct tu_cs *cs,
 844                           uint32_t a,
 845                           uint32_t gmem_a)
 846 {
 847    if (cmd->state.pass->attachments[a].store_op == VK_ATTACHMENT_STORE_OP_DONT_CARE)
 848       return;
 849
 850    tu6_emit_blit_info(cmd, cs,
 851                       cmd->state.framebuffer->attachments[a].attachment,
 852                       cmd->state.pass->attachments[gmem_a].gmem_offset, true);
 853    tu6_emit_blit(cmd, cs);
 854 }
 855
 856 static void
 857 tu6_emit_tile_store(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 858 {
 859    const struct tu_render_pass *pass = cmd->state.pass;
 860    const struct tu_subpass *subpass = &pass->subpasses[pass->subpass_count-1];
 861
 862    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
 863    tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
 864                      CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
 865                      CP_SET_DRAW_STATE__0_GROUP_ID(0));
 866    tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
 867    tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
 868
 869    tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
 870    tu_cs_emit(cs, 0x0);
 871
 872    tu6_emit_marker(cmd, cs);
 873    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
 874    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_RESOLVE) | 0x10);
 875    tu6_emit_marker(cmd, cs);
 876
 877    tu6_emit_blit_scissor(cmd, cs, true);
 878
 879    for (uint32_t a = 0; a < pass->attachment_count; ++a) {
 880       if (pass->attachments[a].gmem_offset >= 0)
 881          tu6_emit_store_attachment(cmd, cs, a, a);
 882    }
 883
 884    if (subpass->resolve_attachments) {
 885       for (unsigned i = 0; i < subpass->color_count; i++) {
 886          uint32_t a = subpass->resolve_attachments[i].attachment;
 887          if (a != VK_ATTACHMENT_UNUSED)
 888             tu6_emit_store_attachment(cmd, cs, a,
 889                                       subpass->color_attachments[i].attachment);
 890       }
 891    }
 892 }
 893
 894 static void
 895 tu6_emit_restart_index(struct tu_cs *cs, uint32_t restart_index)
 896 {
 897    tu_cs_emit_regs(cs,
 898                    A6XX_PC_RESTART_INDEX(restart_index));
 899 }
 900
 901 static void
 902 tu6_init_hw(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 903 {
 904    VkResult result = tu_cs_reserve_space(cmd->device, cs, 256);
 905    if (result != VK_SUCCESS) {
 906       cmd->record_result = result;
 907       return;
 908    }
 909
 910    tu6_emit_cache_flush(cmd, cs);
 911
 912    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UPDATE_CNTL, 0xfffff);
 913
 914    tu_cs_emit_write_reg(cs, REG_A6XX_RB_CCU_CNTL, 0x7c400004);
 915    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8E04, 0x00100000);
 916    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE04, 0x8);
 917    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE00, 0);
 918    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE0F, 0x3f);
 919    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B605, 0x44);
 920    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B600, 0x100000);
 921    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE00, 0x80);
 922    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE01, 0);
 923
 924    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9600, 0);
 925    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8600, 0x880);
 926    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE04, 0);
 927    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE03, 0x00000410);
 928    tu_cs_emit_write_reg(cs, REG_A6XX_SP_IBO_COUNT, 0);
 929    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B182, 0);
 930    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BB11, 0);
 931    tu_cs_emit_write_reg(cs, REG_A6XX_UCHE_UNKNOWN_0E12, 0x3200000);
 932    tu_cs_emit_write_reg(cs, REG_A6XX_UCHE_CLIENT_PF, 4);
 933    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8E01, 0x0);
 934    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AB00, 0x5);
 935    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_ADD_OFFSET, A6XX_VFD_ADD_OFFSET_VERTEX);
 936    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8811, 0x00000010);
 937    tu_cs_emit_write_reg(cs, REG_A6XX_PC_MODE_CNTL, 0x1f);
 938
 939    tu_cs_emit_write_reg(cs, REG_A6XX_RB_SRGB_CNTL, 0);
 940
 941    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8101, 0);
 942    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_SAMPLE_CNTL, 0);
 943    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8110, 0);
 944
 945    tu_cs_emit_write_reg(cs, REG_A6XX_RB_RENDER_CONTROL0, 0x401);
 946    tu_cs_emit_write_reg(cs, REG_A6XX_RB_RENDER_CONTROL1, 0);
 947    tu_cs_emit_write_reg(cs, REG_A6XX_RB_FS_OUTPUT_CNTL0, 0);
 948    tu_cs_emit_write_reg(cs, REG_A6XX_RB_SAMPLE_CNTL, 0);
 949    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8818, 0);
 950    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8819, 0);
 951    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881A, 0);
 952    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881B, 0);
 953    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881C, 0);
 954    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881D, 0);
 955    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881E, 0);
 956    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_88F0, 0);
 957
 958    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9101, 0xffff00);
 959    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9107, 0);
 960
 961    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9236, 1);
 962    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9300, 0);
 963
 964    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_SO_OVERRIDE,
 965                         A6XX_VPC_SO_OVERRIDE_SO_DISABLE);
 966
 967    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9801, 0);
 968    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9806, 0);
 969    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9980, 0);
 970
 971    tu_cs_emit_write_reg(cs, REG_A6XX_PC_PRIMITIVE_CNTL_6, 0);
 972    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9B07, 0);
 973
 974    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_A81B, 0);
 975
 976    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B183, 0);
 977
 978    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8099, 0);
 979    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_809B, 0);
 980    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A0, 2);
 981    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80AF, 0);
 982    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9210, 0);
 983    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9211, 0);
 984    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9602, 0);
 985    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9981, 0x3);
 986    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9E72, 0);
 987    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9108, 0x3);
 988    tu_cs_emit_write_reg(cs, REG_A6XX_SP_TP_UNKNOWN_B304, 0);
 989    tu_cs_emit_write_reg(cs, REG_A6XX_SP_TP_UNKNOWN_B309, 0x000000a2);
 990    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8804, 0);
 991    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A4, 0);
 992    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A5, 0);
 993    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A6, 0);
 994    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8805, 0);
 995    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8806, 0);
 996    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8878, 0);
 997    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8879, 0);
 998    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_CONTROL_5_REG, 0xfc);
 999
1000    tu6_emit_marker(cmd, cs);
1001
1002    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_MODE_CNTL, 0x00000000);
1003
1004    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_UNKNOWN_A008, 0);
1005
1006    tu_cs_emit_write_reg(cs, REG_A6XX_PC_MODE_CNTL, 0x0000001f);
1007
1008    /* we don't use this yet.. probably best to disable.. */
1009    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
1010    tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
1011                      CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
1012                      CP_SET_DRAW_STATE__0_GROUP_ID(0));
1013    tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
1014    tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
1015
1016    tu_cs_emit_regs(cs,
1017                    A6XX_VPC_SO_BUFFER_BASE(0),
1018                    A6XX_VPC_SO_BUFFER_SIZE(0));
1019
1020    tu_cs_emit_regs(cs,
1021                    A6XX_VPC_SO_FLUSH_BASE(0));
1022
1023    tu_cs_emit_regs(cs,
1024                    A6XX_VPC_SO_BUF_CNTL(0));
1025
1026    tu_cs_emit_regs(cs,
1027                    A6XX_VPC_SO_BUFFER_OFFSET(0, 0));
1028
1029    tu_cs_emit_regs(cs,
1030                    A6XX_VPC_SO_BUFFER_BASE(1, 0),
1031                    A6XX_VPC_SO_BUFFER_SIZE(1, 0));
1032
1033    tu_cs_emit_regs(cs,
1034                    A6XX_VPC_SO_BUFFER_OFFSET(1, 0),
1035                    A6XX_VPC_SO_FLUSH_BASE(1, 0),
1036                    A6XX_VPC_SO_BUFFER_BASE(2, 0),
1037                    A6XX_VPC_SO_BUFFER_SIZE(2, 0));
1038
1039    tu_cs_emit_regs(cs,
1040                    A6XX_VPC_SO_BUFFER_OFFSET(2, 0),
1041                    A6XX_VPC_SO_FLUSH_BASE(2, 0),
1042                    A6XX_VPC_SO_BUFFER_BASE(3, 0),
1043                    A6XX_VPC_SO_BUFFER_SIZE(3, 0));
1044
1045    tu_cs_emit_regs(cs,
1046                    A6XX_VPC_SO_BUFFER_OFFSET(3, 0),
1047                    A6XX_VPC_SO_FLUSH_BASE(3, 0));
1048
1049    tu_cs_emit_regs(cs,
1050                    A6XX_SP_HS_CTRL_REG0(0));
1051
1052    tu_cs_emit_regs(cs,
1053                    A6XX_SP_GS_CTRL_REG0(0));
1054
1055    tu_cs_emit_regs(cs,
1056                    A6XX_GRAS_LRZ_CNTL(0));
1057
1058    tu_cs_emit_regs(cs,
1059                    A6XX_RB_LRZ_CNTL(0));
1060
1061    tu_cs_sanity_check(cs);
1062 }
1063
1064 static void
1065 tu6_cache_flush(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1066 {
1067    unsigned seqno;
1068
1069    seqno = tu6_emit_event_write(cmd, cs, CACHE_FLUSH_AND_INV_EVENT, true);
1070
1071    tu_cs_emit_pkt7(cs, CP_WAIT_REG_MEM, 6);
1072    tu_cs_emit(cs, CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ) |
1073                   CP_WAIT_REG_MEM_0_POLL_MEMORY);
1074    tu_cs_emit_qw(cs, cmd->scratch_bo.iova);
1075    tu_cs_emit(cs, CP_WAIT_REG_MEM_3_REF(seqno));
1076    tu_cs_emit(cs, CP_WAIT_REG_MEM_4_MASK(~0));
1077    tu_cs_emit(cs, CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(16));
1078
1079    seqno = tu6_emit_event_write(cmd, cs, CACHE_FLUSH_TS, true);
1080
1081    tu_cs_emit_pkt7(cs, CP_WAIT_MEM_GTE, 4);
1082    tu_cs_emit(cs, CP_WAIT_MEM_GTE_0_RESERVED(0));
1083    tu_cs_emit_qw(cs, cmd->scratch_bo.iova);
1084    tu_cs_emit(cs, CP_WAIT_MEM_GTE_3_REF(seqno));
1085 }
1086
1087 static void
1088 update_vsc_pipe(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1089 {
1090    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1091
1092    tu_cs_emit_regs(cs,
1093                    A6XX_VSC_BIN_SIZE(.width = tiling->tile0.extent.width,
1094                                      .height = tiling->tile0.extent.height),
1095                    A6XX_VSC_SIZE_ADDRESS(.bo = &cmd->vsc_data,
1096                                          .bo_offset = 32 * cmd->vsc_data_pitch));
1097
1098    tu_cs_emit_regs(cs,
1099                    A6XX_VSC_BIN_COUNT(.nx = tiling->tile_count.width,
1100                                       .ny = tiling->tile_count.height));
1101
1102    tu_cs_emit_pkt4(cs, REG_A6XX_VSC_PIPE_CONFIG_REG(0), 32);
1103    for (unsigned i = 0; i < 32; i++)
1104       tu_cs_emit(cs, tiling->pipe_config[i]);
1105
1106    tu_cs_emit_regs(cs,
1107                    A6XX_VSC_PIPE_DATA2_ADDRESS(.bo = &cmd->vsc_data2),
1108                    A6XX_VSC_PIPE_DATA2_PITCH(cmd->vsc_data2_pitch),
1109                    A6XX_VSC_PIPE_DATA2_ARRAY_PITCH(cmd->vsc_data2.size));
1110
1111    tu_cs_emit_regs(cs,
1112                    A6XX_VSC_PIPE_DATA_ADDRESS(.bo = &cmd->vsc_data),
1113                    A6XX_VSC_PIPE_DATA_PITCH(cmd->vsc_data_pitch),
1114                    A6XX_VSC_PIPE_DATA_ARRAY_PITCH(cmd->vsc_data.size));
1115 }
1116
1117 static void
1118 emit_vsc_overflow_test(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1119 {
1120    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1121    const uint32_t used_pipe_count =
1122       tiling->pipe_count.width * tiling->pipe_count.height;
1123
1124    /* Clear vsc_scratch: */
1125    tu_cs_emit_pkt7(cs, CP_MEM_WRITE, 3);
1126    tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1127    tu_cs_emit(cs, 0x0);
1128
1129    /* Check for overflow, write vsc_scratch if detected: */
1130    for (int i = 0; i < used_pipe_count; i++) {
1131       tu_cs_emit_pkt7(cs, CP_COND_WRITE5, 8);
1132       tu_cs_emit(cs, CP_COND_WRITE5_0_FUNCTION(WRITE_GE) |
1133             CP_COND_WRITE5_0_WRITE_MEMORY);
1134       tu_cs_emit(cs, CP_COND_WRITE5_1_POLL_ADDR_LO(REG_A6XX_VSC_SIZE_REG(i)));
1135       tu_cs_emit(cs, CP_COND_WRITE5_2_POLL_ADDR_HI(0));
1136       tu_cs_emit(cs, CP_COND_WRITE5_3_REF(cmd->vsc_data_pitch));
1137       tu_cs_emit(cs, CP_COND_WRITE5_4_MASK(~0));
1138       tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1139       tu_cs_emit(cs, CP_COND_WRITE5_7_WRITE_DATA(1 + cmd->vsc_data_pitch));
1140
1141       tu_cs_emit_pkt7(cs, CP_COND_WRITE5, 8);
1142       tu_cs_emit(cs, CP_COND_WRITE5_0_FUNCTION(WRITE_GE) |
1143             CP_COND_WRITE5_0_WRITE_MEMORY);
1144       tu_cs_emit(cs, CP_COND_WRITE5_1_POLL_ADDR_LO(REG_A6XX_VSC_SIZE2_REG(i)));
1145       tu_cs_emit(cs, CP_COND_WRITE5_2_POLL_ADDR_HI(0));
1146       tu_cs_emit(cs, CP_COND_WRITE5_3_REF(cmd->vsc_data2_pitch));
1147       tu_cs_emit(cs, CP_COND_WRITE5_4_MASK(~0));
1148       tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1149       tu_cs_emit(cs, CP_COND_WRITE5_7_WRITE_DATA(3 + cmd->vsc_data2_pitch));
1150    }
1151
1152    tu_cs_emit_pkt7(cs, CP_WAIT_MEM_WRITES, 0);
1153
1154    tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
1155
1156    tu_cs_emit_pkt7(cs, CP_MEM_TO_REG, 3);
1157    tu_cs_emit(cs, CP_MEM_TO_REG_0_REG(OVERFLOW_FLAG_REG) |
1158          CP_MEM_TO_REG_0_CNT(1 - 1));
1159    tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1160
1161    /*
1162     * This is a bit awkward, we really want a way to invert the
1163     * CP_REG_TEST/CP_COND_REG_EXEC logic, so that we can conditionally
1164     * execute cmds to use hwbinning when a bit is *not* set.  This
1165     * dance is to invert OVERFLOW_FLAG_REG
1166     *
1167     * A CP_NOP packet is used to skip executing the 'else' clause
1168     * if (b0 set)..
1169     */
1170
1171    /* b0 will be set if VSC_DATA or VSC_DATA2 overflow: */
1172    tu_cs_emit_pkt7(cs, CP_REG_TEST, 1);
1173    tu_cs_emit(cs, A6XX_CP_REG_TEST_0_REG(OVERFLOW_FLAG_REG) |
1174          A6XX_CP_REG_TEST_0_BIT(0) |
1175          A6XX_CP_REG_TEST_0_WAIT_FOR_ME);
1176
1177    tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
1178    tu_cs_emit(cs, CP_COND_REG_EXEC_0_MODE(PRED_TEST));
1179    tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(7));
1180
1181    /* if (b0 set) */ {
1182       /*
1183        * On overflow, mirror the value to control->vsc_overflow
1184        * which CPU is checking to detect overflow (see
1185        * check_vsc_overflow())
1186        */
1187       tu_cs_emit_pkt7(cs, CP_REG_TO_MEM, 3);
1188       tu_cs_emit(cs, CP_REG_TO_MEM_0_REG(OVERFLOW_FLAG_REG) |
1189             CP_REG_TO_MEM_0_CNT(0));
1190       tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_OVERFLOW);
1191
1192       tu_cs_emit_pkt4(cs, OVERFLOW_FLAG_REG, 1);
1193       tu_cs_emit(cs, 0x0);
1194
1195       tu_cs_emit_pkt7(cs, CP_NOP, 2);  /* skip 'else' when 'if' is taken */
1196    } /* else */ {
1197       tu_cs_emit_pkt4(cs, OVERFLOW_FLAG_REG, 1);
1198       tu_cs_emit(cs, 0x1);
1199    }
1200 }
1201
1202 static void
1203 tu6_emit_binning_pass(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1204 {
1205    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1206
1207    uint32_t x1 = tiling->tile0.offset.x;
1208    uint32_t y1 = tiling->tile0.offset.y;
1209    uint32_t x2 = tiling->render_area.offset.x + tiling->render_area.extent.width - 1;
1210    uint32_t y2 = tiling->render_area.offset.y + tiling->render_area.extent.height - 1;
1211
1212    tu6_emit_window_scissor(cmd, cs, x1, y1, x2, y2);
1213
1214    tu6_emit_marker(cmd, cs);
1215    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1216    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_BINNING));
1217    tu6_emit_marker(cmd, cs);
1218
1219    tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
1220    tu_cs_emit(cs, 0x1);
1221
1222    tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
1223    tu_cs_emit(cs, 0x1);
1224
1225    tu_cs_emit_wfi(cs);
1226
1227    tu_cs_emit_regs(cs,
1228                    A6XX_VFD_MODE_CNTL(.binning_pass = true));
1229
1230    update_vsc_pipe(cmd, cs);
1231
1232    tu_cs_emit_regs(cs,
1233                    A6XX_PC_UNKNOWN_9805(.unknown = 0x1));
1234
1235    tu_cs_emit_regs(cs,
1236                    A6XX_SP_UNKNOWN_A0F8(.unknown = 0x1));
1237
1238    tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, 1);
1239    tu_cs_emit(cs, UNK_2C);
1240
1241    tu_cs_emit_regs(cs,
1242                    A6XX_RB_WINDOW_OFFSET(.x = 0, .y = 0));
1243
1244    tu_cs_emit_regs(cs,
1245                    A6XX_SP_TP_WINDOW_OFFSET(.x = 0, .y = 0));
1246
1247    /* emit IB to binning drawcmds: */
1248    tu_cs_emit_call(cs, &cmd->draw_cs);
1249
1250    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
1251    tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
1252                   CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
1253                   CP_SET_DRAW_STATE__0_GROUP_ID(0));
1254    tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
1255    tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
1256
1257    tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, 1);
1258    tu_cs_emit(cs, UNK_2D);
1259
1260    tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE, false);
1261    tu6_cache_flush(cmd, cs);
1262
1263    tu_cs_emit_wfi(cs);
1264
1265    tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
1266
1267    emit_vsc_overflow_test(cmd, cs);
1268
1269    tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
1270    tu_cs_emit(cs, 0x0);
1271
1272    tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
1273    tu_cs_emit(cs, 0x0);
1274
1275    tu_cs_emit_wfi(cs);
1276
1277    tu_cs_emit_regs(cs,
1278                    A6XX_RB_CCU_CNTL(.unknown = 0x7c400004));
1279
1280    cmd->wait_for_idle = false;
1281 }
1282
1283 static void
1284 tu6_render_begin(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1285 {
1286    VkResult result = tu_cs_reserve_space(cmd->device, cs, 1024);
1287    if (result != VK_SUCCESS) {
1288       cmd->record_result = result;
1289       return;
1290    }
1291
1292    tu6_emit_lrz_flush(cmd, cs);
1293
1294    /* lrz clear? */
1295
1296    tu6_emit_cache_flush(cmd, cs);
1297
1298    tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1299    tu_cs_emit(cs, 0x0);
1300
1301    /* 0x10000000 for BYPASS.. 0x7c13c080 for GMEM: */
1302    tu6_emit_wfi(cmd, cs);
1303    tu_cs_emit_regs(cs,
1304                    A6XX_RB_CCU_CNTL(0x7c400004));
1305
1306    if (use_hw_binning(cmd)) {
1307       tu6_emit_bin_size(cmd, cs, A6XX_RB_BIN_CONTROL_BINNING_PASS | 0x6000000);
1308
1309       tu6_emit_render_cntl(cmd, cs, true);
1310
1311       tu6_emit_binning_pass(cmd, cs);
1312
1313       tu6_emit_bin_size(cmd, cs, A6XX_RB_BIN_CONTROL_USE_VIZ | 0x6000000);
1314
1315       tu_cs_emit_regs(cs,
1316                       A6XX_VFD_MODE_CNTL(0));
1317
1318       tu_cs_emit_regs(cs, A6XX_PC_UNKNOWN_9805(.unknown = 0x1));
1319
1320       tu_cs_emit_regs(cs, A6XX_SP_UNKNOWN_A0F8(.unknown = 0x1));
1321
1322       tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1323       tu_cs_emit(cs, 0x1);
1324    } else {
1325       tu6_emit_bin_size(cmd, cs, 0x6000000);
1326    }
1327
1328    tu6_emit_render_cntl(cmd, cs, false);
1329
1330    tu_cs_sanity_check(cs);
1331 }
1332
1333 static void
1334 tu6_render_tile(struct tu_cmd_buffer *cmd,
1335                 struct tu_cs *cs,
1336                 const struct tu_tile *tile)
1337 {
1338    const uint32_t render_tile_space = 256 + tu_cs_get_call_size(&cmd->draw_cs);
1339    VkResult result = tu_cs_reserve_space(cmd->device, cs, render_tile_space);
1340    if (result != VK_SUCCESS) {
1341       cmd->record_result = result;
1342       return;
1343    }
1344
1345    tu6_emit_tile_select(cmd, cs, tile);
1346    tu_cs_emit_ib(cs, &cmd->state.tile_load_ib);
1347
1348    tu_cs_emit_call(cs, &cmd->draw_cs);
1349    cmd->wait_for_idle = true;
1350
1351    if (use_hw_binning(cmd)) {
1352       tu_cs_emit_pkt7(cs, CP_REG_TEST, 1);
1353       tu_cs_emit(cs, A6XX_CP_REG_TEST_0_REG(OVERFLOW_FLAG_REG) |
1354                      A6XX_CP_REG_TEST_0_BIT(0) |
1355                      A6XX_CP_REG_TEST_0_WAIT_FOR_ME);
1356
1357       tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
1358       tu_cs_emit(cs, 0x10000000);
1359       tu_cs_emit(cs, 2);  /* conditionally execute next 2 dwords */
1360
1361       /* if (no overflow) */ {
1362          tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1363          tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(0x5) | 0x10);
1364       }
1365    }
1366
1367    tu_cs_emit_ib(cs, &cmd->state.tile_store_ib);
1368
1369    tu_cs_sanity_check(cs);
1370 }
1371
1372 static void
1373 tu6_render_end(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1374 {
1375    const uint32_t space = 16 + tu_cs_get_call_size(&cmd->draw_epilogue_cs);
1376    VkResult result = tu_cs_reserve_space(cmd->device, cs, space);
1377    if (result != VK_SUCCESS) {
1378       cmd->record_result = result;
1379       return;
1380    }
1381
1382    tu_cs_emit_call(cs, &cmd->draw_epilogue_cs);
1383
1384    tu_cs_emit_regs(cs,
1385                    A6XX_GRAS_LRZ_CNTL(0));
1386
1387    tu6_emit_lrz_flush(cmd, cs);
1388
1389    tu6_emit_event_write(cmd, cs, CACHE_FLUSH_TS, true);
1390
1391    tu_cs_sanity_check(cs);
1392 }
1393
1394 static void
1395 tu_cmd_render_tiles(struct tu_cmd_buffer *cmd)
1396 {
1397    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1398
1399    tu6_render_begin(cmd, &cmd->cs);
1400
1401    for (uint32_t y = 0; y < tiling->tile_count.height; y++) {
1402       for (uint32_t x = 0; x < tiling->tile_count.width; x++) {
1403          struct tu_tile tile;
1404          tu_tiling_config_get_tile(tiling, cmd->device, x, y, &tile);
1405          tu6_render_tile(cmd, &cmd->cs, &tile);
1406       }
1407    }
1408
1409    tu6_render_end(cmd, &cmd->cs);
1410 }
1411
1412 static void
1413 tu_cmd_prepare_tile_load_ib(struct tu_cmd_buffer *cmd,
1414                             const VkRenderPassBeginInfo *info)
1415 {
1416    const uint32_t tile_load_space =
1417       8 + (23+19) * cmd->state.pass->attachment_count +
1418       21 + (13 * cmd->state.subpass->color_count + 8) + 11;
1419
1420    struct tu_cs sub_cs;
1421
1422    VkResult result = tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs,
1423                                             tile_load_space, &sub_cs);
1424    if (result != VK_SUCCESS) {
1425       cmd->record_result = result;
1426       return;
1427    }
1428
1429    tu6_emit_blit_scissor(cmd, &sub_cs, true);
1430
1431    for (uint32_t i = 0; i < cmd->state.pass->attachment_count; ++i)
1432       tu6_emit_load_attachment(cmd, &sub_cs, i);
1433
1434    tu6_emit_blit_scissor(cmd, &sub_cs, false);
1435
1436    for (uint32_t i = 0; i < cmd->state.pass->attachment_count; ++i)
1437       tu6_emit_clear_attachment(cmd, &sub_cs, i, info);
1438
1439    /* invalidate because reading input attachments will cache GMEM and
1440     * the cache isn''t updated when GMEM is written
1441     * TODO: is there a no-cache bit for textures?
1442     */
1443    if (cmd->state.subpass->input_count)
1444       tu6_emit_event_write(cmd, &sub_cs, CACHE_INVALIDATE, false);
1445
1446    tu6_emit_zs(cmd, cmd->state.subpass, &sub_cs);
1447    tu6_emit_mrt(cmd, cmd->state.subpass, &sub_cs);
1448    tu6_emit_msaa(cmd, cmd->state.subpass, &sub_cs);
1449
1450    cmd->state.tile_load_ib = tu_cs_end_sub_stream(&cmd->sub_cs, &sub_cs);
1451 }
1452
1453 static void
1454 tu_cmd_prepare_tile_store_ib(struct tu_cmd_buffer *cmd)
1455 {
1456    const uint32_t tile_store_space = 32 + 23 * cmd->state.pass->attachment_count;
1457    struct tu_cs sub_cs;
1458
1459    VkResult result = tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs,
1460                                             tile_store_space, &sub_cs);
1461    if (result != VK_SUCCESS) {
1462       cmd->record_result = result;
1463       return;
1464    }
1465
1466    /* emit to tile-store sub_cs */
1467    tu6_emit_tile_store(cmd, &sub_cs);
1468
1469    cmd->state.tile_store_ib = tu_cs_end_sub_stream(&cmd->sub_cs, &sub_cs);
1470 }
1471
1472 static void
1473 tu_cmd_update_tiling_config(struct tu_cmd_buffer *cmd,
1474                             const VkRect2D *render_area)
1475 {
1476    const struct tu_device *dev = cmd->device;
1477    struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1478
1479    tiling->render_area = *render_area;
1480
1481    tu_tiling_config_update_tile_layout(tiling, dev, cmd->state.pass->gmem_pixels);
1482    tu_tiling_config_update_pipe_layout(tiling, dev);
1483    tu_tiling_config_update_pipes(tiling, dev);
1484 }
1485
1486 const struct tu_dynamic_state default_dynamic_state = {
1487    .viewport =
1488      {
1489        .count = 0,
1490      },
1491    .scissor =
1492      {
1493        .count = 0,
1494      },
1495    .line_width = 1.0f,
1496    .depth_bias =
1497      {
1498        .bias = 0.0f,
1499        .clamp = 0.0f,
1500        .slope = 0.0f,
1501      },
1502    .blend_constants = { 0.0f, 0.0f, 0.0f, 0.0f },
1503    .depth_bounds =
1504      {
1505        .min = 0.0f,
1506        .max = 1.0f,
1507      },
1508    .stencil_compare_mask =
1509      {
1510        .front = ~0u,
1511        .back = ~0u,
1512      },
1513    .stencil_write_mask =
1514      {
1515        .front = ~0u,
1516        .back = ~0u,
1517      },
1518    .stencil_reference =
1519      {
1520        .front = 0u,
1521        .back = 0u,
1522      },
1523 };
1524
1525 static void UNUSED /* FINISHME */
1526 tu_bind_dynamic_state(struct tu_cmd_buffer *cmd_buffer,
1527                       const struct tu_dynamic_state *src)
1528 {
1529    struct tu_dynamic_state *dest = &cmd_buffer->state.dynamic;
1530    uint32_t copy_mask = src->mask;
1531    uint32_t dest_mask = 0;
1532
1533    tu_use_args(cmd_buffer); /* FINISHME */
1534
1535    /* Make sure to copy the number of viewports/scissors because they can
1536     * only be specified at pipeline creation time.
1537     */
1538    dest->viewport.count = src->viewport.count;
1539    dest->scissor.count = src->scissor.count;
1540    dest->discard_rectangle.count = src->discard_rectangle.count;
1541
1542    if (copy_mask & TU_DYNAMIC_VIEWPORT) {
1543       if (memcmp(&dest->viewport.viewports, &src->viewport.viewports,
1544                  src->viewport.count * sizeof(VkViewport))) {
1545          typed_memcpy(dest->viewport.viewports, src->viewport.viewports,
1546                       src->viewport.count);
1547          dest_mask |= TU_DYNAMIC_VIEWPORT;
1548       }
1549    }
1550
1551    if (copy_mask & TU_DYNAMIC_SCISSOR) {
1552       if (memcmp(&dest->scissor.scissors, &src->scissor.scissors,
1553                  src->scissor.count * sizeof(VkRect2D))) {
1554          typed_memcpy(dest->scissor.scissors, src->scissor.scissors,
1555                       src->scissor.count);
1556          dest_mask |= TU_DYNAMIC_SCISSOR;
1557       }
1558    }
1559
1560    if (copy_mask & TU_DYNAMIC_LINE_WIDTH) {
1561       if (dest->line_width != src->line_width) {
1562          dest->line_width = src->line_width;
1563          dest_mask |= TU_DYNAMIC_LINE_WIDTH;
1564       }
1565    }
1566
1567    if (copy_mask & TU_DYNAMIC_DEPTH_BIAS) {
1568       if (memcmp(&dest->depth_bias, &src->depth_bias,
1569                  sizeof(src->depth_bias))) {
1570          dest->depth_bias = src->depth_bias;
1571          dest_mask |= TU_DYNAMIC_DEPTH_BIAS;
1572       }
1573    }
1574
1575    if (copy_mask & TU_DYNAMIC_BLEND_CONSTANTS) {
1576       if (memcmp(&dest->blend_constants, &src->blend_constants,
1577                  sizeof(src->blend_constants))) {
1578          typed_memcpy(dest->blend_constants, src->blend_constants, 4);
1579          dest_mask |= TU_DYNAMIC_BLEND_CONSTANTS;
1580       }
1581    }
1582
1583    if (copy_mask & TU_DYNAMIC_DEPTH_BOUNDS) {
1584       if (memcmp(&dest->depth_bounds, &src->depth_bounds,
1585                  sizeof(src->depth_bounds))) {
1586          dest->depth_bounds = src->depth_bounds;
1587          dest_mask |= TU_DYNAMIC_DEPTH_BOUNDS;
1588       }
1589    }
1590
1591    if (copy_mask & TU_DYNAMIC_STENCIL_COMPARE_MASK) {
1592       if (memcmp(&dest->stencil_compare_mask, &src->stencil_compare_mask,
1593                  sizeof(src->stencil_compare_mask))) {
1594          dest->stencil_compare_mask = src->stencil_compare_mask;
1595          dest_mask |= TU_DYNAMIC_STENCIL_COMPARE_MASK;
1596       }
1597    }
1598
1599    if (copy_mask & TU_DYNAMIC_STENCIL_WRITE_MASK) {
1600       if (memcmp(&dest->stencil_write_mask, &src->stencil_write_mask,
1601                  sizeof(src->stencil_write_mask))) {
1602          dest->stencil_write_mask = src->stencil_write_mask;
1603          dest_mask |= TU_DYNAMIC_STENCIL_WRITE_MASK;
1604       }
1605    }
1606
1607    if (copy_mask & TU_DYNAMIC_STENCIL_REFERENCE) {
1608       if (memcmp(&dest->stencil_reference, &src->stencil_reference,
1609                  sizeof(src->stencil_reference))) {
1610          dest->stencil_reference = src->stencil_reference;
1611          dest_mask |= TU_DYNAMIC_STENCIL_REFERENCE;
1612       }
1613    }
1614
1615    if (copy_mask & TU_DYNAMIC_DISCARD_RECTANGLE) {
1616       if (memcmp(&dest->discard_rectangle.rectangles,
1617                  &src->discard_rectangle.rectangles,
1618                  src->discard_rectangle.count * sizeof(VkRect2D))) {
1619          typed_memcpy(dest->discard_rectangle.rectangles,
1620                       src->discard_rectangle.rectangles,
1621                       src->discard_rectangle.count);
1622          dest_mask |= TU_DYNAMIC_DISCARD_RECTANGLE;
1623       }
1624    }
1625 }
1626
1627 static VkResult
1628 tu_create_cmd_buffer(struct tu_device *device,
1629                      struct tu_cmd_pool *pool,
1630                      VkCommandBufferLevel level,
1631                      VkCommandBuffer *pCommandBuffer)
1632 {
1633    struct tu_cmd_buffer *cmd_buffer;
1634    cmd_buffer = vk_zalloc(&pool->alloc, sizeof(*cmd_buffer), 8,
1635                           VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1636    if (cmd_buffer == NULL)
1637       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
1638
1639    cmd_buffer->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
1640    cmd_buffer->device = device;
1641    cmd_buffer->pool = pool;
1642    cmd_buffer->level = level;
1643
1644    if (pool) {
1645       list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers);
1646       cmd_buffer->queue_family_index = pool->queue_family_index;
1647
1648    } else {
1649       /* Init the pool_link so we can safely call list_del when we destroy
1650        * the command buffer
1651        */
1652       list_inithead(&cmd_buffer->pool_link);
1653       cmd_buffer->queue_family_index = TU_QUEUE_GENERAL;
1654    }
1655
1656    tu_bo_list_init(&cmd_buffer->bo_list);
1657    tu_cs_init(&cmd_buffer->cs, TU_CS_MODE_GROW, 4096);
1658    tu_cs_init(&cmd_buffer->draw_cs, TU_CS_MODE_GROW, 4096);
1659    tu_cs_init(&cmd_buffer->draw_epilogue_cs, TU_CS_MODE_GROW, 4096);
1660    tu_cs_init(&cmd_buffer->sub_cs, TU_CS_MODE_SUB_STREAM, 2048);
1661
1662    *pCommandBuffer = tu_cmd_buffer_to_handle(cmd_buffer);
1663
1664    list_inithead(&cmd_buffer->upload.list);
1665
1666    cmd_buffer->marker_reg = REG_A6XX_CP_SCRATCH_REG(
1667       cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY ? 7 : 6);
1668
1669    VkResult result = tu_bo_init_new(device, &cmd_buffer->scratch_bo, 0x1000);
1670    if (result != VK_SUCCESS)
1671       goto fail_scratch_bo;
1672
1673 #define VSC_DATA_SIZE(pitch)  ((pitch) * 32 + 0x100)  /* extra size to store VSC_SIZE */
1674 #define VSC_DATA2_SIZE(pitch) ((pitch) * 32)
1675
1676    /* TODO: resize on overflow or compute a max size from # of vertices in renderpass?? */
1677    cmd_buffer->vsc_data_pitch = 0x440 * 4;
1678    cmd_buffer->vsc_data2_pitch = 0x1040 * 4;
1679
1680    result = tu_bo_init_new(device, &cmd_buffer->vsc_data, VSC_DATA_SIZE(cmd_buffer->vsc_data_pitch));
1681    if (result != VK_SUCCESS)
1682       goto fail_vsc_data;
1683
1684    result = tu_bo_init_new(device, &cmd_buffer->vsc_data2, VSC_DATA2_SIZE(cmd_buffer->vsc_data2_pitch));
1685    if (result != VK_SUCCESS)
1686       goto fail_vsc_data2;
1687
1688    return VK_SUCCESS;
1689
1690 fail_vsc_data2:
1691    tu_bo_finish(cmd_buffer->device, &cmd_buffer->vsc_data);
1692 fail_vsc_data:
1693    tu_bo_finish(cmd_buffer->device, &cmd_buffer->scratch_bo);
1694 fail_scratch_bo:
1695    list_del(&cmd_buffer->pool_link);
1696    return result;
1697 }
1698
1699 static void
1700 tu_cmd_buffer_destroy(struct tu_cmd_buffer *cmd_buffer)
1701 {
1702    tu_bo_finish(cmd_buffer->device, &cmd_buffer->scratch_bo);
1703    tu_bo_finish(cmd_buffer->device, &cmd_buffer->vsc_data);
1704    tu_bo_finish(cmd_buffer->device, &cmd_buffer->vsc_data2);
1705
1706    list_del(&cmd_buffer->pool_link);
1707
1708    for (unsigned i = 0; i < VK_PIPELINE_BIND_POINT_RANGE_SIZE; i++)
1709       free(cmd_buffer->descriptors[i].push_set.set.mapped_ptr);
1710
1711    tu_cs_finish(cmd_buffer->device, &cmd_buffer->cs);
1712    tu_cs_finish(cmd_buffer->device, &cmd_buffer->draw_cs);
1713    tu_cs_finish(cmd_buffer->device, &cmd_buffer->draw_epilogue_cs);
1714    tu_cs_finish(cmd_buffer->device, &cmd_buffer->sub_cs);
1715
1716    tu_bo_list_destroy(&cmd_buffer->bo_list);
1717    vk_free(&cmd_buffer->pool->alloc, cmd_buffer);
1718 }
1719
1720 static VkResult
1721 tu_reset_cmd_buffer(struct tu_cmd_buffer *cmd_buffer)
1722 {
1723    cmd_buffer->wait_for_idle = true;
1724
1725    cmd_buffer->record_result = VK_SUCCESS;
1726
1727    tu_bo_list_reset(&cmd_buffer->bo_list);
1728    tu_cs_reset(cmd_buffer->device, &cmd_buffer->cs);
1729    tu_cs_reset(cmd_buffer->device, &cmd_buffer->draw_cs);
1730    tu_cs_reset(cmd_buffer->device, &cmd_buffer->draw_epilogue_cs);
1731    tu_cs_reset(cmd_buffer->device, &cmd_buffer->sub_cs);
1732
1733    for (unsigned i = 0; i < VK_PIPELINE_BIND_POINT_RANGE_SIZE; i++) {
1734       cmd_buffer->descriptors[i].valid = 0;
1735       cmd_buffer->descriptors[i].push_dirty = false;
1736    }
1737
1738    cmd_buffer->status = TU_CMD_BUFFER_STATUS_INITIAL;
1739
1740    return cmd_buffer->record_result;
1741 }
1742
1743 VkResult
1744 tu_AllocateCommandBuffers(VkDevice _device,
1745                           const VkCommandBufferAllocateInfo *pAllocateInfo,
1746                           VkCommandBuffer *pCommandBuffers)
1747 {
1748    TU_FROM_HANDLE(tu_device, device, _device);
1749    TU_FROM_HANDLE(tu_cmd_pool, pool, pAllocateInfo->commandPool);
1750
1751    VkResult result = VK_SUCCESS;
1752    uint32_t i;
1753
1754    for (i = 0; i < pAllocateInfo->commandBufferCount; i++) {
1755
1756       if (!list_is_empty(&pool->free_cmd_buffers)) {
1757          struct tu_cmd_buffer *cmd_buffer = list_first_entry(
1758             &pool->free_cmd_buffers, struct tu_cmd_buffer, pool_link);
1759
1760          list_del(&cmd_buffer->pool_link);
1761          list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers);
1762
1763          result = tu_reset_cmd_buffer(cmd_buffer);
1764          cmd_buffer->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
1765          cmd_buffer->level = pAllocateInfo->level;
1766
1767          pCommandBuffers[i] = tu_cmd_buffer_to_handle(cmd_buffer);
1768       } else {
1769          result = tu_create_cmd_buffer(device, pool, pAllocateInfo->level,
1770                                        &pCommandBuffers[i]);
1771       }
1772       if (result != VK_SUCCESS)
1773          break;
1774    }
1775
1776    if (result != VK_SUCCESS) {
1777       tu_FreeCommandBuffers(_device, pAllocateInfo->commandPool, i,
1778                             pCommandBuffers);
1779
1780       /* From the Vulkan 1.0.66 spec:
1781        *
1782        * "vkAllocateCommandBuffers can be used to create multiple
1783        *  command buffers. If the creation of any of those command
1784        *  buffers fails, the implementation must destroy all
1785        *  successfully created command buffer objects from this
1786        *  command, set all entries of the pCommandBuffers array to
1787        *  NULL and return the error."
1788        */
1789       memset(pCommandBuffers, 0,
1790              sizeof(*pCommandBuffers) * pAllocateInfo->commandBufferCount);
1791    }
1792
1793    return result;
1794 }
1795
1796 void
1797 tu_FreeCommandBuffers(VkDevice device,
1798                       VkCommandPool commandPool,
1799                       uint32_t commandBufferCount,
1800                       const VkCommandBuffer *pCommandBuffers)
1801 {
1802    for (uint32_t i = 0; i < commandBufferCount; i++) {
1803       TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, pCommandBuffers[i]);
1804
1805       if (cmd_buffer) {
1806          if (cmd_buffer->pool) {
1807             list_del(&cmd_buffer->pool_link);
1808             list_addtail(&cmd_buffer->pool_link,
1809                          &cmd_buffer->pool->free_cmd_buffers);
1810          } else
1811             tu_cmd_buffer_destroy(cmd_buffer);
1812       }
1813    }
1814 }
1815
1816 VkResult
1817 tu_ResetCommandBuffer(VkCommandBuffer commandBuffer,
1818                       VkCommandBufferResetFlags flags)
1819 {
1820    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
1821    return tu_reset_cmd_buffer(cmd_buffer);
1822 }
1823
1824 VkResult
1825 tu_BeginCommandBuffer(VkCommandBuffer commandBuffer,
1826                       const VkCommandBufferBeginInfo *pBeginInfo)
1827 {
1828    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
1829    VkResult result = VK_SUCCESS;
1830
1831    if (cmd_buffer->status != TU_CMD_BUFFER_STATUS_INITIAL) {
1832       /* If the command buffer has already been resetted with
1833        * vkResetCommandBuffer, no need to do it again.
1834        */
1835       result = tu_reset_cmd_buffer(cmd_buffer);
1836       if (result != VK_SUCCESS)
1837          return result;
1838    }
1839
1840    memset(&cmd_buffer->state, 0, sizeof(cmd_buffer->state));
1841    cmd_buffer->usage_flags = pBeginInfo->flags;
1842
1843    tu_cs_begin(&cmd_buffer->cs);
1844    tu_cs_begin(&cmd_buffer->draw_cs);
1845    tu_cs_begin(&cmd_buffer->draw_epilogue_cs);
1846
1847    cmd_buffer->marker_seqno = 0;
1848    cmd_buffer->scratch_seqno = 0;
1849
1850    /* setup initial configuration into command buffer */
1851    if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY) {
1852       switch (cmd_buffer->queue_family_index) {
1853       case TU_QUEUE_GENERAL:
1854          tu6_init_hw(cmd_buffer, &cmd_buffer->cs);
1855          break;
1856       default:
1857          break;
1858       }
1859    } else if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY &&
1860               (pBeginInfo->flags & VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT)) {
1861       assert(pBeginInfo->pInheritanceInfo);
1862       cmd_buffer->state.pass = tu_render_pass_from_handle(pBeginInfo->pInheritanceInfo->renderPass);
1863       cmd_buffer->state.subpass = &cmd_buffer->state.pass->subpasses[pBeginInfo->pInheritanceInfo->subpass];
1864    }
1865
1866    cmd_buffer->status = TU_CMD_BUFFER_STATUS_RECORDING;
1867
1868    return VK_SUCCESS;
1869 }
1870
1871 void
1872 tu_CmdBindVertexBuffers(VkCommandBuffer commandBuffer,
1873                         uint32_t firstBinding,
1874                         uint32_t bindingCount,
1875                         const VkBuffer *pBuffers,
1876                         const VkDeviceSize *pOffsets)
1877 {
1878    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
1879
1880    assert(firstBinding + bindingCount <= MAX_VBS);
1881
1882    for (uint32_t i = 0; i < bindingCount; i++) {
1883       cmd->state.vb.buffers[firstBinding + i] =
1884          tu_buffer_from_handle(pBuffers[i]);
1885       cmd->state.vb.offsets[firstBinding + i] = pOffsets[i];
1886    }
1887
1888    /* VB states depend on VkPipelineVertexInputStateCreateInfo */
1889    cmd->state.dirty |= TU_CMD_DIRTY_VERTEX_BUFFERS;
1890 }
1891
1892 void
1893 tu_CmdBindIndexBuffer(VkCommandBuffer commandBuffer,
1894                       VkBuffer buffer,
1895                       VkDeviceSize offset,
1896                       VkIndexType indexType)
1897 {
1898    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
1899    TU_FROM_HANDLE(tu_buffer, buf, buffer);
1900
1901    /* initialize/update the restart index */
1902    if (!cmd->state.index_buffer || cmd->state.index_type != indexType) {
1903       struct tu_cs *draw_cs = &cmd->draw_cs;
1904       VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 2);
1905       if (result != VK_SUCCESS) {
1906          cmd->record_result = result;
1907          return;
1908       }
1909
1910       tu6_emit_restart_index(
1911          draw_cs, indexType == VK_INDEX_TYPE_UINT32 ? 0xffffffff : 0xffff);
1912
1913       tu_cs_sanity_check(draw_cs);
1914    }
1915
1916    /* track the BO */
1917    if (cmd->state.index_buffer != buf)
1918       tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
1919
1920    cmd->state.index_buffer = buf;
1921    cmd->state.index_offset = offset;
1922    cmd->state.index_type = indexType;
1923 }
1924
1925 void
1926 tu_CmdBindDescriptorSets(VkCommandBuffer commandBuffer,
1927                          VkPipelineBindPoint pipelineBindPoint,
1928                          VkPipelineLayout _layout,
1929                          uint32_t firstSet,
1930                          uint32_t descriptorSetCount,
1931                          const VkDescriptorSet *pDescriptorSets,
1932                          uint32_t dynamicOffsetCount,
1933                          const uint32_t *pDynamicOffsets)
1934 {
1935    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
1936    TU_FROM_HANDLE(tu_pipeline_layout, layout, _layout);
1937    unsigned dyn_idx = 0;
1938
1939    struct tu_descriptor_state *descriptors_state =
1940       tu_get_descriptors_state(cmd_buffer, pipelineBindPoint);
1941
1942    for (unsigned i = 0; i < descriptorSetCount; ++i) {
1943       unsigned idx = i + firstSet;
1944       TU_FROM_HANDLE(tu_descriptor_set, set, pDescriptorSets[i]);
1945
1946       descriptors_state->sets[idx] = set;
1947       descriptors_state->valid |= (1u << idx);
1948
1949       for(unsigned j = 0; j < set->layout->dynamic_offset_count; ++j, ++dyn_idx) {
1950          unsigned idx = j + layout->set[i + firstSet].dynamic_offset_start;
1951          assert(dyn_idx < dynamicOffsetCount);
1952
1953          descriptors_state->dynamic_buffers[idx] =
1954          set->dynamic_descriptors[j].va + pDynamicOffsets[dyn_idx];
1955       }
1956    }
1957
1958    cmd_buffer->state.dirty |= TU_CMD_DIRTY_DESCRIPTOR_SETS;
1959 }
1960
1961 void
1962 tu_CmdPushConstants(VkCommandBuffer commandBuffer,
1963                     VkPipelineLayout layout,
1964                     VkShaderStageFlags stageFlags,
1965                     uint32_t offset,
1966                     uint32_t size,
1967                     const void *pValues)
1968 {
1969    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
1970    memcpy((void*) cmd->push_constants + offset, pValues, size);
1971    cmd->state.dirty |= TU_CMD_DIRTY_PUSH_CONSTANTS;
1972 }
1973
1974 VkResult
1975 tu_EndCommandBuffer(VkCommandBuffer commandBuffer)
1976 {
1977    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
1978
1979    if (cmd_buffer->scratch_seqno) {
1980       tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->scratch_bo,
1981                      MSM_SUBMIT_BO_WRITE);
1982    }
1983
1984    if (cmd_buffer->use_vsc_data) {
1985       tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->vsc_data,
1986                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
1987       tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->vsc_data2,
1988                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
1989    }
1990
1991    for (uint32_t i = 0; i < cmd_buffer->draw_cs.bo_count; i++) {
1992       tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->draw_cs.bos[i],
1993                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
1994    }
1995
1996    for (uint32_t i = 0; i < cmd_buffer->draw_epilogue_cs.bo_count; i++) {
1997       tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->draw_epilogue_cs.bos[i],
1998                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
1999    }
2000
2001    for (uint32_t i = 0; i < cmd_buffer->sub_cs.bo_count; i++) {
2002       tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->sub_cs.bos[i],
2003                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2004    }
2005
2006    tu_cs_end(&cmd_buffer->cs);
2007    tu_cs_end(&cmd_buffer->draw_cs);
2008    tu_cs_end(&cmd_buffer->draw_epilogue_cs);
2009
2010    cmd_buffer->status = TU_CMD_BUFFER_STATUS_EXECUTABLE;
2011
2012    return cmd_buffer->record_result;
2013 }
2014
2015 void
2016 tu_CmdBindPipeline(VkCommandBuffer commandBuffer,
2017                    VkPipelineBindPoint pipelineBindPoint,
2018                    VkPipeline _pipeline)
2019 {
2020    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2021    TU_FROM_HANDLE(tu_pipeline, pipeline, _pipeline);
2022
2023    switch (pipelineBindPoint) {
2024    case VK_PIPELINE_BIND_POINT_GRAPHICS:
2025       cmd->state.pipeline = pipeline;
2026       cmd->state.dirty |= TU_CMD_DIRTY_PIPELINE;
2027       break;
2028    case VK_PIPELINE_BIND_POINT_COMPUTE:
2029       cmd->state.compute_pipeline = pipeline;
2030       cmd->state.dirty |= TU_CMD_DIRTY_COMPUTE_PIPELINE;
2031       break;
2032    default:
2033       unreachable("unrecognized pipeline bind point");
2034       break;
2035    }
2036
2037    tu_bo_list_add(&cmd->bo_list, &pipeline->program.binary_bo,
2038                   MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2039    for (uint32_t i = 0; i < pipeline->cs.bo_count; i++) {
2040       tu_bo_list_add(&cmd->bo_list, pipeline->cs.bos[i],
2041                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2042    }
2043 }
2044
2045 void
2046 tu_CmdSetViewport(VkCommandBuffer commandBuffer,
2047                   uint32_t firstViewport,
2048                   uint32_t viewportCount,
2049                   const VkViewport *pViewports)
2050 {
2051    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2052    struct tu_cs *draw_cs = &cmd->draw_cs;
2053
2054    VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 12);
2055    if (result != VK_SUCCESS) {
2056       cmd->record_result = result;
2057       return;
2058    }
2059
2060    assert(firstViewport == 0 && viewportCount == 1);
2061    tu6_emit_viewport(draw_cs, pViewports);
2062
2063    tu_cs_sanity_check(draw_cs);
2064 }
2065
2066 void
2067 tu_CmdSetScissor(VkCommandBuffer commandBuffer,
2068                  uint32_t firstScissor,
2069                  uint32_t scissorCount,
2070                  const VkRect2D *pScissors)
2071 {
2072    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2073    struct tu_cs *draw_cs = &cmd->draw_cs;
2074
2075    VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 3);
2076    if (result != VK_SUCCESS) {
2077       cmd->record_result = result;
2078       return;
2079    }
2080
2081    assert(firstScissor == 0 && scissorCount == 1);
2082    tu6_emit_scissor(draw_cs, pScissors);
2083
2084    tu_cs_sanity_check(draw_cs);
2085 }
2086
2087 void
2088 tu_CmdSetLineWidth(VkCommandBuffer commandBuffer, float lineWidth)
2089 {
2090    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2091
2092    cmd->state.dynamic.line_width = lineWidth;
2093
2094    /* line width depends on VkPipelineRasterizationStateCreateInfo */
2095    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_LINE_WIDTH;
2096 }
2097
2098 void
2099 tu_CmdSetDepthBias(VkCommandBuffer commandBuffer,
2100                    float depthBiasConstantFactor,
2101                    float depthBiasClamp,
2102                    float depthBiasSlopeFactor)
2103 {
2104    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2105    struct tu_cs *draw_cs = &cmd->draw_cs;
2106
2107    VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 4);
2108    if (result != VK_SUCCESS) {
2109       cmd->record_result = result;
2110       return;
2111    }
2112
2113    tu6_emit_depth_bias(draw_cs, depthBiasConstantFactor, depthBiasClamp,
2114                        depthBiasSlopeFactor);
2115
2116    tu_cs_sanity_check(draw_cs);
2117 }
2118
2119 void
2120 tu_CmdSetBlendConstants(VkCommandBuffer commandBuffer,
2121                         const float blendConstants[4])
2122 {
2123    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2124    struct tu_cs *draw_cs = &cmd->draw_cs;
2125
2126    VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 5);
2127    if (result != VK_SUCCESS) {
2128       cmd->record_result = result;
2129       return;
2130    }
2131
2132    tu6_emit_blend_constants(draw_cs, blendConstants);
2133
2134    tu_cs_sanity_check(draw_cs);
2135 }
2136
2137 void
2138 tu_CmdSetDepthBounds(VkCommandBuffer commandBuffer,
2139                      float minDepthBounds,
2140                      float maxDepthBounds)
2141 {
2142 }
2143
2144 void
2145 tu_CmdSetStencilCompareMask(VkCommandBuffer commandBuffer,
2146                             VkStencilFaceFlags faceMask,
2147                             uint32_t compareMask)
2148 {
2149    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2150
2151    if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
2152       cmd->state.dynamic.stencil_compare_mask.front = compareMask;
2153    if (faceMask & VK_STENCIL_FACE_BACK_BIT)
2154       cmd->state.dynamic.stencil_compare_mask.back = compareMask;
2155
2156    /* the front/back compare masks must be updated together */
2157    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK;
2158 }
2159
2160 void
2161 tu_CmdSetStencilWriteMask(VkCommandBuffer commandBuffer,
2162                           VkStencilFaceFlags faceMask,
2163                           uint32_t writeMask)
2164 {
2165    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2166
2167    if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
2168       cmd->state.dynamic.stencil_write_mask.front = writeMask;
2169    if (faceMask & VK_STENCIL_FACE_BACK_BIT)
2170       cmd->state.dynamic.stencil_write_mask.back = writeMask;
2171
2172    /* the front/back write masks must be updated together */
2173    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK;
2174 }
2175
2176 void
2177 tu_CmdSetStencilReference(VkCommandBuffer commandBuffer,
2178                           VkStencilFaceFlags faceMask,
2179                           uint32_t reference)
2180 {
2181    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2182
2183    if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
2184       cmd->state.dynamic.stencil_reference.front = reference;
2185    if (faceMask & VK_STENCIL_FACE_BACK_BIT)
2186       cmd->state.dynamic.stencil_reference.back = reference;
2187
2188    /* the front/back references must be updated together */
2189    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE;
2190 }
2191
2192 void
2193 tu_CmdExecuteCommands(VkCommandBuffer commandBuffer,
2194                       uint32_t commandBufferCount,
2195                       const VkCommandBuffer *pCmdBuffers)
2196 {
2197    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2198    VkResult result;
2199
2200    assert(commandBufferCount > 0);
2201
2202    for (uint32_t i = 0; i < commandBufferCount; i++) {
2203       TU_FROM_HANDLE(tu_cmd_buffer, secondary, pCmdBuffers[i]);
2204
2205       result = tu_bo_list_merge(&cmd->bo_list, &secondary->bo_list);
2206       if (result != VK_SUCCESS) {
2207          cmd->record_result = result;
2208          break;
2209       }
2210
2211       result = tu_cs_add_entries(&cmd->draw_cs, &secondary->draw_cs);
2212       if (result != VK_SUCCESS) {
2213          cmd->record_result = result;
2214          break;
2215       }
2216
2217       result = tu_cs_add_entries(&cmd->draw_epilogue_cs,
2218             &secondary->draw_epilogue_cs);
2219       if (result != VK_SUCCESS) {
2220          cmd->record_result = result;
2221          break;
2222       }
2223    }
2224    cmd->state.dirty = ~0u; /* TODO: set dirty only what needs to be */
2225 }
2226
2227 VkResult
2228 tu_CreateCommandPool(VkDevice _device,
2229                      const VkCommandPoolCreateInfo *pCreateInfo,
2230                      const VkAllocationCallbacks *pAllocator,
2231                      VkCommandPool *pCmdPool)
2232 {
2233    TU_FROM_HANDLE(tu_device, device, _device);
2234    struct tu_cmd_pool *pool;
2235
2236    pool = vk_alloc2(&device->alloc, pAllocator, sizeof(*pool), 8,
2237                     VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2238    if (pool == NULL)
2239       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
2240
2241    if (pAllocator)
2242       pool->alloc = *pAllocator;
2243    else
2244       pool->alloc = device->alloc;
2245
2246    list_inithead(&pool->cmd_buffers);
2247    list_inithead(&pool->free_cmd_buffers);
2248
2249    pool->queue_family_index = pCreateInfo->queueFamilyIndex;
2250
2251    *pCmdPool = tu_cmd_pool_to_handle(pool);
2252
2253    return VK_SUCCESS;
2254 }
2255
2256 void
2257 tu_DestroyCommandPool(VkDevice _device,
2258                       VkCommandPool commandPool,
2259                       const VkAllocationCallbacks *pAllocator)
2260 {
2261    TU_FROM_HANDLE(tu_device, device, _device);
2262    TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2263
2264    if (!pool)
2265       return;
2266
2267    list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2268                             &pool->cmd_buffers, pool_link)
2269    {
2270       tu_cmd_buffer_destroy(cmd_buffer);
2271    }
2272
2273    list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2274                             &pool->free_cmd_buffers, pool_link)
2275    {
2276       tu_cmd_buffer_destroy(cmd_buffer);
2277    }
2278
2279    vk_free2(&device->alloc, pAllocator, pool);
2280 }
2281
2282 VkResult
2283 tu_ResetCommandPool(VkDevice device,
2284                     VkCommandPool commandPool,
2285                     VkCommandPoolResetFlags flags)
2286 {
2287    TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2288    VkResult result;
2289
2290    list_for_each_entry(struct tu_cmd_buffer, cmd_buffer, &pool->cmd_buffers,
2291                        pool_link)
2292    {
2293       result = tu_reset_cmd_buffer(cmd_buffer);
2294       if (result != VK_SUCCESS)
2295          return result;
2296    }
2297
2298    return VK_SUCCESS;
2299 }
2300
2301 void
2302 tu_TrimCommandPool(VkDevice device,
2303                    VkCommandPool commandPool,
2304                    VkCommandPoolTrimFlags flags)
2305 {
2306    TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2307
2308    if (!pool)
2309       return;
2310
2311    list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2312                             &pool->free_cmd_buffers, pool_link)
2313    {
2314       tu_cmd_buffer_destroy(cmd_buffer);
2315    }
2316 }
2317
2318 void
2319 tu_CmdBeginRenderPass(VkCommandBuffer commandBuffer,
2320                       const VkRenderPassBeginInfo *pRenderPassBegin,
2321                       VkSubpassContents contents)
2322 {
2323    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2324    TU_FROM_HANDLE(tu_render_pass, pass, pRenderPassBegin->renderPass);
2325    TU_FROM_HANDLE(tu_framebuffer, fb, pRenderPassBegin->framebuffer);
2326
2327    cmd->state.pass = pass;
2328    cmd->state.subpass = pass->subpasses;
2329    cmd->state.framebuffer = fb;
2330
2331    tu_cmd_update_tiling_config(cmd, &pRenderPassBegin->renderArea);
2332    tu_cmd_prepare_tile_load_ib(cmd, pRenderPassBegin);
2333    tu_cmd_prepare_tile_store_ib(cmd);
2334
2335    /* note: use_hw_binning only checks tiling config */
2336    if (use_hw_binning(cmd))
2337       cmd->use_vsc_data = true;
2338
2339    for (uint32_t i = 0; i < fb->attachment_count; ++i) {
2340       const struct tu_image_view *iview = fb->attachments[i].attachment;
2341       tu_bo_list_add(&cmd->bo_list, iview->image->bo,
2342                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
2343    }
2344 }
2345
2346 void
2347 tu_CmdBeginRenderPass2(VkCommandBuffer commandBuffer,
2348                        const VkRenderPassBeginInfo *pRenderPassBeginInfo,
2349                        const VkSubpassBeginInfoKHR *pSubpassBeginInfo)
2350 {
2351    tu_CmdBeginRenderPass(commandBuffer, pRenderPassBeginInfo,
2352                          pSubpassBeginInfo->contents);
2353 }
2354
2355 void
2356 tu_CmdNextSubpass(VkCommandBuffer commandBuffer, VkSubpassContents contents)
2357 {
2358    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2359    const struct tu_render_pass *pass = cmd->state.pass;
2360    struct tu_cs *cs = &cmd->draw_cs;
2361
2362    VkResult result = tu_cs_reserve_space(cmd->device, cs, 1024);
2363    if (result != VK_SUCCESS) {
2364       cmd->record_result = result;
2365       return;
2366    }
2367
2368    const struct tu_subpass *subpass = cmd->state.subpass++;
2369    /* TODO:
2370     * if msaa samples change between subpasses,
2371     * attachment store is broken for some attachments
2372     */
2373    if (subpass->resolve_attachments) {
2374       tu6_emit_blit_scissor(cmd, cs, true);
2375       for (unsigned i = 0; i < subpass->color_count; i++) {
2376          uint32_t a = subpass->resolve_attachments[i].attachment;
2377          if (a != VK_ATTACHMENT_UNUSED) {
2378                tu6_emit_store_attachment(cmd, cs, a,
2379                                          subpass->color_attachments[i].attachment);
2380          }
2381       }
2382    }
2383
2384    /* invalidate because reading input attachments will cache GMEM and
2385     * the cache isn''t updated when GMEM is written
2386     * TODO: is there a no-cache bit for textures?
2387     */
2388    if (cmd->state.subpass->input_count)
2389       tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE, false);
2390
2391    /* emit mrt/zs/msaa state for the subpass that is starting */
2392    tu6_emit_zs(cmd, cmd->state.subpass, cs);
2393    tu6_emit_mrt(cmd, cmd->state.subpass, cs);
2394    tu6_emit_msaa(cmd, cmd->state.subpass, cs);
2395
2396    /* TODO:
2397     * since we don't know how to do GMEM->GMEM resolve,
2398     * resolve attachments are resolved to memory then loaded to GMEM again if needed
2399     */
2400    if (subpass->resolve_attachments) {
2401       for (unsigned i = 0; i < subpass->color_count; i++) {
2402          uint32_t a = subpass->resolve_attachments[i].attachment;
2403          const struct tu_image_view *iview =
2404             cmd->state.framebuffer->attachments[a].attachment;
2405          if (a != VK_ATTACHMENT_UNUSED && pass->attachments[a].gmem_offset >= 0) {
2406                tu_finishme("missing GMEM->GMEM resolve, performance will suffer\n");
2407                tu6_emit_blit_info(cmd, cs, iview, pass->attachments[a].gmem_offset, false);
2408                tu6_emit_blit(cmd, cs);
2409          }
2410       }
2411    }
2412 }
2413
2414 void
2415 tu_CmdNextSubpass2(VkCommandBuffer commandBuffer,
2416                    const VkSubpassBeginInfoKHR *pSubpassBeginInfo,
2417                    const VkSubpassEndInfoKHR *pSubpassEndInfo)
2418 {
2419    tu_CmdNextSubpass(commandBuffer, pSubpassBeginInfo->contents);
2420 }
2421
2422 struct tu_draw_info
2423 {
2424    /**
2425     * Number of vertices.
2426     */
2427    uint32_t count;
2428
2429    /**
2430     * Index of the first vertex.
2431     */
2432    int32_t vertex_offset;
2433
2434    /**
2435     * First instance id.
2436     */
2437    uint32_t first_instance;
2438
2439    /**
2440     * Number of instances.
2441     */
2442    uint32_t instance_count;
2443
2444    /**
2445     * First index (indexed draws only).
2446     */
2447    uint32_t first_index;
2448
2449    /**
2450     * Whether it's an indexed draw.
2451     */
2452    bool indexed;
2453
2454    /**
2455     * Indirect draw parameters resource.
2456     */
2457    struct tu_buffer *indirect;
2458    uint64_t indirect_offset;
2459    uint32_t stride;
2460
2461    /**
2462     * Draw count parameters resource.
2463     */
2464    struct tu_buffer *count_buffer;
2465    uint64_t count_buffer_offset;
2466 };
2467
2468 #define ENABLE_ALL (CP_SET_DRAW_STATE__0_BINNING | CP_SET_DRAW_STATE__0_GMEM | CP_SET_DRAW_STATE__0_SYSMEM)
2469 #define ENABLE_DRAW (CP_SET_DRAW_STATE__0_GMEM | CP_SET_DRAW_STATE__0_SYSMEM)
2470
2471 enum tu_draw_state_group_id
2472 {
2473    TU_DRAW_STATE_PROGRAM,
2474    TU_DRAW_STATE_PROGRAM_BINNING,
2475    TU_DRAW_STATE_VI,
2476    TU_DRAW_STATE_VI_BINNING,
2477    TU_DRAW_STATE_VP,
2478    TU_DRAW_STATE_RAST,
2479    TU_DRAW_STATE_DS,
2480    TU_DRAW_STATE_BLEND,
2481    TU_DRAW_STATE_VS_CONST,
2482    TU_DRAW_STATE_FS_CONST,
2483    TU_DRAW_STATE_VS_TEX,
2484    TU_DRAW_STATE_FS_TEX,
2485    TU_DRAW_STATE_FS_IBO,
2486    TU_DRAW_STATE_VS_PARAMS,
2487
2488    TU_DRAW_STATE_COUNT,
2489 };
2490
2491 struct tu_draw_state_group
2492 {
2493    enum tu_draw_state_group_id id;
2494    uint32_t enable_mask;
2495    struct tu_cs_entry ib;
2496 };
2497
2498 const static struct tu_sampler*
2499 sampler_ptr(struct tu_descriptor_state *descriptors_state,
2500             const struct tu_descriptor_map *map, unsigned i,
2501             unsigned array_index)
2502 {
2503    assert(descriptors_state->valid & (1 << map->set[i]));
2504
2505    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2506    assert(map->binding[i] < set->layout->binding_count);
2507
2508    const struct tu_descriptor_set_binding_layout *layout =
2509       &set->layout->binding[map->binding[i]];
2510
2511    if (layout->immutable_samplers_offset) {
2512       const struct tu_sampler *immutable_samplers =
2513          tu_immutable_samplers(set->layout, layout);
2514
2515       return &immutable_samplers[array_index];
2516    }
2517
2518    switch (layout->type) {
2519    case VK_DESCRIPTOR_TYPE_SAMPLER:
2520       return (struct tu_sampler*) &set->mapped_ptr[layout->offset / 4];
2521    case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
2522       return (struct tu_sampler*) &set->mapped_ptr[layout->offset / 4 + A6XX_TEX_CONST_DWORDS +
2523                                                    array_index *
2524                                                    (A6XX_TEX_CONST_DWORDS +
2525                                                     sizeof(struct tu_sampler) / 4)];
2526    default:
2527       unreachable("unimplemented descriptor type");
2528       break;
2529    }
2530 }
2531
2532 static void
2533 write_tex_const(struct tu_cmd_buffer *cmd,
2534                 uint32_t *dst,
2535                 struct tu_descriptor_state *descriptors_state,
2536                 const struct tu_descriptor_map *map,
2537                 unsigned i, unsigned array_index)
2538 {
2539    assert(descriptors_state->valid & (1 << map->set[i]));
2540
2541    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2542    assert(map->binding[i] < set->layout->binding_count);
2543
2544    const struct tu_descriptor_set_binding_layout *layout =
2545       &set->layout->binding[map->binding[i]];
2546
2547    switch (layout->type) {
2548    case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
2549    case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
2550    case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
2551    case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
2552       memcpy(dst, &set->mapped_ptr[layout->offset / 4 +
2553                                    array_index * A6XX_TEX_CONST_DWORDS],
2554              A6XX_TEX_CONST_DWORDS * 4);
2555       break;
2556    case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
2557       memcpy(dst, &set->mapped_ptr[layout->offset / 4 +
2558                                    array_index *
2559                                    (A6XX_TEX_CONST_DWORDS +
2560                                     sizeof(struct tu_sampler) / 4)],
2561              A6XX_TEX_CONST_DWORDS * 4);
2562       break;
2563    default:
2564       unreachable("unimplemented descriptor type");
2565       break;
2566    }
2567
2568    if (layout->type == VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT) {
2569       const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
2570       uint32_t a = cmd->state.subpass->input_attachments[map->value[i] +
2571                                                          array_index].attachment;
2572       const struct tu_render_pass_attachment *att = &cmd->state.pass->attachments[a];
2573
2574       assert(att->gmem_offset >= 0);
2575
2576       dst[0] &= ~(A6XX_TEX_CONST_0_SWAP__MASK | A6XX_TEX_CONST_0_TILE_MODE__MASK);
2577       dst[0] |= A6XX_TEX_CONST_0_TILE_MODE(TILE6_2);
2578       dst[2] &= ~(A6XX_TEX_CONST_2_TYPE__MASK | A6XX_TEX_CONST_2_PITCH__MASK);
2579       dst[2] |=
2580          A6XX_TEX_CONST_2_TYPE(A6XX_TEX_2D) |
2581          A6XX_TEX_CONST_2_PITCH(tiling->tile0.extent.width * att->cpp);
2582       dst[3] = 0;
2583       dst[4] = 0x100000 + att->gmem_offset;
2584       dst[5] = A6XX_TEX_CONST_5_DEPTH(1);
2585       for (unsigned i = 6; i < A6XX_TEX_CONST_DWORDS; i++)
2586          dst[i] = 0;
2587
2588       if (cmd->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY)
2589          tu_finishme("patch input attachment pitch for secondary cmd buffer");
2590    }
2591 }
2592
2593 static void
2594 write_image_ibo(struct tu_cmd_buffer *cmd,
2595                 uint32_t *dst,
2596                 struct tu_descriptor_state *descriptors_state,
2597                 const struct tu_descriptor_map *map,
2598                 unsigned i, unsigned array_index)
2599 {
2600    assert(descriptors_state->valid & (1 << map->set[i]));
2601
2602    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2603    assert(map->binding[i] < set->layout->binding_count);
2604
2605    const struct tu_descriptor_set_binding_layout *layout =
2606       &set->layout->binding[map->binding[i]];
2607
2608    assert(layout->type == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE);
2609
2610    memcpy(dst, &set->mapped_ptr[layout->offset / 4 +
2611                                 (array_index * 2 + 1) * A6XX_TEX_CONST_DWORDS],
2612           A6XX_TEX_CONST_DWORDS * 4);
2613 }
2614
2615 static uint64_t
2616 buffer_ptr(struct tu_descriptor_state *descriptors_state,
2617            const struct tu_descriptor_map *map,
2618            unsigned i, unsigned array_index)
2619 {
2620    assert(descriptors_state->valid & (1 << map->set[i]));
2621
2622    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2623    assert(map->binding[i] < set->layout->binding_count);
2624
2625    const struct tu_descriptor_set_binding_layout *layout =
2626       &set->layout->binding[map->binding[i]];
2627
2628    switch (layout->type) {
2629    case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
2630    case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
2631       return descriptors_state->dynamic_buffers[layout->dynamic_offset_offset +
2632                                                 array_index];
2633    case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
2634    case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
2635       return (uint64_t) set->mapped_ptr[layout->offset / 4 + array_index * 2 + 1] << 32 |
2636                         set->mapped_ptr[layout->offset / 4 + array_index * 2];
2637    default:
2638       unreachable("unimplemented descriptor type");
2639       break;
2640    }
2641 }
2642
2643 static inline uint32_t
2644 tu6_stage2opcode(gl_shader_stage type)
2645 {
2646    switch (type) {
2647    case MESA_SHADER_VERTEX:
2648    case MESA_SHADER_TESS_CTRL:
2649    case MESA_SHADER_TESS_EVAL:
2650    case MESA_SHADER_GEOMETRY:
2651       return CP_LOAD_STATE6_GEOM;
2652    case MESA_SHADER_FRAGMENT:
2653    case MESA_SHADER_COMPUTE:
2654    case MESA_SHADER_KERNEL:
2655       return CP_LOAD_STATE6_FRAG;
2656    default:
2657       unreachable("bad shader type");
2658    }
2659 }
2660
2661 static inline enum a6xx_state_block
2662 tu6_stage2shadersb(gl_shader_stage type)
2663 {
2664    switch (type) {
2665    case MESA_SHADER_VERTEX:
2666       return SB6_VS_SHADER;
2667    case MESA_SHADER_FRAGMENT:
2668       return SB6_FS_SHADER;
2669    case MESA_SHADER_COMPUTE:
2670    case MESA_SHADER_KERNEL:
2671       return SB6_CS_SHADER;
2672    default:
2673       unreachable("bad shader type");
2674       return ~0;
2675    }
2676 }
2677
2678 static void
2679 tu6_emit_user_consts(struct tu_cs *cs, const struct tu_pipeline *pipeline,
2680                      struct tu_descriptor_state *descriptors_state,
2681                      gl_shader_stage type,
2682                      uint32_t *push_constants)
2683 {
2684    const struct tu_program_descriptor_linkage *link =
2685       &pipeline->program.link[type];
2686    const struct ir3_ubo_analysis_state *state = &link->ubo_state;
2687
2688    for (uint32_t i = 0; i < ARRAY_SIZE(state->range); i++) {
2689       if (state->range[i].start < state->range[i].end) {
2690          uint32_t size = state->range[i].end - state->range[i].start;
2691          uint32_t offset = state->range[i].start;
2692
2693          /* and even if the start of the const buffer is before
2694           * first_immediate, the end may not be:
2695           */
2696          size = MIN2(size, (16 * link->constlen) - state->range[i].offset);
2697
2698          if (size == 0)
2699             continue;
2700
2701          /* things should be aligned to vec4: */
2702          debug_assert((state->range[i].offset % 16) == 0);
2703          debug_assert((size % 16) == 0);
2704          debug_assert((offset % 16) == 0);
2705
2706          if (i == 0) {
2707             /* push constants */
2708             tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + (size / 4));
2709             tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(state->range[i].offset / 16) |
2710                   CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2711                   CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
2712                   CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2713                   CP_LOAD_STATE6_0_NUM_UNIT(size / 16));
2714             tu_cs_emit(cs, 0);
2715             tu_cs_emit(cs, 0);
2716             for (unsigned i = 0; i < size / 4; i++)
2717                tu_cs_emit(cs, push_constants[i + offset / 4]);
2718             continue;
2719          }
2720
2721          /* Look through the UBO map to find our UBO index, and get the VA for
2722           * that UBO.
2723           */
2724          uint64_t va = 0;
2725          uint32_t ubo_idx = i - 1;
2726          uint32_t ubo_map_base = 0;
2727          for (int j = 0; j < link->ubo_map.num; j++) {
2728             if (ubo_idx >= ubo_map_base &&
2729                 ubo_idx < ubo_map_base + link->ubo_map.array_size[j]) {
2730                va = buffer_ptr(descriptors_state, &link->ubo_map, j,
2731                                ubo_idx - ubo_map_base);
2732                break;
2733             }
2734             ubo_map_base += link->ubo_map.array_size[j];
2735          }
2736          assert(va);
2737
2738          tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3);
2739          tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(state->range[i].offset / 16) |
2740                CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2741                CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
2742                CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2743                CP_LOAD_STATE6_0_NUM_UNIT(size / 16));
2744          tu_cs_emit_qw(cs, va + offset);
2745       }
2746    }
2747 }
2748
2749 static void
2750 tu6_emit_ubos(struct tu_cs *cs, const struct tu_pipeline *pipeline,
2751               struct tu_descriptor_state *descriptors_state,
2752               gl_shader_stage type)
2753 {
2754    const struct tu_program_descriptor_linkage *link =
2755       &pipeline->program.link[type];
2756
2757    uint32_t num = MIN2(link->ubo_map.num_desc, link->const_state.num_ubos);
2758    uint32_t anum = align(num, 2);
2759
2760    if (!num)
2761       return;
2762
2763    tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + (2 * anum));
2764    tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(link->const_state.offsets.ubo) |
2765          CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2766          CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
2767          CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2768          CP_LOAD_STATE6_0_NUM_UNIT(anum/2));
2769    tu_cs_emit(cs, CP_LOAD_STATE6_1_EXT_SRC_ADDR(0));
2770    tu_cs_emit(cs, CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI(0));
2771
2772    unsigned emitted = 0;
2773    for (unsigned i = 0; emitted < num && i < link->ubo_map.num; i++) {
2774       for (unsigned j = 0; emitted < num && j < link->ubo_map.array_size[i]; j++) {
2775          tu_cs_emit_qw(cs, buffer_ptr(descriptors_state, &link->ubo_map, i, j));
2776          emitted++;
2777       }
2778    }
2779
2780    for (; emitted < anum; emitted++) {
2781       tu_cs_emit(cs, 0xffffffff);
2782       tu_cs_emit(cs, 0xffffffff);
2783    }
2784 }
2785
2786 static struct tu_cs_entry
2787 tu6_emit_consts(struct tu_cmd_buffer *cmd,
2788                 const struct tu_pipeline *pipeline,
2789                 struct tu_descriptor_state *descriptors_state,
2790                 gl_shader_stage type)
2791 {
2792    struct tu_cs cs;
2793    tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs, 512, &cs); /* TODO: maximum size? */
2794
2795    tu6_emit_user_consts(&cs, pipeline, descriptors_state, type, cmd->push_constants);
2796    tu6_emit_ubos(&cs, pipeline, descriptors_state, type);
2797
2798    return tu_cs_end_sub_stream(&cmd->sub_cs, &cs);
2799 }
2800
2801 static VkResult
2802 tu6_emit_vs_params(struct tu_cmd_buffer *cmd,
2803                    const struct tu_draw_info *draw,
2804                    struct tu_cs_entry *entry)
2805 {
2806    /* TODO: fill out more than just base instance */
2807    const struct tu_program_descriptor_linkage *link =
2808       &cmd->state.pipeline->program.link[MESA_SHADER_VERTEX];
2809    const struct ir3_const_state *const_state = &link->const_state;
2810    struct tu_cs cs;
2811
2812    if (const_state->offsets.driver_param >= link->constlen) {
2813       *entry = (struct tu_cs_entry) {};
2814       return VK_SUCCESS;
2815    }
2816
2817    VkResult result = tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs, 8, &cs);
2818    if (result != VK_SUCCESS)
2819       return result;
2820
2821    tu_cs_emit_pkt7(&cs, CP_LOAD_STATE6_GEOM, 3 + 4);
2822    tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(const_state->offsets.driver_param) |
2823          CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2824          CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
2825          CP_LOAD_STATE6_0_STATE_BLOCK(SB6_VS_SHADER) |
2826          CP_LOAD_STATE6_0_NUM_UNIT(1));
2827    tu_cs_emit(&cs, 0);
2828    tu_cs_emit(&cs, 0);
2829
2830    STATIC_ASSERT(IR3_DP_INSTID_BASE == 2);
2831
2832    tu_cs_emit(&cs, 0);
2833    tu_cs_emit(&cs, 0);
2834    tu_cs_emit(&cs, draw->first_instance);
2835    tu_cs_emit(&cs, 0);
2836
2837    *entry = tu_cs_end_sub_stream(&cmd->sub_cs, &cs);
2838    return VK_SUCCESS;
2839 }
2840
2841 static VkResult
2842 tu6_emit_textures(struct tu_cmd_buffer *cmd,
2843                   const struct tu_pipeline *pipeline,
2844                   struct tu_descriptor_state *descriptors_state,
2845                   gl_shader_stage type,
2846                   struct tu_cs_entry *entry,
2847                   bool *needs_border)
2848 {
2849    struct tu_device *device = cmd->device;
2850    struct tu_cs *draw_state = &cmd->sub_cs;
2851    const struct tu_program_descriptor_linkage *link =
2852       &pipeline->program.link[type];
2853    VkResult result;
2854
2855    if (link->texture_map.num_desc == 0 && link->sampler_map.num_desc == 0) {
2856       *entry = (struct tu_cs_entry) {};
2857       return VK_SUCCESS;
2858    }
2859
2860    /* allocate and fill texture state */
2861    struct ts_cs_memory tex_const;
2862    result = tu_cs_alloc(device, draw_state, link->texture_map.num_desc,
2863                         A6XX_TEX_CONST_DWORDS, &tex_const);
2864    if (result != VK_SUCCESS)
2865       return result;
2866
2867    int tex_index = 0;
2868    for (unsigned i = 0; i < link->texture_map.num; i++) {
2869       for (int j = 0; j < link->texture_map.array_size[i]; j++) {
2870          write_tex_const(cmd,
2871                          &tex_const.map[A6XX_TEX_CONST_DWORDS * tex_index++],
2872                          descriptors_state, &link->texture_map, i, j);
2873       }
2874    }
2875
2876    /* allocate and fill sampler state */
2877    struct ts_cs_memory tex_samp = { 0 };
2878    if (link->sampler_map.num_desc) {
2879       result = tu_cs_alloc(device, draw_state, link->sampler_map.num_desc,
2880                            A6XX_TEX_SAMP_DWORDS, &tex_samp);
2881       if (result != VK_SUCCESS)
2882          return result;
2883
2884       int sampler_index = 0;
2885       for (unsigned i = 0; i < link->sampler_map.num; i++) {
2886          for (int j = 0; j < link->sampler_map.array_size[i]; j++) {
2887             const struct tu_sampler *sampler = sampler_ptr(descriptors_state,
2888                                                            &link->sampler_map,
2889                                                            i, j);
2890             memcpy(&tex_samp.map[A6XX_TEX_SAMP_DWORDS * sampler_index++],
2891                    sampler->state, sizeof(sampler->state));
2892             *needs_border |= sampler->needs_border;
2893          }
2894       }
2895    }
2896
2897    unsigned tex_samp_reg, tex_const_reg, tex_count_reg;
2898    enum a6xx_state_block sb;
2899
2900    switch (type) {
2901    case MESA_SHADER_VERTEX:
2902       sb = SB6_VS_TEX;
2903       tex_samp_reg = REG_A6XX_SP_VS_TEX_SAMP_LO;
2904       tex_const_reg = REG_A6XX_SP_VS_TEX_CONST_LO;
2905       tex_count_reg = REG_A6XX_SP_VS_TEX_COUNT;
2906       break;
2907    case MESA_SHADER_FRAGMENT:
2908       sb = SB6_FS_TEX;
2909       tex_samp_reg = REG_A6XX_SP_FS_TEX_SAMP_LO;
2910       tex_const_reg = REG_A6XX_SP_FS_TEX_CONST_LO;
2911       tex_count_reg = REG_A6XX_SP_FS_TEX_COUNT;
2912       break;
2913    case MESA_SHADER_COMPUTE:
2914       sb = SB6_CS_TEX;
2915       tex_samp_reg = REG_A6XX_SP_CS_TEX_SAMP_LO;
2916       tex_const_reg = REG_A6XX_SP_CS_TEX_CONST_LO;
2917       tex_count_reg = REG_A6XX_SP_CS_TEX_COUNT;
2918       break;
2919    default:
2920       unreachable("bad state block");
2921    }
2922
2923    struct tu_cs cs;
2924    result = tu_cs_begin_sub_stream(device, draw_state, 16, &cs);
2925    if (result != VK_SUCCESS)
2926       return result;
2927
2928    if (link->sampler_map.num_desc) {
2929       /* output sampler state: */
2930       tu_cs_emit_pkt7(&cs, tu6_stage2opcode(type), 3);
2931       tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
2932                  CP_LOAD_STATE6_0_STATE_TYPE(ST6_SHADER) |
2933                  CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
2934                  CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
2935                  CP_LOAD_STATE6_0_NUM_UNIT(link->sampler_map.num_desc));
2936       tu_cs_emit_qw(&cs, tex_samp.iova); /* SRC_ADDR_LO/HI */
2937
2938       tu_cs_emit_pkt4(&cs, tex_samp_reg, 2);
2939       tu_cs_emit_qw(&cs, tex_samp.iova); /* SRC_ADDR_LO/HI */
2940    }
2941
2942    /* emit texture state: */
2943    tu_cs_emit_pkt7(&cs, tu6_stage2opcode(type), 3);
2944    tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
2945       CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2946       CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
2947       CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
2948       CP_LOAD_STATE6_0_NUM_UNIT(link->texture_map.num_desc));
2949    tu_cs_emit_qw(&cs, tex_const.iova); /* SRC_ADDR_LO/HI */
2950
2951    tu_cs_emit_pkt4(&cs, tex_const_reg, 2);
2952    tu_cs_emit_qw(&cs, tex_const.iova); /* SRC_ADDR_LO/HI */
2953
2954    tu_cs_emit_pkt4(&cs, tex_count_reg, 1);
2955    tu_cs_emit(&cs, link->texture_map.num_desc);
2956
2957    *entry = tu_cs_end_sub_stream(draw_state, &cs);
2958    return VK_SUCCESS;
2959 }
2960
2961 static VkResult
2962 tu6_emit_ibo(struct tu_cmd_buffer *cmd,
2963              const struct tu_pipeline *pipeline,
2964              struct tu_descriptor_state *descriptors_state,
2965              gl_shader_stage type,
2966              struct tu_cs_entry *entry)
2967 {
2968    struct tu_device *device = cmd->device;
2969    struct tu_cs *draw_state = &cmd->sub_cs;
2970    const struct tu_program_descriptor_linkage *link =
2971       &pipeline->program.link[type];
2972    VkResult result;
2973
2974    unsigned num_desc = link->ssbo_map.num_desc + link->image_map.num_desc;
2975
2976    if (num_desc == 0) {
2977       *entry = (struct tu_cs_entry) {};
2978       return VK_SUCCESS;
2979    }
2980
2981    struct ts_cs_memory ibo_const;
2982    result = tu_cs_alloc(device, draw_state, num_desc,
2983                         A6XX_TEX_CONST_DWORDS, &ibo_const);
2984    if (result != VK_SUCCESS)
2985       return result;
2986
2987    int ssbo_index = 0;
2988    for (unsigned i = 0; i < link->ssbo_map.num; i++) {
2989       for (int j = 0; j < link->ssbo_map.array_size[i]; j++) {
2990          uint32_t *dst = &ibo_const.map[A6XX_TEX_CONST_DWORDS * ssbo_index];
2991
2992          uint64_t va = buffer_ptr(descriptors_state, &link->ssbo_map, i, j);
2993          /* We don't expose robustBufferAccess, so leave the size unlimited. */
2994          uint32_t sz = MAX_STORAGE_BUFFER_RANGE / 4;
2995
2996          dst[0] = A6XX_IBO_0_FMT(TFMT6_32_UINT);
2997          dst[1] = A6XX_IBO_1_WIDTH(sz & MASK(15)) |
2998                   A6XX_IBO_1_HEIGHT(sz >> 15);
2999          dst[2] = A6XX_IBO_2_UNK4 |
3000                   A6XX_IBO_2_UNK31 |
3001                   A6XX_IBO_2_TYPE(A6XX_TEX_1D);
3002          dst[3] = 0;
3003          dst[4] = va;
3004          dst[5] = va >> 32;
3005          for (int i = 6; i < A6XX_TEX_CONST_DWORDS; i++)
3006             dst[i] = 0;
3007
3008          ssbo_index++;
3009       }
3010    }
3011
3012    for (unsigned i = 0; i < link->image_map.num; i++) {
3013       for (int j = 0; j < link->image_map.array_size[i]; j++) {
3014          uint32_t *dst = &ibo_const.map[A6XX_TEX_CONST_DWORDS * ssbo_index];
3015
3016          write_image_ibo(cmd, dst,
3017                          descriptors_state, &link->image_map, i, j);
3018
3019          ssbo_index++;
3020       }
3021    }
3022
3023    assert(ssbo_index == num_desc);
3024
3025    struct tu_cs cs;
3026    result = tu_cs_begin_sub_stream(device, draw_state, 7, &cs);
3027    if (result != VK_SUCCESS)
3028       return result;
3029
3030    uint32_t opcode, ibo_addr_reg;
3031    enum a6xx_state_block sb;
3032    enum a6xx_state_type st;
3033
3034    switch (type) {
3035    case MESA_SHADER_FRAGMENT:
3036       opcode = CP_LOAD_STATE6;
3037       st = ST6_SHADER;
3038       sb = SB6_IBO;
3039       ibo_addr_reg = REG_A6XX_SP_IBO_LO;
3040       break;
3041    case MESA_SHADER_COMPUTE:
3042       opcode = CP_LOAD_STATE6_FRAG;
3043       st = ST6_IBO;
3044       sb = SB6_CS_SHADER;
3045       ibo_addr_reg = REG_A6XX_SP_CS_IBO_LO;
3046       break;
3047    default:
3048       unreachable("unsupported stage for ibos");
3049    }
3050
3051    /* emit texture state: */
3052    tu_cs_emit_pkt7(&cs, opcode, 3);
3053    tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
3054               CP_LOAD_STATE6_0_STATE_TYPE(st) |
3055               CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
3056               CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
3057               CP_LOAD_STATE6_0_NUM_UNIT(num_desc));
3058    tu_cs_emit_qw(&cs, ibo_const.iova); /* SRC_ADDR_LO/HI */
3059
3060    tu_cs_emit_pkt4(&cs, ibo_addr_reg, 2);
3061    tu_cs_emit_qw(&cs, ibo_const.iova); /* SRC_ADDR_LO/HI */
3062
3063    *entry = tu_cs_end_sub_stream(draw_state, &cs);
3064    return VK_SUCCESS;
3065 }
3066
3067 struct PACKED bcolor_entry {
3068    uint32_t fp32[4];
3069    uint16_t ui16[4];
3070    int16_t  si16[4];
3071    uint16_t fp16[4];
3072    uint16_t rgb565;
3073    uint16_t rgb5a1;
3074    uint16_t rgba4;
3075    uint8_t __pad0[2];
3076    uint8_t  ui8[4];
3077    int8_t   si8[4];
3078    uint32_t rgb10a2;
3079    uint32_t z24; /* also s8? */
3080    uint16_t srgb[4];      /* appears to duplicate fp16[], but clamped, used for srgb */
3081    uint8_t  __pad1[56];
3082 } border_color[] = {
3083    [VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK] = {},
3084    [VK_BORDER_COLOR_INT_TRANSPARENT_BLACK] = {},
3085    [VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK] = {
3086       .fp32[3] = 0x3f800000,
3087       .ui16[3] = 0xffff,
3088       .si16[3] = 0x7fff,
3089       .fp16[3] = 0x3c00,
3090       .rgb5a1 = 0x8000,
3091       .rgba4 = 0xf000,
3092       .ui8[3] = 0xff,
3093       .si8[3] = 0x7f,
3094       .rgb10a2 = 0xc0000000,
3095       .srgb[3] = 0x3c00,
3096    },
3097    [VK_BORDER_COLOR_INT_OPAQUE_BLACK] = {
3098       .fp32[3] = 1,
3099       .fp16[3] = 1,
3100    },
3101    [VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE] = {
3102       .fp32[0 ... 3] = 0x3f800000,
3103       .ui16[0 ... 3] = 0xffff,
3104       .si16[0 ... 3] = 0x7fff,
3105       .fp16[0 ... 3] = 0x3c00,
3106       .rgb565 = 0xffff,
3107       .rgb5a1 = 0xffff,
3108       .rgba4 = 0xffff,
3109       .ui8[0 ... 3] = 0xff,
3110       .si8[0 ... 3] = 0x7f,
3111       .rgb10a2 = 0xffffffff,
3112       .z24 = 0xffffff,
3113       .srgb[0 ... 3] = 0x3c00,
3114    },
3115    [VK_BORDER_COLOR_INT_OPAQUE_WHITE] = {
3116       .fp32[0 ... 3] = 1,
3117       .fp16[0 ... 3] = 1,
3118    },
3119 };
3120
3121 static VkResult
3122 tu6_emit_border_color(struct tu_cmd_buffer *cmd,
3123                       struct tu_cs *cs)
3124 {
3125    STATIC_ASSERT(sizeof(struct bcolor_entry) == 128);
3126
3127    const struct tu_pipeline *pipeline = cmd->state.pipeline;
3128    struct tu_descriptor_state *descriptors_state =
3129       &cmd->descriptors[VK_PIPELINE_BIND_POINT_GRAPHICS];
3130    const struct tu_descriptor_map *vs_sampler =
3131       &pipeline->program.link[MESA_SHADER_VERTEX].sampler_map;
3132    const struct tu_descriptor_map *fs_sampler =
3133       &pipeline->program.link[MESA_SHADER_FRAGMENT].sampler_map;
3134    struct ts_cs_memory ptr;
3135
3136    VkResult result = tu_cs_alloc(cmd->device, &cmd->sub_cs,
3137                                  vs_sampler->num_desc + fs_sampler->num_desc,
3138                                  128 / 4,
3139                                  &ptr);
3140    if (result != VK_SUCCESS)
3141       return result;
3142
3143    for (unsigned i = 0; i < vs_sampler->num; i++) {
3144       for (unsigned j = 0; j < vs_sampler->array_size[i]; j++) {
3145          const struct tu_sampler *sampler = sampler_ptr(descriptors_state,
3146                                                         vs_sampler, i, j);
3147          memcpy(ptr.map, &border_color[sampler->border], 128);
3148          ptr.map += 128 / 4;
3149       }
3150    }
3151
3152    for (unsigned i = 0; i < fs_sampler->num; i++) {
3153       for (unsigned j = 0; j < fs_sampler->array_size[i]; j++) {
3154          const struct tu_sampler *sampler = sampler_ptr(descriptors_state,
3155                                                         fs_sampler, i, j);
3156          memcpy(ptr.map, &border_color[sampler->border], 128);
3157          ptr.map += 128 / 4;
3158       }
3159    }
3160
3161    tu_cs_emit_pkt4(cs, REG_A6XX_SP_TP_BORDER_COLOR_BASE_ADDR_LO, 2);
3162    tu_cs_emit_qw(cs, ptr.iova);
3163    return VK_SUCCESS;
3164 }
3165
3166 static VkResult
3167 tu6_bind_draw_states(struct tu_cmd_buffer *cmd,
3168                      struct tu_cs *cs,
3169                      const struct tu_draw_info *draw)
3170 {
3171    const struct tu_pipeline *pipeline = cmd->state.pipeline;
3172    const struct tu_dynamic_state *dynamic = &cmd->state.dynamic;
3173    struct tu_draw_state_group draw_state_groups[TU_DRAW_STATE_COUNT];
3174    uint32_t draw_state_group_count = 0;
3175
3176    struct tu_descriptor_state *descriptors_state =
3177       &cmd->descriptors[VK_PIPELINE_BIND_POINT_GRAPHICS];
3178
3179    VkResult result = tu_cs_reserve_space(cmd->device, cs, 256);
3180    if (result != VK_SUCCESS)
3181       return result;
3182
3183    /* TODO lrz */
3184
3185    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9806, 0);
3186    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9990, 0);
3187    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_UNKNOWN_A008, 0);
3188
3189    tu_cs_emit_regs(cs,
3190                    A6XX_PC_PRIMITIVE_CNTL_0(.primitive_restart =
3191                                             pipeline->ia.primitive_restart && draw->indexed));
3192
3193    if (cmd->state.dirty &
3194           (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_DYNAMIC_LINE_WIDTH) &&
3195        (pipeline->dynamic_state.mask & TU_DYNAMIC_LINE_WIDTH)) {
3196       tu6_emit_gras_su_cntl(cs, pipeline->rast.gras_su_cntl,
3197                             dynamic->line_width);
3198    }
3199
3200    if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK) &&
3201        (pipeline->dynamic_state.mask & TU_DYNAMIC_STENCIL_COMPARE_MASK)) {
3202       tu6_emit_stencil_compare_mask(cs, dynamic->stencil_compare_mask.front,
3203                                     dynamic->stencil_compare_mask.back);
3204    }
3205
3206    if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK) &&
3207        (pipeline->dynamic_state.mask & TU_DYNAMIC_STENCIL_WRITE_MASK)) {
3208       tu6_emit_stencil_write_mask(cs, dynamic->stencil_write_mask.front,
3209                                   dynamic->stencil_write_mask.back);
3210    }
3211
3212    if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE) &&
3213        (pipeline->dynamic_state.mask & TU_DYNAMIC_STENCIL_REFERENCE)) {
3214       tu6_emit_stencil_reference(cs, dynamic->stencil_reference.front,
3215                                  dynamic->stencil_reference.back);
3216    }
3217
3218    if (cmd->state.dirty &
3219        (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_VERTEX_BUFFERS)) {
3220       for (uint32_t i = 0; i < pipeline->vi.count; i++) {
3221          const uint32_t binding = pipeline->vi.bindings[i];
3222          const uint32_t stride = pipeline->vi.strides[i];
3223          const struct tu_buffer *buf = cmd->state.vb.buffers[binding];
3224          const VkDeviceSize offset = buf->bo_offset +
3225                                      cmd->state.vb.offsets[binding] +
3226                                      pipeline->vi.offsets[i];
3227          const VkDeviceSize size =
3228             offset < buf->bo->size ? buf->bo->size - offset : 0;
3229
3230          tu_cs_emit_regs(cs,
3231                          A6XX_VFD_FETCH_BASE(i, .bo = buf->bo, .bo_offset = offset),
3232                          A6XX_VFD_FETCH_SIZE(i, size),
3233                          A6XX_VFD_FETCH_STRIDE(i, stride));
3234       }
3235    }
3236
3237    if (cmd->state.dirty & TU_CMD_DIRTY_PIPELINE) {
3238       draw_state_groups[draw_state_group_count++] =
3239          (struct tu_draw_state_group) {
3240             .id = TU_DRAW_STATE_PROGRAM,
3241             .enable_mask = ENABLE_DRAW,
3242             .ib = pipeline->program.state_ib,
3243          };
3244       draw_state_groups[draw_state_group_count++] =
3245          (struct tu_draw_state_group) {
3246             .id = TU_DRAW_STATE_PROGRAM_BINNING,
3247             .enable_mask = CP_SET_DRAW_STATE__0_BINNING,
3248             .ib = pipeline->program.binning_state_ib,
3249          };
3250       draw_state_groups[draw_state_group_count++] =
3251          (struct tu_draw_state_group) {
3252             .id = TU_DRAW_STATE_VI,
3253             .enable_mask = ENABLE_DRAW,
3254             .ib = pipeline->vi.state_ib,
3255          };
3256       draw_state_groups[draw_state_group_count++] =
3257          (struct tu_draw_state_group) {
3258             .id = TU_DRAW_STATE_VI_BINNING,
3259             .enable_mask = CP_SET_DRAW_STATE__0_BINNING,
3260             .ib = pipeline->vi.binning_state_ib,
3261          };
3262       draw_state_groups[draw_state_group_count++] =
3263          (struct tu_draw_state_group) {
3264             .id = TU_DRAW_STATE_VP,
3265             .enable_mask = ENABLE_ALL,
3266             .ib = pipeline->vp.state_ib,
3267          };
3268       draw_state_groups[draw_state_group_count++] =
3269          (struct tu_draw_state_group) {
3270             .id = TU_DRAW_STATE_RAST,
3271             .enable_mask = ENABLE_ALL,
3272             .ib = pipeline->rast.state_ib,
3273          };
3274       draw_state_groups[draw_state_group_count++] =
3275          (struct tu_draw_state_group) {
3276             .id = TU_DRAW_STATE_DS,
3277             .enable_mask = ENABLE_ALL,
3278             .ib = pipeline->ds.state_ib,
3279          };
3280       draw_state_groups[draw_state_group_count++] =
3281          (struct tu_draw_state_group) {
3282             .id = TU_DRAW_STATE_BLEND,
3283             .enable_mask = ENABLE_ALL,
3284             .ib = pipeline->blend.state_ib,
3285          };
3286    }
3287
3288    if (cmd->state.dirty &
3289          (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_DESCRIPTOR_SETS | TU_CMD_DIRTY_PUSH_CONSTANTS)) {
3290       draw_state_groups[draw_state_group_count++] =
3291          (struct tu_draw_state_group) {
3292             .id = TU_DRAW_STATE_VS_CONST,
3293             .enable_mask = ENABLE_ALL,
3294             .ib = tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_VERTEX)
3295          };
3296       draw_state_groups[draw_state_group_count++] =
3297          (struct tu_draw_state_group) {
3298             .id = TU_DRAW_STATE_FS_CONST,
3299             .enable_mask = ENABLE_DRAW,
3300             .ib = tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_FRAGMENT)
3301          };
3302    }
3303
3304    if (cmd->state.dirty &
3305          (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_DESCRIPTOR_SETS)) {
3306       bool needs_border = false;
3307       struct tu_cs_entry vs_tex, fs_tex, fs_ibo;
3308
3309       result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3310                                  MESA_SHADER_VERTEX, &vs_tex, &needs_border);
3311       if (result != VK_SUCCESS)
3312          return result;
3313
3314       result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3315                                  MESA_SHADER_FRAGMENT, &fs_tex, &needs_border);
3316       if (result != VK_SUCCESS)
3317          return result;
3318
3319       result = tu6_emit_ibo(cmd, pipeline, descriptors_state,
3320                             MESA_SHADER_FRAGMENT, &fs_ibo);
3321       if (result != VK_SUCCESS)
3322          return result;
3323
3324       draw_state_groups[draw_state_group_count++] =
3325          (struct tu_draw_state_group) {
3326             .id = TU_DRAW_STATE_VS_TEX,
3327             .enable_mask = ENABLE_ALL,
3328             .ib = vs_tex,
3329          };
3330       draw_state_groups[draw_state_group_count++] =
3331          (struct tu_draw_state_group) {
3332             .id = TU_DRAW_STATE_FS_TEX,
3333             .enable_mask = ENABLE_DRAW,
3334             .ib = fs_tex,
3335          };
3336       draw_state_groups[draw_state_group_count++] =
3337          (struct tu_draw_state_group) {
3338             .id = TU_DRAW_STATE_FS_IBO,
3339             .enable_mask = ENABLE_DRAW,
3340             .ib = fs_ibo,
3341          };
3342
3343       if (needs_border) {
3344          result = tu6_emit_border_color(cmd, cs);
3345          if (result != VK_SUCCESS)
3346             return result;
3347       }
3348    }
3349
3350    struct tu_cs_entry vs_params;
3351    result = tu6_emit_vs_params(cmd, draw, &vs_params);
3352    if (result != VK_SUCCESS)
3353       return result;
3354
3355    draw_state_groups[draw_state_group_count++] =
3356       (struct tu_draw_state_group) {
3357          .id = TU_DRAW_STATE_VS_PARAMS,
3358          .enable_mask = ENABLE_ALL,
3359          .ib = vs_params,
3360       };
3361
3362    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3 * draw_state_group_count);
3363    for (uint32_t i = 0; i < draw_state_group_count; i++) {
3364       const struct tu_draw_state_group *group = &draw_state_groups[i];
3365       debug_assert((group->enable_mask & ~ENABLE_ALL) == 0);
3366       uint32_t cp_set_draw_state =
3367          CP_SET_DRAW_STATE__0_COUNT(group->ib.size / 4) |
3368          group->enable_mask |
3369          CP_SET_DRAW_STATE__0_GROUP_ID(group->id);
3370       uint64_t iova;
3371       if (group->ib.size) {
3372          iova = group->ib.bo->iova + group->ib.offset;
3373       } else {
3374          cp_set_draw_state |= CP_SET_DRAW_STATE__0_DISABLE;
3375          iova = 0;
3376       }
3377
3378       tu_cs_emit(cs, cp_set_draw_state);
3379       tu_cs_emit_qw(cs, iova);
3380    }
3381
3382    tu_cs_sanity_check(cs);
3383
3384    /* track BOs */
3385    if (cmd->state.dirty & TU_CMD_DIRTY_VERTEX_BUFFERS) {
3386       for (uint32_t i = 0; i < MAX_VBS; i++) {
3387          const struct tu_buffer *buf = cmd->state.vb.buffers[i];
3388          if (buf)
3389             tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
3390       }
3391    }
3392    if (cmd->state.dirty & TU_CMD_DIRTY_DESCRIPTOR_SETS) {
3393       unsigned i;
3394       for_each_bit(i, descriptors_state->valid) {
3395          struct tu_descriptor_set *set = descriptors_state->sets[i];
3396          for (unsigned j = 0; j < set->layout->buffer_count; ++j)
3397             if (set->descriptors[j]) {
3398                tu_bo_list_add(&cmd->bo_list, set->descriptors[j],
3399                               MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3400             }
3401       }
3402    }
3403
3404    /* Fragment shader state overwrites compute shader state, so flag the
3405     * compute pipeline for re-emit.
3406     */
3407    cmd->state.dirty = TU_CMD_DIRTY_COMPUTE_PIPELINE;
3408    return VK_SUCCESS;
3409 }
3410
3411 static void
3412 tu6_emit_draw_direct(struct tu_cmd_buffer *cmd,
3413                      struct tu_cs *cs,
3414                      const struct tu_draw_info *draw)
3415 {
3416
3417    const enum pc_di_primtype primtype = cmd->state.pipeline->ia.primtype;
3418
3419    tu_cs_emit_regs(cs,
3420                    A6XX_VFD_INDEX_OFFSET(draw->vertex_offset),
3421                    A6XX_VFD_INSTANCE_START_OFFSET(draw->first_instance));
3422
3423    /* TODO hw binning */
3424    if (draw->indexed) {
3425       const enum a4xx_index_size index_size =
3426          tu6_index_size(cmd->state.index_type);
3427       const uint32_t index_bytes =
3428          (cmd->state.index_type == VK_INDEX_TYPE_UINT32) ? 4 : 2;
3429       const struct tu_buffer *buf = cmd->state.index_buffer;
3430       const VkDeviceSize offset = buf->bo_offset + cmd->state.index_offset +
3431                                   index_bytes * draw->first_index;
3432       const uint32_t size = index_bytes * draw->count;
3433
3434       const uint32_t cp_draw_indx =
3435          CP_DRAW_INDX_OFFSET_0_PRIM_TYPE(primtype) |
3436          CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT(DI_SRC_SEL_DMA) |
3437          CP_DRAW_INDX_OFFSET_0_INDEX_SIZE(index_size) |
3438          CP_DRAW_INDX_OFFSET_0_VIS_CULL(USE_VISIBILITY) | 0x2000;
3439
3440       tu_cs_emit_pkt7(cs, CP_DRAW_INDX_OFFSET, 7);
3441       tu_cs_emit(cs, cp_draw_indx);
3442       tu_cs_emit(cs, draw->instance_count);
3443       tu_cs_emit(cs, draw->count);
3444       tu_cs_emit(cs, 0x0); /* XXX */
3445       tu_cs_emit_qw(cs, buf->bo->iova + offset);
3446       tu_cs_emit(cs, size);
3447    } else {
3448       const uint32_t cp_draw_indx =
3449          CP_DRAW_INDX_OFFSET_0_PRIM_TYPE(primtype) |
3450          CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT(DI_SRC_SEL_AUTO_INDEX) |
3451          CP_DRAW_INDX_OFFSET_0_VIS_CULL(USE_VISIBILITY) | 0x2000;
3452
3453       tu_cs_emit_pkt7(cs, CP_DRAW_INDX_OFFSET, 3);
3454       tu_cs_emit(cs, cp_draw_indx);
3455       tu_cs_emit(cs, draw->instance_count);
3456       tu_cs_emit(cs, draw->count);
3457    }
3458 }
3459
3460 static void
3461 tu_draw(struct tu_cmd_buffer *cmd, const struct tu_draw_info *draw)
3462 {
3463    struct tu_cs *cs = &cmd->draw_cs;
3464    VkResult result;
3465
3466    result = tu6_bind_draw_states(cmd, cs, draw);
3467    if (result != VK_SUCCESS) {
3468       cmd->record_result = result;
3469       return;
3470    }
3471
3472    result = tu_cs_reserve_space(cmd->device, cs, 32);
3473    if (result != VK_SUCCESS) {
3474       cmd->record_result = result;
3475       return;
3476    }
3477
3478    if (draw->indirect) {
3479       tu_finishme("indirect draw");
3480       return;
3481    }
3482
3483    /* TODO tu6_emit_marker should pick different regs depending on cs */
3484
3485    tu6_emit_marker(cmd, cs);
3486    tu6_emit_draw_direct(cmd, cs, draw);
3487    tu6_emit_marker(cmd, cs);
3488
3489    cmd->wait_for_idle = true;
3490
3491    tu_cs_sanity_check(cs);
3492 }
3493
3494 void
3495 tu_CmdDraw(VkCommandBuffer commandBuffer,
3496            uint32_t vertexCount,
3497            uint32_t instanceCount,
3498            uint32_t firstVertex,
3499            uint32_t firstInstance)
3500 {
3501    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3502    struct tu_draw_info info = {};
3503
3504    info.count = vertexCount;
3505    info.instance_count = instanceCount;
3506    info.first_instance = firstInstance;
3507    info.vertex_offset = firstVertex;
3508
3509    tu_draw(cmd_buffer, &info);
3510 }
3511
3512 void
3513 tu_CmdDrawIndexed(VkCommandBuffer commandBuffer,
3514                   uint32_t indexCount,
3515                   uint32_t instanceCount,
3516                   uint32_t firstIndex,
3517                   int32_t vertexOffset,
3518                   uint32_t firstInstance)
3519 {
3520    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3521    struct tu_draw_info info = {};
3522
3523    info.indexed = true;
3524    info.count = indexCount;
3525    info.instance_count = instanceCount;
3526    info.first_index = firstIndex;
3527    info.vertex_offset = vertexOffset;
3528    info.first_instance = firstInstance;
3529
3530    tu_draw(cmd_buffer, &info);
3531 }
3532
3533 void
3534 tu_CmdDrawIndirect(VkCommandBuffer commandBuffer,
3535                    VkBuffer _buffer,
3536                    VkDeviceSize offset,
3537                    uint32_t drawCount,
3538                    uint32_t stride)
3539 {
3540    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3541    TU_FROM_HANDLE(tu_buffer, buffer, _buffer);
3542    struct tu_draw_info info = {};
3543
3544    info.count = drawCount;
3545    info.indirect = buffer;
3546    info.indirect_offset = offset;
3547    info.stride = stride;
3548
3549    tu_draw(cmd_buffer, &info);
3550 }
3551
3552 void
3553 tu_CmdDrawIndexedIndirect(VkCommandBuffer commandBuffer,
3554                           VkBuffer _buffer,
3555                           VkDeviceSize offset,
3556                           uint32_t drawCount,
3557                           uint32_t stride)
3558 {
3559    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3560    TU_FROM_HANDLE(tu_buffer, buffer, _buffer);
3561    struct tu_draw_info info = {};
3562
3563    info.indexed = true;
3564    info.count = drawCount;
3565    info.indirect = buffer;
3566    info.indirect_offset = offset;
3567    info.stride = stride;
3568
3569    tu_draw(cmd_buffer, &info);
3570 }
3571
3572 struct tu_dispatch_info
3573 {
3574    /**
3575     * Determine the layout of the grid (in block units) to be used.
3576     */
3577    uint32_t blocks[3];
3578
3579    /**
3580     * A starting offset for the grid. If unaligned is set, the offset
3581     * must still be aligned.
3582     */
3583    uint32_t offsets[3];
3584    /**
3585     * Whether it's an unaligned compute dispatch.
3586     */
3587    bool unaligned;
3588
3589    /**
3590     * Indirect compute parameters resource.
3591     */
3592    struct tu_buffer *indirect;
3593    uint64_t indirect_offset;
3594 };
3595
3596 static void
3597 tu_emit_compute_driver_params(struct tu_cs *cs, struct tu_pipeline *pipeline,
3598                               const struct tu_dispatch_info *info)
3599 {
3600    gl_shader_stage type = MESA_SHADER_COMPUTE;
3601    const struct tu_program_descriptor_linkage *link =
3602       &pipeline->program.link[type];
3603    const struct ir3_const_state *const_state = &link->const_state;
3604    uint32_t offset = const_state->offsets.driver_param;
3605
3606    if (link->constlen <= offset)
3607       return;
3608
3609    if (!info->indirect) {
3610       uint32_t driver_params[IR3_DP_CS_COUNT] = {
3611          [IR3_DP_NUM_WORK_GROUPS_X] = info->blocks[0],
3612          [IR3_DP_NUM_WORK_GROUPS_Y] = info->blocks[1],
3613          [IR3_DP_NUM_WORK_GROUPS_Z] = info->blocks[2],
3614          [IR3_DP_LOCAL_GROUP_SIZE_X] = pipeline->compute.local_size[0],
3615          [IR3_DP_LOCAL_GROUP_SIZE_Y] = pipeline->compute.local_size[1],
3616          [IR3_DP_LOCAL_GROUP_SIZE_Z] = pipeline->compute.local_size[2],
3617       };
3618
3619       uint32_t num_consts = MIN2(const_state->num_driver_params,
3620                                  (link->constlen - offset) * 4);
3621       /* push constants */
3622       tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + num_consts);
3623       tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(offset) |
3624                  CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
3625                  CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
3626                  CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
3627                  CP_LOAD_STATE6_0_NUM_UNIT(num_consts / 4));
3628       tu_cs_emit(cs, 0);
3629       tu_cs_emit(cs, 0);
3630       uint32_t i;
3631       for (i = 0; i < num_consts; i++)
3632          tu_cs_emit(cs, driver_params[i]);
3633    } else {
3634       tu_finishme("Indirect driver params");
3635    }
3636 }
3637
3638 static void
3639 tu_dispatch(struct tu_cmd_buffer *cmd,
3640             const struct tu_dispatch_info *info)
3641 {
3642    struct tu_cs *cs = &cmd->cs;
3643    struct tu_pipeline *pipeline = cmd->state.compute_pipeline;
3644    struct tu_descriptor_state *descriptors_state =
3645       &cmd->descriptors[VK_PIPELINE_BIND_POINT_COMPUTE];
3646
3647    VkResult result = tu_cs_reserve_space(cmd->device, cs, 256);
3648    if (result != VK_SUCCESS) {
3649       cmd->record_result = result;
3650       return;
3651    }
3652
3653    if (cmd->state.dirty & TU_CMD_DIRTY_COMPUTE_PIPELINE)
3654       tu_cs_emit_ib(cs, &pipeline->program.state_ib);
3655
3656    struct tu_cs_entry ib;
3657
3658    ib = tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_COMPUTE);
3659    if (ib.size)
3660       tu_cs_emit_ib(cs, &ib);
3661
3662    tu_emit_compute_driver_params(cs, pipeline, info);
3663
3664    bool needs_border;
3665    result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3666                               MESA_SHADER_COMPUTE, &ib, &needs_border);
3667    if (result != VK_SUCCESS) {
3668       cmd->record_result = result;
3669       return;
3670    }
3671
3672    if (ib.size)
3673       tu_cs_emit_ib(cs, &ib);
3674
3675    if (needs_border)
3676       tu_finishme("compute border color");
3677
3678    result = tu6_emit_ibo(cmd, pipeline, descriptors_state, MESA_SHADER_COMPUTE, &ib);
3679    if (result != VK_SUCCESS) {
3680       cmd->record_result = result;
3681       return;
3682    }
3683
3684    if (ib.size)
3685       tu_cs_emit_ib(cs, &ib);
3686
3687    /* track BOs */
3688    if (cmd->state.dirty & TU_CMD_DIRTY_DESCRIPTOR_SETS) {
3689       unsigned i;
3690       for_each_bit(i, descriptors_state->valid) {
3691          struct tu_descriptor_set *set = descriptors_state->sets[i];
3692          for (unsigned j = 0; j < set->layout->buffer_count; ++j)
3693             if (set->descriptors[j]) {
3694                tu_bo_list_add(&cmd->bo_list, set->descriptors[j],
3695                               MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3696             }
3697       }
3698    }
3699
3700    /* Compute shader state overwrites fragment shader state, so we flag the
3701     * graphics pipeline for re-emit.
3702     */
3703    cmd->state.dirty = TU_CMD_DIRTY_PIPELINE;
3704
3705    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
3706    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(0x8));
3707
3708    const uint32_t *local_size = pipeline->compute.local_size;
3709    const uint32_t *num_groups = info->blocks;
3710    tu_cs_emit_regs(cs,
3711                    A6XX_HLSQ_CS_NDRANGE_0(.kerneldim = 3,
3712                                           .localsizex = local_size[0] - 1,
3713                                           .localsizey = local_size[1] - 1,
3714                                           .localsizez = local_size[2] - 1),
3715                    A6XX_HLSQ_CS_NDRANGE_1(.globalsize_x = local_size[0] * num_groups[0]),
3716                    A6XX_HLSQ_CS_NDRANGE_2(.globaloff_x = 0),
3717                    A6XX_HLSQ_CS_NDRANGE_3(.globalsize_y = local_size[1] * num_groups[1]),
3718                    A6XX_HLSQ_CS_NDRANGE_4(.globaloff_y = 0),
3719                    A6XX_HLSQ_CS_NDRANGE_5(.globalsize_z = local_size[2] * num_groups[2]),
3720                    A6XX_HLSQ_CS_NDRANGE_6(.globaloff_z = 0));
3721
3722    tu_cs_emit_regs(cs,
3723                    A6XX_HLSQ_CS_KERNEL_GROUP_X(1),
3724                    A6XX_HLSQ_CS_KERNEL_GROUP_Y(1),
3725                    A6XX_HLSQ_CS_KERNEL_GROUP_Z(1));
3726
3727    if (info->indirect) {
3728       uint64_t iova = tu_buffer_iova(info->indirect) + info->indirect_offset;
3729
3730       tu_bo_list_add(&cmd->bo_list, info->indirect->bo,
3731                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3732
3733       tu_cs_emit_pkt7(cs, CP_EXEC_CS_INDIRECT, 4);
3734       tu_cs_emit(cs, 0x00000000);
3735       tu_cs_emit_qw(cs, iova);
3736       tu_cs_emit(cs,
3737                  A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX(local_size[0] - 1) |
3738                  A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY(local_size[1] - 1) |
3739                  A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ(local_size[2] - 1));
3740    } else {
3741       tu_cs_emit_pkt7(cs, CP_EXEC_CS, 4);
3742       tu_cs_emit(cs, 0x00000000);
3743       tu_cs_emit(cs, CP_EXEC_CS_1_NGROUPS_X(info->blocks[0]));
3744       tu_cs_emit(cs, CP_EXEC_CS_2_NGROUPS_Y(info->blocks[1]));
3745       tu_cs_emit(cs, CP_EXEC_CS_3_NGROUPS_Z(info->blocks[2]));
3746    }
3747
3748    tu_cs_emit_wfi(cs);
3749
3750    tu6_emit_cache_flush(cmd, cs);
3751 }
3752
3753 void
3754 tu_CmdDispatchBase(VkCommandBuffer commandBuffer,
3755                    uint32_t base_x,
3756                    uint32_t base_y,
3757                    uint32_t base_z,
3758                    uint32_t x,
3759                    uint32_t y,
3760                    uint32_t z)
3761 {
3762    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3763    struct tu_dispatch_info info = {};
3764
3765    info.blocks[0] = x;
3766    info.blocks[1] = y;
3767    info.blocks[2] = z;
3768
3769    info.offsets[0] = base_x;
3770    info.offsets[1] = base_y;
3771    info.offsets[2] = base_z;
3772    tu_dispatch(cmd_buffer, &info);
3773 }
3774
3775 void
3776 tu_CmdDispatch(VkCommandBuffer commandBuffer,
3777                uint32_t x,
3778                uint32_t y,
3779                uint32_t z)
3780 {
3781    tu_CmdDispatchBase(commandBuffer, 0, 0, 0, x, y, z);
3782 }
3783
3784 void
3785 tu_CmdDispatchIndirect(VkCommandBuffer commandBuffer,
3786                        VkBuffer _buffer,
3787                        VkDeviceSize offset)
3788 {
3789    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3790    TU_FROM_HANDLE(tu_buffer, buffer, _buffer);
3791    struct tu_dispatch_info info = {};
3792
3793    info.indirect = buffer;
3794    info.indirect_offset = offset;
3795
3796    tu_dispatch(cmd_buffer, &info);
3797 }
3798
3799 void
3800 tu_CmdEndRenderPass(VkCommandBuffer commandBuffer)
3801 {
3802    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3803
3804    tu_cs_end(&cmd_buffer->draw_cs);
3805    tu_cs_end(&cmd_buffer->draw_epilogue_cs);
3806
3807    tu_cmd_render_tiles(cmd_buffer);
3808
3809    /* discard draw_cs and draw_epilogue_cs entries now that the tiles are
3810       rendered */
3811    tu_cs_discard_entries(&cmd_buffer->draw_cs);
3812    tu_cs_begin(&cmd_buffer->draw_cs);
3813    tu_cs_discard_entries(&cmd_buffer->draw_epilogue_cs);
3814    tu_cs_begin(&cmd_buffer->draw_epilogue_cs);
3815
3816    cmd_buffer->state.pass = NULL;
3817    cmd_buffer->state.subpass = NULL;
3818    cmd_buffer->state.framebuffer = NULL;
3819 }
3820
3821 void
3822 tu_CmdEndRenderPass2(VkCommandBuffer commandBuffer,
3823                      const VkSubpassEndInfoKHR *pSubpassEndInfo)
3824 {
3825    tu_CmdEndRenderPass(commandBuffer);
3826 }
3827
3828 struct tu_barrier_info
3829 {
3830    uint32_t eventCount;
3831    const VkEvent *pEvents;
3832    VkPipelineStageFlags srcStageMask;
3833 };
3834
3835 static void
3836 tu_barrier(struct tu_cmd_buffer *cmd_buffer,
3837            uint32_t memoryBarrierCount,
3838            const VkMemoryBarrier *pMemoryBarriers,
3839            uint32_t bufferMemoryBarrierCount,
3840            const VkBufferMemoryBarrier *pBufferMemoryBarriers,
3841            uint32_t imageMemoryBarrierCount,
3842            const VkImageMemoryBarrier *pImageMemoryBarriers,
3843            const struct tu_barrier_info *info)
3844 {
3845 }
3846
3847 void
3848 tu_CmdPipelineBarrier(VkCommandBuffer commandBuffer,
3849                       VkPipelineStageFlags srcStageMask,
3850                       VkPipelineStageFlags destStageMask,
3851                       VkBool32 byRegion,
3852                       uint32_t memoryBarrierCount,
3853                       const VkMemoryBarrier *pMemoryBarriers,
3854                       uint32_t bufferMemoryBarrierCount,
3855                       const VkBufferMemoryBarrier *pBufferMemoryBarriers,
3856                       uint32_t imageMemoryBarrierCount,
3857                       const VkImageMemoryBarrier *pImageMemoryBarriers)
3858 {
3859    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3860    struct tu_barrier_info info;
3861
3862    info.eventCount = 0;
3863    info.pEvents = NULL;
3864    info.srcStageMask = srcStageMask;
3865
3866    tu_barrier(cmd_buffer, memoryBarrierCount, pMemoryBarriers,
3867               bufferMemoryBarrierCount, pBufferMemoryBarriers,
3868               imageMemoryBarrierCount, pImageMemoryBarriers, &info);
3869 }
3870
3871 static void
3872 write_event(struct tu_cmd_buffer *cmd, struct tu_event *event, unsigned value)
3873 {
3874    struct tu_cs *cs = &cmd->cs;
3875
3876    VkResult result = tu_cs_reserve_space(cmd->device, cs, 4);
3877    if (result != VK_SUCCESS) {
3878       cmd->record_result = result;
3879       return;
3880    }
3881
3882    tu_bo_list_add(&cmd->bo_list, &event->bo, MSM_SUBMIT_BO_WRITE);
3883
3884    /* TODO: any flush required before/after ? */
3885
3886    tu_cs_emit_pkt7(cs, CP_MEM_WRITE, 3);
3887    tu_cs_emit_qw(cs, event->bo.iova); /* ADDR_LO/HI */
3888    tu_cs_emit(cs, value);
3889 }
3890
3891 void
3892 tu_CmdSetEvent(VkCommandBuffer commandBuffer,
3893                VkEvent _event,
3894                VkPipelineStageFlags stageMask)
3895 {
3896    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
3897    TU_FROM_HANDLE(tu_event, event, _event);
3898
3899    write_event(cmd, event, 1);
3900 }
3901
3902 void
3903 tu_CmdResetEvent(VkCommandBuffer commandBuffer,
3904                  VkEvent _event,
3905                  VkPipelineStageFlags stageMask)
3906 {
3907    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
3908    TU_FROM_HANDLE(tu_event, event, _event);
3909
3910    write_event(cmd, event, 0);
3911 }
3912
3913 void
3914 tu_CmdWaitEvents(VkCommandBuffer commandBuffer,
3915                  uint32_t eventCount,
3916                  const VkEvent *pEvents,
3917                  VkPipelineStageFlags srcStageMask,
3918                  VkPipelineStageFlags dstStageMask,
3919                  uint32_t memoryBarrierCount,
3920                  const VkMemoryBarrier *pMemoryBarriers,
3921                  uint32_t bufferMemoryBarrierCount,
3922                  const VkBufferMemoryBarrier *pBufferMemoryBarriers,
3923                  uint32_t imageMemoryBarrierCount,
3924                  const VkImageMemoryBarrier *pImageMemoryBarriers)
3925 {
3926    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
3927    struct tu_cs *cs = &cmd->cs;
3928
3929    VkResult result = tu_cs_reserve_space(cmd->device, cs, eventCount * 7);
3930    if (result != VK_SUCCESS) {
3931       cmd->record_result = result;
3932       return;
3933    }
3934
3935    /* TODO: any flush required before/after? (CP_WAIT_FOR_ME?) */
3936
3937    for (uint32_t i = 0; i < eventCount; i++) {
3938       const struct tu_event *event = (const struct tu_event*) pEvents[i];
3939
3940       tu_bo_list_add(&cmd->bo_list, &event->bo, MSM_SUBMIT_BO_READ);
3941
3942       tu_cs_emit_pkt7(cs, CP_WAIT_REG_MEM, 6);
3943       tu_cs_emit(cs, CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ) |
3944                      CP_WAIT_REG_MEM_0_POLL_MEMORY);
3945       tu_cs_emit_qw(cs, event->bo.iova); /* POLL_ADDR_LO/HI */
3946       tu_cs_emit(cs, CP_WAIT_REG_MEM_3_REF(1));
3947       tu_cs_emit(cs, CP_WAIT_REG_MEM_4_MASK(~0u));
3948       tu_cs_emit(cs, CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(20));
3949    }
3950 }
3951
3952 void
3953 tu_CmdSetDeviceMask(VkCommandBuffer commandBuffer, uint32_t deviceMask)
3954 {
3955    /* No-op */
3956 }