src/freedreno/vulkan/tu_cmd_buffer.c

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * based in part on anv driver which is:
   6  * Copyright © 2015 Intel Corporation
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25  * DEALINGS IN THE SOFTWARE.
  26  */
  27
  28 #include "tu_private.h"
  29
  30 #include "registers/adreno_pm4.xml.h"
  31 #include "registers/adreno_common.xml.h"
  32
  33 #include "vk_format.h"
  34
  35 #include "tu_cs.h"
  36 #include "tu_blit.h"
  37
  38 #define OVERFLOW_FLAG_REG REG_A6XX_CP_SCRATCH_REG(0)
  39
  40 void
  41 tu_bo_list_init(struct tu_bo_list *list)
  42 {
  43    list->count = list->capacity = 0;
  44    list->bo_infos = NULL;
  45 }
  46
  47 void
  48 tu_bo_list_destroy(struct tu_bo_list *list)
  49 {
  50    free(list->bo_infos);
  51 }
  52
  53 void
  54 tu_bo_list_reset(struct tu_bo_list *list)
  55 {
  56    list->count = 0;
  57 }
  58
  59 /**
  60  * \a flags consists of MSM_SUBMIT_BO_FLAGS.
  61  */
  62 static uint32_t
  63 tu_bo_list_add_info(struct tu_bo_list *list,
  64                     const struct drm_msm_gem_submit_bo *bo_info)
  65 {
  66    assert(bo_info->handle != 0);
  67
  68    for (uint32_t i = 0; i < list->count; ++i) {
  69       if (list->bo_infos[i].handle == bo_info->handle) {
  70          assert(list->bo_infos[i].presumed == bo_info->presumed);
  71          list->bo_infos[i].flags |= bo_info->flags;
  72          return i;
  73       }
  74    }
  75
  76    /* grow list->bo_infos if needed */
  77    if (list->count == list->capacity) {
  78       uint32_t new_capacity = MAX2(2 * list->count, 16);
  79       struct drm_msm_gem_submit_bo *new_bo_infos = realloc(
  80          list->bo_infos, new_capacity * sizeof(struct drm_msm_gem_submit_bo));
  81       if (!new_bo_infos)
  82          return TU_BO_LIST_FAILED;
  83       list->bo_infos = new_bo_infos;
  84       list->capacity = new_capacity;
  85    }
  86
  87    list->bo_infos[list->count] = *bo_info;
  88    return list->count++;
  89 }
  90
  91 uint32_t
  92 tu_bo_list_add(struct tu_bo_list *list,
  93                const struct tu_bo *bo,
  94                uint32_t flags)
  95 {
  96    return tu_bo_list_add_info(list, &(struct drm_msm_gem_submit_bo) {
  97                                        .flags = flags,
  98                                        .handle = bo->gem_handle,
  99                                        .presumed = bo->iova,
 100                                     });
 101 }
 102
 103 VkResult
 104 tu_bo_list_merge(struct tu_bo_list *list, const struct tu_bo_list *other)
 105 {
 106    for (uint32_t i = 0; i < other->count; i++) {
 107       if (tu_bo_list_add_info(list, other->bo_infos + i) == TU_BO_LIST_FAILED)
 108          return VK_ERROR_OUT_OF_HOST_MEMORY;
 109    }
 110
 111    return VK_SUCCESS;
 112 }
 113
 114 static void
 115 tu_tiling_config_update_tile_layout(struct tu_tiling_config *tiling,
 116                                     const struct tu_device *dev,
 117                                     uint32_t pixels)
 118 {
 119    const uint32_t tile_align_w = dev->physical_device->tile_align_w;
 120    const uint32_t tile_align_h = dev->physical_device->tile_align_h;
 121    const uint32_t max_tile_width = 1024; /* A6xx */
 122
 123    tiling->tile0.offset = (VkOffset2D) {
 124       .x = tiling->render_area.offset.x & ~(tile_align_w - 1),
 125       .y = tiling->render_area.offset.y & ~(tile_align_h - 1),
 126    };
 127
 128    const uint32_t ra_width =
 129       tiling->render_area.extent.width +
 130       (tiling->render_area.offset.x - tiling->tile0.offset.x);
 131    const uint32_t ra_height =
 132       tiling->render_area.extent.height +
 133       (tiling->render_area.offset.y - tiling->tile0.offset.y);
 134
 135    /* start from 1 tile */
 136    tiling->tile_count = (VkExtent2D) {
 137       .width = 1,
 138       .height = 1,
 139    };
 140    tiling->tile0.extent = (VkExtent2D) {
 141       .width = align(ra_width, tile_align_w),
 142       .height = align(ra_height, tile_align_h),
 143    };
 144
 145    /* do not exceed max tile width */
 146    while (tiling->tile0.extent.width > max_tile_width) {
 147       tiling->tile_count.width++;
 148       tiling->tile0.extent.width =
 149          align(ra_width / tiling->tile_count.width, tile_align_w);
 150    }
 151
 152    /* do not exceed gmem size */
 153    while (tiling->tile0.extent.width * tiling->tile0.extent.height > pixels) {
 154       if (tiling->tile0.extent.width > MAX2(tile_align_w, tiling->tile0.extent.height)) {
 155          tiling->tile_count.width++;
 156          tiling->tile0.extent.width =
 157             align(DIV_ROUND_UP(ra_width, tiling->tile_count.width), tile_align_w);
 158       } else {
 159          /* if this assert fails then layout is impossible.. */
 160          assert(tiling->tile0.extent.height > tile_align_h);
 161          tiling->tile_count.height++;
 162          tiling->tile0.extent.height =
 163             align(DIV_ROUND_UP(ra_height, tiling->tile_count.height), tile_align_h);
 164       }
 165    }
 166 }
 167
 168 static void
 169 tu_tiling_config_update_pipe_layout(struct tu_tiling_config *tiling,
 170                                     const struct tu_device *dev)
 171 {
 172    const uint32_t max_pipe_count = 32; /* A6xx */
 173
 174    /* start from 1 tile per pipe */
 175    tiling->pipe0 = (VkExtent2D) {
 176       .width = 1,
 177       .height = 1,
 178    };
 179    tiling->pipe_count = tiling->tile_count;
 180
 181    /* do not exceed max pipe count vertically */
 182    while (tiling->pipe_count.height > max_pipe_count) {
 183       tiling->pipe0.height += 2;
 184       tiling->pipe_count.height =
 185          (tiling->tile_count.height + tiling->pipe0.height - 1) /
 186          tiling->pipe0.height;
 187    }
 188
 189    /* do not exceed max pipe count */
 190    while (tiling->pipe_count.width * tiling->pipe_count.height >
 191           max_pipe_count) {
 192       tiling->pipe0.width += 1;
 193       tiling->pipe_count.width =
 194          (tiling->tile_count.width + tiling->pipe0.width - 1) /
 195          tiling->pipe0.width;
 196    }
 197 }
 198
 199 static void
 200 tu_tiling_config_update_pipes(struct tu_tiling_config *tiling,
 201                               const struct tu_device *dev)
 202 {
 203    const uint32_t max_pipe_count = 32; /* A6xx */
 204    const uint32_t used_pipe_count =
 205       tiling->pipe_count.width * tiling->pipe_count.height;
 206    const VkExtent2D last_pipe = {
 207       .width = (tiling->tile_count.width - 1) % tiling->pipe0.width + 1,
 208       .height = (tiling->tile_count.height - 1) % tiling->pipe0.height + 1,
 209    };
 210
 211    assert(used_pipe_count <= max_pipe_count);
 212    assert(max_pipe_count <= ARRAY_SIZE(tiling->pipe_config));
 213
 214    for (uint32_t y = 0; y < tiling->pipe_count.height; y++) {
 215       for (uint32_t x = 0; x < tiling->pipe_count.width; x++) {
 216          const uint32_t pipe_x = tiling->pipe0.width * x;
 217          const uint32_t pipe_y = tiling->pipe0.height * y;
 218          const uint32_t pipe_w = (x == tiling->pipe_count.width - 1)
 219                                     ? last_pipe.width
 220                                     : tiling->pipe0.width;
 221          const uint32_t pipe_h = (y == tiling->pipe_count.height - 1)
 222                                     ? last_pipe.height
 223                                     : tiling->pipe0.height;
 224          const uint32_t n = tiling->pipe_count.width * y + x;
 225
 226          tiling->pipe_config[n] = A6XX_VSC_PIPE_CONFIG_REG_X(pipe_x) |
 227                                   A6XX_VSC_PIPE_CONFIG_REG_Y(pipe_y) |
 228                                   A6XX_VSC_PIPE_CONFIG_REG_W(pipe_w) |
 229                                   A6XX_VSC_PIPE_CONFIG_REG_H(pipe_h);
 230          tiling->pipe_sizes[n] = CP_SET_BIN_DATA5_0_VSC_SIZE(pipe_w * pipe_h);
 231       }
 232    }
 233
 234    memset(tiling->pipe_config + used_pipe_count, 0,
 235           sizeof(uint32_t) * (max_pipe_count - used_pipe_count));
 236 }
 237
 238 static void
 239 tu_tiling_config_get_tile(const struct tu_tiling_config *tiling,
 240                           const struct tu_device *dev,
 241                           uint32_t tx,
 242                           uint32_t ty,
 243                           struct tu_tile *tile)
 244 {
 245    /* find the pipe and the slot for tile (tx, ty) */
 246    const uint32_t px = tx / tiling->pipe0.width;
 247    const uint32_t py = ty / tiling->pipe0.height;
 248    const uint32_t sx = tx - tiling->pipe0.width * px;
 249    const uint32_t sy = ty - tiling->pipe0.height * py;
 250
 251    assert(tx < tiling->tile_count.width && ty < tiling->tile_count.height);
 252    assert(px < tiling->pipe_count.width && py < tiling->pipe_count.height);
 253    assert(sx < tiling->pipe0.width && sy < tiling->pipe0.height);
 254
 255    /* convert to 1D indices */
 256    tile->pipe = tiling->pipe_count.width * py + px;
 257    tile->slot = tiling->pipe0.width * sy + sx;
 258
 259    /* get the blit area for the tile */
 260    tile->begin = (VkOffset2D) {
 261       .x = tiling->tile0.offset.x + tiling->tile0.extent.width * tx,
 262       .y = tiling->tile0.offset.y + tiling->tile0.extent.height * ty,
 263    };
 264    tile->end.x =
 265       (tx == tiling->tile_count.width - 1)
 266          ? tiling->render_area.offset.x + tiling->render_area.extent.width
 267          : tile->begin.x + tiling->tile0.extent.width;
 268    tile->end.y =
 269       (ty == tiling->tile_count.height - 1)
 270          ? tiling->render_area.offset.y + tiling->render_area.extent.height
 271          : tile->begin.y + tiling->tile0.extent.height;
 272 }
 273
 274 enum a3xx_msaa_samples
 275 tu_msaa_samples(uint32_t samples)
 276 {
 277    switch (samples) {
 278    case 1:
 279       return MSAA_ONE;
 280    case 2:
 281       return MSAA_TWO;
 282    case 4:
 283       return MSAA_FOUR;
 284    case 8:
 285       return MSAA_EIGHT;
 286    default:
 287       assert(!"invalid sample count");
 288       return MSAA_ONE;
 289    }
 290 }
 291
 292 static enum a4xx_index_size
 293 tu6_index_size(VkIndexType type)
 294 {
 295    switch (type) {
 296    case VK_INDEX_TYPE_UINT16:
 297       return INDEX4_SIZE_16_BIT;
 298    case VK_INDEX_TYPE_UINT32:
 299       return INDEX4_SIZE_32_BIT;
 300    default:
 301       unreachable("invalid VkIndexType");
 302       return INDEX4_SIZE_8_BIT;
 303    }
 304 }
 305
 306 static void
 307 tu6_emit_marker(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 308 {
 309    tu_cs_emit_write_reg(cs, cmd->marker_reg, ++cmd->marker_seqno);
 310 }
 311
 312 unsigned
 313 tu6_emit_event_write(struct tu_cmd_buffer *cmd,
 314                      struct tu_cs *cs,
 315                      enum vgt_event_type event,
 316                      bool need_seqno)
 317 {
 318    unsigned seqno = 0;
 319
 320    tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, need_seqno ? 4 : 1);
 321    tu_cs_emit(cs, CP_EVENT_WRITE_0_EVENT(event));
 322    if (need_seqno) {
 323       tu_cs_emit_qw(cs, cmd->scratch_bo.iova);
 324       seqno = ++cmd->scratch_seqno;
 325       tu_cs_emit(cs, seqno);
 326    }
 327
 328    return seqno;
 329 }
 330
 331 static void
 332 tu6_emit_cache_flush(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 333 {
 334    tu6_emit_event_write(cmd, cs, 0x31, false);
 335 }
 336
 337 static void
 338 tu6_emit_lrz_flush(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 339 {
 340    tu6_emit_event_write(cmd, cs, LRZ_FLUSH, false);
 341 }
 342
 343 static void
 344 tu6_emit_wfi(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 345 {
 346    if (cmd->wait_for_idle) {
 347       tu_cs_emit_wfi(cs);
 348       cmd->wait_for_idle = false;
 349    }
 350 }
 351
 352 #define tu_image_view_ubwc_pitches(iview)                                \
 353    .pitch = tu_image_ubwc_pitch(iview->image, iview->base_mip),          \
 354    .array_pitch = tu_image_ubwc_size(iview->image, iview->base_mip) >> 2
 355
 356 static void
 357 tu6_emit_zs(struct tu_cmd_buffer *cmd,
 358             const struct tu_subpass *subpass,
 359             struct tu_cs *cs)
 360 {
 361    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 362
 363    const uint32_t a = subpass->depth_stencil_attachment.attachment;
 364    if (a == VK_ATTACHMENT_UNUSED) {
 365       tu_cs_emit_regs(cs,
 366                       A6XX_RB_DEPTH_BUFFER_INFO(.depth_format = DEPTH6_NONE),
 367                       A6XX_RB_DEPTH_BUFFER_PITCH(0),
 368                       A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH(0),
 369                       A6XX_RB_DEPTH_BUFFER_BASE(0),
 370                       A6XX_RB_DEPTH_BUFFER_BASE_GMEM(0));
 371
 372       tu_cs_emit_regs(cs,
 373                       A6XX_GRAS_SU_DEPTH_BUFFER_INFO(.depth_format = DEPTH6_NONE));
 374
 375       tu_cs_emit_regs(cs,
 376                       A6XX_GRAS_LRZ_BUFFER_BASE(0),
 377                       A6XX_GRAS_LRZ_BUFFER_PITCH(0),
 378                       A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE(0));
 379
 380       tu_cs_emit_regs(cs, A6XX_RB_STENCIL_INFO(0));
 381
 382       return;
 383    }
 384
 385    const struct tu_image_view *iview = fb->attachments[a].attachment;
 386    enum a6xx_depth_format fmt = tu6_pipe2depth(iview->vk_format);
 387
 388    tu_cs_emit_regs(cs,
 389                    A6XX_RB_DEPTH_BUFFER_INFO(.depth_format = fmt),
 390                    A6XX_RB_DEPTH_BUFFER_PITCH(tu_image_stride(iview->image, iview->base_mip)),
 391                    A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH(iview->image->layout.layer_size),
 392                    A6XX_RB_DEPTH_BUFFER_BASE(tu_image_view_base_ref(iview)),
 393                    A6XX_RB_DEPTH_BUFFER_BASE_GMEM(cmd->state.pass->attachments[a].gmem_offset));
 394
 395    tu_cs_emit_regs(cs,
 396                    A6XX_GRAS_SU_DEPTH_BUFFER_INFO(.depth_format = fmt));
 397
 398    tu_cs_emit_regs(cs,
 399                    A6XX_RB_DEPTH_FLAG_BUFFER_BASE(tu_image_view_ubwc_base_ref(iview)),
 400                    A6XX_RB_DEPTH_FLAG_BUFFER_PITCH(tu_image_view_ubwc_pitches(iview)));
 401
 402    tu_cs_emit_regs(cs,
 403                    A6XX_GRAS_LRZ_BUFFER_BASE(0),
 404                    A6XX_GRAS_LRZ_BUFFER_PITCH(0),
 405                    A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE(0));
 406
 407    tu_cs_emit_regs(cs,
 408                    A6XX_RB_STENCIL_INFO(0));
 409
 410    /* enable zs? */
 411 }
 412
 413 static void
 414 tu6_emit_mrt(struct tu_cmd_buffer *cmd,
 415              const struct tu_subpass *subpass,
 416              struct tu_cs *cs)
 417 {
 418    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 419    unsigned char mrt_comp[MAX_RTS] = { 0 };
 420    unsigned srgb_cntl = 0;
 421
 422    for (uint32_t i = 0; i < subpass->color_count; ++i) {
 423       uint32_t a = subpass->color_attachments[i].attachment;
 424       if (a == VK_ATTACHMENT_UNUSED)
 425          continue;
 426
 427       const struct tu_image_view *iview = fb->attachments[a].attachment;
 428       const enum a6xx_tile_mode tile_mode =
 429          tu6_get_image_tile_mode(iview->image, iview->base_mip);
 430
 431       mrt_comp[i] = 0xf;
 432
 433       if (vk_format_is_srgb(iview->vk_format))
 434          srgb_cntl |= (1 << i);
 435
 436       const struct tu_native_format *format =
 437          tu6_get_native_format(iview->vk_format);
 438       assert(format && format->rb >= 0);
 439
 440       tu_cs_emit_regs(cs,
 441                       A6XX_RB_MRT_BUF_INFO(i,
 442                                            .color_tile_mode = tile_mode,
 443                                            .color_format = format->rb,
 444                                            .color_swap = format->swap),
 445                       A6XX_RB_MRT_PITCH(i, tu_image_stride(iview->image, iview->base_mip)),
 446                       A6XX_RB_MRT_ARRAY_PITCH(i, iview->image->layout.layer_size),
 447                       A6XX_RB_MRT_BASE(i, tu_image_view_base_ref(iview)),
 448                       A6XX_RB_MRT_BASE_GMEM(i, cmd->state.pass->attachments[a].gmem_offset));
 449
 450       tu_cs_emit_regs(cs,
 451                       A6XX_SP_FS_MRT_REG(i,
 452                                          .color_format = format->rb,
 453                                          .color_sint = vk_format_is_sint(iview->vk_format),
 454                                          .color_uint = vk_format_is_uint(iview->vk_format)));
 455
 456       tu_cs_emit_regs(cs,
 457                       A6XX_RB_MRT_FLAG_BUFFER_ADDR(i, tu_image_view_ubwc_base_ref(iview)),
 458                       A6XX_RB_MRT_FLAG_BUFFER_PITCH(i, tu_image_view_ubwc_pitches(iview)));
 459    }
 460
 461    tu_cs_emit_regs(cs,
 462                    A6XX_RB_SRGB_CNTL(srgb_cntl));
 463
 464    tu_cs_emit_regs(cs,
 465                    A6XX_SP_SRGB_CNTL(srgb_cntl));
 466
 467    tu_cs_emit_regs(cs,
 468                    A6XX_RB_RENDER_COMPONENTS(
 469                       .rt0 = mrt_comp[0],
 470                       .rt1 = mrt_comp[1],
 471                       .rt2 = mrt_comp[2],
 472                       .rt3 = mrt_comp[3],
 473                       .rt4 = mrt_comp[4],
 474                       .rt5 = mrt_comp[5],
 475                       .rt6 = mrt_comp[6],
 476                       .rt7 = mrt_comp[7]));
 477
 478    tu_cs_emit_regs(cs,
 479                    A6XX_SP_FS_RENDER_COMPONENTS(
 480                       .rt0 = mrt_comp[0],
 481                       .rt1 = mrt_comp[1],
 482                       .rt2 = mrt_comp[2],
 483                       .rt3 = mrt_comp[3],
 484                       .rt4 = mrt_comp[4],
 485                       .rt5 = mrt_comp[5],
 486                       .rt6 = mrt_comp[6],
 487                       .rt7 = mrt_comp[7]));
 488 }
 489
 490 static void
 491 tu6_emit_msaa(struct tu_cmd_buffer *cmd,
 492               const struct tu_subpass *subpass,
 493               struct tu_cs *cs)
 494 {
 495    const enum a3xx_msaa_samples samples = tu_msaa_samples(subpass->samples);
 496    bool msaa_disable = samples == MSAA_ONE;
 497
 498    tu_cs_emit_regs(cs,
 499                    A6XX_SP_TP_RAS_MSAA_CNTL(samples),
 500                    A6XX_SP_TP_DEST_MSAA_CNTL(.samples = samples,
 501                                              .msaa_disable = msaa_disable));
 502
 503    tu_cs_emit_regs(cs,
 504                    A6XX_GRAS_RAS_MSAA_CNTL(samples),
 505                    A6XX_GRAS_DEST_MSAA_CNTL(.samples = samples,
 506                                             .msaa_disable = msaa_disable));
 507
 508    tu_cs_emit_regs(cs,
 509                    A6XX_RB_RAS_MSAA_CNTL(samples),
 510                    A6XX_RB_DEST_MSAA_CNTL(.samples = samples,
 511                                           .msaa_disable = msaa_disable));
 512
 513    tu_cs_emit_regs(cs,
 514                    A6XX_RB_MSAA_CNTL(samples));
 515 }
 516
 517 static void
 518 tu6_emit_bin_size(struct tu_cmd_buffer *cmd, struct tu_cs *cs, uint32_t flags)
 519 {
 520    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
 521    const uint32_t bin_w = tiling->tile0.extent.width;
 522    const uint32_t bin_h = tiling->tile0.extent.height;
 523
 524    tu_cs_emit_regs(cs,
 525                    A6XX_GRAS_BIN_CONTROL(.binw = bin_w,
 526                                          .binh = bin_h,
 527                                          .dword = flags));
 528
 529    tu_cs_emit_regs(cs,
 530                    A6XX_RB_BIN_CONTROL(.binw = bin_w,
 531                                        .binh = bin_h,
 532                                        .dword = flags));
 533
 534    /* no flag for RB_BIN_CONTROL2... */
 535    tu_cs_emit_regs(cs,
 536                    A6XX_RB_BIN_CONTROL2(.binw = bin_w,
 537                                         .binh = bin_h));
 538 }
 539
 540 static void
 541 tu6_emit_render_cntl(struct tu_cmd_buffer *cmd,
 542                      struct tu_cs *cs,
 543                      bool binning)
 544 {
 545    uint32_t cntl = 0;
 546    cntl |= A6XX_RB_RENDER_CNTL_UNK4;
 547    if (binning)
 548       cntl |= A6XX_RB_RENDER_CNTL_BINNING;
 549
 550    tu_cs_emit_pkt7(cs, CP_REG_WRITE, 3);
 551    tu_cs_emit(cs, 0x2);
 552    tu_cs_emit(cs, REG_A6XX_RB_RENDER_CNTL);
 553    tu_cs_emit(cs, cntl);
 554 }
 555
 556 static void
 557 tu6_emit_blit_scissor(struct tu_cmd_buffer *cmd, struct tu_cs *cs, bool align)
 558 {
 559    const VkRect2D *render_area = &cmd->state.tiling_config.render_area;
 560    uint32_t x1 = render_area->offset.x;
 561    uint32_t y1 = render_area->offset.y;
 562    uint32_t x2 = x1 + render_area->extent.width - 1;
 563    uint32_t y2 = y1 + render_area->extent.height - 1;
 564
 565    /* TODO: alignment requirement seems to be less than tile_align_w/h */
 566    if (align) {
 567       x1 = x1 & ~cmd->device->physical_device->tile_align_w;
 568       y1 = y1 & ~cmd->device->physical_device->tile_align_h;
 569       x2 = ALIGN_POT(x2 + 1, cmd->device->physical_device->tile_align_w) - 1;
 570       y2 = ALIGN_POT(y2 + 1, cmd->device->physical_device->tile_align_h) - 1;
 571    }
 572
 573    tu_cs_emit_regs(cs,
 574                    A6XX_RB_BLIT_SCISSOR_TL(.x = x1, .y = y1),
 575                    A6XX_RB_BLIT_SCISSOR_BR(.x = x2, .y = y2));
 576 }
 577
 578 static void
 579 tu6_emit_blit_info(struct tu_cmd_buffer *cmd,
 580                    struct tu_cs *cs,
 581                    const struct tu_image_view *iview,
 582                    uint32_t gmem_offset,
 583                    bool resolve)
 584 {
 585    tu_cs_emit_regs(cs,
 586                    A6XX_RB_BLIT_INFO(.unk0 = !resolve, .gmem = !resolve));
 587
 588    const struct tu_native_format *format =
 589       tu6_get_native_format(iview->vk_format);
 590    assert(format && format->rb >= 0);
 591
 592    enum a6xx_tile_mode tile_mode =
 593       tu6_get_image_tile_mode(iview->image, iview->base_mip);
 594    tu_cs_emit_regs(cs,
 595                    A6XX_RB_BLIT_DST_INFO(
 596                       .tile_mode = tile_mode,
 597                       .samples = tu_msaa_samples(iview->image->samples),
 598                       .color_format = format->rb,
 599                       .color_swap = format->swap,
 600                       .flags = iview->image->layout.ubwc_size != 0),
 601                    A6XX_RB_BLIT_DST(tu_image_view_base_ref(iview)),
 602                    A6XX_RB_BLIT_DST_PITCH(tu_image_stride(iview->image, iview->base_mip)),
 603                    A6XX_RB_BLIT_DST_ARRAY_PITCH(iview->image->layout.layer_size));
 604
 605    if (iview->image->layout.ubwc_size) {
 606       tu_cs_emit_regs(cs,
 607                       A6XX_RB_BLIT_FLAG_DST(tu_image_view_ubwc_base_ref(iview)),
 608                       A6XX_RB_BLIT_FLAG_DST_PITCH(tu_image_view_ubwc_pitches(iview)));
 609    }
 610
 611    tu_cs_emit_regs(cs,
 612                    A6XX_RB_BLIT_BASE_GMEM(gmem_offset));
 613 }
 614
 615 static void
 616 tu6_emit_blit(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 617 {
 618    tu6_emit_marker(cmd, cs);
 619    tu6_emit_event_write(cmd, cs, BLIT, false);
 620    tu6_emit_marker(cmd, cs);
 621 }
 622
 623 static void
 624 tu6_emit_window_scissor(struct tu_cmd_buffer *cmd,
 625                         struct tu_cs *cs,
 626                         uint32_t x1,
 627                         uint32_t y1,
 628                         uint32_t x2,
 629                         uint32_t y2)
 630 {
 631    tu_cs_emit_regs(cs,
 632                    A6XX_GRAS_SC_WINDOW_SCISSOR_TL(.x = x1, .y = y1),
 633                    A6XX_GRAS_SC_WINDOW_SCISSOR_BR(.x = x2, .y = y2));
 634
 635    tu_cs_emit_regs(cs,
 636                    A6XX_GRAS_RESOLVE_CNTL_1(.x = x1, .y = y1),
 637                    A6XX_GRAS_RESOLVE_CNTL_2(.x = x2, .y = y2));
 638 }
 639
 640 static void
 641 tu6_emit_window_offset(struct tu_cmd_buffer *cmd,
 642                        struct tu_cs *cs,
 643                        uint32_t x1,
 644                        uint32_t y1)
 645 {
 646    tu_cs_emit_regs(cs,
 647                    A6XX_RB_WINDOW_OFFSET(.x = x1, .y = y1));
 648
 649    tu_cs_emit_regs(cs,
 650                    A6XX_RB_WINDOW_OFFSET2(.x = x1, .y = y1));
 651
 652    tu_cs_emit_regs(cs,
 653                    A6XX_SP_WINDOW_OFFSET(.x = x1, .y = y1));
 654
 655    tu_cs_emit_regs(cs,
 656                    A6XX_SP_TP_WINDOW_OFFSET(.x = x1, .y = y1));
 657 }
 658
 659 static bool
 660 use_hw_binning(struct tu_cmd_buffer *cmd)
 661 {
 662    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
 663
 664    if (unlikely(cmd->device->physical_device->instance->debug_flags & TU_DEBUG_NOBIN))
 665       return false;
 666
 667    return (tiling->tile_count.width * tiling->tile_count.height) > 2;
 668 }
 669
 670 static void
 671 tu6_emit_tile_select(struct tu_cmd_buffer *cmd,
 672                      struct tu_cs *cs,
 673                      const struct tu_tile *tile)
 674 {
 675    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
 676    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(0x7));
 677
 678    tu6_emit_marker(cmd, cs);
 679    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
 680    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_GMEM) | 0x10);
 681    tu6_emit_marker(cmd, cs);
 682
 683    const uint32_t x1 = tile->begin.x;
 684    const uint32_t y1 = tile->begin.y;
 685    const uint32_t x2 = tile->end.x - 1;
 686    const uint32_t y2 = tile->end.y - 1;
 687    tu6_emit_window_scissor(cmd, cs, x1, y1, x2, y2);
 688    tu6_emit_window_offset(cmd, cs, x1, y1);
 689
 690    tu_cs_emit_regs(cs,
 691                    A6XX_VPC_SO_OVERRIDE(.so_disable = true));
 692
 693    if (use_hw_binning(cmd)) {
 694       tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
 695
 696       tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
 697       tu_cs_emit(cs, 0x0);
 698
 699       tu_cs_emit_pkt7(cs, CP_REG_TEST, 1);
 700       tu_cs_emit(cs, A6XX_CP_REG_TEST_0_REG(OVERFLOW_FLAG_REG) |
 701                      A6XX_CP_REG_TEST_0_BIT(0) |
 702                      A6XX_CP_REG_TEST_0_WAIT_FOR_ME);
 703
 704       tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
 705       tu_cs_emit(cs, CP_COND_REG_EXEC_0_MODE(PRED_TEST));
 706       tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(11));
 707
 708       /* if (no overflow) */ {
 709          tu_cs_emit_pkt7(cs, CP_SET_BIN_DATA5, 7);
 710          tu_cs_emit(cs, cmd->state.tiling_config.pipe_sizes[tile->pipe] |
 711                         CP_SET_BIN_DATA5_0_VSC_N(tile->slot));
 712          tu_cs_emit_qw(cs, cmd->vsc_data.iova + tile->pipe * cmd->vsc_data_pitch);
 713          tu_cs_emit_qw(cs, cmd->vsc_data.iova + (tile->pipe * 4) + (32 * cmd->vsc_data_pitch));
 714          tu_cs_emit_qw(cs, cmd->vsc_data2.iova + (tile->pipe * cmd->vsc_data2_pitch));
 715
 716          tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
 717          tu_cs_emit(cs, 0x0);
 718
 719          /* use a NOP packet to skip over the 'else' side: */
 720          tu_cs_emit_pkt7(cs, CP_NOP, 2);
 721       } /* else */ {
 722          tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
 723          tu_cs_emit(cs, 0x1);
 724       }
 725
 726       tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
 727       tu_cs_emit(cs, 0x0);
 728
 729       tu_cs_emit_regs(cs,
 730                       A6XX_RB_UNKNOWN_8804(0));
 731
 732       tu_cs_emit_regs(cs,
 733                       A6XX_SP_TP_UNKNOWN_B304(0));
 734
 735       tu_cs_emit_regs(cs,
 736                       A6XX_GRAS_UNKNOWN_80A4(0));
 737    } else {
 738       tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
 739       tu_cs_emit(cs, 0x1);
 740
 741       tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
 742       tu_cs_emit(cs, 0x0);
 743    }
 744 }
 745
 746 static void
 747 tu6_emit_load_attachment(struct tu_cmd_buffer *cmd, struct tu_cs *cs, uint32_t a)
 748 {
 749    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
 750    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 751    const struct tu_image_view *iview = fb->attachments[a].attachment;
 752    const struct tu_render_pass_attachment *attachment =
 753       &cmd->state.pass->attachments[a];
 754
 755    if (attachment->gmem_offset < 0)
 756       return;
 757
 758    const uint32_t x1 = tiling->render_area.offset.x;
 759    const uint32_t y1 = tiling->render_area.offset.y;
 760    const uint32_t x2 = x1 + tiling->render_area.extent.width;
 761    const uint32_t y2 = y1 + tiling->render_area.extent.height;
 762    const uint32_t tile_x2 =
 763       tiling->tile0.offset.x + tiling->tile0.extent.width * tiling->tile_count.width;
 764    const uint32_t tile_y2 =
 765       tiling->tile0.offset.y + tiling->tile0.extent.height * tiling->tile_count.height;
 766    bool need_load =
 767       x1 != tiling->tile0.offset.x || x2 != MIN2(fb->width, tile_x2) ||
 768       y1 != tiling->tile0.offset.y || y2 != MIN2(fb->height, tile_y2);
 769
 770    if (need_load)
 771       tu_finishme("improve handling of unaligned render area");
 772
 773    if (attachment->load_op == VK_ATTACHMENT_LOAD_OP_LOAD)
 774       need_load = true;
 775
 776    if (vk_format_has_stencil(iview->vk_format) &&
 777        attachment->stencil_load_op == VK_ATTACHMENT_LOAD_OP_LOAD)
 778       need_load = true;
 779
 780    if (need_load) {
 781       tu6_emit_blit_info(cmd, cs, iview, attachment->gmem_offset, false);
 782       tu6_emit_blit(cmd, cs);
 783    }
 784 }
 785
 786 static void
 787 tu6_emit_clear_attachment(struct tu_cmd_buffer *cmd, struct tu_cs *cs,
 788                           uint32_t a,
 789                           const VkRenderPassBeginInfo *info)
 790 {
 791    const struct tu_framebuffer *fb = cmd->state.framebuffer;
 792    const struct tu_image_view *iview = fb->attachments[a].attachment;
 793    const struct tu_render_pass_attachment *attachment =
 794       &cmd->state.pass->attachments[a];
 795    unsigned clear_mask = 0;
 796
 797    /* note: this means it isn't used by any subpass and shouldn't be cleared anyway */
 798    if (attachment->gmem_offset < 0)
 799       return;
 800
 801    if (attachment->load_op == VK_ATTACHMENT_LOAD_OP_CLEAR)
 802       clear_mask = 0xf;
 803
 804    if (vk_format_has_stencil(iview->vk_format)) {
 805       clear_mask &= 0x1;
 806       if (attachment->stencil_load_op == VK_ATTACHMENT_LOAD_OP_CLEAR)
 807          clear_mask |= 0x2;
 808    }
 809    if (!clear_mask)
 810       return;
 811
 812    const struct tu_native_format *format =
 813       tu6_get_native_format(iview->vk_format);
 814    assert(format && format->rb >= 0);
 815
 816    tu_cs_emit_regs(cs,
 817                    A6XX_RB_BLIT_DST_INFO(.color_format = format->rb));
 818
 819    tu_cs_emit_regs(cs,
 820                    A6XX_RB_BLIT_INFO(.gmem = true,
 821                                      .clear_mask = clear_mask));
 822
 823    tu_cs_emit_regs(cs,
 824                    A6XX_RB_BLIT_BASE_GMEM(attachment->gmem_offset));
 825
 826    tu_cs_emit_regs(cs,
 827                    A6XX_RB_UNKNOWN_88D0(0));
 828
 829    uint32_t clear_vals[4] = { 0 };
 830    tu_pack_clear_value(&info->pClearValues[a], iview->vk_format, clear_vals);
 831
 832    tu_cs_emit_regs(cs,
 833                    A6XX_RB_BLIT_CLEAR_COLOR_DW0(clear_vals[0]),
 834                    A6XX_RB_BLIT_CLEAR_COLOR_DW1(clear_vals[1]),
 835                    A6XX_RB_BLIT_CLEAR_COLOR_DW2(clear_vals[2]),
 836                    A6XX_RB_BLIT_CLEAR_COLOR_DW3(clear_vals[3]));
 837
 838    tu6_emit_blit(cmd, cs);
 839 }
 840
 841 static void
 842 tu6_emit_store_attachment(struct tu_cmd_buffer *cmd,
 843                           struct tu_cs *cs,
 844                           uint32_t a,
 845                           uint32_t gmem_a)
 846 {
 847    if (cmd->state.pass->attachments[a].store_op == VK_ATTACHMENT_STORE_OP_DONT_CARE)
 848       return;
 849
 850    tu6_emit_blit_info(cmd, cs,
 851                       cmd->state.framebuffer->attachments[a].attachment,
 852                       cmd->state.pass->attachments[gmem_a].gmem_offset, true);
 853    tu6_emit_blit(cmd, cs);
 854 }
 855
 856 static void
 857 tu6_emit_tile_store(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 858 {
 859    const struct tu_render_pass *pass = cmd->state.pass;
 860    const struct tu_subpass *subpass = &pass->subpasses[pass->subpass_count-1];
 861
 862    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
 863    tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
 864                      CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
 865                      CP_SET_DRAW_STATE__0_GROUP_ID(0));
 866    tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
 867    tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
 868
 869    tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
 870    tu_cs_emit(cs, 0x0);
 871
 872    tu6_emit_marker(cmd, cs);
 873    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
 874    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_RESOLVE) | 0x10);
 875    tu6_emit_marker(cmd, cs);
 876
 877    tu6_emit_blit_scissor(cmd, cs, true);
 878
 879    for (uint32_t a = 0; a < pass->attachment_count; ++a) {
 880       if (pass->attachments[a].gmem_offset >= 0)
 881          tu6_emit_store_attachment(cmd, cs, a, a);
 882    }
 883
 884    if (subpass->resolve_attachments) {
 885       for (unsigned i = 0; i < subpass->color_count; i++) {
 886          uint32_t a = subpass->resolve_attachments[i].attachment;
 887          if (a != VK_ATTACHMENT_UNUSED)
 888             tu6_emit_store_attachment(cmd, cs, a,
 889                                       subpass->color_attachments[i].attachment);
 890       }
 891    }
 892 }
 893
 894 static void
 895 tu6_emit_restart_index(struct tu_cs *cs, uint32_t restart_index)
 896 {
 897    tu_cs_emit_regs(cs,
 898                    A6XX_PC_RESTART_INDEX(restart_index));
 899 }
 900
 901 static void
 902 tu6_init_hw(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
 903 {
 904    VkResult result = tu_cs_reserve_space(cmd->device, cs, 256);
 905    if (result != VK_SUCCESS) {
 906       cmd->record_result = result;
 907       return;
 908    }
 909
 910    tu6_emit_cache_flush(cmd, cs);
 911
 912    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UPDATE_CNTL, 0xfffff);
 913
 914    tu_cs_emit_write_reg(cs, REG_A6XX_RB_CCU_CNTL, 0x7c400004);
 915    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8E04, 0x00100000);
 916    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE04, 0x8);
 917    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE00, 0);
 918    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE0F, 0x3f);
 919    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B605, 0x44);
 920    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B600, 0x100000);
 921    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE00, 0x80);
 922    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE01, 0);
 923
 924    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9600, 0);
 925    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8600, 0x880);
 926    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE04, 0);
 927    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE03, 0x00000410);
 928    tu_cs_emit_write_reg(cs, REG_A6XX_SP_IBO_COUNT, 0);
 929    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B182, 0);
 930    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BB11, 0);
 931    tu_cs_emit_write_reg(cs, REG_A6XX_UCHE_UNKNOWN_0E12, 0x3200000);
 932    tu_cs_emit_write_reg(cs, REG_A6XX_UCHE_CLIENT_PF, 4);
 933    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8E01, 0x0);
 934    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AB00, 0x5);
 935    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_ADD_OFFSET, A6XX_VFD_ADD_OFFSET_VERTEX);
 936    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8811, 0x00000010);
 937    tu_cs_emit_write_reg(cs, REG_A6XX_PC_MODE_CNTL, 0x1f);
 938
 939    tu_cs_emit_write_reg(cs, REG_A6XX_RB_SRGB_CNTL, 0);
 940
 941    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8101, 0);
 942    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_SAMPLE_CNTL, 0);
 943    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8110, 0);
 944
 945    tu_cs_emit_write_reg(cs, REG_A6XX_RB_RENDER_CONTROL0, 0x401);
 946    tu_cs_emit_write_reg(cs, REG_A6XX_RB_RENDER_CONTROL1, 0);
 947    tu_cs_emit_write_reg(cs, REG_A6XX_RB_FS_OUTPUT_CNTL0, 0);
 948    tu_cs_emit_write_reg(cs, REG_A6XX_RB_SAMPLE_CNTL, 0);
 949    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8818, 0);
 950    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8819, 0);
 951    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881A, 0);
 952    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881B, 0);
 953    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881C, 0);
 954    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881D, 0);
 955    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881E, 0);
 956    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_88F0, 0);
 957
 958    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9101, 0xffff00);
 959    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9107, 0);
 960
 961    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9236, 1);
 962    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9300, 0);
 963
 964    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_SO_OVERRIDE,
 965                         A6XX_VPC_SO_OVERRIDE_SO_DISABLE);
 966
 967    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9801, 0);
 968    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9806, 0);
 969    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9980, 0);
 970
 971    tu_cs_emit_write_reg(cs, REG_A6XX_PC_PRIMITIVE_CNTL_6, 0);
 972    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9B07, 0);
 973
 974    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_A81B, 0);
 975
 976    tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B183, 0);
 977
 978    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8099, 0);
 979    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_809B, 0);
 980    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A0, 2);
 981    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80AF, 0);
 982    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9210, 0);
 983    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9211, 0);
 984    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9602, 0);
 985    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9981, 0x3);
 986    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9E72, 0);
 987    tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9108, 0x3);
 988    tu_cs_emit_write_reg(cs, REG_A6XX_SP_TP_UNKNOWN_B304, 0);
 989    tu_cs_emit_write_reg(cs, REG_A6XX_SP_TP_UNKNOWN_B309, 0x000000a2);
 990    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8804, 0);
 991    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A4, 0);
 992    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A5, 0);
 993    tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A6, 0);
 994    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8805, 0);
 995    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8806, 0);
 996    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8878, 0);
 997    tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8879, 0);
 998    tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_CONTROL_5_REG, 0xfc);
 999
1000    tu6_emit_marker(cmd, cs);
1001
1002    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_MODE_CNTL, 0x00000000);
1003
1004    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_UNKNOWN_A008, 0);
1005
1006    tu_cs_emit_write_reg(cs, REG_A6XX_PC_MODE_CNTL, 0x0000001f);
1007
1008    /* we don't use this yet.. probably best to disable.. */
1009    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
1010    tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
1011                      CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
1012                      CP_SET_DRAW_STATE__0_GROUP_ID(0));
1013    tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
1014    tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
1015
1016    tu_cs_emit_regs(cs,
1017                    A6XX_VPC_SO_BUFFER_BASE(0),
1018                    A6XX_VPC_SO_BUFFER_SIZE(0));
1019
1020    tu_cs_emit_regs(cs,
1021                    A6XX_VPC_SO_FLUSH_BASE(0));
1022
1023    tu_cs_emit_regs(cs,
1024                    A6XX_VPC_SO_BUF_CNTL(0));
1025
1026    tu_cs_emit_regs(cs,
1027                    A6XX_VPC_SO_BUFFER_OFFSET(0, 0));
1028
1029    tu_cs_emit_regs(cs,
1030                    A6XX_VPC_SO_BUFFER_BASE(1, 0),
1031                    A6XX_VPC_SO_BUFFER_SIZE(1, 0));
1032
1033    tu_cs_emit_regs(cs,
1034                    A6XX_VPC_SO_BUFFER_OFFSET(1, 0),
1035                    A6XX_VPC_SO_FLUSH_BASE(1, 0),
1036                    A6XX_VPC_SO_BUFFER_BASE(2, 0),
1037                    A6XX_VPC_SO_BUFFER_SIZE(2, 0));
1038
1039    tu_cs_emit_regs(cs,
1040                    A6XX_VPC_SO_BUFFER_OFFSET(2, 0),
1041                    A6XX_VPC_SO_FLUSH_BASE(2, 0),
1042                    A6XX_VPC_SO_BUFFER_BASE(3, 0),
1043                    A6XX_VPC_SO_BUFFER_SIZE(3, 0));
1044
1045    tu_cs_emit_regs(cs,
1046                    A6XX_VPC_SO_BUFFER_OFFSET(3, 0),
1047                    A6XX_VPC_SO_FLUSH_BASE(3, 0));
1048
1049    tu_cs_emit_regs(cs,
1050                    A6XX_SP_HS_CTRL_REG0(0));
1051
1052    tu_cs_emit_regs(cs,
1053                    A6XX_SP_GS_CTRL_REG0(0));
1054
1055    tu_cs_emit_regs(cs,
1056                    A6XX_GRAS_LRZ_CNTL(0));
1057
1058    tu_cs_emit_regs(cs,
1059                    A6XX_RB_LRZ_CNTL(0));
1060
1061    tu_cs_sanity_check(cs);
1062 }
1063
1064 static void
1065 tu6_cache_flush(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1066 {
1067    unsigned seqno;
1068
1069    seqno = tu6_emit_event_write(cmd, cs, CACHE_FLUSH_AND_INV_EVENT, true);
1070
1071    tu_cs_emit_pkt7(cs, CP_WAIT_REG_MEM, 6);
1072    tu_cs_emit(cs, CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ) |
1073                   CP_WAIT_REG_MEM_0_POLL_MEMORY);
1074    tu_cs_emit_qw(cs, cmd->scratch_bo.iova);
1075    tu_cs_emit(cs, CP_WAIT_REG_MEM_3_REF(seqno));
1076    tu_cs_emit(cs, CP_WAIT_REG_MEM_4_MASK(~0));
1077    tu_cs_emit(cs, CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(16));
1078
1079    seqno = tu6_emit_event_write(cmd, cs, CACHE_FLUSH_TS, true);
1080
1081    tu_cs_emit_pkt7(cs, CP_WAIT_MEM_GTE, 4);
1082    tu_cs_emit(cs, CP_WAIT_MEM_GTE_0_RESERVED(0));
1083    tu_cs_emit_qw(cs, cmd->scratch_bo.iova);
1084    tu_cs_emit(cs, CP_WAIT_MEM_GTE_3_REF(seqno));
1085 }
1086
1087 static void
1088 update_vsc_pipe(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1089 {
1090    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1091
1092    tu_cs_emit_regs(cs,
1093                    A6XX_VSC_BIN_SIZE(.width = tiling->tile0.extent.width,
1094                                      .height = tiling->tile0.extent.height),
1095                    A6XX_VSC_SIZE_ADDRESS(.bo = &cmd->vsc_data,
1096                                          .bo_offset = 32 * cmd->vsc_data_pitch));
1097
1098    tu_cs_emit_regs(cs,
1099                    A6XX_VSC_BIN_COUNT(.nx = tiling->tile_count.width,
1100                                       .ny = tiling->tile_count.height));
1101
1102    tu_cs_emit_pkt4(cs, REG_A6XX_VSC_PIPE_CONFIG_REG(0), 32);
1103    for (unsigned i = 0; i < 32; i++)
1104       tu_cs_emit(cs, tiling->pipe_config[i]);
1105
1106    tu_cs_emit_regs(cs,
1107                    A6XX_VSC_PIPE_DATA2_ADDRESS(.bo = &cmd->vsc_data2),
1108                    A6XX_VSC_PIPE_DATA2_PITCH(cmd->vsc_data2_pitch),
1109                    A6XX_VSC_PIPE_DATA2_ARRAY_PITCH(cmd->vsc_data2.size));
1110
1111    tu_cs_emit_regs(cs,
1112                    A6XX_VSC_PIPE_DATA_ADDRESS(.bo = &cmd->vsc_data),
1113                    A6XX_VSC_PIPE_DATA_PITCH(cmd->vsc_data_pitch),
1114                    A6XX_VSC_PIPE_DATA_ARRAY_PITCH(cmd->vsc_data.size));
1115 }
1116
1117 static void
1118 emit_vsc_overflow_test(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1119 {
1120    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1121    const uint32_t used_pipe_count =
1122       tiling->pipe_count.width * tiling->pipe_count.height;
1123
1124    /* Clear vsc_scratch: */
1125    tu_cs_emit_pkt7(cs, CP_MEM_WRITE, 3);
1126    tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1127    tu_cs_emit(cs, 0x0);
1128
1129    /* Check for overflow, write vsc_scratch if detected: */
1130    for (int i = 0; i < used_pipe_count; i++) {
1131       tu_cs_emit_pkt7(cs, CP_COND_WRITE5, 8);
1132       tu_cs_emit(cs, CP_COND_WRITE5_0_FUNCTION(WRITE_GE) |
1133             CP_COND_WRITE5_0_WRITE_MEMORY);
1134       tu_cs_emit(cs, CP_COND_WRITE5_1_POLL_ADDR_LO(REG_A6XX_VSC_SIZE_REG(i)));
1135       tu_cs_emit(cs, CP_COND_WRITE5_2_POLL_ADDR_HI(0));
1136       tu_cs_emit(cs, CP_COND_WRITE5_3_REF(cmd->vsc_data_pitch));
1137       tu_cs_emit(cs, CP_COND_WRITE5_4_MASK(~0));
1138       tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1139       tu_cs_emit(cs, CP_COND_WRITE5_7_WRITE_DATA(1 + cmd->vsc_data_pitch));
1140
1141       tu_cs_emit_pkt7(cs, CP_COND_WRITE5, 8);
1142       tu_cs_emit(cs, CP_COND_WRITE5_0_FUNCTION(WRITE_GE) |
1143             CP_COND_WRITE5_0_WRITE_MEMORY);
1144       tu_cs_emit(cs, CP_COND_WRITE5_1_POLL_ADDR_LO(REG_A6XX_VSC_SIZE2_REG(i)));
1145       tu_cs_emit(cs, CP_COND_WRITE5_2_POLL_ADDR_HI(0));
1146       tu_cs_emit(cs, CP_COND_WRITE5_3_REF(cmd->vsc_data2_pitch));
1147       tu_cs_emit(cs, CP_COND_WRITE5_4_MASK(~0));
1148       tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1149       tu_cs_emit(cs, CP_COND_WRITE5_7_WRITE_DATA(3 + cmd->vsc_data2_pitch));
1150    }
1151
1152    tu_cs_emit_pkt7(cs, CP_WAIT_MEM_WRITES, 0);
1153
1154    tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
1155
1156    tu_cs_emit_pkt7(cs, CP_MEM_TO_REG, 3);
1157    tu_cs_emit(cs, CP_MEM_TO_REG_0_REG(OVERFLOW_FLAG_REG) |
1158          CP_MEM_TO_REG_0_CNT(1 - 1));
1159    tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_SCRATCH);
1160
1161    /*
1162     * This is a bit awkward, we really want a way to invert the
1163     * CP_REG_TEST/CP_COND_REG_EXEC logic, so that we can conditionally
1164     * execute cmds to use hwbinning when a bit is *not* set.  This
1165     * dance is to invert OVERFLOW_FLAG_REG
1166     *
1167     * A CP_NOP packet is used to skip executing the 'else' clause
1168     * if (b0 set)..
1169     */
1170
1171    /* b0 will be set if VSC_DATA or VSC_DATA2 overflow: */
1172    tu_cs_emit_pkt7(cs, CP_REG_TEST, 1);
1173    tu_cs_emit(cs, A6XX_CP_REG_TEST_0_REG(OVERFLOW_FLAG_REG) |
1174          A6XX_CP_REG_TEST_0_BIT(0) |
1175          A6XX_CP_REG_TEST_0_WAIT_FOR_ME);
1176
1177    tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
1178    tu_cs_emit(cs, CP_COND_REG_EXEC_0_MODE(PRED_TEST));
1179    tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(7));
1180
1181    /* if (b0 set) */ {
1182       /*
1183        * On overflow, mirror the value to control->vsc_overflow
1184        * which CPU is checking to detect overflow (see
1185        * check_vsc_overflow())
1186        */
1187       tu_cs_emit_pkt7(cs, CP_REG_TO_MEM, 3);
1188       tu_cs_emit(cs, CP_REG_TO_MEM_0_REG(OVERFLOW_FLAG_REG) |
1189             CP_REG_TO_MEM_0_CNT(0));
1190       tu_cs_emit_qw(cs, cmd->scratch_bo.iova + VSC_OVERFLOW);
1191
1192       tu_cs_emit_pkt4(cs, OVERFLOW_FLAG_REG, 1);
1193       tu_cs_emit(cs, 0x0);
1194
1195       tu_cs_emit_pkt7(cs, CP_NOP, 2);  /* skip 'else' when 'if' is taken */
1196    } /* else */ {
1197       tu_cs_emit_pkt4(cs, OVERFLOW_FLAG_REG, 1);
1198       tu_cs_emit(cs, 0x1);
1199    }
1200 }
1201
1202 static void
1203 tu6_emit_binning_pass(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1204 {
1205    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1206
1207    uint32_t x1 = tiling->tile0.offset.x;
1208    uint32_t y1 = tiling->tile0.offset.y;
1209    uint32_t x2 = tiling->render_area.offset.x + tiling->render_area.extent.width - 1;
1210    uint32_t y2 = tiling->render_area.offset.y + tiling->render_area.extent.height - 1;
1211
1212    tu6_emit_window_scissor(cmd, cs, x1, y1, x2, y2);
1213
1214    tu6_emit_marker(cmd, cs);
1215    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1216    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_BINNING));
1217    tu6_emit_marker(cmd, cs);
1218
1219    tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
1220    tu_cs_emit(cs, 0x1);
1221
1222    tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
1223    tu_cs_emit(cs, 0x1);
1224
1225    tu_cs_emit_wfi(cs);
1226
1227    tu_cs_emit_regs(cs,
1228                    A6XX_VFD_MODE_CNTL(.binning_pass = true));
1229
1230    update_vsc_pipe(cmd, cs);
1231
1232    tu_cs_emit_regs(cs,
1233                    A6XX_PC_UNKNOWN_9805(.unknown = 0x1));
1234
1235    tu_cs_emit_regs(cs,
1236                    A6XX_SP_UNKNOWN_A0F8(.unknown = 0x1));
1237
1238    tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, 1);
1239    tu_cs_emit(cs, UNK_2C);
1240
1241    tu_cs_emit_regs(cs,
1242                    A6XX_RB_WINDOW_OFFSET(.x = 0, .y = 0));
1243
1244    tu_cs_emit_regs(cs,
1245                    A6XX_SP_TP_WINDOW_OFFSET(.x = 0, .y = 0));
1246
1247    /* emit IB to binning drawcmds: */
1248    tu_cs_emit_call(cs, &cmd->draw_cs);
1249
1250    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
1251    tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
1252                   CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
1253                   CP_SET_DRAW_STATE__0_GROUP_ID(0));
1254    tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
1255    tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
1256
1257    tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, 1);
1258    tu_cs_emit(cs, UNK_2D);
1259
1260    tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE, false);
1261    tu6_cache_flush(cmd, cs);
1262
1263    tu_cs_emit_wfi(cs);
1264
1265    tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
1266
1267    emit_vsc_overflow_test(cmd, cs);
1268
1269    tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
1270    tu_cs_emit(cs, 0x0);
1271
1272    tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
1273    tu_cs_emit(cs, 0x0);
1274
1275    tu_cs_emit_wfi(cs);
1276
1277    tu_cs_emit_regs(cs,
1278                    A6XX_RB_CCU_CNTL(.unknown = 0x7c400004));
1279
1280    cmd->wait_for_idle = false;
1281 }
1282
1283 static void
1284 tu6_render_begin(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1285 {
1286    VkResult result = tu_cs_reserve_space(cmd->device, cs, 1024);
1287    if (result != VK_SUCCESS) {
1288       cmd->record_result = result;
1289       return;
1290    }
1291
1292    tu6_emit_lrz_flush(cmd, cs);
1293
1294    /* lrz clear? */
1295
1296    tu6_emit_cache_flush(cmd, cs);
1297
1298    tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1299    tu_cs_emit(cs, 0x0);
1300
1301    /* 0x10000000 for BYPASS.. 0x7c13c080 for GMEM: */
1302    tu6_emit_wfi(cmd, cs);
1303    tu_cs_emit_regs(cs,
1304                    A6XX_RB_CCU_CNTL(0x7c400004));
1305
1306    if (use_hw_binning(cmd)) {
1307       tu6_emit_bin_size(cmd, cs, A6XX_RB_BIN_CONTROL_BINNING_PASS | 0x6000000);
1308
1309       tu6_emit_render_cntl(cmd, cs, true);
1310
1311       tu6_emit_binning_pass(cmd, cs);
1312
1313       tu6_emit_bin_size(cmd, cs, A6XX_RB_BIN_CONTROL_USE_VIZ | 0x6000000);
1314
1315       tu_cs_emit_regs(cs,
1316                       A6XX_VFD_MODE_CNTL(0));
1317
1318       tu_cs_emit_regs(cs, A6XX_PC_UNKNOWN_9805(.unknown = 0x1));
1319
1320       tu_cs_emit_regs(cs, A6XX_SP_UNKNOWN_A0F8(.unknown = 0x1));
1321
1322       tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1323       tu_cs_emit(cs, 0x1);
1324    } else {
1325       tu6_emit_bin_size(cmd, cs, 0x6000000);
1326    }
1327
1328    tu6_emit_render_cntl(cmd, cs, false);
1329
1330    tu_cs_sanity_check(cs);
1331 }
1332
1333 static void
1334 tu6_render_tile(struct tu_cmd_buffer *cmd,
1335                 struct tu_cs *cs,
1336                 const struct tu_tile *tile)
1337 {
1338    const uint32_t render_tile_space = 256 + tu_cs_get_call_size(&cmd->draw_cs);
1339    VkResult result = tu_cs_reserve_space(cmd->device, cs, render_tile_space);
1340    if (result != VK_SUCCESS) {
1341       cmd->record_result = result;
1342       return;
1343    }
1344
1345    tu6_emit_tile_select(cmd, cs, tile);
1346    tu_cs_emit_ib(cs, &cmd->state.tile_load_ib);
1347
1348    tu_cs_emit_call(cs, &cmd->draw_cs);
1349    cmd->wait_for_idle = true;
1350
1351    if (use_hw_binning(cmd)) {
1352       tu_cs_emit_pkt7(cs, CP_REG_TEST, 1);
1353       tu_cs_emit(cs, A6XX_CP_REG_TEST_0_REG(OVERFLOW_FLAG_REG) |
1354                      A6XX_CP_REG_TEST_0_BIT(0) |
1355                      A6XX_CP_REG_TEST_0_WAIT_FOR_ME);
1356
1357       tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
1358       tu_cs_emit(cs, 0x10000000);
1359       tu_cs_emit(cs, 2);  /* conditionally execute next 2 dwords */
1360
1361       /* if (no overflow) */ {
1362          tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1363          tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(0x5) | 0x10);
1364       }
1365    }
1366
1367    tu_cs_emit_ib(cs, &cmd->state.tile_store_ib);
1368
1369    tu_cs_sanity_check(cs);
1370 }
1371
1372 static void
1373 tu6_render_end(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1374 {
1375    const uint32_t space = 16 + tu_cs_get_call_size(&cmd->draw_epilogue_cs);
1376    VkResult result = tu_cs_reserve_space(cmd->device, cs, space);
1377    if (result != VK_SUCCESS) {
1378       cmd->record_result = result;
1379       return;
1380    }
1381
1382    tu_cs_emit_call(cs, &cmd->draw_epilogue_cs);
1383
1384    tu_cs_emit_regs(cs,
1385                    A6XX_GRAS_LRZ_CNTL(0));
1386
1387    tu6_emit_lrz_flush(cmd, cs);
1388
1389    tu6_emit_event_write(cmd, cs, CACHE_FLUSH_TS, true);
1390
1391    tu_cs_sanity_check(cs);
1392 }
1393
1394 static void
1395 tu_cmd_render_tiles(struct tu_cmd_buffer *cmd)
1396 {
1397    const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1398
1399    tu6_render_begin(cmd, &cmd->cs);
1400
1401    for (uint32_t y = 0; y < tiling->tile_count.height; y++) {
1402       for (uint32_t x = 0; x < tiling->tile_count.width; x++) {
1403          struct tu_tile tile;
1404          tu_tiling_config_get_tile(tiling, cmd->device, x, y, &tile);
1405          tu6_render_tile(cmd, &cmd->cs, &tile);
1406       }
1407    }
1408
1409    tu6_render_end(cmd, &cmd->cs);
1410 }
1411
1412 static void
1413 tu_cmd_prepare_tile_load_ib(struct tu_cmd_buffer *cmd,
1414                             const VkRenderPassBeginInfo *info)
1415 {
1416    const uint32_t tile_load_space =
1417       8 + (23+19) * cmd->state.pass->attachment_count +
1418       21 + (13 * cmd->state.subpass->color_count + 8) + 11;
1419
1420    struct tu_cs sub_cs;
1421
1422    VkResult result = tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs,
1423                                             tile_load_space, &sub_cs);
1424    if (result != VK_SUCCESS) {
1425       cmd->record_result = result;
1426       return;
1427    }
1428
1429    tu6_emit_blit_scissor(cmd, &sub_cs, true);
1430
1431    for (uint32_t i = 0; i < cmd->state.pass->attachment_count; ++i)
1432       tu6_emit_load_attachment(cmd, &sub_cs, i);
1433
1434    tu6_emit_blit_scissor(cmd, &sub_cs, false);
1435
1436    for (uint32_t i = 0; i < cmd->state.pass->attachment_count; ++i)
1437       tu6_emit_clear_attachment(cmd, &sub_cs, i, info);
1438
1439    /* invalidate because reading input attachments will cache GMEM and
1440     * the cache isn''t updated when GMEM is written
1441     * TODO: is there a no-cache bit for textures?
1442     */
1443    if (cmd->state.subpass->input_count)
1444       tu6_emit_event_write(cmd, &sub_cs, CACHE_INVALIDATE, false);
1445
1446    tu6_emit_zs(cmd, cmd->state.subpass, &sub_cs);
1447    tu6_emit_mrt(cmd, cmd->state.subpass, &sub_cs);
1448    tu6_emit_msaa(cmd, cmd->state.subpass, &sub_cs);
1449
1450    cmd->state.tile_load_ib = tu_cs_end_sub_stream(&cmd->sub_cs, &sub_cs);
1451 }
1452
1453 static void
1454 tu_cmd_prepare_tile_store_ib(struct tu_cmd_buffer *cmd)
1455 {
1456    const uint32_t tile_store_space = 32 + 23 * cmd->state.pass->attachment_count;
1457    struct tu_cs sub_cs;
1458
1459    VkResult result = tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs,
1460                                             tile_store_space, &sub_cs);
1461    if (result != VK_SUCCESS) {
1462       cmd->record_result = result;
1463       return;
1464    }
1465
1466    /* emit to tile-store sub_cs */
1467    tu6_emit_tile_store(cmd, &sub_cs);
1468
1469    cmd->state.tile_store_ib = tu_cs_end_sub_stream(&cmd->sub_cs, &sub_cs);
1470 }
1471
1472 static void
1473 tu_cmd_update_tiling_config(struct tu_cmd_buffer *cmd,
1474                             const VkRect2D *render_area)
1475 {
1476    const struct tu_device *dev = cmd->device;
1477    struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1478
1479    tiling->render_area = *render_area;
1480
1481    tu_tiling_config_update_tile_layout(tiling, dev, cmd->state.pass->gmem_pixels);
1482    tu_tiling_config_update_pipe_layout(tiling, dev);
1483    tu_tiling_config_update_pipes(tiling, dev);
1484 }
1485
1486 const struct tu_dynamic_state default_dynamic_state = {
1487    .viewport =
1488      {
1489        .count = 0,
1490      },
1491    .scissor =
1492      {
1493        .count = 0,
1494      },
1495    .line_width = 1.0f,
1496    .depth_bias =
1497      {
1498        .bias = 0.0f,
1499        .clamp = 0.0f,
1500        .slope = 0.0f,
1501      },
1502    .blend_constants = { 0.0f, 0.0f, 0.0f, 0.0f },
1503    .depth_bounds =
1504      {
1505        .min = 0.0f,
1506        .max = 1.0f,
1507      },
1508    .stencil_compare_mask =
1509      {
1510        .front = ~0u,
1511        .back = ~0u,
1512      },
1513    .stencil_write_mask =
1514      {
1515        .front = ~0u,
1516        .back = ~0u,
1517      },
1518    .stencil_reference =
1519      {
1520        .front = 0u,
1521        .back = 0u,
1522      },
1523 };
1524
1525 static void UNUSED /* FINISHME */
1526 tu_bind_dynamic_state(struct tu_cmd_buffer *cmd_buffer,
1527                       const struct tu_dynamic_state *src)
1528 {
1529    struct tu_dynamic_state *dest = &cmd_buffer->state.dynamic;
1530    uint32_t copy_mask = src->mask;
1531    uint32_t dest_mask = 0;
1532
1533    tu_use_args(cmd_buffer); /* FINISHME */
1534
1535    /* Make sure to copy the number of viewports/scissors because they can
1536     * only be specified at pipeline creation time.
1537     */
1538    dest->viewport.count = src->viewport.count;
1539    dest->scissor.count = src->scissor.count;
1540    dest->discard_rectangle.count = src->discard_rectangle.count;
1541
1542    if (copy_mask & TU_DYNAMIC_VIEWPORT) {
1543       if (memcmp(&dest->viewport.viewports, &src->viewport.viewports,
1544                  src->viewport.count * sizeof(VkViewport))) {
1545          typed_memcpy(dest->viewport.viewports, src->viewport.viewports,
1546                       src->viewport.count);
1547          dest_mask |= TU_DYNAMIC_VIEWPORT;
1548       }
1549    }
1550
1551    if (copy_mask & TU_DYNAMIC_SCISSOR) {
1552       if (memcmp(&dest->scissor.scissors, &src->scissor.scissors,
1553                  src->scissor.count * sizeof(VkRect2D))) {
1554          typed_memcpy(dest->scissor.scissors, src->scissor.scissors,
1555                       src->scissor.count);
1556          dest_mask |= TU_DYNAMIC_SCISSOR;
1557       }
1558    }
1559
1560    if (copy_mask & TU_DYNAMIC_LINE_WIDTH) {
1561       if (dest->line_width != src->line_width) {
1562          dest->line_width = src->line_width;
1563          dest_mask |= TU_DYNAMIC_LINE_WIDTH;
1564       }
1565    }
1566
1567    if (copy_mask & TU_DYNAMIC_DEPTH_BIAS) {
1568       if (memcmp(&dest->depth_bias, &src->depth_bias,
1569                  sizeof(src->depth_bias))) {
1570          dest->depth_bias = src->depth_bias;
1571          dest_mask |= TU_DYNAMIC_DEPTH_BIAS;
1572       }
1573    }
1574
1575    if (copy_mask & TU_DYNAMIC_BLEND_CONSTANTS) {
1576       if (memcmp(&dest->blend_constants, &src->blend_constants,
1577                  sizeof(src->blend_constants))) {
1578          typed_memcpy(dest->blend_constants, src->blend_constants, 4);
1579          dest_mask |= TU_DYNAMIC_BLEND_CONSTANTS;
1580       }
1581    }
1582
1583    if (copy_mask & TU_DYNAMIC_DEPTH_BOUNDS) {
1584       if (memcmp(&dest->depth_bounds, &src->depth_bounds,
1585                  sizeof(src->depth_bounds))) {
1586          dest->depth_bounds = src->depth_bounds;
1587          dest_mask |= TU_DYNAMIC_DEPTH_BOUNDS;
1588       }
1589    }
1590
1591    if (copy_mask & TU_DYNAMIC_STENCIL_COMPARE_MASK) {
1592       if (memcmp(&dest->stencil_compare_mask, &src->stencil_compare_mask,
1593                  sizeof(src->stencil_compare_mask))) {
1594          dest->stencil_compare_mask = src->stencil_compare_mask;
1595          dest_mask |= TU_DYNAMIC_STENCIL_COMPARE_MASK;
1596       }
1597    }
1598
1599    if (copy_mask & TU_DYNAMIC_STENCIL_WRITE_MASK) {
1600       if (memcmp(&dest->stencil_write_mask, &src->stencil_write_mask,
1601                  sizeof(src->stencil_write_mask))) {
1602          dest->stencil_write_mask = src->stencil_write_mask;
1603          dest_mask |= TU_DYNAMIC_STENCIL_WRITE_MASK;
1604       }
1605    }
1606
1607    if (copy_mask & TU_DYNAMIC_STENCIL_REFERENCE) {
1608       if (memcmp(&dest->stencil_reference, &src->stencil_reference,
1609                  sizeof(src->stencil_reference))) {
1610          dest->stencil_reference = src->stencil_reference;
1611          dest_mask |= TU_DYNAMIC_STENCIL_REFERENCE;
1612       }
1613    }
1614
1615    if (copy_mask & TU_DYNAMIC_DISCARD_RECTANGLE) {
1616       if (memcmp(&dest->discard_rectangle.rectangles,
1617                  &src->discard_rectangle.rectangles,
1618                  src->discard_rectangle.count * sizeof(VkRect2D))) {
1619          typed_memcpy(dest->discard_rectangle.rectangles,
1620                       src->discard_rectangle.rectangles,
1621                       src->discard_rectangle.count);
1622          dest_mask |= TU_DYNAMIC_DISCARD_RECTANGLE;
1623       }
1624    }
1625 }
1626
1627 static VkResult
1628 tu_create_cmd_buffer(struct tu_device *device,
1629                      struct tu_cmd_pool *pool,
1630                      VkCommandBufferLevel level,
1631                      VkCommandBuffer *pCommandBuffer)
1632 {
1633    struct tu_cmd_buffer *cmd_buffer;
1634    cmd_buffer = vk_zalloc(&pool->alloc, sizeof(*cmd_buffer), 8,
1635                           VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1636    if (cmd_buffer == NULL)
1637       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
1638
1639    cmd_buffer->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
1640    cmd_buffer->device = device;
1641    cmd_buffer->pool = pool;
1642    cmd_buffer->level = level;
1643
1644    if (pool) {
1645       list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers);
1646       cmd_buffer->queue_family_index = pool->queue_family_index;
1647
1648    } else {
1649       /* Init the pool_link so we can safely call list_del when we destroy
1650        * the command buffer
1651        */
1652       list_inithead(&cmd_buffer->pool_link);
1653       cmd_buffer->queue_family_index = TU_QUEUE_GENERAL;
1654    }
1655
1656    tu_bo_list_init(&cmd_buffer->bo_list);
1657    tu_cs_init(&cmd_buffer->cs, TU_CS_MODE_GROW, 4096);
1658    tu_cs_init(&cmd_buffer->draw_cs, TU_CS_MODE_GROW, 4096);
1659    tu_cs_init(&cmd_buffer->draw_epilogue_cs, TU_CS_MODE_GROW, 4096);
1660    tu_cs_init(&cmd_buffer->sub_cs, TU_CS_MODE_SUB_STREAM, 2048);
1661
1662    *pCommandBuffer = tu_cmd_buffer_to_handle(cmd_buffer);
1663
1664    list_inithead(&cmd_buffer->upload.list);
1665
1666    cmd_buffer->marker_reg = REG_A6XX_CP_SCRATCH_REG(
1667       cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY ? 7 : 6);
1668
1669    VkResult result = tu_bo_init_new(device, &cmd_buffer->scratch_bo, 0x1000);
1670    if (result != VK_SUCCESS)
1671       goto fail_scratch_bo;
1672
1673 #define VSC_DATA_SIZE(pitch)  ((pitch) * 32 + 0x100)  /* extra size to store VSC_SIZE */
1674 #define VSC_DATA2_SIZE(pitch) ((pitch) * 32)
1675
1676    /* TODO: resize on overflow or compute a max size from # of vertices in renderpass?? */
1677    cmd_buffer->vsc_data_pitch = 0x440 * 4;
1678    cmd_buffer->vsc_data2_pitch = 0x1040 * 4;
1679
1680    result = tu_bo_init_new(device, &cmd_buffer->vsc_data, VSC_DATA_SIZE(cmd_buffer->vsc_data_pitch));
1681    if (result != VK_SUCCESS)
1682       goto fail_vsc_data;
1683
1684    result = tu_bo_init_new(device, &cmd_buffer->vsc_data2, VSC_DATA2_SIZE(cmd_buffer->vsc_data2_pitch));
1685    if (result != VK_SUCCESS)
1686       goto fail_vsc_data2;
1687
1688    return VK_SUCCESS;
1689
1690 fail_vsc_data2:
1691    tu_bo_finish(cmd_buffer->device, &cmd_buffer->vsc_data);
1692 fail_vsc_data:
1693    tu_bo_finish(cmd_buffer->device, &cmd_buffer->scratch_bo);
1694 fail_scratch_bo:
1695    list_del(&cmd_buffer->pool_link);
1696    return result;
1697 }
1698
1699 static void
1700 tu_cmd_buffer_destroy(struct tu_cmd_buffer *cmd_buffer)
1701 {
1702    tu_bo_finish(cmd_buffer->device, &cmd_buffer->scratch_bo);
1703    tu_bo_finish(cmd_buffer->device, &cmd_buffer->vsc_data);
1704    tu_bo_finish(cmd_buffer->device, &cmd_buffer->vsc_data2);
1705
1706    list_del(&cmd_buffer->pool_link);
1707
1708    for (unsigned i = 0; i < VK_PIPELINE_BIND_POINT_RANGE_SIZE; i++)
1709       free(cmd_buffer->descriptors[i].push_set.set.mapped_ptr);
1710
1711    tu_cs_finish(cmd_buffer->device, &cmd_buffer->cs);
1712    tu_cs_finish(cmd_buffer->device, &cmd_buffer->draw_cs);
1713    tu_cs_finish(cmd_buffer->device, &cmd_buffer->draw_epilogue_cs);
1714    tu_cs_finish(cmd_buffer->device, &cmd_buffer->sub_cs);
1715
1716    tu_bo_list_destroy(&cmd_buffer->bo_list);
1717    vk_free(&cmd_buffer->pool->alloc, cmd_buffer);
1718 }
1719
1720 static VkResult
1721 tu_reset_cmd_buffer(struct tu_cmd_buffer *cmd_buffer)
1722 {
1723    cmd_buffer->wait_for_idle = true;
1724
1725    cmd_buffer->record_result = VK_SUCCESS;
1726
1727    tu_bo_list_reset(&cmd_buffer->bo_list);
1728    tu_cs_reset(cmd_buffer->device, &cmd_buffer->cs);
1729    tu_cs_reset(cmd_buffer->device, &cmd_buffer->draw_cs);
1730    tu_cs_reset(cmd_buffer->device, &cmd_buffer->draw_epilogue_cs);
1731    tu_cs_reset(cmd_buffer->device, &cmd_buffer->sub_cs);
1732
1733    for (unsigned i = 0; i < VK_PIPELINE_BIND_POINT_RANGE_SIZE; i++) {
1734       cmd_buffer->descriptors[i].dirty = 0;
1735       cmd_buffer->descriptors[i].valid = 0;
1736       cmd_buffer->descriptors[i].push_dirty = false;
1737    }
1738
1739    cmd_buffer->status = TU_CMD_BUFFER_STATUS_INITIAL;
1740
1741    return cmd_buffer->record_result;
1742 }
1743
1744 VkResult
1745 tu_AllocateCommandBuffers(VkDevice _device,
1746                           const VkCommandBufferAllocateInfo *pAllocateInfo,
1747                           VkCommandBuffer *pCommandBuffers)
1748 {
1749    TU_FROM_HANDLE(tu_device, device, _device);
1750    TU_FROM_HANDLE(tu_cmd_pool, pool, pAllocateInfo->commandPool);
1751
1752    VkResult result = VK_SUCCESS;
1753    uint32_t i;
1754
1755    for (i = 0; i < pAllocateInfo->commandBufferCount; i++) {
1756
1757       if (!list_is_empty(&pool->free_cmd_buffers)) {
1758          struct tu_cmd_buffer *cmd_buffer = list_first_entry(
1759             &pool->free_cmd_buffers, struct tu_cmd_buffer, pool_link);
1760
1761          list_del(&cmd_buffer->pool_link);
1762          list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers);
1763
1764          result = tu_reset_cmd_buffer(cmd_buffer);
1765          cmd_buffer->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
1766          cmd_buffer->level = pAllocateInfo->level;
1767
1768          pCommandBuffers[i] = tu_cmd_buffer_to_handle(cmd_buffer);
1769       } else {
1770          result = tu_create_cmd_buffer(device, pool, pAllocateInfo->level,
1771                                        &pCommandBuffers[i]);
1772       }
1773       if (result != VK_SUCCESS)
1774          break;
1775    }
1776
1777    if (result != VK_SUCCESS) {
1778       tu_FreeCommandBuffers(_device, pAllocateInfo->commandPool, i,
1779                             pCommandBuffers);
1780
1781       /* From the Vulkan 1.0.66 spec:
1782        *
1783        * "vkAllocateCommandBuffers can be used to create multiple
1784        *  command buffers. If the creation of any of those command
1785        *  buffers fails, the implementation must destroy all
1786        *  successfully created command buffer objects from this
1787        *  command, set all entries of the pCommandBuffers array to
1788        *  NULL and return the error."
1789        */
1790       memset(pCommandBuffers, 0,
1791              sizeof(*pCommandBuffers) * pAllocateInfo->commandBufferCount);
1792    }
1793
1794    return result;
1795 }
1796
1797 void
1798 tu_FreeCommandBuffers(VkDevice device,
1799                       VkCommandPool commandPool,
1800                       uint32_t commandBufferCount,
1801                       const VkCommandBuffer *pCommandBuffers)
1802 {
1803    for (uint32_t i = 0; i < commandBufferCount; i++) {
1804       TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, pCommandBuffers[i]);
1805
1806       if (cmd_buffer) {
1807          if (cmd_buffer->pool) {
1808             list_del(&cmd_buffer->pool_link);
1809             list_addtail(&cmd_buffer->pool_link,
1810                          &cmd_buffer->pool->free_cmd_buffers);
1811          } else
1812             tu_cmd_buffer_destroy(cmd_buffer);
1813       }
1814    }
1815 }
1816
1817 VkResult
1818 tu_ResetCommandBuffer(VkCommandBuffer commandBuffer,
1819                       VkCommandBufferResetFlags flags)
1820 {
1821    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
1822    return tu_reset_cmd_buffer(cmd_buffer);
1823 }
1824
1825 VkResult
1826 tu_BeginCommandBuffer(VkCommandBuffer commandBuffer,
1827                       const VkCommandBufferBeginInfo *pBeginInfo)
1828 {
1829    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
1830    VkResult result = VK_SUCCESS;
1831
1832    if (cmd_buffer->status != TU_CMD_BUFFER_STATUS_INITIAL) {
1833       /* If the command buffer has already been resetted with
1834        * vkResetCommandBuffer, no need to do it again.
1835        */
1836       result = tu_reset_cmd_buffer(cmd_buffer);
1837       if (result != VK_SUCCESS)
1838          return result;
1839    }
1840
1841    memset(&cmd_buffer->state, 0, sizeof(cmd_buffer->state));
1842    cmd_buffer->usage_flags = pBeginInfo->flags;
1843
1844    tu_cs_begin(&cmd_buffer->cs);
1845    tu_cs_begin(&cmd_buffer->draw_cs);
1846    tu_cs_begin(&cmd_buffer->draw_epilogue_cs);
1847
1848    cmd_buffer->marker_seqno = 0;
1849    cmd_buffer->scratch_seqno = 0;
1850
1851    /* setup initial configuration into command buffer */
1852    if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY) {
1853       switch (cmd_buffer->queue_family_index) {
1854       case TU_QUEUE_GENERAL:
1855          tu6_init_hw(cmd_buffer, &cmd_buffer->cs);
1856          break;
1857       default:
1858          break;
1859       }
1860    } else if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY &&
1861               (pBeginInfo->flags & VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT)) {
1862       assert(pBeginInfo->pInheritanceInfo);
1863       cmd_buffer->state.pass = tu_render_pass_from_handle(pBeginInfo->pInheritanceInfo->renderPass);
1864       cmd_buffer->state.subpass = &cmd_buffer->state.pass->subpasses[pBeginInfo->pInheritanceInfo->subpass];
1865    }
1866
1867    cmd_buffer->status = TU_CMD_BUFFER_STATUS_RECORDING;
1868
1869    return VK_SUCCESS;
1870 }
1871
1872 void
1873 tu_CmdBindVertexBuffers(VkCommandBuffer commandBuffer,
1874                         uint32_t firstBinding,
1875                         uint32_t bindingCount,
1876                         const VkBuffer *pBuffers,
1877                         const VkDeviceSize *pOffsets)
1878 {
1879    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
1880
1881    assert(firstBinding + bindingCount <= MAX_VBS);
1882
1883    for (uint32_t i = 0; i < bindingCount; i++) {
1884       cmd->state.vb.buffers[firstBinding + i] =
1885          tu_buffer_from_handle(pBuffers[i]);
1886       cmd->state.vb.offsets[firstBinding + i] = pOffsets[i];
1887    }
1888
1889    /* VB states depend on VkPipelineVertexInputStateCreateInfo */
1890    cmd->state.dirty |= TU_CMD_DIRTY_VERTEX_BUFFERS;
1891 }
1892
1893 void
1894 tu_CmdBindIndexBuffer(VkCommandBuffer commandBuffer,
1895                       VkBuffer buffer,
1896                       VkDeviceSize offset,
1897                       VkIndexType indexType)
1898 {
1899    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
1900    TU_FROM_HANDLE(tu_buffer, buf, buffer);
1901
1902    /* initialize/update the restart index */
1903    if (!cmd->state.index_buffer || cmd->state.index_type != indexType) {
1904       struct tu_cs *draw_cs = &cmd->draw_cs;
1905       VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 2);
1906       if (result != VK_SUCCESS) {
1907          cmd->record_result = result;
1908          return;
1909       }
1910
1911       tu6_emit_restart_index(
1912          draw_cs, indexType == VK_INDEX_TYPE_UINT32 ? 0xffffffff : 0xffff);
1913
1914       tu_cs_sanity_check(draw_cs);
1915    }
1916
1917    /* track the BO */
1918    if (cmd->state.index_buffer != buf)
1919       tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
1920
1921    cmd->state.index_buffer = buf;
1922    cmd->state.index_offset = offset;
1923    cmd->state.index_type = indexType;
1924 }
1925
1926 void
1927 tu_CmdBindDescriptorSets(VkCommandBuffer commandBuffer,
1928                          VkPipelineBindPoint pipelineBindPoint,
1929                          VkPipelineLayout _layout,
1930                          uint32_t firstSet,
1931                          uint32_t descriptorSetCount,
1932                          const VkDescriptorSet *pDescriptorSets,
1933                          uint32_t dynamicOffsetCount,
1934                          const uint32_t *pDynamicOffsets)
1935 {
1936    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
1937    TU_FROM_HANDLE(tu_pipeline_layout, layout, _layout);
1938    unsigned dyn_idx = 0;
1939
1940    struct tu_descriptor_state *descriptors_state =
1941       tu_get_descriptors_state(cmd_buffer, pipelineBindPoint);
1942
1943    for (unsigned i = 0; i < descriptorSetCount; ++i) {
1944       unsigned idx = i + firstSet;
1945       TU_FROM_HANDLE(tu_descriptor_set, set, pDescriptorSets[i]);
1946
1947       descriptors_state->sets[idx] = set;
1948       descriptors_state->valid |= (1u << idx);
1949
1950       for(unsigned j = 0; j < set->layout->dynamic_offset_count; ++j, ++dyn_idx) {
1951          unsigned idx = j + layout->set[i + firstSet].dynamic_offset_start;
1952          assert(dyn_idx < dynamicOffsetCount);
1953
1954          descriptors_state->dynamic_buffers[idx] =
1955          set->dynamic_descriptors[j].va + pDynamicOffsets[dyn_idx];
1956       }
1957    }
1958
1959    cmd_buffer->state.dirty |= TU_CMD_DIRTY_DESCRIPTOR_SETS;
1960 }
1961
1962 void
1963 tu_CmdPushConstants(VkCommandBuffer commandBuffer,
1964                     VkPipelineLayout layout,
1965                     VkShaderStageFlags stageFlags,
1966                     uint32_t offset,
1967                     uint32_t size,
1968                     const void *pValues)
1969 {
1970    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
1971    memcpy((void*) cmd->push_constants + offset, pValues, size);
1972    cmd->state.dirty |= TU_CMD_DIRTY_PUSH_CONSTANTS;
1973 }
1974
1975 VkResult
1976 tu_EndCommandBuffer(VkCommandBuffer commandBuffer)
1977 {
1978    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
1979
1980    if (cmd_buffer->scratch_seqno) {
1981       tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->scratch_bo,
1982                      MSM_SUBMIT_BO_WRITE);
1983    }
1984
1985    if (cmd_buffer->use_vsc_data) {
1986       tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->vsc_data,
1987                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
1988       tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->vsc_data2,
1989                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
1990    }
1991
1992    for (uint32_t i = 0; i < cmd_buffer->draw_cs.bo_count; i++) {
1993       tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->draw_cs.bos[i],
1994                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
1995    }
1996
1997    for (uint32_t i = 0; i < cmd_buffer->draw_epilogue_cs.bo_count; i++) {
1998       tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->draw_epilogue_cs.bos[i],
1999                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2000    }
2001
2002    for (uint32_t i = 0; i < cmd_buffer->sub_cs.bo_count; i++) {
2003       tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->sub_cs.bos[i],
2004                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2005    }
2006
2007    tu_cs_end(&cmd_buffer->cs);
2008    tu_cs_end(&cmd_buffer->draw_cs);
2009    tu_cs_end(&cmd_buffer->draw_epilogue_cs);
2010
2011    cmd_buffer->status = TU_CMD_BUFFER_STATUS_EXECUTABLE;
2012
2013    return cmd_buffer->record_result;
2014 }
2015
2016 void
2017 tu_CmdBindPipeline(VkCommandBuffer commandBuffer,
2018                    VkPipelineBindPoint pipelineBindPoint,
2019                    VkPipeline _pipeline)
2020 {
2021    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2022    TU_FROM_HANDLE(tu_pipeline, pipeline, _pipeline);
2023
2024    switch (pipelineBindPoint) {
2025    case VK_PIPELINE_BIND_POINT_GRAPHICS:
2026       cmd->state.pipeline = pipeline;
2027       cmd->state.dirty |= TU_CMD_DIRTY_PIPELINE;
2028       break;
2029    case VK_PIPELINE_BIND_POINT_COMPUTE:
2030       cmd->state.compute_pipeline = pipeline;
2031       cmd->state.dirty |= TU_CMD_DIRTY_COMPUTE_PIPELINE;
2032       break;
2033    default:
2034       unreachable("unrecognized pipeline bind point");
2035       break;
2036    }
2037
2038    tu_bo_list_add(&cmd->bo_list, &pipeline->program.binary_bo,
2039                   MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2040    for (uint32_t i = 0; i < pipeline->cs.bo_count; i++) {
2041       tu_bo_list_add(&cmd->bo_list, pipeline->cs.bos[i],
2042                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2043    }
2044 }
2045
2046 void
2047 tu_CmdSetViewport(VkCommandBuffer commandBuffer,
2048                   uint32_t firstViewport,
2049                   uint32_t viewportCount,
2050                   const VkViewport *pViewports)
2051 {
2052    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2053    struct tu_cs *draw_cs = &cmd->draw_cs;
2054
2055    VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 12);
2056    if (result != VK_SUCCESS) {
2057       cmd->record_result = result;
2058       return;
2059    }
2060
2061    assert(firstViewport == 0 && viewportCount == 1);
2062    tu6_emit_viewport(draw_cs, pViewports);
2063
2064    tu_cs_sanity_check(draw_cs);
2065 }
2066
2067 void
2068 tu_CmdSetScissor(VkCommandBuffer commandBuffer,
2069                  uint32_t firstScissor,
2070                  uint32_t scissorCount,
2071                  const VkRect2D *pScissors)
2072 {
2073    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2074    struct tu_cs *draw_cs = &cmd->draw_cs;
2075
2076    VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 3);
2077    if (result != VK_SUCCESS) {
2078       cmd->record_result = result;
2079       return;
2080    }
2081
2082    assert(firstScissor == 0 && scissorCount == 1);
2083    tu6_emit_scissor(draw_cs, pScissors);
2084
2085    tu_cs_sanity_check(draw_cs);
2086 }
2087
2088 void
2089 tu_CmdSetLineWidth(VkCommandBuffer commandBuffer, float lineWidth)
2090 {
2091    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2092
2093    cmd->state.dynamic.line_width = lineWidth;
2094
2095    /* line width depends on VkPipelineRasterizationStateCreateInfo */
2096    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_LINE_WIDTH;
2097 }
2098
2099 void
2100 tu_CmdSetDepthBias(VkCommandBuffer commandBuffer,
2101                    float depthBiasConstantFactor,
2102                    float depthBiasClamp,
2103                    float depthBiasSlopeFactor)
2104 {
2105    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2106    struct tu_cs *draw_cs = &cmd->draw_cs;
2107
2108    VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 4);
2109    if (result != VK_SUCCESS) {
2110       cmd->record_result = result;
2111       return;
2112    }
2113
2114    tu6_emit_depth_bias(draw_cs, depthBiasConstantFactor, depthBiasClamp,
2115                        depthBiasSlopeFactor);
2116
2117    tu_cs_sanity_check(draw_cs);
2118 }
2119
2120 void
2121 tu_CmdSetBlendConstants(VkCommandBuffer commandBuffer,
2122                         const float blendConstants[4])
2123 {
2124    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2125    struct tu_cs *draw_cs = &cmd->draw_cs;
2126
2127    VkResult result = tu_cs_reserve_space(cmd->device, draw_cs, 5);
2128    if (result != VK_SUCCESS) {
2129       cmd->record_result = result;
2130       return;
2131    }
2132
2133    tu6_emit_blend_constants(draw_cs, blendConstants);
2134
2135    tu_cs_sanity_check(draw_cs);
2136 }
2137
2138 void
2139 tu_CmdSetDepthBounds(VkCommandBuffer commandBuffer,
2140                      float minDepthBounds,
2141                      float maxDepthBounds)
2142 {
2143 }
2144
2145 void
2146 tu_CmdSetStencilCompareMask(VkCommandBuffer commandBuffer,
2147                             VkStencilFaceFlags faceMask,
2148                             uint32_t compareMask)
2149 {
2150    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2151
2152    if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
2153       cmd->state.dynamic.stencil_compare_mask.front = compareMask;
2154    if (faceMask & VK_STENCIL_FACE_BACK_BIT)
2155       cmd->state.dynamic.stencil_compare_mask.back = compareMask;
2156
2157    /* the front/back compare masks must be updated together */
2158    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK;
2159 }
2160
2161 void
2162 tu_CmdSetStencilWriteMask(VkCommandBuffer commandBuffer,
2163                           VkStencilFaceFlags faceMask,
2164                           uint32_t writeMask)
2165 {
2166    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2167
2168    if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
2169       cmd->state.dynamic.stencil_write_mask.front = writeMask;
2170    if (faceMask & VK_STENCIL_FACE_BACK_BIT)
2171       cmd->state.dynamic.stencil_write_mask.back = writeMask;
2172
2173    /* the front/back write masks must be updated together */
2174    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK;
2175 }
2176
2177 void
2178 tu_CmdSetStencilReference(VkCommandBuffer commandBuffer,
2179                           VkStencilFaceFlags faceMask,
2180                           uint32_t reference)
2181 {
2182    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2183
2184    if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
2185       cmd->state.dynamic.stencil_reference.front = reference;
2186    if (faceMask & VK_STENCIL_FACE_BACK_BIT)
2187       cmd->state.dynamic.stencil_reference.back = reference;
2188
2189    /* the front/back references must be updated together */
2190    cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE;
2191 }
2192
2193 void
2194 tu_CmdExecuteCommands(VkCommandBuffer commandBuffer,
2195                       uint32_t commandBufferCount,
2196                       const VkCommandBuffer *pCmdBuffers)
2197 {
2198    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2199    VkResult result;
2200
2201    assert(commandBufferCount > 0);
2202
2203    for (uint32_t i = 0; i < commandBufferCount; i++) {
2204       TU_FROM_HANDLE(tu_cmd_buffer, secondary, pCmdBuffers[i]);
2205
2206       result = tu_bo_list_merge(&cmd->bo_list, &secondary->bo_list);
2207       if (result != VK_SUCCESS) {
2208          cmd->record_result = result;
2209          break;
2210       }
2211
2212       result = tu_cs_add_entries(&cmd->draw_cs, &secondary->draw_cs);
2213       if (result != VK_SUCCESS) {
2214          cmd->record_result = result;
2215          break;
2216       }
2217
2218       result = tu_cs_add_entries(&cmd->draw_epilogue_cs,
2219             &secondary->draw_epilogue_cs);
2220       if (result != VK_SUCCESS) {
2221          cmd->record_result = result;
2222          break;
2223       }
2224    }
2225    cmd->state.dirty = ~0u; /* TODO: set dirty only what needs to be */
2226 }
2227
2228 VkResult
2229 tu_CreateCommandPool(VkDevice _device,
2230                      const VkCommandPoolCreateInfo *pCreateInfo,
2231                      const VkAllocationCallbacks *pAllocator,
2232                      VkCommandPool *pCmdPool)
2233 {
2234    TU_FROM_HANDLE(tu_device, device, _device);
2235    struct tu_cmd_pool *pool;
2236
2237    pool = vk_alloc2(&device->alloc, pAllocator, sizeof(*pool), 8,
2238                     VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2239    if (pool == NULL)
2240       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
2241
2242    if (pAllocator)
2243       pool->alloc = *pAllocator;
2244    else
2245       pool->alloc = device->alloc;
2246
2247    list_inithead(&pool->cmd_buffers);
2248    list_inithead(&pool->free_cmd_buffers);
2249
2250    pool->queue_family_index = pCreateInfo->queueFamilyIndex;
2251
2252    *pCmdPool = tu_cmd_pool_to_handle(pool);
2253
2254    return VK_SUCCESS;
2255 }
2256
2257 void
2258 tu_DestroyCommandPool(VkDevice _device,
2259                       VkCommandPool commandPool,
2260                       const VkAllocationCallbacks *pAllocator)
2261 {
2262    TU_FROM_HANDLE(tu_device, device, _device);
2263    TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2264
2265    if (!pool)
2266       return;
2267
2268    list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2269                             &pool->cmd_buffers, pool_link)
2270    {
2271       tu_cmd_buffer_destroy(cmd_buffer);
2272    }
2273
2274    list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2275                             &pool->free_cmd_buffers, pool_link)
2276    {
2277       tu_cmd_buffer_destroy(cmd_buffer);
2278    }
2279
2280    vk_free2(&device->alloc, pAllocator, pool);
2281 }
2282
2283 VkResult
2284 tu_ResetCommandPool(VkDevice device,
2285                     VkCommandPool commandPool,
2286                     VkCommandPoolResetFlags flags)
2287 {
2288    TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2289    VkResult result;
2290
2291    list_for_each_entry(struct tu_cmd_buffer, cmd_buffer, &pool->cmd_buffers,
2292                        pool_link)
2293    {
2294       result = tu_reset_cmd_buffer(cmd_buffer);
2295       if (result != VK_SUCCESS)
2296          return result;
2297    }
2298
2299    return VK_SUCCESS;
2300 }
2301
2302 void
2303 tu_TrimCommandPool(VkDevice device,
2304                    VkCommandPool commandPool,
2305                    VkCommandPoolTrimFlags flags)
2306 {
2307    TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2308
2309    if (!pool)
2310       return;
2311
2312    list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2313                             &pool->free_cmd_buffers, pool_link)
2314    {
2315       tu_cmd_buffer_destroy(cmd_buffer);
2316    }
2317 }
2318
2319 void
2320 tu_CmdBeginRenderPass(VkCommandBuffer commandBuffer,
2321                       const VkRenderPassBeginInfo *pRenderPassBegin,
2322                       VkSubpassContents contents)
2323 {
2324    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2325    TU_FROM_HANDLE(tu_render_pass, pass, pRenderPassBegin->renderPass);
2326    TU_FROM_HANDLE(tu_framebuffer, fb, pRenderPassBegin->framebuffer);
2327
2328    cmd->state.pass = pass;
2329    cmd->state.subpass = pass->subpasses;
2330    cmd->state.framebuffer = fb;
2331
2332    tu_cmd_update_tiling_config(cmd, &pRenderPassBegin->renderArea);
2333    tu_cmd_prepare_tile_load_ib(cmd, pRenderPassBegin);
2334    tu_cmd_prepare_tile_store_ib(cmd);
2335
2336    /* note: use_hw_binning only checks tiling config */
2337    if (use_hw_binning(cmd))
2338       cmd->use_vsc_data = true;
2339
2340    for (uint32_t i = 0; i < fb->attachment_count; ++i) {
2341       const struct tu_image_view *iview = fb->attachments[i].attachment;
2342       tu_bo_list_add(&cmd->bo_list, iview->image->bo,
2343                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
2344    }
2345 }
2346
2347 void
2348 tu_CmdBeginRenderPass2(VkCommandBuffer commandBuffer,
2349                        const VkRenderPassBeginInfo *pRenderPassBeginInfo,
2350                        const VkSubpassBeginInfoKHR *pSubpassBeginInfo)
2351 {
2352    tu_CmdBeginRenderPass(commandBuffer, pRenderPassBeginInfo,
2353                          pSubpassBeginInfo->contents);
2354 }
2355
2356 void
2357 tu_CmdNextSubpass(VkCommandBuffer commandBuffer, VkSubpassContents contents)
2358 {
2359    TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2360    const struct tu_render_pass *pass = cmd->state.pass;
2361    struct tu_cs *cs = &cmd->draw_cs;
2362
2363    VkResult result = tu_cs_reserve_space(cmd->device, cs, 1024);
2364    if (result != VK_SUCCESS) {
2365       cmd->record_result = result;
2366       return;
2367    }
2368
2369    const struct tu_subpass *subpass = cmd->state.subpass++;
2370    /* TODO:
2371     * if msaa samples change between subpasses,
2372     * attachment store is broken for some attachments
2373     */
2374    if (subpass->resolve_attachments) {
2375       tu6_emit_blit_scissor(cmd, cs, true);
2376       for (unsigned i = 0; i < subpass->color_count; i++) {
2377          uint32_t a = subpass->resolve_attachments[i].attachment;
2378          if (a != VK_ATTACHMENT_UNUSED) {
2379                tu6_emit_store_attachment(cmd, cs, a,
2380                                          subpass->color_attachments[i].attachment);
2381          }
2382       }
2383    }
2384
2385    /* invalidate because reading input attachments will cache GMEM and
2386     * the cache isn''t updated when GMEM is written
2387     * TODO: is there a no-cache bit for textures?
2388     */
2389    if (cmd->state.subpass->input_count)
2390       tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE, false);
2391
2392    /* emit mrt/zs/msaa state for the subpass that is starting */
2393    tu6_emit_zs(cmd, cmd->state.subpass, cs);
2394    tu6_emit_mrt(cmd, cmd->state.subpass, cs);
2395    tu6_emit_msaa(cmd, cmd->state.subpass, cs);
2396
2397    /* TODO:
2398     * since we don't know how to do GMEM->GMEM resolve,
2399     * resolve attachments are resolved to memory then loaded to GMEM again if needed
2400     */
2401    if (subpass->resolve_attachments) {
2402       for (unsigned i = 0; i < subpass->color_count; i++) {
2403          uint32_t a = subpass->resolve_attachments[i].attachment;
2404          const struct tu_image_view *iview =
2405             cmd->state.framebuffer->attachments[a].attachment;
2406          if (a != VK_ATTACHMENT_UNUSED && pass->attachments[a].gmem_offset >= 0) {
2407                tu_finishme("missing GMEM->GMEM resolve, performance will suffer\n");
2408                tu6_emit_blit_info(cmd, cs, iview, pass->attachments[a].gmem_offset, false);
2409                tu6_emit_blit(cmd, cs);
2410          }
2411       }
2412    }
2413 }
2414
2415 void
2416 tu_CmdNextSubpass2(VkCommandBuffer commandBuffer,
2417                    const VkSubpassBeginInfoKHR *pSubpassBeginInfo,
2418                    const VkSubpassEndInfoKHR *pSubpassEndInfo)
2419 {
2420    tu_CmdNextSubpass(commandBuffer, pSubpassBeginInfo->contents);
2421 }
2422
2423 struct tu_draw_info
2424 {
2425    /**
2426     * Number of vertices.
2427     */
2428    uint32_t count;
2429
2430    /**
2431     * Index of the first vertex.
2432     */
2433    int32_t vertex_offset;
2434
2435    /**
2436     * First instance id.
2437     */
2438    uint32_t first_instance;
2439
2440    /**
2441     * Number of instances.
2442     */
2443    uint32_t instance_count;
2444
2445    /**
2446     * First index (indexed draws only).
2447     */
2448    uint32_t first_index;
2449
2450    /**
2451     * Whether it's an indexed draw.
2452     */
2453    bool indexed;
2454
2455    /**
2456     * Indirect draw parameters resource.
2457     */
2458    struct tu_buffer *indirect;
2459    uint64_t indirect_offset;
2460    uint32_t stride;
2461
2462    /**
2463     * Draw count parameters resource.
2464     */
2465    struct tu_buffer *count_buffer;
2466    uint64_t count_buffer_offset;
2467 };
2468
2469 #define ENABLE_ALL (CP_SET_DRAW_STATE__0_BINNING | CP_SET_DRAW_STATE__0_GMEM | CP_SET_DRAW_STATE__0_SYSMEM)
2470 #define ENABLE_DRAW (CP_SET_DRAW_STATE__0_GMEM | CP_SET_DRAW_STATE__0_SYSMEM)
2471
2472 enum tu_draw_state_group_id
2473 {
2474    TU_DRAW_STATE_PROGRAM,
2475    TU_DRAW_STATE_PROGRAM_BINNING,
2476    TU_DRAW_STATE_VI,
2477    TU_DRAW_STATE_VI_BINNING,
2478    TU_DRAW_STATE_VP,
2479    TU_DRAW_STATE_RAST,
2480    TU_DRAW_STATE_DS,
2481    TU_DRAW_STATE_BLEND,
2482    TU_DRAW_STATE_VS_CONST,
2483    TU_DRAW_STATE_FS_CONST,
2484    TU_DRAW_STATE_VS_TEX,
2485    TU_DRAW_STATE_FS_TEX,
2486    TU_DRAW_STATE_FS_IBO,
2487    TU_DRAW_STATE_VS_PARAMS,
2488
2489    TU_DRAW_STATE_COUNT,
2490 };
2491
2492 struct tu_draw_state_group
2493 {
2494    enum tu_draw_state_group_id id;
2495    uint32_t enable_mask;
2496    struct tu_cs_entry ib;
2497 };
2498
2499 const static struct tu_sampler*
2500 sampler_ptr(struct tu_descriptor_state *descriptors_state,
2501             const struct tu_descriptor_map *map, unsigned i,
2502             unsigned array_index)
2503 {
2504    assert(descriptors_state->valid & (1 << map->set[i]));
2505
2506    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2507    assert(map->binding[i] < set->layout->binding_count);
2508
2509    const struct tu_descriptor_set_binding_layout *layout =
2510       &set->layout->binding[map->binding[i]];
2511
2512    if (layout->immutable_samplers_offset) {
2513       const struct tu_sampler *immutable_samplers =
2514          tu_immutable_samplers(set->layout, layout);
2515
2516       return &immutable_samplers[array_index];
2517    }
2518
2519    switch (layout->type) {
2520    case VK_DESCRIPTOR_TYPE_SAMPLER:
2521       return (struct tu_sampler*) &set->mapped_ptr[layout->offset / 4];
2522    case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
2523       return (struct tu_sampler*) &set->mapped_ptr[layout->offset / 4 + A6XX_TEX_CONST_DWORDS +
2524                                                    array_index *
2525                                                    (A6XX_TEX_CONST_DWORDS +
2526                                                     sizeof(struct tu_sampler) / 4)];
2527    default:
2528       unreachable("unimplemented descriptor type");
2529       break;
2530    }
2531 }
2532
2533 static void
2534 write_tex_const(struct tu_cmd_buffer *cmd,
2535                 uint32_t *dst,
2536                 struct tu_descriptor_state *descriptors_state,
2537                 const struct tu_descriptor_map *map,
2538                 unsigned i, unsigned array_index)
2539 {
2540    assert(descriptors_state->valid & (1 << map->set[i]));
2541
2542    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2543    assert(map->binding[i] < set->layout->binding_count);
2544
2545    const struct tu_descriptor_set_binding_layout *layout =
2546       &set->layout->binding[map->binding[i]];
2547
2548    switch (layout->type) {
2549    case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
2550    case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
2551    case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
2552    case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
2553       memcpy(dst, &set->mapped_ptr[layout->offset / 4 +
2554                                    array_index * A6XX_TEX_CONST_DWORDS],
2555              A6XX_TEX_CONST_DWORDS * 4);
2556       break;
2557    case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
2558       memcpy(dst, &set->mapped_ptr[layout->offset / 4 +
2559                                    array_index *
2560                                    (A6XX_TEX_CONST_DWORDS +
2561                                     sizeof(struct tu_sampler) / 4)],
2562              A6XX_TEX_CONST_DWORDS * 4);
2563       break;
2564    default:
2565       unreachable("unimplemented descriptor type");
2566       break;
2567    }
2568
2569    if (layout->type == VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT) {
2570       const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
2571       uint32_t a = cmd->state.subpass->input_attachments[map->value[i] +
2572                                                          array_index].attachment;
2573       const struct tu_render_pass_attachment *att = &cmd->state.pass->attachments[a];
2574
2575       assert(att->gmem_offset >= 0);
2576
2577       dst[0] &= ~(A6XX_TEX_CONST_0_SWAP__MASK | A6XX_TEX_CONST_0_TILE_MODE__MASK);
2578       dst[0] |= A6XX_TEX_CONST_0_TILE_MODE(TILE6_2);
2579       dst[2] &= ~(A6XX_TEX_CONST_2_TYPE__MASK | A6XX_TEX_CONST_2_PITCH__MASK);
2580       dst[2] |=
2581          A6XX_TEX_CONST_2_TYPE(A6XX_TEX_2D) |
2582          A6XX_TEX_CONST_2_PITCH(tiling->tile0.extent.width * att->cpp);
2583       dst[3] = 0;
2584       dst[4] = 0x100000 + att->gmem_offset;
2585       dst[5] = A6XX_TEX_CONST_5_DEPTH(1);
2586       for (unsigned i = 6; i < A6XX_TEX_CONST_DWORDS; i++)
2587          dst[i] = 0;
2588
2589       if (cmd->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY)
2590          tu_finishme("patch input attachment pitch for secondary cmd buffer");
2591    }
2592 }
2593
2594 static void
2595 write_image_ibo(struct tu_cmd_buffer *cmd,
2596                 uint32_t *dst,
2597                 struct tu_descriptor_state *descriptors_state,
2598                 const struct tu_descriptor_map *map,
2599                 unsigned i, unsigned array_index)
2600 {
2601    assert(descriptors_state->valid & (1 << map->set[i]));
2602
2603    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2604    assert(map->binding[i] < set->layout->binding_count);
2605
2606    const struct tu_descriptor_set_binding_layout *layout =
2607       &set->layout->binding[map->binding[i]];
2608
2609    assert(layout->type == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE);
2610
2611    memcpy(dst, &set->mapped_ptr[layout->offset / 4 +
2612                                 (array_index * 2 + 1) * A6XX_TEX_CONST_DWORDS],
2613           A6XX_TEX_CONST_DWORDS * 4);
2614 }
2615
2616 static uint64_t
2617 buffer_ptr(struct tu_descriptor_state *descriptors_state,
2618            const struct tu_descriptor_map *map,
2619            unsigned i, unsigned array_index)
2620 {
2621    assert(descriptors_state->valid & (1 << map->set[i]));
2622
2623    struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2624    assert(map->binding[i] < set->layout->binding_count);
2625
2626    const struct tu_descriptor_set_binding_layout *layout =
2627       &set->layout->binding[map->binding[i]];
2628
2629    switch (layout->type) {
2630    case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
2631    case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
2632       return descriptors_state->dynamic_buffers[layout->dynamic_offset_offset +
2633                                                 array_index];
2634    case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
2635    case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
2636       return (uint64_t) set->mapped_ptr[layout->offset / 4 + array_index * 2 + 1] << 32 |
2637                         set->mapped_ptr[layout->offset / 4 + array_index * 2];
2638    default:
2639       unreachable("unimplemented descriptor type");
2640       break;
2641    }
2642 }
2643
2644 static inline uint32_t
2645 tu6_stage2opcode(gl_shader_stage type)
2646 {
2647    switch (type) {
2648    case MESA_SHADER_VERTEX:
2649    case MESA_SHADER_TESS_CTRL:
2650    case MESA_SHADER_TESS_EVAL:
2651    case MESA_SHADER_GEOMETRY:
2652       return CP_LOAD_STATE6_GEOM;
2653    case MESA_SHADER_FRAGMENT:
2654    case MESA_SHADER_COMPUTE:
2655    case MESA_SHADER_KERNEL:
2656       return CP_LOAD_STATE6_FRAG;
2657    default:
2658       unreachable("bad shader type");
2659    }
2660 }
2661
2662 static inline enum a6xx_state_block
2663 tu6_stage2shadersb(gl_shader_stage type)
2664 {
2665    switch (type) {
2666    case MESA_SHADER_VERTEX:
2667       return SB6_VS_SHADER;
2668    case MESA_SHADER_FRAGMENT:
2669       return SB6_FS_SHADER;
2670    case MESA_SHADER_COMPUTE:
2671    case MESA_SHADER_KERNEL:
2672       return SB6_CS_SHADER;
2673    default:
2674       unreachable("bad shader type");
2675       return ~0;
2676    }
2677 }
2678
2679 static void
2680 tu6_emit_user_consts(struct tu_cs *cs, const struct tu_pipeline *pipeline,
2681                      struct tu_descriptor_state *descriptors_state,
2682                      gl_shader_stage type,
2683                      uint32_t *push_constants)
2684 {
2685    const struct tu_program_descriptor_linkage *link =
2686       &pipeline->program.link[type];
2687    const struct ir3_ubo_analysis_state *state = &link->ubo_state;
2688
2689    for (uint32_t i = 0; i < ARRAY_SIZE(state->range); i++) {
2690       if (state->range[i].start < state->range[i].end) {
2691          uint32_t size = state->range[i].end - state->range[i].start;
2692          uint32_t offset = state->range[i].start;
2693
2694          /* and even if the start of the const buffer is before
2695           * first_immediate, the end may not be:
2696           */
2697          size = MIN2(size, (16 * link->constlen) - state->range[i].offset);
2698
2699          if (size == 0)
2700             continue;
2701
2702          /* things should be aligned to vec4: */
2703          debug_assert((state->range[i].offset % 16) == 0);
2704          debug_assert((size % 16) == 0);
2705          debug_assert((offset % 16) == 0);
2706
2707          if (i == 0) {
2708             /* push constants */
2709             tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + (size / 4));
2710             tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(state->range[i].offset / 16) |
2711                   CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2712                   CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
2713                   CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2714                   CP_LOAD_STATE6_0_NUM_UNIT(size / 16));
2715             tu_cs_emit(cs, 0);
2716             tu_cs_emit(cs, 0);
2717             for (unsigned i = 0; i < size / 4; i++)
2718                tu_cs_emit(cs, push_constants[i + offset / 4]);
2719             continue;
2720          }
2721
2722          /* Look through the UBO map to find our UBO index, and get the VA for
2723           * that UBO.
2724           */
2725          uint64_t va = 0;
2726          uint32_t ubo_idx = i - 1;
2727          uint32_t ubo_map_base = 0;
2728          for (int j = 0; j < link->ubo_map.num; j++) {
2729             if (ubo_idx >= ubo_map_base &&
2730                 ubo_idx < ubo_map_base + link->ubo_map.array_size[j]) {
2731                va = buffer_ptr(descriptors_state, &link->ubo_map, j,
2732                                ubo_idx - ubo_map_base);
2733                break;
2734             }
2735             ubo_map_base += link->ubo_map.array_size[j];
2736          }
2737          assert(va);
2738
2739          tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3);
2740          tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(state->range[i].offset / 16) |
2741                CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2742                CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
2743                CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2744                CP_LOAD_STATE6_0_NUM_UNIT(size / 16));
2745          tu_cs_emit_qw(cs, va + offset);
2746       }
2747    }
2748 }
2749
2750 static void
2751 tu6_emit_ubos(struct tu_cs *cs, const struct tu_pipeline *pipeline,
2752               struct tu_descriptor_state *descriptors_state,
2753               gl_shader_stage type)
2754 {
2755    const struct tu_program_descriptor_linkage *link =
2756       &pipeline->program.link[type];
2757
2758    uint32_t num = MIN2(link->ubo_map.num_desc, link->const_state.num_ubos);
2759    uint32_t anum = align(num, 2);
2760
2761    if (!num)
2762       return;
2763
2764    tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + (2 * anum));
2765    tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(link->const_state.offsets.ubo) |
2766          CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2767          CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
2768          CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2769          CP_LOAD_STATE6_0_NUM_UNIT(anum/2));
2770    tu_cs_emit(cs, CP_LOAD_STATE6_1_EXT_SRC_ADDR(0));
2771    tu_cs_emit(cs, CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI(0));
2772
2773    unsigned emitted = 0;
2774    for (unsigned i = 0; emitted < num && i < link->ubo_map.num; i++) {
2775       for (unsigned j = 0; emitted < num && j < link->ubo_map.array_size[i]; j++) {
2776          tu_cs_emit_qw(cs, buffer_ptr(descriptors_state, &link->ubo_map, i, j));
2777          emitted++;
2778       }
2779    }
2780
2781    for (; emitted < anum; emitted++) {
2782       tu_cs_emit(cs, 0xffffffff);
2783       tu_cs_emit(cs, 0xffffffff);
2784    }
2785 }
2786
2787 static struct tu_cs_entry
2788 tu6_emit_consts(struct tu_cmd_buffer *cmd,
2789                 const struct tu_pipeline *pipeline,
2790                 struct tu_descriptor_state *descriptors_state,
2791                 gl_shader_stage type)
2792 {
2793    struct tu_cs cs;
2794    tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs, 512, &cs); /* TODO: maximum size? */
2795
2796    tu6_emit_user_consts(&cs, pipeline, descriptors_state, type, cmd->push_constants);
2797    tu6_emit_ubos(&cs, pipeline, descriptors_state, type);
2798
2799    return tu_cs_end_sub_stream(&cmd->sub_cs, &cs);
2800 }
2801
2802 static VkResult
2803 tu6_emit_vs_params(struct tu_cmd_buffer *cmd,
2804                    const struct tu_draw_info *draw,
2805                    struct tu_cs_entry *entry)
2806 {
2807    /* TODO: fill out more than just base instance */
2808    const struct tu_program_descriptor_linkage *link =
2809       &cmd->state.pipeline->program.link[MESA_SHADER_VERTEX];
2810    const struct ir3_const_state *const_state = &link->const_state;
2811    struct tu_cs cs;
2812
2813    if (const_state->offsets.driver_param >= link->constlen) {
2814       *entry = (struct tu_cs_entry) {};
2815       return VK_SUCCESS;
2816    }
2817
2818    VkResult result = tu_cs_begin_sub_stream(cmd->device, &cmd->sub_cs, 8, &cs);
2819    if (result != VK_SUCCESS)
2820       return result;
2821
2822    tu_cs_emit_pkt7(&cs, CP_LOAD_STATE6_GEOM, 3 + 4);
2823    tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(const_state->offsets.driver_param) |
2824          CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2825          CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
2826          CP_LOAD_STATE6_0_STATE_BLOCK(SB6_VS_SHADER) |
2827          CP_LOAD_STATE6_0_NUM_UNIT(1));
2828    tu_cs_emit(&cs, 0);
2829    tu_cs_emit(&cs, 0);
2830
2831    STATIC_ASSERT(IR3_DP_INSTID_BASE == 2);
2832
2833    tu_cs_emit(&cs, 0);
2834    tu_cs_emit(&cs, 0);
2835    tu_cs_emit(&cs, draw->first_instance);
2836    tu_cs_emit(&cs, 0);
2837
2838    *entry = tu_cs_end_sub_stream(&cmd->sub_cs, &cs);
2839    return VK_SUCCESS;
2840 }
2841
2842 static VkResult
2843 tu6_emit_textures(struct tu_cmd_buffer *cmd,
2844                   const struct tu_pipeline *pipeline,
2845                   struct tu_descriptor_state *descriptors_state,
2846                   gl_shader_stage type,
2847                   struct tu_cs_entry *entry,
2848                   bool *needs_border)
2849 {
2850    struct tu_device *device = cmd->device;
2851    struct tu_cs *draw_state = &cmd->sub_cs;
2852    const struct tu_program_descriptor_linkage *link =
2853       &pipeline->program.link[type];
2854    VkResult result;
2855
2856    if (link->texture_map.num_desc == 0 && link->sampler_map.num_desc == 0) {
2857       *entry = (struct tu_cs_entry) {};
2858       return VK_SUCCESS;
2859    }
2860
2861    /* allocate and fill texture state */
2862    struct ts_cs_memory tex_const;
2863    result = tu_cs_alloc(device, draw_state, link->texture_map.num_desc,
2864                         A6XX_TEX_CONST_DWORDS, &tex_const);
2865    if (result != VK_SUCCESS)
2866       return result;
2867
2868    int tex_index = 0;
2869    for (unsigned i = 0; i < link->texture_map.num; i++) {
2870       for (int j = 0; j < link->texture_map.array_size[i]; j++) {
2871          write_tex_const(cmd,
2872                          &tex_const.map[A6XX_TEX_CONST_DWORDS * tex_index++],
2873                          descriptors_state, &link->texture_map, i, j);
2874       }
2875    }
2876
2877    /* allocate and fill sampler state */
2878    struct ts_cs_memory tex_samp = { 0 };
2879    if (link->sampler_map.num_desc) {
2880       result = tu_cs_alloc(device, draw_state, link->sampler_map.num_desc,
2881                            A6XX_TEX_SAMP_DWORDS, &tex_samp);
2882       if (result != VK_SUCCESS)
2883          return result;
2884
2885       int sampler_index = 0;
2886       for (unsigned i = 0; i < link->sampler_map.num; i++) {
2887          for (int j = 0; j < link->sampler_map.array_size[i]; j++) {
2888             const struct tu_sampler *sampler = sampler_ptr(descriptors_state,
2889                                                            &link->sampler_map,
2890                                                            i, j);
2891             memcpy(&tex_samp.map[A6XX_TEX_SAMP_DWORDS * sampler_index++],
2892                    sampler->state, sizeof(sampler->state));
2893             *needs_border |= sampler->needs_border;
2894          }
2895       }
2896    }
2897
2898    unsigned tex_samp_reg, tex_const_reg, tex_count_reg;
2899    enum a6xx_state_block sb;
2900
2901    switch (type) {
2902    case MESA_SHADER_VERTEX:
2903       sb = SB6_VS_TEX;
2904       tex_samp_reg = REG_A6XX_SP_VS_TEX_SAMP_LO;
2905       tex_const_reg = REG_A6XX_SP_VS_TEX_CONST_LO;
2906       tex_count_reg = REG_A6XX_SP_VS_TEX_COUNT;
2907       break;
2908    case MESA_SHADER_FRAGMENT:
2909       sb = SB6_FS_TEX;
2910       tex_samp_reg = REG_A6XX_SP_FS_TEX_SAMP_LO;
2911       tex_const_reg = REG_A6XX_SP_FS_TEX_CONST_LO;
2912       tex_count_reg = REG_A6XX_SP_FS_TEX_COUNT;
2913       break;
2914    case MESA_SHADER_COMPUTE:
2915       sb = SB6_CS_TEX;
2916       tex_samp_reg = REG_A6XX_SP_CS_TEX_SAMP_LO;
2917       tex_const_reg = REG_A6XX_SP_CS_TEX_CONST_LO;
2918       tex_count_reg = REG_A6XX_SP_CS_TEX_COUNT;
2919       break;
2920    default:
2921       unreachable("bad state block");
2922    }
2923
2924    struct tu_cs cs;
2925    result = tu_cs_begin_sub_stream(device, draw_state, 16, &cs);
2926    if (result != VK_SUCCESS)
2927       return result;
2928
2929    if (link->sampler_map.num_desc) {
2930       /* output sampler state: */
2931       tu_cs_emit_pkt7(&cs, tu6_stage2opcode(type), 3);
2932       tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
2933                  CP_LOAD_STATE6_0_STATE_TYPE(ST6_SHADER) |
2934                  CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
2935                  CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
2936                  CP_LOAD_STATE6_0_NUM_UNIT(link->sampler_map.num_desc));
2937       tu_cs_emit_qw(&cs, tex_samp.iova); /* SRC_ADDR_LO/HI */
2938
2939       tu_cs_emit_pkt4(&cs, tex_samp_reg, 2);
2940       tu_cs_emit_qw(&cs, tex_samp.iova); /* SRC_ADDR_LO/HI */
2941    }
2942
2943    /* emit texture state: */
2944    tu_cs_emit_pkt7(&cs, tu6_stage2opcode(type), 3);
2945    tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
2946       CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2947       CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
2948       CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
2949       CP_LOAD_STATE6_0_NUM_UNIT(link->texture_map.num_desc));
2950    tu_cs_emit_qw(&cs, tex_const.iova); /* SRC_ADDR_LO/HI */
2951
2952    tu_cs_emit_pkt4(&cs, tex_const_reg, 2);
2953    tu_cs_emit_qw(&cs, tex_const.iova); /* SRC_ADDR_LO/HI */
2954
2955    tu_cs_emit_pkt4(&cs, tex_count_reg, 1);
2956    tu_cs_emit(&cs, link->texture_map.num_desc);
2957
2958    *entry = tu_cs_end_sub_stream(draw_state, &cs);
2959    return VK_SUCCESS;
2960 }
2961
2962 static VkResult
2963 tu6_emit_ibo(struct tu_cmd_buffer *cmd,
2964              const struct tu_pipeline *pipeline,
2965              struct tu_descriptor_state *descriptors_state,
2966              gl_shader_stage type,
2967              struct tu_cs_entry *entry)
2968 {
2969    struct tu_device *device = cmd->device;
2970    struct tu_cs *draw_state = &cmd->sub_cs;
2971    const struct tu_program_descriptor_linkage *link =
2972       &pipeline->program.link[type];
2973    VkResult result;
2974
2975    unsigned num_desc = link->ssbo_map.num_desc + link->image_map.num_desc;
2976
2977    if (num_desc == 0) {
2978       *entry = (struct tu_cs_entry) {};
2979       return VK_SUCCESS;
2980    }
2981
2982    struct ts_cs_memory ibo_const;
2983    result = tu_cs_alloc(device, draw_state, num_desc,
2984                         A6XX_TEX_CONST_DWORDS, &ibo_const);
2985    if (result != VK_SUCCESS)
2986       return result;
2987
2988    int ssbo_index = 0;
2989    for (unsigned i = 0; i < link->ssbo_map.num; i++) {
2990       for (int j = 0; j < link->ssbo_map.array_size[i]; j++) {
2991          uint32_t *dst = &ibo_const.map[A6XX_TEX_CONST_DWORDS * ssbo_index];
2992
2993          uint64_t va = buffer_ptr(descriptors_state, &link->ssbo_map, i, j);
2994          /* We don't expose robustBufferAccess, so leave the size unlimited. */
2995          uint32_t sz = MAX_STORAGE_BUFFER_RANGE / 4;
2996
2997          dst[0] = A6XX_IBO_0_FMT(TFMT6_32_UINT);
2998          dst[1] = A6XX_IBO_1_WIDTH(sz & MASK(15)) |
2999                   A6XX_IBO_1_HEIGHT(sz >> 15);
3000          dst[2] = A6XX_IBO_2_UNK4 |
3001                   A6XX_IBO_2_UNK31 |
3002                   A6XX_IBO_2_TYPE(A6XX_TEX_1D);
3003          dst[3] = 0;
3004          dst[4] = va;
3005          dst[5] = va >> 32;
3006          for (int i = 6; i < A6XX_TEX_CONST_DWORDS; i++)
3007             dst[i] = 0;
3008
3009          ssbo_index++;
3010       }
3011    }
3012
3013    for (unsigned i = 0; i < link->image_map.num; i++) {
3014       for (int j = 0; j < link->image_map.array_size[i]; j++) {
3015          uint32_t *dst = &ibo_const.map[A6XX_TEX_CONST_DWORDS * ssbo_index];
3016
3017          write_image_ibo(cmd, dst,
3018                          descriptors_state, &link->image_map, i, j);
3019
3020          ssbo_index++;
3021       }
3022    }
3023
3024    assert(ssbo_index == num_desc);
3025
3026    struct tu_cs cs;
3027    result = tu_cs_begin_sub_stream(device, draw_state, 7, &cs);
3028    if (result != VK_SUCCESS)
3029       return result;
3030
3031    uint32_t opcode, ibo_addr_reg;
3032    enum a6xx_state_block sb;
3033    enum a6xx_state_type st;
3034
3035    switch (type) {
3036    case MESA_SHADER_FRAGMENT:
3037       opcode = CP_LOAD_STATE6;
3038       st = ST6_SHADER;
3039       sb = SB6_IBO;
3040       ibo_addr_reg = REG_A6XX_SP_IBO_LO;
3041       break;
3042    case MESA_SHADER_COMPUTE:
3043       opcode = CP_LOAD_STATE6_FRAG;
3044       st = ST6_IBO;
3045       sb = SB6_CS_SHADER;
3046       ibo_addr_reg = REG_A6XX_SP_CS_IBO_LO;
3047       break;
3048    default:
3049       unreachable("unsupported stage for ibos");
3050    }
3051
3052    /* emit texture state: */
3053    tu_cs_emit_pkt7(&cs, opcode, 3);
3054    tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
3055               CP_LOAD_STATE6_0_STATE_TYPE(st) |
3056               CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
3057               CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
3058               CP_LOAD_STATE6_0_NUM_UNIT(num_desc));
3059    tu_cs_emit_qw(&cs, ibo_const.iova); /* SRC_ADDR_LO/HI */
3060
3061    tu_cs_emit_pkt4(&cs, ibo_addr_reg, 2);
3062    tu_cs_emit_qw(&cs, ibo_const.iova); /* SRC_ADDR_LO/HI */
3063
3064    *entry = tu_cs_end_sub_stream(draw_state, &cs);
3065    return VK_SUCCESS;
3066 }
3067
3068 struct PACKED bcolor_entry {
3069    uint32_t fp32[4];
3070    uint16_t ui16[4];
3071    int16_t  si16[4];
3072    uint16_t fp16[4];
3073    uint16_t rgb565;
3074    uint16_t rgb5a1;
3075    uint16_t rgba4;
3076    uint8_t __pad0[2];
3077    uint8_t  ui8[4];
3078    int8_t   si8[4];
3079    uint32_t rgb10a2;
3080    uint32_t z24; /* also s8? */
3081    uint16_t srgb[4];      /* appears to duplicate fp16[], but clamped, used for srgb */
3082    uint8_t  __pad1[56];
3083 } border_color[] = {
3084    [VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK] = {},
3085    [VK_BORDER_COLOR_INT_TRANSPARENT_BLACK] = {},
3086    [VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK] = {
3087       .fp32[3] = 0x3f800000,
3088       .ui16[3] = 0xffff,
3089       .si16[3] = 0x7fff,
3090       .fp16[3] = 0x3c00,
3091       .rgb5a1 = 0x8000,
3092       .rgba4 = 0xf000,
3093       .ui8[3] = 0xff,
3094       .si8[3] = 0x7f,
3095       .rgb10a2 = 0xc0000000,
3096       .srgb[3] = 0x3c00,
3097    },
3098    [VK_BORDER_COLOR_INT_OPAQUE_BLACK] = {
3099       .fp32[3] = 1,
3100       .fp16[3] = 1,
3101    },
3102    [VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE] = {
3103       .fp32[0 ... 3] = 0x3f800000,
3104       .ui16[0 ... 3] = 0xffff,
3105       .si16[0 ... 3] = 0x7fff,
3106       .fp16[0 ... 3] = 0x3c00,
3107       .rgb565 = 0xffff,
3108       .rgb5a1 = 0xffff,
3109       .rgba4 = 0xffff,
3110       .ui8[0 ... 3] = 0xff,
3111       .si8[0 ... 3] = 0x7f,
3112       .rgb10a2 = 0xffffffff,
3113       .z24 = 0xffffff,
3114       .srgb[0 ... 3] = 0x3c00,
3115    },
3116    [VK_BORDER_COLOR_INT_OPAQUE_WHITE] = {
3117       .fp32[0 ... 3] = 1,
3118       .fp16[0 ... 3] = 1,
3119    },
3120 };
3121
3122 static VkResult
3123 tu6_emit_border_color(struct tu_cmd_buffer *cmd,
3124                       struct tu_cs *cs)
3125 {
3126    STATIC_ASSERT(sizeof(struct bcolor_entry) == 128);
3127
3128    const struct tu_pipeline *pipeline = cmd->state.pipeline;
3129    struct tu_descriptor_state *descriptors_state =
3130       &cmd->descriptors[VK_PIPELINE_BIND_POINT_GRAPHICS];
3131    const struct tu_descriptor_map *vs_sampler =
3132       &pipeline->program.link[MESA_SHADER_VERTEX].sampler_map;
3133    const struct tu_descriptor_map *fs_sampler =
3134       &pipeline->program.link[MESA_SHADER_FRAGMENT].sampler_map;
3135    struct ts_cs_memory ptr;
3136
3137    VkResult result = tu_cs_alloc(cmd->device, &cmd->sub_cs,
3138                                  vs_sampler->num_desc + fs_sampler->num_desc,
3139                                  128 / 4,
3140                                  &ptr);
3141    if (result != VK_SUCCESS)
3142       return result;
3143
3144    for (unsigned i = 0; i < vs_sampler->num; i++) {
3145       for (unsigned j = 0; j < vs_sampler->array_size[i]; j++) {
3146          const struct tu_sampler *sampler = sampler_ptr(descriptors_state,
3147                                                         vs_sampler, i, j);
3148          memcpy(ptr.map, &border_color[sampler->border], 128);
3149          ptr.map += 128 / 4;
3150       }
3151    }
3152
3153    for (unsigned i = 0; i < fs_sampler->num; i++) {
3154       for (unsigned j = 0; j < fs_sampler->array_size[i]; j++) {
3155          const struct tu_sampler *sampler = sampler_ptr(descriptors_state,
3156                                                         fs_sampler, i, j);
3157          memcpy(ptr.map, &border_color[sampler->border], 128);
3158          ptr.map += 128 / 4;
3159       }
3160    }
3161
3162    tu_cs_emit_pkt4(cs, REG_A6XX_SP_TP_BORDER_COLOR_BASE_ADDR_LO, 2);
3163    tu_cs_emit_qw(cs, ptr.iova);
3164    return VK_SUCCESS;
3165 }
3166
3167 static VkResult
3168 tu6_bind_draw_states(struct tu_cmd_buffer *cmd,
3169                      struct tu_cs *cs,
3170                      const struct tu_draw_info *draw)
3171 {
3172    const struct tu_pipeline *pipeline = cmd->state.pipeline;
3173    const struct tu_dynamic_state *dynamic = &cmd->state.dynamic;
3174    struct tu_draw_state_group draw_state_groups[TU_DRAW_STATE_COUNT];
3175    uint32_t draw_state_group_count = 0;
3176
3177    struct tu_descriptor_state *descriptors_state =
3178       &cmd->descriptors[VK_PIPELINE_BIND_POINT_GRAPHICS];
3179
3180    VkResult result = tu_cs_reserve_space(cmd->device, cs, 256);
3181    if (result != VK_SUCCESS)
3182       return result;
3183
3184    /* TODO lrz */
3185
3186    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9806, 0);
3187    tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9990, 0);
3188    tu_cs_emit_write_reg(cs, REG_A6XX_VFD_UNKNOWN_A008, 0);
3189
3190    tu_cs_emit_regs(cs,
3191                    A6XX_PC_PRIMITIVE_CNTL_0(.primitive_restart =
3192                                             pipeline->ia.primitive_restart && draw->indexed));
3193
3194    if (cmd->state.dirty &
3195           (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_DYNAMIC_LINE_WIDTH) &&
3196        (pipeline->dynamic_state.mask & TU_DYNAMIC_LINE_WIDTH)) {
3197       tu6_emit_gras_su_cntl(cs, pipeline->rast.gras_su_cntl,
3198                             dynamic->line_width);
3199    }
3200
3201    if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK) &&
3202        (pipeline->dynamic_state.mask & TU_DYNAMIC_STENCIL_COMPARE_MASK)) {
3203       tu6_emit_stencil_compare_mask(cs, dynamic->stencil_compare_mask.front,
3204                                     dynamic->stencil_compare_mask.back);
3205    }
3206
3207    if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK) &&
3208        (pipeline->dynamic_state.mask & TU_DYNAMIC_STENCIL_WRITE_MASK)) {
3209       tu6_emit_stencil_write_mask(cs, dynamic->stencil_write_mask.front,
3210                                   dynamic->stencil_write_mask.back);
3211    }
3212
3213    if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE) &&
3214        (pipeline->dynamic_state.mask & TU_DYNAMIC_STENCIL_REFERENCE)) {
3215       tu6_emit_stencil_reference(cs, dynamic->stencil_reference.front,
3216                                  dynamic->stencil_reference.back);
3217    }
3218
3219    if (cmd->state.dirty &
3220        (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_VERTEX_BUFFERS)) {
3221       for (uint32_t i = 0; i < pipeline->vi.count; i++) {
3222          const uint32_t binding = pipeline->vi.bindings[i];
3223          const uint32_t stride = pipeline->vi.strides[i];
3224          const struct tu_buffer *buf = cmd->state.vb.buffers[binding];
3225          const VkDeviceSize offset = buf->bo_offset +
3226                                      cmd->state.vb.offsets[binding] +
3227                                      pipeline->vi.offsets[i];
3228          const VkDeviceSize size =
3229             offset < buf->bo->size ? buf->bo->size - offset : 0;
3230
3231          tu_cs_emit_regs(cs,
3232                          A6XX_VFD_FETCH_BASE(i, .bo = buf->bo, .bo_offset = offset),
3233                          A6XX_VFD_FETCH_SIZE(i, size),
3234                          A6XX_VFD_FETCH_STRIDE(i, stride));
3235       }
3236    }
3237
3238    if (cmd->state.dirty & TU_CMD_DIRTY_PIPELINE) {
3239       draw_state_groups[draw_state_group_count++] =
3240          (struct tu_draw_state_group) {
3241             .id = TU_DRAW_STATE_PROGRAM,
3242             .enable_mask = ENABLE_DRAW,
3243             .ib = pipeline->program.state_ib,
3244          };
3245       draw_state_groups[draw_state_group_count++] =
3246          (struct tu_draw_state_group) {
3247             .id = TU_DRAW_STATE_PROGRAM_BINNING,
3248             .enable_mask = CP_SET_DRAW_STATE__0_BINNING,
3249             .ib = pipeline->program.binning_state_ib,
3250          };
3251       draw_state_groups[draw_state_group_count++] =
3252          (struct tu_draw_state_group) {
3253             .id = TU_DRAW_STATE_VI,
3254             .enable_mask = ENABLE_DRAW,
3255             .ib = pipeline->vi.state_ib,
3256          };
3257       draw_state_groups[draw_state_group_count++] =
3258          (struct tu_draw_state_group) {
3259             .id = TU_DRAW_STATE_VI_BINNING,
3260             .enable_mask = CP_SET_DRAW_STATE__0_BINNING,
3261             .ib = pipeline->vi.binning_state_ib,
3262          };
3263       draw_state_groups[draw_state_group_count++] =
3264          (struct tu_draw_state_group) {
3265             .id = TU_DRAW_STATE_VP,
3266             .enable_mask = ENABLE_ALL,
3267             .ib = pipeline->vp.state_ib,
3268          };
3269       draw_state_groups[draw_state_group_count++] =
3270          (struct tu_draw_state_group) {
3271             .id = TU_DRAW_STATE_RAST,
3272             .enable_mask = ENABLE_ALL,
3273             .ib = pipeline->rast.state_ib,
3274          };
3275       draw_state_groups[draw_state_group_count++] =
3276          (struct tu_draw_state_group) {
3277             .id = TU_DRAW_STATE_DS,
3278             .enable_mask = ENABLE_ALL,
3279             .ib = pipeline->ds.state_ib,
3280          };
3281       draw_state_groups[draw_state_group_count++] =
3282          (struct tu_draw_state_group) {
3283             .id = TU_DRAW_STATE_BLEND,
3284             .enable_mask = ENABLE_ALL,
3285             .ib = pipeline->blend.state_ib,
3286          };
3287    }
3288
3289    if (cmd->state.dirty &
3290          (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_DESCRIPTOR_SETS | TU_CMD_DIRTY_PUSH_CONSTANTS)) {
3291       draw_state_groups[draw_state_group_count++] =
3292          (struct tu_draw_state_group) {
3293             .id = TU_DRAW_STATE_VS_CONST,
3294             .enable_mask = ENABLE_ALL,
3295             .ib = tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_VERTEX)
3296          };
3297       draw_state_groups[draw_state_group_count++] =
3298          (struct tu_draw_state_group) {
3299             .id = TU_DRAW_STATE_FS_CONST,
3300             .enable_mask = ENABLE_DRAW,
3301             .ib = tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_FRAGMENT)
3302          };
3303    }
3304
3305    if (cmd->state.dirty &
3306          (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_DESCRIPTOR_SETS)) {
3307       bool needs_border = false;
3308       struct tu_cs_entry vs_tex, fs_tex, fs_ibo;
3309
3310       result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3311                                  MESA_SHADER_VERTEX, &vs_tex, &needs_border);
3312       if (result != VK_SUCCESS)
3313          return result;
3314
3315       result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3316                                  MESA_SHADER_FRAGMENT, &fs_tex, &needs_border);
3317       if (result != VK_SUCCESS)
3318          return result;
3319
3320       result = tu6_emit_ibo(cmd, pipeline, descriptors_state,
3321                             MESA_SHADER_FRAGMENT, &fs_ibo);
3322       if (result != VK_SUCCESS)
3323          return result;
3324
3325       draw_state_groups[draw_state_group_count++] =
3326          (struct tu_draw_state_group) {
3327             .id = TU_DRAW_STATE_VS_TEX,
3328             .enable_mask = ENABLE_ALL,
3329             .ib = vs_tex,
3330          };
3331       draw_state_groups[draw_state_group_count++] =
3332          (struct tu_draw_state_group) {
3333             .id = TU_DRAW_STATE_FS_TEX,
3334             .enable_mask = ENABLE_DRAW,
3335             .ib = fs_tex,
3336          };
3337       draw_state_groups[draw_state_group_count++] =
3338          (struct tu_draw_state_group) {
3339             .id = TU_DRAW_STATE_FS_IBO,
3340             .enable_mask = ENABLE_DRAW,
3341             .ib = fs_ibo,
3342          };
3343
3344       if (needs_border) {
3345          result = tu6_emit_border_color(cmd, cs);
3346          if (result != VK_SUCCESS)
3347             return result;
3348       }
3349    }
3350
3351    struct tu_cs_entry vs_params;
3352    result = tu6_emit_vs_params(cmd, draw, &vs_params);
3353    if (result != VK_SUCCESS)
3354       return result;
3355
3356    draw_state_groups[draw_state_group_count++] =
3357       (struct tu_draw_state_group) {
3358          .id = TU_DRAW_STATE_VS_PARAMS,
3359          .enable_mask = ENABLE_ALL,
3360          .ib = vs_params,
3361       };
3362
3363    tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3 * draw_state_group_count);
3364    for (uint32_t i = 0; i < draw_state_group_count; i++) {
3365       const struct tu_draw_state_group *group = &draw_state_groups[i];
3366       debug_assert((group->enable_mask & ~ENABLE_ALL) == 0);
3367       uint32_t cp_set_draw_state =
3368          CP_SET_DRAW_STATE__0_COUNT(group->ib.size / 4) |
3369          group->enable_mask |
3370          CP_SET_DRAW_STATE__0_GROUP_ID(group->id);
3371       uint64_t iova;
3372       if (group->ib.size) {
3373          iova = group->ib.bo->iova + group->ib.offset;
3374       } else {
3375          cp_set_draw_state |= CP_SET_DRAW_STATE__0_DISABLE;
3376          iova = 0;
3377       }
3378
3379       tu_cs_emit(cs, cp_set_draw_state);
3380       tu_cs_emit_qw(cs, iova);
3381    }
3382
3383    tu_cs_sanity_check(cs);
3384
3385    /* track BOs */
3386    if (cmd->state.dirty & TU_CMD_DIRTY_VERTEX_BUFFERS) {
3387       for (uint32_t i = 0; i < MAX_VBS; i++) {
3388          const struct tu_buffer *buf = cmd->state.vb.buffers[i];
3389          if (buf)
3390             tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
3391       }
3392    }
3393    if (cmd->state.dirty & TU_CMD_DIRTY_DESCRIPTOR_SETS) {
3394       unsigned i;
3395       for_each_bit(i, descriptors_state->valid) {
3396          struct tu_descriptor_set *set = descriptors_state->sets[i];
3397          for (unsigned j = 0; j < set->layout->buffer_count; ++j)
3398             if (set->descriptors[j]) {
3399                tu_bo_list_add(&cmd->bo_list, set->descriptors[j],
3400                               MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3401             }
3402       }
3403    }
3404
3405    /* Fragment shader state overwrites compute shader state, so flag the
3406     * compute pipeline for re-emit.
3407     */
3408    cmd->state.dirty = TU_CMD_DIRTY_COMPUTE_PIPELINE;
3409    return VK_SUCCESS;
3410 }
3411
3412 static void
3413 tu6_emit_draw_direct(struct tu_cmd_buffer *cmd,
3414                      struct tu_cs *cs,
3415                      const struct tu_draw_info *draw)
3416 {
3417
3418    const enum pc_di_primtype primtype = cmd->state.pipeline->ia.primtype;
3419
3420    tu_cs_emit_regs(cs,
3421                    A6XX_VFD_INDEX_OFFSET(draw->vertex_offset),
3422                    A6XX_VFD_INSTANCE_START_OFFSET(draw->first_instance));
3423
3424    /* TODO hw binning */
3425    if (draw->indexed) {
3426       const enum a4xx_index_size index_size =
3427          tu6_index_size(cmd->state.index_type);
3428       const uint32_t index_bytes =
3429          (cmd->state.index_type == VK_INDEX_TYPE_UINT32) ? 4 : 2;
3430       const struct tu_buffer *buf = cmd->state.index_buffer;
3431       const VkDeviceSize offset = buf->bo_offset + cmd->state.index_offset +
3432                                   index_bytes * draw->first_index;
3433       const uint32_t size = index_bytes * draw->count;
3434
3435       const uint32_t cp_draw_indx =
3436          CP_DRAW_INDX_OFFSET_0_PRIM_TYPE(primtype) |
3437          CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT(DI_SRC_SEL_DMA) |
3438          CP_DRAW_INDX_OFFSET_0_INDEX_SIZE(index_size) |
3439          CP_DRAW_INDX_OFFSET_0_VIS_CULL(USE_VISIBILITY) | 0x2000;
3440
3441       tu_cs_emit_pkt7(cs, CP_DRAW_INDX_OFFSET, 7);
3442       tu_cs_emit(cs, cp_draw_indx);
3443       tu_cs_emit(cs, draw->instance_count);
3444       tu_cs_emit(cs, draw->count);
3445       tu_cs_emit(cs, 0x0); /* XXX */
3446       tu_cs_emit_qw(cs, buf->bo->iova + offset);
3447       tu_cs_emit(cs, size);
3448    } else {
3449       const uint32_t cp_draw_indx =
3450          CP_DRAW_INDX_OFFSET_0_PRIM_TYPE(primtype) |
3451          CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT(DI_SRC_SEL_AUTO_INDEX) |
3452          CP_DRAW_INDX_OFFSET_0_VIS_CULL(USE_VISIBILITY) | 0x2000;
3453
3454       tu_cs_emit_pkt7(cs, CP_DRAW_INDX_OFFSET, 3);
3455       tu_cs_emit(cs, cp_draw_indx);
3456       tu_cs_emit(cs, draw->instance_count);
3457       tu_cs_emit(cs, draw->count);
3458    }
3459 }
3460
3461 static void
3462 tu_draw(struct tu_cmd_buffer *cmd, const struct tu_draw_info *draw)
3463 {
3464    struct tu_cs *cs = &cmd->draw_cs;
3465    VkResult result;
3466
3467    result = tu6_bind_draw_states(cmd, cs, draw);
3468    if (result != VK_SUCCESS) {
3469       cmd->record_result = result;
3470       return;
3471    }
3472
3473    result = tu_cs_reserve_space(cmd->device, cs, 32);
3474    if (result != VK_SUCCESS) {
3475       cmd->record_result = result;
3476       return;
3477    }
3478
3479    if (draw->indirect) {
3480       tu_finishme("indirect draw");
3481       return;
3482    }
3483
3484    /* TODO tu6_emit_marker should pick different regs depending on cs */
3485
3486    tu6_emit_marker(cmd, cs);
3487    tu6_emit_draw_direct(cmd, cs, draw);
3488    tu6_emit_marker(cmd, cs);
3489
3490    cmd->wait_for_idle = true;
3491
3492    tu_cs_sanity_check(cs);
3493 }
3494
3495 void
3496 tu_CmdDraw(VkCommandBuffer commandBuffer,
3497            uint32_t vertexCount,
3498            uint32_t instanceCount,
3499            uint32_t firstVertex,
3500            uint32_t firstInstance)
3501 {
3502    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3503    struct tu_draw_info info = {};
3504
3505    info.count = vertexCount;
3506    info.instance_count = instanceCount;
3507    info.first_instance = firstInstance;
3508    info.vertex_offset = firstVertex;
3509
3510    tu_draw(cmd_buffer, &info);
3511 }
3512
3513 void
3514 tu_CmdDrawIndexed(VkCommandBuffer commandBuffer,
3515                   uint32_t indexCount,
3516                   uint32_t instanceCount,
3517                   uint32_t firstIndex,
3518                   int32_t vertexOffset,
3519                   uint32_t firstInstance)
3520 {
3521    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3522    struct tu_draw_info info = {};
3523
3524    info.indexed = true;
3525    info.count = indexCount;
3526    info.instance_count = instanceCount;
3527    info.first_index = firstIndex;
3528    info.vertex_offset = vertexOffset;
3529    info.first_instance = firstInstance;
3530
3531    tu_draw(cmd_buffer, &info);
3532 }
3533
3534 void
3535 tu_CmdDrawIndirect(VkCommandBuffer commandBuffer,
3536                    VkBuffer _buffer,
3537                    VkDeviceSize offset,
3538                    uint32_t drawCount,
3539                    uint32_t stride)
3540 {
3541    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3542    TU_FROM_HANDLE(tu_buffer, buffer, _buffer);
3543    struct tu_draw_info info = {};
3544
3545    info.count = drawCount;
3546    info.indirect = buffer;
3547    info.indirect_offset = offset;
3548    info.stride = stride;
3549
3550    tu_draw(cmd_buffer, &info);
3551 }
3552
3553 void
3554 tu_CmdDrawIndexedIndirect(VkCommandBuffer commandBuffer,
3555                           VkBuffer _buffer,
3556                           VkDeviceSize offset,
3557                           uint32_t drawCount,
3558                           uint32_t stride)
3559 {
3560    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3561    TU_FROM_HANDLE(tu_buffer, buffer, _buffer);
3562    struct tu_draw_info info = {};
3563
3564    info.indexed = true;
3565    info.count = drawCount;
3566    info.indirect = buffer;
3567    info.indirect_offset = offset;
3568    info.stride = stride;
3569
3570    tu_draw(cmd_buffer, &info);
3571 }
3572
3573 struct tu_dispatch_info
3574 {
3575    /**
3576     * Determine the layout of the grid (in block units) to be used.
3577     */
3578    uint32_t blocks[3];
3579
3580    /**
3581     * A starting offset for the grid. If unaligned is set, the offset
3582     * must still be aligned.
3583     */
3584    uint32_t offsets[3];
3585    /**
3586     * Whether it's an unaligned compute dispatch.
3587     */
3588    bool unaligned;
3589
3590    /**
3591     * Indirect compute parameters resource.
3592     */
3593    struct tu_buffer *indirect;
3594    uint64_t indirect_offset;
3595 };
3596
3597 static void
3598 tu_emit_compute_driver_params(struct tu_cs *cs, struct tu_pipeline *pipeline,
3599                               const struct tu_dispatch_info *info)
3600 {
3601    gl_shader_stage type = MESA_SHADER_COMPUTE;
3602    const struct tu_program_descriptor_linkage *link =
3603       &pipeline->program.link[type];
3604    const struct ir3_const_state *const_state = &link->const_state;
3605    uint32_t offset = const_state->offsets.driver_param;
3606
3607    if (link->constlen <= offset)
3608       return;
3609
3610    if (!info->indirect) {
3611       uint32_t driver_params[IR3_DP_CS_COUNT] = {
3612          [IR3_DP_NUM_WORK_GROUPS_X] = info->blocks[0],
3613          [IR3_DP_NUM_WORK_GROUPS_Y] = info->blocks[1],
3614          [IR3_DP_NUM_WORK_GROUPS_Z] = info->blocks[2],
3615          [IR3_DP_LOCAL_GROUP_SIZE_X] = pipeline->compute.local_size[0],
3616          [IR3_DP_LOCAL_GROUP_SIZE_Y] = pipeline->compute.local_size[1],
3617          [IR3_DP_LOCAL_GROUP_SIZE_Z] = pipeline->compute.local_size[2],
3618       };
3619
3620       uint32_t num_consts = MIN2(const_state->num_driver_params,
3621                                  (link->constlen - offset) * 4);
3622       /* push constants */
3623       tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + num_consts);
3624       tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(offset) |
3625                  CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
3626                  CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
3627                  CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
3628                  CP_LOAD_STATE6_0_NUM_UNIT(num_consts / 4));
3629       tu_cs_emit(cs, 0);
3630       tu_cs_emit(cs, 0);
3631       uint32_t i;
3632       for (i = 0; i < num_consts; i++)
3633          tu_cs_emit(cs, driver_params[i]);
3634    } else {
3635       tu_finishme("Indirect driver params");
3636    }
3637 }
3638
3639 static void
3640 tu_dispatch(struct tu_cmd_buffer *cmd,
3641             const struct tu_dispatch_info *info)
3642 {
3643    struct tu_cs *cs = &cmd->cs;
3644    struct tu_pipeline *pipeline = cmd->state.compute_pipeline;
3645    struct tu_descriptor_state *descriptors_state =
3646       &cmd->descriptors[VK_PIPELINE_BIND_POINT_COMPUTE];
3647
3648    VkResult result = tu_cs_reserve_space(cmd->device, cs, 256);
3649    if (result != VK_SUCCESS) {
3650       cmd->record_result = result;
3651       return;
3652    }
3653
3654    if (cmd->state.dirty & TU_CMD_DIRTY_COMPUTE_PIPELINE)
3655       tu_cs_emit_ib(cs, &pipeline->program.state_ib);
3656
3657    struct tu_cs_entry ib;
3658
3659    ib = tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_COMPUTE);
3660    if (ib.size)
3661       tu_cs_emit_ib(cs, &ib);
3662
3663    tu_emit_compute_driver_params(cs, pipeline, info);
3664
3665    bool needs_border;
3666    result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3667                               MESA_SHADER_COMPUTE, &ib, &needs_border);
3668    if (result != VK_SUCCESS) {
3669       cmd->record_result = result;
3670       return;
3671    }
3672
3673    if (ib.size)
3674       tu_cs_emit_ib(cs, &ib);
3675
3676    if (needs_border)
3677       tu_finishme("compute border color");
3678
3679    result = tu6_emit_ibo(cmd, pipeline, descriptors_state, MESA_SHADER_COMPUTE, &ib);
3680    if (result != VK_SUCCESS) {
3681       cmd->record_result = result;
3682       return;
3683    }
3684
3685    if (ib.size)
3686       tu_cs_emit_ib(cs, &ib);
3687
3688    /* track BOs */
3689    if (cmd->state.dirty & TU_CMD_DIRTY_DESCRIPTOR_SETS) {
3690       unsigned i;
3691       for_each_bit(i, descriptors_state->valid) {
3692          struct tu_descriptor_set *set = descriptors_state->sets[i];
3693          for (unsigned j = 0; j < set->layout->buffer_count; ++j)
3694             if (set->descriptors[j]) {
3695                tu_bo_list_add(&cmd->bo_list, set->descriptors[j],
3696                               MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3697             }
3698       }
3699    }
3700
3701    /* Compute shader state overwrites fragment shader state, so we flag the
3702     * graphics pipeline for re-emit.
3703     */
3704    cmd->state.dirty = TU_CMD_DIRTY_PIPELINE;
3705
3706    tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
3707    tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(0x8));
3708
3709    const uint32_t *local_size = pipeline->compute.local_size;
3710    const uint32_t *num_groups = info->blocks;
3711    tu_cs_emit_regs(cs,
3712                    A6XX_HLSQ_CS_NDRANGE_0(.kerneldim = 3,
3713                                           .localsizex = local_size[0] - 1,
3714                                           .localsizey = local_size[1] - 1,
3715                                           .localsizez = local_size[2] - 1),
3716                    A6XX_HLSQ_CS_NDRANGE_1(.globalsize_x = local_size[0] * num_groups[0]),
3717                    A6XX_HLSQ_CS_NDRANGE_2(.globaloff_x = 0),
3718                    A6XX_HLSQ_CS_NDRANGE_3(.globalsize_y = local_size[1] * num_groups[1]),
3719                    A6XX_HLSQ_CS_NDRANGE_4(.globaloff_y = 0),
3720                    A6XX_HLSQ_CS_NDRANGE_5(.globalsize_z = local_size[2] * num_groups[2]),
3721                    A6XX_HLSQ_CS_NDRANGE_6(.globaloff_z = 0));
3722
3723    tu_cs_emit_regs(cs,
3724                    A6XX_HLSQ_CS_KERNEL_GROUP_X(1),
3725                    A6XX_HLSQ_CS_KERNEL_GROUP_Y(1),
3726                    A6XX_HLSQ_CS_KERNEL_GROUP_Z(1));
3727
3728    if (info->indirect) {
3729       uint64_t iova = tu_buffer_iova(info->indirect) + info->indirect_offset;
3730
3731       tu_bo_list_add(&cmd->bo_list, info->indirect->bo,
3732                      MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3733
3734       tu_cs_emit_pkt7(cs, CP_EXEC_CS_INDIRECT, 4);
3735       tu_cs_emit(cs, 0x00000000);
3736       tu_cs_emit_qw(cs, iova);
3737       tu_cs_emit(cs,
3738                  A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX(local_size[0] - 1) |
3739                  A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY(local_size[1] - 1) |
3740                  A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ(local_size[2] - 1));
3741    } else {
3742       tu_cs_emit_pkt7(cs, CP_EXEC_CS, 4);
3743       tu_cs_emit(cs, 0x00000000);
3744       tu_cs_emit(cs, CP_EXEC_CS_1_NGROUPS_X(info->blocks[0]));
3745       tu_cs_emit(cs, CP_EXEC_CS_2_NGROUPS_Y(info->blocks[1]));
3746       tu_cs_emit(cs, CP_EXEC_CS_3_NGROUPS_Z(info->blocks[2]));
3747    }
3748
3749    tu_cs_emit_wfi(cs);
3750
3751    tu6_emit_cache_flush(cmd, cs);
3752 }
3753
3754 void
3755 tu_CmdDispatchBase(VkCommandBuffer commandBuffer,
3756                    uint32_t base_x,
3757                    uint32_t base_y,
3758                    uint32_t base_z,
3759                    uint32_t x,
3760                    uint32_t y,
3761                    uint32_t z)
3762 {
3763    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3764    struct tu_dispatch_info info = {};
3765
3766    info.blocks[0] = x;
3767    info.blocks[1] = y;
3768    info.blocks[2] = z;
3769
3770    info.offsets[0] = base_x;
3771    info.offsets[1] = base_y;
3772    info.offsets[2] = base_z;
3773    tu_dispatch(cmd_buffer, &info);
3774 }
3775
3776 void
3777 tu_CmdDispatch(VkCommandBuffer commandBuffer,
3778                uint32_t x,
3779                uint32_t y,
3780                uint32_t z)
3781 {
3782    tu_CmdDispatchBase(commandBuffer, 0, 0, 0, x, y, z);
3783 }
3784
3785 void
3786 tu_CmdDispatchIndirect(VkCommandBuffer commandBuffer,
3787                        VkBuffer _buffer,
3788                        VkDeviceSize offset)
3789 {
3790    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3791    TU_FROM_HANDLE(tu_buffer, buffer, _buffer);
3792    struct tu_dispatch_info info = {};
3793
3794    info.indirect = buffer;
3795    info.indirect_offset = offset;
3796
3797    tu_dispatch(cmd_buffer, &info);
3798 }
3799
3800 void
3801 tu_CmdEndRenderPass(VkCommandBuffer commandBuffer)
3802 {
3803    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3804
3805    tu_cs_end(&cmd_buffer->draw_cs);
3806    tu_cs_end(&cmd_buffer->draw_epilogue_cs);
3807
3808    tu_cmd_render_tiles(cmd_buffer);
3809
3810    /* discard draw_cs and draw_epilogue_cs entries now that the tiles are
3811       rendered */
3812    tu_cs_discard_entries(&cmd_buffer->draw_cs);
3813    tu_cs_begin(&cmd_buffer->draw_cs);
3814    tu_cs_discard_entries(&cmd_buffer->draw_epilogue_cs);
3815    tu_cs_begin(&cmd_buffer->draw_epilogue_cs);
3816
3817    cmd_buffer->state.pass = NULL;
3818    cmd_buffer->state.subpass = NULL;
3819    cmd_buffer->state.framebuffer = NULL;
3820 }
3821
3822 void
3823 tu_CmdEndRenderPass2(VkCommandBuffer commandBuffer,
3824                      const VkSubpassEndInfoKHR *pSubpassEndInfo)
3825 {
3826    tu_CmdEndRenderPass(commandBuffer);
3827 }
3828
3829 struct tu_barrier_info
3830 {
3831    uint32_t eventCount;
3832    const VkEvent *pEvents;
3833    VkPipelineStageFlags srcStageMask;
3834 };
3835
3836 static void
3837 tu_barrier(struct tu_cmd_buffer *cmd_buffer,
3838            uint32_t memoryBarrierCount,
3839            const VkMemoryBarrier *pMemoryBarriers,
3840            uint32_t bufferMemoryBarrierCount,
3841            const VkBufferMemoryBarrier *pBufferMemoryBarriers,
3842            uint32_t imageMemoryBarrierCount,
3843            const VkImageMemoryBarrier *pImageMemoryBarriers,
3844            const struct tu_barrier_info *info)
3845 {
3846 }
3847
3848 void
3849 tu_CmdPipelineBarrier(VkCommandBuffer commandBuffer,
3850                       VkPipelineStageFlags srcStageMask,
3851                       VkPipelineStageFlags destStageMask,
3852                       VkBool32 byRegion,
3853                       uint32_t memoryBarrierCount,
3854                       const VkMemoryBarrier *pMemoryBarriers,
3855                       uint32_t bufferMemoryBarrierCount,
3856                       const VkBufferMemoryBarrier *pBufferMemoryBarriers,
3857                       uint32_t imageMemoryBarrierCount,
3858                       const VkImageMemoryBarrier *pImageMemoryBarriers)
3859 {
3860    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3861    struct tu_barrier_info info;
3862
3863    info.eventCount = 0;
3864    info.pEvents = NULL;
3865    info.srcStageMask = srcStageMask;
3866
3867    tu_barrier(cmd_buffer, memoryBarrierCount, pMemoryBarriers,
3868               bufferMemoryBarrierCount, pBufferMemoryBarriers,
3869               imageMemoryBarrierCount, pImageMemoryBarriers, &info);
3870 }
3871
3872 static void
3873 write_event(struct tu_cmd_buffer *cmd_buffer,
3874             struct tu_event *event,
3875             VkPipelineStageFlags stageMask,
3876             unsigned value)
3877 {
3878 }
3879
3880 void
3881 tu_CmdSetEvent(VkCommandBuffer commandBuffer,
3882                VkEvent _event,
3883                VkPipelineStageFlags stageMask)
3884 {
3885    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3886    TU_FROM_HANDLE(tu_event, event, _event);
3887
3888    write_event(cmd_buffer, event, stageMask, 1);
3889 }
3890
3891 void
3892 tu_CmdResetEvent(VkCommandBuffer commandBuffer,
3893                  VkEvent _event,
3894                  VkPipelineStageFlags stageMask)
3895 {
3896    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3897    TU_FROM_HANDLE(tu_event, event, _event);
3898
3899    write_event(cmd_buffer, event, stageMask, 0);
3900 }
3901
3902 void
3903 tu_CmdWaitEvents(VkCommandBuffer commandBuffer,
3904                  uint32_t eventCount,
3905                  const VkEvent *pEvents,
3906                  VkPipelineStageFlags srcStageMask,
3907                  VkPipelineStageFlags dstStageMask,
3908                  uint32_t memoryBarrierCount,
3909                  const VkMemoryBarrier *pMemoryBarriers,
3910                  uint32_t bufferMemoryBarrierCount,
3911                  const VkBufferMemoryBarrier *pBufferMemoryBarriers,
3912                  uint32_t imageMemoryBarrierCount,
3913                  const VkImageMemoryBarrier *pImageMemoryBarriers)
3914 {
3915    TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3916    struct tu_barrier_info info;
3917
3918    info.eventCount = eventCount;
3919    info.pEvents = pEvents;
3920    info.srcStageMask = 0;
3921
3922    tu_barrier(cmd_buffer, memoryBarrierCount, pMemoryBarriers,
3923               bufferMemoryBarrierCount, pBufferMemoryBarriers,
3924               imageMemoryBarrierCount, pImageMemoryBarriers, &info);
3925 }
3926
3927 void
3928 tu_CmdSetDeviceMask(VkCommandBuffer commandBuffer, uint32_t deviceMask)
3929 {
3930    /* No-op */
3931 }