projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
turnip: new clear/blit implementation with shader path fallback
[mesa.git] / src / freedreno / vulkan / tu_cmd_buffer.c
1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 *
5 * based in part on anv driver which is:
6 * Copyright © 2015 Intel Corporation
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the "Software"),
10 * to deal in the Software without restriction, including without limitation
11 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 * and/or sell copies of the Software, and to permit persons to whom the
13 * Software is furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the next
16 * paragraph) shall be included in all copies or substantial portions of the
17 * Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25 * DEALINGS IN THE SOFTWARE.
26 */
27
28 #include "tu_private.h"
29
30 #include "registers/adreno_pm4.xml.h"
31 #include "registers/adreno_common.xml.h"
32
33 #include "vk_format.h"
34
35 #include "tu_cs.h"
36
37 #define OVERFLOW_FLAG_REG REG_A6XX_CP_SCRATCH_REG(0)
38
39 void
40 tu_bo_list_init(struct tu_bo_list *list)
41 {
42 list->count = list->capacity = 0;
43 list->bo_infos = NULL;
44 }
45
46 void
47 tu_bo_list_destroy(struct tu_bo_list *list)
48 {
49 free(list->bo_infos);
50 }
51
52 void
53 tu_bo_list_reset(struct tu_bo_list *list)
54 {
55 list->count = 0;
56 }
57
58 /**
59 * \a flags consists of MSM_SUBMIT_BO_FLAGS.
60 */
61 static uint32_t
62 tu_bo_list_add_info(struct tu_bo_list *list,
63 const struct drm_msm_gem_submit_bo *bo_info)
64 {
65 assert(bo_info->handle != 0);
66
67 for (uint32_t i = 0; i < list->count; ++i) {
68 if (list->bo_infos[i].handle == bo_info->handle) {
69 assert(list->bo_infos[i].presumed == bo_info->presumed);
70 list->bo_infos[i].flags |= bo_info->flags;
71 return i;
72 }
73 }
74
75 /* grow list->bo_infos if needed */
76 if (list->count == list->capacity) {
77 uint32_t new_capacity = MAX2(2 * list->count, 16);
78 struct drm_msm_gem_submit_bo *new_bo_infos = realloc(
79 list->bo_infos, new_capacity * sizeof(struct drm_msm_gem_submit_bo));
80 if (!new_bo_infos)
81 return TU_BO_LIST_FAILED;
82 list->bo_infos = new_bo_infos;
83 list->capacity = new_capacity;
84 }
85
86 list->bo_infos[list->count] = *bo_info;
87 return list->count++;
88 }
89
90 uint32_t
91 tu_bo_list_add(struct tu_bo_list *list,
92 const struct tu_bo *bo,
93 uint32_t flags)
94 {
95 return tu_bo_list_add_info(list, &(struct drm_msm_gem_submit_bo) {
96 .flags = flags,
97 .handle = bo->gem_handle,
98 .presumed = bo->iova,
99 });
100 }
101
102 VkResult
103 tu_bo_list_merge(struct tu_bo_list *list, const struct tu_bo_list *other)
104 {
105 for (uint32_t i = 0; i < other->count; i++) {
106 if (tu_bo_list_add_info(list, other->bo_infos + i) == TU_BO_LIST_FAILED)
107 return VK_ERROR_OUT_OF_HOST_MEMORY;
108 }
109
110 return VK_SUCCESS;
111 }
112
113 static void
114 tu_tiling_config_update_tile_layout(struct tu_tiling_config *tiling,
115 const struct tu_device *dev,
116 uint32_t pixels)
117 {
118 const uint32_t tile_align_w = 64; /* note: 32 when no input attachments */
119 const uint32_t tile_align_h = 16;
120 const uint32_t max_tile_width = 1024;
121
122 /* note: don't offset the tiling config by render_area.offset,
123 * because binning pass can't deal with it
124 * this means we might end up with more tiles than necessary,
125 * but load/store/etc are still scissored to the render_area
126 */
127 tiling->tile0.offset = (VkOffset2D) {};
128
129 const uint32_t ra_width =
130 tiling->render_area.extent.width +
131 (tiling->render_area.offset.x - tiling->tile0.offset.x);
132 const uint32_t ra_height =
133 tiling->render_area.extent.height +
134 (tiling->render_area.offset.y - tiling->tile0.offset.y);
135
136 /* start from 1 tile */
137 tiling->tile_count = (VkExtent2D) {
138 .width = 1,
139 .height = 1,
140 };
141 tiling->tile0.extent = (VkExtent2D) {
142 .width = align(ra_width, tile_align_w),
143 .height = align(ra_height, tile_align_h),
144 };
145
146 if (unlikely(dev->physical_device->instance->debug_flags & TU_DEBUG_FORCEBIN)) {
147 /* start with 2x2 tiles */
148 tiling->tile_count.width = 2;
149 tiling->tile_count.height = 2;
150 tiling->tile0.extent.width = align(DIV_ROUND_UP(ra_width, 2), tile_align_w);
151 tiling->tile0.extent.height = align(DIV_ROUND_UP(ra_height, 2), tile_align_h);
152 }
153
154 /* do not exceed max tile width */
155 while (tiling->tile0.extent.width > max_tile_width) {
156 tiling->tile_count.width++;
157 tiling->tile0.extent.width =
158 align(DIV_ROUND_UP(ra_width, tiling->tile_count.width), tile_align_w);
159 }
160
161 /* will force to sysmem, don't bother trying to have a valid tile config
162 * TODO: just skip all GMEM stuff when sysmem is forced?
163 */
164 if (!pixels)
165 return;
166
167 /* do not exceed gmem size */
168 while (tiling->tile0.extent.width * tiling->tile0.extent.height > pixels) {
169 if (tiling->tile0.extent.width > MAX2(tile_align_w, tiling->tile0.extent.height)) {
170 tiling->tile_count.width++;
171 tiling->tile0.extent.width =
172 align(DIV_ROUND_UP(ra_width, tiling->tile_count.width), tile_align_w);
173 } else {
174 /* if this assert fails then layout is impossible.. */
175 assert(tiling->tile0.extent.height > tile_align_h);
176 tiling->tile_count.height++;
177 tiling->tile0.extent.height =
178 align(DIV_ROUND_UP(ra_height, tiling->tile_count.height), tile_align_h);
179 }
180 }
181 }
182
183 static void
184 tu_tiling_config_update_pipe_layout(struct tu_tiling_config *tiling,
185 const struct tu_device *dev)
186 {
187 const uint32_t max_pipe_count = 32; /* A6xx */
188
189 /* start from 1 tile per pipe */
190 tiling->pipe0 = (VkExtent2D) {
191 .width = 1,
192 .height = 1,
193 };
194 tiling->pipe_count = tiling->tile_count;
195
196 while (tiling->pipe_count.width * tiling->pipe_count.height > max_pipe_count) {
197 if (tiling->pipe0.width < tiling->pipe0.height) {
198 tiling->pipe0.width += 1;
199 tiling->pipe_count.width =
200 DIV_ROUND_UP(tiling->tile_count.width, tiling->pipe0.width);
201 } else {
202 tiling->pipe0.height += 1;
203 tiling->pipe_count.height =
204 DIV_ROUND_UP(tiling->tile_count.height, tiling->pipe0.height);
205 }
206 }
207 }
208
209 static void
210 tu_tiling_config_update_pipes(struct tu_tiling_config *tiling,
211 const struct tu_device *dev)
212 {
213 const uint32_t max_pipe_count = 32; /* A6xx */
214 const uint32_t used_pipe_count =
215 tiling->pipe_count.width * tiling->pipe_count.height;
216 const VkExtent2D last_pipe = {
217 .width = (tiling->tile_count.width - 1) % tiling->pipe0.width + 1,
218 .height = (tiling->tile_count.height - 1) % tiling->pipe0.height + 1,
219 };
220
221 assert(used_pipe_count <= max_pipe_count);
222 assert(max_pipe_count <= ARRAY_SIZE(tiling->pipe_config));
223
224 for (uint32_t y = 0; y < tiling->pipe_count.height; y++) {
225 for (uint32_t x = 0; x < tiling->pipe_count.width; x++) {
226 const uint32_t pipe_x = tiling->pipe0.width * x;
227 const uint32_t pipe_y = tiling->pipe0.height * y;
228 const uint32_t pipe_w = (x == tiling->pipe_count.width - 1)
229 ? last_pipe.width
230 : tiling->pipe0.width;
231 const uint32_t pipe_h = (y == tiling->pipe_count.height - 1)
232 ? last_pipe.height
233 : tiling->pipe0.height;
234 const uint32_t n = tiling->pipe_count.width * y + x;
235
236 tiling->pipe_config[n] = A6XX_VSC_PIPE_CONFIG_REG_X(pipe_x) |
237 A6XX_VSC_PIPE_CONFIG_REG_Y(pipe_y) |
238 A6XX_VSC_PIPE_CONFIG_REG_W(pipe_w) |
239 A6XX_VSC_PIPE_CONFIG_REG_H(pipe_h);
240 tiling->pipe_sizes[n] = CP_SET_BIN_DATA5_0_VSC_SIZE(pipe_w * pipe_h);
241 }
242 }
243
244 memset(tiling->pipe_config + used_pipe_count, 0,
245 sizeof(uint32_t) * (max_pipe_count - used_pipe_count));
246 }
247
248 static void
249 tu_tiling_config_get_tile(const struct tu_tiling_config *tiling,
250 const struct tu_device *dev,
251 uint32_t tx,
252 uint32_t ty,
253 struct tu_tile *tile)
254 {
255 /* find the pipe and the slot for tile (tx, ty) */
256 const uint32_t px = tx / tiling->pipe0.width;
257 const uint32_t py = ty / tiling->pipe0.height;
258 const uint32_t sx = tx - tiling->pipe0.width * px;
259 const uint32_t sy = ty - tiling->pipe0.height * py;
260 /* last pipe has different width */
261 const uint32_t pipe_width =
262 MIN2(tiling->pipe0.width,
263 tiling->tile_count.width - px * tiling->pipe0.width);
264
265 assert(tx < tiling->tile_count.width && ty < tiling->tile_count.height);
266 assert(px < tiling->pipe_count.width && py < tiling->pipe_count.height);
267 assert(sx < tiling->pipe0.width && sy < tiling->pipe0.height);
268
269 /* convert to 1D indices */
270 tile->pipe = tiling->pipe_count.width * py + px;
271 tile->slot = pipe_width * sy + sx;
272
273 /* get the blit area for the tile */
274 tile->begin = (VkOffset2D) {
275 .x = tiling->tile0.offset.x + tiling->tile0.extent.width * tx,
276 .y = tiling->tile0.offset.y + tiling->tile0.extent.height * ty,
277 };
278 tile->end.x =
279 (tx == tiling->tile_count.width - 1)
280 ? tiling->render_area.offset.x + tiling->render_area.extent.width
281 : tile->begin.x + tiling->tile0.extent.width;
282 tile->end.y =
283 (ty == tiling->tile_count.height - 1)
284 ? tiling->render_area.offset.y + tiling->render_area.extent.height
285 : tile->begin.y + tiling->tile0.extent.height;
286 }
287
288 enum a3xx_msaa_samples
289 tu_msaa_samples(uint32_t samples)
290 {
291 switch (samples) {
292 case 1:
293 return MSAA_ONE;
294 case 2:
295 return MSAA_TWO;
296 case 4:
297 return MSAA_FOUR;
298 case 8:
299 return MSAA_EIGHT;
300 default:
301 assert(!"invalid sample count");
302 return MSAA_ONE;
303 }
304 }
305
306 static enum a4xx_index_size
307 tu6_index_size(VkIndexType type)
308 {
309 switch (type) {
310 case VK_INDEX_TYPE_UINT16:
311 return INDEX4_SIZE_16_BIT;
312 case VK_INDEX_TYPE_UINT32:
313 return INDEX4_SIZE_32_BIT;
314 default:
315 unreachable("invalid VkIndexType");
316 return INDEX4_SIZE_8_BIT;
317 }
318 }
319
320 unsigned
321 tu6_emit_event_write(struct tu_cmd_buffer *cmd,
322 struct tu_cs *cs,
323 enum vgt_event_type event,
324 bool need_seqno)
325 {
326 unsigned seqno = 0;
327
328 tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, need_seqno ? 4 : 1);
329 tu_cs_emit(cs, CP_EVENT_WRITE_0_EVENT(event));
330 if (need_seqno) {
331 tu_cs_emit_qw(cs, cmd->scratch_bo.iova);
332 seqno = ++cmd->scratch_seqno;
333 tu_cs_emit(cs, seqno);
334 }
335
336 return seqno;
337 }
338
339 static void
340 tu6_emit_cache_flush(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
341 {
342 tu6_emit_event_write(cmd, cs, 0x31, false);
343 }
344
345 static void
346 tu6_emit_lrz_flush(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
347 {
348 tu6_emit_event_write(cmd, cs, LRZ_FLUSH, false);
349 }
350
351 static void
352 tu6_emit_wfi(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
353 {
354 if (cmd->wait_for_idle) {
355 tu_cs_emit_wfi(cs);
356 cmd->wait_for_idle = false;
357 }
358 }
359
360 static void
361 tu6_emit_zs(struct tu_cmd_buffer *cmd,
362 const struct tu_subpass *subpass,
363 struct tu_cs *cs)
364 {
365 const struct tu_framebuffer *fb = cmd->state.framebuffer;
366
367 const uint32_t a = subpass->depth_stencil_attachment.attachment;
368 if (a == VK_ATTACHMENT_UNUSED) {
369 tu_cs_emit_regs(cs,
370 A6XX_RB_DEPTH_BUFFER_INFO(.depth_format = DEPTH6_NONE),
371 A6XX_RB_DEPTH_BUFFER_PITCH(0),
372 A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH(0),
373 A6XX_RB_DEPTH_BUFFER_BASE(0),
374 A6XX_RB_DEPTH_BUFFER_BASE_GMEM(0));
375
376 tu_cs_emit_regs(cs,
377 A6XX_GRAS_SU_DEPTH_BUFFER_INFO(.depth_format = DEPTH6_NONE));
378
379 tu_cs_emit_regs(cs,
380 A6XX_GRAS_LRZ_BUFFER_BASE(0),
381 A6XX_GRAS_LRZ_BUFFER_PITCH(0),
382 A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE(0));
383
384 tu_cs_emit_regs(cs, A6XX_RB_STENCIL_INFO(0));
385
386 return;
387 }
388
389 const struct tu_image_view *iview = fb->attachments[a].attachment;
390 enum a6xx_depth_format fmt = tu6_pipe2depth(iview->vk_format);
391
392 tu_cs_emit_regs(cs,
393 A6XX_RB_DEPTH_BUFFER_INFO(.depth_format = fmt),
394 A6XX_RB_DEPTH_BUFFER_PITCH(tu_image_stride(iview->image, iview->base_mip)),
395 A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH(iview->image->layout.layer_size),
396 A6XX_RB_DEPTH_BUFFER_BASE(tu_image_view_base_ref(iview)),
397 A6XX_RB_DEPTH_BUFFER_BASE_GMEM(cmd->state.pass->attachments[a].gmem_offset));
398
399 tu_cs_emit_regs(cs,
400 A6XX_GRAS_SU_DEPTH_BUFFER_INFO(.depth_format = fmt));
401
402 tu_cs_emit_regs(cs,
403 A6XX_RB_DEPTH_FLAG_BUFFER_BASE(tu_image_view_ubwc_base_ref(iview)),
404 A6XX_RB_DEPTH_FLAG_BUFFER_PITCH(tu_image_view_ubwc_pitches(iview)));
405
406 tu_cs_emit_regs(cs,
407 A6XX_GRAS_LRZ_BUFFER_BASE(0),
408 A6XX_GRAS_LRZ_BUFFER_PITCH(0),
409 A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE(0));
410
411 tu_cs_emit_regs(cs,
412 A6XX_RB_STENCIL_INFO(0));
413
414 /* enable zs? */
415 }
416
417 static void
418 tu6_emit_mrt(struct tu_cmd_buffer *cmd,
419 const struct tu_subpass *subpass,
420 struct tu_cs *cs)
421 {
422 const struct tu_framebuffer *fb = cmd->state.framebuffer;
423 unsigned char mrt_comp[MAX_RTS] = { 0 };
424 unsigned srgb_cntl = 0;
425
426 for (uint32_t i = 0; i < subpass->color_count; ++i) {
427 uint32_t a = subpass->color_attachments[i].attachment;
428 if (a == VK_ATTACHMENT_UNUSED)
429 continue;
430
431 const struct tu_image_view *iview = fb->attachments[a].attachment;
432
433 mrt_comp[i] = 0xf;
434
435 if (vk_format_is_srgb(iview->vk_format))
436 srgb_cntl |= (1 << i);
437
438 struct tu_native_format format =
439 tu6_format_image(iview->image, iview->vk_format, iview->base_mip);
440
441 tu_cs_emit_regs(cs,
442 A6XX_RB_MRT_BUF_INFO(i,
443 .color_tile_mode = format.tile_mode,
444 .color_format = format.fmt,
445 .color_swap = format.swap),
446 A6XX_RB_MRT_PITCH(i, tu_image_stride(iview->image, iview->base_mip)),
447 A6XX_RB_MRT_ARRAY_PITCH(i, iview->image->layout.layer_size),
448 A6XX_RB_MRT_BASE(i, tu_image_view_base_ref(iview)),
449 A6XX_RB_MRT_BASE_GMEM(i, cmd->state.pass->attachments[a].gmem_offset));
450
451 tu_cs_emit_regs(cs,
452 A6XX_SP_FS_MRT_REG(i,
453 .color_format = format.fmt,
454 .color_sint = vk_format_is_sint(iview->vk_format),
455 .color_uint = vk_format_is_uint(iview->vk_format)));
456
457 tu_cs_emit_regs(cs,
458 A6XX_RB_MRT_FLAG_BUFFER_ADDR(i, tu_image_view_ubwc_base_ref(iview)),
459 A6XX_RB_MRT_FLAG_BUFFER_PITCH(i, tu_image_view_ubwc_pitches(iview)));
460 }
461
462 tu_cs_emit_regs(cs,
463 A6XX_RB_SRGB_CNTL(.dword = srgb_cntl));
464
465 tu_cs_emit_regs(cs,
466 A6XX_SP_SRGB_CNTL(.dword = srgb_cntl));
467
468 tu_cs_emit_regs(cs,
469 A6XX_RB_RENDER_COMPONENTS(
470 .rt0 = mrt_comp[0],
471 .rt1 = mrt_comp[1],
472 .rt2 = mrt_comp[2],
473 .rt3 = mrt_comp[3],
474 .rt4 = mrt_comp[4],
475 .rt5 = mrt_comp[5],
476 .rt6 = mrt_comp[6],
477 .rt7 = mrt_comp[7]));
478
479 tu_cs_emit_regs(cs,
480 A6XX_SP_FS_RENDER_COMPONENTS(
481 .rt0 = mrt_comp[0],
482 .rt1 = mrt_comp[1],
483 .rt2 = mrt_comp[2],
484 .rt3 = mrt_comp[3],
485 .rt4 = mrt_comp[4],
486 .rt5 = mrt_comp[5],
487 .rt6 = mrt_comp[6],
488 .rt7 = mrt_comp[7]));
489
490 // XXX: We probably can't hardcode LAYER_CNTL_TYPE.
491 tu_cs_emit_regs(cs,
492 A6XX_GRAS_LAYER_CNTL(.layered = fb->layers > 1,
493 .type = LAYER_2D_ARRAY));
494 }
495
496 void
497 tu6_emit_msaa(struct tu_cs *cs, VkSampleCountFlagBits vk_samples)
498 {
499 const enum a3xx_msaa_samples samples = tu_msaa_samples(vk_samples);
500 bool msaa_disable = samples == MSAA_ONE;
501
502 tu_cs_emit_regs(cs,
503 A6XX_SP_TP_RAS_MSAA_CNTL(samples),
504 A6XX_SP_TP_DEST_MSAA_CNTL(.samples = samples,
505 .msaa_disable = msaa_disable));
506
507 tu_cs_emit_regs(cs,
508 A6XX_GRAS_RAS_MSAA_CNTL(samples),
509 A6XX_GRAS_DEST_MSAA_CNTL(.samples = samples,
510 .msaa_disable = msaa_disable));
511
512 tu_cs_emit_regs(cs,
513 A6XX_RB_RAS_MSAA_CNTL(samples),
514 A6XX_RB_DEST_MSAA_CNTL(.samples = samples,
515 .msaa_disable = msaa_disable));
516
517 tu_cs_emit_regs(cs,
518 A6XX_RB_MSAA_CNTL(samples));
519 }
520
521 static void
522 tu6_emit_bin_size(struct tu_cs *cs,
523 uint32_t bin_w, uint32_t bin_h, uint32_t flags)
524 {
525 tu_cs_emit_regs(cs,
526 A6XX_GRAS_BIN_CONTROL(.binw = bin_w,
527 .binh = bin_h,
528 .dword = flags));
529
530 tu_cs_emit_regs(cs,
531 A6XX_RB_BIN_CONTROL(.binw = bin_w,
532 .binh = bin_h,
533 .dword = flags));
534
535 /* no flag for RB_BIN_CONTROL2... */
536 tu_cs_emit_regs(cs,
537 A6XX_RB_BIN_CONTROL2(.binw = bin_w,
538 .binh = bin_h));
539 }
540
541 static void
542 tu6_emit_render_cntl(struct tu_cmd_buffer *cmd,
543 const struct tu_subpass *subpass,
544 struct tu_cs *cs,
545 bool binning)
546 {
547 const struct tu_framebuffer *fb = cmd->state.framebuffer;
548 uint32_t cntl = 0;
549 cntl |= A6XX_RB_RENDER_CNTL_UNK4;
550 if (binning) {
551 cntl |= A6XX_RB_RENDER_CNTL_BINNING;
552 } else {
553 uint32_t mrts_ubwc_enable = 0;
554 for (uint32_t i = 0; i < subpass->color_count; ++i) {
555 uint32_t a = subpass->color_attachments[i].attachment;
556 if (a == VK_ATTACHMENT_UNUSED)
557 continue;
558
559 const struct tu_image_view *iview = fb->attachments[a].attachment;
560 if (iview->image->layout.ubwc_layer_size != 0)
561 mrts_ubwc_enable |= 1 << i;
562 }
563
564 cntl |= A6XX_RB_RENDER_CNTL_FLAG_MRTS(mrts_ubwc_enable);
565
566 const uint32_t a = subpass->depth_stencil_attachment.attachment;
567 if (a != VK_ATTACHMENT_UNUSED) {
568 const struct tu_image_view *iview = fb->attachments[a].attachment;
569 if (iview->image->layout.ubwc_layer_size != 0)
570 cntl |= A6XX_RB_RENDER_CNTL_FLAG_DEPTH;
571 }
572
573 /* In the !binning case, we need to set RB_RENDER_CNTL in the draw_cs
574 * in order to set it correctly for the different subpasses. However,
575 * that means the packets we're emitting also happen during binning. So
576 * we need to guard the write on !BINNING at CP execution time.
577 */
578 tu_cs_reserve(cs, 3 + 4);
579 tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
580 tu_cs_emit(cs, CP_COND_REG_EXEC_0_MODE(RENDER_MODE) |
581 CP_COND_REG_EXEC_0_GMEM | CP_COND_REG_EXEC_0_SYSMEM);
582 tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(4));
583 }
584
585 tu_cs_emit_pkt7(cs, CP_REG_WRITE, 3);
586 tu_cs_emit(cs, CP_REG_WRITE_0_TRACKER(TRACK_RENDER_CNTL));
587 tu_cs_emit(cs, REG_A6XX_RB_RENDER_CNTL);
588 tu_cs_emit(cs, cntl);
589 }
590
591 static void
592 tu6_emit_blit_scissor(struct tu_cmd_buffer *cmd, struct tu_cs *cs, bool align)
593 {
594 const VkRect2D *render_area = &cmd->state.tiling_config.render_area;
595 uint32_t x1 = render_area->offset.x;
596 uint32_t y1 = render_area->offset.y;
597 uint32_t x2 = x1 + render_area->extent.width - 1;
598 uint32_t y2 = y1 + render_area->extent.height - 1;
599
600 if (align) {
601 x1 = x1 & ~(GMEM_ALIGN_W - 1);
602 y1 = y1 & ~(GMEM_ALIGN_H - 1);
603 x2 = ALIGN_POT(x2 + 1, GMEM_ALIGN_W) - 1;
604 y2 = ALIGN_POT(y2 + 1, GMEM_ALIGN_H) - 1;
605 }
606
607 tu_cs_emit_regs(cs,
608 A6XX_RB_BLIT_SCISSOR_TL(.x = x1, .y = y1),
609 A6XX_RB_BLIT_SCISSOR_BR(.x = x2, .y = y2));
610 }
611
612 void
613 tu6_emit_window_scissor(struct tu_cs *cs,
614 uint32_t x1,
615 uint32_t y1,
616 uint32_t x2,
617 uint32_t y2)
618 {
619 tu_cs_emit_regs(cs,
620 A6XX_GRAS_SC_WINDOW_SCISSOR_TL(.x = x1, .y = y1),
621 A6XX_GRAS_SC_WINDOW_SCISSOR_BR(.x = x2, .y = y2));
622
623 tu_cs_emit_regs(cs,
624 A6XX_GRAS_RESOLVE_CNTL_1(.x = x1, .y = y1),
625 A6XX_GRAS_RESOLVE_CNTL_2(.x = x2, .y = y2));
626 }
627
628 void
629 tu6_emit_window_offset(struct tu_cs *cs, uint32_t x1, uint32_t y1)
630 {
631 tu_cs_emit_regs(cs,
632 A6XX_RB_WINDOW_OFFSET(.x = x1, .y = y1));
633
634 tu_cs_emit_regs(cs,
635 A6XX_RB_WINDOW_OFFSET2(.x = x1, .y = y1));
636
637 tu_cs_emit_regs(cs,
638 A6XX_SP_WINDOW_OFFSET(.x = x1, .y = y1));
639
640 tu_cs_emit_regs(cs,
641 A6XX_SP_TP_WINDOW_OFFSET(.x = x1, .y = y1));
642 }
643
644 static bool
645 use_hw_binning(struct tu_cmd_buffer *cmd)
646 {
647 const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
648
649 if (unlikely(cmd->device->physical_device->instance->debug_flags & TU_DEBUG_NOBIN))
650 return false;
651
652 if (unlikely(cmd->device->physical_device->instance->debug_flags & TU_DEBUG_FORCEBIN))
653 return true;
654
655 return (tiling->tile_count.width * tiling->tile_count.height) > 2;
656 }
657
658 static bool
659 use_sysmem_rendering(struct tu_cmd_buffer *cmd)
660 {
661 if (unlikely(cmd->device->physical_device->instance->debug_flags & TU_DEBUG_SYSMEM))
662 return true;
663
664 /* can't fit attachments into gmem */
665 if (!cmd->state.pass->gmem_pixels)
666 return true;
667
668 if (cmd->state.framebuffer->layers > 1)
669 return true;
670
671 return cmd->state.tiling_config.force_sysmem;
672 }
673
674 static void
675 tu6_emit_tile_select(struct tu_cmd_buffer *cmd,
676 struct tu_cs *cs,
677 const struct tu_tile *tile)
678 {
679 tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
680 tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_YIELD));
681
682 tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
683 tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_GMEM));
684
685 const uint32_t x1 = tile->begin.x;
686 const uint32_t y1 = tile->begin.y;
687 const uint32_t x2 = tile->end.x - 1;
688 const uint32_t y2 = tile->end.y - 1;
689 tu6_emit_window_scissor(cs, x1, y1, x2, y2);
690 tu6_emit_window_offset(cs, x1, y1);
691
692 tu_cs_emit_regs(cs,
693 A6XX_VPC_SO_OVERRIDE(.so_disable = false));
694
695 if (use_hw_binning(cmd)) {
696 tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
697
698 tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
699 tu_cs_emit(cs, 0x0);
700
701 tu_cs_emit_pkt7(cs, CP_REG_TEST, 1);
702 tu_cs_emit(cs, A6XX_CP_REG_TEST_0_REG(OVERFLOW_FLAG_REG) |
703 A6XX_CP_REG_TEST_0_BIT(0) |
704 A6XX_CP_REG_TEST_0_WAIT_FOR_ME);
705
706 tu_cs_reserve(cs, 3 + 11);
707 tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
708 tu_cs_emit(cs, CP_COND_REG_EXEC_0_MODE(PRED_TEST));
709 tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(11));
710
711 /* if (no overflow) */ {
712 tu_cs_emit_pkt7(cs, CP_SET_BIN_DATA5, 7);
713 tu_cs_emit(cs, cmd->state.tiling_config.pipe_sizes[tile->pipe] |
714 CP_SET_BIN_DATA5_0_VSC_N(tile->slot));
715 tu_cs_emit_qw(cs, cmd->vsc_data.iova + tile->pipe * cmd->vsc_data_pitch);
716 tu_cs_emit_qw(cs, cmd->vsc_data.iova + (tile->pipe * 4) + (32 * cmd->vsc_data_pitch));
717 tu_cs_emit_qw(cs, cmd->vsc_data2.iova + (tile->pipe * cmd->vsc_data2_pitch));
718
719 tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
720 tu_cs_emit(cs, 0x0);
721
722 /* use a NOP packet to skip over the 'else' side: */
723 tu_cs_emit_pkt7(cs, CP_NOP, 2);
724 } /* else */ {
725 tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
726 tu_cs_emit(cs, 0x1);
727 }
728
729 tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
730 tu_cs_emit(cs, 0x0);
731
732 tu_cs_emit_regs(cs,
733 A6XX_RB_UNKNOWN_8804(0));
734
735 tu_cs_emit_regs(cs,
736 A6XX_SP_TP_UNKNOWN_B304(0));
737
738 tu_cs_emit_regs(cs,
739 A6XX_GRAS_UNKNOWN_80A4(0));
740 } else {
741 tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
742 tu_cs_emit(cs, 0x1);
743
744 tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
745 tu_cs_emit(cs, 0x0);
746 }
747 }
748
749 static void
750 tu6_emit_sysmem_resolve(struct tu_cmd_buffer *cmd,
751 struct tu_cs *cs,
752 uint32_t a,
753 uint32_t gmem_a)
754 {
755 const struct tu_framebuffer *fb = cmd->state.framebuffer;
756 struct tu_image_view *dst = fb->attachments[a].attachment;
757 struct tu_image_view *src = fb->attachments[gmem_a].attachment;
758
759 tu_resolve_sysmem(cmd, cs, src, dst, fb->layers, &cmd->state.tiling_config.render_area);
760 }
761
762 static void
763 tu6_emit_tile_store(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
764 {
765 const struct tu_render_pass *pass = cmd->state.pass;
766 const struct tu_subpass *subpass = &pass->subpasses[pass->subpass_count-1];
767
768 tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
769 tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
770 CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
771 CP_SET_DRAW_STATE__0_GROUP_ID(0));
772 tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
773 tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
774
775 tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
776 tu_cs_emit(cs, 0x0);
777
778 tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
779 tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_RESOLVE));
780
781 /* blit scissor may have been changed by CmdClearAttachments */
782 tu6_emit_blit_scissor(cmd, cs, false);
783
784 for (uint32_t a = 0; a < pass->attachment_count; ++a) {
785 if (pass->attachments[a].gmem_offset >= 0)
786 tu_store_gmem_attachment(cmd, cs, a, a);
787 }
788
789 if (subpass->resolve_attachments) {
790 for (unsigned i = 0; i < subpass->color_count; i++) {
791 uint32_t a = subpass->resolve_attachments[i].attachment;
792 if (a != VK_ATTACHMENT_UNUSED)
793 tu_store_gmem_attachment(cmd, cs, a,
794 subpass->color_attachments[i].attachment);
795 }
796 }
797 }
798
799 static void
800 tu6_emit_restart_index(struct tu_cs *cs, uint32_t restart_index)
801 {
802 tu_cs_emit_regs(cs,
803 A6XX_PC_RESTART_INDEX(restart_index));
804 }
805
806 static void
807 tu6_init_hw(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
808 {
809 const struct tu_physical_device *phys_dev = cmd->device->physical_device;
810
811 tu6_emit_cache_flush(cmd, cs);
812
813 tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UPDATE_CNTL, 0xfffff);
814
815 tu_cs_emit_regs(cs,
816 A6XX_RB_CCU_CNTL(.offset = phys_dev->ccu_offset_bypass));
817 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8E04, 0x00100000);
818 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE04, 0x8);
819 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE00, 0);
820 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE0F, 0x3f);
821 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B605, 0x44);
822 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B600, 0x100000);
823 tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE00, 0x80);
824 tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE01, 0);
825
826 tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9600, 0);
827 tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8600, 0x880);
828 tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE04, 0);
829 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE03, 0x00000410);
830 tu_cs_emit_write_reg(cs, REG_A6XX_SP_IBO_COUNT, 0);
831 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B182, 0);
832 tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BB11, 0);
833 tu_cs_emit_write_reg(cs, REG_A6XX_UCHE_UNKNOWN_0E12, 0x3200000);
834 tu_cs_emit_write_reg(cs, REG_A6XX_UCHE_CLIENT_PF, 4);
835 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8E01, 0x0);
836 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_A982, 0);
837 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_A9A8, 0);
838 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AB00, 0x5);
839 tu_cs_emit_write_reg(cs, REG_A6XX_VPC_GS_SIV_CNTL, 0x0000ffff);
840
841 tu_cs_emit_write_reg(cs, REG_A6XX_VFD_ADD_OFFSET, A6XX_VFD_ADD_OFFSET_VERTEX);
842 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8811, 0x00000010);
843 tu_cs_emit_write_reg(cs, REG_A6XX_PC_MODE_CNTL, 0x1f);
844
845 tu_cs_emit_write_reg(cs, REG_A6XX_RB_SRGB_CNTL, 0);
846
847 tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8110, 0);
848
849 tu_cs_emit_write_reg(cs, REG_A6XX_RB_RENDER_CONTROL0, 0x401);
850 tu_cs_emit_write_reg(cs, REG_A6XX_RB_RENDER_CONTROL1, 0);
851 tu_cs_emit_write_reg(cs, REG_A6XX_RB_FS_OUTPUT_CNTL0, 0);
852 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8818, 0);
853 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8819, 0);
854 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881A, 0);
855 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881B, 0);
856 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881C, 0);
857 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881D, 0);
858 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881E, 0);
859 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_88F0, 0);
860
861 tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9101, 0xffff00);
862 tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9107, 0);
863
864 tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9236, 1);
865 tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9300, 0);
866
867 tu_cs_emit_write_reg(cs, REG_A6XX_VPC_SO_OVERRIDE,
868 A6XX_VPC_SO_OVERRIDE_SO_DISABLE);
869
870 tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9801, 0);
871 tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9806, 0);
872 tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9980, 0);
873 tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9990, 0);
874
875 tu_cs_emit_write_reg(cs, REG_A6XX_PC_PRIMITIVE_CNTL_6, 0);
876 tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9B07, 0);
877
878 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_A81B, 0);
879
880 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B183, 0);
881
882 tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8099, 0);
883 tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_809B, 0);
884 tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A0, 2);
885 tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80AF, 0);
886 tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9210, 0);
887 tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9211, 0);
888 tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9602, 0);
889 tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9981, 0x3);
890 tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9E72, 0);
891 tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9108, 0x3);
892 tu_cs_emit_write_reg(cs, REG_A6XX_SP_TP_UNKNOWN_B304, 0);
893 tu_cs_emit_write_reg(cs, REG_A6XX_SP_TP_UNKNOWN_B309, 0x000000a2);
894 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8804, 0);
895 tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A4, 0);
896 tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A5, 0);
897 tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A6, 0);
898 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8805, 0);
899 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8806, 0);
900 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8878, 0);
901 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8879, 0);
902 tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_CONTROL_5_REG, 0xfc);
903
904 tu_cs_emit_write_reg(cs, REG_A6XX_VFD_MODE_CNTL, 0x00000000);
905
906 tu_cs_emit_write_reg(cs, REG_A6XX_VFD_UNKNOWN_A008, 0);
907
908 tu_cs_emit_write_reg(cs, REG_A6XX_PC_MODE_CNTL, 0x0000001f);
909
910 /* we don't use this yet.. probably best to disable.. */
911 tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
912 tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
913 CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
914 CP_SET_DRAW_STATE__0_GROUP_ID(0));
915 tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
916 tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
917
918 /* Set not to use streamout by default, */
919 tu_cs_emit_pkt7(cs, CP_CONTEXT_REG_BUNCH, 4);
920 tu_cs_emit(cs, REG_A6XX_VPC_SO_CNTL);
921 tu_cs_emit(cs, 0);
922 tu_cs_emit(cs, REG_A6XX_VPC_SO_BUF_CNTL);
923 tu_cs_emit(cs, 0);
924
925 tu_cs_emit_regs(cs,
926 A6XX_SP_HS_CTRL_REG0(0));
927
928 tu_cs_emit_regs(cs,
929 A6XX_SP_GS_CTRL_REG0(0));
930
931 tu_cs_emit_regs(cs,
932 A6XX_GRAS_LRZ_CNTL(0));
933
934 tu_cs_emit_regs(cs,
935 A6XX_RB_LRZ_CNTL(0));
936
937 tu_cs_emit_regs(cs,
938 A6XX_SP_TP_BORDER_COLOR_BASE_ADDR(.bo = &cmd->device->border_color));
939 tu_cs_emit_regs(cs,
940 A6XX_SP_PS_TP_BORDER_COLOR_BASE_ADDR(.bo = &cmd->device->border_color));
941
942 tu_cs_sanity_check(cs);
943 }
944
945 static void
946 tu6_cache_flush(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
947 {
948 unsigned seqno;
949
950 seqno = tu6_emit_event_write(cmd, cs, RB_DONE_TS, true);
951
952 tu_cs_emit_pkt7(cs, CP_WAIT_REG_MEM, 6);
953 tu_cs_emit(cs, CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ) |
954 CP_WAIT_REG_MEM_0_POLL_MEMORY);
955 tu_cs_emit_qw(cs, cmd->scratch_bo.iova);
956 tu_cs_emit(cs, CP_WAIT_REG_MEM_3_REF(seqno));
957 tu_cs_emit(cs, CP_WAIT_REG_MEM_4_MASK(~0));
958 tu_cs_emit(cs, CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(16));
959
960 seqno = tu6_emit_event_write(cmd, cs, CACHE_FLUSH_TS, true);
961
962 tu_cs_emit_pkt7(cs, CP_WAIT_MEM_GTE, 4);
963 tu_cs_emit(cs, CP_WAIT_MEM_GTE_0_RESERVED(0));
964 tu_cs_emit_qw(cs, cmd->scratch_bo.iova);
965 tu_cs_emit(cs, CP_WAIT_MEM_GTE_3_REF(seqno));
966 }
967
968 static void
969 update_vsc_pipe(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
970 {
971 const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
972
973 tu_cs_emit_regs(cs,
974 A6XX_VSC_BIN_SIZE(.width = tiling->tile0.extent.width,
975 .height = tiling->tile0.extent.height),
976 A6XX_VSC_SIZE_ADDRESS(.bo = &cmd->vsc_data,
977 .bo_offset = 32 * cmd->vsc_data_pitch));
978
979 tu_cs_emit_regs(cs,
980 A6XX_VSC_BIN_COUNT(.nx = tiling->tile_count.width,
981 .ny = tiling->tile_count.height));
982
983 tu_cs_emit_pkt4(cs, REG_A6XX_VSC_PIPE_CONFIG_REG(0), 32);
984 for (unsigned i = 0; i < 32; i++)
985 tu_cs_emit(cs, tiling->pipe_config[i]);
986
987 tu_cs_emit_regs(cs,
988 A6XX_VSC_PIPE_DATA2_ADDRESS(.bo = &cmd->vsc_data2),
989 A6XX_VSC_PIPE_DATA2_PITCH(cmd->vsc_data2_pitch),
990 A6XX_VSC_PIPE_DATA2_ARRAY_PITCH(cmd->vsc_data2.size));
991
992 tu_cs_emit_regs(cs,
993 A6XX_VSC_PIPE_DATA_ADDRESS(.bo = &cmd->vsc_data),
994 A6XX_VSC_PIPE_DATA_PITCH(cmd->vsc_data_pitch),
995 A6XX_VSC_PIPE_DATA_ARRAY_PITCH(cmd->vsc_data.size));
996 }
997
998 static void
999 emit_vsc_overflow_test(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1000 {
1001 const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1002 const uint32_t used_pipe_count =
1003 tiling->pipe_count.width * tiling->pipe_count.height;
1004
1005 /* Clear vsc_scratch: */
1006 tu_cs_emit_pkt7(cs, CP_MEM_WRITE, 3);
1007 tu_cs_emit_qw(cs, cmd->scratch_bo.iova + ctrl_offset(vsc_scratch));
1008 tu_cs_emit(cs, 0x0);
1009
1010 /* Check for overflow, write vsc_scratch if detected: */
1011 for (int i = 0; i < used_pipe_count; i++) {
1012 tu_cs_emit_pkt7(cs, CP_COND_WRITE5, 8);
1013 tu_cs_emit(cs, CP_COND_WRITE5_0_FUNCTION(WRITE_GE) |
1014 CP_COND_WRITE5_0_WRITE_MEMORY);
1015 tu_cs_emit(cs, CP_COND_WRITE5_1_POLL_ADDR_LO(REG_A6XX_VSC_SIZE_REG(i)));
1016 tu_cs_emit(cs, CP_COND_WRITE5_2_POLL_ADDR_HI(0));
1017 tu_cs_emit(cs, CP_COND_WRITE5_3_REF(cmd->vsc_data_pitch));
1018 tu_cs_emit(cs, CP_COND_WRITE5_4_MASK(~0));
1019 tu_cs_emit_qw(cs, cmd->scratch_bo.iova + ctrl_offset(vsc_scratch));
1020 tu_cs_emit(cs, CP_COND_WRITE5_7_WRITE_DATA(1 + cmd->vsc_data_pitch));
1021
1022 tu_cs_emit_pkt7(cs, CP_COND_WRITE5, 8);
1023 tu_cs_emit(cs, CP_COND_WRITE5_0_FUNCTION(WRITE_GE) |
1024 CP_COND_WRITE5_0_WRITE_MEMORY);
1025 tu_cs_emit(cs, CP_COND_WRITE5_1_POLL_ADDR_LO(REG_A6XX_VSC_SIZE2_REG(i)));
1026 tu_cs_emit(cs, CP_COND_WRITE5_2_POLL_ADDR_HI(0));
1027 tu_cs_emit(cs, CP_COND_WRITE5_3_REF(cmd->vsc_data2_pitch));
1028 tu_cs_emit(cs, CP_COND_WRITE5_4_MASK(~0));
1029 tu_cs_emit_qw(cs, cmd->scratch_bo.iova + ctrl_offset(vsc_scratch));
1030 tu_cs_emit(cs, CP_COND_WRITE5_7_WRITE_DATA(3 + cmd->vsc_data2_pitch));
1031 }
1032
1033 tu_cs_emit_pkt7(cs, CP_WAIT_MEM_WRITES, 0);
1034
1035 tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
1036
1037 tu_cs_emit_pkt7(cs, CP_MEM_TO_REG, 3);
1038 tu_cs_emit(cs, CP_MEM_TO_REG_0_REG(OVERFLOW_FLAG_REG) |
1039 CP_MEM_TO_REG_0_CNT(1 - 1));
1040 tu_cs_emit_qw(cs, cmd->scratch_bo.iova + ctrl_offset(vsc_scratch));
1041
1042 /*
1043 * This is a bit awkward, we really want a way to invert the
1044 * CP_REG_TEST/CP_COND_REG_EXEC logic, so that we can conditionally
1045 * execute cmds to use hwbinning when a bit is *not* set. This
1046 * dance is to invert OVERFLOW_FLAG_REG
1047 *
1048 * A CP_NOP packet is used to skip executing the 'else' clause
1049 * if (b0 set)..
1050 */
1051
1052 /* b0 will be set if VSC_DATA or VSC_DATA2 overflow: */
1053 tu_cs_emit_pkt7(cs, CP_REG_TEST, 1);
1054 tu_cs_emit(cs, A6XX_CP_REG_TEST_0_REG(OVERFLOW_FLAG_REG) |
1055 A6XX_CP_REG_TEST_0_BIT(0) |
1056 A6XX_CP_REG_TEST_0_WAIT_FOR_ME);
1057
1058 tu_cs_reserve(cs, 3 + 7);
1059 tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
1060 tu_cs_emit(cs, CP_COND_REG_EXEC_0_MODE(PRED_TEST));
1061 tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(7));
1062
1063 /* if (b0 set) */ {
1064 /*
1065 * On overflow, mirror the value to control->vsc_overflow
1066 * which CPU is checking to detect overflow (see
1067 * check_vsc_overflow())
1068 */
1069 tu_cs_emit_pkt7(cs, CP_REG_TO_MEM, 3);
1070 tu_cs_emit(cs, CP_REG_TO_MEM_0_REG(OVERFLOW_FLAG_REG) |
1071 CP_REG_TO_MEM_0_CNT(0));
1072 tu_cs_emit_qw(cs, cmd->scratch_bo.iova + ctrl_offset(vsc_overflow));
1073
1074 tu_cs_emit_pkt4(cs, OVERFLOW_FLAG_REG, 1);
1075 tu_cs_emit(cs, 0x0);
1076
1077 tu_cs_emit_pkt7(cs, CP_NOP, 2); /* skip 'else' when 'if' is taken */
1078 } /* else */ {
1079 tu_cs_emit_pkt4(cs, OVERFLOW_FLAG_REG, 1);
1080 tu_cs_emit(cs, 0x1);
1081 }
1082 }
1083
1084 static void
1085 tu6_emit_binning_pass(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1086 {
1087 struct tu_physical_device *phys_dev = cmd->device->physical_device;
1088 const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1089
1090 uint32_t x1 = tiling->tile0.offset.x;
1091 uint32_t y1 = tiling->tile0.offset.y;
1092 uint32_t x2 = tiling->render_area.offset.x + tiling->render_area.extent.width - 1;
1093 uint32_t y2 = tiling->render_area.offset.y + tiling->render_area.extent.height - 1;
1094
1095 tu6_emit_window_scissor(cs, x1, y1, x2, y2);
1096
1097 tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1098 tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_BINNING));
1099
1100 tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
1101 tu_cs_emit(cs, 0x1);
1102
1103 tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
1104 tu_cs_emit(cs, 0x1);
1105
1106 tu_cs_emit_wfi(cs);
1107
1108 tu_cs_emit_regs(cs,
1109 A6XX_VFD_MODE_CNTL(.binning_pass = true));
1110
1111 update_vsc_pipe(cmd, cs);
1112
1113 tu_cs_emit_regs(cs,
1114 A6XX_PC_UNKNOWN_9805(.unknown = phys_dev->magic.PC_UNKNOWN_9805));
1115
1116 tu_cs_emit_regs(cs,
1117 A6XX_SP_UNKNOWN_A0F8(.unknown = phys_dev->magic.SP_UNKNOWN_A0F8));
1118
1119 tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, 1);
1120 tu_cs_emit(cs, UNK_2C);
1121
1122 tu_cs_emit_regs(cs,
1123 A6XX_RB_WINDOW_OFFSET(.x = 0, .y = 0));
1124
1125 tu_cs_emit_regs(cs,
1126 A6XX_SP_TP_WINDOW_OFFSET(.x = 0, .y = 0));
1127
1128 /* emit IB to binning drawcmds: */
1129 tu_cs_emit_call(cs, &cmd->draw_cs);
1130
1131 tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
1132 tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
1133 CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
1134 CP_SET_DRAW_STATE__0_GROUP_ID(0));
1135 tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
1136 tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
1137
1138 tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, 1);
1139 tu_cs_emit(cs, UNK_2D);
1140
1141 tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE, false);
1142 tu6_cache_flush(cmd, cs);
1143
1144 tu_cs_emit_wfi(cs);
1145
1146 tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
1147
1148 emit_vsc_overflow_test(cmd, cs);
1149
1150 tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
1151 tu_cs_emit(cs, 0x0);
1152
1153 tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
1154 tu_cs_emit(cs, 0x0);
1155
1156 cmd->wait_for_idle = false;
1157 }
1158
1159 static void
1160 tu_emit_load_clear(struct tu_cmd_buffer *cmd,
1161 const VkRenderPassBeginInfo *info)
1162 {
1163 struct tu_cs *cs = &cmd->draw_cs;
1164
1165 tu_cond_exec_start(cs, CP_COND_EXEC_0_RENDER_MODE_GMEM);
1166
1167 tu6_emit_blit_scissor(cmd, cs, true);
1168
1169 for (uint32_t i = 0; i < cmd->state.pass->attachment_count; ++i)
1170 tu_load_gmem_attachment(cmd, cs, i);
1171
1172 tu6_emit_blit_scissor(cmd, cs, false);
1173
1174 for (uint32_t i = 0; i < cmd->state.pass->attachment_count; ++i)
1175 tu_clear_gmem_attachment(cmd, cs, i, info);
1176
1177 tu_cond_exec_end(cs);
1178
1179 tu_cond_exec_start(cs, CP_COND_EXEC_0_RENDER_MODE_SYSMEM);
1180
1181 for (uint32_t i = 0; i < cmd->state.pass->attachment_count; ++i)
1182 tu_clear_sysmem_attachment(cmd, cs, i, info);
1183
1184 tu_cond_exec_end(cs);
1185 }
1186
1187 static void
1188 tu6_sysmem_render_begin(struct tu_cmd_buffer *cmd, struct tu_cs *cs,
1189 const struct VkRect2D *renderArea)
1190 {
1191 const struct tu_physical_device *phys_dev = cmd->device->physical_device;
1192 const struct tu_framebuffer *fb = cmd->state.framebuffer;
1193
1194 assert(fb->width > 0 && fb->height > 0);
1195 tu6_emit_window_scissor(cs, 0, 0, fb->width - 1, fb->height - 1);
1196 tu6_emit_window_offset(cs, 0, 0);
1197
1198 tu6_emit_bin_size(cs, 0, 0, 0xc00000); /* 0xc00000 = BYPASS? */
1199
1200 tu6_emit_lrz_flush(cmd, cs);
1201
1202 tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1203 tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_BYPASS));
1204
1205 tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1206 tu_cs_emit(cs, 0x0);
1207
1208 tu6_emit_event_write(cmd, cs, PC_CCU_INVALIDATE_COLOR, false);
1209 tu6_emit_event_write(cmd, cs, PC_CCU_INVALIDATE_DEPTH, false);
1210 tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE, false);
1211
1212 tu6_emit_wfi(cmd, cs);
1213 tu_cs_emit_regs(cs,
1214 A6XX_RB_CCU_CNTL(.offset = phys_dev->ccu_offset_bypass));
1215
1216 /* enable stream-out, with sysmem there is only one pass: */
1217 tu_cs_emit_regs(cs,
1218 A6XX_VPC_SO_OVERRIDE(.so_disable = false));
1219
1220 tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
1221 tu_cs_emit(cs, 0x1);
1222
1223 tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
1224 tu_cs_emit(cs, 0x0);
1225
1226 tu_cs_sanity_check(cs);
1227 }
1228
1229 static void
1230 tu6_sysmem_render_end(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1231 {
1232 /* Do any resolves of the last subpass. These are handled in the
1233 * tile_store_ib in the gmem path.
1234 */
1235 const struct tu_subpass *subpass = cmd->state.subpass;
1236 if (subpass->resolve_attachments) {
1237 for (unsigned i = 0; i < subpass->color_count; i++) {
1238 uint32_t a = subpass->resolve_attachments[i].attachment;
1239 if (a != VK_ATTACHMENT_UNUSED)
1240 tu6_emit_sysmem_resolve(cmd, cs, a,
1241 subpass->color_attachments[i].attachment);
1242 }
1243 }
1244
1245 tu_cs_emit_call(cs, &cmd->draw_epilogue_cs);
1246
1247 tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1248 tu_cs_emit(cs, 0x0);
1249
1250 tu6_emit_lrz_flush(cmd, cs);
1251
1252 tu6_emit_event_write(cmd, cs, PC_CCU_FLUSH_COLOR_TS, true);
1253 tu6_emit_event_write(cmd, cs, PC_CCU_FLUSH_DEPTH_TS, true);
1254
1255 tu_cs_sanity_check(cs);
1256 }
1257
1258
1259 static void
1260 tu6_tile_render_begin(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1261 {
1262 struct tu_physical_device *phys_dev = cmd->device->physical_device;
1263
1264 tu6_emit_lrz_flush(cmd, cs);
1265
1266 /* lrz clear? */
1267
1268 tu6_emit_cache_flush(cmd, cs);
1269
1270 tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1271 tu_cs_emit(cs, 0x0);
1272
1273 /* TODO: flushing with barriers instead of blindly always flushing */
1274 tu6_emit_event_write(cmd, cs, PC_CCU_FLUSH_COLOR_TS, true);
1275 tu6_emit_event_write(cmd, cs, PC_CCU_FLUSH_DEPTH_TS, true);
1276 tu6_emit_event_write(cmd, cs, PC_CCU_INVALIDATE_COLOR, false);
1277 tu6_emit_event_write(cmd, cs, PC_CCU_INVALIDATE_DEPTH, false);
1278
1279 tu_cs_emit_wfi(cs);
1280 tu_cs_emit_regs(cs,
1281 A6XX_RB_CCU_CNTL(.offset = phys_dev->ccu_offset_gmem, .gmem = 1));
1282
1283 const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1284 if (use_hw_binning(cmd)) {
1285 /* enable stream-out during binning pass: */
1286 tu_cs_emit_regs(cs, A6XX_VPC_SO_OVERRIDE(.so_disable=false));
1287
1288 tu6_emit_bin_size(cs,
1289 tiling->tile0.extent.width,
1290 tiling->tile0.extent.height,
1291 A6XX_RB_BIN_CONTROL_BINNING_PASS | 0x6000000);
1292
1293 tu6_emit_render_cntl(cmd, cmd->state.subpass, cs, true);
1294
1295 tu6_emit_binning_pass(cmd, cs);
1296
1297 /* and disable stream-out for draw pass: */
1298 tu_cs_emit_regs(cs, A6XX_VPC_SO_OVERRIDE(.so_disable=true));
1299
1300 tu6_emit_bin_size(cs,
1301 tiling->tile0.extent.width,
1302 tiling->tile0.extent.height,
1303 A6XX_RB_BIN_CONTROL_USE_VIZ | 0x6000000);
1304
1305 tu_cs_emit_regs(cs,
1306 A6XX_VFD_MODE_CNTL(0));
1307
1308 tu_cs_emit_regs(cs, A6XX_PC_UNKNOWN_9805(.unknown = phys_dev->magic.PC_UNKNOWN_9805));
1309
1310 tu_cs_emit_regs(cs, A6XX_SP_UNKNOWN_A0F8(.unknown = phys_dev->magic.SP_UNKNOWN_A0F8));
1311
1312 tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1313 tu_cs_emit(cs, 0x1);
1314 } else {
1315 /* no binning pass, so enable stream-out for draw pass:: */
1316 tu_cs_emit_regs(cs, A6XX_VPC_SO_OVERRIDE(.so_disable=false));
1317
1318 tu6_emit_bin_size(cs,
1319 tiling->tile0.extent.width,
1320 tiling->tile0.extent.height,
1321 0x6000000);
1322 }
1323
1324 tu_cs_sanity_check(cs);
1325 }
1326
1327 static void
1328 tu6_render_tile(struct tu_cmd_buffer *cmd,
1329 struct tu_cs *cs,
1330 const struct tu_tile *tile)
1331 {
1332 tu6_emit_tile_select(cmd, cs, tile);
1333
1334 tu_cs_emit_call(cs, &cmd->draw_cs);
1335 cmd->wait_for_idle = true;
1336
1337 if (use_hw_binning(cmd)) {
1338 tu_cs_emit_pkt7(cs, CP_REG_TEST, 1);
1339 tu_cs_emit(cs, A6XX_CP_REG_TEST_0_REG(OVERFLOW_FLAG_REG) |
1340 A6XX_CP_REG_TEST_0_BIT(0) |
1341 A6XX_CP_REG_TEST_0_WAIT_FOR_ME);
1342
1343 tu_cs_reserve(cs, 3 + 2);
1344 tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
1345 tu_cs_emit(cs, CP_COND_REG_EXEC_0_MODE(PRED_TEST));
1346 tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(2));
1347
1348 /* if (no overflow) */ {
1349 tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1350 tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_ENDVIS));
1351 }
1352 }
1353
1354 tu_cs_emit_ib(cs, &cmd->state.tile_store_ib);
1355
1356 tu_cs_sanity_check(cs);
1357 }
1358
1359 static void
1360 tu6_tile_render_end(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1361 {
1362 tu_cs_emit_call(cs, &cmd->draw_epilogue_cs);
1363
1364 tu_cs_emit_regs(cs,
1365 A6XX_GRAS_LRZ_CNTL(0));
1366
1367 tu6_emit_lrz_flush(cmd, cs);
1368
1369 tu6_emit_event_write(cmd, cs, CACHE_FLUSH_TS, true);
1370
1371 tu_cs_sanity_check(cs);
1372 }
1373
1374 static void
1375 tu_cmd_render_tiles(struct tu_cmd_buffer *cmd)
1376 {
1377 const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1378
1379 tu6_tile_render_begin(cmd, &cmd->cs);
1380
1381 for (uint32_t y = 0; y < tiling->tile_count.height; y++) {
1382 for (uint32_t x = 0; x < tiling->tile_count.width; x++) {
1383 struct tu_tile tile;
1384 tu_tiling_config_get_tile(tiling, cmd->device, x, y, &tile);
1385 tu6_render_tile(cmd, &cmd->cs, &tile);
1386 }
1387 }
1388
1389 tu6_tile_render_end(cmd, &cmd->cs);
1390 }
1391
1392 static void
1393 tu_cmd_render_sysmem(struct tu_cmd_buffer *cmd)
1394 {
1395 const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1396
1397 tu6_sysmem_render_begin(cmd, &cmd->cs, &tiling->render_area);
1398
1399 tu_cs_emit_call(&cmd->cs, &cmd->draw_cs);
1400 cmd->wait_for_idle = true;
1401
1402 tu6_sysmem_render_end(cmd, &cmd->cs);
1403 }
1404
1405 static void
1406 tu_cmd_prepare_tile_store_ib(struct tu_cmd_buffer *cmd)
1407 {
1408 const uint32_t tile_store_space = 11 + (35 * 2) * cmd->state.pass->attachment_count;
1409 struct tu_cs sub_cs;
1410
1411 VkResult result =
1412 tu_cs_begin_sub_stream(&cmd->sub_cs, tile_store_space, &sub_cs);
1413 if (result != VK_SUCCESS) {
1414 cmd->record_result = result;
1415 return;
1416 }
1417
1418 /* emit to tile-store sub_cs */
1419 tu6_emit_tile_store(cmd, &sub_cs);
1420
1421 cmd->state.tile_store_ib = tu_cs_end_sub_stream(&cmd->sub_cs, &sub_cs);
1422 }
1423
1424 static void
1425 tu_cmd_update_tiling_config(struct tu_cmd_buffer *cmd,
1426 const VkRect2D *render_area)
1427 {
1428 const struct tu_device *dev = cmd->device;
1429 struct tu_tiling_config *tiling = &cmd->state.tiling_config;
1430
1431 tiling->render_area = *render_area;
1432 tiling->force_sysmem = false;
1433
1434 tu_tiling_config_update_tile_layout(tiling, dev, cmd->state.pass->gmem_pixels);
1435 tu_tiling_config_update_pipe_layout(tiling, dev);
1436 tu_tiling_config_update_pipes(tiling, dev);
1437 }
1438
1439 const struct tu_dynamic_state default_dynamic_state = {
1440 .viewport =
1441 {
1442 .count = 0,
1443 },
1444 .scissor =
1445 {
1446 .count = 0,
1447 },
1448 .line_width = 1.0f,
1449 .depth_bias =
1450 {
1451 .bias = 0.0f,
1452 .clamp = 0.0f,
1453 .slope = 0.0f,
1454 },
1455 .blend_constants = { 0.0f, 0.0f, 0.0f, 0.0f },
1456 .depth_bounds =
1457 {
1458 .min = 0.0f,
1459 .max = 1.0f,
1460 },
1461 .stencil_compare_mask =
1462 {
1463 .front = ~0u,
1464 .back = ~0u,
1465 },
1466 .stencil_write_mask =
1467 {
1468 .front = ~0u,
1469 .back = ~0u,
1470 },
1471 .stencil_reference =
1472 {
1473 .front = 0u,
1474 .back = 0u,
1475 },
1476 };
1477
1478 static void UNUSED /* FINISHME */
1479 tu_bind_dynamic_state(struct tu_cmd_buffer *cmd_buffer,
1480 const struct tu_dynamic_state *src)
1481 {
1482 struct tu_dynamic_state *dest = &cmd_buffer->state.dynamic;
1483 uint32_t copy_mask = src->mask;
1484 uint32_t dest_mask = 0;
1485
1486 tu_use_args(cmd_buffer); /* FINISHME */
1487
1488 /* Make sure to copy the number of viewports/scissors because they can
1489 * only be specified at pipeline creation time.
1490 */
1491 dest->viewport.count = src->viewport.count;
1492 dest->scissor.count = src->scissor.count;
1493 dest->discard_rectangle.count = src->discard_rectangle.count;
1494
1495 if (copy_mask & TU_DYNAMIC_VIEWPORT) {
1496 if (memcmp(&dest->viewport.viewports, &src->viewport.viewports,
1497 src->viewport.count * sizeof(VkViewport))) {
1498 typed_memcpy(dest->viewport.viewports, src->viewport.viewports,
1499 src->viewport.count);
1500 dest_mask |= TU_DYNAMIC_VIEWPORT;
1501 }
1502 }
1503
1504 if (copy_mask & TU_DYNAMIC_SCISSOR) {
1505 if (memcmp(&dest->scissor.scissors, &src->scissor.scissors,
1506 src->scissor.count * sizeof(VkRect2D))) {
1507 typed_memcpy(dest->scissor.scissors, src->scissor.scissors,
1508 src->scissor.count);
1509 dest_mask |= TU_DYNAMIC_SCISSOR;
1510 }
1511 }
1512
1513 if (copy_mask & TU_DYNAMIC_LINE_WIDTH) {
1514 if (dest->line_width != src->line_width) {
1515 dest->line_width = src->line_width;
1516 dest_mask |= TU_DYNAMIC_LINE_WIDTH;
1517 }
1518 }
1519
1520 if (copy_mask & TU_DYNAMIC_DEPTH_BIAS) {
1521 if (memcmp(&dest->depth_bias, &src->depth_bias,
1522 sizeof(src->depth_bias))) {
1523 dest->depth_bias = src->depth_bias;
1524 dest_mask |= TU_DYNAMIC_DEPTH_BIAS;
1525 }
1526 }
1527
1528 if (copy_mask & TU_DYNAMIC_BLEND_CONSTANTS) {
1529 if (memcmp(&dest->blend_constants, &src->blend_constants,
1530 sizeof(src->blend_constants))) {
1531 typed_memcpy(dest->blend_constants, src->blend_constants, 4);
1532 dest_mask |= TU_DYNAMIC_BLEND_CONSTANTS;
1533 }
1534 }
1535
1536 if (copy_mask & TU_DYNAMIC_DEPTH_BOUNDS) {
1537 if (memcmp(&dest->depth_bounds, &src->depth_bounds,
1538 sizeof(src->depth_bounds))) {
1539 dest->depth_bounds = src->depth_bounds;
1540 dest_mask |= TU_DYNAMIC_DEPTH_BOUNDS;
1541 }
1542 }
1543
1544 if (copy_mask & TU_DYNAMIC_STENCIL_COMPARE_MASK) {
1545 if (memcmp(&dest->stencil_compare_mask, &src->stencil_compare_mask,
1546 sizeof(src->stencil_compare_mask))) {
1547 dest->stencil_compare_mask = src->stencil_compare_mask;
1548 dest_mask |= TU_DYNAMIC_STENCIL_COMPARE_MASK;
1549 }
1550 }
1551
1552 if (copy_mask & TU_DYNAMIC_STENCIL_WRITE_MASK) {
1553 if (memcmp(&dest->stencil_write_mask, &src->stencil_write_mask,
1554 sizeof(src->stencil_write_mask))) {
1555 dest->stencil_write_mask = src->stencil_write_mask;
1556 dest_mask |= TU_DYNAMIC_STENCIL_WRITE_MASK;
1557 }
1558 }
1559
1560 if (copy_mask & TU_DYNAMIC_STENCIL_REFERENCE) {
1561 if (memcmp(&dest->stencil_reference, &src->stencil_reference,
1562 sizeof(src->stencil_reference))) {
1563 dest->stencil_reference = src->stencil_reference;
1564 dest_mask |= TU_DYNAMIC_STENCIL_REFERENCE;
1565 }
1566 }
1567
1568 if (copy_mask & TU_DYNAMIC_DISCARD_RECTANGLE) {
1569 if (memcmp(&dest->discard_rectangle.rectangles,
1570 &src->discard_rectangle.rectangles,
1571 src->discard_rectangle.count * sizeof(VkRect2D))) {
1572 typed_memcpy(dest->discard_rectangle.rectangles,
1573 src->discard_rectangle.rectangles,
1574 src->discard_rectangle.count);
1575 dest_mask |= TU_DYNAMIC_DISCARD_RECTANGLE;
1576 }
1577 }
1578 }
1579
1580 static VkResult
1581 tu_create_cmd_buffer(struct tu_device *device,
1582 struct tu_cmd_pool *pool,
1583 VkCommandBufferLevel level,
1584 VkCommandBuffer *pCommandBuffer)
1585 {
1586 struct tu_cmd_buffer *cmd_buffer;
1587 cmd_buffer = vk_zalloc(&pool->alloc, sizeof(*cmd_buffer), 8,
1588 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1589 if (cmd_buffer == NULL)
1590 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
1591
1592 cmd_buffer->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
1593 cmd_buffer->device = device;
1594 cmd_buffer->pool = pool;
1595 cmd_buffer->level = level;
1596
1597 if (pool) {
1598 list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers);
1599 cmd_buffer->queue_family_index = pool->queue_family_index;
1600
1601 } else {
1602 /* Init the pool_link so we can safely call list_del when we destroy
1603 * the command buffer
1604 */
1605 list_inithead(&cmd_buffer->pool_link);
1606 cmd_buffer->queue_family_index = TU_QUEUE_GENERAL;
1607 }
1608
1609 tu_bo_list_init(&cmd_buffer->bo_list);
1610 tu_cs_init(&cmd_buffer->cs, device, TU_CS_MODE_GROW, 4096);
1611 tu_cs_init(&cmd_buffer->draw_cs, device, TU_CS_MODE_GROW, 4096);
1612 tu_cs_init(&cmd_buffer->draw_epilogue_cs, device, TU_CS_MODE_GROW, 4096);
1613 tu_cs_init(&cmd_buffer->sub_cs, device, TU_CS_MODE_SUB_STREAM, 2048);
1614
1615 *pCommandBuffer = tu_cmd_buffer_to_handle(cmd_buffer);
1616
1617 list_inithead(&cmd_buffer->upload.list);
1618
1619 VkResult result = tu_bo_init_new(device, &cmd_buffer->scratch_bo, 0x1000);
1620 if (result != VK_SUCCESS)
1621 goto fail_scratch_bo;
1622
1623 /* TODO: resize on overflow */
1624 cmd_buffer->vsc_data_pitch = device->vsc_data_pitch;
1625 cmd_buffer->vsc_data2_pitch = device->vsc_data2_pitch;
1626 cmd_buffer->vsc_data = device->vsc_data;
1627 cmd_buffer->vsc_data2 = device->vsc_data2;
1628
1629 return VK_SUCCESS;
1630
1631 fail_scratch_bo:
1632 list_del(&cmd_buffer->pool_link);
1633 return result;
1634 }
1635
1636 static void
1637 tu_cmd_buffer_destroy(struct tu_cmd_buffer *cmd_buffer)
1638 {
1639 tu_bo_finish(cmd_buffer->device, &cmd_buffer->scratch_bo);
1640
1641 list_del(&cmd_buffer->pool_link);
1642
1643 for (unsigned i = 0; i < VK_PIPELINE_BIND_POINT_RANGE_SIZE; i++)
1644 free(cmd_buffer->descriptors[i].push_set.set.mapped_ptr);
1645
1646 tu_cs_finish(&cmd_buffer->cs);
1647 tu_cs_finish(&cmd_buffer->draw_cs);
1648 tu_cs_finish(&cmd_buffer->draw_epilogue_cs);
1649 tu_cs_finish(&cmd_buffer->sub_cs);
1650
1651 tu_bo_list_destroy(&cmd_buffer->bo_list);
1652 vk_free(&cmd_buffer->pool->alloc, cmd_buffer);
1653 }
1654
1655 static VkResult
1656 tu_reset_cmd_buffer(struct tu_cmd_buffer *cmd_buffer)
1657 {
1658 cmd_buffer->wait_for_idle = true;
1659
1660 cmd_buffer->record_result = VK_SUCCESS;
1661
1662 tu_bo_list_reset(&cmd_buffer->bo_list);
1663 tu_cs_reset(&cmd_buffer->cs);
1664 tu_cs_reset(&cmd_buffer->draw_cs);
1665 tu_cs_reset(&cmd_buffer->draw_epilogue_cs);
1666 tu_cs_reset(&cmd_buffer->sub_cs);
1667
1668 for (unsigned i = 0; i < VK_PIPELINE_BIND_POINT_RANGE_SIZE; i++) {
1669 cmd_buffer->descriptors[i].valid = 0;
1670 cmd_buffer->descriptors[i].push_dirty = false;
1671 }
1672
1673 cmd_buffer->status = TU_CMD_BUFFER_STATUS_INITIAL;
1674
1675 return cmd_buffer->record_result;
1676 }
1677
1678 VkResult
1679 tu_AllocateCommandBuffers(VkDevice _device,
1680 const VkCommandBufferAllocateInfo *pAllocateInfo,
1681 VkCommandBuffer *pCommandBuffers)
1682 {
1683 TU_FROM_HANDLE(tu_device, device, _device);
1684 TU_FROM_HANDLE(tu_cmd_pool, pool, pAllocateInfo->commandPool);
1685
1686 VkResult result = VK_SUCCESS;
1687 uint32_t i;
1688
1689 for (i = 0; i < pAllocateInfo->commandBufferCount; i++) {
1690
1691 if (!list_is_empty(&pool->free_cmd_buffers)) {
1692 struct tu_cmd_buffer *cmd_buffer = list_first_entry(
1693 &pool->free_cmd_buffers, struct tu_cmd_buffer, pool_link);
1694
1695 list_del(&cmd_buffer->pool_link);
1696 list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers);
1697
1698 result = tu_reset_cmd_buffer(cmd_buffer);
1699 cmd_buffer->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
1700 cmd_buffer->level = pAllocateInfo->level;
1701
1702 pCommandBuffers[i] = tu_cmd_buffer_to_handle(cmd_buffer);
1703 } else {
1704 result = tu_create_cmd_buffer(device, pool, pAllocateInfo->level,
1705 &pCommandBuffers[i]);
1706 }
1707 if (result != VK_SUCCESS)
1708 break;
1709 }
1710
1711 if (result != VK_SUCCESS) {
1712 tu_FreeCommandBuffers(_device, pAllocateInfo->commandPool, i,
1713 pCommandBuffers);
1714
1715 /* From the Vulkan 1.0.66 spec:
1716 *
1717 * "vkAllocateCommandBuffers can be used to create multiple
1718 * command buffers. If the creation of any of those command
1719 * buffers fails, the implementation must destroy all
1720 * successfully created command buffer objects from this
1721 * command, set all entries of the pCommandBuffers array to
1722 * NULL and return the error."
1723 */
1724 memset(pCommandBuffers, 0,
1725 sizeof(*pCommandBuffers) * pAllocateInfo->commandBufferCount);
1726 }
1727
1728 return result;
1729 }
1730
1731 void
1732 tu_FreeCommandBuffers(VkDevice device,
1733 VkCommandPool commandPool,
1734 uint32_t commandBufferCount,
1735 const VkCommandBuffer *pCommandBuffers)
1736 {
1737 for (uint32_t i = 0; i < commandBufferCount; i++) {
1738 TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, pCommandBuffers[i]);
1739
1740 if (cmd_buffer) {
1741 if (cmd_buffer->pool) {
1742 list_del(&cmd_buffer->pool_link);
1743 list_addtail(&cmd_buffer->pool_link,
1744 &cmd_buffer->pool->free_cmd_buffers);
1745 } else
1746 tu_cmd_buffer_destroy(cmd_buffer);
1747 }
1748 }
1749 }
1750
1751 VkResult
1752 tu_ResetCommandBuffer(VkCommandBuffer commandBuffer,
1753 VkCommandBufferResetFlags flags)
1754 {
1755 TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
1756 return tu_reset_cmd_buffer(cmd_buffer);
1757 }
1758
1759 VkResult
1760 tu_BeginCommandBuffer(VkCommandBuffer commandBuffer,
1761 const VkCommandBufferBeginInfo *pBeginInfo)
1762 {
1763 TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
1764 VkResult result = VK_SUCCESS;
1765
1766 if (cmd_buffer->status != TU_CMD_BUFFER_STATUS_INITIAL) {
1767 /* If the command buffer has already been resetted with
1768 * vkResetCommandBuffer, no need to do it again.
1769 */
1770 result = tu_reset_cmd_buffer(cmd_buffer);
1771 if (result != VK_SUCCESS)
1772 return result;
1773 }
1774
1775 memset(&cmd_buffer->state, 0, sizeof(cmd_buffer->state));
1776 cmd_buffer->usage_flags = pBeginInfo->flags;
1777
1778 tu_cs_begin(&cmd_buffer->cs);
1779 tu_cs_begin(&cmd_buffer->draw_cs);
1780 tu_cs_begin(&cmd_buffer->draw_epilogue_cs);
1781
1782 cmd_buffer->scratch_seqno = 0;
1783
1784 /* setup initial configuration into command buffer */
1785 if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY) {
1786 switch (cmd_buffer->queue_family_index) {
1787 case TU_QUEUE_GENERAL:
1788 tu6_init_hw(cmd_buffer, &cmd_buffer->cs);
1789 break;
1790 default:
1791 break;
1792 }
1793 } else if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY &&
1794 (pBeginInfo->flags & VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT)) {
1795 assert(pBeginInfo->pInheritanceInfo);
1796 cmd_buffer->state.pass = tu_render_pass_from_handle(pBeginInfo->pInheritanceInfo->renderPass);
1797 cmd_buffer->state.subpass = &cmd_buffer->state.pass->subpasses[pBeginInfo->pInheritanceInfo->subpass];
1798 }
1799
1800 cmd_buffer->status = TU_CMD_BUFFER_STATUS_RECORDING;
1801
1802 return VK_SUCCESS;
1803 }
1804
1805 void
1806 tu_CmdBindVertexBuffers(VkCommandBuffer commandBuffer,
1807 uint32_t firstBinding,
1808 uint32_t bindingCount,
1809 const VkBuffer *pBuffers,
1810 const VkDeviceSize *pOffsets)
1811 {
1812 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
1813
1814 assert(firstBinding + bindingCount <= MAX_VBS);
1815
1816 for (uint32_t i = 0; i < bindingCount; i++) {
1817 cmd->state.vb.buffers[firstBinding + i] =
1818 tu_buffer_from_handle(pBuffers[i]);
1819 cmd->state.vb.offsets[firstBinding + i] = pOffsets[i];
1820 }
1821
1822 /* VB states depend on VkPipelineVertexInputStateCreateInfo */
1823 cmd->state.dirty |= TU_CMD_DIRTY_VERTEX_BUFFERS;
1824 }
1825
1826 void
1827 tu_CmdBindIndexBuffer(VkCommandBuffer commandBuffer,
1828 VkBuffer buffer,
1829 VkDeviceSize offset,
1830 VkIndexType indexType)
1831 {
1832 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
1833 TU_FROM_HANDLE(tu_buffer, buf, buffer);
1834
1835 /* initialize/update the restart index */
1836 if (!cmd->state.index_buffer || cmd->state.index_type != indexType) {
1837 struct tu_cs *draw_cs = &cmd->draw_cs;
1838
1839 tu6_emit_restart_index(
1840 draw_cs, indexType == VK_INDEX_TYPE_UINT32 ? 0xffffffff : 0xffff);
1841
1842 tu_cs_sanity_check(draw_cs);
1843 }
1844
1845 /* track the BO */
1846 if (cmd->state.index_buffer != buf)
1847 tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
1848
1849 cmd->state.index_buffer = buf;
1850 cmd->state.index_offset = offset;
1851 cmd->state.index_type = indexType;
1852 }
1853
1854 void
1855 tu_CmdBindDescriptorSets(VkCommandBuffer commandBuffer,
1856 VkPipelineBindPoint pipelineBindPoint,
1857 VkPipelineLayout _layout,
1858 uint32_t firstSet,
1859 uint32_t descriptorSetCount,
1860 const VkDescriptorSet *pDescriptorSets,
1861 uint32_t dynamicOffsetCount,
1862 const uint32_t *pDynamicOffsets)
1863 {
1864 TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
1865 TU_FROM_HANDLE(tu_pipeline_layout, layout, _layout);
1866 unsigned dyn_idx = 0;
1867
1868 struct tu_descriptor_state *descriptors_state =
1869 tu_get_descriptors_state(cmd_buffer, pipelineBindPoint);
1870
1871 for (unsigned i = 0; i < descriptorSetCount; ++i) {
1872 unsigned idx = i + firstSet;
1873 TU_FROM_HANDLE(tu_descriptor_set, set, pDescriptorSets[i]);
1874
1875 descriptors_state->sets[idx] = set;
1876 descriptors_state->valid |= (1u << idx);
1877
1878 for(unsigned j = 0; j < set->layout->dynamic_offset_count; ++j, ++dyn_idx) {
1879 unsigned idx = j + layout->set[i + firstSet].dynamic_offset_start;
1880 assert(dyn_idx < dynamicOffsetCount);
1881
1882 descriptors_state->dynamic_buffers[idx] =
1883 set->dynamic_descriptors[j].va + pDynamicOffsets[dyn_idx];
1884 }
1885 }
1886
1887 cmd_buffer->state.dirty |= TU_CMD_DIRTY_DESCRIPTOR_SETS;
1888 }
1889
1890 void tu_CmdBindTransformFeedbackBuffersEXT(VkCommandBuffer commandBuffer,
1891 uint32_t firstBinding,
1892 uint32_t bindingCount,
1893 const VkBuffer *pBuffers,
1894 const VkDeviceSize *pOffsets,
1895 const VkDeviceSize *pSizes)
1896 {
1897 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
1898 assert(firstBinding + bindingCount <= IR3_MAX_SO_BUFFERS);
1899
1900 for (uint32_t i = 0; i < bindingCount; i++) {
1901 uint32_t idx = firstBinding + i;
1902 TU_FROM_HANDLE(tu_buffer, buf, pBuffers[i]);
1903
1904 if (pOffsets[i] != 0)
1905 cmd->state.streamout_reset |= 1 << idx;
1906
1907 cmd->state.streamout_buf.buffers[idx] = buf;
1908 cmd->state.streamout_buf.offsets[idx] = pOffsets[i];
1909 cmd->state.streamout_buf.sizes[idx] = pSizes[i];
1910
1911 cmd->state.streamout_enabled |= 1 << idx;
1912 }
1913
1914 cmd->state.dirty |= TU_CMD_DIRTY_STREAMOUT_BUFFERS;
1915 }
1916
1917 void tu_CmdBeginTransformFeedbackEXT(VkCommandBuffer commandBuffer,
1918 uint32_t firstCounterBuffer,
1919 uint32_t counterBufferCount,
1920 const VkBuffer *pCounterBuffers,
1921 const VkDeviceSize *pCounterBufferOffsets)
1922 {
1923 assert(firstCounterBuffer + counterBufferCount <= IR3_MAX_SO_BUFFERS);
1924 /* TODO do something with counter buffer? */
1925 }
1926
1927 void tu_CmdEndTransformFeedbackEXT(VkCommandBuffer commandBuffer,
1928 uint32_t firstCounterBuffer,
1929 uint32_t counterBufferCount,
1930 const VkBuffer *pCounterBuffers,
1931 const VkDeviceSize *pCounterBufferOffsets)
1932 {
1933 assert(firstCounterBuffer + counterBufferCount <= IR3_MAX_SO_BUFFERS);
1934 /* TODO do something with counter buffer? */
1935
1936 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
1937 cmd->state.streamout_enabled = 0;
1938 }
1939
1940 void
1941 tu_CmdPushConstants(VkCommandBuffer commandBuffer,
1942 VkPipelineLayout layout,
1943 VkShaderStageFlags stageFlags,
1944 uint32_t offset,
1945 uint32_t size,
1946 const void *pValues)
1947 {
1948 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
1949 memcpy((void*) cmd->push_constants + offset, pValues, size);
1950 cmd->state.dirty |= TU_CMD_DIRTY_PUSH_CONSTANTS;
1951 }
1952
1953 VkResult
1954 tu_EndCommandBuffer(VkCommandBuffer commandBuffer)
1955 {
1956 TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
1957
1958 if (cmd_buffer->scratch_seqno) {
1959 tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->scratch_bo,
1960 MSM_SUBMIT_BO_WRITE);
1961 }
1962
1963 if (cmd_buffer->use_vsc_data) {
1964 tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->vsc_data,
1965 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
1966 tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->vsc_data2,
1967 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
1968 }
1969
1970 tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->device->border_color,
1971 MSM_SUBMIT_BO_READ);
1972
1973 for (uint32_t i = 0; i < cmd_buffer->draw_cs.bo_count; i++) {
1974 tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->draw_cs.bos[i],
1975 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
1976 }
1977
1978 for (uint32_t i = 0; i < cmd_buffer->draw_epilogue_cs.bo_count; i++) {
1979 tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->draw_epilogue_cs.bos[i],
1980 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
1981 }
1982
1983 for (uint32_t i = 0; i < cmd_buffer->sub_cs.bo_count; i++) {
1984 tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->sub_cs.bos[i],
1985 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
1986 }
1987
1988 tu_cs_end(&cmd_buffer->cs);
1989 tu_cs_end(&cmd_buffer->draw_cs);
1990 tu_cs_end(&cmd_buffer->draw_epilogue_cs);
1991
1992 cmd_buffer->status = TU_CMD_BUFFER_STATUS_EXECUTABLE;
1993
1994 return cmd_buffer->record_result;
1995 }
1996
1997 void
1998 tu_CmdBindPipeline(VkCommandBuffer commandBuffer,
1999 VkPipelineBindPoint pipelineBindPoint,
2000 VkPipeline _pipeline)
2001 {
2002 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2003 TU_FROM_HANDLE(tu_pipeline, pipeline, _pipeline);
2004
2005 switch (pipelineBindPoint) {
2006 case VK_PIPELINE_BIND_POINT_GRAPHICS:
2007 cmd->state.pipeline = pipeline;
2008 cmd->state.dirty |= TU_CMD_DIRTY_PIPELINE;
2009 break;
2010 case VK_PIPELINE_BIND_POINT_COMPUTE:
2011 cmd->state.compute_pipeline = pipeline;
2012 cmd->state.dirty |= TU_CMD_DIRTY_COMPUTE_PIPELINE;
2013 break;
2014 default:
2015 unreachable("unrecognized pipeline bind point");
2016 break;
2017 }
2018
2019 tu_bo_list_add(&cmd->bo_list, &pipeline->program.binary_bo,
2020 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2021 for (uint32_t i = 0; i < pipeline->cs.bo_count; i++) {
2022 tu_bo_list_add(&cmd->bo_list, pipeline->cs.bos[i],
2023 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2024 }
2025 }
2026
2027 void
2028 tu_CmdSetViewport(VkCommandBuffer commandBuffer,
2029 uint32_t firstViewport,
2030 uint32_t viewportCount,
2031 const VkViewport *pViewports)
2032 {
2033 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2034
2035 assert(firstViewport == 0 && viewportCount == 1);
2036 cmd->state.dynamic.viewport.viewports[0] = pViewports[0];
2037 cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_VIEWPORT;
2038 }
2039
2040 void
2041 tu_CmdSetScissor(VkCommandBuffer commandBuffer,
2042 uint32_t firstScissor,
2043 uint32_t scissorCount,
2044 const VkRect2D *pScissors)
2045 {
2046 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2047
2048 assert(firstScissor == 0 && scissorCount == 1);
2049 cmd->state.dynamic.scissor.scissors[0] = pScissors[0];
2050 cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_SCISSOR;
2051 }
2052
2053 void
2054 tu_CmdSetLineWidth(VkCommandBuffer commandBuffer, float lineWidth)
2055 {
2056 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2057
2058 cmd->state.dynamic.line_width = lineWidth;
2059
2060 /* line width depends on VkPipelineRasterizationStateCreateInfo */
2061 cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_LINE_WIDTH;
2062 }
2063
2064 void
2065 tu_CmdSetDepthBias(VkCommandBuffer commandBuffer,
2066 float depthBiasConstantFactor,
2067 float depthBiasClamp,
2068 float depthBiasSlopeFactor)
2069 {
2070 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2071 struct tu_cs *draw_cs = &cmd->draw_cs;
2072
2073 tu6_emit_depth_bias(draw_cs, depthBiasConstantFactor, depthBiasClamp,
2074 depthBiasSlopeFactor);
2075
2076 tu_cs_sanity_check(draw_cs);
2077 }
2078
2079 void
2080 tu_CmdSetBlendConstants(VkCommandBuffer commandBuffer,
2081 const float blendConstants[4])
2082 {
2083 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2084 struct tu_cs *draw_cs = &cmd->draw_cs;
2085
2086 tu6_emit_blend_constants(draw_cs, blendConstants);
2087
2088 tu_cs_sanity_check(draw_cs);
2089 }
2090
2091 void
2092 tu_CmdSetDepthBounds(VkCommandBuffer commandBuffer,
2093 float minDepthBounds,
2094 float maxDepthBounds)
2095 {
2096 }
2097
2098 void
2099 tu_CmdSetStencilCompareMask(VkCommandBuffer commandBuffer,
2100 VkStencilFaceFlags faceMask,
2101 uint32_t compareMask)
2102 {
2103 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2104
2105 if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
2106 cmd->state.dynamic.stencil_compare_mask.front = compareMask;
2107 if (faceMask & VK_STENCIL_FACE_BACK_BIT)
2108 cmd->state.dynamic.stencil_compare_mask.back = compareMask;
2109
2110 /* the front/back compare masks must be updated together */
2111 cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK;
2112 }
2113
2114 void
2115 tu_CmdSetStencilWriteMask(VkCommandBuffer commandBuffer,
2116 VkStencilFaceFlags faceMask,
2117 uint32_t writeMask)
2118 {
2119 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2120
2121 if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
2122 cmd->state.dynamic.stencil_write_mask.front = writeMask;
2123 if (faceMask & VK_STENCIL_FACE_BACK_BIT)
2124 cmd->state.dynamic.stencil_write_mask.back = writeMask;
2125
2126 /* the front/back write masks must be updated together */
2127 cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK;
2128 }
2129
2130 void
2131 tu_CmdSetStencilReference(VkCommandBuffer commandBuffer,
2132 VkStencilFaceFlags faceMask,
2133 uint32_t reference)
2134 {
2135 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2136
2137 if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
2138 cmd->state.dynamic.stencil_reference.front = reference;
2139 if (faceMask & VK_STENCIL_FACE_BACK_BIT)
2140 cmd->state.dynamic.stencil_reference.back = reference;
2141
2142 /* the front/back references must be updated together */
2143 cmd->state.dirty |= TU_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE;
2144 }
2145
2146 void
2147 tu_CmdExecuteCommands(VkCommandBuffer commandBuffer,
2148 uint32_t commandBufferCount,
2149 const VkCommandBuffer *pCmdBuffers)
2150 {
2151 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2152 VkResult result;
2153
2154 assert(commandBufferCount > 0);
2155
2156 for (uint32_t i = 0; i < commandBufferCount; i++) {
2157 TU_FROM_HANDLE(tu_cmd_buffer, secondary, pCmdBuffers[i]);
2158
2159 result = tu_bo_list_merge(&cmd->bo_list, &secondary->bo_list);
2160 if (result != VK_SUCCESS) {
2161 cmd->record_result = result;
2162 break;
2163 }
2164
2165 if (secondary->usage_flags &
2166 VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT) {
2167 assert(tu_cs_is_empty(&secondary->cs));
2168
2169 result = tu_cs_add_entries(&cmd->draw_cs, &secondary->draw_cs);
2170 if (result != VK_SUCCESS) {
2171 cmd->record_result = result;
2172 break;
2173 }
2174
2175 result = tu_cs_add_entries(&cmd->draw_epilogue_cs,
2176 &secondary->draw_epilogue_cs);
2177 if (result != VK_SUCCESS) {
2178 cmd->record_result = result;
2179 break;
2180 }
2181 } else {
2182 assert(tu_cs_is_empty(&secondary->draw_cs));
2183 assert(tu_cs_is_empty(&secondary->draw_epilogue_cs));
2184
2185 for (uint32_t j = 0; j < secondary->cs.bo_count; j++) {
2186 tu_bo_list_add(&cmd->bo_list, secondary->cs.bos[j],
2187 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2188 }
2189
2190 tu_cs_emit_call(&cmd->cs, &secondary->cs);
2191 }
2192 }
2193 cmd->state.dirty = ~0u; /* TODO: set dirty only what needs to be */
2194 }
2195
2196 VkResult
2197 tu_CreateCommandPool(VkDevice _device,
2198 const VkCommandPoolCreateInfo *pCreateInfo,
2199 const VkAllocationCallbacks *pAllocator,
2200 VkCommandPool *pCmdPool)
2201 {
2202 TU_FROM_HANDLE(tu_device, device, _device);
2203 struct tu_cmd_pool *pool;
2204
2205 pool = vk_alloc2(&device->alloc, pAllocator, sizeof(*pool), 8,
2206 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2207 if (pool == NULL)
2208 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
2209
2210 if (pAllocator)
2211 pool->alloc = *pAllocator;
2212 else
2213 pool->alloc = device->alloc;
2214
2215 list_inithead(&pool->cmd_buffers);
2216 list_inithead(&pool->free_cmd_buffers);
2217
2218 pool->queue_family_index = pCreateInfo->queueFamilyIndex;
2219
2220 *pCmdPool = tu_cmd_pool_to_handle(pool);
2221
2222 return VK_SUCCESS;
2223 }
2224
2225 void
2226 tu_DestroyCommandPool(VkDevice _device,
2227 VkCommandPool commandPool,
2228 const VkAllocationCallbacks *pAllocator)
2229 {
2230 TU_FROM_HANDLE(tu_device, device, _device);
2231 TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2232
2233 if (!pool)
2234 return;
2235
2236 list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2237 &pool->cmd_buffers, pool_link)
2238 {
2239 tu_cmd_buffer_destroy(cmd_buffer);
2240 }
2241
2242 list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2243 &pool->free_cmd_buffers, pool_link)
2244 {
2245 tu_cmd_buffer_destroy(cmd_buffer);
2246 }
2247
2248 vk_free2(&device->alloc, pAllocator, pool);
2249 }
2250
2251 VkResult
2252 tu_ResetCommandPool(VkDevice device,
2253 VkCommandPool commandPool,
2254 VkCommandPoolResetFlags flags)
2255 {
2256 TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2257 VkResult result;
2258
2259 list_for_each_entry(struct tu_cmd_buffer, cmd_buffer, &pool->cmd_buffers,
2260 pool_link)
2261 {
2262 result = tu_reset_cmd_buffer(cmd_buffer);
2263 if (result != VK_SUCCESS)
2264 return result;
2265 }
2266
2267 return VK_SUCCESS;
2268 }
2269
2270 void
2271 tu_TrimCommandPool(VkDevice device,
2272 VkCommandPool commandPool,
2273 VkCommandPoolTrimFlags flags)
2274 {
2275 TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2276
2277 if (!pool)
2278 return;
2279
2280 list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2281 &pool->free_cmd_buffers, pool_link)
2282 {
2283 tu_cmd_buffer_destroy(cmd_buffer);
2284 }
2285 }
2286
2287 void
2288 tu_CmdBeginRenderPass(VkCommandBuffer commandBuffer,
2289 const VkRenderPassBeginInfo *pRenderPassBegin,
2290 VkSubpassContents contents)
2291 {
2292 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2293 TU_FROM_HANDLE(tu_render_pass, pass, pRenderPassBegin->renderPass);
2294 TU_FROM_HANDLE(tu_framebuffer, fb, pRenderPassBegin->framebuffer);
2295
2296 cmd->state.pass = pass;
2297 cmd->state.subpass = pass->subpasses;
2298 cmd->state.framebuffer = fb;
2299
2300 tu_cmd_update_tiling_config(cmd, &pRenderPassBegin->renderArea);
2301 tu_cmd_prepare_tile_store_ib(cmd);
2302
2303 tu_emit_load_clear(cmd, pRenderPassBegin);
2304
2305 tu6_emit_zs(cmd, cmd->state.subpass, &cmd->draw_cs);
2306 tu6_emit_mrt(cmd, cmd->state.subpass, &cmd->draw_cs);
2307 tu6_emit_msaa(&cmd->draw_cs, cmd->state.subpass->samples);
2308 tu6_emit_render_cntl(cmd, cmd->state.subpass, &cmd->draw_cs, false);
2309
2310 /* note: use_hw_binning only checks tiling config */
2311 if (use_hw_binning(cmd))
2312 cmd->use_vsc_data = true;
2313
2314 for (uint32_t i = 0; i < fb->attachment_count; ++i) {
2315 const struct tu_image_view *iview = fb->attachments[i].attachment;
2316 tu_bo_list_add(&cmd->bo_list, iview->image->bo,
2317 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
2318 }
2319 }
2320
2321 void
2322 tu_CmdBeginRenderPass2(VkCommandBuffer commandBuffer,
2323 const VkRenderPassBeginInfo *pRenderPassBeginInfo,
2324 const VkSubpassBeginInfoKHR *pSubpassBeginInfo)
2325 {
2326 tu_CmdBeginRenderPass(commandBuffer, pRenderPassBeginInfo,
2327 pSubpassBeginInfo->contents);
2328 }
2329
2330 void
2331 tu_CmdNextSubpass(VkCommandBuffer commandBuffer, VkSubpassContents contents)
2332 {
2333 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2334 const struct tu_render_pass *pass = cmd->state.pass;
2335 struct tu_cs *cs = &cmd->draw_cs;
2336
2337 const struct tu_subpass *subpass = cmd->state.subpass++;
2338
2339 tu_cond_exec_start(cs, CP_COND_EXEC_0_RENDER_MODE_GMEM);
2340
2341 if (subpass->resolve_attachments) {
2342 for (unsigned i = 0; i < subpass->color_count; i++) {
2343 uint32_t a = subpass->resolve_attachments[i].attachment;
2344 if (a == VK_ATTACHMENT_UNUSED)
2345 continue;
2346
2347 tu_store_gmem_attachment(cmd, cs, a,
2348 subpass->color_attachments[i].attachment);
2349
2350 if (pass->attachments[a].gmem_offset < 0)
2351 continue;
2352
2353 /* TODO:
2354 * check if the resolved attachment is needed by later subpasses,
2355 * if it is, should be doing a GMEM->GMEM resolve instead of GMEM->MEM->GMEM..
2356 */
2357 tu_finishme("missing GMEM->GMEM resolve path\n");
2358 tu_emit_load_gmem_attachment(cmd, cs, a);
2359 }
2360 }
2361
2362 tu_cond_exec_end(cs);
2363
2364 tu_cond_exec_start(cs, CP_COND_EXEC_0_RENDER_MODE_SYSMEM);
2365
2366 /* Emit flushes so that input attachments will read the correct value.
2367 * TODO: use subpass dependencies to flush or not
2368 */
2369 tu6_emit_event_write(cmd, cs, PC_CCU_FLUSH_COLOR_TS, true);
2370 tu6_emit_event_write(cmd, cs, PC_CCU_FLUSH_DEPTH_TS, true);
2371
2372 if (subpass->resolve_attachments) {
2373 tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE, false);
2374
2375 for (unsigned i = 0; i < subpass->color_count; i++) {
2376 uint32_t a = subpass->resolve_attachments[i].attachment;
2377 if (a == VK_ATTACHMENT_UNUSED)
2378 continue;
2379
2380 tu6_emit_sysmem_resolve(cmd, cs, a,
2381 subpass->color_attachments[i].attachment);
2382 }
2383
2384 tu6_emit_event_write(cmd, cs, PC_CCU_FLUSH_COLOR_TS, true);
2385 }
2386
2387 tu_cond_exec_end(cs);
2388
2389 /* subpass->input_count > 0 then texture cache invalidate is likely to be needed */
2390 if (cmd->state.subpass->input_count)
2391 tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE, false);
2392
2393 /* emit mrt/zs/msaa/ubwc state for the subpass that is starting */
2394 tu6_emit_zs(cmd, cmd->state.subpass, cs);
2395 tu6_emit_mrt(cmd, cmd->state.subpass, cs);
2396 tu6_emit_msaa(cs, cmd->state.subpass->samples);
2397 tu6_emit_render_cntl(cmd, cmd->state.subpass, cs, false);
2398 }
2399
2400 void
2401 tu_CmdNextSubpass2(VkCommandBuffer commandBuffer,
2402 const VkSubpassBeginInfoKHR *pSubpassBeginInfo,
2403 const VkSubpassEndInfoKHR *pSubpassEndInfo)
2404 {
2405 tu_CmdNextSubpass(commandBuffer, pSubpassBeginInfo->contents);
2406 }
2407
2408 struct tu_draw_info
2409 {
2410 /**
2411 * Number of vertices.
2412 */
2413 uint32_t count;
2414
2415 /**
2416 * Index of the first vertex.
2417 */
2418 int32_t vertex_offset;
2419
2420 /**
2421 * First instance id.
2422 */
2423 uint32_t first_instance;
2424
2425 /**
2426 * Number of instances.
2427 */
2428 uint32_t instance_count;
2429
2430 /**
2431 * First index (indexed draws only).
2432 */
2433 uint32_t first_index;
2434
2435 /**
2436 * Whether it's an indexed draw.
2437 */
2438 bool indexed;
2439
2440 /**
2441 * Indirect draw parameters resource.
2442 */
2443 struct tu_buffer *indirect;
2444 uint64_t indirect_offset;
2445 uint32_t stride;
2446
2447 /**
2448 * Draw count parameters resource.
2449 */
2450 struct tu_buffer *count_buffer;
2451 uint64_t count_buffer_offset;
2452
2453 /**
2454 * Stream output parameters resource.
2455 */
2456 struct tu_buffer *streamout_buffer;
2457 uint64_t streamout_buffer_offset;
2458 };
2459
2460 #define ENABLE_ALL (CP_SET_DRAW_STATE__0_BINNING | CP_SET_DRAW_STATE__0_GMEM | CP_SET_DRAW_STATE__0_SYSMEM)
2461 #define ENABLE_DRAW (CP_SET_DRAW_STATE__0_GMEM | CP_SET_DRAW_STATE__0_SYSMEM)
2462
2463 enum tu_draw_state_group_id
2464 {
2465 TU_DRAW_STATE_PROGRAM,
2466 TU_DRAW_STATE_PROGRAM_BINNING,
2467 TU_DRAW_STATE_VI,
2468 TU_DRAW_STATE_VI_BINNING,
2469 TU_DRAW_STATE_VP,
2470 TU_DRAW_STATE_RAST,
2471 TU_DRAW_STATE_DS,
2472 TU_DRAW_STATE_BLEND,
2473 TU_DRAW_STATE_VS_CONST,
2474 TU_DRAW_STATE_FS_CONST,
2475 TU_DRAW_STATE_VS_TEX,
2476 TU_DRAW_STATE_FS_TEX_SYSMEM,
2477 TU_DRAW_STATE_FS_TEX_GMEM,
2478 TU_DRAW_STATE_FS_IBO,
2479 TU_DRAW_STATE_VS_PARAMS,
2480
2481 TU_DRAW_STATE_COUNT,
2482 };
2483
2484 struct tu_draw_state_group
2485 {
2486 enum tu_draw_state_group_id id;
2487 uint32_t enable_mask;
2488 struct tu_cs_entry ib;
2489 };
2490
2491 const static void *
2492 sampler_ptr(struct tu_descriptor_state *descriptors_state,
2493 const struct tu_descriptor_map *map, unsigned i,
2494 unsigned array_index)
2495 {
2496 assert(descriptors_state->valid & (1 << map->set[i]));
2497
2498 struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2499 assert(map->binding[i] < set->layout->binding_count);
2500
2501 const struct tu_descriptor_set_binding_layout *layout =
2502 &set->layout->binding[map->binding[i]];
2503
2504 if (layout->immutable_samplers_offset) {
2505 const uint32_t *immutable_samplers =
2506 tu_immutable_samplers(set->layout, layout);
2507
2508 return &immutable_samplers[array_index * A6XX_TEX_SAMP_DWORDS];
2509 }
2510
2511 switch (layout->type) {
2512 case VK_DESCRIPTOR_TYPE_SAMPLER:
2513 return &set->mapped_ptr[layout->offset / 4];
2514 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
2515 return &set->mapped_ptr[layout->offset / 4 + A6XX_TEX_CONST_DWORDS +
2516 array_index * (A6XX_TEX_CONST_DWORDS + A6XX_TEX_SAMP_DWORDS)];
2517 default:
2518 unreachable("unimplemented descriptor type");
2519 break;
2520 }
2521 }
2522
2523 static void
2524 write_tex_const(struct tu_cmd_buffer *cmd,
2525 uint32_t *dst,
2526 struct tu_descriptor_state *descriptors_state,
2527 const struct tu_descriptor_map *map,
2528 unsigned i, unsigned array_index, bool is_sysmem)
2529 {
2530 assert(descriptors_state->valid & (1 << map->set[i]));
2531
2532 struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2533 assert(map->binding[i] < set->layout->binding_count);
2534
2535 const struct tu_descriptor_set_binding_layout *layout =
2536 &set->layout->binding[map->binding[i]];
2537
2538 switch (layout->type) {
2539 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
2540 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
2541 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
2542 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
2543 memcpy(dst, &set->mapped_ptr[layout->offset / 4 +
2544 array_index * A6XX_TEX_CONST_DWORDS],
2545 A6XX_TEX_CONST_DWORDS * 4);
2546 break;
2547 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
2548 memcpy(dst, &set->mapped_ptr[layout->offset / 4 +
2549 array_index *
2550 (A6XX_TEX_CONST_DWORDS +
2551 A6XX_TEX_SAMP_DWORDS)],
2552 A6XX_TEX_CONST_DWORDS * 4);
2553 break;
2554 default:
2555 unreachable("unimplemented descriptor type");
2556 break;
2557 }
2558
2559 if (layout->type == VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT && !is_sysmem) {
2560 const struct tu_tiling_config *tiling = &cmd->state.tiling_config;
2561 uint32_t a = cmd->state.subpass->input_attachments[map->value[i] +
2562 array_index].attachment;
2563 const struct tu_render_pass_attachment *att = &cmd->state.pass->attachments[a];
2564
2565 assert(att->gmem_offset >= 0);
2566
2567 dst[0] &= ~(A6XX_TEX_CONST_0_SWAP__MASK | A6XX_TEX_CONST_0_TILE_MODE__MASK);
2568 dst[0] |= A6XX_TEX_CONST_0_TILE_MODE(TILE6_2);
2569 dst[2] &= ~(A6XX_TEX_CONST_2_TYPE__MASK | A6XX_TEX_CONST_2_PITCH__MASK);
2570 dst[2] |=
2571 A6XX_TEX_CONST_2_TYPE(A6XX_TEX_2D) |
2572 A6XX_TEX_CONST_2_PITCH(tiling->tile0.extent.width * att->cpp);
2573 dst[3] = 0;
2574 dst[4] = cmd->device->physical_device->gmem_base + att->gmem_offset;
2575 dst[5] = A6XX_TEX_CONST_5_DEPTH(1);
2576 for (unsigned i = 6; i < A6XX_TEX_CONST_DWORDS; i++)
2577 dst[i] = 0;
2578
2579 if (cmd->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY)
2580 tu_finishme("patch input attachment pitch for secondary cmd buffer");
2581 }
2582 }
2583
2584 static void
2585 write_image_ibo(struct tu_cmd_buffer *cmd,
2586 uint32_t *dst,
2587 struct tu_descriptor_state *descriptors_state,
2588 const struct tu_descriptor_map *map,
2589 unsigned i, unsigned array_index)
2590 {
2591 assert(descriptors_state->valid & (1 << map->set[i]));
2592
2593 struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2594 assert(map->binding[i] < set->layout->binding_count);
2595
2596 const struct tu_descriptor_set_binding_layout *layout =
2597 &set->layout->binding[map->binding[i]];
2598
2599 assert(layout->type == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE);
2600
2601 memcpy(dst, &set->mapped_ptr[layout->offset / 4 +
2602 (array_index * 2 + 1) * A6XX_TEX_CONST_DWORDS],
2603 A6XX_TEX_CONST_DWORDS * 4);
2604 }
2605
2606 static uint64_t
2607 buffer_ptr(struct tu_descriptor_state *descriptors_state,
2608 const struct tu_descriptor_map *map,
2609 unsigned i, unsigned array_index)
2610 {
2611 assert(descriptors_state->valid & (1 << map->set[i]));
2612
2613 struct tu_descriptor_set *set = descriptors_state->sets[map->set[i]];
2614 assert(map->binding[i] < set->layout->binding_count);
2615
2616 const struct tu_descriptor_set_binding_layout *layout =
2617 &set->layout->binding[map->binding[i]];
2618
2619 switch (layout->type) {
2620 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
2621 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
2622 return descriptors_state->dynamic_buffers[layout->dynamic_offset_offset +
2623 array_index];
2624 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
2625 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
2626 return (uint64_t) set->mapped_ptr[layout->offset / 4 + array_index * 2 + 1] << 32 |
2627 set->mapped_ptr[layout->offset / 4 + array_index * 2];
2628 default:
2629 unreachable("unimplemented descriptor type");
2630 break;
2631 }
2632 }
2633
2634 static inline uint32_t
2635 tu6_stage2opcode(gl_shader_stage type)
2636 {
2637 switch (type) {
2638 case MESA_SHADER_VERTEX:
2639 case MESA_SHADER_TESS_CTRL:
2640 case MESA_SHADER_TESS_EVAL:
2641 case MESA_SHADER_GEOMETRY:
2642 return CP_LOAD_STATE6_GEOM;
2643 case MESA_SHADER_FRAGMENT:
2644 case MESA_SHADER_COMPUTE:
2645 case MESA_SHADER_KERNEL:
2646 return CP_LOAD_STATE6_FRAG;
2647 default:
2648 unreachable("bad shader type");
2649 }
2650 }
2651
2652 static inline enum a6xx_state_block
2653 tu6_stage2shadersb(gl_shader_stage type)
2654 {
2655 switch (type) {
2656 case MESA_SHADER_VERTEX:
2657 return SB6_VS_SHADER;
2658 case MESA_SHADER_FRAGMENT:
2659 return SB6_FS_SHADER;
2660 case MESA_SHADER_COMPUTE:
2661 case MESA_SHADER_KERNEL:
2662 return SB6_CS_SHADER;
2663 default:
2664 unreachable("bad shader type");
2665 return ~0;
2666 }
2667 }
2668
2669 static void
2670 tu6_emit_user_consts(struct tu_cs *cs, const struct tu_pipeline *pipeline,
2671 struct tu_descriptor_state *descriptors_state,
2672 gl_shader_stage type,
2673 uint32_t *push_constants)
2674 {
2675 const struct tu_program_descriptor_linkage *link =
2676 &pipeline->program.link[type];
2677 const struct ir3_ubo_analysis_state *state = &link->ubo_state;
2678
2679 for (uint32_t i = 0; i < ARRAY_SIZE(state->range); i++) {
2680 if (state->range[i].start < state->range[i].end) {
2681 uint32_t size = state->range[i].end - state->range[i].start;
2682 uint32_t offset = state->range[i].start;
2683
2684 /* and even if the start of the const buffer is before
2685 * first_immediate, the end may not be:
2686 */
2687 size = MIN2(size, (16 * link->constlen) - state->range[i].offset);
2688
2689 if (size == 0)
2690 continue;
2691
2692 /* things should be aligned to vec4: */
2693 debug_assert((state->range[i].offset % 16) == 0);
2694 debug_assert((size % 16) == 0);
2695 debug_assert((offset % 16) == 0);
2696
2697 if (i == 0) {
2698 /* push constants */
2699 tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + (size / 4));
2700 tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(state->range[i].offset / 16) |
2701 CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2702 CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
2703 CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2704 CP_LOAD_STATE6_0_NUM_UNIT(size / 16));
2705 tu_cs_emit(cs, 0);
2706 tu_cs_emit(cs, 0);
2707 for (unsigned i = 0; i < size / 4; i++)
2708 tu_cs_emit(cs, push_constants[i + offset / 4]);
2709 continue;
2710 }
2711
2712 /* Look through the UBO map to find our UBO index, and get the VA for
2713 * that UBO.
2714 */
2715 uint64_t va = 0;
2716 uint32_t ubo_idx = i - 1;
2717 uint32_t ubo_map_base = 0;
2718 for (int j = 0; j < link->ubo_map.num; j++) {
2719 if (ubo_idx >= ubo_map_base &&
2720 ubo_idx < ubo_map_base + link->ubo_map.array_size[j]) {
2721 va = buffer_ptr(descriptors_state, &link->ubo_map, j,
2722 ubo_idx - ubo_map_base);
2723 break;
2724 }
2725 ubo_map_base += link->ubo_map.array_size[j];
2726 }
2727 assert(va);
2728
2729 tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3);
2730 tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(state->range[i].offset / 16) |
2731 CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2732 CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
2733 CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2734 CP_LOAD_STATE6_0_NUM_UNIT(size / 16));
2735 tu_cs_emit_qw(cs, va + offset);
2736 }
2737 }
2738 }
2739
2740 static void
2741 tu6_emit_ubos(struct tu_cs *cs, const struct tu_pipeline *pipeline,
2742 struct tu_descriptor_state *descriptors_state,
2743 gl_shader_stage type)
2744 {
2745 const struct tu_program_descriptor_linkage *link =
2746 &pipeline->program.link[type];
2747
2748 uint32_t num = MIN2(link->ubo_map.num_desc, link->const_state.num_ubos);
2749 uint32_t anum = align(num, 2);
2750
2751 if (!num)
2752 return;
2753
2754 tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + (2 * anum));
2755 tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(link->const_state.offsets.ubo) |
2756 CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2757 CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
2758 CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2759 CP_LOAD_STATE6_0_NUM_UNIT(anum/2));
2760 tu_cs_emit(cs, CP_LOAD_STATE6_1_EXT_SRC_ADDR(0));
2761 tu_cs_emit(cs, CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI(0));
2762
2763 unsigned emitted = 0;
2764 for (unsigned i = 0; emitted < num && i < link->ubo_map.num; i++) {
2765 for (unsigned j = 0; emitted < num && j < link->ubo_map.array_size[i]; j++) {
2766 tu_cs_emit_qw(cs, buffer_ptr(descriptors_state, &link->ubo_map, i, j));
2767 emitted++;
2768 }
2769 }
2770
2771 for (; emitted < anum; emitted++) {
2772 tu_cs_emit(cs, 0xffffffff);
2773 tu_cs_emit(cs, 0xffffffff);
2774 }
2775 }
2776
2777 static struct tu_cs_entry
2778 tu6_emit_consts(struct tu_cmd_buffer *cmd,
2779 const struct tu_pipeline *pipeline,
2780 struct tu_descriptor_state *descriptors_state,
2781 gl_shader_stage type)
2782 {
2783 struct tu_cs cs;
2784 tu_cs_begin_sub_stream(&cmd->sub_cs, 512, &cs); /* TODO: maximum size? */
2785
2786 tu6_emit_user_consts(&cs, pipeline, descriptors_state, type, cmd->push_constants);
2787 tu6_emit_ubos(&cs, pipeline, descriptors_state, type);
2788
2789 return tu_cs_end_sub_stream(&cmd->sub_cs, &cs);
2790 }
2791
2792 static VkResult
2793 tu6_emit_vs_params(struct tu_cmd_buffer *cmd,
2794 const struct tu_draw_info *draw,
2795 struct tu_cs_entry *entry)
2796 {
2797 /* TODO: fill out more than just base instance */
2798 const struct tu_program_descriptor_linkage *link =
2799 &cmd->state.pipeline->program.link[MESA_SHADER_VERTEX];
2800 const struct ir3_const_state *const_state = &link->const_state;
2801 struct tu_cs cs;
2802
2803 if (const_state->offsets.driver_param >= link->constlen) {
2804 *entry = (struct tu_cs_entry) {};
2805 return VK_SUCCESS;
2806 }
2807
2808 VkResult result = tu_cs_begin_sub_stream(&cmd->sub_cs, 8, &cs);
2809 if (result != VK_SUCCESS)
2810 return result;
2811
2812 tu_cs_emit_pkt7(&cs, CP_LOAD_STATE6_GEOM, 3 + 4);
2813 tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(const_state->offsets.driver_param) |
2814 CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2815 CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
2816 CP_LOAD_STATE6_0_STATE_BLOCK(SB6_VS_SHADER) |
2817 CP_LOAD_STATE6_0_NUM_UNIT(1));
2818 tu_cs_emit(&cs, 0);
2819 tu_cs_emit(&cs, 0);
2820
2821 STATIC_ASSERT(IR3_DP_INSTID_BASE == 2);
2822
2823 tu_cs_emit(&cs, 0);
2824 tu_cs_emit(&cs, 0);
2825 tu_cs_emit(&cs, draw->first_instance);
2826 tu_cs_emit(&cs, 0);
2827
2828 *entry = tu_cs_end_sub_stream(&cmd->sub_cs, &cs);
2829 return VK_SUCCESS;
2830 }
2831
2832 static VkResult
2833 tu6_emit_textures(struct tu_cmd_buffer *cmd,
2834 const struct tu_pipeline *pipeline,
2835 struct tu_descriptor_state *descriptors_state,
2836 gl_shader_stage type,
2837 struct tu_cs_entry *entry,
2838 bool is_sysmem)
2839 {
2840 struct tu_cs *draw_state = &cmd->sub_cs;
2841 const struct tu_program_descriptor_linkage *link =
2842 &pipeline->program.link[type];
2843 VkResult result;
2844
2845 if (link->texture_map.num_desc == 0 && link->sampler_map.num_desc == 0) {
2846 *entry = (struct tu_cs_entry) {};
2847 return VK_SUCCESS;
2848 }
2849
2850 /* allocate and fill texture state */
2851 struct ts_cs_memory tex_const;
2852 result = tu_cs_alloc(draw_state, link->texture_map.num_desc,
2853 A6XX_TEX_CONST_DWORDS, &tex_const);
2854 if (result != VK_SUCCESS)
2855 return result;
2856
2857 int tex_index = 0;
2858 for (unsigned i = 0; i < link->texture_map.num; i++) {
2859 for (int j = 0; j < link->texture_map.array_size[i]; j++) {
2860 write_tex_const(cmd,
2861 &tex_const.map[A6XX_TEX_CONST_DWORDS * tex_index++],
2862 descriptors_state, &link->texture_map, i, j,
2863 is_sysmem);
2864 }
2865 }
2866
2867 /* allocate and fill sampler state */
2868 struct ts_cs_memory tex_samp = { 0 };
2869 if (link->sampler_map.num_desc) {
2870 result = tu_cs_alloc(draw_state, link->sampler_map.num_desc,
2871 A6XX_TEX_SAMP_DWORDS, &tex_samp);
2872 if (result != VK_SUCCESS)
2873 return result;
2874
2875 int sampler_index = 0;
2876 for (unsigned i = 0; i < link->sampler_map.num; i++) {
2877 for (int j = 0; j < link->sampler_map.array_size[i]; j++) {
2878 const uint32_t *sampler = sampler_ptr(descriptors_state,
2879 &link->sampler_map,
2880 i, j);
2881 memcpy(&tex_samp.map[A6XX_TEX_SAMP_DWORDS * sampler_index++],
2882 sampler, A6XX_TEX_SAMP_DWORDS * 4);
2883 }
2884 }
2885 }
2886
2887 unsigned tex_samp_reg, tex_const_reg, tex_count_reg;
2888 enum a6xx_state_block sb;
2889
2890 switch (type) {
2891 case MESA_SHADER_VERTEX:
2892 sb = SB6_VS_TEX;
2893 tex_samp_reg = REG_A6XX_SP_VS_TEX_SAMP_LO;
2894 tex_const_reg = REG_A6XX_SP_VS_TEX_CONST_LO;
2895 tex_count_reg = REG_A6XX_SP_VS_TEX_COUNT;
2896 break;
2897 case MESA_SHADER_FRAGMENT:
2898 sb = SB6_FS_TEX;
2899 tex_samp_reg = REG_A6XX_SP_FS_TEX_SAMP_LO;
2900 tex_const_reg = REG_A6XX_SP_FS_TEX_CONST_LO;
2901 tex_count_reg = REG_A6XX_SP_FS_TEX_COUNT;
2902 break;
2903 case MESA_SHADER_COMPUTE:
2904 sb = SB6_CS_TEX;
2905 tex_samp_reg = REG_A6XX_SP_CS_TEX_SAMP_LO;
2906 tex_const_reg = REG_A6XX_SP_CS_TEX_CONST_LO;
2907 tex_count_reg = REG_A6XX_SP_CS_TEX_COUNT;
2908 break;
2909 default:
2910 unreachable("bad state block");
2911 }
2912
2913 struct tu_cs cs;
2914 result = tu_cs_begin_sub_stream(draw_state, 16, &cs);
2915 if (result != VK_SUCCESS)
2916 return result;
2917
2918 if (link->sampler_map.num_desc) {
2919 /* output sampler state: */
2920 tu_cs_emit_pkt7(&cs, tu6_stage2opcode(type), 3);
2921 tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
2922 CP_LOAD_STATE6_0_STATE_TYPE(ST6_SHADER) |
2923 CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
2924 CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
2925 CP_LOAD_STATE6_0_NUM_UNIT(link->sampler_map.num_desc));
2926 tu_cs_emit_qw(&cs, tex_samp.iova); /* SRC_ADDR_LO/HI */
2927
2928 tu_cs_emit_pkt4(&cs, tex_samp_reg, 2);
2929 tu_cs_emit_qw(&cs, tex_samp.iova); /* SRC_ADDR_LO/HI */
2930 }
2931
2932 /* emit texture state: */
2933 tu_cs_emit_pkt7(&cs, tu6_stage2opcode(type), 3);
2934 tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
2935 CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2936 CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
2937 CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
2938 CP_LOAD_STATE6_0_NUM_UNIT(link->texture_map.num_desc));
2939 tu_cs_emit_qw(&cs, tex_const.iova); /* SRC_ADDR_LO/HI */
2940
2941 tu_cs_emit_pkt4(&cs, tex_const_reg, 2);
2942 tu_cs_emit_qw(&cs, tex_const.iova); /* SRC_ADDR_LO/HI */
2943
2944 tu_cs_emit_pkt4(&cs, tex_count_reg, 1);
2945 tu_cs_emit(&cs, link->texture_map.num_desc);
2946
2947 *entry = tu_cs_end_sub_stream(draw_state, &cs);
2948 return VK_SUCCESS;
2949 }
2950
2951 static VkResult
2952 tu6_emit_ibo(struct tu_cmd_buffer *cmd,
2953 const struct tu_pipeline *pipeline,
2954 struct tu_descriptor_state *descriptors_state,
2955 gl_shader_stage type,
2956 struct tu_cs_entry *entry)
2957 {
2958 struct tu_cs *draw_state = &cmd->sub_cs;
2959 const struct tu_program_descriptor_linkage *link =
2960 &pipeline->program.link[type];
2961 VkResult result;
2962
2963 unsigned num_desc = link->ssbo_map.num_desc + link->image_map.num_desc;
2964
2965 if (num_desc == 0) {
2966 *entry = (struct tu_cs_entry) {};
2967 return VK_SUCCESS;
2968 }
2969
2970 struct ts_cs_memory ibo_const;
2971 result = tu_cs_alloc(draw_state, num_desc,
2972 A6XX_TEX_CONST_DWORDS, &ibo_const);
2973 if (result != VK_SUCCESS)
2974 return result;
2975
2976 int ssbo_index = 0;
2977 for (unsigned i = 0; i < link->ssbo_map.num; i++) {
2978 for (int j = 0; j < link->ssbo_map.array_size[i]; j++) {
2979 uint32_t *dst = &ibo_const.map[A6XX_TEX_CONST_DWORDS * ssbo_index];
2980
2981 uint64_t va = buffer_ptr(descriptors_state, &link->ssbo_map, i, j);
2982 /* We don't expose robustBufferAccess, so leave the size unlimited. */
2983 uint32_t sz = MAX_STORAGE_BUFFER_RANGE / 4;
2984
2985 dst[0] = A6XX_IBO_0_FMT(FMT6_32_UINT);
2986 dst[1] = A6XX_IBO_1_WIDTH(sz & MASK(15)) |
2987 A6XX_IBO_1_HEIGHT(sz >> 15);
2988 dst[2] = A6XX_IBO_2_UNK4 |
2989 A6XX_IBO_2_UNK31 |
2990 A6XX_IBO_2_TYPE(A6XX_TEX_1D);
2991 dst[3] = 0;
2992 dst[4] = va;
2993 dst[5] = va >> 32;
2994 for (int i = 6; i < A6XX_TEX_CONST_DWORDS; i++)
2995 dst[i] = 0;
2996
2997 ssbo_index++;
2998 }
2999 }
3000
3001 for (unsigned i = 0; i < link->image_map.num; i++) {
3002 for (int j = 0; j < link->image_map.array_size[i]; j++) {
3003 uint32_t *dst = &ibo_const.map[A6XX_TEX_CONST_DWORDS * ssbo_index];
3004
3005 write_image_ibo(cmd, dst,
3006 descriptors_state, &link->image_map, i, j);
3007
3008 ssbo_index++;
3009 }
3010 }
3011
3012 assert(ssbo_index == num_desc);
3013
3014 struct tu_cs cs;
3015 result = tu_cs_begin_sub_stream(draw_state, 7, &cs);
3016 if (result != VK_SUCCESS)
3017 return result;
3018
3019 uint32_t opcode, ibo_addr_reg;
3020 enum a6xx_state_block sb;
3021 enum a6xx_state_type st;
3022
3023 switch (type) {
3024 case MESA_SHADER_FRAGMENT:
3025 opcode = CP_LOAD_STATE6;
3026 st = ST6_SHADER;
3027 sb = SB6_IBO;
3028 ibo_addr_reg = REG_A6XX_SP_IBO_LO;
3029 break;
3030 case MESA_SHADER_COMPUTE:
3031 opcode = CP_LOAD_STATE6_FRAG;
3032 st = ST6_IBO;
3033 sb = SB6_CS_SHADER;
3034 ibo_addr_reg = REG_A6XX_SP_CS_IBO_LO;
3035 break;
3036 default:
3037 unreachable("unsupported stage for ibos");
3038 }
3039
3040 /* emit texture state: */
3041 tu_cs_emit_pkt7(&cs, opcode, 3);
3042 tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
3043 CP_LOAD_STATE6_0_STATE_TYPE(st) |
3044 CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
3045 CP_LOAD_STATE6_0_STATE_BLOCK(sb) |
3046 CP_LOAD_STATE6_0_NUM_UNIT(num_desc));
3047 tu_cs_emit_qw(&cs, ibo_const.iova); /* SRC_ADDR_LO/HI */
3048
3049 tu_cs_emit_pkt4(&cs, ibo_addr_reg, 2);
3050 tu_cs_emit_qw(&cs, ibo_const.iova); /* SRC_ADDR_LO/HI */
3051
3052 *entry = tu_cs_end_sub_stream(draw_state, &cs);
3053 return VK_SUCCESS;
3054 }
3055
3056 static void
3057 tu6_emit_streamout(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
3058 {
3059 struct tu_streamout_state *tf = &cmd->state.pipeline->streamout;
3060
3061 for (unsigned i = 0; i < IR3_MAX_SO_BUFFERS; i++) {
3062 struct tu_buffer *buf = cmd->state.streamout_buf.buffers[i];
3063 if (!buf)
3064 continue;
3065
3066 uint32_t offset;
3067 offset = cmd->state.streamout_buf.offsets[i];
3068
3069 tu_cs_emit_regs(cs, A6XX_VPC_SO_BUFFER_BASE(i, .bo = buf->bo,
3070 .bo_offset = buf->bo_offset));
3071 tu_cs_emit_regs(cs, A6XX_VPC_SO_BUFFER_SIZE(i, buf->size));
3072
3073 if (cmd->state.streamout_reset & (1 << i)) {
3074 offset *= tf->stride[i];
3075
3076 tu_cs_emit_regs(cs, A6XX_VPC_SO_BUFFER_OFFSET(i, offset));
3077 cmd->state.streamout_reset &= ~(1 << i);
3078 } else {
3079 tu_cs_emit_pkt7(cs, CP_MEM_TO_REG, 3);
3080 tu_cs_emit(cs, CP_MEM_TO_REG_0_REG(REG_A6XX_VPC_SO_BUFFER_OFFSET(i)) |
3081 CP_MEM_TO_REG_0_SHIFT_BY_2 | CP_MEM_TO_REG_0_UNK31 |
3082 CP_MEM_TO_REG_0_CNT(0));
3083 tu_cs_emit_qw(cs, cmd->scratch_bo.iova +
3084 ctrl_offset(flush_base[i].offset));
3085 }
3086
3087 tu_cs_emit_regs(cs, A6XX_VPC_SO_FLUSH_BASE(i, .bo = &cmd->scratch_bo,
3088 .bo_offset =
3089 ctrl_offset(flush_base[i])));
3090 }
3091
3092 if (cmd->state.streamout_enabled) {
3093 tu_cs_emit_pkt7(cs, CP_CONTEXT_REG_BUNCH, 12 + (2 * tf->prog_count));
3094 tu_cs_emit(cs, REG_A6XX_VPC_SO_BUF_CNTL);
3095 tu_cs_emit(cs, tf->vpc_so_buf_cntl);
3096 tu_cs_emit(cs, REG_A6XX_VPC_SO_NCOMP(0));
3097 tu_cs_emit(cs, tf->ncomp[0]);
3098 tu_cs_emit(cs, REG_A6XX_VPC_SO_NCOMP(1));
3099 tu_cs_emit(cs, tf->ncomp[1]);
3100 tu_cs_emit(cs, REG_A6XX_VPC_SO_NCOMP(2));
3101 tu_cs_emit(cs, tf->ncomp[2]);
3102 tu_cs_emit(cs, REG_A6XX_VPC_SO_NCOMP(3));
3103 tu_cs_emit(cs, tf->ncomp[3]);
3104 tu_cs_emit(cs, REG_A6XX_VPC_SO_CNTL);
3105 tu_cs_emit(cs, A6XX_VPC_SO_CNTL_ENABLE);
3106 for (unsigned i = 0; i < tf->prog_count; i++) {
3107 tu_cs_emit(cs, REG_A6XX_VPC_SO_PROG);
3108 tu_cs_emit(cs, tf->prog[i]);
3109 }
3110 } else {
3111 tu_cs_emit_pkt7(cs, CP_CONTEXT_REG_BUNCH, 4);
3112 tu_cs_emit(cs, REG_A6XX_VPC_SO_CNTL);
3113 tu_cs_emit(cs, 0);
3114 tu_cs_emit(cs, REG_A6XX_VPC_SO_BUF_CNTL);
3115 tu_cs_emit(cs, 0);
3116 }
3117 }
3118
3119 static VkResult
3120 tu6_bind_draw_states(struct tu_cmd_buffer *cmd,
3121 struct tu_cs *cs,
3122 const struct tu_draw_info *draw)
3123 {
3124 const struct tu_pipeline *pipeline = cmd->state.pipeline;
3125 const struct tu_dynamic_state *dynamic = &cmd->state.dynamic;
3126 struct tu_draw_state_group draw_state_groups[TU_DRAW_STATE_COUNT];
3127 uint32_t draw_state_group_count = 0;
3128 VkResult result;
3129
3130 struct tu_descriptor_state *descriptors_state =
3131 &cmd->descriptors[VK_PIPELINE_BIND_POINT_GRAPHICS];
3132
3133 /* TODO lrz */
3134
3135 tu_cs_emit_regs(cs,
3136 A6XX_PC_PRIMITIVE_CNTL_0(.primitive_restart =
3137 pipeline->ia.primitive_restart && draw->indexed));
3138
3139 if (cmd->state.dirty &
3140 (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_DYNAMIC_LINE_WIDTH) &&
3141 (pipeline->dynamic_state.mask & TU_DYNAMIC_LINE_WIDTH)) {
3142 tu6_emit_gras_su_cntl(cs, pipeline->rast.gras_su_cntl,
3143 dynamic->line_width);
3144 }
3145
3146 if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK) &&
3147 (pipeline->dynamic_state.mask & TU_DYNAMIC_STENCIL_COMPARE_MASK)) {
3148 tu6_emit_stencil_compare_mask(cs, dynamic->stencil_compare_mask.front,
3149 dynamic->stencil_compare_mask.back);
3150 }
3151
3152 if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK) &&
3153 (pipeline->dynamic_state.mask & TU_DYNAMIC_STENCIL_WRITE_MASK)) {
3154 tu6_emit_stencil_write_mask(cs, dynamic->stencil_write_mask.front,
3155 dynamic->stencil_write_mask.back);
3156 }
3157
3158 if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE) &&
3159 (pipeline->dynamic_state.mask & TU_DYNAMIC_STENCIL_REFERENCE)) {
3160 tu6_emit_stencil_reference(cs, dynamic->stencil_reference.front,
3161 dynamic->stencil_reference.back);
3162 }
3163
3164 if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_VIEWPORT) &&
3165 (pipeline->dynamic_state.mask & TU_DYNAMIC_VIEWPORT)) {
3166 tu6_emit_viewport(cs, &cmd->state.dynamic.viewport.viewports[0]);
3167 }
3168
3169 if ((cmd->state.dirty & TU_CMD_DIRTY_DYNAMIC_SCISSOR) &&
3170 (pipeline->dynamic_state.mask & TU_DYNAMIC_SCISSOR)) {
3171 tu6_emit_scissor(cs, &cmd->state.dynamic.scissor.scissors[0]);
3172 }
3173
3174 if (cmd->state.dirty &
3175 (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_VERTEX_BUFFERS)) {
3176 for (uint32_t i = 0; i < pipeline->vi.count; i++) {
3177 const uint32_t binding = pipeline->vi.bindings[i];
3178 const struct tu_buffer *buf = cmd->state.vb.buffers[binding];
3179 const VkDeviceSize offset = buf->bo_offset +
3180 cmd->state.vb.offsets[binding];
3181 const VkDeviceSize size =
3182 offset < buf->size ? buf->size - offset : 0;
3183
3184 tu_cs_emit_regs(cs,
3185 A6XX_VFD_FETCH_BASE(i, .bo = buf->bo, .bo_offset = offset),
3186 A6XX_VFD_FETCH_SIZE(i, size));
3187 }
3188 }
3189
3190 if (cmd->state.dirty & TU_CMD_DIRTY_PIPELINE) {
3191 draw_state_groups[draw_state_group_count++] =
3192 (struct tu_draw_state_group) {
3193 .id = TU_DRAW_STATE_PROGRAM,
3194 .enable_mask = ENABLE_DRAW,
3195 .ib = pipeline->program.state_ib,
3196 };
3197 draw_state_groups[draw_state_group_count++] =
3198 (struct tu_draw_state_group) {
3199 .id = TU_DRAW_STATE_PROGRAM_BINNING,
3200 .enable_mask = CP_SET_DRAW_STATE__0_BINNING,
3201 .ib = pipeline->program.binning_state_ib,
3202 };
3203 draw_state_groups[draw_state_group_count++] =
3204 (struct tu_draw_state_group) {
3205 .id = TU_DRAW_STATE_VI,
3206 .enable_mask = ENABLE_DRAW,
3207 .ib = pipeline->vi.state_ib,
3208 };
3209 draw_state_groups[draw_state_group_count++] =
3210 (struct tu_draw_state_group) {
3211 .id = TU_DRAW_STATE_VI_BINNING,
3212 .enable_mask = CP_SET_DRAW_STATE__0_BINNING,
3213 .ib = pipeline->vi.binning_state_ib,
3214 };
3215 draw_state_groups[draw_state_group_count++] =
3216 (struct tu_draw_state_group) {
3217 .id = TU_DRAW_STATE_VP,
3218 .enable_mask = ENABLE_ALL,
3219 .ib = pipeline->vp.state_ib,
3220 };
3221 draw_state_groups[draw_state_group_count++] =
3222 (struct tu_draw_state_group) {
3223 .id = TU_DRAW_STATE_RAST,
3224 .enable_mask = ENABLE_ALL,
3225 .ib = pipeline->rast.state_ib,
3226 };
3227 draw_state_groups[draw_state_group_count++] =
3228 (struct tu_draw_state_group) {
3229 .id = TU_DRAW_STATE_DS,
3230 .enable_mask = ENABLE_ALL,
3231 .ib = pipeline->ds.state_ib,
3232 };
3233 draw_state_groups[draw_state_group_count++] =
3234 (struct tu_draw_state_group) {
3235 .id = TU_DRAW_STATE_BLEND,
3236 .enable_mask = ENABLE_ALL,
3237 .ib = pipeline->blend.state_ib,
3238 };
3239 }
3240
3241 if (cmd->state.dirty &
3242 (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_DESCRIPTOR_SETS | TU_CMD_DIRTY_PUSH_CONSTANTS)) {
3243 draw_state_groups[draw_state_group_count++] =
3244 (struct tu_draw_state_group) {
3245 .id = TU_DRAW_STATE_VS_CONST,
3246 .enable_mask = ENABLE_ALL,
3247 .ib = tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_VERTEX)
3248 };
3249 draw_state_groups[draw_state_group_count++] =
3250 (struct tu_draw_state_group) {
3251 .id = TU_DRAW_STATE_FS_CONST,
3252 .enable_mask = ENABLE_DRAW,
3253 .ib = tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_FRAGMENT)
3254 };
3255 }
3256
3257 if (cmd->state.dirty & TU_CMD_DIRTY_STREAMOUT_BUFFERS)
3258 tu6_emit_streamout(cmd, cs);
3259
3260 if (cmd->state.dirty &
3261 (TU_CMD_DIRTY_PIPELINE | TU_CMD_DIRTY_DESCRIPTOR_SETS)) {
3262 struct tu_cs_entry vs_tex, fs_tex_sysmem, fs_tex_gmem, fs_ibo;
3263
3264 result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3265 MESA_SHADER_VERTEX, &vs_tex, false);
3266 if (result != VK_SUCCESS)
3267 return result;
3268
3269 /* TODO: we could emit just one texture descriptor draw state when there
3270 * are no input attachments, which is the most common case. We could
3271 * also split out the sampler state, which doesn't change even for input
3272 * attachments.
3273 */
3274 result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3275 MESA_SHADER_FRAGMENT, &fs_tex_sysmem, true);
3276 if (result != VK_SUCCESS)
3277 return result;
3278
3279 result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3280 MESA_SHADER_FRAGMENT, &fs_tex_gmem, false);
3281 if (result != VK_SUCCESS)
3282 return result;
3283
3284 result = tu6_emit_ibo(cmd, pipeline, descriptors_state,
3285 MESA_SHADER_FRAGMENT, &fs_ibo);
3286 if (result != VK_SUCCESS)
3287 return result;
3288
3289 draw_state_groups[draw_state_group_count++] =
3290 (struct tu_draw_state_group) {
3291 .id = TU_DRAW_STATE_VS_TEX,
3292 .enable_mask = ENABLE_ALL,
3293 .ib = vs_tex,
3294 };
3295 draw_state_groups[draw_state_group_count++] =
3296 (struct tu_draw_state_group) {
3297 .id = TU_DRAW_STATE_FS_TEX_GMEM,
3298 .enable_mask = CP_SET_DRAW_STATE__0_GMEM,
3299 .ib = fs_tex_gmem,
3300 };
3301 draw_state_groups[draw_state_group_count++] =
3302 (struct tu_draw_state_group) {
3303 .id = TU_DRAW_STATE_FS_TEX_SYSMEM,
3304 .enable_mask = CP_SET_DRAW_STATE__0_SYSMEM,
3305 .ib = fs_tex_sysmem,
3306 };
3307 draw_state_groups[draw_state_group_count++] =
3308 (struct tu_draw_state_group) {
3309 .id = TU_DRAW_STATE_FS_IBO,
3310 .enable_mask = ENABLE_DRAW,
3311 .ib = fs_ibo,
3312 };
3313 }
3314
3315 struct tu_cs_entry vs_params;
3316 result = tu6_emit_vs_params(cmd, draw, &vs_params);
3317 if (result != VK_SUCCESS)
3318 return result;
3319
3320 draw_state_groups[draw_state_group_count++] =
3321 (struct tu_draw_state_group) {
3322 .id = TU_DRAW_STATE_VS_PARAMS,
3323 .enable_mask = ENABLE_ALL,
3324 .ib = vs_params,
3325 };
3326
3327 tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3 * draw_state_group_count);
3328 for (uint32_t i = 0; i < draw_state_group_count; i++) {
3329 const struct tu_draw_state_group *group = &draw_state_groups[i];
3330 debug_assert((group->enable_mask & ~ENABLE_ALL) == 0);
3331 uint32_t cp_set_draw_state =
3332 CP_SET_DRAW_STATE__0_COUNT(group->ib.size / 4) |
3333 group->enable_mask |
3334 CP_SET_DRAW_STATE__0_GROUP_ID(group->id);
3335 uint64_t iova;
3336 if (group->ib.size) {
3337 iova = group->ib.bo->iova + group->ib.offset;
3338 } else {
3339 cp_set_draw_state |= CP_SET_DRAW_STATE__0_DISABLE;
3340 iova = 0;
3341 }
3342
3343 tu_cs_emit(cs, cp_set_draw_state);
3344 tu_cs_emit_qw(cs, iova);
3345 }
3346
3347 tu_cs_sanity_check(cs);
3348
3349 /* track BOs */
3350 if (cmd->state.dirty & TU_CMD_DIRTY_VERTEX_BUFFERS) {
3351 for (uint32_t i = 0; i < MAX_VBS; i++) {
3352 const struct tu_buffer *buf = cmd->state.vb.buffers[i];
3353 if (buf)
3354 tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
3355 }
3356 }
3357 if (cmd->state.dirty & TU_CMD_DIRTY_DESCRIPTOR_SETS) {
3358 unsigned i;
3359 for_each_bit(i, descriptors_state->valid) {
3360 struct tu_descriptor_set *set = descriptors_state->sets[i];
3361 for (unsigned j = 0; j < set->layout->buffer_count; ++j)
3362 if (set->descriptors[j]) {
3363 tu_bo_list_add(&cmd->bo_list, set->descriptors[j],
3364 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3365 }
3366 }
3367 }
3368 if (cmd->state.dirty & TU_CMD_DIRTY_STREAMOUT_BUFFERS) {
3369 for (unsigned i = 0; i < IR3_MAX_SO_BUFFERS; i++) {
3370 const struct tu_buffer *buf = cmd->state.streamout_buf.buffers[i];
3371 if (buf) {
3372 tu_bo_list_add(&cmd->bo_list, buf->bo,
3373 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3374 }
3375 }
3376 }
3377
3378 /* Fragment shader state overwrites compute shader state, so flag the
3379 * compute pipeline for re-emit.
3380 */
3381 cmd->state.dirty = TU_CMD_DIRTY_COMPUTE_PIPELINE;
3382 return VK_SUCCESS;
3383 }
3384
3385 static void
3386 tu6_emit_draw_indirect(struct tu_cmd_buffer *cmd,
3387 struct tu_cs *cs,
3388 const struct tu_draw_info *draw)
3389 {
3390 const enum pc_di_primtype primtype = cmd->state.pipeline->ia.primtype;
3391 bool has_gs = cmd->state.pipeline->active_stages &
3392 VK_SHADER_STAGE_GEOMETRY_BIT;
3393
3394 tu_cs_emit_regs(cs,
3395 A6XX_VFD_INDEX_OFFSET(draw->vertex_offset),
3396 A6XX_VFD_INSTANCE_START_OFFSET(draw->first_instance));
3397
3398 if (draw->indexed) {
3399 const enum a4xx_index_size index_size =
3400 tu6_index_size(cmd->state.index_type);
3401 const uint32_t index_bytes =
3402 (cmd->state.index_type == VK_INDEX_TYPE_UINT32) ? 4 : 2;
3403 const struct tu_buffer *index_buf = cmd->state.index_buffer;
3404 unsigned max_indicies =
3405 (index_buf->size - cmd->state.index_offset) / index_bytes;
3406
3407 const uint32_t cp_draw_indx =
3408 CP_DRAW_INDX_OFFSET_0_PRIM_TYPE(primtype) |
3409 CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT(DI_SRC_SEL_DMA) |
3410 CP_DRAW_INDX_OFFSET_0_INDEX_SIZE(index_size) |
3411 CP_DRAW_INDX_OFFSET_0_VIS_CULL(USE_VISIBILITY) |
3412 COND(has_gs, CP_DRAW_INDX_OFFSET_0_GS_ENABLE) | 0x2000;
3413
3414 tu_cs_emit_pkt7(cs, CP_DRAW_INDX_INDIRECT, 6);
3415 tu_cs_emit(cs, cp_draw_indx);
3416 tu_cs_emit_qw(cs, index_buf->bo->iova + cmd->state.index_offset);
3417 tu_cs_emit(cs, A5XX_CP_DRAW_INDX_INDIRECT_3_MAX_INDICES(max_indicies));
3418 tu_cs_emit_qw(cs, draw->indirect->bo->iova + draw->indirect_offset);
3419 } else {
3420 const uint32_t cp_draw_indx =
3421 CP_DRAW_INDX_OFFSET_0_PRIM_TYPE(primtype) |
3422 CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT(DI_SRC_SEL_AUTO_INDEX) |
3423 CP_DRAW_INDX_OFFSET_0_VIS_CULL(USE_VISIBILITY) |
3424 COND(has_gs, CP_DRAW_INDX_OFFSET_0_GS_ENABLE) | 0x2000;
3425
3426 tu_cs_emit_pkt7(cs, CP_DRAW_INDIRECT, 3);
3427 tu_cs_emit(cs, cp_draw_indx);
3428 tu_cs_emit_qw(cs, draw->indirect->bo->iova + draw->indirect_offset);
3429 }
3430
3431 tu_bo_list_add(&cmd->bo_list, draw->indirect->bo, MSM_SUBMIT_BO_READ);
3432 }
3433
3434 static void
3435 tu6_emit_draw_direct(struct tu_cmd_buffer *cmd,
3436 struct tu_cs *cs,
3437 const struct tu_draw_info *draw)
3438 {
3439
3440 const enum pc_di_primtype primtype = cmd->state.pipeline->ia.primtype;
3441 bool has_gs = cmd->state.pipeline->active_stages &
3442 VK_SHADER_STAGE_GEOMETRY_BIT;
3443
3444 tu_cs_emit_regs(cs,
3445 A6XX_VFD_INDEX_OFFSET(draw->vertex_offset),
3446 A6XX_VFD_INSTANCE_START_OFFSET(draw->first_instance));
3447
3448 /* TODO hw binning */
3449 if (draw->indexed) {
3450 const enum a4xx_index_size index_size =
3451 tu6_index_size(cmd->state.index_type);
3452 const uint32_t index_bytes =
3453 (cmd->state.index_type == VK_INDEX_TYPE_UINT32) ? 4 : 2;
3454 const struct tu_buffer *buf = cmd->state.index_buffer;
3455 const VkDeviceSize offset = buf->bo_offset + cmd->state.index_offset +
3456 index_bytes * draw->first_index;
3457 const uint32_t size = index_bytes * draw->count;
3458
3459 const uint32_t cp_draw_indx =
3460 CP_DRAW_INDX_OFFSET_0_PRIM_TYPE(primtype) |
3461 CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT(DI_SRC_SEL_DMA) |
3462 CP_DRAW_INDX_OFFSET_0_INDEX_SIZE(index_size) |
3463 CP_DRAW_INDX_OFFSET_0_VIS_CULL(USE_VISIBILITY) |
3464 COND(has_gs, CP_DRAW_INDX_OFFSET_0_GS_ENABLE) | 0x2000;
3465
3466 tu_cs_emit_pkt7(cs, CP_DRAW_INDX_OFFSET, 7);
3467 tu_cs_emit(cs, cp_draw_indx);
3468 tu_cs_emit(cs, draw->instance_count);
3469 tu_cs_emit(cs, draw->count);
3470 tu_cs_emit(cs, 0x0); /* XXX */
3471 tu_cs_emit_qw(cs, buf->bo->iova + offset);
3472 tu_cs_emit(cs, size);
3473 } else {
3474 const uint32_t cp_draw_indx =
3475 CP_DRAW_INDX_OFFSET_0_PRIM_TYPE(primtype) |
3476 CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT(DI_SRC_SEL_AUTO_INDEX) |
3477 CP_DRAW_INDX_OFFSET_0_VIS_CULL(USE_VISIBILITY) |
3478 COND(has_gs, CP_DRAW_INDX_OFFSET_0_GS_ENABLE) | 0x2000;
3479
3480 tu_cs_emit_pkt7(cs, CP_DRAW_INDX_OFFSET, 3);
3481 tu_cs_emit(cs, cp_draw_indx);
3482 tu_cs_emit(cs, draw->instance_count);
3483 tu_cs_emit(cs, draw->count);
3484 }
3485 }
3486
3487 static void
3488 tu_draw(struct tu_cmd_buffer *cmd, const struct tu_draw_info *draw)
3489 {
3490 struct tu_cs *cs = &cmd->draw_cs;
3491 VkResult result;
3492
3493 result = tu6_bind_draw_states(cmd, cs, draw);
3494 if (result != VK_SUCCESS) {
3495 cmd->record_result = result;
3496 return;
3497 }
3498
3499 if (draw->indirect)
3500 tu6_emit_draw_indirect(cmd, cs, draw);
3501 else
3502 tu6_emit_draw_direct(cmd, cs, draw);
3503
3504 if (cmd->state.streamout_enabled) {
3505 for (unsigned i = 0; i < IR3_MAX_SO_BUFFERS; i++) {
3506 if (cmd->state.streamout_enabled & (1 << i))
3507 tu6_emit_event_write(cmd, cs, FLUSH_SO_0 + i, false);
3508 }
3509 }
3510
3511 cmd->wait_for_idle = true;
3512
3513 tu_cs_sanity_check(cs);
3514 }
3515
3516 void
3517 tu_CmdDraw(VkCommandBuffer commandBuffer,
3518 uint32_t vertexCount,
3519 uint32_t instanceCount,
3520 uint32_t firstVertex,
3521 uint32_t firstInstance)
3522 {
3523 TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3524 struct tu_draw_info info = {};
3525
3526 info.count = vertexCount;
3527 info.instance_count = instanceCount;
3528 info.first_instance = firstInstance;
3529 info.vertex_offset = firstVertex;
3530
3531 tu_draw(cmd_buffer, &info);
3532 }
3533
3534 void
3535 tu_CmdDrawIndexed(VkCommandBuffer commandBuffer,
3536 uint32_t indexCount,
3537 uint32_t instanceCount,
3538 uint32_t firstIndex,
3539 int32_t vertexOffset,
3540 uint32_t firstInstance)
3541 {
3542 TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3543 struct tu_draw_info info = {};
3544
3545 info.indexed = true;
3546 info.count = indexCount;
3547 info.instance_count = instanceCount;
3548 info.first_index = firstIndex;
3549 info.vertex_offset = vertexOffset;
3550 info.first_instance = firstInstance;
3551
3552 tu_draw(cmd_buffer, &info);
3553 }
3554
3555 void
3556 tu_CmdDrawIndirect(VkCommandBuffer commandBuffer,
3557 VkBuffer _buffer,
3558 VkDeviceSize offset,
3559 uint32_t drawCount,
3560 uint32_t stride)
3561 {
3562 TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3563 TU_FROM_HANDLE(tu_buffer, buffer, _buffer);
3564 struct tu_draw_info info = {};
3565
3566 info.count = drawCount;
3567 info.indirect = buffer;
3568 info.indirect_offset = offset;
3569 info.stride = stride;
3570
3571 tu_draw(cmd_buffer, &info);
3572 }
3573
3574 void
3575 tu_CmdDrawIndexedIndirect(VkCommandBuffer commandBuffer,
3576 VkBuffer _buffer,
3577 VkDeviceSize offset,
3578 uint32_t drawCount,
3579 uint32_t stride)
3580 {
3581 TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3582 TU_FROM_HANDLE(tu_buffer, buffer, _buffer);
3583 struct tu_draw_info info = {};
3584
3585 info.indexed = true;
3586 info.count = drawCount;
3587 info.indirect = buffer;
3588 info.indirect_offset = offset;
3589 info.stride = stride;
3590
3591 tu_draw(cmd_buffer, &info);
3592 }
3593
3594 void tu_CmdDrawIndirectByteCountEXT(VkCommandBuffer commandBuffer,
3595 uint32_t instanceCount,
3596 uint32_t firstInstance,
3597 VkBuffer _counterBuffer,
3598 VkDeviceSize counterBufferOffset,
3599 uint32_t counterOffset,
3600 uint32_t vertexStride)
3601 {
3602 TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3603 TU_FROM_HANDLE(tu_buffer, buffer, _counterBuffer);
3604
3605 struct tu_draw_info info = {};
3606
3607 info.instance_count = instanceCount;
3608 info.first_instance = firstInstance;
3609 info.streamout_buffer = buffer;
3610 info.streamout_buffer_offset = counterBufferOffset;
3611 info.stride = vertexStride;
3612
3613 tu_draw(cmd_buffer, &info);
3614 }
3615
3616 struct tu_dispatch_info
3617 {
3618 /**
3619 * Determine the layout of the grid (in block units) to be used.
3620 */
3621 uint32_t blocks[3];
3622
3623 /**
3624 * A starting offset for the grid. If unaligned is set, the offset
3625 * must still be aligned.
3626 */
3627 uint32_t offsets[3];
3628 /**
3629 * Whether it's an unaligned compute dispatch.
3630 */
3631 bool unaligned;
3632
3633 /**
3634 * Indirect compute parameters resource.
3635 */
3636 struct tu_buffer *indirect;
3637 uint64_t indirect_offset;
3638 };
3639
3640 static void
3641 tu_emit_compute_driver_params(struct tu_cs *cs, struct tu_pipeline *pipeline,
3642 const struct tu_dispatch_info *info)
3643 {
3644 gl_shader_stage type = MESA_SHADER_COMPUTE;
3645 const struct tu_program_descriptor_linkage *link =
3646 &pipeline->program.link[type];
3647 const struct ir3_const_state *const_state = &link->const_state;
3648 uint32_t offset = const_state->offsets.driver_param;
3649
3650 if (link->constlen <= offset)
3651 return;
3652
3653 if (!info->indirect) {
3654 uint32_t driver_params[IR3_DP_CS_COUNT] = {
3655 [IR3_DP_NUM_WORK_GROUPS_X] = info->blocks[0],
3656 [IR3_DP_NUM_WORK_GROUPS_Y] = info->blocks[1],
3657 [IR3_DP_NUM_WORK_GROUPS_Z] = info->blocks[2],
3658 [IR3_DP_LOCAL_GROUP_SIZE_X] = pipeline->compute.local_size[0],
3659 [IR3_DP_LOCAL_GROUP_SIZE_Y] = pipeline->compute.local_size[1],
3660 [IR3_DP_LOCAL_GROUP_SIZE_Z] = pipeline->compute.local_size[2],
3661 };
3662
3663 uint32_t num_consts = MIN2(const_state->num_driver_params,
3664 (link->constlen - offset) * 4);
3665 /* push constants */
3666 tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + num_consts);
3667 tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(offset) |
3668 CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
3669 CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
3670 CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
3671 CP_LOAD_STATE6_0_NUM_UNIT(num_consts / 4));
3672 tu_cs_emit(cs, 0);
3673 tu_cs_emit(cs, 0);
3674 uint32_t i;
3675 for (i = 0; i < num_consts; i++)
3676 tu_cs_emit(cs, driver_params[i]);
3677 } else {
3678 tu_finishme("Indirect driver params");
3679 }
3680 }
3681
3682 static void
3683 tu_dispatch(struct tu_cmd_buffer *cmd,
3684 const struct tu_dispatch_info *info)
3685 {
3686 struct tu_cs *cs = &cmd->cs;
3687 struct tu_pipeline *pipeline = cmd->state.compute_pipeline;
3688 struct tu_descriptor_state *descriptors_state =
3689 &cmd->descriptors[VK_PIPELINE_BIND_POINT_COMPUTE];
3690 VkResult result;
3691
3692 if (cmd->state.dirty & TU_CMD_DIRTY_COMPUTE_PIPELINE)
3693 tu_cs_emit_ib(cs, &pipeline->program.state_ib);
3694
3695 struct tu_cs_entry ib;
3696
3697 ib = tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_COMPUTE);
3698 if (ib.size)
3699 tu_cs_emit_ib(cs, &ib);
3700
3701 tu_emit_compute_driver_params(cs, pipeline, info);
3702
3703 result = tu6_emit_textures(cmd, pipeline, descriptors_state,
3704 MESA_SHADER_COMPUTE, &ib, false);
3705 if (result != VK_SUCCESS) {
3706 cmd->record_result = result;
3707 return;
3708 }
3709
3710 if (ib.size)
3711 tu_cs_emit_ib(cs, &ib);
3712
3713 result = tu6_emit_ibo(cmd, pipeline, descriptors_state, MESA_SHADER_COMPUTE, &ib);
3714 if (result != VK_SUCCESS) {
3715 cmd->record_result = result;
3716 return;
3717 }
3718
3719 if (ib.size)
3720 tu_cs_emit_ib(cs, &ib);
3721
3722 /* track BOs */
3723 if (cmd->state.dirty & TU_CMD_DIRTY_DESCRIPTOR_SETS) {
3724 unsigned i;
3725 for_each_bit(i, descriptors_state->valid) {
3726 struct tu_descriptor_set *set = descriptors_state->sets[i];
3727 for (unsigned j = 0; j < set->layout->buffer_count; ++j)
3728 if (set->descriptors[j]) {
3729 tu_bo_list_add(&cmd->bo_list, set->descriptors[j],
3730 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3731 }
3732 }
3733 }
3734
3735 /* Compute shader state overwrites fragment shader state, so we flag the
3736 * graphics pipeline for re-emit.
3737 */
3738 cmd->state.dirty = TU_CMD_DIRTY_PIPELINE;
3739
3740 tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
3741 tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_COMPUTE));
3742
3743 const uint32_t *local_size = pipeline->compute.local_size;
3744 const uint32_t *num_groups = info->blocks;
3745 tu_cs_emit_regs(cs,
3746 A6XX_HLSQ_CS_NDRANGE_0(.kerneldim = 3,
3747 .localsizex = local_size[0] - 1,
3748 .localsizey = local_size[1] - 1,
3749 .localsizez = local_size[2] - 1),
3750 A6XX_HLSQ_CS_NDRANGE_1(.globalsize_x = local_size[0] * num_groups[0]),
3751 A6XX_HLSQ_CS_NDRANGE_2(.globaloff_x = 0),
3752 A6XX_HLSQ_CS_NDRANGE_3(.globalsize_y = local_size[1] * num_groups[1]),
3753 A6XX_HLSQ_CS_NDRANGE_4(.globaloff_y = 0),
3754 A6XX_HLSQ_CS_NDRANGE_5(.globalsize_z = local_size[2] * num_groups[2]),
3755 A6XX_HLSQ_CS_NDRANGE_6(.globaloff_z = 0));
3756
3757 tu_cs_emit_regs(cs,
3758 A6XX_HLSQ_CS_KERNEL_GROUP_X(1),
3759 A6XX_HLSQ_CS_KERNEL_GROUP_Y(1),
3760 A6XX_HLSQ_CS_KERNEL_GROUP_Z(1));
3761
3762 if (info->indirect) {
3763 uint64_t iova = tu_buffer_iova(info->indirect) + info->indirect_offset;
3764
3765 tu_bo_list_add(&cmd->bo_list, info->indirect->bo,
3766 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3767
3768 tu_cs_emit_pkt7(cs, CP_EXEC_CS_INDIRECT, 4);
3769 tu_cs_emit(cs, 0x00000000);
3770 tu_cs_emit_qw(cs, iova);
3771 tu_cs_emit(cs,
3772 A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX(local_size[0] - 1) |
3773 A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY(local_size[1] - 1) |
3774 A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ(local_size[2] - 1));
3775 } else {
3776 tu_cs_emit_pkt7(cs, CP_EXEC_CS, 4);
3777 tu_cs_emit(cs, 0x00000000);
3778 tu_cs_emit(cs, CP_EXEC_CS_1_NGROUPS_X(info->blocks[0]));
3779 tu_cs_emit(cs, CP_EXEC_CS_2_NGROUPS_Y(info->blocks[1]));
3780 tu_cs_emit(cs, CP_EXEC_CS_3_NGROUPS_Z(info->blocks[2]));
3781 }
3782
3783 tu_cs_emit_wfi(cs);
3784
3785 tu6_emit_cache_flush(cmd, cs);
3786 }
3787
3788 void
3789 tu_CmdDispatchBase(VkCommandBuffer commandBuffer,
3790 uint32_t base_x,
3791 uint32_t base_y,
3792 uint32_t base_z,
3793 uint32_t x,
3794 uint32_t y,
3795 uint32_t z)
3796 {
3797 TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3798 struct tu_dispatch_info info = {};
3799
3800 info.blocks[0] = x;
3801 info.blocks[1] = y;
3802 info.blocks[2] = z;
3803
3804 info.offsets[0] = base_x;
3805 info.offsets[1] = base_y;
3806 info.offsets[2] = base_z;
3807 tu_dispatch(cmd_buffer, &info);
3808 }
3809
3810 void
3811 tu_CmdDispatch(VkCommandBuffer commandBuffer,
3812 uint32_t x,
3813 uint32_t y,
3814 uint32_t z)
3815 {
3816 tu_CmdDispatchBase(commandBuffer, 0, 0, 0, x, y, z);
3817 }
3818
3819 void
3820 tu_CmdDispatchIndirect(VkCommandBuffer commandBuffer,
3821 VkBuffer _buffer,
3822 VkDeviceSize offset)
3823 {
3824 TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3825 TU_FROM_HANDLE(tu_buffer, buffer, _buffer);
3826 struct tu_dispatch_info info = {};
3827
3828 info.indirect = buffer;
3829 info.indirect_offset = offset;
3830
3831 tu_dispatch(cmd_buffer, &info);
3832 }
3833
3834 void
3835 tu_CmdEndRenderPass(VkCommandBuffer commandBuffer)
3836 {
3837 TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3838
3839 tu_cs_end(&cmd_buffer->draw_cs);
3840 tu_cs_end(&cmd_buffer->draw_epilogue_cs);
3841
3842 if (use_sysmem_rendering(cmd_buffer))
3843 tu_cmd_render_sysmem(cmd_buffer);
3844 else
3845 tu_cmd_render_tiles(cmd_buffer);
3846
3847 /* discard draw_cs and draw_epilogue_cs entries now that the tiles are
3848 rendered */
3849 tu_cs_discard_entries(&cmd_buffer->draw_cs);
3850 tu_cs_begin(&cmd_buffer->draw_cs);
3851 tu_cs_discard_entries(&cmd_buffer->draw_epilogue_cs);
3852 tu_cs_begin(&cmd_buffer->draw_epilogue_cs);
3853
3854 cmd_buffer->state.pass = NULL;
3855 cmd_buffer->state.subpass = NULL;
3856 cmd_buffer->state.framebuffer = NULL;
3857 }
3858
3859 void
3860 tu_CmdEndRenderPass2(VkCommandBuffer commandBuffer,
3861 const VkSubpassEndInfoKHR *pSubpassEndInfo)
3862 {
3863 tu_CmdEndRenderPass(commandBuffer);
3864 }
3865
3866 struct tu_barrier_info
3867 {
3868 uint32_t eventCount;
3869 const VkEvent *pEvents;
3870 VkPipelineStageFlags srcStageMask;
3871 };
3872
3873 static void
3874 tu_barrier(struct tu_cmd_buffer *cmd,
3875 uint32_t memoryBarrierCount,
3876 const VkMemoryBarrier *pMemoryBarriers,
3877 uint32_t bufferMemoryBarrierCount,
3878 const VkBufferMemoryBarrier *pBufferMemoryBarriers,
3879 uint32_t imageMemoryBarrierCount,
3880 const VkImageMemoryBarrier *pImageMemoryBarriers,
3881 const struct tu_barrier_info *info)
3882 {
3883 /* renderpass case is only for subpass self-dependencies
3884 * which means syncing the render output with texture cache
3885 * note: only the CACHE_INVALIDATE is needed in GMEM mode
3886 * and in sysmem mode we might not need either color/depth flush
3887 */
3888 if (cmd->state.pass) {
3889 tu6_emit_event_write(cmd, &cmd->draw_cs, PC_CCU_FLUSH_COLOR_TS, true);
3890 tu6_emit_event_write(cmd, &cmd->draw_cs, PC_CCU_FLUSH_DEPTH_TS, true);
3891 tu6_emit_event_write(cmd, &cmd->draw_cs, CACHE_INVALIDATE, false);
3892 return;
3893 }
3894 }
3895
3896 void
3897 tu_CmdPipelineBarrier(VkCommandBuffer commandBuffer,
3898 VkPipelineStageFlags srcStageMask,
3899 VkPipelineStageFlags dstStageMask,
3900 VkDependencyFlags dependencyFlags,
3901 uint32_t memoryBarrierCount,
3902 const VkMemoryBarrier *pMemoryBarriers,
3903 uint32_t bufferMemoryBarrierCount,
3904 const VkBufferMemoryBarrier *pBufferMemoryBarriers,
3905 uint32_t imageMemoryBarrierCount,
3906 const VkImageMemoryBarrier *pImageMemoryBarriers)
3907 {
3908 TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
3909 struct tu_barrier_info info;
3910
3911 info.eventCount = 0;
3912 info.pEvents = NULL;
3913 info.srcStageMask = srcStageMask;
3914
3915 tu_barrier(cmd_buffer, memoryBarrierCount, pMemoryBarriers,
3916 bufferMemoryBarrierCount, pBufferMemoryBarriers,
3917 imageMemoryBarrierCount, pImageMemoryBarriers, &info);
3918 }
3919
3920 static void
3921 write_event(struct tu_cmd_buffer *cmd, struct tu_event *event, unsigned value)
3922 {
3923 struct tu_cs *cs = &cmd->cs;
3924
3925 tu_bo_list_add(&cmd->bo_list, &event->bo, MSM_SUBMIT_BO_WRITE);
3926
3927 /* TODO: any flush required before/after ? */
3928
3929 tu_cs_emit_pkt7(cs, CP_MEM_WRITE, 3);
3930 tu_cs_emit_qw(cs, event->bo.iova); /* ADDR_LO/HI */
3931 tu_cs_emit(cs, value);
3932 }
3933
3934 void
3935 tu_CmdSetEvent(VkCommandBuffer commandBuffer,
3936 VkEvent _event,
3937 VkPipelineStageFlags stageMask)
3938 {
3939 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
3940 TU_FROM_HANDLE(tu_event, event, _event);
3941
3942 write_event(cmd, event, 1);
3943 }
3944
3945 void
3946 tu_CmdResetEvent(VkCommandBuffer commandBuffer,
3947 VkEvent _event,
3948 VkPipelineStageFlags stageMask)
3949 {
3950 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
3951 TU_FROM_HANDLE(tu_event, event, _event);
3952
3953 write_event(cmd, event, 0);
3954 }
3955
3956 void
3957 tu_CmdWaitEvents(VkCommandBuffer commandBuffer,
3958 uint32_t eventCount,
3959 const VkEvent *pEvents,
3960 VkPipelineStageFlags srcStageMask,
3961 VkPipelineStageFlags dstStageMask,
3962 uint32_t memoryBarrierCount,
3963 const VkMemoryBarrier *pMemoryBarriers,
3964 uint32_t bufferMemoryBarrierCount,
3965 const VkBufferMemoryBarrier *pBufferMemoryBarriers,
3966 uint32_t imageMemoryBarrierCount,
3967 const VkImageMemoryBarrier *pImageMemoryBarriers)
3968 {
3969 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
3970 struct tu_cs *cs = &cmd->cs;
3971
3972 /* TODO: any flush required before/after? (CP_WAIT_FOR_ME?) */
3973
3974 for (uint32_t i = 0; i < eventCount; i++) {
3975 TU_FROM_HANDLE(tu_event, event, pEvents[i]);
3976
3977 tu_bo_list_add(&cmd->bo_list, &event->bo, MSM_SUBMIT_BO_READ);
3978
3979 tu_cs_emit_pkt7(cs, CP_WAIT_REG_MEM, 6);
3980 tu_cs_emit(cs, CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ) |
3981 CP_WAIT_REG_MEM_0_POLL_MEMORY);
3982 tu_cs_emit_qw(cs, event->bo.iova); /* POLL_ADDR_LO/HI */
3983 tu_cs_emit(cs, CP_WAIT_REG_MEM_3_REF(1));
3984 tu_cs_emit(cs, CP_WAIT_REG_MEM_4_MASK(~0u));
3985 tu_cs_emit(cs, CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(20));
3986 }
3987 }
3988
3989 void
3990 tu_CmdSetDeviceMask(VkCommandBuffer commandBuffer, uint32_t deviceMask)
3991 {
3992 /* No-op */
3993 }