36d0e462edc24f1ae94e52c35dc42dc7fd931c5e
[mesa.git] / src / freedreno / vulkan / tu_cmd_buffer.c
1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 *
5 * based in part on anv driver which is:
6 * Copyright © 2015 Intel Corporation
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the "Software"),
10 * to deal in the Software without restriction, including without limitation
11 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 * and/or sell copies of the Software, and to permit persons to whom the
13 * Software is furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the next
16 * paragraph) shall be included in all copies or substantial portions of the
17 * Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25 * DEALINGS IN THE SOFTWARE.
26 */
27
28 #include "tu_private.h"
29
30 #include "adreno_pm4.xml.h"
31 #include "adreno_common.xml.h"
32
33 #include "vk_format.h"
34 #include "vk_util.h"
35
36 #include "tu_cs.h"
37
38 void
39 tu_bo_list_init(struct tu_bo_list *list)
40 {
41 list->count = list->capacity = 0;
42 list->bo_infos = NULL;
43 }
44
45 void
46 tu_bo_list_destroy(struct tu_bo_list *list)
47 {
48 free(list->bo_infos);
49 }
50
51 void
52 tu_bo_list_reset(struct tu_bo_list *list)
53 {
54 list->count = 0;
55 }
56
57 /**
58 * \a flags consists of MSM_SUBMIT_BO_FLAGS.
59 */
60 static uint32_t
61 tu_bo_list_add_info(struct tu_bo_list *list,
62 const struct drm_msm_gem_submit_bo *bo_info)
63 {
64 assert(bo_info->handle != 0);
65
66 for (uint32_t i = 0; i < list->count; ++i) {
67 if (list->bo_infos[i].handle == bo_info->handle) {
68 assert(list->bo_infos[i].presumed == bo_info->presumed);
69 list->bo_infos[i].flags |= bo_info->flags;
70 return i;
71 }
72 }
73
74 /* grow list->bo_infos if needed */
75 if (list->count == list->capacity) {
76 uint32_t new_capacity = MAX2(2 * list->count, 16);
77 struct drm_msm_gem_submit_bo *new_bo_infos = realloc(
78 list->bo_infos, new_capacity * sizeof(struct drm_msm_gem_submit_bo));
79 if (!new_bo_infos)
80 return TU_BO_LIST_FAILED;
81 list->bo_infos = new_bo_infos;
82 list->capacity = new_capacity;
83 }
84
85 list->bo_infos[list->count] = *bo_info;
86 return list->count++;
87 }
88
89 uint32_t
90 tu_bo_list_add(struct tu_bo_list *list,
91 const struct tu_bo *bo,
92 uint32_t flags)
93 {
94 return tu_bo_list_add_info(list, &(struct drm_msm_gem_submit_bo) {
95 .flags = flags,
96 .handle = bo->gem_handle,
97 .presumed = bo->iova,
98 });
99 }
100
101 VkResult
102 tu_bo_list_merge(struct tu_bo_list *list, const struct tu_bo_list *other)
103 {
104 for (uint32_t i = 0; i < other->count; i++) {
105 if (tu_bo_list_add_info(list, other->bo_infos + i) == TU_BO_LIST_FAILED)
106 return VK_ERROR_OUT_OF_HOST_MEMORY;
107 }
108
109 return VK_SUCCESS;
110 }
111
112 void
113 tu6_emit_event_write(struct tu_cmd_buffer *cmd,
114 struct tu_cs *cs,
115 enum vgt_event_type event)
116 {
117 bool need_seqno = false;
118 switch (event) {
119 case CACHE_FLUSH_TS:
120 case WT_DONE_TS:
121 case RB_DONE_TS:
122 case PC_CCU_FLUSH_DEPTH_TS:
123 case PC_CCU_FLUSH_COLOR_TS:
124 case PC_CCU_RESOLVE_TS:
125 need_seqno = true;
126 break;
127 default:
128 break;
129 }
130
131 tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, need_seqno ? 4 : 1);
132 tu_cs_emit(cs, CP_EVENT_WRITE_0_EVENT(event));
133 if (need_seqno) {
134 tu_cs_emit_qw(cs, global_iova(cmd, seqno_dummy));
135 tu_cs_emit(cs, 0);
136 }
137 }
138
139 static void
140 tu6_emit_flushes(struct tu_cmd_buffer *cmd_buffer,
141 struct tu_cs *cs,
142 enum tu_cmd_flush_bits flushes)
143 {
144 /* Experiments show that invalidating CCU while it still has data in it
145 * doesn't work, so make sure to always flush before invalidating in case
146 * any data remains that hasn't yet been made available through a barrier.
147 * However it does seem to work for UCHE.
148 */
149 if (flushes & (TU_CMD_FLAG_CCU_FLUSH_COLOR |
150 TU_CMD_FLAG_CCU_INVALIDATE_COLOR))
151 tu6_emit_event_write(cmd_buffer, cs, PC_CCU_FLUSH_COLOR_TS);
152 if (flushes & (TU_CMD_FLAG_CCU_FLUSH_DEPTH |
153 TU_CMD_FLAG_CCU_INVALIDATE_DEPTH))
154 tu6_emit_event_write(cmd_buffer, cs, PC_CCU_FLUSH_DEPTH_TS);
155 if (flushes & TU_CMD_FLAG_CCU_INVALIDATE_COLOR)
156 tu6_emit_event_write(cmd_buffer, cs, PC_CCU_INVALIDATE_COLOR);
157 if (flushes & TU_CMD_FLAG_CCU_INVALIDATE_DEPTH)
158 tu6_emit_event_write(cmd_buffer, cs, PC_CCU_INVALIDATE_DEPTH);
159 if (flushes & TU_CMD_FLAG_CACHE_FLUSH)
160 tu6_emit_event_write(cmd_buffer, cs, CACHE_FLUSH_TS);
161 if (flushes & TU_CMD_FLAG_CACHE_INVALIDATE)
162 tu6_emit_event_write(cmd_buffer, cs, CACHE_INVALIDATE);
163 if (flushes & TU_CMD_FLAG_WAIT_MEM_WRITES)
164 tu_cs_emit_pkt7(cs, CP_WAIT_MEM_WRITES, 0);
165 if (flushes & TU_CMD_FLAG_WAIT_FOR_IDLE)
166 tu_cs_emit_wfi(cs);
167 if (flushes & TU_CMD_FLAG_WAIT_FOR_ME)
168 tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
169 }
170
171 /* "Normal" cache flushes, that don't require any special handling */
172
173 static void
174 tu_emit_cache_flush(struct tu_cmd_buffer *cmd_buffer,
175 struct tu_cs *cs)
176 {
177 tu6_emit_flushes(cmd_buffer, cs, cmd_buffer->state.cache.flush_bits);
178 cmd_buffer->state.cache.flush_bits = 0;
179 }
180
181 /* Renderpass cache flushes */
182
183 void
184 tu_emit_cache_flush_renderpass(struct tu_cmd_buffer *cmd_buffer,
185 struct tu_cs *cs)
186 {
187 tu6_emit_flushes(cmd_buffer, cs, cmd_buffer->state.renderpass_cache.flush_bits);
188 cmd_buffer->state.renderpass_cache.flush_bits = 0;
189 }
190
191 /* Cache flushes for things that use the color/depth read/write path (i.e.
192 * blits and draws). This deals with changing CCU state as well as the usual
193 * cache flushing.
194 */
195
196 void
197 tu_emit_cache_flush_ccu(struct tu_cmd_buffer *cmd_buffer,
198 struct tu_cs *cs,
199 enum tu_cmd_ccu_state ccu_state)
200 {
201 enum tu_cmd_flush_bits flushes = cmd_buffer->state.cache.flush_bits;
202
203 assert(ccu_state != TU_CMD_CCU_UNKNOWN);
204
205 /* Changing CCU state must involve invalidating the CCU. In sysmem mode,
206 * the CCU may also contain data that we haven't flushed out yet, so we
207 * also need to flush. Also, in order to program RB_CCU_CNTL, we need to
208 * emit a WFI as it isn't pipelined.
209 */
210 if (ccu_state != cmd_buffer->state.ccu_state) {
211 if (cmd_buffer->state.ccu_state != TU_CMD_CCU_GMEM) {
212 flushes |=
213 TU_CMD_FLAG_CCU_FLUSH_COLOR |
214 TU_CMD_FLAG_CCU_FLUSH_DEPTH;
215 cmd_buffer->state.cache.pending_flush_bits &= ~(
216 TU_CMD_FLAG_CCU_FLUSH_COLOR |
217 TU_CMD_FLAG_CCU_FLUSH_DEPTH);
218 }
219 flushes |=
220 TU_CMD_FLAG_CCU_INVALIDATE_COLOR |
221 TU_CMD_FLAG_CCU_INVALIDATE_DEPTH |
222 TU_CMD_FLAG_WAIT_FOR_IDLE;
223 cmd_buffer->state.cache.pending_flush_bits &= ~(
224 TU_CMD_FLAG_CCU_INVALIDATE_COLOR |
225 TU_CMD_FLAG_CCU_INVALIDATE_DEPTH |
226 TU_CMD_FLAG_WAIT_FOR_IDLE);
227 }
228
229 tu6_emit_flushes(cmd_buffer, cs, flushes);
230 cmd_buffer->state.cache.flush_bits = 0;
231
232 if (ccu_state != cmd_buffer->state.ccu_state) {
233 struct tu_physical_device *phys_dev = cmd_buffer->device->physical_device;
234 tu_cs_emit_regs(cs,
235 A6XX_RB_CCU_CNTL(.offset =
236 ccu_state == TU_CMD_CCU_GMEM ?
237 phys_dev->ccu_offset_gmem :
238 phys_dev->ccu_offset_bypass,
239 .gmem = ccu_state == TU_CMD_CCU_GMEM));
240 cmd_buffer->state.ccu_state = ccu_state;
241 }
242 }
243
244 static void
245 tu6_emit_zs(struct tu_cmd_buffer *cmd,
246 const struct tu_subpass *subpass,
247 struct tu_cs *cs)
248 {
249 const struct tu_framebuffer *fb = cmd->state.framebuffer;
250
251 const uint32_t a = subpass->depth_stencil_attachment.attachment;
252 if (a == VK_ATTACHMENT_UNUSED) {
253 tu_cs_emit_regs(cs,
254 A6XX_RB_DEPTH_BUFFER_INFO(.depth_format = DEPTH6_NONE),
255 A6XX_RB_DEPTH_BUFFER_PITCH(0),
256 A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH(0),
257 A6XX_RB_DEPTH_BUFFER_BASE(0),
258 A6XX_RB_DEPTH_BUFFER_BASE_GMEM(0));
259
260 tu_cs_emit_regs(cs,
261 A6XX_GRAS_SU_DEPTH_BUFFER_INFO(.depth_format = DEPTH6_NONE));
262
263 tu_cs_emit_regs(cs,
264 A6XX_GRAS_LRZ_BUFFER_BASE(0),
265 A6XX_GRAS_LRZ_BUFFER_PITCH(0),
266 A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE(0));
267
268 tu_cs_emit_regs(cs, A6XX_RB_STENCIL_INFO(0));
269
270 return;
271 }
272
273 const struct tu_image_view *iview = fb->attachments[a].attachment;
274 const struct tu_render_pass_attachment *attachment =
275 &cmd->state.pass->attachments[a];
276 enum a6xx_depth_format fmt = tu6_pipe2depth(attachment->format);
277
278 tu_cs_emit_pkt4(cs, REG_A6XX_RB_DEPTH_BUFFER_INFO, 6);
279 tu_cs_emit(cs, A6XX_RB_DEPTH_BUFFER_INFO(.depth_format = fmt).value);
280 tu_cs_image_ref(cs, iview, 0);
281 tu_cs_emit(cs, attachment->gmem_offset);
282
283 tu_cs_emit_regs(cs,
284 A6XX_GRAS_SU_DEPTH_BUFFER_INFO(.depth_format = fmt));
285
286 tu_cs_emit_pkt4(cs, REG_A6XX_RB_DEPTH_FLAG_BUFFER_BASE_LO, 3);
287 tu_cs_image_flag_ref(cs, iview, 0);
288
289 tu_cs_emit_regs(cs,
290 A6XX_GRAS_LRZ_BUFFER_BASE(0),
291 A6XX_GRAS_LRZ_BUFFER_PITCH(0),
292 A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE(0));
293
294 if (attachment->format == VK_FORMAT_D32_SFLOAT_S8_UINT ||
295 attachment->format == VK_FORMAT_S8_UINT) {
296
297 tu_cs_emit_pkt4(cs, REG_A6XX_RB_STENCIL_INFO, 6);
298 tu_cs_emit(cs, A6XX_RB_STENCIL_INFO(.separate_stencil = true).value);
299 if (attachment->format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
300 tu_cs_image_stencil_ref(cs, iview, 0);
301 tu_cs_emit(cs, attachment->gmem_offset_stencil);
302 } else {
303 tu_cs_image_ref(cs, iview, 0);
304 tu_cs_emit(cs, attachment->gmem_offset);
305 }
306 } else {
307 tu_cs_emit_regs(cs,
308 A6XX_RB_STENCIL_INFO(0));
309 }
310 }
311
312 static void
313 tu6_emit_mrt(struct tu_cmd_buffer *cmd,
314 const struct tu_subpass *subpass,
315 struct tu_cs *cs)
316 {
317 const struct tu_framebuffer *fb = cmd->state.framebuffer;
318
319 for (uint32_t i = 0; i < subpass->color_count; ++i) {
320 uint32_t a = subpass->color_attachments[i].attachment;
321 if (a == VK_ATTACHMENT_UNUSED)
322 continue;
323
324 const struct tu_image_view *iview = fb->attachments[a].attachment;
325
326 tu_cs_emit_pkt4(cs, REG_A6XX_RB_MRT_BUF_INFO(i), 6);
327 tu_cs_emit(cs, iview->RB_MRT_BUF_INFO);
328 tu_cs_image_ref(cs, iview, 0);
329 tu_cs_emit(cs, cmd->state.pass->attachments[a].gmem_offset);
330
331 tu_cs_emit_regs(cs,
332 A6XX_SP_FS_MRT_REG(i, .dword = iview->SP_FS_MRT_REG));
333
334 tu_cs_emit_pkt4(cs, REG_A6XX_RB_MRT_FLAG_BUFFER_ADDR_LO(i), 3);
335 tu_cs_image_flag_ref(cs, iview, 0);
336 }
337
338 tu_cs_emit_regs(cs,
339 A6XX_RB_SRGB_CNTL(.dword = subpass->srgb_cntl));
340 tu_cs_emit_regs(cs,
341 A6XX_SP_SRGB_CNTL(.dword = subpass->srgb_cntl));
342
343 tu_cs_emit_regs(cs, A6XX_GRAS_MAX_LAYER_INDEX(fb->layers - 1));
344 }
345
346 void
347 tu6_emit_msaa(struct tu_cs *cs, VkSampleCountFlagBits vk_samples)
348 {
349 const enum a3xx_msaa_samples samples = tu_msaa_samples(vk_samples);
350 bool msaa_disable = samples == MSAA_ONE;
351
352 tu_cs_emit_regs(cs,
353 A6XX_SP_TP_RAS_MSAA_CNTL(samples),
354 A6XX_SP_TP_DEST_MSAA_CNTL(.samples = samples,
355 .msaa_disable = msaa_disable));
356
357 tu_cs_emit_regs(cs,
358 A6XX_GRAS_RAS_MSAA_CNTL(samples),
359 A6XX_GRAS_DEST_MSAA_CNTL(.samples = samples,
360 .msaa_disable = msaa_disable));
361
362 tu_cs_emit_regs(cs,
363 A6XX_RB_RAS_MSAA_CNTL(samples),
364 A6XX_RB_DEST_MSAA_CNTL(.samples = samples,
365 .msaa_disable = msaa_disable));
366
367 tu_cs_emit_regs(cs,
368 A6XX_RB_MSAA_CNTL(samples));
369 }
370
371 static void
372 tu6_emit_bin_size(struct tu_cs *cs,
373 uint32_t bin_w, uint32_t bin_h, uint32_t flags)
374 {
375 tu_cs_emit_regs(cs,
376 A6XX_GRAS_BIN_CONTROL(.binw = bin_w,
377 .binh = bin_h,
378 .dword = flags));
379
380 tu_cs_emit_regs(cs,
381 A6XX_RB_BIN_CONTROL(.binw = bin_w,
382 .binh = bin_h,
383 .dword = flags));
384
385 /* no flag for RB_BIN_CONTROL2... */
386 tu_cs_emit_regs(cs,
387 A6XX_RB_BIN_CONTROL2(.binw = bin_w,
388 .binh = bin_h));
389 }
390
391 static void
392 tu6_emit_render_cntl(struct tu_cmd_buffer *cmd,
393 const struct tu_subpass *subpass,
394 struct tu_cs *cs,
395 bool binning)
396 {
397 const struct tu_framebuffer *fb = cmd->state.framebuffer;
398 uint32_t cntl = 0;
399 cntl |= A6XX_RB_RENDER_CNTL_UNK4;
400 if (binning) {
401 cntl |= A6XX_RB_RENDER_CNTL_BINNING;
402 } else {
403 uint32_t mrts_ubwc_enable = 0;
404 for (uint32_t i = 0; i < subpass->color_count; ++i) {
405 uint32_t a = subpass->color_attachments[i].attachment;
406 if (a == VK_ATTACHMENT_UNUSED)
407 continue;
408
409 const struct tu_image_view *iview = fb->attachments[a].attachment;
410 if (iview->ubwc_enabled)
411 mrts_ubwc_enable |= 1 << i;
412 }
413
414 cntl |= A6XX_RB_RENDER_CNTL_FLAG_MRTS(mrts_ubwc_enable);
415
416 const uint32_t a = subpass->depth_stencil_attachment.attachment;
417 if (a != VK_ATTACHMENT_UNUSED) {
418 const struct tu_image_view *iview = fb->attachments[a].attachment;
419 if (iview->ubwc_enabled)
420 cntl |= A6XX_RB_RENDER_CNTL_FLAG_DEPTH;
421 }
422
423 /* In the !binning case, we need to set RB_RENDER_CNTL in the draw_cs
424 * in order to set it correctly for the different subpasses. However,
425 * that means the packets we're emitting also happen during binning. So
426 * we need to guard the write on !BINNING at CP execution time.
427 */
428 tu_cs_reserve(cs, 3 + 4);
429 tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
430 tu_cs_emit(cs, CP_COND_REG_EXEC_0_MODE(RENDER_MODE) |
431 CP_COND_REG_EXEC_0_GMEM | CP_COND_REG_EXEC_0_SYSMEM);
432 tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(4));
433 }
434
435 tu_cs_emit_pkt7(cs, CP_REG_WRITE, 3);
436 tu_cs_emit(cs, CP_REG_WRITE_0_TRACKER(TRACK_RENDER_CNTL));
437 tu_cs_emit(cs, REG_A6XX_RB_RENDER_CNTL);
438 tu_cs_emit(cs, cntl);
439 }
440
441 static void
442 tu6_emit_blit_scissor(struct tu_cmd_buffer *cmd, struct tu_cs *cs, bool align)
443 {
444
445 const VkRect2D *render_area = &cmd->state.render_area;
446
447 /* Avoid assertion fails with an empty render area at (0, 0) where the
448 * subtraction below wraps around. Empty render areas should be forced to
449 * the sysmem path by use_sysmem_rendering(). It's not even clear whether
450 * an empty scissor here works, and the blob seems to force sysmem too as
451 * it sets something wrong (non-empty) for the scissor.
452 */
453 if (render_area->extent.width == 0 ||
454 render_area->extent.height == 0)
455 return;
456
457 uint32_t x1 = render_area->offset.x;
458 uint32_t y1 = render_area->offset.y;
459 uint32_t x2 = x1 + render_area->extent.width - 1;
460 uint32_t y2 = y1 + render_area->extent.height - 1;
461
462 if (align) {
463 x1 = x1 & ~(GMEM_ALIGN_W - 1);
464 y1 = y1 & ~(GMEM_ALIGN_H - 1);
465 x2 = ALIGN_POT(x2 + 1, GMEM_ALIGN_W) - 1;
466 y2 = ALIGN_POT(y2 + 1, GMEM_ALIGN_H) - 1;
467 }
468
469 tu_cs_emit_regs(cs,
470 A6XX_RB_BLIT_SCISSOR_TL(.x = x1, .y = y1),
471 A6XX_RB_BLIT_SCISSOR_BR(.x = x2, .y = y2));
472 }
473
474 void
475 tu6_emit_window_scissor(struct tu_cs *cs,
476 uint32_t x1,
477 uint32_t y1,
478 uint32_t x2,
479 uint32_t y2)
480 {
481 tu_cs_emit_regs(cs,
482 A6XX_GRAS_SC_WINDOW_SCISSOR_TL(.x = x1, .y = y1),
483 A6XX_GRAS_SC_WINDOW_SCISSOR_BR(.x = x2, .y = y2));
484
485 tu_cs_emit_regs(cs,
486 A6XX_GRAS_2D_RESOLVE_CNTL_1(.x = x1, .y = y1),
487 A6XX_GRAS_2D_RESOLVE_CNTL_2(.x = x2, .y = y2));
488 }
489
490 void
491 tu6_emit_window_offset(struct tu_cs *cs, uint32_t x1, uint32_t y1)
492 {
493 tu_cs_emit_regs(cs,
494 A6XX_RB_WINDOW_OFFSET(.x = x1, .y = y1));
495
496 tu_cs_emit_regs(cs,
497 A6XX_RB_WINDOW_OFFSET2(.x = x1, .y = y1));
498
499 tu_cs_emit_regs(cs,
500 A6XX_SP_WINDOW_OFFSET(.x = x1, .y = y1));
501
502 tu_cs_emit_regs(cs,
503 A6XX_SP_TP_WINDOW_OFFSET(.x = x1, .y = y1));
504 }
505
506 static void
507 tu_cs_emit_draw_state(struct tu_cs *cs, uint32_t id, struct tu_draw_state state)
508 {
509 uint32_t enable_mask;
510 switch (id) {
511 case TU_DRAW_STATE_PROGRAM:
512 case TU_DRAW_STATE_VI:
513 case TU_DRAW_STATE_FS_CONST:
514 /* The blob seems to not enable this (DESC_SETS_LOAD) for binning, even
515 * when resources would actually be used in the binning shader.
516 * Presumably the overhead of prefetching the resources isn't
517 * worth it.
518 */
519 case TU_DRAW_STATE_DESC_SETS_LOAD:
520 enable_mask = CP_SET_DRAW_STATE__0_GMEM |
521 CP_SET_DRAW_STATE__0_SYSMEM;
522 break;
523 case TU_DRAW_STATE_PROGRAM_BINNING:
524 case TU_DRAW_STATE_VI_BINNING:
525 enable_mask = CP_SET_DRAW_STATE__0_BINNING;
526 break;
527 case TU_DRAW_STATE_INPUT_ATTACHMENTS_GMEM:
528 enable_mask = CP_SET_DRAW_STATE__0_GMEM;
529 break;
530 case TU_DRAW_STATE_INPUT_ATTACHMENTS_SYSMEM:
531 enable_mask = CP_SET_DRAW_STATE__0_SYSMEM;
532 break;
533 default:
534 enable_mask = CP_SET_DRAW_STATE__0_GMEM |
535 CP_SET_DRAW_STATE__0_SYSMEM |
536 CP_SET_DRAW_STATE__0_BINNING;
537 break;
538 }
539
540 /* We need to reload the descriptors every time the descriptor sets
541 * change. However, the commands we send only depend on the pipeline
542 * because the whole point is to cache descriptors which are used by the
543 * pipeline. There's a problem here, in that the firmware has an
544 * "optimization" which skips executing groups that are set to the same
545 * value as the last draw. This means that if the descriptor sets change
546 * but not the pipeline, we'd try to re-execute the same buffer which
547 * the firmware would ignore and we wouldn't pre-load the new
548 * descriptors. Set the DIRTY bit to avoid this optimization
549 */
550 if (id == TU_DRAW_STATE_DESC_SETS_LOAD)
551 enable_mask |= CP_SET_DRAW_STATE__0_DIRTY;
552
553 tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(state.size) |
554 enable_mask |
555 CP_SET_DRAW_STATE__0_GROUP_ID(id) |
556 COND(!state.size, CP_SET_DRAW_STATE__0_DISABLE));
557 tu_cs_emit_qw(cs, state.iova);
558 }
559
560 static bool
561 use_hw_binning(struct tu_cmd_buffer *cmd)
562 {
563 const struct tu_framebuffer *fb = cmd->state.framebuffer;
564
565 /* XFB commands are emitted for BINNING || SYSMEM, which makes it incompatible
566 * with non-hw binning GMEM rendering. this is required because some of the
567 * XFB commands need to only be executed once
568 */
569 if (cmd->state.xfb_used)
570 return true;
571
572 /* Some devices have a newer a630_sqe.fw in which, only in CP_DRAW_INDX and
573 * CP_DRAW_INDX_OFFSET, visibility-based skipping happens *before*
574 * predication-based skipping. It seems this breaks predication, because
575 * draws skipped by predication will not be executed in the binning phase,
576 * and therefore won't have an entry in the draw stream, but the
577 * visibility-based skipping will expect it to have an entry. The result is
578 * a GPU hang when actually executing the first non-predicated draw.
579 * However, it seems that things still work if the whole renderpass is
580 * predicated. Affected tests are
581 * dEQP-VK.conditional_rendering.draw_clear.draw.case_2 as well as a few
582 * other case_N.
583 *
584 * Broken FW version: 016ee181
585 * linux-firmware (working) FW version: 016ee176
586 *
587 * All known a650_sqe.fw versions don't have this bug.
588 *
589 * TODO: we should do version detection of the FW so that devices using the
590 * linux-firmware version of a630_sqe.fw don't need this workaround.
591 */
592 if (cmd->state.has_subpass_predication && cmd->device->physical_device->gpu_id != 650)
593 return false;
594
595 if (unlikely(cmd->device->physical_device->instance->debug_flags & TU_DEBUG_NOBIN))
596 return false;
597
598 if (unlikely(cmd->device->physical_device->instance->debug_flags & TU_DEBUG_FORCEBIN))
599 return true;
600
601 return (fb->tile_count.width * fb->tile_count.height) > 2;
602 }
603
604 static bool
605 use_sysmem_rendering(struct tu_cmd_buffer *cmd)
606 {
607 if (unlikely(cmd->device->physical_device->instance->debug_flags & TU_DEBUG_SYSMEM))
608 return true;
609
610 /* If hw binning is required because of XFB but doesn't work because of the
611 * conditional rendering bug, fallback to sysmem.
612 */
613 if (cmd->state.xfb_used && cmd->state.has_subpass_predication &&
614 cmd->device->physical_device->gpu_id != 650)
615 return true;
616
617 /* can't fit attachments into gmem */
618 if (!cmd->state.pass->gmem_pixels)
619 return true;
620
621 if (cmd->state.framebuffer->layers > 1)
622 return true;
623
624 /* Use sysmem for empty render areas */
625 if (cmd->state.render_area.extent.width == 0 ||
626 cmd->state.render_area.extent.height == 0)
627 return true;
628
629 if (cmd->state.has_tess)
630 return true;
631
632 return false;
633 }
634
635 static void
636 tu6_emit_tile_select(struct tu_cmd_buffer *cmd,
637 struct tu_cs *cs,
638 uint32_t tx, uint32_t ty, uint32_t pipe, uint32_t slot)
639 {
640 const struct tu_framebuffer *fb = cmd->state.framebuffer;
641
642 tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
643 tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_YIELD));
644
645 tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
646 tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_GMEM));
647
648 const uint32_t x1 = fb->tile0.width * tx;
649 const uint32_t y1 = fb->tile0.height * ty;
650 const uint32_t x2 = x1 + fb->tile0.width - 1;
651 const uint32_t y2 = y1 + fb->tile0.height - 1;
652 tu6_emit_window_scissor(cs, x1, y1, x2, y2);
653 tu6_emit_window_offset(cs, x1, y1);
654
655 tu_cs_emit_regs(cs, A6XX_VPC_SO_DISABLE(false));
656
657 if (use_hw_binning(cmd)) {
658 tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
659
660 tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
661 tu_cs_emit(cs, 0x0);
662
663 tu_cs_emit_pkt7(cs, CP_SET_BIN_DATA5_OFFSET, 4);
664 tu_cs_emit(cs, fb->pipe_sizes[pipe] |
665 CP_SET_BIN_DATA5_0_VSC_N(slot));
666 tu_cs_emit(cs, pipe * cmd->vsc_draw_strm_pitch);
667 tu_cs_emit(cs, pipe * 4);
668 tu_cs_emit(cs, pipe * cmd->vsc_prim_strm_pitch);
669
670 tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
671 tu_cs_emit(cs, 0x0);
672
673 tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
674 tu_cs_emit(cs, 0x0);
675 } else {
676 tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
677 tu_cs_emit(cs, 0x1);
678
679 tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
680 tu_cs_emit(cs, 0x0);
681 }
682 }
683
684 static void
685 tu6_emit_sysmem_resolve(struct tu_cmd_buffer *cmd,
686 struct tu_cs *cs,
687 uint32_t a,
688 uint32_t gmem_a)
689 {
690 const struct tu_framebuffer *fb = cmd->state.framebuffer;
691 struct tu_image_view *dst = fb->attachments[a].attachment;
692 struct tu_image_view *src = fb->attachments[gmem_a].attachment;
693
694 tu_resolve_sysmem(cmd, cs, src, dst, fb->layers, &cmd->state.render_area);
695 }
696
697 static void
698 tu6_emit_sysmem_resolves(struct tu_cmd_buffer *cmd,
699 struct tu_cs *cs,
700 const struct tu_subpass *subpass)
701 {
702 if (subpass->resolve_attachments) {
703 /* From the documentation for vkCmdNextSubpass, section 7.4 "Render Pass
704 * Commands":
705 *
706 * End-of-subpass multisample resolves are treated as color
707 * attachment writes for the purposes of synchronization. That is,
708 * they are considered to execute in the
709 * VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT pipeline stage and
710 * their writes are synchronized with
711 * VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT. Synchronization between
712 * rendering within a subpass and any resolve operations at the end
713 * of the subpass occurs automatically, without need for explicit
714 * dependencies or pipeline barriers. However, if the resolve
715 * attachment is also used in a different subpass, an explicit
716 * dependency is needed.
717 *
718 * We use the CP_BLIT path for sysmem resolves, which is really a
719 * transfer command, so we have to manually flush similar to the gmem
720 * resolve case. However, a flush afterwards isn't needed because of the
721 * last sentence and the fact that we're in sysmem mode.
722 */
723 tu6_emit_event_write(cmd, cs, PC_CCU_FLUSH_COLOR_TS);
724 tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE);
725
726 /* Wait for the flushes to land before using the 2D engine */
727 tu_cs_emit_wfi(cs);
728
729 for (unsigned i = 0; i < subpass->color_count; i++) {
730 uint32_t a = subpass->resolve_attachments[i].attachment;
731 if (a == VK_ATTACHMENT_UNUSED)
732 continue;
733
734 tu6_emit_sysmem_resolve(cmd, cs, a,
735 subpass->color_attachments[i].attachment);
736 }
737 }
738 }
739
740 static void
741 tu6_emit_tile_store(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
742 {
743 const struct tu_render_pass *pass = cmd->state.pass;
744 const struct tu_subpass *subpass = &pass->subpasses[pass->subpass_count-1];
745
746 tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
747 tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
748 CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
749 CP_SET_DRAW_STATE__0_GROUP_ID(0));
750 tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
751 tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
752
753 tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
754 tu_cs_emit(cs, 0x0);
755
756 tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
757 tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_RESOLVE));
758
759 tu6_emit_blit_scissor(cmd, cs, true);
760
761 for (uint32_t a = 0; a < pass->attachment_count; ++a) {
762 if (pass->attachments[a].gmem_offset >= 0)
763 tu_store_gmem_attachment(cmd, cs, a, a);
764 }
765
766 if (subpass->resolve_attachments) {
767 for (unsigned i = 0; i < subpass->color_count; i++) {
768 uint32_t a = subpass->resolve_attachments[i].attachment;
769 if (a != VK_ATTACHMENT_UNUSED)
770 tu_store_gmem_attachment(cmd, cs, a,
771 subpass->color_attachments[i].attachment);
772 }
773 }
774 }
775
776 static void
777 tu6_init_hw(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
778 {
779 struct tu_device *dev = cmd->device;
780 const struct tu_physical_device *phys_dev = dev->physical_device;
781
782 tu6_emit_event_write(cmd, cs, CACHE_INVALIDATE);
783
784 tu_cs_emit_regs(cs, A6XX_HLSQ_INVALIDATE_CMD(
785 .vs_state = true,
786 .hs_state = true,
787 .ds_state = true,
788 .gs_state = true,
789 .fs_state = true,
790 .cs_state = true,
791 .gfx_ibo = true,
792 .cs_ibo = true,
793 .gfx_shared_const = true,
794 .cs_shared_const = true,
795 .gfx_bindless = 0x1f,
796 .cs_bindless = 0x1f));
797
798 tu_cs_emit_wfi(cs);
799
800 cmd->state.cache.pending_flush_bits &=
801 ~(TU_CMD_FLAG_WAIT_FOR_IDLE | TU_CMD_FLAG_CACHE_INVALIDATE);
802
803 tu_cs_emit_regs(cs,
804 A6XX_RB_CCU_CNTL(.offset = phys_dev->ccu_offset_bypass));
805 cmd->state.ccu_state = TU_CMD_CCU_SYSMEM;
806 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8E04, 0x00100000);
807 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE04, 0x8);
808 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE00, 0);
809 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE0F, 0x3f);
810 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B605, 0x44);
811 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B600, 0x100000);
812 tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE00, 0x80);
813 tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE01, 0);
814
815 tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9600, 0);
816 tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8600, 0x880);
817 tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_UNKNOWN_BE04, 0);
818 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_AE03, 0x00000410);
819 tu_cs_emit_write_reg(cs, REG_A6XX_SP_IBO_COUNT, 0);
820 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B182, 0);
821 tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_SHARED_CONSTS, 0);
822 tu_cs_emit_write_reg(cs, REG_A6XX_UCHE_UNKNOWN_0E12, 0x3200000);
823 tu_cs_emit_write_reg(cs, REG_A6XX_UCHE_CLIENT_PF, 4);
824 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8E01, 0x0);
825 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_A982, 0);
826 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_A9A8, 0);
827 tu_cs_emit_write_reg(cs, REG_A6XX_SP_MODE_CONTROL,
828 A6XX_SP_MODE_CONTROL_CONSTANT_DEMOTION_ENABLE | 4);
829
830 /* TODO: set A6XX_VFD_ADD_OFFSET_INSTANCE and fix ir3 to avoid adding base instance */
831 tu_cs_emit_write_reg(cs, REG_A6XX_VFD_ADD_OFFSET, A6XX_VFD_ADD_OFFSET_VERTEX);
832 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8811, 0x00000010);
833 tu_cs_emit_write_reg(cs, REG_A6XX_PC_MODE_CNTL, 0x1f);
834
835 tu_cs_emit_write_reg(cs, REG_A6XX_RB_SRGB_CNTL, 0);
836
837 tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8110, 0);
838
839 tu_cs_emit_write_reg(cs, REG_A6XX_RB_RENDER_CONTROL0, 0x401);
840 tu_cs_emit_write_reg(cs, REG_A6XX_RB_RENDER_CONTROL1, 0);
841 tu_cs_emit_write_reg(cs, REG_A6XX_RB_FS_OUTPUT_CNTL0, 0);
842 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8818, 0);
843 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_8819, 0);
844 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881A, 0);
845 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881B, 0);
846 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881C, 0);
847 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881D, 0);
848 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_881E, 0);
849 tu_cs_emit_write_reg(cs, REG_A6XX_RB_UNKNOWN_88F0, 0);
850
851 tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9107, 0);
852
853 tu_cs_emit_regs(cs, A6XX_VPC_POINT_COORD_INVERT(false));
854 tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9300, 0);
855
856 tu_cs_emit_regs(cs, A6XX_VPC_SO_DISABLE(true));
857
858 tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9980, 0);
859
860 tu_cs_emit_write_reg(cs, REG_A6XX_PC_PRIMITIVE_CNTL_6, 0);
861 tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9B07, 0);
862
863 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_A81B, 0);
864
865 tu_cs_emit_write_reg(cs, REG_A6XX_SP_UNKNOWN_B183, 0);
866
867 tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_8099, 0);
868 tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80A0, 2);
869 tu_cs_emit_write_reg(cs, REG_A6XX_GRAS_UNKNOWN_80AF, 0);
870 tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9210, 0);
871 tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9211, 0);
872 tu_cs_emit_write_reg(cs, REG_A6XX_VPC_UNKNOWN_9602, 0);
873 tu_cs_emit_write_reg(cs, REG_A6XX_PC_UNKNOWN_9E72, 0);
874 tu_cs_emit_write_reg(cs, REG_A6XX_SP_TP_UNKNOWN_B309, 0x000000a2);
875 tu_cs_emit_write_reg(cs, REG_A6XX_HLSQ_CONTROL_5_REG, 0xfc);
876
877 tu_cs_emit_write_reg(cs, REG_A6XX_VFD_MODE_CNTL, 0x00000000);
878
879 tu_cs_emit_write_reg(cs, REG_A6XX_VFD_UNKNOWN_A008, 0);
880
881 tu_cs_emit_write_reg(cs, REG_A6XX_PC_MODE_CNTL, 0x0000001f);
882
883 /* we don't use this yet.. probably best to disable.. */
884 tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
885 tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
886 CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
887 CP_SET_DRAW_STATE__0_GROUP_ID(0));
888 tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
889 tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
890
891 tu_cs_emit_regs(cs,
892 A6XX_SP_HS_CTRL_REG0(0));
893
894 tu_cs_emit_regs(cs,
895 A6XX_SP_GS_CTRL_REG0(0));
896
897 tu_cs_emit_regs(cs,
898 A6XX_GRAS_LRZ_CNTL(0));
899
900 tu_cs_emit_regs(cs,
901 A6XX_RB_LRZ_CNTL(0));
902
903 tu_cs_emit_regs(cs,
904 A6XX_SP_TP_BORDER_COLOR_BASE_ADDR(.bo = &dev->global_bo,
905 .bo_offset = gb_offset(border_color)));
906 tu_cs_emit_regs(cs,
907 A6XX_SP_PS_TP_BORDER_COLOR_BASE_ADDR(.bo = &dev->global_bo,
908 .bo_offset = gb_offset(border_color)));
909
910 /* VSC buffers:
911 * use vsc pitches from the largest values used so far with this device
912 * if there hasn't been overflow, there will already be a scratch bo
913 * allocated for these sizes
914 *
915 * if overflow is detected, the stream size is increased by 2x
916 */
917 mtx_lock(&dev->vsc_pitch_mtx);
918
919 struct tu6_global *global = dev->global_bo.map;
920
921 uint32_t vsc_draw_overflow = global->vsc_draw_overflow;
922 uint32_t vsc_prim_overflow = global->vsc_prim_overflow;
923
924 if (vsc_draw_overflow >= dev->vsc_draw_strm_pitch)
925 dev->vsc_draw_strm_pitch = (dev->vsc_draw_strm_pitch - VSC_PAD) * 2 + VSC_PAD;
926
927 if (vsc_prim_overflow >= dev->vsc_prim_strm_pitch)
928 dev->vsc_prim_strm_pitch = (dev->vsc_prim_strm_pitch - VSC_PAD) * 2 + VSC_PAD;
929
930 cmd->vsc_prim_strm_pitch = dev->vsc_prim_strm_pitch;
931 cmd->vsc_draw_strm_pitch = dev->vsc_draw_strm_pitch;
932
933 mtx_unlock(&dev->vsc_pitch_mtx);
934
935 struct tu_bo *vsc_bo;
936 uint32_t size0 = cmd->vsc_prim_strm_pitch * MAX_VSC_PIPES +
937 cmd->vsc_draw_strm_pitch * MAX_VSC_PIPES;
938
939 tu_get_scratch_bo(dev, size0 + MAX_VSC_PIPES * 4, &vsc_bo);
940
941 tu_cs_emit_regs(cs,
942 A6XX_VSC_DRAW_STRM_SIZE_ADDRESS(.bo = vsc_bo, .bo_offset = size0));
943 tu_cs_emit_regs(cs,
944 A6XX_VSC_PRIM_STRM_ADDRESS(.bo = vsc_bo));
945 tu_cs_emit_regs(cs,
946 A6XX_VSC_DRAW_STRM_ADDRESS(.bo = vsc_bo,
947 .bo_offset = cmd->vsc_prim_strm_pitch * MAX_VSC_PIPES));
948
949 tu_bo_list_add(&cmd->bo_list, vsc_bo, MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
950
951 tu_cs_sanity_check(cs);
952 }
953
954 static void
955 update_vsc_pipe(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
956 {
957 const struct tu_framebuffer *fb = cmd->state.framebuffer;
958
959 tu_cs_emit_regs(cs,
960 A6XX_VSC_BIN_SIZE(.width = fb->tile0.width,
961 .height = fb->tile0.height));
962
963 tu_cs_emit_regs(cs,
964 A6XX_VSC_BIN_COUNT(.nx = fb->tile_count.width,
965 .ny = fb->tile_count.height));
966
967 tu_cs_emit_pkt4(cs, REG_A6XX_VSC_PIPE_CONFIG_REG(0), 32);
968 tu_cs_emit_array(cs, fb->pipe_config, 32);
969
970 tu_cs_emit_regs(cs,
971 A6XX_VSC_PRIM_STRM_PITCH(cmd->vsc_prim_strm_pitch),
972 A6XX_VSC_PRIM_STRM_LIMIT(cmd->vsc_prim_strm_pitch - VSC_PAD));
973
974 tu_cs_emit_regs(cs,
975 A6XX_VSC_DRAW_STRM_PITCH(cmd->vsc_draw_strm_pitch),
976 A6XX_VSC_DRAW_STRM_LIMIT(cmd->vsc_draw_strm_pitch - VSC_PAD));
977 }
978
979 static void
980 emit_vsc_overflow_test(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
981 {
982 const struct tu_framebuffer *fb = cmd->state.framebuffer;
983 const uint32_t used_pipe_count =
984 fb->pipe_count.width * fb->pipe_count.height;
985
986 for (int i = 0; i < used_pipe_count; i++) {
987 tu_cs_emit_pkt7(cs, CP_COND_WRITE5, 8);
988 tu_cs_emit(cs, CP_COND_WRITE5_0_FUNCTION(WRITE_GE) |
989 CP_COND_WRITE5_0_WRITE_MEMORY);
990 tu_cs_emit(cs, CP_COND_WRITE5_1_POLL_ADDR_LO(REG_A6XX_VSC_DRAW_STRM_SIZE_REG(i)));
991 tu_cs_emit(cs, CP_COND_WRITE5_2_POLL_ADDR_HI(0));
992 tu_cs_emit(cs, CP_COND_WRITE5_3_REF(cmd->vsc_draw_strm_pitch - VSC_PAD));
993 tu_cs_emit(cs, CP_COND_WRITE5_4_MASK(~0));
994 tu_cs_emit_qw(cs, global_iova(cmd, vsc_draw_overflow));
995 tu_cs_emit(cs, CP_COND_WRITE5_7_WRITE_DATA(cmd->vsc_draw_strm_pitch));
996
997 tu_cs_emit_pkt7(cs, CP_COND_WRITE5, 8);
998 tu_cs_emit(cs, CP_COND_WRITE5_0_FUNCTION(WRITE_GE) |
999 CP_COND_WRITE5_0_WRITE_MEMORY);
1000 tu_cs_emit(cs, CP_COND_WRITE5_1_POLL_ADDR_LO(REG_A6XX_VSC_PRIM_STRM_SIZE_REG(i)));
1001 tu_cs_emit(cs, CP_COND_WRITE5_2_POLL_ADDR_HI(0));
1002 tu_cs_emit(cs, CP_COND_WRITE5_3_REF(cmd->vsc_prim_strm_pitch - VSC_PAD));
1003 tu_cs_emit(cs, CP_COND_WRITE5_4_MASK(~0));
1004 tu_cs_emit_qw(cs, global_iova(cmd, vsc_prim_overflow));
1005 tu_cs_emit(cs, CP_COND_WRITE5_7_WRITE_DATA(cmd->vsc_prim_strm_pitch));
1006 }
1007
1008 tu_cs_emit_pkt7(cs, CP_WAIT_MEM_WRITES, 0);
1009 }
1010
1011 static void
1012 tu6_emit_binning_pass(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1013 {
1014 struct tu_physical_device *phys_dev = cmd->device->physical_device;
1015 const struct tu_framebuffer *fb = cmd->state.framebuffer;
1016
1017 tu6_emit_window_scissor(cs, 0, 0, fb->width - 1, fb->height - 1);
1018
1019 tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1020 tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_BINNING));
1021
1022 tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
1023 tu_cs_emit(cs, 0x1);
1024
1025 tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
1026 tu_cs_emit(cs, 0x1);
1027
1028 tu_cs_emit_wfi(cs);
1029
1030 tu_cs_emit_regs(cs,
1031 A6XX_VFD_MODE_CNTL(.binning_pass = true));
1032
1033 update_vsc_pipe(cmd, cs);
1034
1035 tu_cs_emit_regs(cs,
1036 A6XX_PC_UNKNOWN_9805(.unknown = phys_dev->magic.PC_UNKNOWN_9805));
1037
1038 tu_cs_emit_regs(cs,
1039 A6XX_SP_UNKNOWN_A0F8(.unknown = phys_dev->magic.SP_UNKNOWN_A0F8));
1040
1041 tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, 1);
1042 tu_cs_emit(cs, UNK_2C);
1043
1044 tu_cs_emit_regs(cs,
1045 A6XX_RB_WINDOW_OFFSET(.x = 0, .y = 0));
1046
1047 tu_cs_emit_regs(cs,
1048 A6XX_SP_TP_WINDOW_OFFSET(.x = 0, .y = 0));
1049
1050 /* emit IB to binning drawcmds: */
1051 tu_cs_emit_call(cs, &cmd->draw_cs);
1052
1053 tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3);
1054 tu_cs_emit(cs, CP_SET_DRAW_STATE__0_COUNT(0) |
1055 CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS |
1056 CP_SET_DRAW_STATE__0_GROUP_ID(0));
1057 tu_cs_emit(cs, CP_SET_DRAW_STATE__1_ADDR_LO(0));
1058 tu_cs_emit(cs, CP_SET_DRAW_STATE__2_ADDR_HI(0));
1059
1060 tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, 1);
1061 tu_cs_emit(cs, UNK_2D);
1062
1063 /* This flush is probably required because the VSC, which produces the
1064 * visibility stream, is a client of UCHE, whereas the CP needs to read the
1065 * visibility stream (without caching) to do draw skipping. The
1066 * WFI+WAIT_FOR_ME combination guarantees that the binning commands
1067 * submitted are finished before reading the VSC regs (in
1068 * emit_vsc_overflow_test) or the VSC_DATA buffer directly (implicitly as
1069 * part of draws).
1070 */
1071 tu6_emit_event_write(cmd, cs, CACHE_FLUSH_TS);
1072
1073 tu_cs_emit_wfi(cs);
1074
1075 tu_cs_emit_pkt7(cs, CP_WAIT_FOR_ME, 0);
1076
1077 emit_vsc_overflow_test(cmd, cs);
1078
1079 tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
1080 tu_cs_emit(cs, 0x0);
1081
1082 tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
1083 tu_cs_emit(cs, 0x0);
1084 }
1085
1086 static struct tu_draw_state
1087 tu_emit_input_attachments(struct tu_cmd_buffer *cmd,
1088 const struct tu_subpass *subpass,
1089 bool gmem)
1090 {
1091 /* note: we can probably emit input attachments just once for the whole
1092 * renderpass, this would avoid emitting both sysmem/gmem versions
1093 *
1094 * emit two texture descriptors for each input, as a workaround for
1095 * d24s8/d32s8, which can be sampled as both float (depth) and integer (stencil)
1096 * tu_shader lowers uint input attachment loads to use the 2nd descriptor
1097 * in the pair
1098 * TODO: a smarter workaround
1099 */
1100
1101 if (!subpass->input_count)
1102 return (struct tu_draw_state) {};
1103
1104 struct tu_cs_memory texture;
1105 VkResult result = tu_cs_alloc(&cmd->sub_cs, subpass->input_count * 2,
1106 A6XX_TEX_CONST_DWORDS, &texture);
1107 assert(result == VK_SUCCESS);
1108
1109 for (unsigned i = 0; i < subpass->input_count * 2; i++) {
1110 uint32_t a = subpass->input_attachments[i / 2].attachment;
1111 if (a == VK_ATTACHMENT_UNUSED)
1112 continue;
1113
1114 struct tu_image_view *iview =
1115 cmd->state.framebuffer->attachments[a].attachment;
1116 const struct tu_render_pass_attachment *att =
1117 &cmd->state.pass->attachments[a];
1118 uint32_t *dst = &texture.map[A6XX_TEX_CONST_DWORDS * i];
1119 uint32_t gmem_offset = att->gmem_offset;
1120 uint32_t cpp = att->cpp;
1121
1122 memcpy(dst, iview->descriptor, A6XX_TEX_CONST_DWORDS * 4);
1123
1124 if (i % 2 == 1 && att->format == VK_FORMAT_D24_UNORM_S8_UINT) {
1125 /* note this works because spec says fb and input attachments
1126 * must use identity swizzle
1127 */
1128 dst[0] &= ~(A6XX_TEX_CONST_0_FMT__MASK |
1129 A6XX_TEX_CONST_0_SWIZ_X__MASK | A6XX_TEX_CONST_0_SWIZ_Y__MASK |
1130 A6XX_TEX_CONST_0_SWIZ_Z__MASK | A6XX_TEX_CONST_0_SWIZ_W__MASK);
1131 if (cmd->device->physical_device->limited_z24s8) {
1132 dst[0] |= A6XX_TEX_CONST_0_FMT(FMT6_8_8_8_8_UINT) |
1133 A6XX_TEX_CONST_0_SWIZ_X(A6XX_TEX_W) |
1134 A6XX_TEX_CONST_0_SWIZ_Y(A6XX_TEX_ZERO) |
1135 A6XX_TEX_CONST_0_SWIZ_Z(A6XX_TEX_ZERO) |
1136 A6XX_TEX_CONST_0_SWIZ_W(A6XX_TEX_ONE);
1137 } else {
1138 dst[0] |= A6XX_TEX_CONST_0_FMT(FMT6_Z24_UINT_S8_UINT) |
1139 A6XX_TEX_CONST_0_SWIZ_X(A6XX_TEX_Y) |
1140 A6XX_TEX_CONST_0_SWIZ_Y(A6XX_TEX_ZERO) |
1141 A6XX_TEX_CONST_0_SWIZ_Z(A6XX_TEX_ZERO) |
1142 A6XX_TEX_CONST_0_SWIZ_W(A6XX_TEX_ONE);
1143 }
1144 }
1145
1146 if (i % 2 == 1 && att->format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
1147 dst[0] &= ~A6XX_TEX_CONST_0_FMT__MASK;
1148 dst[0] |= A6XX_TEX_CONST_0_FMT(FMT6_8_UINT);
1149 dst[2] &= ~(A6XX_TEX_CONST_2_PITCHALIGN__MASK | A6XX_TEX_CONST_2_PITCH__MASK);
1150 dst[2] |= A6XX_TEX_CONST_2_PITCH(iview->stencil_PITCH << 6);
1151 dst[3] = 0;
1152 dst[4] = iview->stencil_base_addr;
1153 dst[5] = (dst[5] & 0xffff) | iview->stencil_base_addr >> 32;
1154
1155 cpp = att->samples;
1156 gmem_offset = att->gmem_offset_stencil;
1157 }
1158
1159 if (!gmem)
1160 continue;
1161
1162 /* patched for gmem */
1163 dst[0] &= ~(A6XX_TEX_CONST_0_SWAP__MASK | A6XX_TEX_CONST_0_TILE_MODE__MASK);
1164 dst[0] |= A6XX_TEX_CONST_0_TILE_MODE(TILE6_2);
1165 dst[2] =
1166 A6XX_TEX_CONST_2_TYPE(A6XX_TEX_2D) |
1167 A6XX_TEX_CONST_2_PITCH(cmd->state.framebuffer->tile0.width * cpp);
1168 dst[3] = 0;
1169 dst[4] = cmd->device->physical_device->gmem_base + gmem_offset;
1170 dst[5] = A6XX_TEX_CONST_5_DEPTH(1);
1171 for (unsigned i = 6; i < A6XX_TEX_CONST_DWORDS; i++)
1172 dst[i] = 0;
1173 }
1174
1175 struct tu_cs cs;
1176 struct tu_draw_state ds = tu_cs_draw_state(&cmd->sub_cs, &cs, 9);
1177
1178 tu_cs_emit_pkt7(&cs, CP_LOAD_STATE6_FRAG, 3);
1179 tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(0) |
1180 CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
1181 CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
1182 CP_LOAD_STATE6_0_STATE_BLOCK(SB6_FS_TEX) |
1183 CP_LOAD_STATE6_0_NUM_UNIT(subpass->input_count * 2));
1184 tu_cs_emit_qw(&cs, texture.iova);
1185
1186 tu_cs_emit_pkt4(&cs, REG_A6XX_SP_FS_TEX_CONST_LO, 2);
1187 tu_cs_emit_qw(&cs, texture.iova);
1188
1189 tu_cs_emit_regs(&cs, A6XX_SP_FS_TEX_COUNT(subpass->input_count * 2));
1190
1191 assert(cs.cur == cs.end); /* validate draw state size */
1192
1193 return ds;
1194 }
1195
1196 static void
1197 tu_set_input_attachments(struct tu_cmd_buffer *cmd, const struct tu_subpass *subpass)
1198 {
1199 struct tu_cs *cs = &cmd->draw_cs;
1200
1201 tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 6);
1202 tu_cs_emit_draw_state(cs, TU_DRAW_STATE_INPUT_ATTACHMENTS_GMEM,
1203 tu_emit_input_attachments(cmd, subpass, true));
1204 tu_cs_emit_draw_state(cs, TU_DRAW_STATE_INPUT_ATTACHMENTS_SYSMEM,
1205 tu_emit_input_attachments(cmd, subpass, false));
1206 }
1207
1208 static void
1209 tu_emit_renderpass_begin(struct tu_cmd_buffer *cmd,
1210 const VkRenderPassBeginInfo *info)
1211 {
1212 struct tu_cs *cs = &cmd->draw_cs;
1213
1214 tu_cond_exec_start(cs, CP_COND_EXEC_0_RENDER_MODE_GMEM);
1215
1216 tu6_emit_blit_scissor(cmd, cs, true);
1217
1218 for (uint32_t i = 0; i < cmd->state.pass->attachment_count; ++i)
1219 tu_load_gmem_attachment(cmd, cs, i, false);
1220
1221 tu6_emit_blit_scissor(cmd, cs, false);
1222
1223 for (uint32_t i = 0; i < cmd->state.pass->attachment_count; ++i)
1224 tu_clear_gmem_attachment(cmd, cs, i, info);
1225
1226 tu_cond_exec_end(cs);
1227
1228 tu_cond_exec_start(cs, CP_COND_EXEC_0_RENDER_MODE_SYSMEM);
1229
1230 for (uint32_t i = 0; i < cmd->state.pass->attachment_count; ++i)
1231 tu_clear_sysmem_attachment(cmd, cs, i, info);
1232
1233 tu_cond_exec_end(cs);
1234 }
1235
1236 static void
1237 tu6_sysmem_render_begin(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1238 {
1239 const struct tu_framebuffer *fb = cmd->state.framebuffer;
1240
1241 assert(fb->width > 0 && fb->height > 0);
1242 tu6_emit_window_scissor(cs, 0, 0, fb->width - 1, fb->height - 1);
1243 tu6_emit_window_offset(cs, 0, 0);
1244
1245 tu6_emit_bin_size(cs, 0, 0, 0xc00000); /* 0xc00000 = BYPASS? */
1246
1247 tu6_emit_event_write(cmd, cs, LRZ_FLUSH);
1248
1249 tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1250 tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_BYPASS));
1251
1252 tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1253 tu_cs_emit(cs, 0x0);
1254
1255 tu_emit_cache_flush_ccu(cmd, cs, TU_CMD_CCU_SYSMEM);
1256
1257 /* enable stream-out, with sysmem there is only one pass: */
1258 tu_cs_emit_regs(cs, A6XX_VPC_SO_DISABLE(false));
1259
1260 tu_cs_emit_pkt7(cs, CP_SET_VISIBILITY_OVERRIDE, 1);
1261 tu_cs_emit(cs, 0x1);
1262
1263 tu_cs_emit_pkt7(cs, CP_SET_MODE, 1);
1264 tu_cs_emit(cs, 0x0);
1265
1266 tu_cs_sanity_check(cs);
1267 }
1268
1269 static void
1270 tu6_sysmem_render_end(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1271 {
1272 /* Do any resolves of the last subpass. These are handled in the
1273 * tile_store_ib in the gmem path.
1274 */
1275 tu6_emit_sysmem_resolves(cmd, cs, cmd->state.subpass);
1276
1277 tu_cs_emit_call(cs, &cmd->draw_epilogue_cs);
1278
1279 tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1280 tu_cs_emit(cs, 0x0);
1281
1282 tu6_emit_event_write(cmd, cs, LRZ_FLUSH);
1283
1284 tu_cs_sanity_check(cs);
1285 }
1286
1287 static void
1288 tu6_tile_render_begin(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1289 {
1290 struct tu_physical_device *phys_dev = cmd->device->physical_device;
1291
1292 tu6_emit_event_write(cmd, cs, LRZ_FLUSH);
1293
1294 /* lrz clear? */
1295
1296 tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1297 tu_cs_emit(cs, 0x0);
1298
1299 tu_emit_cache_flush_ccu(cmd, cs, TU_CMD_CCU_GMEM);
1300
1301 const struct tu_framebuffer *fb = cmd->state.framebuffer;
1302 if (use_hw_binning(cmd)) {
1303 /* enable stream-out during binning pass: */
1304 tu_cs_emit_regs(cs, A6XX_VPC_SO_DISABLE(false));
1305
1306 tu6_emit_bin_size(cs, fb->tile0.width, fb->tile0.height,
1307 A6XX_RB_BIN_CONTROL_BINNING_PASS | 0x6000000);
1308
1309 tu6_emit_render_cntl(cmd, cmd->state.subpass, cs, true);
1310
1311 tu6_emit_binning_pass(cmd, cs);
1312
1313 /* and disable stream-out for draw pass: */
1314 tu_cs_emit_regs(cs, A6XX_VPC_SO_DISABLE(true));
1315
1316 tu6_emit_bin_size(cs, fb->tile0.width, fb->tile0.height,
1317 A6XX_RB_BIN_CONTROL_USE_VIZ | 0x6000000);
1318
1319 tu_cs_emit_regs(cs,
1320 A6XX_VFD_MODE_CNTL(0));
1321
1322 tu_cs_emit_regs(cs, A6XX_PC_UNKNOWN_9805(.unknown = phys_dev->magic.PC_UNKNOWN_9805));
1323
1324 tu_cs_emit_regs(cs, A6XX_SP_UNKNOWN_A0F8(.unknown = phys_dev->magic.SP_UNKNOWN_A0F8));
1325
1326 tu_cs_emit_pkt7(cs, CP_SKIP_IB2_ENABLE_GLOBAL, 1);
1327 tu_cs_emit(cs, 0x1);
1328 } else {
1329 /* no binning pass, so enable stream-out for draw pass:: */
1330 tu_cs_emit_regs(cs, A6XX_VPC_SO_DISABLE(false));
1331
1332 tu6_emit_bin_size(cs, fb->tile0.width, fb->tile0.height, 0x6000000);
1333 }
1334
1335 tu_cs_sanity_check(cs);
1336 }
1337
1338 static void
1339 tu6_render_tile(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1340 {
1341 tu_cs_emit_call(cs, &cmd->draw_cs);
1342
1343 if (use_hw_binning(cmd)) {
1344 tu_cs_emit_pkt7(cs, CP_SET_MARKER, 1);
1345 tu_cs_emit(cs, A6XX_CP_SET_MARKER_0_MODE(RM6_ENDVIS));
1346 }
1347
1348 tu_cs_emit_ib(cs, &cmd->state.tile_store_ib);
1349
1350 tu_cs_sanity_check(cs);
1351 }
1352
1353 static void
1354 tu6_tile_render_end(struct tu_cmd_buffer *cmd, struct tu_cs *cs)
1355 {
1356 tu_cs_emit_call(cs, &cmd->draw_epilogue_cs);
1357
1358 tu_cs_emit_regs(cs,
1359 A6XX_GRAS_LRZ_CNTL(0));
1360
1361 tu6_emit_event_write(cmd, cs, LRZ_FLUSH);
1362
1363 tu6_emit_event_write(cmd, cs, PC_CCU_RESOLVE_TS);
1364
1365 tu_cs_sanity_check(cs);
1366 }
1367
1368 static void
1369 tu_cmd_render_tiles(struct tu_cmd_buffer *cmd)
1370 {
1371 const struct tu_framebuffer *fb = cmd->state.framebuffer;
1372
1373 tu6_tile_render_begin(cmd, &cmd->cs);
1374
1375 uint32_t pipe = 0;
1376 for (uint32_t py = 0; py < fb->pipe_count.height; py++) {
1377 for (uint32_t px = 0; px < fb->pipe_count.width; px++, pipe++) {
1378 uint32_t tx1 = px * fb->pipe0.width;
1379 uint32_t ty1 = py * fb->pipe0.height;
1380 uint32_t tx2 = MIN2(tx1 + fb->pipe0.width, fb->tile_count.width);
1381 uint32_t ty2 = MIN2(ty1 + fb->pipe0.height, fb->tile_count.height);
1382 uint32_t slot = 0;
1383 for (uint32_t ty = ty1; ty < ty2; ty++) {
1384 for (uint32_t tx = tx1; tx < tx2; tx++, slot++) {
1385 tu6_emit_tile_select(cmd, &cmd->cs, tx, ty, pipe, slot);
1386 tu6_render_tile(cmd, &cmd->cs);
1387 }
1388 }
1389 }
1390 }
1391
1392 tu6_tile_render_end(cmd, &cmd->cs);
1393 }
1394
1395 static void
1396 tu_cmd_render_sysmem(struct tu_cmd_buffer *cmd)
1397 {
1398 tu6_sysmem_render_begin(cmd, &cmd->cs);
1399
1400 tu_cs_emit_call(&cmd->cs, &cmd->draw_cs);
1401
1402 tu6_sysmem_render_end(cmd, &cmd->cs);
1403 }
1404
1405 static void
1406 tu_cmd_prepare_tile_store_ib(struct tu_cmd_buffer *cmd)
1407 {
1408 const uint32_t tile_store_space = 11 + (35 * 2) * cmd->state.pass->attachment_count;
1409 struct tu_cs sub_cs;
1410
1411 VkResult result =
1412 tu_cs_begin_sub_stream(&cmd->sub_cs, tile_store_space, &sub_cs);
1413 if (result != VK_SUCCESS) {
1414 cmd->record_result = result;
1415 return;
1416 }
1417
1418 /* emit to tile-store sub_cs */
1419 tu6_emit_tile_store(cmd, &sub_cs);
1420
1421 cmd->state.tile_store_ib = tu_cs_end_sub_stream(&cmd->sub_cs, &sub_cs);
1422 }
1423
1424 static VkResult
1425 tu_create_cmd_buffer(struct tu_device *device,
1426 struct tu_cmd_pool *pool,
1427 VkCommandBufferLevel level,
1428 VkCommandBuffer *pCommandBuffer)
1429 {
1430 struct tu_cmd_buffer *cmd_buffer;
1431
1432 cmd_buffer = vk_object_zalloc(&device->vk, NULL, sizeof(*cmd_buffer),
1433 VK_OBJECT_TYPE_COMMAND_BUFFER);
1434 if (cmd_buffer == NULL)
1435 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
1436
1437 cmd_buffer->device = device;
1438 cmd_buffer->pool = pool;
1439 cmd_buffer->level = level;
1440
1441 if (pool) {
1442 list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers);
1443 cmd_buffer->queue_family_index = pool->queue_family_index;
1444
1445 } else {
1446 /* Init the pool_link so we can safely call list_del when we destroy
1447 * the command buffer
1448 */
1449 list_inithead(&cmd_buffer->pool_link);
1450 cmd_buffer->queue_family_index = TU_QUEUE_GENERAL;
1451 }
1452
1453 tu_bo_list_init(&cmd_buffer->bo_list);
1454 tu_cs_init(&cmd_buffer->cs, device, TU_CS_MODE_GROW, 4096);
1455 tu_cs_init(&cmd_buffer->draw_cs, device, TU_CS_MODE_GROW, 4096);
1456 tu_cs_init(&cmd_buffer->draw_epilogue_cs, device, TU_CS_MODE_GROW, 4096);
1457 tu_cs_init(&cmd_buffer->sub_cs, device, TU_CS_MODE_SUB_STREAM, 2048);
1458
1459 *pCommandBuffer = tu_cmd_buffer_to_handle(cmd_buffer);
1460
1461 list_inithead(&cmd_buffer->upload.list);
1462
1463 return VK_SUCCESS;
1464 }
1465
1466 static void
1467 tu_cmd_buffer_destroy(struct tu_cmd_buffer *cmd_buffer)
1468 {
1469 list_del(&cmd_buffer->pool_link);
1470
1471 tu_cs_finish(&cmd_buffer->cs);
1472 tu_cs_finish(&cmd_buffer->draw_cs);
1473 tu_cs_finish(&cmd_buffer->draw_epilogue_cs);
1474 tu_cs_finish(&cmd_buffer->sub_cs);
1475
1476 tu_bo_list_destroy(&cmd_buffer->bo_list);
1477 vk_object_free(&cmd_buffer->device->vk, &cmd_buffer->pool->alloc, cmd_buffer);
1478 }
1479
1480 static VkResult
1481 tu_reset_cmd_buffer(struct tu_cmd_buffer *cmd_buffer)
1482 {
1483 cmd_buffer->record_result = VK_SUCCESS;
1484
1485 tu_bo_list_reset(&cmd_buffer->bo_list);
1486 tu_cs_reset(&cmd_buffer->cs);
1487 tu_cs_reset(&cmd_buffer->draw_cs);
1488 tu_cs_reset(&cmd_buffer->draw_epilogue_cs);
1489 tu_cs_reset(&cmd_buffer->sub_cs);
1490
1491 for (unsigned i = 0; i < MAX_BIND_POINTS; i++)
1492 memset(&cmd_buffer->descriptors[i].sets, 0, sizeof(cmd_buffer->descriptors[i].sets));
1493
1494 cmd_buffer->status = TU_CMD_BUFFER_STATUS_INITIAL;
1495
1496 return cmd_buffer->record_result;
1497 }
1498
1499 VkResult
1500 tu_AllocateCommandBuffers(VkDevice _device,
1501 const VkCommandBufferAllocateInfo *pAllocateInfo,
1502 VkCommandBuffer *pCommandBuffers)
1503 {
1504 TU_FROM_HANDLE(tu_device, device, _device);
1505 TU_FROM_HANDLE(tu_cmd_pool, pool, pAllocateInfo->commandPool);
1506
1507 VkResult result = VK_SUCCESS;
1508 uint32_t i;
1509
1510 for (i = 0; i < pAllocateInfo->commandBufferCount; i++) {
1511
1512 if (!list_is_empty(&pool->free_cmd_buffers)) {
1513 struct tu_cmd_buffer *cmd_buffer = list_first_entry(
1514 &pool->free_cmd_buffers, struct tu_cmd_buffer, pool_link);
1515
1516 list_del(&cmd_buffer->pool_link);
1517 list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers);
1518
1519 result = tu_reset_cmd_buffer(cmd_buffer);
1520 cmd_buffer->level = pAllocateInfo->level;
1521
1522 pCommandBuffers[i] = tu_cmd_buffer_to_handle(cmd_buffer);
1523 } else {
1524 result = tu_create_cmd_buffer(device, pool, pAllocateInfo->level,
1525 &pCommandBuffers[i]);
1526 }
1527 if (result != VK_SUCCESS)
1528 break;
1529 }
1530
1531 if (result != VK_SUCCESS) {
1532 tu_FreeCommandBuffers(_device, pAllocateInfo->commandPool, i,
1533 pCommandBuffers);
1534
1535 /* From the Vulkan 1.0.66 spec:
1536 *
1537 * "vkAllocateCommandBuffers can be used to create multiple
1538 * command buffers. If the creation of any of those command
1539 * buffers fails, the implementation must destroy all
1540 * successfully created command buffer objects from this
1541 * command, set all entries of the pCommandBuffers array to
1542 * NULL and return the error."
1543 */
1544 memset(pCommandBuffers, 0,
1545 sizeof(*pCommandBuffers) * pAllocateInfo->commandBufferCount);
1546 }
1547
1548 return result;
1549 }
1550
1551 void
1552 tu_FreeCommandBuffers(VkDevice device,
1553 VkCommandPool commandPool,
1554 uint32_t commandBufferCount,
1555 const VkCommandBuffer *pCommandBuffers)
1556 {
1557 for (uint32_t i = 0; i < commandBufferCount; i++) {
1558 TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, pCommandBuffers[i]);
1559
1560 if (cmd_buffer) {
1561 if (cmd_buffer->pool) {
1562 list_del(&cmd_buffer->pool_link);
1563 list_addtail(&cmd_buffer->pool_link,
1564 &cmd_buffer->pool->free_cmd_buffers);
1565 } else
1566 tu_cmd_buffer_destroy(cmd_buffer);
1567 }
1568 }
1569 }
1570
1571 VkResult
1572 tu_ResetCommandBuffer(VkCommandBuffer commandBuffer,
1573 VkCommandBufferResetFlags flags)
1574 {
1575 TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
1576 return tu_reset_cmd_buffer(cmd_buffer);
1577 }
1578
1579 /* Initialize the cache, assuming all necessary flushes have happened but *not*
1580 * invalidations.
1581 */
1582 static void
1583 tu_cache_init(struct tu_cache_state *cache)
1584 {
1585 cache->flush_bits = 0;
1586 cache->pending_flush_bits = TU_CMD_FLAG_ALL_INVALIDATE;
1587 }
1588
1589 VkResult
1590 tu_BeginCommandBuffer(VkCommandBuffer commandBuffer,
1591 const VkCommandBufferBeginInfo *pBeginInfo)
1592 {
1593 TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
1594 VkResult result = VK_SUCCESS;
1595
1596 if (cmd_buffer->status != TU_CMD_BUFFER_STATUS_INITIAL) {
1597 /* If the command buffer has already been resetted with
1598 * vkResetCommandBuffer, no need to do it again.
1599 */
1600 result = tu_reset_cmd_buffer(cmd_buffer);
1601 if (result != VK_SUCCESS)
1602 return result;
1603 }
1604
1605 memset(&cmd_buffer->state, 0, sizeof(cmd_buffer->state));
1606 cmd_buffer->state.index_size = 0xff; /* dirty restart index */
1607
1608 tu_cache_init(&cmd_buffer->state.cache);
1609 tu_cache_init(&cmd_buffer->state.renderpass_cache);
1610 cmd_buffer->usage_flags = pBeginInfo->flags;
1611
1612 tu_cs_begin(&cmd_buffer->cs);
1613 tu_cs_begin(&cmd_buffer->draw_cs);
1614 tu_cs_begin(&cmd_buffer->draw_epilogue_cs);
1615
1616 /* setup initial configuration into command buffer */
1617 if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY) {
1618 switch (cmd_buffer->queue_family_index) {
1619 case TU_QUEUE_GENERAL:
1620 tu6_init_hw(cmd_buffer, &cmd_buffer->cs);
1621 break;
1622 default:
1623 break;
1624 }
1625 } else if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY) {
1626 assert(pBeginInfo->pInheritanceInfo);
1627
1628 vk_foreach_struct(ext, pBeginInfo->pInheritanceInfo) {
1629 switch (ext->sType) {
1630 case VK_STRUCTURE_TYPE_COMMAND_BUFFER_INHERITANCE_CONDITIONAL_RENDERING_INFO_EXT: {
1631 const VkCommandBufferInheritanceConditionalRenderingInfoEXT *cond_rend = (void *) ext;
1632 cmd_buffer->state.predication_active = cond_rend->conditionalRenderingEnable;
1633 break;
1634 default:
1635 break;
1636 }
1637 }
1638 }
1639
1640 if (pBeginInfo->flags & VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT) {
1641 cmd_buffer->state.pass = tu_render_pass_from_handle(pBeginInfo->pInheritanceInfo->renderPass);
1642 cmd_buffer->state.subpass =
1643 &cmd_buffer->state.pass->subpasses[pBeginInfo->pInheritanceInfo->subpass];
1644 } else {
1645 /* When executing in the middle of another command buffer, the CCU
1646 * state is unknown.
1647 */
1648 cmd_buffer->state.ccu_state = TU_CMD_CCU_UNKNOWN;
1649 }
1650 }
1651
1652 cmd_buffer->status = TU_CMD_BUFFER_STATUS_RECORDING;
1653
1654 return VK_SUCCESS;
1655 }
1656
1657 /* Sets vertex buffers to HW binding points. We emit VBs in SDS (so that bin
1658 * rendering can skip over unused state), so we need to collect all the
1659 * bindings together into a single state emit at draw time.
1660 */
1661 void
1662 tu_CmdBindVertexBuffers(VkCommandBuffer commandBuffer,
1663 uint32_t firstBinding,
1664 uint32_t bindingCount,
1665 const VkBuffer *pBuffers,
1666 const VkDeviceSize *pOffsets)
1667 {
1668 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
1669
1670 assert(firstBinding + bindingCount <= MAX_VBS);
1671
1672 for (uint32_t i = 0; i < bindingCount; i++) {
1673 struct tu_buffer *buf = tu_buffer_from_handle(pBuffers[i]);
1674
1675 cmd->state.vb.buffers[firstBinding + i] = buf;
1676 cmd->state.vb.offsets[firstBinding + i] = pOffsets[i];
1677
1678 tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
1679 }
1680
1681 cmd->state.dirty |= TU_CMD_DIRTY_VERTEX_BUFFERS;
1682 }
1683
1684 void
1685 tu_CmdBindIndexBuffer(VkCommandBuffer commandBuffer,
1686 VkBuffer buffer,
1687 VkDeviceSize offset,
1688 VkIndexType indexType)
1689 {
1690 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
1691 TU_FROM_HANDLE(tu_buffer, buf, buffer);
1692
1693
1694
1695 uint32_t index_size, index_shift, restart_index;
1696
1697 switch (indexType) {
1698 case VK_INDEX_TYPE_UINT16:
1699 index_size = INDEX4_SIZE_16_BIT;
1700 index_shift = 1;
1701 restart_index = 0xffff;
1702 break;
1703 case VK_INDEX_TYPE_UINT32:
1704 index_size = INDEX4_SIZE_32_BIT;
1705 index_shift = 2;
1706 restart_index = 0xffffffff;
1707 break;
1708 case VK_INDEX_TYPE_UINT8_EXT:
1709 index_size = INDEX4_SIZE_8_BIT;
1710 index_shift = 0;
1711 restart_index = 0xff;
1712 break;
1713 default:
1714 unreachable("invalid VkIndexType");
1715 }
1716
1717 /* initialize/update the restart index */
1718 if (cmd->state.index_size != index_size)
1719 tu_cs_emit_regs(&cmd->draw_cs, A6XX_PC_RESTART_INDEX(restart_index));
1720
1721 assert(buf->size >= offset);
1722
1723 cmd->state.index_va = buf->bo->iova + buf->bo_offset + offset;
1724 cmd->state.max_index_count = (buf->size - offset) >> index_shift;
1725 cmd->state.index_size = index_size;
1726
1727 tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
1728 }
1729
1730 void
1731 tu_CmdBindDescriptorSets(VkCommandBuffer commandBuffer,
1732 VkPipelineBindPoint pipelineBindPoint,
1733 VkPipelineLayout _layout,
1734 uint32_t firstSet,
1735 uint32_t descriptorSetCount,
1736 const VkDescriptorSet *pDescriptorSets,
1737 uint32_t dynamicOffsetCount,
1738 const uint32_t *pDynamicOffsets)
1739 {
1740 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
1741 TU_FROM_HANDLE(tu_pipeline_layout, layout, _layout);
1742 unsigned dyn_idx = 0;
1743
1744 struct tu_descriptor_state *descriptors_state =
1745 tu_get_descriptors_state(cmd, pipelineBindPoint);
1746
1747 for (unsigned i = 0; i < descriptorSetCount; ++i) {
1748 unsigned idx = i + firstSet;
1749 TU_FROM_HANDLE(tu_descriptor_set, set, pDescriptorSets[i]);
1750
1751 descriptors_state->sets[idx] = set;
1752
1753 for(unsigned j = 0; j < set->layout->dynamic_offset_count; ++j, ++dyn_idx) {
1754 /* update the contents of the dynamic descriptor set */
1755 unsigned src_idx = j;
1756 unsigned dst_idx = j + layout->set[idx].dynamic_offset_start;
1757 assert(dyn_idx < dynamicOffsetCount);
1758
1759 uint32_t *dst =
1760 &descriptors_state->dynamic_descriptors[dst_idx * A6XX_TEX_CONST_DWORDS];
1761 uint32_t *src =
1762 &set->dynamic_descriptors[src_idx * A6XX_TEX_CONST_DWORDS];
1763 uint32_t offset = pDynamicOffsets[dyn_idx];
1764
1765 /* Patch the storage/uniform descriptors right away. */
1766 if (layout->set[idx].layout->dynamic_ubo & (1 << j)) {
1767 /* Note: we can assume here that the addition won't roll over and
1768 * change the SIZE field.
1769 */
1770 uint64_t va = src[0] | ((uint64_t)src[1] << 32);
1771 va += offset;
1772 dst[0] = va;
1773 dst[1] = va >> 32;
1774 } else {
1775 memcpy(dst, src, A6XX_TEX_CONST_DWORDS * 4);
1776 /* Note: A6XX_IBO_5_DEPTH is always 0 */
1777 uint64_t va = dst[4] | ((uint64_t)dst[5] << 32);
1778 va += offset;
1779 dst[4] = va;
1780 dst[5] = va >> 32;
1781 }
1782 }
1783
1784 for (unsigned j = 0; j < set->layout->buffer_count; ++j) {
1785 if (set->buffers[j]) {
1786 tu_bo_list_add(&cmd->bo_list, set->buffers[j],
1787 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
1788 }
1789 }
1790
1791 if (set->size > 0) {
1792 tu_bo_list_add(&cmd->bo_list, &set->pool->bo,
1793 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
1794 }
1795 }
1796 assert(dyn_idx == dynamicOffsetCount);
1797
1798 uint32_t sp_bindless_base_reg, hlsq_bindless_base_reg, hlsq_invalidate_value;
1799 uint64_t addr[MAX_SETS + 1] = {};
1800 struct tu_cs *cs, state_cs;
1801
1802 for (uint32_t i = 0; i < MAX_SETS; i++) {
1803 struct tu_descriptor_set *set = descriptors_state->sets[i];
1804 if (set)
1805 addr[i] = set->va | 3;
1806 }
1807
1808 if (layout->dynamic_offset_count) {
1809 /* allocate and fill out dynamic descriptor set */
1810 struct tu_cs_memory dynamic_desc_set;
1811 VkResult result = tu_cs_alloc(&cmd->sub_cs, layout->dynamic_offset_count,
1812 A6XX_TEX_CONST_DWORDS, &dynamic_desc_set);
1813 assert(result == VK_SUCCESS);
1814
1815 memcpy(dynamic_desc_set.map, descriptors_state->dynamic_descriptors,
1816 layout->dynamic_offset_count * A6XX_TEX_CONST_DWORDS * 4);
1817 addr[MAX_SETS] = dynamic_desc_set.iova | 3;
1818 }
1819
1820 if (pipelineBindPoint == VK_PIPELINE_BIND_POINT_GRAPHICS) {
1821 sp_bindless_base_reg = REG_A6XX_SP_BINDLESS_BASE(0);
1822 hlsq_bindless_base_reg = REG_A6XX_HLSQ_BINDLESS_BASE(0);
1823 hlsq_invalidate_value = A6XX_HLSQ_INVALIDATE_CMD_GFX_BINDLESS(0x1f);
1824
1825 cmd->state.desc_sets = tu_cs_draw_state(&cmd->sub_cs, &state_cs, 24);
1826 cmd->state.dirty |= TU_CMD_DIRTY_DESC_SETS_LOAD | TU_CMD_DIRTY_SHADER_CONSTS;
1827 cs = &state_cs;
1828 } else {
1829 assert(pipelineBindPoint == VK_PIPELINE_BIND_POINT_COMPUTE);
1830
1831 sp_bindless_base_reg = REG_A6XX_SP_CS_BINDLESS_BASE(0);
1832 hlsq_bindless_base_reg = REG_A6XX_HLSQ_CS_BINDLESS_BASE(0);
1833 hlsq_invalidate_value = A6XX_HLSQ_INVALIDATE_CMD_CS_BINDLESS(0x1f);
1834
1835 cmd->state.dirty |= TU_CMD_DIRTY_COMPUTE_DESC_SETS_LOAD;
1836 cs = &cmd->cs;
1837 }
1838
1839 tu_cs_emit_pkt4(cs, sp_bindless_base_reg, 10);
1840 tu_cs_emit_array(cs, (const uint32_t*) addr, 10);
1841 tu_cs_emit_pkt4(cs, hlsq_bindless_base_reg, 10);
1842 tu_cs_emit_array(cs, (const uint32_t*) addr, 10);
1843 tu_cs_emit_regs(cs, A6XX_HLSQ_INVALIDATE_CMD(.dword = hlsq_invalidate_value));
1844
1845 if (pipelineBindPoint == VK_PIPELINE_BIND_POINT_GRAPHICS) {
1846 assert(cs->cur == cs->end); /* validate draw state size */
1847 tu_cs_emit_pkt7(&cmd->draw_cs, CP_SET_DRAW_STATE, 3);
1848 tu_cs_emit_draw_state(&cmd->draw_cs, TU_DRAW_STATE_DESC_SETS, cmd->state.desc_sets);
1849 }
1850 }
1851
1852 void tu_CmdBindTransformFeedbackBuffersEXT(VkCommandBuffer commandBuffer,
1853 uint32_t firstBinding,
1854 uint32_t bindingCount,
1855 const VkBuffer *pBuffers,
1856 const VkDeviceSize *pOffsets,
1857 const VkDeviceSize *pSizes)
1858 {
1859 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
1860 struct tu_cs *cs = &cmd->draw_cs;
1861
1862 /* using COND_REG_EXEC for xfb commands matches the blob behavior
1863 * presumably there isn't any benefit using a draw state when the
1864 * condition is (SYSMEM | BINNING)
1865 */
1866 tu_cond_exec_start(cs, CP_COND_REG_EXEC_0_MODE(RENDER_MODE) |
1867 CP_COND_REG_EXEC_0_SYSMEM |
1868 CP_COND_REG_EXEC_0_BINNING);
1869
1870 for (uint32_t i = 0; i < bindingCount; i++) {
1871 TU_FROM_HANDLE(tu_buffer, buf, pBuffers[i]);
1872 uint64_t iova = buf->bo->iova + pOffsets[i];
1873 uint32_t size = buf->bo->size - pOffsets[i];
1874 uint32_t idx = i + firstBinding;
1875
1876 if (pSizes && pSizes[i] != VK_WHOLE_SIZE)
1877 size = pSizes[i];
1878
1879 /* BUFFER_BASE is 32-byte aligned, add remaining offset to BUFFER_OFFSET */
1880 uint32_t offset = iova & 0x1f;
1881 iova &= ~(uint64_t) 0x1f;
1882
1883 tu_cs_emit_pkt4(cs, REG_A6XX_VPC_SO_BUFFER_BASE(idx), 3);
1884 tu_cs_emit_qw(cs, iova);
1885 tu_cs_emit(cs, size + offset);
1886
1887 cmd->state.streamout_offset[idx] = offset;
1888
1889 tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_WRITE);
1890 }
1891
1892 tu_cond_exec_end(cs);
1893 }
1894
1895 void
1896 tu_CmdBeginTransformFeedbackEXT(VkCommandBuffer commandBuffer,
1897 uint32_t firstCounterBuffer,
1898 uint32_t counterBufferCount,
1899 const VkBuffer *pCounterBuffers,
1900 const VkDeviceSize *pCounterBufferOffsets)
1901 {
1902 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
1903 struct tu_cs *cs = &cmd->draw_cs;
1904
1905 tu_cond_exec_start(cs, CP_COND_REG_EXEC_0_MODE(RENDER_MODE) |
1906 CP_COND_REG_EXEC_0_SYSMEM |
1907 CP_COND_REG_EXEC_0_BINNING);
1908
1909 /* TODO: only update offset for active buffers */
1910 for (uint32_t i = 0; i < IR3_MAX_SO_BUFFERS; i++)
1911 tu_cs_emit_regs(cs, A6XX_VPC_SO_BUFFER_OFFSET(i, cmd->state.streamout_offset[i]));
1912
1913 for (uint32_t i = 0; i < counterBufferCount; i++) {
1914 uint32_t idx = firstCounterBuffer + i;
1915 uint32_t offset = cmd->state.streamout_offset[idx];
1916
1917 if (!pCounterBuffers[i])
1918 continue;
1919
1920 TU_FROM_HANDLE(tu_buffer, buf, pCounterBuffers[i]);
1921
1922 tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
1923
1924 tu_cs_emit_pkt7(cs, CP_MEM_TO_REG, 3);
1925 tu_cs_emit(cs, CP_MEM_TO_REG_0_REG(REG_A6XX_VPC_SO_BUFFER_OFFSET(idx)) |
1926 CP_MEM_TO_REG_0_UNK31 |
1927 CP_MEM_TO_REG_0_CNT(1));
1928 tu_cs_emit_qw(cs, buf->bo->iova + pCounterBufferOffsets[i]);
1929
1930 if (offset) {
1931 tu_cs_emit_pkt7(cs, CP_REG_RMW, 3);
1932 tu_cs_emit(cs, CP_REG_RMW_0_DST_REG(REG_A6XX_VPC_SO_BUFFER_OFFSET(idx)) |
1933 CP_REG_RMW_0_SRC1_ADD);
1934 tu_cs_emit_qw(cs, 0xffffffff);
1935 tu_cs_emit_qw(cs, offset);
1936 }
1937 }
1938
1939 tu_cond_exec_end(cs);
1940 }
1941
1942 void tu_CmdEndTransformFeedbackEXT(VkCommandBuffer commandBuffer,
1943 uint32_t firstCounterBuffer,
1944 uint32_t counterBufferCount,
1945 const VkBuffer *pCounterBuffers,
1946 const VkDeviceSize *pCounterBufferOffsets)
1947 {
1948 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
1949 struct tu_cs *cs = &cmd->draw_cs;
1950
1951 tu_cond_exec_start(cs, CP_COND_REG_EXEC_0_MODE(RENDER_MODE) |
1952 CP_COND_REG_EXEC_0_SYSMEM |
1953 CP_COND_REG_EXEC_0_BINNING);
1954
1955 /* TODO: only flush buffers that need to be flushed */
1956 for (uint32_t i = 0; i < IR3_MAX_SO_BUFFERS; i++) {
1957 /* note: FLUSH_BASE is always the same, so it could go in init_hw()? */
1958 tu_cs_emit_pkt4(cs, REG_A6XX_VPC_SO_FLUSH_BASE(i), 2);
1959 tu_cs_emit_qw(cs, global_iova(cmd, flush_base[i]));
1960 tu6_emit_event_write(cmd, cs, FLUSH_SO_0 + i);
1961 }
1962
1963 for (uint32_t i = 0; i < counterBufferCount; i++) {
1964 uint32_t idx = firstCounterBuffer + i;
1965 uint32_t offset = cmd->state.streamout_offset[idx];
1966
1967 if (!pCounterBuffers[i])
1968 continue;
1969
1970 TU_FROM_HANDLE(tu_buffer, buf, pCounterBuffers[i]);
1971
1972 tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_WRITE);
1973
1974 /* VPC_SO_FLUSH_BASE has dwords counter, but counter should be in bytes */
1975 tu_cs_emit_pkt7(cs, CP_MEM_TO_REG, 3);
1976 tu_cs_emit(cs, CP_MEM_TO_REG_0_REG(REG_A6XX_CP_SCRATCH_REG(0)) |
1977 CP_MEM_TO_REG_0_SHIFT_BY_2 |
1978 0x40000 | /* ??? */
1979 CP_MEM_TO_REG_0_UNK31 |
1980 CP_MEM_TO_REG_0_CNT(1));
1981 tu_cs_emit_qw(cs, global_iova(cmd, flush_base[idx]));
1982
1983 if (offset) {
1984 tu_cs_emit_pkt7(cs, CP_REG_RMW, 3);
1985 tu_cs_emit(cs, CP_REG_RMW_0_DST_REG(REG_A6XX_CP_SCRATCH_REG(0)) |
1986 CP_REG_RMW_0_SRC1_ADD);
1987 tu_cs_emit_qw(cs, 0xffffffff);
1988 tu_cs_emit_qw(cs, -offset);
1989 }
1990
1991 tu_cs_emit_pkt7(cs, CP_REG_TO_MEM, 3);
1992 tu_cs_emit(cs, CP_REG_TO_MEM_0_REG(REG_A6XX_CP_SCRATCH_REG(0)) |
1993 CP_REG_TO_MEM_0_CNT(1));
1994 tu_cs_emit_qw(cs, buf->bo->iova + pCounterBufferOffsets[i]);
1995 }
1996
1997 tu_cond_exec_end(cs);
1998
1999 cmd->state.xfb_used = true;
2000 }
2001
2002 void
2003 tu_CmdPushConstants(VkCommandBuffer commandBuffer,
2004 VkPipelineLayout layout,
2005 VkShaderStageFlags stageFlags,
2006 uint32_t offset,
2007 uint32_t size,
2008 const void *pValues)
2009 {
2010 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2011 memcpy((void*) cmd->push_constants + offset, pValues, size);
2012 cmd->state.dirty |= TU_CMD_DIRTY_SHADER_CONSTS;
2013 }
2014
2015 /* Flush everything which has been made available but we haven't actually
2016 * flushed yet.
2017 */
2018 static void
2019 tu_flush_all_pending(struct tu_cache_state *cache)
2020 {
2021 cache->flush_bits |= cache->pending_flush_bits & TU_CMD_FLAG_ALL_FLUSH;
2022 cache->pending_flush_bits &= ~TU_CMD_FLAG_ALL_FLUSH;
2023 }
2024
2025 VkResult
2026 tu_EndCommandBuffer(VkCommandBuffer commandBuffer)
2027 {
2028 TU_FROM_HANDLE(tu_cmd_buffer, cmd_buffer, commandBuffer);
2029
2030 /* We currently flush CCU at the end of the command buffer, like
2031 * what the blob does. There's implicit synchronization around every
2032 * vkQueueSubmit, but the kernel only flushes the UCHE, and we don't
2033 * know yet if this command buffer will be the last in the submit so we
2034 * have to defensively flush everything else.
2035 *
2036 * TODO: We could definitely do better than this, since these flushes
2037 * aren't required by Vulkan, but we'd need kernel support to do that.
2038 * Ideally, we'd like the kernel to flush everything afterwards, so that we
2039 * wouldn't have to do any flushes here, and when submitting multiple
2040 * command buffers there wouldn't be any unnecessary flushes in between.
2041 */
2042 if (cmd_buffer->state.pass) {
2043 tu_flush_all_pending(&cmd_buffer->state.renderpass_cache);
2044 tu_emit_cache_flush_renderpass(cmd_buffer, &cmd_buffer->draw_cs);
2045 } else {
2046 tu_flush_all_pending(&cmd_buffer->state.cache);
2047 cmd_buffer->state.cache.flush_bits |=
2048 TU_CMD_FLAG_CCU_FLUSH_COLOR |
2049 TU_CMD_FLAG_CCU_FLUSH_DEPTH;
2050 tu_emit_cache_flush(cmd_buffer, &cmd_buffer->cs);
2051 }
2052
2053 tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->device->global_bo,
2054 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
2055
2056 for (uint32_t i = 0; i < cmd_buffer->draw_cs.bo_count; i++) {
2057 tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->draw_cs.bos[i],
2058 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2059 }
2060
2061 for (uint32_t i = 0; i < cmd_buffer->draw_epilogue_cs.bo_count; i++) {
2062 tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->draw_epilogue_cs.bos[i],
2063 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2064 }
2065
2066 for (uint32_t i = 0; i < cmd_buffer->sub_cs.bo_count; i++) {
2067 tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->sub_cs.bos[i],
2068 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2069 }
2070
2071 tu_cs_end(&cmd_buffer->cs);
2072 tu_cs_end(&cmd_buffer->draw_cs);
2073 tu_cs_end(&cmd_buffer->draw_epilogue_cs);
2074
2075 cmd_buffer->status = TU_CMD_BUFFER_STATUS_EXECUTABLE;
2076
2077 return cmd_buffer->record_result;
2078 }
2079
2080 static struct tu_cs
2081 tu_cmd_dynamic_state(struct tu_cmd_buffer *cmd, uint32_t id, uint32_t size)
2082 {
2083 struct tu_cs cs;
2084
2085 assert(id < ARRAY_SIZE(cmd->state.dynamic_state));
2086 cmd->state.dynamic_state[id] = tu_cs_draw_state(&cmd->sub_cs, &cs, size);
2087
2088 tu_cs_emit_pkt7(&cmd->draw_cs, CP_SET_DRAW_STATE, 3);
2089 tu_cs_emit_draw_state(&cmd->draw_cs, TU_DRAW_STATE_DYNAMIC + id, cmd->state.dynamic_state[id]);
2090
2091 return cs;
2092 }
2093
2094 void
2095 tu_CmdBindPipeline(VkCommandBuffer commandBuffer,
2096 VkPipelineBindPoint pipelineBindPoint,
2097 VkPipeline _pipeline)
2098 {
2099 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2100 TU_FROM_HANDLE(tu_pipeline, pipeline, _pipeline);
2101
2102 for (uint32_t i = 0; i < pipeline->cs.bo_count; i++) {
2103 tu_bo_list_add(&cmd->bo_list, pipeline->cs.bos[i],
2104 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2105 }
2106
2107 if (pipelineBindPoint == VK_PIPELINE_BIND_POINT_COMPUTE) {
2108 cmd->state.compute_pipeline = pipeline;
2109 tu_cs_emit_state_ib(&cmd->cs, pipeline->program.state);
2110 return;
2111 }
2112
2113 assert(pipelineBindPoint == VK_PIPELINE_BIND_POINT_GRAPHICS);
2114
2115 cmd->state.pipeline = pipeline;
2116 cmd->state.dirty |= TU_CMD_DIRTY_DESC_SETS_LOAD | TU_CMD_DIRTY_SHADER_CONSTS;
2117
2118 struct tu_cs *cs = &cmd->draw_cs;
2119 uint32_t mask = ~pipeline->dynamic_state_mask & BITFIELD_MASK(TU_DYNAMIC_STATE_COUNT);
2120 uint32_t i;
2121
2122 tu_cs_emit_pkt7(cs, CP_SET_DRAW_STATE, 3 * (7 + util_bitcount(mask)));
2123 tu_cs_emit_draw_state(cs, TU_DRAW_STATE_PROGRAM, pipeline->program.state);
2124 tu_cs_emit_draw_state(cs, TU_DRAW_STATE_PROGRAM_BINNING, pipeline->program.binning_state);
2125 tu_cs_emit_draw_state(cs, TU_DRAW_STATE_VI, pipeline->vi.state);
2126 tu_cs_emit_draw_state(cs, TU_DRAW_STATE_VI_BINNING, pipeline->vi.binning_state);
2127 tu_cs_emit_draw_state(cs, TU_DRAW_STATE_RAST, pipeline->rast_state);
2128 tu_cs_emit_draw_state(cs, TU_DRAW_STATE_DS, pipeline->ds_state);
2129 tu_cs_emit_draw_state(cs, TU_DRAW_STATE_BLEND, pipeline->blend_state);
2130 for_each_bit(i, mask)
2131 tu_cs_emit_draw_state(cs, TU_DRAW_STATE_DYNAMIC + i, pipeline->dynamic_state[i]);
2132
2133 /* If the new pipeline requires more VBs than we had previously set up, we
2134 * need to re-emit them in SDS. If it requires the same set or fewer, we
2135 * can just re-use the old SDS.
2136 */
2137 if (pipeline->vi.bindings_used & ~cmd->vertex_bindings_set)
2138 cmd->state.dirty |= TU_CMD_DIRTY_VERTEX_BUFFERS;
2139
2140 /* dynamic linewidth state depends pipeline state's gras_su_cntl
2141 * so the dynamic state ib must be updated when pipeline changes
2142 */
2143 if (pipeline->dynamic_state_mask & BIT(VK_DYNAMIC_STATE_LINE_WIDTH)) {
2144 struct tu_cs cs = tu_cmd_dynamic_state(cmd, VK_DYNAMIC_STATE_LINE_WIDTH, 2);
2145
2146 cmd->state.dynamic_gras_su_cntl &= A6XX_GRAS_SU_CNTL_LINEHALFWIDTH__MASK;
2147 cmd->state.dynamic_gras_su_cntl |= pipeline->gras_su_cntl;
2148
2149 tu_cs_emit_regs(&cs, A6XX_GRAS_SU_CNTL(.dword = cmd->state.dynamic_gras_su_cntl));
2150 }
2151 }
2152
2153 void
2154 tu_CmdSetViewport(VkCommandBuffer commandBuffer,
2155 uint32_t firstViewport,
2156 uint32_t viewportCount,
2157 const VkViewport *pViewports)
2158 {
2159 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2160 struct tu_cs cs = tu_cmd_dynamic_state(cmd, VK_DYNAMIC_STATE_VIEWPORT, 18);
2161
2162 assert(firstViewport == 0 && viewportCount == 1);
2163
2164 tu6_emit_viewport(&cs, pViewports);
2165 }
2166
2167 void
2168 tu_CmdSetScissor(VkCommandBuffer commandBuffer,
2169 uint32_t firstScissor,
2170 uint32_t scissorCount,
2171 const VkRect2D *pScissors)
2172 {
2173 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2174 struct tu_cs cs = tu_cmd_dynamic_state(cmd, VK_DYNAMIC_STATE_SCISSOR, 3);
2175
2176 assert(firstScissor == 0 && scissorCount == 1);
2177
2178 tu6_emit_scissor(&cs, pScissors);
2179 }
2180
2181 void
2182 tu_CmdSetLineWidth(VkCommandBuffer commandBuffer, float lineWidth)
2183 {
2184 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2185 struct tu_cs cs = tu_cmd_dynamic_state(cmd, VK_DYNAMIC_STATE_LINE_WIDTH, 2);
2186
2187 cmd->state.dynamic_gras_su_cntl &= ~A6XX_GRAS_SU_CNTL_LINEHALFWIDTH__MASK;
2188 cmd->state.dynamic_gras_su_cntl |= A6XX_GRAS_SU_CNTL_LINEHALFWIDTH(lineWidth / 2.0f);
2189
2190 tu_cs_emit_regs(&cs, A6XX_GRAS_SU_CNTL(.dword = cmd->state.dynamic_gras_su_cntl));
2191 }
2192
2193 void
2194 tu_CmdSetDepthBias(VkCommandBuffer commandBuffer,
2195 float depthBiasConstantFactor,
2196 float depthBiasClamp,
2197 float depthBiasSlopeFactor)
2198 {
2199 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2200 struct tu_cs cs = tu_cmd_dynamic_state(cmd, VK_DYNAMIC_STATE_DEPTH_BIAS, 4);
2201
2202 tu6_emit_depth_bias(&cs, depthBiasConstantFactor, depthBiasClamp, depthBiasSlopeFactor);
2203 }
2204
2205 void
2206 tu_CmdSetBlendConstants(VkCommandBuffer commandBuffer,
2207 const float blendConstants[4])
2208 {
2209 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2210 struct tu_cs cs = tu_cmd_dynamic_state(cmd, VK_DYNAMIC_STATE_BLEND_CONSTANTS, 5);
2211
2212 tu_cs_emit_pkt4(&cs, REG_A6XX_RB_BLEND_RED_F32, 4);
2213 tu_cs_emit_array(&cs, (const uint32_t *) blendConstants, 4);
2214 }
2215
2216 void
2217 tu_CmdSetDepthBounds(VkCommandBuffer commandBuffer,
2218 float minDepthBounds,
2219 float maxDepthBounds)
2220 {
2221 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2222 struct tu_cs cs = tu_cmd_dynamic_state(cmd, VK_DYNAMIC_STATE_DEPTH_BOUNDS, 3);
2223
2224 tu_cs_emit_regs(&cs,
2225 A6XX_RB_Z_BOUNDS_MIN(minDepthBounds),
2226 A6XX_RB_Z_BOUNDS_MAX(maxDepthBounds));
2227 }
2228
2229 static void
2230 update_stencil_mask(uint32_t *value, VkStencilFaceFlags face, uint32_t mask)
2231 {
2232 if (face & VK_STENCIL_FACE_FRONT_BIT)
2233 *value = (*value & 0xff00) | (mask & 0xff);
2234 if (face & VK_STENCIL_FACE_BACK_BIT)
2235 *value = (*value & 0xff) | (mask & 0xff) << 8;
2236 }
2237
2238 void
2239 tu_CmdSetStencilCompareMask(VkCommandBuffer commandBuffer,
2240 VkStencilFaceFlags faceMask,
2241 uint32_t compareMask)
2242 {
2243 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2244 struct tu_cs cs = tu_cmd_dynamic_state(cmd, VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK, 2);
2245
2246 update_stencil_mask(&cmd->state.dynamic_stencil_mask, faceMask, compareMask);
2247
2248 tu_cs_emit_regs(&cs, A6XX_RB_STENCILMASK(.dword = cmd->state.dynamic_stencil_mask));
2249 }
2250
2251 void
2252 tu_CmdSetStencilWriteMask(VkCommandBuffer commandBuffer,
2253 VkStencilFaceFlags faceMask,
2254 uint32_t writeMask)
2255 {
2256 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2257 struct tu_cs cs = tu_cmd_dynamic_state(cmd, VK_DYNAMIC_STATE_STENCIL_WRITE_MASK, 2);
2258
2259 update_stencil_mask(&cmd->state.dynamic_stencil_wrmask, faceMask, writeMask);
2260
2261 tu_cs_emit_regs(&cs, A6XX_RB_STENCILWRMASK(.dword = cmd->state.dynamic_stencil_wrmask));
2262 }
2263
2264 void
2265 tu_CmdSetStencilReference(VkCommandBuffer commandBuffer,
2266 VkStencilFaceFlags faceMask,
2267 uint32_t reference)
2268 {
2269 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2270 struct tu_cs cs = tu_cmd_dynamic_state(cmd, VK_DYNAMIC_STATE_STENCIL_REFERENCE, 2);
2271
2272 update_stencil_mask(&cmd->state.dynamic_stencil_ref, faceMask, reference);
2273
2274 tu_cs_emit_regs(&cs, A6XX_RB_STENCILREF(.dword = cmd->state.dynamic_stencil_ref));
2275 }
2276
2277 void
2278 tu_CmdSetSampleLocationsEXT(VkCommandBuffer commandBuffer,
2279 const VkSampleLocationsInfoEXT* pSampleLocationsInfo)
2280 {
2281 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2282 struct tu_cs cs = tu_cmd_dynamic_state(cmd, TU_DYNAMIC_STATE_SAMPLE_LOCATIONS, 9);
2283
2284 assert(pSampleLocationsInfo);
2285
2286 tu6_emit_sample_locations(&cs, pSampleLocationsInfo);
2287 }
2288
2289 static void
2290 tu_flush_for_access(struct tu_cache_state *cache,
2291 enum tu_cmd_access_mask src_mask,
2292 enum tu_cmd_access_mask dst_mask)
2293 {
2294 enum tu_cmd_flush_bits flush_bits = 0;
2295
2296 if (src_mask & TU_ACCESS_HOST_WRITE) {
2297 /* Host writes are always visible to CP, so only invalidate GPU caches */
2298 cache->pending_flush_bits |= TU_CMD_FLAG_GPU_INVALIDATE;
2299 }
2300
2301 if (src_mask & TU_ACCESS_SYSMEM_WRITE) {
2302 /* Invalidate CP and 2D engine (make it do WFI + WFM if necessary) as
2303 * well.
2304 */
2305 cache->pending_flush_bits |= TU_CMD_FLAG_ALL_INVALIDATE;
2306 }
2307
2308 if (src_mask & TU_ACCESS_CP_WRITE) {
2309 /* Flush the CP write queue. However a WFI shouldn't be necessary as
2310 * WAIT_MEM_WRITES should cover it.
2311 */
2312 cache->pending_flush_bits |=
2313 TU_CMD_FLAG_WAIT_MEM_WRITES |
2314 TU_CMD_FLAG_GPU_INVALIDATE |
2315 TU_CMD_FLAG_WAIT_FOR_ME;
2316 }
2317
2318 #define SRC_FLUSH(domain, flush, invalidate) \
2319 if (src_mask & TU_ACCESS_##domain##_WRITE) { \
2320 cache->pending_flush_bits |= TU_CMD_FLAG_##flush | \
2321 (TU_CMD_FLAG_ALL_INVALIDATE & ~TU_CMD_FLAG_##invalidate); \
2322 }
2323
2324 SRC_FLUSH(UCHE, CACHE_FLUSH, CACHE_INVALIDATE)
2325 SRC_FLUSH(CCU_COLOR, CCU_FLUSH_COLOR, CCU_INVALIDATE_COLOR)
2326 SRC_FLUSH(CCU_DEPTH, CCU_FLUSH_DEPTH, CCU_INVALIDATE_DEPTH)
2327
2328 #undef SRC_FLUSH
2329
2330 #define SRC_INCOHERENT_FLUSH(domain, flush, invalidate) \
2331 if (src_mask & TU_ACCESS_##domain##_INCOHERENT_WRITE) { \
2332 flush_bits |= TU_CMD_FLAG_##flush; \
2333 cache->pending_flush_bits |= \
2334 (TU_CMD_FLAG_ALL_INVALIDATE & ~TU_CMD_FLAG_##invalidate); \
2335 }
2336
2337 SRC_INCOHERENT_FLUSH(CCU_COLOR, CCU_FLUSH_COLOR, CCU_INVALIDATE_COLOR)
2338 SRC_INCOHERENT_FLUSH(CCU_DEPTH, CCU_FLUSH_DEPTH, CCU_INVALIDATE_DEPTH)
2339
2340 #undef SRC_INCOHERENT_FLUSH
2341
2342 /* Treat host & sysmem write accesses the same, since the kernel implicitly
2343 * drains the queue before signalling completion to the host.
2344 */
2345 if (dst_mask & (TU_ACCESS_SYSMEM_READ | TU_ACCESS_SYSMEM_WRITE |
2346 TU_ACCESS_HOST_READ | TU_ACCESS_HOST_WRITE)) {
2347 flush_bits |= cache->pending_flush_bits & TU_CMD_FLAG_ALL_FLUSH;
2348 }
2349
2350 #define DST_FLUSH(domain, flush, invalidate) \
2351 if (dst_mask & (TU_ACCESS_##domain##_READ | \
2352 TU_ACCESS_##domain##_WRITE)) { \
2353 flush_bits |= cache->pending_flush_bits & \
2354 (TU_CMD_FLAG_##invalidate | \
2355 (TU_CMD_FLAG_ALL_FLUSH & ~TU_CMD_FLAG_##flush)); \
2356 }
2357
2358 DST_FLUSH(UCHE, CACHE_FLUSH, CACHE_INVALIDATE)
2359 DST_FLUSH(CCU_COLOR, CCU_FLUSH_COLOR, CCU_INVALIDATE_COLOR)
2360 DST_FLUSH(CCU_DEPTH, CCU_FLUSH_DEPTH, CCU_INVALIDATE_DEPTH)
2361
2362 #undef DST_FLUSH
2363
2364 #define DST_INCOHERENT_FLUSH(domain, flush, invalidate) \
2365 if (dst_mask & (TU_ACCESS_##domain##_INCOHERENT_READ | \
2366 TU_ACCESS_##domain##_INCOHERENT_WRITE)) { \
2367 flush_bits |= TU_CMD_FLAG_##invalidate | \
2368 (cache->pending_flush_bits & \
2369 (TU_CMD_FLAG_ALL_FLUSH & ~TU_CMD_FLAG_##flush)); \
2370 }
2371
2372 DST_INCOHERENT_FLUSH(CCU_COLOR, CCU_FLUSH_COLOR, CCU_INVALIDATE_COLOR)
2373 DST_INCOHERENT_FLUSH(CCU_DEPTH, CCU_FLUSH_DEPTH, CCU_INVALIDATE_DEPTH)
2374
2375 #undef DST_INCOHERENT_FLUSH
2376
2377 if (dst_mask & TU_ACCESS_WFI_READ) {
2378 flush_bits |= cache->pending_flush_bits &
2379 (TU_CMD_FLAG_ALL_FLUSH | TU_CMD_FLAG_WAIT_FOR_IDLE);
2380 }
2381
2382 if (dst_mask & TU_ACCESS_WFM_READ) {
2383 flush_bits |= cache->pending_flush_bits &
2384 (TU_CMD_FLAG_ALL_FLUSH | TU_CMD_FLAG_WAIT_FOR_ME);
2385 }
2386
2387 cache->flush_bits |= flush_bits;
2388 cache->pending_flush_bits &= ~flush_bits;
2389 }
2390
2391 static enum tu_cmd_access_mask
2392 vk2tu_access(VkAccessFlags flags, bool gmem)
2393 {
2394 enum tu_cmd_access_mask mask = 0;
2395
2396 /* If the GPU writes a buffer that is then read by an indirect draw
2397 * command, we theoretically need to emit a WFI to wait for any cache
2398 * flushes, and then a WAIT_FOR_ME to wait on the CP for the WFI to
2399 * complete. Waiting for the WFI to complete is performed as part of the
2400 * draw by the firmware, so we just need to execute the WFI.
2401 *
2402 * Transform feedback counters are read via CP_MEM_TO_REG, which implicitly
2403 * does CP_WAIT_FOR_ME, but we still need a WFI if the GPU writes it.
2404 *
2405 * Currently we read the draw predicate using CP_MEM_TO_MEM, which
2406 * also implicitly does CP_WAIT_FOR_ME. However CP_DRAW_PRED_SET does *not*
2407 * implicitly do CP_WAIT_FOR_ME, it seems to only wait for counters to
2408 * complete since it's written for DX11 where you can only predicate on the
2409 * result of a query object. So if we implement 64-bit comparisons in the
2410 * future, or if CP_DRAW_PRED_SET grows the capability to do 32-bit
2411 * comparisons, then this will have to be dealt with.
2412 */
2413 if (flags &
2414 (VK_ACCESS_INDIRECT_COMMAND_READ_BIT |
2415 VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_READ_BIT_EXT |
2416 VK_ACCESS_CONDITIONAL_RENDERING_READ_BIT_EXT |
2417 VK_ACCESS_MEMORY_READ_BIT)) {
2418 mask |= TU_ACCESS_WFI_READ;
2419 }
2420
2421 if (flags &
2422 (VK_ACCESS_INDIRECT_COMMAND_READ_BIT | /* Read performed by CP */
2423 VK_ACCESS_CONDITIONAL_RENDERING_READ_BIT_EXT | /* Read performed by CP */
2424 VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_READ_BIT_EXT | /* Read performed by CP */
2425 VK_ACCESS_MEMORY_READ_BIT)) {
2426 mask |= TU_ACCESS_SYSMEM_READ;
2427 }
2428
2429 if (flags &
2430 (VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_WRITE_BIT_EXT |
2431 VK_ACCESS_MEMORY_WRITE_BIT)) {
2432 mask |= TU_ACCESS_CP_WRITE;
2433 }
2434
2435 if (flags &
2436 (VK_ACCESS_HOST_READ_BIT |
2437 VK_ACCESS_MEMORY_WRITE_BIT)) {
2438 mask |= TU_ACCESS_HOST_READ;
2439 }
2440
2441 if (flags &
2442 (VK_ACCESS_HOST_WRITE_BIT |
2443 VK_ACCESS_MEMORY_WRITE_BIT)) {
2444 mask |= TU_ACCESS_HOST_WRITE;
2445 }
2446
2447 if (flags &
2448 (VK_ACCESS_INDEX_READ_BIT | /* Read performed by PC, I think */
2449 VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | /* Read performed by VFD */
2450 VK_ACCESS_UNIFORM_READ_BIT | /* Read performed by SP */
2451 /* TODO: Is there a no-cache bit for textures so that we can ignore
2452 * these?
2453 */
2454 VK_ACCESS_INPUT_ATTACHMENT_READ_BIT | /* Read performed by TP */
2455 VK_ACCESS_SHADER_READ_BIT | /* Read perfomed by SP/TP */
2456 VK_ACCESS_MEMORY_READ_BIT)) {
2457 mask |= TU_ACCESS_UCHE_READ;
2458 }
2459
2460 if (flags &
2461 (VK_ACCESS_SHADER_WRITE_BIT | /* Write performed by SP */
2462 VK_ACCESS_TRANSFORM_FEEDBACK_WRITE_BIT_EXT | /* Write performed by VPC */
2463 VK_ACCESS_MEMORY_WRITE_BIT)) {
2464 mask |= TU_ACCESS_UCHE_WRITE;
2465 }
2466
2467 /* When using GMEM, the CCU is always flushed automatically to GMEM, and
2468 * then GMEM is flushed to sysmem. Furthermore, we already had to flush any
2469 * previous writes in sysmem mode when transitioning to GMEM. Therefore we
2470 * can ignore CCU and pretend that color attachments and transfers use
2471 * sysmem directly.
2472 */
2473
2474 if (flags &
2475 (VK_ACCESS_COLOR_ATTACHMENT_READ_BIT |
2476 VK_ACCESS_COLOR_ATTACHMENT_READ_NONCOHERENT_BIT_EXT |
2477 VK_ACCESS_MEMORY_READ_BIT)) {
2478 if (gmem)
2479 mask |= TU_ACCESS_SYSMEM_READ;
2480 else
2481 mask |= TU_ACCESS_CCU_COLOR_INCOHERENT_READ;
2482 }
2483
2484 if (flags &
2485 (VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT |
2486 VK_ACCESS_MEMORY_READ_BIT)) {
2487 if (gmem)
2488 mask |= TU_ACCESS_SYSMEM_READ;
2489 else
2490 mask |= TU_ACCESS_CCU_DEPTH_INCOHERENT_READ;
2491 }
2492
2493 if (flags &
2494 (VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT |
2495 VK_ACCESS_MEMORY_WRITE_BIT)) {
2496 if (gmem) {
2497 mask |= TU_ACCESS_SYSMEM_WRITE;
2498 } else {
2499 mask |= TU_ACCESS_CCU_COLOR_INCOHERENT_WRITE;
2500 }
2501 }
2502
2503 if (flags &
2504 (VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT |
2505 VK_ACCESS_MEMORY_WRITE_BIT)) {
2506 if (gmem) {
2507 mask |= TU_ACCESS_SYSMEM_WRITE;
2508 } else {
2509 mask |= TU_ACCESS_CCU_DEPTH_INCOHERENT_WRITE;
2510 }
2511 }
2512
2513 /* When the dst access is a transfer read/write, it seems we sometimes need
2514 * to insert a WFI after any flushes, to guarantee that the flushes finish
2515 * before the 2D engine starts. However the opposite (i.e. a WFI after
2516 * CP_BLIT and before any subsequent flush) does not seem to be needed, and
2517 * the blob doesn't emit such a WFI.
2518 */
2519
2520 if (flags &
2521 (VK_ACCESS_TRANSFER_WRITE_BIT |
2522 VK_ACCESS_MEMORY_WRITE_BIT)) {
2523 if (gmem) {
2524 mask |= TU_ACCESS_SYSMEM_WRITE;
2525 } else {
2526 mask |= TU_ACCESS_CCU_COLOR_WRITE;
2527 }
2528 mask |= TU_ACCESS_WFI_READ;
2529 }
2530
2531 if (flags &
2532 (VK_ACCESS_TRANSFER_READ_BIT | /* Access performed by TP */
2533 VK_ACCESS_MEMORY_READ_BIT)) {
2534 mask |= TU_ACCESS_UCHE_READ | TU_ACCESS_WFI_READ;
2535 }
2536
2537 return mask;
2538 }
2539
2540
2541 void
2542 tu_CmdExecuteCommands(VkCommandBuffer commandBuffer,
2543 uint32_t commandBufferCount,
2544 const VkCommandBuffer *pCmdBuffers)
2545 {
2546 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2547 VkResult result;
2548
2549 assert(commandBufferCount > 0);
2550
2551 /* Emit any pending flushes. */
2552 if (cmd->state.pass) {
2553 tu_flush_all_pending(&cmd->state.renderpass_cache);
2554 tu_emit_cache_flush_renderpass(cmd, &cmd->draw_cs);
2555 } else {
2556 tu_flush_all_pending(&cmd->state.cache);
2557 tu_emit_cache_flush(cmd, &cmd->cs);
2558 }
2559
2560 for (uint32_t i = 0; i < commandBufferCount; i++) {
2561 TU_FROM_HANDLE(tu_cmd_buffer, secondary, pCmdBuffers[i]);
2562
2563 result = tu_bo_list_merge(&cmd->bo_list, &secondary->bo_list);
2564 if (result != VK_SUCCESS) {
2565 cmd->record_result = result;
2566 break;
2567 }
2568
2569 if (secondary->usage_flags &
2570 VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT) {
2571 assert(tu_cs_is_empty(&secondary->cs));
2572
2573 result = tu_cs_add_entries(&cmd->draw_cs, &secondary->draw_cs);
2574 if (result != VK_SUCCESS) {
2575 cmd->record_result = result;
2576 break;
2577 }
2578
2579 result = tu_cs_add_entries(&cmd->draw_epilogue_cs,
2580 &secondary->draw_epilogue_cs);
2581 if (result != VK_SUCCESS) {
2582 cmd->record_result = result;
2583 break;
2584 }
2585
2586 if (secondary->state.has_tess)
2587 cmd->state.has_tess = true;
2588 if (secondary->state.has_subpass_predication)
2589 cmd->state.has_subpass_predication = true;
2590 } else {
2591 assert(tu_cs_is_empty(&secondary->draw_cs));
2592 assert(tu_cs_is_empty(&secondary->draw_epilogue_cs));
2593
2594 for (uint32_t j = 0; j < secondary->cs.bo_count; j++) {
2595 tu_bo_list_add(&cmd->bo_list, secondary->cs.bos[j],
2596 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
2597 }
2598
2599 tu_cs_add_entries(&cmd->cs, &secondary->cs);
2600 }
2601
2602 cmd->state.index_size = secondary->state.index_size; /* for restart index update */
2603 }
2604 cmd->state.dirty = ~0u; /* TODO: set dirty only what needs to be */
2605
2606 /* After executing secondary command buffers, there may have been arbitrary
2607 * flushes executed, so when we encounter a pipeline barrier with a
2608 * srcMask, we have to assume that we need to invalidate. Therefore we need
2609 * to re-initialize the cache with all pending invalidate bits set.
2610 */
2611 if (cmd->state.pass) {
2612 tu_cache_init(&cmd->state.renderpass_cache);
2613 } else {
2614 tu_cache_init(&cmd->state.cache);
2615 }
2616 }
2617
2618 VkResult
2619 tu_CreateCommandPool(VkDevice _device,
2620 const VkCommandPoolCreateInfo *pCreateInfo,
2621 const VkAllocationCallbacks *pAllocator,
2622 VkCommandPool *pCmdPool)
2623 {
2624 TU_FROM_HANDLE(tu_device, device, _device);
2625 struct tu_cmd_pool *pool;
2626
2627 pool = vk_object_alloc(&device->vk, pAllocator, sizeof(*pool),
2628 VK_OBJECT_TYPE_COMMAND_POOL);
2629 if (pool == NULL)
2630 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
2631
2632 if (pAllocator)
2633 pool->alloc = *pAllocator;
2634 else
2635 pool->alloc = device->vk.alloc;
2636
2637 list_inithead(&pool->cmd_buffers);
2638 list_inithead(&pool->free_cmd_buffers);
2639
2640 pool->queue_family_index = pCreateInfo->queueFamilyIndex;
2641
2642 *pCmdPool = tu_cmd_pool_to_handle(pool);
2643
2644 return VK_SUCCESS;
2645 }
2646
2647 void
2648 tu_DestroyCommandPool(VkDevice _device,
2649 VkCommandPool commandPool,
2650 const VkAllocationCallbacks *pAllocator)
2651 {
2652 TU_FROM_HANDLE(tu_device, device, _device);
2653 TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2654
2655 if (!pool)
2656 return;
2657
2658 list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2659 &pool->cmd_buffers, pool_link)
2660 {
2661 tu_cmd_buffer_destroy(cmd_buffer);
2662 }
2663
2664 list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2665 &pool->free_cmd_buffers, pool_link)
2666 {
2667 tu_cmd_buffer_destroy(cmd_buffer);
2668 }
2669
2670 vk_object_free(&device->vk, pAllocator, pool);
2671 }
2672
2673 VkResult
2674 tu_ResetCommandPool(VkDevice device,
2675 VkCommandPool commandPool,
2676 VkCommandPoolResetFlags flags)
2677 {
2678 TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2679 VkResult result;
2680
2681 list_for_each_entry(struct tu_cmd_buffer, cmd_buffer, &pool->cmd_buffers,
2682 pool_link)
2683 {
2684 result = tu_reset_cmd_buffer(cmd_buffer);
2685 if (result != VK_SUCCESS)
2686 return result;
2687 }
2688
2689 return VK_SUCCESS;
2690 }
2691
2692 void
2693 tu_TrimCommandPool(VkDevice device,
2694 VkCommandPool commandPool,
2695 VkCommandPoolTrimFlags flags)
2696 {
2697 TU_FROM_HANDLE(tu_cmd_pool, pool, commandPool);
2698
2699 if (!pool)
2700 return;
2701
2702 list_for_each_entry_safe(struct tu_cmd_buffer, cmd_buffer,
2703 &pool->free_cmd_buffers, pool_link)
2704 {
2705 tu_cmd_buffer_destroy(cmd_buffer);
2706 }
2707 }
2708
2709 static void
2710 tu_subpass_barrier(struct tu_cmd_buffer *cmd_buffer,
2711 const struct tu_subpass_barrier *barrier,
2712 bool external)
2713 {
2714 /* Note: we don't know until the end of the subpass whether we'll use
2715 * sysmem, so assume sysmem here to be safe.
2716 */
2717 struct tu_cache_state *cache =
2718 external ? &cmd_buffer->state.cache : &cmd_buffer->state.renderpass_cache;
2719 enum tu_cmd_access_mask src_flags =
2720 vk2tu_access(barrier->src_access_mask, false);
2721 enum tu_cmd_access_mask dst_flags =
2722 vk2tu_access(barrier->dst_access_mask, false);
2723
2724 if (barrier->incoherent_ccu_color)
2725 src_flags |= TU_ACCESS_CCU_COLOR_INCOHERENT_WRITE;
2726 if (barrier->incoherent_ccu_depth)
2727 src_flags |= TU_ACCESS_CCU_DEPTH_INCOHERENT_WRITE;
2728
2729 tu_flush_for_access(cache, src_flags, dst_flags);
2730 }
2731
2732 void
2733 tu_CmdBeginRenderPass(VkCommandBuffer commandBuffer,
2734 const VkRenderPassBeginInfo *pRenderPassBegin,
2735 VkSubpassContents contents)
2736 {
2737 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2738 TU_FROM_HANDLE(tu_render_pass, pass, pRenderPassBegin->renderPass);
2739 TU_FROM_HANDLE(tu_framebuffer, fb, pRenderPassBegin->framebuffer);
2740
2741 cmd->state.pass = pass;
2742 cmd->state.subpass = pass->subpasses;
2743 cmd->state.framebuffer = fb;
2744 cmd->state.render_area = pRenderPassBegin->renderArea;
2745
2746 tu_cmd_prepare_tile_store_ib(cmd);
2747
2748 /* Note: because this is external, any flushes will happen before draw_cs
2749 * gets called. However deferred flushes could have to happen later as part
2750 * of the subpass.
2751 */
2752 tu_subpass_barrier(cmd, &pass->subpasses[0].start_barrier, true);
2753 cmd->state.renderpass_cache.pending_flush_bits =
2754 cmd->state.cache.pending_flush_bits;
2755 cmd->state.renderpass_cache.flush_bits = 0;
2756
2757 tu_emit_renderpass_begin(cmd, pRenderPassBegin);
2758
2759 tu6_emit_zs(cmd, cmd->state.subpass, &cmd->draw_cs);
2760 tu6_emit_mrt(cmd, cmd->state.subpass, &cmd->draw_cs);
2761 tu6_emit_msaa(&cmd->draw_cs, cmd->state.subpass->samples);
2762 tu6_emit_render_cntl(cmd, cmd->state.subpass, &cmd->draw_cs, false);
2763
2764 tu_set_input_attachments(cmd, cmd->state.subpass);
2765
2766 for (uint32_t i = 0; i < fb->attachment_count; ++i) {
2767 const struct tu_image_view *iview = fb->attachments[i].attachment;
2768 tu_bo_list_add(&cmd->bo_list, iview->image->bo,
2769 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
2770 }
2771
2772 cmd->state.dirty |= TU_CMD_DIRTY_DRAW_STATE;
2773 }
2774
2775 void
2776 tu_CmdBeginRenderPass2(VkCommandBuffer commandBuffer,
2777 const VkRenderPassBeginInfo *pRenderPassBeginInfo,
2778 const VkSubpassBeginInfoKHR *pSubpassBeginInfo)
2779 {
2780 tu_CmdBeginRenderPass(commandBuffer, pRenderPassBeginInfo,
2781 pSubpassBeginInfo->contents);
2782 }
2783
2784 void
2785 tu_CmdNextSubpass(VkCommandBuffer commandBuffer, VkSubpassContents contents)
2786 {
2787 TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
2788 const struct tu_render_pass *pass = cmd->state.pass;
2789 struct tu_cs *cs = &cmd->draw_cs;
2790
2791 const struct tu_subpass *subpass = cmd->state.subpass++;
2792
2793 tu_cond_exec_start(cs, CP_COND_EXEC_0_RENDER_MODE_GMEM);
2794
2795 if (subpass->resolve_attachments) {
2796 tu6_emit_blit_scissor(cmd, cs, true);
2797
2798 for (unsigned i = 0; i < subpass->color_count; i++) {
2799 uint32_t a = subpass->resolve_attachments[i].attachment;
2800 if (a == VK_ATTACHMENT_UNUSED)
2801 continue;
2802
2803 tu_store_gmem_attachment(cmd, cs, a,
2804 subpass->color_attachments[i].attachment);
2805
2806 if (pass->attachments[a].gmem_offset < 0)
2807 continue;
2808
2809 /* TODO:
2810 * check if the resolved attachment is needed by later subpasses,
2811 * if it is, should be doing a GMEM->GMEM resolve instead of GMEM->MEM->GMEM..
2812 */
2813 tu_finishme("missing GMEM->GMEM resolve path\n");
2814 tu_load_gmem_attachment(cmd, cs, a, true);
2815 }
2816 }
2817
2818 tu_cond_exec_end(cs);
2819
2820 tu_cond_exec_start(cs, CP_COND_EXEC_0_RENDER_MODE_SYSMEM);
2821
2822 tu6_emit_sysmem_resolves(cmd, cs, subpass);
2823
2824 tu_cond_exec_end(cs);
2825
2826 /* Handle dependencies for the next subpass */
2827 tu_subpass_barrier(cmd, &cmd->state.subpass->start_barrier, false);
2828
2829 /* emit mrt/zs/msaa/ubwc state for the subpass that is starting */
2830 tu6_emit_zs(cmd, cmd->state.subpass, cs);
2831 tu6_emit_mrt(cmd, cmd->state.subpass, cs);
2832 tu6_emit_msaa(cs, cmd->state.subpass->samples);
2833 tu6_emit_render_cntl(cmd, cmd->state.subpass, cs, false);
2834
2835 tu_set_input_attachments(cmd, cmd->state.subpass);
2836 }
2837
2838 void
2839 tu_CmdNextSubpass2(VkCommandBuffer commandBuffer,
2840 const VkSubpassBeginInfoKHR *pSubpassBeginInfo,
2841 const VkSubpassEndInfoKHR *pSubpassEndInfo)
2842 {
2843 tu_CmdNextSubpass(commandBuffer, pSubpassBeginInfo->contents);
2844 }
2845
2846 static void
2847 tu6_emit_user_consts(struct tu_cs *cs, const struct tu_pipeline *pipeline,
2848 struct tu_descriptor_state *descriptors_state,
2849 gl_shader_stage type,
2850 uint32_t *push_constants)
2851 {
2852 const struct tu_program_descriptor_linkage *link =
2853 &pipeline->program.link[type];
2854 const struct ir3_ubo_analysis_state *state = &link->const_state.ubo_state;
2855
2856 if (link->push_consts.count > 0) {
2857 unsigned num_units = link->push_consts.count;
2858 unsigned offset = link->push_consts.lo;
2859 tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3 + num_units * 4);
2860 tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(offset) |
2861 CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2862 CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
2863 CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2864 CP_LOAD_STATE6_0_NUM_UNIT(num_units));
2865 tu_cs_emit(cs, 0);
2866 tu_cs_emit(cs, 0);
2867 for (unsigned i = 0; i < num_units * 4; i++)
2868 tu_cs_emit(cs, push_constants[i + offset * 4]);
2869 }
2870
2871 for (uint32_t i = 0; i < state->num_enabled; i++) {
2872 uint32_t size = state->range[i].end - state->range[i].start;
2873 uint32_t offset = state->range[i].start;
2874
2875 /* and even if the start of the const buffer is before
2876 * first_immediate, the end may not be:
2877 */
2878 size = MIN2(size, (16 * link->constlen) - state->range[i].offset);
2879
2880 if (size == 0)
2881 continue;
2882
2883 /* things should be aligned to vec4: */
2884 debug_assert((state->range[i].offset % 16) == 0);
2885 debug_assert((size % 16) == 0);
2886 debug_assert((offset % 16) == 0);
2887
2888 /* Dig out the descriptor from the descriptor state and read the VA from
2889 * it.
2890 */
2891 assert(state->range[i].ubo.bindless);
2892 uint32_t *base = state->range[i].ubo.bindless_base == MAX_SETS ?
2893 descriptors_state->dynamic_descriptors :
2894 descriptors_state->sets[state->range[i].ubo.bindless_base]->mapped_ptr;
2895 unsigned block = state->range[i].ubo.block;
2896 uint32_t *desc = base + block * A6XX_TEX_CONST_DWORDS;
2897 uint64_t va = desc[0] | ((uint64_t)(desc[1] & A6XX_UBO_1_BASE_HI__MASK) << 32);
2898 assert(va);
2899
2900 tu_cs_emit_pkt7(cs, tu6_stage2opcode(type), 3);
2901 tu_cs_emit(cs, CP_LOAD_STATE6_0_DST_OFF(state->range[i].offset / 16) |
2902 CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
2903 CP_LOAD_STATE6_0_STATE_SRC(SS6_INDIRECT) |
2904 CP_LOAD_STATE6_0_STATE_BLOCK(tu6_stage2shadersb(type)) |
2905 CP_LOAD_STATE6_0_NUM_UNIT(size / 16));
2906 tu_cs_emit_qw(cs, va + offset);
2907 }
2908 }
2909
2910 static struct tu_draw_state
2911 tu6_emit_consts(struct tu_cmd_buffer *cmd,
2912 const struct tu_pipeline *pipeline,
2913 struct tu_descriptor_state *descriptors_state,
2914 gl_shader_stage type)
2915 {
2916 struct tu_cs cs;
2917 tu_cs_begin_sub_stream(&cmd->sub_cs, 512, &cs); /* TODO: maximum size? */
2918
2919 tu6_emit_user_consts(&cs, pipeline, descriptors_state, type, cmd->push_constants);
2920
2921 return tu_cs_end_draw_state(&cmd->sub_cs, &cs);
2922 }
2923
2924 static struct tu_draw_state
2925 tu6_emit_vertex_buffers(struct tu_cmd_buffer *cmd,
2926 const struct tu_pipeline *pipeline)
2927 {
2928 struct tu_cs cs;
2929 tu_cs_begin_sub_stream(&cmd->sub_cs, 4 * MAX_VBS, &cs);
2930
2931 int binding;
2932 for_each_bit(binding, pipeline->vi.bindings_used) {
2933 const struct tu_buffer *buf = cmd->state.vb.buffers[binding];
2934 const VkDeviceSize offset = buf->bo_offset +
2935 cmd->state.vb.offsets[binding];
2936
2937 tu_cs_emit_regs(&cs,
2938 A6XX_VFD_FETCH_BASE(binding, .bo = buf->bo, .bo_offset = offset),
2939 A6XX_VFD_FETCH_SIZE(binding, buf->size - offset));
2940
2941 }
2942
2943 cmd->vertex_bindings_set = pipeline->vi.bindings_used;
2944
2945 return tu_cs_end_draw_state(&cmd->sub_cs, &cs);
2946 }
2947
2948 static uint64_t
2949 get_tess_param_bo_size(const struct tu_pipeline *pipeline,
2950 uint32_t draw_count)
2951 {
2952 /* TODO: For indirect draws, we can't compute the BO size ahead of time.
2953 * Still not sure what to do here, so just allocate a reasonably large
2954 * BO and hope for the best for now. */
2955 if (!draw_count)
2956 draw_count = 2048;
2957
2958 /* the tess param BO is pipeline->tess.param_stride bytes per patch,
2959 * which includes both the per-vertex outputs and per-patch outputs
2960 * build_primitive_map in ir3 calculates this stride
2961 */
2962 uint32_t verts_per_patch = pipeline->ia.primtype - DI_PT_PATCHES0;
2963 uint32_t num_patches = draw_count / verts_per_patch;
2964 return num_patches * pipeline->tess.param_stride;
2965 }
2966
2967 static uint64_t
2968 get_tess_factor_bo_size(const struct tu_pipeline *pipeline,
2969 uint32_t draw_count)
2970 {
2971 /* TODO: For indirect draws, we can't compute the BO size ahead of time.
2972 * Still not sure what to do here, so just allocate a reasonably large
2973 * BO and hope for the best for now. */
2974 if (!draw_count)
2975 draw_count = 2048;
2976
2977 /* Each distinct patch gets its own tess factor output. */
2978 uint32_t verts_per_patch = pipeline->ia.primtype - DI_PT_PATCHES0;
2979 uint32_t num_patches = draw_count / verts_per_patch;
2980 uint32_t factor_stride;
2981 switch (pipeline->tess.patch_type) {
2982 case IR3_TESS_ISOLINES:
2983 factor_stride = 12;
2984 break;
2985 case IR3_TESS_TRIANGLES:
2986 factor_stride = 20;
2987 break;
2988 case IR3_TESS_QUADS:
2989 factor_stride = 28;
2990 break;
2991 default:
2992 unreachable("bad tessmode");
2993 }
2994 return factor_stride * num_patches;
2995 }
2996
2997 static VkResult
2998 tu6_emit_tess_consts(struct tu_cmd_buffer *cmd,
2999 uint32_t draw_count,
3000 const struct tu_pipeline *pipeline,
3001 struct tu_draw_state *state,
3002 uint64_t *factor_iova)
3003 {
3004 struct tu_cs cs;
3005 VkResult result = tu_cs_begin_sub_stream(&cmd->sub_cs, 16, &cs);
3006 if (result != VK_SUCCESS)
3007 return result;
3008
3009 uint64_t tess_factor_size = get_tess_factor_bo_size(pipeline, draw_count);
3010 uint64_t tess_param_size = get_tess_param_bo_size(pipeline, draw_count);
3011 uint64_t tess_bo_size = tess_factor_size + tess_param_size;
3012 if (tess_bo_size > 0) {
3013 struct tu_bo *tess_bo;
3014 result = tu_get_scratch_bo(cmd->device, tess_bo_size, &tess_bo);
3015 if (result != VK_SUCCESS)
3016 return result;
3017
3018 tu_bo_list_add(&cmd->bo_list, tess_bo,
3019 MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
3020 uint64_t tess_factor_iova = tess_bo->iova;
3021 uint64_t tess_param_iova = tess_factor_iova + tess_factor_size;
3022
3023 tu_cs_emit_pkt7(&cs, CP_LOAD_STATE6_GEOM, 3 + 4);
3024 tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(pipeline->tess.hs_bo_regid) |
3025 CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
3026 CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
3027 CP_LOAD_STATE6_0_STATE_BLOCK(SB6_HS_SHADER) |
3028 CP_LOAD_STATE6_0_NUM_UNIT(1));
3029 tu_cs_emit(&cs, CP_LOAD_STATE6_1_EXT_SRC_ADDR(0));
3030 tu_cs_emit(&cs, CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI(0));
3031 tu_cs_emit_qw(&cs, tess_param_iova);
3032 tu_cs_emit_qw(&cs, tess_factor_iova);
3033
3034 tu_cs_emit_pkt7(&cs, CP_LOAD_STATE6_GEOM, 3 + 4);
3035 tu_cs_emit(&cs, CP_LOAD_STATE6_0_DST_OFF(pipeline->tess.ds_bo_regid) |
3036 CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) |
3037 CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) |
3038 CP_LOAD_STATE6_0_STATE_BLOCK(SB6_DS_SHADER) |
3039 CP_LOAD_STATE6_0_NUM_UNIT(1));
3040 tu_cs_emit(&cs, CP_LOAD_STATE6_1_EXT_SRC_ADDR(0));
3041 tu_cs_emit(&cs, CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI(0));
3042 tu_cs_emit_qw(&cs, tess_param_iova);
3043 tu_cs_emit_qw(&cs, tess_factor_iova);
3044
3045 *factor_iova = tess_factor_iova;
3046 }
3047 *state = tu_cs_end_draw_state(&cmd->sub_cs, &cs);
3048 return VK_SUCCESS;
3049 }
3050
3051 static VkResult
3052 tu6_draw_common(struct tu_cmd_buffer *cmd,
3053 struct tu_cs *cs,
3054 bool indexed,
3055 /* note: draw_count is 0 for indirect */
3056 uint32_t draw_count)
3057 {
3058 const struct tu_pipeline *pipeline = cmd->state.pipeline;
3059 VkResult result;
3060
3061 struct tu_descriptor_state *descriptors_state =
3062 &cmd->descriptors[VK_PIPELINE_BIND_POINT_GRAPHICS];
3063
3064 tu_emit_cache_flush_renderpass(cmd, cs);
3065
3066 /* TODO lrz */
3067
3068 tu_cs_emit_regs(cs, A6XX_PC_PRIMITIVE_CNTL_0(
3069 .primitive_restart =
3070 pipeline->ia.primitive_restart && indexed,
3071 .tess_upper_left_domain_origin =
3072 pipeline->tess.upper_left_domain_origin));
3073
3074 if (cmd->state.dirty & TU_CMD_DIRTY_SHADER_CONSTS) {
3075 cmd->state.shader_const[MESA_SHADER_VERTEX] =
3076 tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_VERTEX);
3077 cmd->state.shader_const[MESA_SHADER_TESS_CTRL] =
3078 tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_TESS_CTRL);
3079 cmd->state.shader_const[MESA_SHADER_TESS_EVAL] =
3080 tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_TESS_EVAL);
3081 cmd->state.shader_const[MESA_SHADER_GEOMETRY] =
3082 tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_GEOMETRY);
3083 cmd->state.shader_const[MESA_SHADER_FRAGMENT] =
3084 tu6_emit_consts(cmd, pipeline, descriptors_state, MESA_SHADER_FRAGMENT);
3085 }
3086
3087 if (cmd->state.dirty & TU_CMD_DIRTY_VERTEX_BUFFERS)
3088 cmd->state.vertex_buffers = tu6_emit_vertex_buffers(cmd, pipeline);
3089