projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
radv: Only break batch on framebuffer change with dfsm.
[mesa.git] / src / amd / vulkan / radv_cmd_buffer.c
1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 *
5 * based in part on anv driver which is:
6 * Copyright © 2015 Intel Corporation
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the "Software"),
10 * to deal in the Software without restriction, including without limitation
11 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 * and/or sell copies of the Software, and to permit persons to whom the
13 * Software is furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the next
16 * paragraph) shall be included in all copies or substantial portions of the
17 * Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 * IN THE SOFTWARE.
26 */
27
28 #include "radv_private.h"
29 #include "radv_radeon_winsys.h"
30 #include "radv_shader.h"
31 #include "radv_cs.h"
32 #include "sid.h"
33 #include "vk_format.h"
34 #include "vk_util.h"
35 #include "radv_debug.h"
36 #include "radv_meta.h"
37
38 #include "ac_debug.h"
39
40 enum {
41 RADV_PREFETCH_VBO_DESCRIPTORS = (1 << 0),
42 RADV_PREFETCH_VS = (1 << 1),
43 RADV_PREFETCH_TCS = (1 << 2),
44 RADV_PREFETCH_TES = (1 << 3),
45 RADV_PREFETCH_GS = (1 << 4),
46 RADV_PREFETCH_PS = (1 << 5),
47 RADV_PREFETCH_SHADERS = (RADV_PREFETCH_VS |
48 RADV_PREFETCH_TCS |
49 RADV_PREFETCH_TES |
50 RADV_PREFETCH_GS |
51 RADV_PREFETCH_PS)
52 };
53
54 static void radv_handle_image_transition(struct radv_cmd_buffer *cmd_buffer,
55 struct radv_image *image,
56 VkImageLayout src_layout,
57 bool src_render_loop,
58 VkImageLayout dst_layout,
59 bool dst_render_loop,
60 uint32_t src_family,
61 uint32_t dst_family,
62 const VkImageSubresourceRange *range,
63 struct radv_sample_locations_state *sample_locs);
64
65 const struct radv_dynamic_state default_dynamic_state = {
66 .viewport = {
67 .count = 0,
68 },
69 .scissor = {
70 .count = 0,
71 },
72 .line_width = 1.0f,
73 .depth_bias = {
74 .bias = 0.0f,
75 .clamp = 0.0f,
76 .slope = 0.0f,
77 },
78 .blend_constants = { 0.0f, 0.0f, 0.0f, 0.0f },
79 .depth_bounds = {
80 .min = 0.0f,
81 .max = 1.0f,
82 },
83 .stencil_compare_mask = {
84 .front = ~0u,
85 .back = ~0u,
86 },
87 .stencil_write_mask = {
88 .front = ~0u,
89 .back = ~0u,
90 },
91 .stencil_reference = {
92 .front = 0u,
93 .back = 0u,
94 },
95 };
96
97 static void
98 radv_bind_dynamic_state(struct radv_cmd_buffer *cmd_buffer,
99 const struct radv_dynamic_state *src)
100 {
101 struct radv_dynamic_state *dest = &cmd_buffer->state.dynamic;
102 uint32_t copy_mask = src->mask;
103 uint32_t dest_mask = 0;
104
105 /* Make sure to copy the number of viewports/scissors because they can
106 * only be specified at pipeline creation time.
107 */
108 dest->viewport.count = src->viewport.count;
109 dest->scissor.count = src->scissor.count;
110 dest->discard_rectangle.count = src->discard_rectangle.count;
111 dest->sample_location.count = src->sample_location.count;
112
113 if (copy_mask & RADV_DYNAMIC_VIEWPORT) {
114 if (memcmp(&dest->viewport.viewports, &src->viewport.viewports,
115 src->viewport.count * sizeof(VkViewport))) {
116 typed_memcpy(dest->viewport.viewports,
117 src->viewport.viewports,
118 src->viewport.count);
119 dest_mask |= RADV_DYNAMIC_VIEWPORT;
120 }
121 }
122
123 if (copy_mask & RADV_DYNAMIC_SCISSOR) {
124 if (memcmp(&dest->scissor.scissors, &src->scissor.scissors,
125 src->scissor.count * sizeof(VkRect2D))) {
126 typed_memcpy(dest->scissor.scissors,
127 src->scissor.scissors, src->scissor.count);
128 dest_mask |= RADV_DYNAMIC_SCISSOR;
129 }
130 }
131
132 if (copy_mask & RADV_DYNAMIC_LINE_WIDTH) {
133 if (dest->line_width != src->line_width) {
134 dest->line_width = src->line_width;
135 dest_mask |= RADV_DYNAMIC_LINE_WIDTH;
136 }
137 }
138
139 if (copy_mask & RADV_DYNAMIC_DEPTH_BIAS) {
140 if (memcmp(&dest->depth_bias, &src->depth_bias,
141 sizeof(src->depth_bias))) {
142 dest->depth_bias = src->depth_bias;
143 dest_mask |= RADV_DYNAMIC_DEPTH_BIAS;
144 }
145 }
146
147 if (copy_mask & RADV_DYNAMIC_BLEND_CONSTANTS) {
148 if (memcmp(&dest->blend_constants, &src->blend_constants,
149 sizeof(src->blend_constants))) {
150 typed_memcpy(dest->blend_constants,
151 src->blend_constants, 4);
152 dest_mask |= RADV_DYNAMIC_BLEND_CONSTANTS;
153 }
154 }
155
156 if (copy_mask & RADV_DYNAMIC_DEPTH_BOUNDS) {
157 if (memcmp(&dest->depth_bounds, &src->depth_bounds,
158 sizeof(src->depth_bounds))) {
159 dest->depth_bounds = src->depth_bounds;
160 dest_mask |= RADV_DYNAMIC_DEPTH_BOUNDS;
161 }
162 }
163
164 if (copy_mask & RADV_DYNAMIC_STENCIL_COMPARE_MASK) {
165 if (memcmp(&dest->stencil_compare_mask,
166 &src->stencil_compare_mask,
167 sizeof(src->stencil_compare_mask))) {
168 dest->stencil_compare_mask = src->stencil_compare_mask;
169 dest_mask |= RADV_DYNAMIC_STENCIL_COMPARE_MASK;
170 }
171 }
172
173 if (copy_mask & RADV_DYNAMIC_STENCIL_WRITE_MASK) {
174 if (memcmp(&dest->stencil_write_mask, &src->stencil_write_mask,
175 sizeof(src->stencil_write_mask))) {
176 dest->stencil_write_mask = src->stencil_write_mask;
177 dest_mask |= RADV_DYNAMIC_STENCIL_WRITE_MASK;
178 }
179 }
180
181 if (copy_mask & RADV_DYNAMIC_STENCIL_REFERENCE) {
182 if (memcmp(&dest->stencil_reference, &src->stencil_reference,
183 sizeof(src->stencil_reference))) {
184 dest->stencil_reference = src->stencil_reference;
185 dest_mask |= RADV_DYNAMIC_STENCIL_REFERENCE;
186 }
187 }
188
189 if (copy_mask & RADV_DYNAMIC_DISCARD_RECTANGLE) {
190 if (memcmp(&dest->discard_rectangle.rectangles, &src->discard_rectangle.rectangles,
191 src->discard_rectangle.count * sizeof(VkRect2D))) {
192 typed_memcpy(dest->discard_rectangle.rectangles,
193 src->discard_rectangle.rectangles,
194 src->discard_rectangle.count);
195 dest_mask |= RADV_DYNAMIC_DISCARD_RECTANGLE;
196 }
197 }
198
199 if (copy_mask & RADV_DYNAMIC_SAMPLE_LOCATIONS) {
200 if (dest->sample_location.per_pixel != src->sample_location.per_pixel ||
201 dest->sample_location.grid_size.width != src->sample_location.grid_size.width ||
202 dest->sample_location.grid_size.height != src->sample_location.grid_size.height ||
203 memcmp(&dest->sample_location.locations,
204 &src->sample_location.locations,
205 src->sample_location.count * sizeof(VkSampleLocationEXT))) {
206 dest->sample_location.per_pixel = src->sample_location.per_pixel;
207 dest->sample_location.grid_size = src->sample_location.grid_size;
208 typed_memcpy(dest->sample_location.locations,
209 src->sample_location.locations,
210 src->sample_location.count);
211 dest_mask |= RADV_DYNAMIC_SAMPLE_LOCATIONS;
212 }
213 }
214
215 cmd_buffer->state.dirty |= dest_mask;
216 }
217
218 static void
219 radv_bind_streamout_state(struct radv_cmd_buffer *cmd_buffer,
220 struct radv_pipeline *pipeline)
221 {
222 struct radv_streamout_state *so = &cmd_buffer->state.streamout;
223 struct radv_shader_info *info;
224
225 if (!pipeline->streamout_shader ||
226 cmd_buffer->device->physical_device->use_ngg_streamout)
227 return;
228
229 info = &pipeline->streamout_shader->info;
230 for (int i = 0; i < MAX_SO_BUFFERS; i++)
231 so->stride_in_dw[i] = info->so.strides[i];
232
233 so->enabled_stream_buffers_mask = info->so.enabled_stream_buffers_mask;
234 }
235
236 bool radv_cmd_buffer_uses_mec(struct radv_cmd_buffer *cmd_buffer)
237 {
238 return cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE &&
239 cmd_buffer->device->physical_device->rad_info.chip_class >= GFX7;
240 }
241
242 enum ring_type radv_queue_family_to_ring(int f) {
243 switch (f) {
244 case RADV_QUEUE_GENERAL:
245 return RING_GFX;
246 case RADV_QUEUE_COMPUTE:
247 return RING_COMPUTE;
248 case RADV_QUEUE_TRANSFER:
249 return RING_DMA;
250 default:
251 unreachable("Unknown queue family");
252 }
253 }
254
255 static VkResult radv_create_cmd_buffer(
256 struct radv_device * device,
257 struct radv_cmd_pool * pool,
258 VkCommandBufferLevel level,
259 VkCommandBuffer* pCommandBuffer)
260 {
261 struct radv_cmd_buffer *cmd_buffer;
262 unsigned ring;
263 cmd_buffer = vk_zalloc(&pool->alloc, sizeof(*cmd_buffer), 8,
264 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
265 if (cmd_buffer == NULL)
266 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
267
268 cmd_buffer->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
269 cmd_buffer->device = device;
270 cmd_buffer->pool = pool;
271 cmd_buffer->level = level;
272
273 if (pool) {
274 list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers);
275 cmd_buffer->queue_family_index = pool->queue_family_index;
276
277 } else {
278 /* Init the pool_link so we can safely call list_del when we destroy
279 * the command buffer
280 */
281 list_inithead(&cmd_buffer->pool_link);
282 cmd_buffer->queue_family_index = RADV_QUEUE_GENERAL;
283 }
284
285 ring = radv_queue_family_to_ring(cmd_buffer->queue_family_index);
286
287 cmd_buffer->cs = device->ws->cs_create(device->ws, ring);
288 if (!cmd_buffer->cs) {
289 vk_free(&cmd_buffer->pool->alloc, cmd_buffer);
290 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
291 }
292
293 *pCommandBuffer = radv_cmd_buffer_to_handle(cmd_buffer);
294
295 list_inithead(&cmd_buffer->upload.list);
296
297 return VK_SUCCESS;
298 }
299
300 static void
301 radv_cmd_buffer_destroy(struct radv_cmd_buffer *cmd_buffer)
302 {
303 list_del(&cmd_buffer->pool_link);
304
305 list_for_each_entry_safe(struct radv_cmd_buffer_upload, up,
306 &cmd_buffer->upload.list, list) {
307 cmd_buffer->device->ws->buffer_destroy(up->upload_bo);
308 list_del(&up->list);
309 free(up);
310 }
311
312 if (cmd_buffer->upload.upload_bo)
313 cmd_buffer->device->ws->buffer_destroy(cmd_buffer->upload.upload_bo);
314 cmd_buffer->device->ws->cs_destroy(cmd_buffer->cs);
315
316 for (unsigned i = 0; i < VK_PIPELINE_BIND_POINT_RANGE_SIZE; i++)
317 free(cmd_buffer->descriptors[i].push_set.set.mapped_ptr);
318
319 vk_free(&cmd_buffer->pool->alloc, cmd_buffer);
320 }
321
322 static VkResult
323 radv_reset_cmd_buffer(struct radv_cmd_buffer *cmd_buffer)
324 {
325 cmd_buffer->device->ws->cs_reset(cmd_buffer->cs);
326
327 list_for_each_entry_safe(struct radv_cmd_buffer_upload, up,
328 &cmd_buffer->upload.list, list) {
329 cmd_buffer->device->ws->buffer_destroy(up->upload_bo);
330 list_del(&up->list);
331 free(up);
332 }
333
334 cmd_buffer->push_constant_stages = 0;
335 cmd_buffer->scratch_size_needed = 0;
336 cmd_buffer->compute_scratch_size_needed = 0;
337 cmd_buffer->esgs_ring_size_needed = 0;
338 cmd_buffer->gsvs_ring_size_needed = 0;
339 cmd_buffer->tess_rings_needed = false;
340 cmd_buffer->gds_needed = false;
341 cmd_buffer->sample_positions_needed = false;
342
343 if (cmd_buffer->upload.upload_bo)
344 radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs,
345 cmd_buffer->upload.upload_bo);
346 cmd_buffer->upload.offset = 0;
347
348 cmd_buffer->record_result = VK_SUCCESS;
349
350 memset(cmd_buffer->vertex_bindings, 0, sizeof(cmd_buffer->vertex_bindings));
351
352 for (unsigned i = 0; i < VK_PIPELINE_BIND_POINT_RANGE_SIZE; i++) {
353 cmd_buffer->descriptors[i].dirty = 0;
354 cmd_buffer->descriptors[i].valid = 0;
355 cmd_buffer->descriptors[i].push_dirty = false;
356 }
357
358 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9 &&
359 cmd_buffer->queue_family_index == RADV_QUEUE_GENERAL) {
360 unsigned num_db = cmd_buffer->device->physical_device->rad_info.num_render_backends;
361 unsigned fence_offset, eop_bug_offset;
362 void *fence_ptr;
363
364 radv_cmd_buffer_upload_alloc(cmd_buffer, 8, 8, &fence_offset,
365 &fence_ptr);
366
367 cmd_buffer->gfx9_fence_va =
368 radv_buffer_get_va(cmd_buffer->upload.upload_bo);
369 cmd_buffer->gfx9_fence_va += fence_offset;
370
371 if (cmd_buffer->device->physical_device->rad_info.chip_class == GFX9) {
372 /* Allocate a buffer for the EOP bug on GFX9. */
373 radv_cmd_buffer_upload_alloc(cmd_buffer, 16 * num_db, 8,
374 &eop_bug_offset, &fence_ptr);
375 cmd_buffer->gfx9_eop_bug_va =
376 radv_buffer_get_va(cmd_buffer->upload.upload_bo);
377 cmd_buffer->gfx9_eop_bug_va += eop_bug_offset;
378 }
379 }
380
381 cmd_buffer->status = RADV_CMD_BUFFER_STATUS_INITIAL;
382
383 return cmd_buffer->record_result;
384 }
385
386 static bool
387 radv_cmd_buffer_resize_upload_buf(struct radv_cmd_buffer *cmd_buffer,
388 uint64_t min_needed)
389 {
390 uint64_t new_size;
391 struct radeon_winsys_bo *bo;
392 struct radv_cmd_buffer_upload *upload;
393 struct radv_device *device = cmd_buffer->device;
394
395 new_size = MAX2(min_needed, 16 * 1024);
396 new_size = MAX2(new_size, 2 * cmd_buffer->upload.size);
397
398 bo = device->ws->buffer_create(device->ws,
399 new_size, 4096,
400 RADEON_DOMAIN_GTT,
401 RADEON_FLAG_CPU_ACCESS|
402 RADEON_FLAG_NO_INTERPROCESS_SHARING |
403 RADEON_FLAG_32BIT,
404 RADV_BO_PRIORITY_UPLOAD_BUFFER);
405
406 if (!bo) {
407 cmd_buffer->record_result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
408 return false;
409 }
410
411 radv_cs_add_buffer(device->ws, cmd_buffer->cs, bo);
412 if (cmd_buffer->upload.upload_bo) {
413 upload = malloc(sizeof(*upload));
414
415 if (!upload) {
416 cmd_buffer->record_result = VK_ERROR_OUT_OF_HOST_MEMORY;
417 device->ws->buffer_destroy(bo);
418 return false;
419 }
420
421 memcpy(upload, &cmd_buffer->upload, sizeof(*upload));
422 list_add(&upload->list, &cmd_buffer->upload.list);
423 }
424
425 cmd_buffer->upload.upload_bo = bo;
426 cmd_buffer->upload.size = new_size;
427 cmd_buffer->upload.offset = 0;
428 cmd_buffer->upload.map = device->ws->buffer_map(cmd_buffer->upload.upload_bo);
429
430 if (!cmd_buffer->upload.map) {
431 cmd_buffer->record_result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
432 return false;
433 }
434
435 return true;
436 }
437
438 bool
439 radv_cmd_buffer_upload_alloc(struct radv_cmd_buffer *cmd_buffer,
440 unsigned size,
441 unsigned alignment,
442 unsigned *out_offset,
443 void **ptr)
444 {
445 assert(util_is_power_of_two_nonzero(alignment));
446
447 uint64_t offset = align(cmd_buffer->upload.offset, alignment);
448 if (offset + size > cmd_buffer->upload.size) {
449 if (!radv_cmd_buffer_resize_upload_buf(cmd_buffer, size))
450 return false;
451 offset = 0;
452 }
453
454 *out_offset = offset;
455 *ptr = cmd_buffer->upload.map + offset;
456
457 cmd_buffer->upload.offset = offset + size;
458 return true;
459 }
460
461 bool
462 radv_cmd_buffer_upload_data(struct radv_cmd_buffer *cmd_buffer,
463 unsigned size, unsigned alignment,
464 const void *data, unsigned *out_offset)
465 {
466 uint8_t *ptr;
467
468 if (!radv_cmd_buffer_upload_alloc(cmd_buffer, size, alignment,
469 out_offset, (void **)&ptr))
470 return false;
471
472 if (ptr)
473 memcpy(ptr, data, size);
474
475 return true;
476 }
477
478 static void
479 radv_emit_write_data_packet(struct radv_cmd_buffer *cmd_buffer, uint64_t va,
480 unsigned count, const uint32_t *data)
481 {
482 struct radeon_cmdbuf *cs = cmd_buffer->cs;
483
484 radeon_check_space(cmd_buffer->device->ws, cs, 4 + count);
485
486 radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 2 + count, 0));
487 radeon_emit(cs, S_370_DST_SEL(V_370_MEM) |
488 S_370_WR_CONFIRM(1) |
489 S_370_ENGINE_SEL(V_370_ME));
490 radeon_emit(cs, va);
491 radeon_emit(cs, va >> 32);
492 radeon_emit_array(cs, data, count);
493 }
494
495 void radv_cmd_buffer_trace_emit(struct radv_cmd_buffer *cmd_buffer)
496 {
497 struct radv_device *device = cmd_buffer->device;
498 struct radeon_cmdbuf *cs = cmd_buffer->cs;
499 uint64_t va;
500
501 va = radv_buffer_get_va(device->trace_bo);
502 if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY)
503 va += 4;
504
505 ++cmd_buffer->state.trace_id;
506 radv_emit_write_data_packet(cmd_buffer, va, 1,
507 &cmd_buffer->state.trace_id);
508
509 radeon_check_space(cmd_buffer->device->ws, cs, 2);
510
511 radeon_emit(cs, PKT3(PKT3_NOP, 0, 0));
512 radeon_emit(cs, AC_ENCODE_TRACE_POINT(cmd_buffer->state.trace_id));
513 }
514
515 static void
516 radv_cmd_buffer_after_draw(struct radv_cmd_buffer *cmd_buffer,
517 enum radv_cmd_flush_bits flags)
518 {
519 if (cmd_buffer->device->instance->debug_flags & RADV_DEBUG_SYNC_SHADERS) {
520 assert(flags & (RADV_CMD_FLAG_PS_PARTIAL_FLUSH |
521 RADV_CMD_FLAG_CS_PARTIAL_FLUSH));
522
523 radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, 4);
524
525 /* Force wait for graphics or compute engines to be idle. */
526 si_cs_emit_cache_flush(cmd_buffer->cs,
527 cmd_buffer->device->physical_device->rad_info.chip_class,
528 &cmd_buffer->gfx9_fence_idx,
529 cmd_buffer->gfx9_fence_va,
530 radv_cmd_buffer_uses_mec(cmd_buffer),
531 flags, cmd_buffer->gfx9_eop_bug_va);
532 }
533
534 if (unlikely(cmd_buffer->device->trace_bo))
535 radv_cmd_buffer_trace_emit(cmd_buffer);
536 }
537
538 static void
539 radv_save_pipeline(struct radv_cmd_buffer *cmd_buffer,
540 struct radv_pipeline *pipeline, enum ring_type ring)
541 {
542 struct radv_device *device = cmd_buffer->device;
543 uint32_t data[2];
544 uint64_t va;
545
546 va = radv_buffer_get_va(device->trace_bo);
547
548 switch (ring) {
549 case RING_GFX:
550 va += 8;
551 break;
552 case RING_COMPUTE:
553 va += 16;
554 break;
555 default:
556 assert(!"invalid ring type");
557 }
558
559 data[0] = (uintptr_t)pipeline;
560 data[1] = (uintptr_t)pipeline >> 32;
561
562 radv_emit_write_data_packet(cmd_buffer, va, 2, data);
563 }
564
565 void radv_set_descriptor_set(struct radv_cmd_buffer *cmd_buffer,
566 VkPipelineBindPoint bind_point,
567 struct radv_descriptor_set *set,
568 unsigned idx)
569 {
570 struct radv_descriptor_state *descriptors_state =
571 radv_get_descriptors_state(cmd_buffer, bind_point);
572
573 descriptors_state->sets[idx] = set;
574
575 descriptors_state->valid |= (1u << idx); /* active descriptors */
576 descriptors_state->dirty |= (1u << idx);
577 }
578
579 static void
580 radv_save_descriptors(struct radv_cmd_buffer *cmd_buffer,
581 VkPipelineBindPoint bind_point)
582 {
583 struct radv_descriptor_state *descriptors_state =
584 radv_get_descriptors_state(cmd_buffer, bind_point);
585 struct radv_device *device = cmd_buffer->device;
586 uint32_t data[MAX_SETS * 2] = {};
587 uint64_t va;
588 unsigned i;
589 va = radv_buffer_get_va(device->trace_bo) + 24;
590
591 for_each_bit(i, descriptors_state->valid) {
592 struct radv_descriptor_set *set = descriptors_state->sets[i];
593 data[i * 2] = (uint64_t)(uintptr_t)set;
594 data[i * 2 + 1] = (uint64_t)(uintptr_t)set >> 32;
595 }
596
597 radv_emit_write_data_packet(cmd_buffer, va, MAX_SETS * 2, data);
598 }
599
600 struct radv_userdata_info *
601 radv_lookup_user_sgpr(struct radv_pipeline *pipeline,
602 gl_shader_stage stage,
603 int idx)
604 {
605 struct radv_shader_variant *shader = radv_get_shader(pipeline, stage);
606 return &shader->info.user_sgprs_locs.shader_data[idx];
607 }
608
609 static void
610 radv_emit_userdata_address(struct radv_cmd_buffer *cmd_buffer,
611 struct radv_pipeline *pipeline,
612 gl_shader_stage stage,
613 int idx, uint64_t va)
614 {
615 struct radv_userdata_info *loc = radv_lookup_user_sgpr(pipeline, stage, idx);
616 uint32_t base_reg = pipeline->user_data_0[stage];
617 if (loc->sgpr_idx == -1)
618 return;
619
620 assert(loc->num_sgprs == 1);
621
622 radv_emit_shader_pointer(cmd_buffer->device, cmd_buffer->cs,
623 base_reg + loc->sgpr_idx * 4, va, false);
624 }
625
626 static void
627 radv_emit_descriptor_pointers(struct radv_cmd_buffer *cmd_buffer,
628 struct radv_pipeline *pipeline,
629 struct radv_descriptor_state *descriptors_state,
630 gl_shader_stage stage)
631 {
632 struct radv_device *device = cmd_buffer->device;
633 struct radeon_cmdbuf *cs = cmd_buffer->cs;
634 uint32_t sh_base = pipeline->user_data_0[stage];
635 struct radv_userdata_locations *locs =
636 &pipeline->shaders[stage]->info.user_sgprs_locs;
637 unsigned mask = locs->descriptor_sets_enabled;
638
639 mask &= descriptors_state->dirty & descriptors_state->valid;
640
641 while (mask) {
642 int start, count;
643
644 u_bit_scan_consecutive_range(&mask, &start, &count);
645
646 struct radv_userdata_info *loc = &locs->descriptor_sets[start];
647 unsigned sh_offset = sh_base + loc->sgpr_idx * 4;
648
649 radv_emit_shader_pointer_head(cs, sh_offset, count, true);
650 for (int i = 0; i < count; i++) {
651 struct radv_descriptor_set *set =
652 descriptors_state->sets[start + i];
653
654 radv_emit_shader_pointer_body(device, cs, set->va, true);
655 }
656 }
657 }
658
659 /**
660 * Convert the user sample locations to hardware sample locations (the values
661 * that will be emitted by PA_SC_AA_SAMPLE_LOCS_PIXEL_*).
662 */
663 static void
664 radv_convert_user_sample_locs(struct radv_sample_locations_state *state,
665 uint32_t x, uint32_t y, VkOffset2D *sample_locs)
666 {
667 uint32_t x_offset = x % state->grid_size.width;
668 uint32_t y_offset = y % state->grid_size.height;
669 uint32_t num_samples = (uint32_t)state->per_pixel;
670 VkSampleLocationEXT *user_locs;
671 uint32_t pixel_offset;
672
673 pixel_offset = (x_offset + y_offset * state->grid_size.width) * num_samples;
674
675 assert(pixel_offset <= MAX_SAMPLE_LOCATIONS);
676 user_locs = &state->locations[pixel_offset];
677
678 for (uint32_t i = 0; i < num_samples; i++) {
679 float shifted_pos_x = user_locs[i].x - 0.5;
680 float shifted_pos_y = user_locs[i].y - 0.5;
681
682 int32_t scaled_pos_x = floor(shifted_pos_x * 16);
683 int32_t scaled_pos_y = floor(shifted_pos_y * 16);
684
685 sample_locs[i].x = CLAMP(scaled_pos_x, -8, 7);
686 sample_locs[i].y = CLAMP(scaled_pos_y, -8, 7);
687 }
688 }
689
690 /**
691 * Compute the PA_SC_AA_SAMPLE_LOCS_PIXEL_* mask based on hardware sample
692 * locations.
693 */
694 static void
695 radv_compute_sample_locs_pixel(uint32_t num_samples, VkOffset2D *sample_locs,
696 uint32_t *sample_locs_pixel)
697 {
698 for (uint32_t i = 0; i < num_samples; i++) {
699 uint32_t sample_reg_idx = i / 4;
700 uint32_t sample_loc_idx = i % 4;
701 int32_t pos_x = sample_locs[i].x;
702 int32_t pos_y = sample_locs[i].y;
703
704 uint32_t shift_x = 8 * sample_loc_idx;
705 uint32_t shift_y = shift_x + 4;
706
707 sample_locs_pixel[sample_reg_idx] |= (pos_x & 0xf) << shift_x;
708 sample_locs_pixel[sample_reg_idx] |= (pos_y & 0xf) << shift_y;
709 }
710 }
711
712 /**
713 * Compute the PA_SC_CENTROID_PRIORITY_* mask based on the top left hardware
714 * sample locations.
715 */
716 static uint64_t
717 radv_compute_centroid_priority(struct radv_cmd_buffer *cmd_buffer,
718 VkOffset2D *sample_locs,
719 uint32_t num_samples)
720 {
721 uint32_t centroid_priorities[num_samples];
722 uint32_t sample_mask = num_samples - 1;
723 uint32_t distances[num_samples];
724 uint64_t centroid_priority = 0;
725
726 /* Compute the distances from center for each sample. */
727 for (int i = 0; i < num_samples; i++) {
728 distances[i] = (sample_locs[i].x * sample_locs[i].x) +
729 (sample_locs[i].y * sample_locs[i].y);
730 }
731
732 /* Compute the centroid priorities by looking at the distances array. */
733 for (int i = 0; i < num_samples; i++) {
734 uint32_t min_idx = 0;
735
736 for (int j = 1; j < num_samples; j++) {
737 if (distances[j] < distances[min_idx])
738 min_idx = j;
739 }
740
741 centroid_priorities[i] = min_idx;
742 distances[min_idx] = 0xffffffff;
743 }
744
745 /* Compute the final centroid priority. */
746 for (int i = 0; i < 8; i++) {
747 centroid_priority |=
748 centroid_priorities[i & sample_mask] << (i * 4);
749 }
750
751 return centroid_priority << 32 | centroid_priority;
752 }
753
754 /**
755 * Emit the sample locations that are specified with VK_EXT_sample_locations.
756 */
757 static void
758 radv_emit_sample_locations(struct radv_cmd_buffer *cmd_buffer)
759 {
760 struct radv_pipeline *pipeline = cmd_buffer->state.pipeline;
761 struct radv_multisample_state *ms = &pipeline->graphics.ms;
762 struct radv_sample_locations_state *sample_location =
763 &cmd_buffer->state.dynamic.sample_location;
764 uint32_t num_samples = (uint32_t)sample_location->per_pixel;
765 struct radeon_cmdbuf *cs = cmd_buffer->cs;
766 uint32_t sample_locs_pixel[4][2] = {};
767 VkOffset2D sample_locs[4][8]; /* 8 is the max. sample count supported */
768 uint32_t max_sample_dist = 0;
769 uint64_t centroid_priority;
770
771 if (!cmd_buffer->state.dynamic.sample_location.count)
772 return;
773
774 /* Convert the user sample locations to hardware sample locations. */
775 radv_convert_user_sample_locs(sample_location, 0, 0, sample_locs[0]);
776 radv_convert_user_sample_locs(sample_location, 1, 0, sample_locs[1]);
777 radv_convert_user_sample_locs(sample_location, 0, 1, sample_locs[2]);
778 radv_convert_user_sample_locs(sample_location, 1, 1, sample_locs[3]);
779
780 /* Compute the PA_SC_AA_SAMPLE_LOCS_PIXEL_* mask. */
781 for (uint32_t i = 0; i < 4; i++) {
782 radv_compute_sample_locs_pixel(num_samples, sample_locs[i],
783 sample_locs_pixel[i]);
784 }
785
786 /* Compute the PA_SC_CENTROID_PRIORITY_* mask. */
787 centroid_priority =
788 radv_compute_centroid_priority(cmd_buffer, sample_locs[0],
789 num_samples);
790
791 /* Compute the maximum sample distance from the specified locations. */
792 for (uint32_t i = 0; i < num_samples; i++) {
793 VkOffset2D offset = sample_locs[0][i];
794 max_sample_dist = MAX2(max_sample_dist,
795 MAX2(abs(offset.x), abs(offset.y)));
796 }
797
798 /* Emit the specified user sample locations. */
799 switch (num_samples) {
800 case 2:
801 case 4:
802 radeon_set_context_reg(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_pixel[0][0]);
803 radeon_set_context_reg(cs, R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_pixel[1][0]);
804 radeon_set_context_reg(cs, R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_pixel[2][0]);
805 radeon_set_context_reg(cs, R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_pixel[3][0]);
806 break;
807 case 8:
808 radeon_set_context_reg(cs, R_028BF8_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0, sample_locs_pixel[0][0]);
809 radeon_set_context_reg(cs, R_028C08_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0, sample_locs_pixel[1][0]);
810 radeon_set_context_reg(cs, R_028C18_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0, sample_locs_pixel[2][0]);
811 radeon_set_context_reg(cs, R_028C28_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0, sample_locs_pixel[3][0]);
812 radeon_set_context_reg(cs, R_028BFC_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1, sample_locs_pixel[0][1]);
813 radeon_set_context_reg(cs, R_028C0C_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1, sample_locs_pixel[1][1]);
814 radeon_set_context_reg(cs, R_028C1C_PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1, sample_locs_pixel[2][1]);
815 radeon_set_context_reg(cs, R_028C2C_PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1, sample_locs_pixel[3][1]);
816 break;
817 default:
818 unreachable("invalid number of samples");
819 }
820
821 /* Emit the maximum sample distance and the centroid priority. */
822 uint32_t pa_sc_aa_config = ms->pa_sc_aa_config;
823
824 pa_sc_aa_config &= C_028BE0_MAX_SAMPLE_DIST;
825 pa_sc_aa_config |= S_028BE0_MAX_SAMPLE_DIST(max_sample_dist);
826
827 radeon_set_context_reg_seq(cs, R_028BE0_PA_SC_AA_CONFIG, 1);
828 radeon_emit(cs, pa_sc_aa_config);
829
830 radeon_set_context_reg_seq(cs, R_028BD4_PA_SC_CENTROID_PRIORITY_0, 2);
831 radeon_emit(cs, centroid_priority);
832 radeon_emit(cs, centroid_priority >> 32);
833
834 /* GFX9: Flush DFSM when the AA mode changes. */
835 if (cmd_buffer->device->dfsm_allowed) {
836 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
837 radeon_emit(cs, EVENT_TYPE(V_028A90_FLUSH_DFSM) | EVENT_INDEX(0));
838 }
839
840 cmd_buffer->state.context_roll_without_scissor_emitted = true;
841 }
842
843 static void
844 radv_emit_inline_push_consts(struct radv_cmd_buffer *cmd_buffer,
845 struct radv_pipeline *pipeline,
846 gl_shader_stage stage,
847 int idx, int count, uint32_t *values)
848 {
849 struct radv_userdata_info *loc = radv_lookup_user_sgpr(pipeline, stage, idx);
850 uint32_t base_reg = pipeline->user_data_0[stage];
851 if (loc->sgpr_idx == -1)
852 return;
853
854 assert(loc->num_sgprs == count);
855
856 radeon_set_sh_reg_seq(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, count);
857 radeon_emit_array(cmd_buffer->cs, values, count);
858 }
859
860 static void
861 radv_update_multisample_state(struct radv_cmd_buffer *cmd_buffer,
862 struct radv_pipeline *pipeline)
863 {
864 int num_samples = pipeline->graphics.ms.num_samples;
865 struct radv_multisample_state *ms = &pipeline->graphics.ms;
866 struct radv_pipeline *old_pipeline = cmd_buffer->state.emitted_pipeline;
867
868 if (pipeline->shaders[MESA_SHADER_FRAGMENT]->info.ps.needs_sample_positions)
869 cmd_buffer->sample_positions_needed = true;
870
871 if (old_pipeline && num_samples == old_pipeline->graphics.ms.num_samples)
872 return;
873
874 radeon_set_context_reg_seq(cmd_buffer->cs, R_028BDC_PA_SC_LINE_CNTL, 2);
875 radeon_emit(cmd_buffer->cs, ms->pa_sc_line_cntl);
876 radeon_emit(cmd_buffer->cs, ms->pa_sc_aa_config);
877
878 radeon_set_context_reg(cmd_buffer->cs, R_028A48_PA_SC_MODE_CNTL_0, ms->pa_sc_mode_cntl_0);
879
880 radv_emit_default_sample_locations(cmd_buffer->cs, num_samples);
881
882 /* GFX9: Flush DFSM when the AA mode changes. */
883 if (cmd_buffer->device->dfsm_allowed) {
884 radeon_emit(cmd_buffer->cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
885 radeon_emit(cmd_buffer->cs, EVENT_TYPE(V_028A90_FLUSH_DFSM) | EVENT_INDEX(0));
886 }
887
888 cmd_buffer->state.context_roll_without_scissor_emitted = true;
889 }
890
891 static void
892 radv_update_binning_state(struct radv_cmd_buffer *cmd_buffer,
893 struct radv_pipeline *pipeline)
894 {
895 const struct radv_pipeline *old_pipeline = cmd_buffer->state.emitted_pipeline;
896
897
898 if (pipeline->device->physical_device->rad_info.chip_class < GFX9)
899 return;
900
901 if (old_pipeline &&
902 old_pipeline->graphics.binning.pa_sc_binner_cntl_0 == pipeline->graphics.binning.pa_sc_binner_cntl_0 &&
903 old_pipeline->graphics.binning.db_dfsm_control == pipeline->graphics.binning.db_dfsm_control)
904 return;
905
906 bool binning_flush = false;
907 if (cmd_buffer->device->physical_device->rad_info.family == CHIP_VEGA12 ||
908 cmd_buffer->device->physical_device->rad_info.family == CHIP_VEGA20 ||
909 cmd_buffer->device->physical_device->rad_info.family == CHIP_RAVEN2 ||
910 cmd_buffer->device->physical_device->rad_info.chip_class >= GFX10) {
911 binning_flush = !old_pipeline ||
912 G_028C44_BINNING_MODE(old_pipeline->graphics.binning.pa_sc_binner_cntl_0) !=
913 G_028C44_BINNING_MODE(pipeline->graphics.binning.pa_sc_binner_cntl_0);
914 }
915
916 radeon_set_context_reg(cmd_buffer->cs, R_028C44_PA_SC_BINNER_CNTL_0,
917 pipeline->graphics.binning.pa_sc_binner_cntl_0 |
918 S_028C44_FLUSH_ON_BINNING_TRANSITION(!!binning_flush));
919
920 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX10) {
921 radeon_set_context_reg(cmd_buffer->cs, R_028038_DB_DFSM_CONTROL,
922 pipeline->graphics.binning.db_dfsm_control);
923 } else {
924 radeon_set_context_reg(cmd_buffer->cs, R_028060_DB_DFSM_CONTROL,
925 pipeline->graphics.binning.db_dfsm_control);
926 }
927
928 cmd_buffer->state.context_roll_without_scissor_emitted = true;
929 }
930
931
932 static void
933 radv_emit_shader_prefetch(struct radv_cmd_buffer *cmd_buffer,
934 struct radv_shader_variant *shader)
935 {
936 uint64_t va;
937
938 if (!shader)
939 return;
940
941 va = radv_buffer_get_va(shader->bo) + shader->bo_offset;
942
943 si_cp_dma_prefetch(cmd_buffer, va, shader->code_size);
944 }
945
946 static void
947 radv_emit_prefetch_L2(struct radv_cmd_buffer *cmd_buffer,
948 struct radv_pipeline *pipeline,
949 bool vertex_stage_only)
950 {
951 struct radv_cmd_state *state = &cmd_buffer->state;
952 uint32_t mask = state->prefetch_L2_mask;
953
954 if (vertex_stage_only) {
955 /* Fast prefetch path for starting draws as soon as possible.
956 */
957 mask = state->prefetch_L2_mask & (RADV_PREFETCH_VS |
958 RADV_PREFETCH_VBO_DESCRIPTORS);
959 }
960
961 if (mask & RADV_PREFETCH_VS)
962 radv_emit_shader_prefetch(cmd_buffer,
963 pipeline->shaders[MESA_SHADER_VERTEX]);
964
965 if (mask & RADV_PREFETCH_VBO_DESCRIPTORS)
966 si_cp_dma_prefetch(cmd_buffer, state->vb_va, state->vb_size);
967
968 if (mask & RADV_PREFETCH_TCS)
969 radv_emit_shader_prefetch(cmd_buffer,
970 pipeline->shaders[MESA_SHADER_TESS_CTRL]);
971
972 if (mask & RADV_PREFETCH_TES)
973 radv_emit_shader_prefetch(cmd_buffer,
974 pipeline->shaders[MESA_SHADER_TESS_EVAL]);
975
976 if (mask & RADV_PREFETCH_GS) {
977 radv_emit_shader_prefetch(cmd_buffer,
978 pipeline->shaders[MESA_SHADER_GEOMETRY]);
979 if (radv_pipeline_has_gs_copy_shader(pipeline))
980 radv_emit_shader_prefetch(cmd_buffer, pipeline->gs_copy_shader);
981 }
982
983 if (mask & RADV_PREFETCH_PS)
984 radv_emit_shader_prefetch(cmd_buffer,
985 pipeline->shaders[MESA_SHADER_FRAGMENT]);
986
987 state->prefetch_L2_mask &= ~mask;
988 }
989
990 static void
991 radv_emit_rbplus_state(struct radv_cmd_buffer *cmd_buffer)
992 {
993 if (!cmd_buffer->device->physical_device->rad_info.rbplus_allowed)
994 return;
995
996 struct radv_pipeline *pipeline = cmd_buffer->state.pipeline;
997 const struct radv_subpass *subpass = cmd_buffer->state.subpass;
998
999 unsigned sx_ps_downconvert = 0;
1000 unsigned sx_blend_opt_epsilon = 0;
1001 unsigned sx_blend_opt_control = 0;
1002
1003 for (unsigned i = 0; i < subpass->color_count; ++i) {
1004 if (subpass->color_attachments[i].attachment == VK_ATTACHMENT_UNUSED) {
1005 sx_blend_opt_control |= S_02875C_MRT0_COLOR_OPT_DISABLE(1) << (i * 4);
1006 sx_blend_opt_control |= S_02875C_MRT0_ALPHA_OPT_DISABLE(1) << (i * 4);
1007 continue;
1008 }
1009
1010 int idx = subpass->color_attachments[i].attachment;
1011 struct radv_color_buffer_info *cb = &cmd_buffer->state.attachments[idx].cb;
1012
1013 unsigned format = G_028C70_FORMAT(cb->cb_color_info);
1014 unsigned swap = G_028C70_COMP_SWAP(cb->cb_color_info);
1015 uint32_t spi_format = (pipeline->graphics.col_format >> (i * 4)) & 0xf;
1016 uint32_t colormask = (pipeline->graphics.cb_target_mask >> (i * 4)) & 0xf;
1017
1018 bool has_alpha, has_rgb;
1019
1020 /* Set if RGB and A are present. */
1021 has_alpha = !G_028C74_FORCE_DST_ALPHA_1(cb->cb_color_attrib);
1022
1023 if (format == V_028C70_COLOR_8 ||
1024 format == V_028C70_COLOR_16 ||
1025 format == V_028C70_COLOR_32)
1026 has_rgb = !has_alpha;
1027 else
1028 has_rgb = true;
1029
1030 /* Check the colormask and export format. */
1031 if (!(colormask & 0x7))
1032 has_rgb = false;
1033 if (!(colormask & 0x8))
1034 has_alpha = false;
1035
1036 if (spi_format == V_028714_SPI_SHADER_ZERO) {
1037 has_rgb = false;
1038 has_alpha = false;
1039 }
1040
1041 /* Disable value checking for disabled channels. */
1042 if (!has_rgb)
1043 sx_blend_opt_control |= S_02875C_MRT0_COLOR_OPT_DISABLE(1) << (i * 4);
1044 if (!has_alpha)
1045 sx_blend_opt_control |= S_02875C_MRT0_ALPHA_OPT_DISABLE(1) << (i * 4);
1046
1047 /* Enable down-conversion for 32bpp and smaller formats. */
1048 switch (format) {
1049 case V_028C70_COLOR_8:
1050 case V_028C70_COLOR_8_8:
1051 case V_028C70_COLOR_8_8_8_8:
1052 /* For 1 and 2-channel formats, use the superset thereof. */
1053 if (spi_format == V_028714_SPI_SHADER_FP16_ABGR ||
1054 spi_format == V_028714_SPI_SHADER_UINT16_ABGR ||
1055 spi_format == V_028714_SPI_SHADER_SINT16_ABGR) {
1056 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_8_8_8_8 << (i * 4);
1057 sx_blend_opt_epsilon |= V_028758_8BIT_FORMAT << (i * 4);
1058 }
1059 break;
1060
1061 case V_028C70_COLOR_5_6_5:
1062 if (spi_format == V_028714_SPI_SHADER_FP16_ABGR) {
1063 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_5_6_5 << (i * 4);
1064 sx_blend_opt_epsilon |= V_028758_6BIT_FORMAT << (i * 4);
1065 }
1066 break;
1067
1068 case V_028C70_COLOR_1_5_5_5:
1069 if (spi_format == V_028714_SPI_SHADER_FP16_ABGR) {
1070 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_1_5_5_5 << (i * 4);
1071 sx_blend_opt_epsilon |= V_028758_5BIT_FORMAT << (i * 4);
1072 }
1073 break;
1074
1075 case V_028C70_COLOR_4_4_4_4:
1076 if (spi_format == V_028714_SPI_SHADER_FP16_ABGR) {
1077 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_4_4_4_4 << (i * 4);
1078 sx_blend_opt_epsilon |= V_028758_4BIT_FORMAT << (i * 4);
1079 }
1080 break;
1081
1082 case V_028C70_COLOR_32:
1083 if (swap == V_028C70_SWAP_STD &&
1084 spi_format == V_028714_SPI_SHADER_32_R)
1085 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_32_R << (i * 4);
1086 else if (swap == V_028C70_SWAP_ALT_REV &&
1087 spi_format == V_028714_SPI_SHADER_32_AR)
1088 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_32_A << (i * 4);
1089 break;
1090
1091 case V_028C70_COLOR_16:
1092 case V_028C70_COLOR_16_16:
1093 /* For 1-channel formats, use the superset thereof. */
1094 if (spi_format == V_028714_SPI_SHADER_UNORM16_ABGR ||
1095 spi_format == V_028714_SPI_SHADER_SNORM16_ABGR ||
1096 spi_format == V_028714_SPI_SHADER_UINT16_ABGR ||
1097 spi_format == V_028714_SPI_SHADER_SINT16_ABGR) {
1098 if (swap == V_028C70_SWAP_STD ||
1099 swap == V_028C70_SWAP_STD_REV)
1100 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_16_16_GR << (i * 4);
1101 else
1102 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_16_16_AR << (i * 4);
1103 }
1104 break;
1105
1106 case V_028C70_COLOR_10_11_11:
1107 if (spi_format == V_028714_SPI_SHADER_FP16_ABGR) {
1108 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_10_11_11 << (i * 4);
1109 sx_blend_opt_epsilon |= V_028758_11BIT_FORMAT << (i * 4);
1110 }
1111 break;
1112
1113 case V_028C70_COLOR_2_10_10_10:
1114 if (spi_format == V_028714_SPI_SHADER_FP16_ABGR) {
1115 sx_ps_downconvert |= V_028754_SX_RT_EXPORT_2_10_10_10 << (i * 4);
1116 sx_blend_opt_epsilon |= V_028758_10BIT_FORMAT << (i * 4);
1117 }
1118 break;
1119 }
1120 }
1121
1122 for (unsigned i = subpass->color_count; i < 8; ++i) {
1123 sx_blend_opt_control |= S_02875C_MRT0_COLOR_OPT_DISABLE(1) << (i * 4);
1124 sx_blend_opt_control |= S_02875C_MRT0_ALPHA_OPT_DISABLE(1) << (i * 4);
1125 }
1126 /* TODO: avoid redundantly setting context registers */
1127 radeon_set_context_reg_seq(cmd_buffer->cs, R_028754_SX_PS_DOWNCONVERT, 3);
1128 radeon_emit(cmd_buffer->cs, sx_ps_downconvert);
1129 radeon_emit(cmd_buffer->cs, sx_blend_opt_epsilon);
1130 radeon_emit(cmd_buffer->cs, sx_blend_opt_control);
1131
1132 cmd_buffer->state.context_roll_without_scissor_emitted = true;
1133 }
1134
1135 static void
1136 radv_emit_graphics_pipeline(struct radv_cmd_buffer *cmd_buffer)
1137 {
1138 struct radv_pipeline *pipeline = cmd_buffer->state.pipeline;
1139
1140 if (!pipeline || cmd_buffer->state.emitted_pipeline == pipeline)
1141 return;
1142
1143 radv_update_multisample_state(cmd_buffer, pipeline);
1144 radv_update_binning_state(cmd_buffer, pipeline);
1145
1146 cmd_buffer->scratch_size_needed =
1147 MAX2(cmd_buffer->scratch_size_needed,
1148 pipeline->max_waves * pipeline->scratch_bytes_per_wave);
1149
1150 if (!cmd_buffer->state.emitted_pipeline ||
1151 cmd_buffer->state.emitted_pipeline->graphics.can_use_guardband !=
1152 pipeline->graphics.can_use_guardband)
1153 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_SCISSOR;
1154
1155 radeon_emit_array(cmd_buffer->cs, pipeline->cs.buf, pipeline->cs.cdw);
1156
1157 if (!cmd_buffer->state.emitted_pipeline ||
1158 cmd_buffer->state.emitted_pipeline->ctx_cs.cdw != pipeline->ctx_cs.cdw ||
1159 cmd_buffer->state.emitted_pipeline->ctx_cs_hash != pipeline->ctx_cs_hash ||
1160 memcmp(cmd_buffer->state.emitted_pipeline->ctx_cs.buf,
1161 pipeline->ctx_cs.buf, pipeline->ctx_cs.cdw * 4)) {
1162 radeon_emit_array(cmd_buffer->cs, pipeline->ctx_cs.buf, pipeline->ctx_cs.cdw);
1163 cmd_buffer->state.context_roll_without_scissor_emitted = true;
1164 }
1165
1166 for (unsigned i = 0; i < MESA_SHADER_COMPUTE; i++) {
1167 if (!pipeline->shaders[i])
1168 continue;
1169
1170 radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs,
1171 pipeline->shaders[i]->bo);
1172 }
1173
1174 if (radv_pipeline_has_gs_copy_shader(pipeline))
1175 radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs,
1176 pipeline->gs_copy_shader->bo);
1177
1178 if (unlikely(cmd_buffer->device->trace_bo))
1179 radv_save_pipeline(cmd_buffer, pipeline, RING_GFX);
1180
1181 cmd_buffer->state.emitted_pipeline = pipeline;
1182
1183 cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_PIPELINE;
1184 }
1185
1186 static void
1187 radv_emit_viewport(struct radv_cmd_buffer *cmd_buffer)
1188 {
1189 si_write_viewport(cmd_buffer->cs, 0, cmd_buffer->state.dynamic.viewport.count,
1190 cmd_buffer->state.dynamic.viewport.viewports);
1191 }
1192
1193 static void
1194 radv_emit_scissor(struct radv_cmd_buffer *cmd_buffer)
1195 {
1196 uint32_t count = cmd_buffer->state.dynamic.scissor.count;
1197
1198 si_write_scissors(cmd_buffer->cs, 0, count,
1199 cmd_buffer->state.dynamic.scissor.scissors,
1200 cmd_buffer->state.dynamic.viewport.viewports,
1201 cmd_buffer->state.emitted_pipeline->graphics.can_use_guardband);
1202
1203 cmd_buffer->state.context_roll_without_scissor_emitted = false;
1204 }
1205
1206 static void
1207 radv_emit_discard_rectangle(struct radv_cmd_buffer *cmd_buffer)
1208 {
1209 if (!cmd_buffer->state.dynamic.discard_rectangle.count)
1210 return;
1211
1212 radeon_set_context_reg_seq(cmd_buffer->cs, R_028210_PA_SC_CLIPRECT_0_TL,
1213 cmd_buffer->state.dynamic.discard_rectangle.count * 2);
1214 for (unsigned i = 0; i < cmd_buffer->state.dynamic.discard_rectangle.count; ++i) {
1215 VkRect2D rect = cmd_buffer->state.dynamic.discard_rectangle.rectangles[i];
1216 radeon_emit(cmd_buffer->cs, S_028210_TL_X(rect.offset.x) | S_028210_TL_Y(rect.offset.y));
1217 radeon_emit(cmd_buffer->cs, S_028214_BR_X(rect.offset.x + rect.extent.width) |
1218 S_028214_BR_Y(rect.offset.y + rect.extent.height));
1219 }
1220 }
1221
1222 static void
1223 radv_emit_line_width(struct radv_cmd_buffer *cmd_buffer)
1224 {
1225 unsigned width = cmd_buffer->state.dynamic.line_width * 8;
1226
1227 radeon_set_context_reg(cmd_buffer->cs, R_028A08_PA_SU_LINE_CNTL,
1228 S_028A08_WIDTH(CLAMP(width, 0, 0xFFF)));
1229 }
1230
1231 static void
1232 radv_emit_blend_constants(struct radv_cmd_buffer *cmd_buffer)
1233 {
1234 struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;
1235
1236 radeon_set_context_reg_seq(cmd_buffer->cs, R_028414_CB_BLEND_RED, 4);
1237 radeon_emit_array(cmd_buffer->cs, (uint32_t *)d->blend_constants, 4);
1238 }
1239
1240 static void
1241 radv_emit_stencil(struct radv_cmd_buffer *cmd_buffer)
1242 {
1243 struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;
1244
1245 radeon_set_context_reg_seq(cmd_buffer->cs,
1246 R_028430_DB_STENCILREFMASK, 2);
1247 radeon_emit(cmd_buffer->cs,
1248 S_028430_STENCILTESTVAL(d->stencil_reference.front) |
1249 S_028430_STENCILMASK(d->stencil_compare_mask.front) |
1250 S_028430_STENCILWRITEMASK(d->stencil_write_mask.front) |
1251 S_028430_STENCILOPVAL(1));
1252 radeon_emit(cmd_buffer->cs,
1253 S_028434_STENCILTESTVAL_BF(d->stencil_reference.back) |
1254 S_028434_STENCILMASK_BF(d->stencil_compare_mask.back) |
1255 S_028434_STENCILWRITEMASK_BF(d->stencil_write_mask.back) |
1256 S_028434_STENCILOPVAL_BF(1));
1257 }
1258
1259 static void
1260 radv_emit_depth_bounds(struct radv_cmd_buffer *cmd_buffer)
1261 {
1262 struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;
1263
1264 radeon_set_context_reg(cmd_buffer->cs, R_028020_DB_DEPTH_BOUNDS_MIN,
1265 fui(d->depth_bounds.min));
1266 radeon_set_context_reg(cmd_buffer->cs, R_028024_DB_DEPTH_BOUNDS_MAX,
1267 fui(d->depth_bounds.max));
1268 }
1269
1270 static void
1271 radv_emit_depth_bias(struct radv_cmd_buffer *cmd_buffer)
1272 {
1273 struct radv_dynamic_state *d = &cmd_buffer->state.dynamic;
1274 unsigned slope = fui(d->depth_bias.slope * 16.0f);
1275 unsigned bias = fui(d->depth_bias.bias * cmd_buffer->state.offset_scale);
1276
1277
1278 radeon_set_context_reg_seq(cmd_buffer->cs,
1279 R_028B7C_PA_SU_POLY_OFFSET_CLAMP, 5);
1280 radeon_emit(cmd_buffer->cs, fui(d->depth_bias.clamp)); /* CLAMP */
1281 radeon_emit(cmd_buffer->cs, slope); /* FRONT SCALE */
1282 radeon_emit(cmd_buffer->cs, bias); /* FRONT OFFSET */
1283 radeon_emit(cmd_buffer->cs, slope); /* BACK SCALE */
1284 radeon_emit(cmd_buffer->cs, bias); /* BACK OFFSET */
1285 }
1286
1287 static void
1288 radv_emit_fb_color_state(struct radv_cmd_buffer *cmd_buffer,
1289 int index,
1290 struct radv_color_buffer_info *cb,
1291 struct radv_image_view *iview,
1292 VkImageLayout layout,
1293 bool in_render_loop)
1294 {
1295 bool is_vi = cmd_buffer->device->physical_device->rad_info.chip_class >= GFX8;
1296 uint32_t cb_color_info = cb->cb_color_info;
1297 struct radv_image *image = iview->image;
1298
1299 if (!radv_layout_dcc_compressed(cmd_buffer->device, image, layout, in_render_loop,
1300 radv_image_queue_family_mask(image,
1301 cmd_buffer->queue_family_index,
1302 cmd_buffer->queue_family_index))) {
1303 cb_color_info &= C_028C70_DCC_ENABLE;
1304 }
1305
1306 if (radv_image_is_tc_compat_cmask(image) &&
1307 (radv_is_fmask_decompress_pipeline(cmd_buffer) ||
1308 radv_is_dcc_decompress_pipeline(cmd_buffer))) {
1309 /* If this bit is set, the FMASK decompression operation
1310 * doesn't occur (DCC_COMPRESS also implies FMASK_DECOMPRESS).
1311 */
1312 cb_color_info &= C_028C70_FMASK_COMPRESS_1FRAG_ONLY;
1313 }
1314
1315 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX10) {
1316 radeon_set_context_reg_seq(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x3c, 11);
1317 radeon_emit(cmd_buffer->cs, cb->cb_color_base);
1318 radeon_emit(cmd_buffer->cs, 0);
1319 radeon_emit(cmd_buffer->cs, 0);
1320 radeon_emit(cmd_buffer->cs, cb->cb_color_view);
1321 radeon_emit(cmd_buffer->cs, cb_color_info);
1322 radeon_emit(cmd_buffer->cs, cb->cb_color_attrib);
1323 radeon_emit(cmd_buffer->cs, cb->cb_dcc_control);
1324 radeon_emit(cmd_buffer->cs, cb->cb_color_cmask);
1325 radeon_emit(cmd_buffer->cs, 0);
1326 radeon_emit(cmd_buffer->cs, cb->cb_color_fmask);
1327 radeon_emit(cmd_buffer->cs, 0);
1328
1329 radeon_set_context_reg_seq(cmd_buffer->cs, R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, 1);
1330 radeon_emit(cmd_buffer->cs, cb->cb_dcc_base);
1331
1332 radeon_set_context_reg(cmd_buffer->cs, R_028E40_CB_COLOR0_BASE_EXT + index * 4,
1333 cb->cb_color_base >> 32);
1334 radeon_set_context_reg(cmd_buffer->cs, R_028E60_CB_COLOR0_CMASK_BASE_EXT + index * 4,
1335 cb->cb_color_cmask >> 32);
1336 radeon_set_context_reg(cmd_buffer->cs, R_028E80_CB_COLOR0_FMASK_BASE_EXT + index * 4,
1337 cb->cb_color_fmask >> 32);
1338 radeon_set_context_reg(cmd_buffer->cs, R_028EA0_CB_COLOR0_DCC_BASE_EXT + index * 4,
1339 cb->cb_dcc_base >> 32);
1340 radeon_set_context_reg(cmd_buffer->cs, R_028EC0_CB_COLOR0_ATTRIB2 + index * 4,
1341 cb->cb_color_attrib2);
1342 radeon_set_context_reg(cmd_buffer->cs, R_028EE0_CB_COLOR0_ATTRIB3 + index * 4,
1343 cb->cb_color_attrib3);
1344 } else if (cmd_buffer->device->physical_device->rad_info.chip_class == GFX9) {
1345 radeon_set_context_reg_seq(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x3c, 11);
1346 radeon_emit(cmd_buffer->cs, cb->cb_color_base);
1347 radeon_emit(cmd_buffer->cs, S_028C64_BASE_256B(cb->cb_color_base >> 32));
1348 radeon_emit(cmd_buffer->cs, cb->cb_color_attrib2);
1349 radeon_emit(cmd_buffer->cs, cb->cb_color_view);
1350 radeon_emit(cmd_buffer->cs, cb_color_info);
1351 radeon_emit(cmd_buffer->cs, cb->cb_color_attrib);
1352 radeon_emit(cmd_buffer->cs, cb->cb_dcc_control);
1353 radeon_emit(cmd_buffer->cs, cb->cb_color_cmask);
1354 radeon_emit(cmd_buffer->cs, S_028C80_BASE_256B(cb->cb_color_cmask >> 32));
1355 radeon_emit(cmd_buffer->cs, cb->cb_color_fmask);
1356 radeon_emit(cmd_buffer->cs, S_028C88_BASE_256B(cb->cb_color_fmask >> 32));
1357
1358 radeon_set_context_reg_seq(cmd_buffer->cs, R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, 2);
1359 radeon_emit(cmd_buffer->cs, cb->cb_dcc_base);
1360 radeon_emit(cmd_buffer->cs, S_028C98_BASE_256B(cb->cb_dcc_base >> 32));
1361
1362 radeon_set_context_reg(cmd_buffer->cs, R_0287A0_CB_MRT0_EPITCH + index * 4,
1363 cb->cb_mrt_epitch);
1364 } else {
1365 radeon_set_context_reg_seq(cmd_buffer->cs, R_028C60_CB_COLOR0_BASE + index * 0x3c, 11);
1366 radeon_emit(cmd_buffer->cs, cb->cb_color_base);
1367 radeon_emit(cmd_buffer->cs, cb->cb_color_pitch);
1368 radeon_emit(cmd_buffer->cs, cb->cb_color_slice);
1369 radeon_emit(cmd_buffer->cs, cb->cb_color_view);
1370 radeon_emit(cmd_buffer->cs, cb_color_info);
1371 radeon_emit(cmd_buffer->cs, cb->cb_color_attrib);
1372 radeon_emit(cmd_buffer->cs, cb->cb_dcc_control);
1373 radeon_emit(cmd_buffer->cs, cb->cb_color_cmask);
1374 radeon_emit(cmd_buffer->cs, cb->cb_color_cmask_slice);
1375 radeon_emit(cmd_buffer->cs, cb->cb_color_fmask);
1376 radeon_emit(cmd_buffer->cs, cb->cb_color_fmask_slice);
1377
1378 if (is_vi) { /* DCC BASE */
1379 radeon_set_context_reg(cmd_buffer->cs, R_028C94_CB_COLOR0_DCC_BASE + index * 0x3c, cb->cb_dcc_base);
1380 }
1381 }
1382
1383 if (radv_dcc_enabled(image, iview->base_mip)) {
1384 /* Drawing with DCC enabled also compresses colorbuffers. */
1385 VkImageSubresourceRange range = {
1386 .aspectMask = iview->aspect_mask,
1387 .baseMipLevel = iview->base_mip,
1388 .levelCount = iview->level_count,
1389 .baseArrayLayer = iview->base_layer,
1390 .layerCount = iview->layer_count,
1391 };
1392
1393 radv_update_dcc_metadata(cmd_buffer, image, &range, true);
1394 }
1395 }
1396
1397 static void
1398 radv_update_zrange_precision(struct radv_cmd_buffer *cmd_buffer,
1399 struct radv_ds_buffer_info *ds,
1400 const struct radv_image_view *iview,
1401 VkImageLayout layout,
1402 bool in_render_loop, bool requires_cond_exec)
1403 {
1404 const struct radv_image *image = iview->image;
1405 uint32_t db_z_info = ds->db_z_info;
1406 uint32_t db_z_info_reg;
1407
1408 if (!cmd_buffer->device->physical_device->rad_info.has_tc_compat_zrange_bug ||
1409 !radv_image_is_tc_compat_htile(image))
1410 return;
1411
1412 if (!radv_layout_has_htile(image, layout, in_render_loop,
1413 radv_image_queue_family_mask(image,
1414 cmd_buffer->queue_family_index,
1415 cmd_buffer->queue_family_index))) {
1416 db_z_info &= C_028040_TILE_SURFACE_ENABLE;
1417 }
1418
1419 db_z_info &= C_028040_ZRANGE_PRECISION;
1420
1421 if (cmd_buffer->device->physical_device->rad_info.chip_class == GFX9) {
1422 db_z_info_reg = R_028038_DB_Z_INFO;
1423 } else {
1424 db_z_info_reg = R_028040_DB_Z_INFO;
1425 }
1426
1427 /* When we don't know the last fast clear value we need to emit a
1428 * conditional packet that will eventually skip the following
1429 * SET_CONTEXT_REG packet.
1430 */
1431 if (requires_cond_exec) {
1432 uint64_t va = radv_get_tc_compat_zrange_va(image, iview->base_mip);
1433
1434 radeon_emit(cmd_buffer->cs, PKT3(PKT3_COND_EXEC, 3, 0));
1435 radeon_emit(cmd_buffer->cs, va);
1436 radeon_emit(cmd_buffer->cs, va >> 32);
1437 radeon_emit(cmd_buffer->cs, 0);
1438 radeon_emit(cmd_buffer->cs, 3); /* SET_CONTEXT_REG size */
1439 }
1440
1441 radeon_set_context_reg(cmd_buffer->cs, db_z_info_reg, db_z_info);
1442 }
1443
1444 static void
1445 radv_emit_fb_ds_state(struct radv_cmd_buffer *cmd_buffer,
1446 struct radv_ds_buffer_info *ds,
1447 struct radv_image_view *iview,
1448 VkImageLayout layout,
1449 bool in_render_loop)
1450 {
1451 const struct radv_image *image = iview->image;
1452 uint32_t db_z_info = ds->db_z_info;
1453 uint32_t db_stencil_info = ds->db_stencil_info;
1454
1455 if (!radv_layout_has_htile(image, layout, in_render_loop,
1456 radv_image_queue_family_mask(image,
1457 cmd_buffer->queue_family_index,
1458 cmd_buffer->queue_family_index))) {
1459 db_z_info &= C_028040_TILE_SURFACE_ENABLE;
1460 db_stencil_info |= S_028044_TILE_STENCIL_DISABLE(1);
1461 }
1462
1463 radeon_set_context_reg(cmd_buffer->cs, R_028008_DB_DEPTH_VIEW, ds->db_depth_view);
1464 radeon_set_context_reg(cmd_buffer->cs, R_028ABC_DB_HTILE_SURFACE, ds->db_htile_surface);
1465
1466 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX10) {
1467 radeon_set_context_reg(cmd_buffer->cs, R_028014_DB_HTILE_DATA_BASE, ds->db_htile_data_base);
1468 radeon_set_context_reg(cmd_buffer->cs, R_02801C_DB_DEPTH_SIZE_XY, ds->db_depth_size);
1469
1470 radeon_set_context_reg_seq(cmd_buffer->cs, R_02803C_DB_DEPTH_INFO, 7);
1471 radeon_emit(cmd_buffer->cs, S_02803C_RESOURCE_LEVEL(1));
1472 radeon_emit(cmd_buffer->cs, db_z_info);
1473 radeon_emit(cmd_buffer->cs, db_stencil_info);
1474 radeon_emit(cmd_buffer->cs, ds->db_z_read_base);
1475 radeon_emit(cmd_buffer->cs, ds->db_stencil_read_base);
1476 radeon_emit(cmd_buffer->cs, ds->db_z_read_base);
1477 radeon_emit(cmd_buffer->cs, ds->db_stencil_read_base);
1478
1479 radeon_set_context_reg_seq(cmd_buffer->cs, R_028068_DB_Z_READ_BASE_HI, 5);
1480 radeon_emit(cmd_buffer->cs, ds->db_z_read_base >> 32);
1481 radeon_emit(cmd_buffer->cs, ds->db_stencil_read_base >> 32);
1482 radeon_emit(cmd_buffer->cs, ds->db_z_read_base >> 32);
1483 radeon_emit(cmd_buffer->cs, ds->db_stencil_read_base >> 32);
1484 radeon_emit(cmd_buffer->cs, ds->db_htile_data_base >> 32);
1485 } else if (cmd_buffer->device->physical_device->rad_info.chip_class == GFX9) {
1486 radeon_set_context_reg_seq(cmd_buffer->cs, R_028014_DB_HTILE_DATA_BASE, 3);
1487 radeon_emit(cmd_buffer->cs, ds->db_htile_data_base);
1488 radeon_emit(cmd_buffer->cs, S_028018_BASE_HI(ds->db_htile_data_base >> 32));
1489 radeon_emit(cmd_buffer->cs, ds->db_depth_size);
1490
1491 radeon_set_context_reg_seq(cmd_buffer->cs, R_028038_DB_Z_INFO, 10);
1492 radeon_emit(cmd_buffer->cs, db_z_info); /* DB_Z_INFO */
1493 radeon_emit(cmd_buffer->cs, db_stencil_info); /* DB_STENCIL_INFO */
1494 radeon_emit(cmd_buffer->cs, ds->db_z_read_base); /* DB_Z_READ_BASE */
1495 radeon_emit(cmd_buffer->cs, S_028044_BASE_HI(ds->db_z_read_base >> 32)); /* DB_Z_READ_BASE_HI */
1496 radeon_emit(cmd_buffer->cs, ds->db_stencil_read_base); /* DB_STENCIL_READ_BASE */
1497 radeon_emit(cmd_buffer->cs, S_02804C_BASE_HI(ds->db_stencil_read_base >> 32)); /* DB_STENCIL_READ_BASE_HI */
1498 radeon_emit(cmd_buffer->cs, ds->db_z_write_base); /* DB_Z_WRITE_BASE */
1499 radeon_emit(cmd_buffer->cs, S_028054_BASE_HI(ds->db_z_write_base >> 32)); /* DB_Z_WRITE_BASE_HI */
1500 radeon_emit(cmd_buffer->cs, ds->db_stencil_write_base); /* DB_STENCIL_WRITE_BASE */
1501 radeon_emit(cmd_buffer->cs, S_02805C_BASE_HI(ds->db_stencil_write_base >> 32)); /* DB_STENCIL_WRITE_BASE_HI */
1502
1503 radeon_set_context_reg_seq(cmd_buffer->cs, R_028068_DB_Z_INFO2, 2);
1504 radeon_emit(cmd_buffer->cs, ds->db_z_info2);
1505 radeon_emit(cmd_buffer->cs, ds->db_stencil_info2);
1506 } else {
1507 radeon_set_context_reg(cmd_buffer->cs, R_028014_DB_HTILE_DATA_BASE, ds->db_htile_data_base);
1508
1509 radeon_set_context_reg_seq(cmd_buffer->cs, R_02803C_DB_DEPTH_INFO, 9);
1510 radeon_emit(cmd_buffer->cs, ds->db_depth_info); /* R_02803C_DB_DEPTH_INFO */
1511 radeon_emit(cmd_buffer->cs, db_z_info); /* R_028040_DB_Z_INFO */
1512 radeon_emit(cmd_buffer->cs, db_stencil_info); /* R_028044_DB_STENCIL_INFO */
1513 radeon_emit(cmd_buffer->cs, ds->db_z_read_base); /* R_028048_DB_Z_READ_BASE */
1514 radeon_emit(cmd_buffer->cs, ds->db_stencil_read_base); /* R_02804C_DB_STENCIL_READ_BASE */
1515 radeon_emit(cmd_buffer->cs, ds->db_z_write_base); /* R_028050_DB_Z_WRITE_BASE */
1516 radeon_emit(cmd_buffer->cs, ds->db_stencil_write_base); /* R_028054_DB_STENCIL_WRITE_BASE */
1517 radeon_emit(cmd_buffer->cs, ds->db_depth_size); /* R_028058_DB_DEPTH_SIZE */
1518 radeon_emit(cmd_buffer->cs, ds->db_depth_slice); /* R_02805C_DB_DEPTH_SLICE */
1519
1520 }
1521
1522 /* Update the ZRANGE_PRECISION value for the TC-compat bug. */
1523 radv_update_zrange_precision(cmd_buffer, ds, iview, layout,
1524 in_render_loop, true);
1525
1526 radeon_set_context_reg(cmd_buffer->cs, R_028B78_PA_SU_POLY_OFFSET_DB_FMT_CNTL,
1527 ds->pa_su_poly_offset_db_fmt_cntl);
1528 }
1529
1530 /**
1531 * Update the fast clear depth/stencil values if the image is bound as a
1532 * depth/stencil buffer.
1533 */
1534 static void
1535 radv_update_bound_fast_clear_ds(struct radv_cmd_buffer *cmd_buffer,
1536 const struct radv_image_view *iview,
1537 VkClearDepthStencilValue ds_clear_value,
1538 VkImageAspectFlags aspects)
1539 {
1540 const struct radv_subpass *subpass = cmd_buffer->state.subpass;
1541 const struct radv_image *image = iview->image;
1542 struct radeon_cmdbuf *cs = cmd_buffer->cs;
1543 uint32_t att_idx;
1544
1545 if (!cmd_buffer->state.attachments || !subpass)
1546 return;
1547
1548 if (!subpass->depth_stencil_attachment)
1549 return;
1550
1551 att_idx = subpass->depth_stencil_attachment->attachment;
1552 if (cmd_buffer->state.attachments[att_idx].iview->image != image)
1553 return;
1554
1555 radeon_set_context_reg_seq(cs, R_028028_DB_STENCIL_CLEAR, 2);
1556 radeon_emit(cs, ds_clear_value.stencil);
1557 radeon_emit(cs, fui(ds_clear_value.depth));
1558
1559 /* Update the ZRANGE_PRECISION value for the TC-compat bug. This is
1560 * only needed when clearing Z to 0.0.
1561 */
1562 if ((aspects & VK_IMAGE_ASPECT_DEPTH_BIT) &&
1563 ds_clear_value.depth == 0.0) {
1564 VkImageLayout layout = subpass->depth_stencil_attachment->layout;
1565 bool in_render_loop = subpass->depth_stencil_attachment->in_render_loop;
1566
1567 radv_update_zrange_precision(cmd_buffer, &cmd_buffer->state.attachments[att_idx].ds,
1568 iview, layout, in_render_loop, false);
1569 }
1570
1571 cmd_buffer->state.context_roll_without_scissor_emitted = true;
1572 }
1573
1574 /**
1575 * Set the clear depth/stencil values to the image's metadata.
1576 */
1577 static void
1578 radv_set_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer,
1579 struct radv_image *image,
1580 const VkImageSubresourceRange *range,
1581 VkClearDepthStencilValue ds_clear_value,
1582 VkImageAspectFlags aspects)
1583 {
1584 struct radeon_cmdbuf *cs = cmd_buffer->cs;
1585 uint64_t va = radv_get_ds_clear_value_va(image, range->baseMipLevel);
1586 uint32_t level_count = radv_get_levelCount(image, range);
1587
1588 if (aspects == (VK_IMAGE_ASPECT_DEPTH_BIT |
1589 VK_IMAGE_ASPECT_STENCIL_BIT)) {
1590 /* Use the fastest way when both aspects are used. */
1591 radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 2 + 2 * level_count, cmd_buffer->state.predicating));
1592 radeon_emit(cs, S_370_DST_SEL(V_370_MEM) |
1593 S_370_WR_CONFIRM(1) |
1594 S_370_ENGINE_SEL(V_370_PFP));
1595 radeon_emit(cs, va);
1596 radeon_emit(cs, va >> 32);
1597
1598 for (uint32_t l = 0; l < level_count; l++) {
1599 radeon_emit(cs, ds_clear_value.stencil);
1600 radeon_emit(cs, fui(ds_clear_value.depth));
1601 }
1602 } else {
1603 /* Otherwise we need one WRITE_DATA packet per level. */
1604 for (uint32_t l = 0; l < level_count; l++) {
1605 uint64_t va = radv_get_ds_clear_value_va(image, range->baseMipLevel + l);
1606 unsigned value;
1607
1608 if (aspects == VK_IMAGE_ASPECT_DEPTH_BIT) {
1609 value = fui(ds_clear_value.depth);
1610 va += 4;
1611 } else {
1612 assert(aspects == VK_IMAGE_ASPECT_STENCIL_BIT);
1613 value = ds_clear_value.stencil;
1614 }
1615
1616 radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 3, cmd_buffer->state.predicating));
1617 radeon_emit(cs, S_370_DST_SEL(V_370_MEM) |
1618 S_370_WR_CONFIRM(1) |
1619 S_370_ENGINE_SEL(V_370_PFP));
1620 radeon_emit(cs, va);
1621 radeon_emit(cs, va >> 32);
1622 radeon_emit(cs, value);
1623 }
1624 }
1625 }
1626
1627 /**
1628 * Update the TC-compat metadata value for this image.
1629 */
1630 static void
1631 radv_set_tc_compat_zrange_metadata(struct radv_cmd_buffer *cmd_buffer,
1632 struct radv_image *image,
1633 const VkImageSubresourceRange *range,
1634 uint32_t value)
1635 {
1636 struct radeon_cmdbuf *cs = cmd_buffer->cs;
1637
1638 if (!cmd_buffer->device->physical_device->rad_info.has_tc_compat_zrange_bug)
1639 return;
1640
1641 uint64_t va = radv_get_tc_compat_zrange_va(image, range->baseMipLevel);
1642 uint32_t level_count = radv_get_levelCount(image, range);
1643
1644 radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 2 + level_count, cmd_buffer->state.predicating));
1645 radeon_emit(cs, S_370_DST_SEL(V_370_MEM) |
1646 S_370_WR_CONFIRM(1) |
1647 S_370_ENGINE_SEL(V_370_PFP));
1648 radeon_emit(cs, va);
1649 radeon_emit(cs, va >> 32);
1650
1651 for (uint32_t l = 0; l < level_count; l++)
1652 radeon_emit(cs, value);
1653 }
1654
1655 static void
1656 radv_update_tc_compat_zrange_metadata(struct radv_cmd_buffer *cmd_buffer,
1657 const struct radv_image_view *iview,
1658 VkClearDepthStencilValue ds_clear_value)
1659 {
1660 VkImageSubresourceRange range = {
1661 .aspectMask = iview->aspect_mask,
1662 .baseMipLevel = iview->base_mip,
1663 .levelCount = iview->level_count,
1664 .baseArrayLayer = iview->base_layer,
1665 .layerCount = iview->layer_count,
1666 };
1667 uint32_t cond_val;
1668
1669 /* Conditionally set DB_Z_INFO.ZRANGE_PRECISION to 0 when the last
1670 * depth clear value is 0.0f.
1671 */
1672 cond_val = ds_clear_value.depth == 0.0f ? UINT_MAX : 0;
1673
1674 radv_set_tc_compat_zrange_metadata(cmd_buffer, iview->image, &range,
1675 cond_val);
1676 }
1677
1678 /**
1679 * Update the clear depth/stencil values for this image.
1680 */
1681 void
1682 radv_update_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer,
1683 const struct radv_image_view *iview,
1684 VkClearDepthStencilValue ds_clear_value,
1685 VkImageAspectFlags aspects)
1686 {
1687 VkImageSubresourceRange range = {
1688 .aspectMask = iview->aspect_mask,
1689 .baseMipLevel = iview->base_mip,
1690 .levelCount = iview->level_count,
1691 .baseArrayLayer = iview->base_layer,
1692 .layerCount = iview->layer_count,
1693 };
1694 struct radv_image *image = iview->image;
1695
1696 assert(radv_image_has_htile(image));
1697
1698 radv_set_ds_clear_metadata(cmd_buffer, iview->image, &range,
1699 ds_clear_value, aspects);
1700
1701 if (radv_image_is_tc_compat_htile(image) &&
1702 (aspects & VK_IMAGE_ASPECT_DEPTH_BIT)) {
1703 radv_update_tc_compat_zrange_metadata(cmd_buffer, iview,
1704 ds_clear_value);
1705 }
1706
1707 radv_update_bound_fast_clear_ds(cmd_buffer, iview, ds_clear_value,
1708 aspects);
1709 }
1710
1711 /**
1712 * Load the clear depth/stencil values from the image's metadata.
1713 */
1714 static void
1715 radv_load_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer,
1716 const struct radv_image_view *iview)
1717 {
1718 struct radeon_cmdbuf *cs = cmd_buffer->cs;
1719 const struct radv_image *image = iview->image;
1720 VkImageAspectFlags aspects = vk_format_aspects(image->vk_format);
1721 uint64_t va = radv_get_ds_clear_value_va(image, iview->base_mip);
1722 unsigned reg_offset = 0, reg_count = 0;
1723
1724 if (!radv_image_has_htile(image))
1725 return;
1726
1727 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1728 ++reg_count;
1729 } else {
1730 ++reg_offset;
1731 va += 4;
1732 }
1733 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT)
1734 ++reg_count;
1735
1736 uint32_t reg = R_028028_DB_STENCIL_CLEAR + 4 * reg_offset;
1737
1738 if (cmd_buffer->device->physical_device->rad_info.has_load_ctx_reg_pkt) {
1739 radeon_emit(cs, PKT3(PKT3_LOAD_CONTEXT_REG, 3, 0));
1740 radeon_emit(cs, va);
1741 radeon_emit(cs, va >> 32);
1742 radeon_emit(cs, (reg - SI_CONTEXT_REG_OFFSET) >> 2);
1743 radeon_emit(cs, reg_count);
1744 } else {
1745 radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
1746 radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) |
1747 COPY_DATA_DST_SEL(COPY_DATA_REG) |
1748 (reg_count == 2 ? COPY_DATA_COUNT_SEL : 0));
1749 radeon_emit(cs, va);
1750 radeon_emit(cs, va >> 32);
1751 radeon_emit(cs, reg >> 2);
1752 radeon_emit(cs, 0);
1753
1754 radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0));
1755 radeon_emit(cs, 0);
1756 }
1757 }
1758
1759 /*
1760 * With DCC some colors don't require CMASK elimination before being
1761 * used as a texture. This sets a predicate value to determine if the
1762 * cmask eliminate is required.
1763 */
1764 void
1765 radv_update_fce_metadata(struct radv_cmd_buffer *cmd_buffer,
1766 struct radv_image *image,
1767 const VkImageSubresourceRange *range, bool value)
1768 {
1769 uint64_t pred_val = value;
1770 uint64_t va = radv_image_get_fce_pred_va(image, range->baseMipLevel);
1771 uint32_t level_count = radv_get_levelCount(image, range);
1772 uint32_t count = 2 * level_count;
1773
1774 assert(radv_dcc_enabled(image, range->baseMipLevel));
1775
1776 radeon_emit(cmd_buffer->cs, PKT3(PKT3_WRITE_DATA, 2 + count, 0));
1777 radeon_emit(cmd_buffer->cs, S_370_DST_SEL(V_370_MEM) |
1778 S_370_WR_CONFIRM(1) |
1779 S_370_ENGINE_SEL(V_370_PFP));
1780 radeon_emit(cmd_buffer->cs, va);
1781 radeon_emit(cmd_buffer->cs, va >> 32);
1782
1783 for (uint32_t l = 0; l < level_count; l++) {
1784 radeon_emit(cmd_buffer->cs, pred_val);
1785 radeon_emit(cmd_buffer->cs, pred_val >> 32);
1786 }
1787 }
1788
1789 /**
1790 * Update the DCC predicate to reflect the compression state.
1791 */
1792 void
1793 radv_update_dcc_metadata(struct radv_cmd_buffer *cmd_buffer,
1794 struct radv_image *image,
1795 const VkImageSubresourceRange *range, bool value)
1796 {
1797 uint64_t pred_val = value;
1798 uint64_t va = radv_image_get_dcc_pred_va(image, range->baseMipLevel);
1799 uint32_t level_count = radv_get_levelCount(image, range);
1800 uint32_t count = 2 * level_count;
1801
1802 assert(radv_dcc_enabled(image, range->baseMipLevel));
1803
1804 radeon_emit(cmd_buffer->cs, PKT3(PKT3_WRITE_DATA, 2 + count, 0));
1805 radeon_emit(cmd_buffer->cs, S_370_DST_SEL(V_370_MEM) |
1806 S_370_WR_CONFIRM(1) |
1807 S_370_ENGINE_SEL(V_370_PFP));
1808 radeon_emit(cmd_buffer->cs, va);
1809 radeon_emit(cmd_buffer->cs, va >> 32);
1810
1811 for (uint32_t l = 0; l < level_count; l++) {
1812 radeon_emit(cmd_buffer->cs, pred_val);
1813 radeon_emit(cmd_buffer->cs, pred_val >> 32);
1814 }
1815 }
1816
1817 /**
1818 * Update the fast clear color values if the image is bound as a color buffer.
1819 */
1820 static void
1821 radv_update_bound_fast_clear_color(struct radv_cmd_buffer *cmd_buffer,
1822 struct radv_image *image,
1823 int cb_idx,
1824 uint32_t color_values[2])
1825 {
1826 const struct radv_subpass *subpass = cmd_buffer->state.subpass;
1827 struct radeon_cmdbuf *cs = cmd_buffer->cs;
1828 uint32_t att_idx;
1829
1830 if (!cmd_buffer->state.attachments || !subpass)
1831 return;
1832
1833 att_idx = subpass->color_attachments[cb_idx].attachment;
1834 if (att_idx == VK_ATTACHMENT_UNUSED)
1835 return;
1836
1837 if (cmd_buffer->state.attachments[att_idx].iview->image != image)
1838 return;
1839
1840 radeon_set_context_reg_seq(cs, R_028C8C_CB_COLOR0_CLEAR_WORD0 + cb_idx * 0x3c, 2);
1841 radeon_emit(cs, color_values[0]);
1842 radeon_emit(cs, color_values[1]);
1843
1844 cmd_buffer->state.context_roll_without_scissor_emitted = true;
1845 }
1846
1847 /**
1848 * Set the clear color values to the image's metadata.
1849 */
1850 static void
1851 radv_set_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer,
1852 struct radv_image *image,
1853 const VkImageSubresourceRange *range,
1854 uint32_t color_values[2])
1855 {
1856 struct radeon_cmdbuf *cs = cmd_buffer->cs;
1857 uint64_t va = radv_image_get_fast_clear_va(image, range->baseMipLevel);
1858 uint32_t level_count = radv_get_levelCount(image, range);
1859 uint32_t count = 2 * level_count;
1860
1861 assert(radv_image_has_cmask(image) ||
1862 radv_dcc_enabled(image, range->baseMipLevel));
1863
1864 radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 2 + count, cmd_buffer->state.predicating));
1865 radeon_emit(cs, S_370_DST_SEL(V_370_MEM) |
1866 S_370_WR_CONFIRM(1) |
1867 S_370_ENGINE_SEL(V_370_PFP));
1868 radeon_emit(cs, va);
1869 radeon_emit(cs, va >> 32);
1870
1871 for (uint32_t l = 0; l < level_count; l++) {
1872 radeon_emit(cs, color_values[0]);
1873 radeon_emit(cs, color_values[1]);
1874 }
1875 }
1876
1877 /**
1878 * Update the clear color values for this image.
1879 */
1880 void
1881 radv_update_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer,
1882 const struct radv_image_view *iview,
1883 int cb_idx,
1884 uint32_t color_values[2])
1885 {
1886 struct radv_image *image = iview->image;
1887 VkImageSubresourceRange range = {
1888 .aspectMask = iview->aspect_mask,
1889 .baseMipLevel = iview->base_mip,
1890 .levelCount = iview->level_count,
1891 .baseArrayLayer = iview->base_layer,
1892 .layerCount = iview->layer_count,
1893 };
1894
1895 assert(radv_image_has_cmask(image) ||
1896 radv_dcc_enabled(image, iview->base_mip));
1897
1898 radv_set_color_clear_metadata(cmd_buffer, image, &range, color_values);
1899
1900 radv_update_bound_fast_clear_color(cmd_buffer, image, cb_idx,
1901 color_values);
1902 }
1903
1904 /**
1905 * Load the clear color values from the image's metadata.
1906 */
1907 static void
1908 radv_load_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer,
1909 struct radv_image_view *iview,
1910 int cb_idx)
1911 {
1912 struct radeon_cmdbuf *cs = cmd_buffer->cs;
1913 struct radv_image *image = iview->image;
1914 uint64_t va = radv_image_get_fast_clear_va(image, iview->base_mip);
1915
1916 if (!radv_image_has_cmask(image) &&
1917 !radv_dcc_enabled(image, iview->base_mip))
1918 return;
1919
1920 uint32_t reg = R_028C8C_CB_COLOR0_CLEAR_WORD0 + cb_idx * 0x3c;
1921
1922 if (cmd_buffer->device->physical_device->rad_info.has_load_ctx_reg_pkt) {
1923 radeon_emit(cs, PKT3(PKT3_LOAD_CONTEXT_REG, 3, cmd_buffer->state.predicating));
1924 radeon_emit(cs, va);
1925 radeon_emit(cs, va >> 32);
1926 radeon_emit(cs, (reg - SI_CONTEXT_REG_OFFSET) >> 2);
1927 radeon_emit(cs, 2);
1928 } else {
1929 radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, cmd_buffer->state.predicating));
1930 radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) |
1931 COPY_DATA_DST_SEL(COPY_DATA_REG) |
1932 COPY_DATA_COUNT_SEL);
1933 radeon_emit(cs, va);
1934 radeon_emit(cs, va >> 32);
1935 radeon_emit(cs, reg >> 2);
1936 radeon_emit(cs, 0);
1937
1938 radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating));
1939 radeon_emit(cs, 0);
1940 }
1941 }
1942
1943 static void
1944 radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer)
1945 {
1946 int i;
1947 struct radv_framebuffer *framebuffer = cmd_buffer->state.framebuffer;
1948 const struct radv_subpass *subpass = cmd_buffer->state.subpass;
1949
1950 /* this may happen for inherited secondary recording */
1951 if (!framebuffer)
1952 return;
1953
1954 for (i = 0; i < 8; ++i) {
1955 if (i >= subpass->color_count || subpass->color_attachments[i].attachment == VK_ATTACHMENT_UNUSED) {
1956 radeon_set_context_reg(cmd_buffer->cs, R_028C70_CB_COLOR0_INFO + i * 0x3C,
1957 S_028C70_FORMAT(V_028C70_COLOR_INVALID));
1958 continue;
1959 }
1960
1961 int idx = subpass->color_attachments[i].attachment;
1962 struct radv_image_view *iview = cmd_buffer->state.attachments[idx].iview;
1963 VkImageLayout layout = subpass->color_attachments[i].layout;
1964 bool in_render_loop = subpass->color_attachments[i].in_render_loop;
1965
1966 radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, iview->bo);
1967
1968 assert(iview->aspect_mask & (VK_IMAGE_ASPECT_COLOR_BIT | VK_IMAGE_ASPECT_PLANE_0_BIT |
1969 VK_IMAGE_ASPECT_PLANE_1_BIT | VK_IMAGE_ASPECT_PLANE_2_BIT));
1970 radv_emit_fb_color_state(cmd_buffer, i, &cmd_buffer->state.attachments[idx].cb, iview, layout, in_render_loop);
1971
1972 radv_load_color_clear_metadata(cmd_buffer, iview, i);
1973 }
1974
1975 if (subpass->depth_stencil_attachment) {
1976 int idx = subpass->depth_stencil_attachment->attachment;
1977 VkImageLayout layout = subpass->depth_stencil_attachment->layout;
1978 bool in_render_loop = subpass->depth_stencil_attachment->in_render_loop;
1979 struct radv_image_view *iview = cmd_buffer->state.attachments[idx].iview;
1980 struct radv_image *image = iview->image;
1981 radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, cmd_buffer->state.attachments[idx].iview->bo);
1982 ASSERTED uint32_t queue_mask = radv_image_queue_family_mask(image,
1983 cmd_buffer->queue_family_index,
1984 cmd_buffer->queue_family_index);
1985 /* We currently don't support writing decompressed HTILE */
1986 assert(radv_layout_has_htile(image, layout, in_render_loop, queue_mask) ==
1987 radv_layout_is_htile_compressed(image, layout, in_render_loop, queue_mask));
1988
1989 radv_emit_fb_ds_state(cmd_buffer, &cmd_buffer->state.attachments[idx].ds, iview, layout, in_render_loop);
1990
1991 if (cmd_buffer->state.attachments[idx].ds.offset_scale != cmd_buffer->state.offset_scale) {
1992 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS;
1993 cmd_buffer->state.offset_scale = cmd_buffer->state.attachments[idx].ds.offset_scale;
1994 }
1995 radv_load_ds_clear_metadata(cmd_buffer, iview);
1996 } else {
1997 if (cmd_buffer->device->physical_device->rad_info.chip_class == GFX9)
1998 radeon_set_context_reg_seq(cmd_buffer->cs, R_028038_DB_Z_INFO, 2);
1999 else
2000 radeon_set_context_reg_seq(cmd_buffer->cs, R_028040_DB_Z_INFO, 2);
2001
2002 radeon_emit(cmd_buffer->cs, S_028040_FORMAT(V_028040_Z_INVALID)); /* DB_Z_INFO */
2003 radeon_emit(cmd_buffer->cs, S_028044_FORMAT(V_028044_STENCIL_INVALID)); /* DB_STENCIL_INFO */
2004 }
2005 radeon_set_context_reg(cmd_buffer->cs, R_028208_PA_SC_WINDOW_SCISSOR_BR,
2006 S_028208_BR_X(framebuffer->width) |
2007 S_028208_BR_Y(framebuffer->height));
2008
2009 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX8) {
2010 bool disable_constant_encode =
2011 cmd_buffer->device->physical_device->rad_info.has_dcc_constant_encode;
2012 enum chip_class chip_class =
2013 cmd_buffer->device->physical_device->rad_info.chip_class;
2014 uint8_t watermark = chip_class >= GFX10 ? 6 : 4;
2015
2016 radeon_set_context_reg(cmd_buffer->cs, R_028424_CB_DCC_CONTROL,
2017 S_028424_OVERWRITE_COMBINER_MRT_SHARING_DISABLE(chip_class <= GFX9) |
2018 S_028424_OVERWRITE_COMBINER_WATERMARK(watermark) |
2019 S_028424_DISABLE_CONSTANT_ENCODE_REG(disable_constant_encode));
2020 }
2021
2022 if (cmd_buffer->device->dfsm_allowed) {
2023 radeon_emit(cmd_buffer->cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
2024 radeon_emit(cmd_buffer->cs, EVENT_TYPE(V_028A90_BREAK_BATCH) | EVENT_INDEX(0));
2025 }
2026
2027 cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_FRAMEBUFFER;
2028 }
2029
2030 static void
2031 radv_emit_index_buffer(struct radv_cmd_buffer *cmd_buffer)
2032 {
2033 struct radeon_cmdbuf *cs = cmd_buffer->cs;
2034 struct radv_cmd_state *state = &cmd_buffer->state;
2035
2036 if (state->index_type != state->last_index_type) {
2037 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9) {
2038 radeon_set_uconfig_reg_idx(cmd_buffer->device->physical_device,
2039 cs, R_03090C_VGT_INDEX_TYPE,
2040 2, state->index_type);
2041 } else {
2042 radeon_emit(cs, PKT3(PKT3_INDEX_TYPE, 0, 0));
2043 radeon_emit(cs, state->index_type);
2044 }
2045
2046 state->last_index_type = state->index_type;
2047 }
2048
2049 radeon_emit(cs, PKT3(PKT3_INDEX_BASE, 1, 0));
2050 radeon_emit(cs, state->index_va);
2051 radeon_emit(cs, state->index_va >> 32);
2052
2053 radeon_emit(cs, PKT3(PKT3_INDEX_BUFFER_SIZE, 0, 0));
2054 radeon_emit(cs, state->max_index_count);
2055
2056 cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_INDEX_BUFFER;
2057 }
2058
2059 void radv_set_db_count_control(struct radv_cmd_buffer *cmd_buffer)
2060 {
2061 bool has_perfect_queries = cmd_buffer->state.perfect_occlusion_queries_enabled;
2062 struct radv_pipeline *pipeline = cmd_buffer->state.pipeline;
2063 uint32_t pa_sc_mode_cntl_1 =
2064 pipeline ? pipeline->graphics.ms.pa_sc_mode_cntl_1 : 0;
2065 uint32_t db_count_control;
2066
2067 if(!cmd_buffer->state.active_occlusion_queries) {
2068 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX7) {
2069 if (G_028A4C_OUT_OF_ORDER_PRIMITIVE_ENABLE(pa_sc_mode_cntl_1) &&
2070 pipeline->graphics.disable_out_of_order_rast_for_occlusion &&
2071 has_perfect_queries) {
2072 /* Re-enable out-of-order rasterization if the
2073 * bound pipeline supports it and if it's has
2074 * been disabled before starting any perfect
2075 * occlusion queries.
2076 */
2077 radeon_set_context_reg(cmd_buffer->cs,
2078 R_028A4C_PA_SC_MODE_CNTL_1,
2079 pa_sc_mode_cntl_1);
2080 }
2081 }
2082 db_count_control = S_028004_ZPASS_INCREMENT_DISABLE(1);
2083 } else {
2084 const struct radv_subpass *subpass = cmd_buffer->state.subpass;
2085 uint32_t sample_rate = subpass ? util_logbase2(subpass->max_sample_count) : 0;
2086 bool gfx10_perfect = cmd_buffer->device->physical_device->rad_info.chip_class >= GFX10 && has_perfect_queries;
2087
2088 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX7) {
2089 db_count_control =
2090 S_028004_PERFECT_ZPASS_COUNTS(has_perfect_queries) |
2091 S_028004_DISABLE_CONSERVATIVE_ZPASS_COUNTS(gfx10_perfect) |
2092 S_028004_SAMPLE_RATE(sample_rate) |
2093 S_028004_ZPASS_ENABLE(1) |
2094 S_028004_SLICE_EVEN_ENABLE(1) |
2095 S_028004_SLICE_ODD_ENABLE(1);
2096
2097 if (G_028A4C_OUT_OF_ORDER_PRIMITIVE_ENABLE(pa_sc_mode_cntl_1) &&
2098 pipeline->graphics.disable_out_of_order_rast_for_occlusion &&
2099 has_perfect_queries) {
2100 /* If the bound pipeline has enabled
2101 * out-of-order rasterization, we should
2102 * disable it before starting any perfect
2103 * occlusion queries.
2104 */
2105 pa_sc_mode_cntl_1 &= C_028A4C_OUT_OF_ORDER_PRIMITIVE_ENABLE;
2106
2107 radeon_set_context_reg(cmd_buffer->cs,
2108 R_028A4C_PA_SC_MODE_CNTL_1,
2109 pa_sc_mode_cntl_1);
2110 }
2111 } else {
2112 db_count_control = S_028004_PERFECT_ZPASS_COUNTS(1) |
2113 S_028004_SAMPLE_RATE(sample_rate);
2114 }
2115 }
2116
2117 radeon_set_context_reg(cmd_buffer->cs, R_028004_DB_COUNT_CONTROL, db_count_control);
2118
2119 cmd_buffer->state.context_roll_without_scissor_emitted = true;
2120 }
2121
2122 static void
2123 radv_cmd_buffer_flush_dynamic_state(struct radv_cmd_buffer *cmd_buffer)
2124 {
2125 uint32_t states = cmd_buffer->state.dirty & cmd_buffer->state.emitted_pipeline->graphics.needed_dynamic_state;
2126
2127 if (states & (RADV_CMD_DIRTY_DYNAMIC_VIEWPORT))
2128 radv_emit_viewport(cmd_buffer);
2129
2130 if (states & (RADV_CMD_DIRTY_DYNAMIC_SCISSOR | RADV_CMD_DIRTY_DYNAMIC_VIEWPORT) &&
2131 !cmd_buffer->device->physical_device->rad_info.has_gfx9_scissor_bug)
2132 radv_emit_scissor(cmd_buffer);
2133
2134 if (states & RADV_CMD_DIRTY_DYNAMIC_LINE_WIDTH)
2135 radv_emit_line_width(cmd_buffer);
2136
2137 if (states & RADV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS)
2138 radv_emit_blend_constants(cmd_buffer);
2139
2140 if (states & (RADV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE |
2141 RADV_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK |
2142 RADV_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK))
2143 radv_emit_stencil(cmd_buffer);
2144
2145 if (states & RADV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS)
2146 radv_emit_depth_bounds(cmd_buffer);
2147
2148 if (states & RADV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS)
2149 radv_emit_depth_bias(cmd_buffer);
2150
2151 if (states & RADV_CMD_DIRTY_DYNAMIC_DISCARD_RECTANGLE)
2152 radv_emit_discard_rectangle(cmd_buffer);
2153
2154 if (states & RADV_CMD_DIRTY_DYNAMIC_SAMPLE_LOCATIONS)
2155 radv_emit_sample_locations(cmd_buffer);
2156
2157 cmd_buffer->state.dirty &= ~states;
2158 }
2159
2160 static void
2161 radv_flush_push_descriptors(struct radv_cmd_buffer *cmd_buffer,
2162 VkPipelineBindPoint bind_point)
2163 {
2164 struct radv_descriptor_state *descriptors_state =
2165 radv_get_descriptors_state(cmd_buffer, bind_point);
2166 struct radv_descriptor_set *set = &descriptors_state->push_set.set;
2167 unsigned bo_offset;
2168
2169 if (!radv_cmd_buffer_upload_data(cmd_buffer, set->size, 32,
2170 set->mapped_ptr,
2171 &bo_offset))
2172 return;
2173
2174 set->va = radv_buffer_get_va(cmd_buffer->upload.upload_bo);
2175 set->va += bo_offset;
2176 }
2177
2178 static void
2179 radv_flush_indirect_descriptor_sets(struct radv_cmd_buffer *cmd_buffer,
2180 VkPipelineBindPoint bind_point)
2181 {
2182 struct radv_descriptor_state *descriptors_state =
2183 radv_get_descriptors_state(cmd_buffer, bind_point);
2184 uint32_t size = MAX_SETS * 4;
2185 uint32_t offset;
2186 void *ptr;
2187
2188 if (!radv_cmd_buffer_upload_alloc(cmd_buffer, size,
2189 256, &offset, &ptr))
2190 return;
2191
2192 for (unsigned i = 0; i < MAX_SETS; i++) {
2193 uint32_t *uptr = ((uint32_t *)ptr) + i;
2194 uint64_t set_va = 0;
2195 struct radv_descriptor_set *set = descriptors_state->sets[i];
2196 if (descriptors_state->valid & (1u << i))
2197 set_va = set->va;
2198 uptr[0] = set_va & 0xffffffff;
2199 }
2200
2201 uint64_t va = radv_buffer_get_va(cmd_buffer->upload.upload_bo);
2202 va += offset;
2203
2204 if (cmd_buffer->state.pipeline) {
2205 if (cmd_buffer->state.pipeline->shaders[MESA_SHADER_VERTEX])
2206 radv_emit_userdata_address(cmd_buffer, cmd_buffer->state.pipeline, MESA_SHADER_VERTEX,
2207 AC_UD_INDIRECT_DESCRIPTOR_SETS, va);
2208
2209 if (cmd_buffer->state.pipeline->shaders[MESA_SHADER_FRAGMENT])
2210 radv_emit_userdata_address(cmd_buffer, cmd_buffer->state.pipeline, MESA_SHADER_FRAGMENT,
2211 AC_UD_INDIRECT_DESCRIPTOR_SETS, va);
2212
2213 if (radv_pipeline_has_gs(cmd_buffer->state.pipeline))
2214 radv_emit_userdata_address(cmd_buffer, cmd_buffer->state.pipeline, MESA_SHADER_GEOMETRY,
2215 AC_UD_INDIRECT_DESCRIPTOR_SETS, va);
2216
2217 if (radv_pipeline_has_tess(cmd_buffer->state.pipeline))
2218 radv_emit_userdata_address(cmd_buffer, cmd_buffer->state.pipeline, MESA_SHADER_TESS_CTRL,
2219 AC_UD_INDIRECT_DESCRIPTOR_SETS, va);
2220
2221 if (radv_pipeline_has_tess(cmd_buffer->state.pipeline))
2222 radv_emit_userdata_address(cmd_buffer, cmd_buffer->state.pipeline, MESA_SHADER_TESS_EVAL,
2223 AC_UD_INDIRECT_DESCRIPTOR_SETS, va);
2224 }
2225
2226 if (cmd_buffer->state.compute_pipeline)
2227 radv_emit_userdata_address(cmd_buffer, cmd_buffer->state.compute_pipeline, MESA_SHADER_COMPUTE,
2228 AC_UD_INDIRECT_DESCRIPTOR_SETS, va);
2229 }
2230
2231 static void
2232 radv_flush_descriptors(struct radv_cmd_buffer *cmd_buffer,
2233 VkShaderStageFlags stages)
2234 {
2235 VkPipelineBindPoint bind_point = stages & VK_SHADER_STAGE_COMPUTE_BIT ?
2236 VK_PIPELINE_BIND_POINT_COMPUTE :
2237 VK_PIPELINE_BIND_POINT_GRAPHICS;
2238 struct radv_descriptor_state *descriptors_state =
2239 radv_get_descriptors_state(cmd_buffer, bind_point);
2240 struct radv_cmd_state *state = &cmd_buffer->state;
2241 bool flush_indirect_descriptors;
2242
2243 if (!descriptors_state->dirty)
2244 return;
2245
2246 if (descriptors_state->push_dirty)
2247 radv_flush_push_descriptors(cmd_buffer, bind_point);
2248
2249 flush_indirect_descriptors =
2250 (bind_point == VK_PIPELINE_BIND_POINT_GRAPHICS &&
2251 state->pipeline && state->pipeline->need_indirect_descriptor_sets) ||
2252 (bind_point == VK_PIPELINE_BIND_POINT_COMPUTE &&
2253 state->compute_pipeline && state->compute_pipeline->need_indirect_descriptor_sets);
2254
2255 if (flush_indirect_descriptors)
2256 radv_flush_indirect_descriptor_sets(cmd_buffer, bind_point);
2257
2258 ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws,
2259 cmd_buffer->cs,
2260 MAX_SETS * MESA_SHADER_STAGES * 4);
2261
2262 if (cmd_buffer->state.pipeline) {
2263 radv_foreach_stage(stage, stages) {
2264 if (!cmd_buffer->state.pipeline->shaders[stage])
2265 continue;
2266
2267 radv_emit_descriptor_pointers(cmd_buffer,
2268 cmd_buffer->state.pipeline,
2269 descriptors_state, stage);
2270 }
2271 }
2272
2273 if (cmd_buffer->state.compute_pipeline &&
2274 (stages & VK_SHADER_STAGE_COMPUTE_BIT)) {
2275 radv_emit_descriptor_pointers(cmd_buffer,
2276 cmd_buffer->state.compute_pipeline,
2277 descriptors_state,
2278 MESA_SHADER_COMPUTE);
2279 }
2280
2281 descriptors_state->dirty = 0;
2282 descriptors_state->push_dirty = false;
2283
2284 assert(cmd_buffer->cs->cdw <= cdw_max);
2285
2286 if (unlikely(cmd_buffer->device->trace_bo))
2287 radv_save_descriptors(cmd_buffer, bind_point);
2288 }
2289
2290 static void
2291 radv_flush_constants(struct radv_cmd_buffer *cmd_buffer,
2292 VkShaderStageFlags stages)
2293 {
2294 struct radv_pipeline *pipeline = stages & VK_SHADER_STAGE_COMPUTE_BIT
2295 ? cmd_buffer->state.compute_pipeline
2296 : cmd_buffer->state.pipeline;
2297 VkPipelineBindPoint bind_point = stages & VK_SHADER_STAGE_COMPUTE_BIT ?
2298 VK_PIPELINE_BIND_POINT_COMPUTE :
2299 VK_PIPELINE_BIND_POINT_GRAPHICS;
2300 struct radv_descriptor_state *descriptors_state =
2301 radv_get_descriptors_state(cmd_buffer, bind_point);
2302 struct radv_pipeline_layout *layout = pipeline->layout;
2303 struct radv_shader_variant *shader, *prev_shader;
2304 bool need_push_constants = false;
2305 unsigned offset;
2306 void *ptr;
2307 uint64_t va;
2308
2309 stages &= cmd_buffer->push_constant_stages;
2310 if (!stages ||
2311 (!layout->push_constant_size && !layout->dynamic_offset_count))
2312 return;
2313
2314 radv_foreach_stage(stage, stages) {
2315 if (!pipeline->shaders[stage])
2316 continue;
2317
2318 need_push_constants |= pipeline->shaders[stage]->info.loads_push_constants;
2319 need_push_constants |= pipeline->shaders[stage]->info.loads_dynamic_offsets;
2320
2321 uint8_t base = pipeline->shaders[stage]->info.base_inline_push_consts;
2322 uint8_t count = pipeline->shaders[stage]->info.num_inline_push_consts;
2323
2324 radv_emit_inline_push_consts(cmd_buffer, pipeline, stage,
2325 AC_UD_INLINE_PUSH_CONSTANTS,
2326 count,
2327 (uint32_t *)&cmd_buffer->push_constants[base * 4]);
2328 }
2329
2330 if (need_push_constants) {
2331 if (!radv_cmd_buffer_upload_alloc(cmd_buffer, layout->push_constant_size +
2332 16 * layout->dynamic_offset_count,
2333 256, &offset, &ptr))
2334 return;
2335
2336 memcpy(ptr, cmd_buffer->push_constants, layout->push_constant_size);
2337 memcpy((char*)ptr + layout->push_constant_size,
2338 descriptors_state->dynamic_buffers,
2339 16 * layout->dynamic_offset_count);
2340
2341 va = radv_buffer_get_va(cmd_buffer->upload.upload_bo);
2342 va += offset;
2343
2344 ASSERTED unsigned cdw_max =
2345 radeon_check_space(cmd_buffer->device->ws,
2346 cmd_buffer->cs, MESA_SHADER_STAGES * 4);
2347
2348 prev_shader = NULL;
2349 radv_foreach_stage(stage, stages) {
2350 shader = radv_get_shader(pipeline, stage);
2351
2352 /* Avoid redundantly emitting the address for merged stages. */
2353 if (shader && shader != prev_shader) {
2354 radv_emit_userdata_address(cmd_buffer, pipeline, stage,
2355 AC_UD_PUSH_CONSTANTS, va);
2356
2357 prev_shader = shader;
2358 }
2359 }
2360 assert(cmd_buffer->cs->cdw <= cdw_max);
2361 }
2362
2363 cmd_buffer->push_constant_stages &= ~stages;
2364 }
2365
2366 static void
2367 radv_flush_vertex_descriptors(struct radv_cmd_buffer *cmd_buffer,
2368 bool pipeline_is_dirty)
2369 {
2370 if ((pipeline_is_dirty ||
2371 (cmd_buffer->state.dirty & RADV_CMD_DIRTY_VERTEX_BUFFER)) &&
2372 cmd_buffer->state.pipeline->num_vertex_bindings &&
2373 radv_get_shader(cmd_buffer->state.pipeline, MESA_SHADER_VERTEX)->info.vs.has_vertex_buffers) {
2374 struct radv_vertex_elements_info *velems = &cmd_buffer->state.pipeline->vertex_elements;
2375 unsigned vb_offset;
2376 void *vb_ptr;
2377 uint32_t i = 0;
2378 uint32_t count = cmd_buffer->state.pipeline->num_vertex_bindings;
2379 uint64_t va;
2380
2381 /* allocate some descriptor state for vertex buffers */
2382 if (!radv_cmd_buffer_upload_alloc(cmd_buffer, count * 16, 256,
2383 &vb_offset, &vb_ptr))
2384 return;
2385
2386 for (i = 0; i < count; i++) {
2387 uint32_t *desc = &((uint32_t *)vb_ptr)[i * 4];
2388 uint32_t offset;
2389 struct radv_buffer *buffer = cmd_buffer->vertex_bindings[i].buffer;
2390 uint32_t stride = cmd_buffer->state.pipeline->binding_stride[i];
2391
2392 if (!buffer)
2393 continue;
2394
2395 va = radv_buffer_get_va(buffer->bo);
2396
2397 offset = cmd_buffer->vertex_bindings[i].offset;
2398 va += offset + buffer->offset;
2399 desc[0] = va;
2400 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) | S_008F04_STRIDE(stride);
2401 if (cmd_buffer->device->physical_device->rad_info.chip_class <= GFX7 && stride)
2402 desc[2] = (buffer->size - offset - velems->format_size[i]) / stride + 1;
2403 else
2404 desc[2] = buffer->size - offset;
2405 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
2406 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
2407 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
2408 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
2409
2410 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX10) {
2411 desc[3] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_UINT) |
2412 S_008F0C_OOB_SELECT(1) |
2413 S_008F0C_RESOURCE_LEVEL(1);
2414 } else {
2415 desc[3] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_UINT) |
2416 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
2417 }
2418 }
2419
2420 va = radv_buffer_get_va(cmd_buffer->upload.upload_bo);
2421 va += vb_offset;
2422
2423 radv_emit_userdata_address(cmd_buffer, cmd_buffer->state.pipeline, MESA_SHADER_VERTEX,
2424 AC_UD_VS_VERTEX_BUFFERS, va);
2425
2426 cmd_buffer->state.vb_va = va;
2427 cmd_buffer->state.vb_size = count * 16;
2428 cmd_buffer->state.prefetch_L2_mask |= RADV_PREFETCH_VBO_DESCRIPTORS;
2429 }
2430 cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_VERTEX_BUFFER;
2431 }
2432
2433 static void
2434 radv_emit_streamout_buffers(struct radv_cmd_buffer *cmd_buffer, uint64_t va)
2435 {
2436 struct radv_pipeline *pipeline = cmd_buffer->state.pipeline;
2437 struct radv_userdata_info *loc;
2438 uint32_t base_reg;
2439
2440 for (unsigned stage = 0; stage < MESA_SHADER_STAGES; ++stage) {
2441 if (!radv_get_shader(pipeline, stage))
2442 continue;
2443
2444 loc = radv_lookup_user_sgpr(pipeline, stage,
2445 AC_UD_STREAMOUT_BUFFERS);
2446 if (loc->sgpr_idx == -1)
2447 continue;
2448
2449 base_reg = pipeline->user_data_0[stage];
2450
2451 radv_emit_shader_pointer(cmd_buffer->device, cmd_buffer->cs,
2452 base_reg + loc->sgpr_idx * 4, va, false);
2453 }
2454
2455 if (radv_pipeline_has_gs_copy_shader(pipeline)) {
2456 loc = &pipeline->gs_copy_shader->info.user_sgprs_locs.shader_data[AC_UD_STREAMOUT_BUFFERS];
2457 if (loc->sgpr_idx != -1) {
2458 base_reg = R_00B130_SPI_SHADER_USER_DATA_VS_0;
2459
2460 radv_emit_shader_pointer(cmd_buffer->device, cmd_buffer->cs,
2461 base_reg + loc->sgpr_idx * 4, va, false);
2462 }
2463 }
2464 }
2465
2466 static void
2467 radv_flush_streamout_descriptors(struct radv_cmd_buffer *cmd_buffer)
2468 {
2469 if (cmd_buffer->state.dirty & RADV_CMD_DIRTY_STREAMOUT_BUFFER) {
2470 struct radv_streamout_binding *sb = cmd_buffer->streamout_bindings;
2471 struct radv_streamout_state *so = &cmd_buffer->state.streamout;
2472 unsigned so_offset;
2473 void *so_ptr;
2474 uint64_t va;
2475
2476 /* Allocate some descriptor state for streamout buffers. */
2477 if (!radv_cmd_buffer_upload_alloc(cmd_buffer,
2478 MAX_SO_BUFFERS * 16, 256,
2479 &so_offset, &so_ptr))
2480 return;
2481
2482 for (uint32_t i = 0; i < MAX_SO_BUFFERS; i++) {
2483 struct radv_buffer *buffer = sb[i].buffer;
2484 uint32_t *desc = &((uint32_t *)so_ptr)[i * 4];
2485
2486 if (!(so->enabled_mask & (1 << i)))
2487 continue;
2488
2489 va = radv_buffer_get_va(buffer->bo) + buffer->offset;
2490
2491 va += sb[i].offset;
2492
2493 /* Set the descriptor.
2494 *
2495 * On GFX8, the format must be non-INVALID, otherwise
2496 * the buffer will be considered not bound and store
2497 * instructions will be no-ops.
2498 */
2499 uint32_t size = 0xffffffff;
2500
2501 /* Compute the correct buffer size for NGG streamout
2502 * because it's used to determine the max emit per
2503 * buffer.
2504 */
2505 if (cmd_buffer->device->physical_device->use_ngg_streamout)
2506 size = buffer->size - sb[i].offset;
2507
2508 desc[0] = va;
2509 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32);
2510 desc[2] = size;
2511 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
2512 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
2513 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
2514 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
2515
2516 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX10) {
2517 desc[3] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
2518 S_008F0C_OOB_SELECT(3) |
2519 S_008F0C_RESOURCE_LEVEL(1);
2520 } else {
2521 desc[3] |= S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
2522 }
2523 }
2524
2525 va = radv_buffer_get_va(cmd_buffer->upload.upload_bo);
2526 va += so_offset;
2527
2528 radv_emit_streamout_buffers(cmd_buffer, va);
2529 }
2530
2531 cmd_buffer->state.dirty &= ~RADV_CMD_DIRTY_STREAMOUT_BUFFER;
2532 }
2533
2534 static void
2535 radv_upload_graphics_shader_descriptors(struct radv_cmd_buffer *cmd_buffer, bool pipeline_is_dirty)
2536 {
2537 radv_flush_vertex_descriptors(cmd_buffer, pipeline_is_dirty);
2538 radv_flush_streamout_descriptors(cmd_buffer);
2539 radv_flush_descriptors(cmd_buffer, VK_SHADER_STAGE_ALL_GRAPHICS);
2540 radv_flush_constants(cmd_buffer, VK_SHADER_STAGE_ALL_GRAPHICS);
2541 }
2542
2543 struct radv_draw_info {
2544 /**
2545 * Number of vertices.
2546 */
2547 uint32_t count;
2548
2549 /**
2550 * Index of the first vertex.
2551 */
2552 int32_t vertex_offset;
2553
2554 /**
2555 * First instance id.
2556 */
2557 uint32_t first_instance;
2558
2559 /**
2560 * Number of instances.
2561 */
2562 uint32_t instance_count;
2563
2564 /**
2565 * First index (indexed draws only).
2566 */
2567 uint32_t first_index;
2568
2569 /**
2570 * Whether it's an indexed draw.
2571 */
2572 bool indexed;
2573
2574 /**
2575 * Indirect draw parameters resource.
2576 */
2577 struct radv_buffer *indirect;
2578 uint64_t indirect_offset;
2579 uint32_t stride;
2580
2581 /**
2582 * Draw count parameters resource.
2583 */
2584 struct radv_buffer *count_buffer;
2585 uint64_t count_buffer_offset;
2586
2587 /**
2588 * Stream output parameters resource.
2589 */
2590 struct radv_buffer *strmout_buffer;
2591 uint64_t strmout_buffer_offset;
2592 };
2593
2594 static uint32_t
2595 radv_get_primitive_reset_index(struct radv_cmd_buffer *cmd_buffer)
2596 {
2597 switch (cmd_buffer->state.index_type) {
2598 case V_028A7C_VGT_INDEX_8:
2599 return 0xffu;
2600 case V_028A7C_VGT_INDEX_16:
2601 return 0xffffu;
2602 case V_028A7C_VGT_INDEX_32:
2603 return 0xffffffffu;
2604 default:
2605 unreachable("invalid index type");
2606 }
2607 }
2608
2609 static void
2610 si_emit_ia_multi_vgt_param(struct radv_cmd_buffer *cmd_buffer,
2611 bool instanced_draw, bool indirect_draw,
2612 bool count_from_stream_output,
2613 uint32_t draw_vertex_count)
2614 {
2615 struct radeon_info *info = &cmd_buffer->device->physical_device->rad_info;
2616 struct radv_cmd_state *state = &cmd_buffer->state;
2617 struct radeon_cmdbuf *cs = cmd_buffer->cs;
2618 unsigned ia_multi_vgt_param;
2619
2620 ia_multi_vgt_param =
2621 si_get_ia_multi_vgt_param(cmd_buffer, instanced_draw,
2622 indirect_draw,
2623 count_from_stream_output,
2624 draw_vertex_count);
2625
2626 if (state->last_ia_multi_vgt_param != ia_multi_vgt_param) {
2627 if (info->chip_class == GFX9) {
2628 radeon_set_uconfig_reg_idx(cmd_buffer->device->physical_device,
2629 cs,
2630 R_030960_IA_MULTI_VGT_PARAM,
2631 4, ia_multi_vgt_param);
2632 } else if (info->chip_class >= GFX7) {
2633 radeon_set_context_reg_idx(cs,
2634 R_028AA8_IA_MULTI_VGT_PARAM,
2635 1, ia_multi_vgt_param);
2636 } else {
2637 radeon_set_context_reg(cs, R_028AA8_IA_MULTI_VGT_PARAM,
2638 ia_multi_vgt_param);
2639 }
2640 state->last_ia_multi_vgt_param = ia_multi_vgt_param;
2641 }
2642 }
2643
2644 static void
2645 radv_emit_draw_registers(struct radv_cmd_buffer *cmd_buffer,
2646 const struct radv_draw_info *draw_info)
2647 {
2648 struct radeon_info *info = &cmd_buffer->device->physical_device->rad_info;
2649 struct radv_cmd_state *state = &cmd_buffer->state;
2650 struct radeon_cmdbuf *cs = cmd_buffer->cs;
2651 int32_t primitive_reset_en;
2652
2653 /* Draw state. */
2654 if (info->chip_class < GFX10) {
2655 si_emit_ia_multi_vgt_param(cmd_buffer, draw_info->instance_count > 1,
2656 draw_info->indirect,
2657 !!draw_info->strmout_buffer,
2658 draw_info->indirect ? 0 : draw_info->count);
2659 }
2660
2661 /* Primitive restart. */
2662 primitive_reset_en =
2663 draw_info->indexed && state->pipeline->graphics.prim_restart_enable;
2664
2665 if (primitive_reset_en != state->last_primitive_reset_en) {
2666 state->last_primitive_reset_en = primitive_reset_en;
2667 if (info->chip_class >= GFX9) {
2668 radeon_set_uconfig_reg(cs,
2669 R_03092C_VGT_MULTI_PRIM_IB_RESET_EN,
2670 primitive_reset_en);
2671 } else {
2672 radeon_set_context_reg(cs,
2673 R_028A94_VGT_MULTI_PRIM_IB_RESET_EN,
2674 primitive_reset_en);
2675 }
2676 }
2677
2678 if (primitive_reset_en) {
2679 uint32_t primitive_reset_index =
2680 radv_get_primitive_reset_index(cmd_buffer);
2681
2682 if (primitive_reset_index != state->last_primitive_reset_index) {
2683 radeon_set_context_reg(cs,
2684 R_02840C_VGT_MULTI_PRIM_IB_RESET_INDX,
2685 primitive_reset_index);
2686 state->last_primitive_reset_index = primitive_reset_index;
2687 }
2688 }
2689
2690 if (draw_info->strmout_buffer) {
2691 uint64_t va = radv_buffer_get_va(draw_info->strmout_buffer->bo);
2692
2693 va += draw_info->strmout_buffer->offset +
2694 draw_info->strmout_buffer_offset;
2695
2696 radeon_set_context_reg(cs, R_028B30_VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE,
2697 draw_info->stride);
2698
2699 radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
2700 radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) |
2701 COPY_DATA_DST_SEL(COPY_DATA_REG) |
2702 COPY_DATA_WR_CONFIRM);
2703 radeon_emit(cs, va);
2704 radeon_emit(cs, va >> 32);
2705 radeon_emit(cs, R_028B2C_VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE >> 2);
2706 radeon_emit(cs, 0); /* unused */
2707
2708 radv_cs_add_buffer(cmd_buffer->device->ws, cs, draw_info->strmout_buffer->bo);
2709 }
2710 }
2711
2712 static void radv_stage_flush(struct radv_cmd_buffer *cmd_buffer,
2713 VkPipelineStageFlags src_stage_mask)
2714 {
2715 if (src_stage_mask & (VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT |
2716 VK_PIPELINE_STAGE_TRANSFER_BIT |
2717 VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT |
2718 VK_PIPELINE_STAGE_ALL_COMMANDS_BIT)) {
2719 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH;
2720 }
2721
2722 if (src_stage_mask & (VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT |
2723 VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT |
2724 VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT |
2725 VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT |
2726 VK_PIPELINE_STAGE_TRANSFER_BIT |
2727 VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT |
2728 VK_PIPELINE_STAGE_ALL_GRAPHICS_BIT |
2729 VK_PIPELINE_STAGE_ALL_COMMANDS_BIT)) {
2730 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_PS_PARTIAL_FLUSH;
2731 } else if (src_stage_mask & (VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT |
2732 VK_PIPELINE_STAGE_VERTEX_INPUT_BIT |
2733 VK_PIPELINE_STAGE_VERTEX_SHADER_BIT |
2734 VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT |
2735 VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT |
2736 VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT |
2737 VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT)) {
2738 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_VS_PARTIAL_FLUSH;
2739 }
2740 }
2741
2742 static enum radv_cmd_flush_bits
2743 radv_src_access_flush(struct radv_cmd_buffer *cmd_buffer,
2744 VkAccessFlags src_flags,
2745 struct radv_image *image)
2746 {
2747 bool flush_CB_meta = true, flush_DB_meta = true;
2748 enum radv_cmd_flush_bits flush_bits = 0;
2749 uint32_t b;
2750
2751 if (image) {
2752 if (!radv_image_has_CB_metadata(image))
2753 flush_CB_meta = false;
2754 if (!radv_image_has_htile(image))
2755 flush_DB_meta = false;
2756 }
2757
2758 for_each_bit(b, src_flags) {
2759 switch ((VkAccessFlagBits)(1 << b)) {
2760 case VK_ACCESS_SHADER_WRITE_BIT:
2761 case VK_ACCESS_TRANSFORM_FEEDBACK_WRITE_BIT_EXT:
2762 case VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_WRITE_BIT_EXT:
2763 flush_bits |= RADV_CMD_FLAG_WB_L2;
2764 break;
2765 case VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT:
2766 flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB;
2767 if (flush_CB_meta)
2768 flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB_META;
2769 break;
2770 case VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT:
2771 flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB;
2772 if (flush_DB_meta)
2773 flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB_META;
2774 break;
2775 case VK_ACCESS_TRANSFER_WRITE_BIT:
2776 flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB |
2777 RADV_CMD_FLAG_FLUSH_AND_INV_DB |
2778 RADV_CMD_FLAG_INV_L2;
2779
2780 if (flush_CB_meta)
2781 flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB_META;
2782 if (flush_DB_meta)
2783 flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB_META;
2784 break;
2785 default:
2786 break;
2787 }
2788 }
2789 return flush_bits;
2790 }
2791
2792 static enum radv_cmd_flush_bits
2793 radv_dst_access_flush(struct radv_cmd_buffer *cmd_buffer,
2794 VkAccessFlags dst_flags,
2795 struct radv_image *image)
2796 {
2797 bool flush_CB_meta = true, flush_DB_meta = true;
2798 enum radv_cmd_flush_bits flush_bits = 0;
2799 bool flush_CB = true, flush_DB = true;
2800 bool image_is_coherent = false;
2801 uint32_t b;
2802
2803 if (image) {
2804 if (!(image->usage & VK_IMAGE_USAGE_STORAGE_BIT)) {
2805 flush_CB = false;
2806 flush_DB = false;
2807 }
2808
2809 if (!radv_image_has_CB_metadata(image))
2810 flush_CB_meta = false;
2811 if (!radv_image_has_htile(image))
2812 flush_DB_meta = false;
2813
2814 /* TODO: implement shader coherent for GFX10 */
2815
2816 if (cmd_buffer->device->physical_device->rad_info.chip_class == GFX9) {
2817 if (image->info.samples == 1 &&
2818 (image->usage & (VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT |
2819 VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT)) &&
2820 !vk_format_is_stencil(image->vk_format)) {
2821 /* Single-sample color and single-sample depth
2822 * (not stencil) are coherent with shaders on
2823 * GFX9.
2824 */
2825 image_is_coherent = true;
2826 }
2827 }
2828 }
2829
2830 for_each_bit(b, dst_flags) {
2831 switch ((VkAccessFlagBits)(1 << b)) {
2832 case VK_ACCESS_INDIRECT_COMMAND_READ_BIT:
2833 case VK_ACCESS_INDEX_READ_BIT:
2834 case VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_WRITE_BIT_EXT:
2835 break;
2836 case VK_ACCESS_UNIFORM_READ_BIT:
2837 flush_bits |= RADV_CMD_FLAG_INV_VCACHE | RADV_CMD_FLAG_INV_SCACHE;
2838 break;
2839 case VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT:
2840 case VK_ACCESS_TRANSFER_READ_BIT:
2841 case VK_ACCESS_INPUT_ATTACHMENT_READ_BIT:
2842 flush_bits |= RADV_CMD_FLAG_INV_VCACHE |
2843 RADV_CMD_FLAG_INV_L2;
2844 break;
2845 case VK_ACCESS_SHADER_READ_BIT:
2846 flush_bits |= RADV_CMD_FLAG_INV_VCACHE;
2847
2848 if (!image_is_coherent)
2849 flush_bits |= RADV_CMD_FLAG_INV_L2;
2850 break;
2851 case VK_ACCESS_COLOR_ATTACHMENT_READ_BIT:
2852 if (flush_CB)
2853 flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB;
2854 if (flush_CB_meta)
2855 flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB_META;
2856 break;
2857 case VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT:
2858 if (flush_DB)
2859 flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB;
2860 if (flush_DB_meta)
2861 flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB_META;
2862 break;
2863 default:
2864 break;
2865 }
2866 }
2867 return flush_bits;
2868 }
2869
2870 void radv_subpass_barrier(struct radv_cmd_buffer *cmd_buffer,
2871 const struct radv_subpass_barrier *barrier)
2872 {
2873 cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, barrier->src_access_mask,
2874 NULL);
2875 radv_stage_flush(cmd_buffer, barrier->src_stage_mask);
2876 cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, barrier->dst_access_mask,
2877 NULL);
2878 }
2879
2880 uint32_t
2881 radv_get_subpass_id(struct radv_cmd_buffer *cmd_buffer)
2882 {
2883 struct radv_cmd_state *state = &cmd_buffer->state;
2884 uint32_t subpass_id = state->subpass - state->pass->subpasses;
2885
2886 /* The id of this subpass shouldn't exceed the number of subpasses in
2887 * this render pass minus 1.
2888 */
2889 assert(subpass_id < state->pass->subpass_count);
2890 return subpass_id;
2891 }
2892
2893 static struct radv_sample_locations_state *
2894 radv_get_attachment_sample_locations(struct radv_cmd_buffer *cmd_buffer,
2895 uint32_t att_idx,
2896 bool begin_subpass)
2897 {
2898 struct radv_cmd_state *state = &cmd_buffer->state;
2899 uint32_t subpass_id = radv_get_subpass_id(cmd_buffer);
2900 struct radv_image_view *view = state->attachments[att_idx].iview;
2901
2902 if (view->image->info.samples == 1)
2903 return NULL;
2904
2905 if (state->pass->attachments[att_idx].first_subpass_idx == subpass_id) {
2906 /* Return the initial sample locations if this is the initial
2907 * layout transition of the given subpass attachemnt.
2908 */
2909 if (state->attachments[att_idx].sample_location.count > 0)
2910 return &state->attachments[att_idx].sample_location;
2911 } else {
2912 /* Otherwise return the subpass sample locations if defined. */
2913 if (state->subpass_sample_locs) {
2914 /* Because the driver sets the current subpass before
2915 * initial layout transitions, we should use the sample
2916 * locations from the previous subpass to avoid an
2917 * off-by-one problem. Otherwise, use the sample
2918 * locations for the current subpass for final layout
2919 * transitions.
2920 */
2921 if (begin_subpass)
2922 subpass_id--;
2923
2924 for (uint32_t i = 0; i < state->num_subpass_sample_locs; i++) {
2925 if (state->subpass_sample_locs[i].subpass_idx == subpass_id)
2926 return &state->subpass_sample_locs[i].sample_location;
2927 }
2928 }
2929 }
2930
2931 return NULL;
2932 }
2933
2934 static void radv_handle_subpass_image_transition(struct radv_cmd_buffer *cmd_buffer,
2935 struct radv_subpass_attachment att,
2936 bool begin_subpass)
2937 {
2938 unsigned idx = att.attachment;
2939 struct radv_image_view *view = cmd_buffer->state.attachments[idx].iview;
2940 struct radv_sample_locations_state *sample_locs;
2941 VkImageSubresourceRange range;
2942 range.aspectMask = 0;
2943 range.baseMipLevel = view->base_mip;
2944 range.levelCount = 1;
2945 range.baseArrayLayer = view->base_layer;
2946 range.layerCount = cmd_buffer->state.framebuffer->layers;
2947
2948 if (cmd_buffer->state.subpass->view_mask) {
2949 /* If the current subpass uses multiview, the driver might have
2950 * performed a fast color/depth clear to the whole image
2951 * (including all layers). To make sure the driver will
2952 * decompress the image correctly (if needed), we have to
2953 * account for the "real" number of layers. If the view mask is
2954 * sparse, this will decompress more layers than needed.
2955 */
2956 range.layerCount = util_last_bit(cmd_buffer->state.subpass->view_mask);
2957 }
2958
2959 /* Get the subpass sample locations for the given attachment, if NULL
2960 * is returned the driver will use the default HW locations.
2961 */
2962 sample_locs = radv_get_attachment_sample_locations(cmd_buffer, idx,
2963 begin_subpass);
2964
2965 radv_handle_image_transition(cmd_buffer,
2966 view->image,
2967 cmd_buffer->state.attachments[idx].current_layout,
2968 cmd_buffer->state.attachments[idx].current_in_render_loop,
2969 att.layout, att.in_render_loop,
2970 0, 0, &range, sample_locs);
2971
2972 cmd_buffer->state.attachments[idx].current_layout = att.layout;
2973 cmd_buffer->state.attachments[idx].current_in_render_loop = att.in_render_loop;
2974
2975
2976 }
2977
2978 void
2979 radv_cmd_buffer_set_subpass(struct radv_cmd_buffer *cmd_buffer,
2980 const struct radv_subpass *subpass)
2981 {
2982 cmd_buffer->state.subpass = subpass;
2983
2984 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_FRAMEBUFFER;
2985 }
2986
2987 static VkResult
2988 radv_cmd_state_setup_sample_locations(struct radv_cmd_buffer *cmd_buffer,
2989 struct radv_render_pass *pass,
2990 const VkRenderPassBeginInfo *info)
2991 {
2992 const struct VkRenderPassSampleLocationsBeginInfoEXT *sample_locs =
2993 vk_find_struct_const(info->pNext,
2994 RENDER_PASS_SAMPLE_LOCATIONS_BEGIN_INFO_EXT);
2995 struct radv_cmd_state *state = &cmd_buffer->state;
2996
2997 if (!sample_locs) {
2998 state->subpass_sample_locs = NULL;
2999 return VK_SUCCESS;
3000 }
3001
3002 for (uint32_t i = 0; i < sample_locs->attachmentInitialSampleLocationsCount; i++) {
3003 const VkAttachmentSampleLocationsEXT *att_sample_locs =
3004 &sample_locs->pAttachmentInitialSampleLocations[i];
3005 uint32_t att_idx = att_sample_locs->attachmentIndex;
3006 struct radv_image *image = cmd_buffer->state.attachments[att_idx].iview->image;
3007
3008 assert(vk_format_is_depth_or_stencil(image->vk_format));
3009
3010 /* From the Vulkan spec 1.1.108:
3011 *
3012 * "If the image referenced by the framebuffer attachment at
3013 * index attachmentIndex was not created with
3014 * VK_IMAGE_CREATE_SAMPLE_LOCATIONS_COMPATIBLE_DEPTH_BIT_EXT
3015 * then the values specified in sampleLocationsInfo are
3016 * ignored."
3017 */
3018 if (!(image->flags & VK_IMAGE_CREATE_SAMPLE_LOCATIONS_COMPATIBLE_DEPTH_BIT_EXT))
3019 continue;
3020
3021 const VkSampleLocationsInfoEXT *sample_locs_info =
3022 &att_sample_locs->sampleLocationsInfo;
3023
3024 state->attachments[att_idx].sample_location.per_pixel =
3025 sample_locs_info->sampleLocationsPerPixel;
3026 state->attachments[att_idx].sample_location.grid_size =
3027 sample_locs_info->sampleLocationGridSize;
3028 state->attachments[att_idx].sample_location.count =
3029 sample_locs_info->sampleLocationsCount;
3030 typed_memcpy(&state->attachments[att_idx].sample_location.locations[0],
3031 sample_locs_info->pSampleLocations,
3032 sample_locs_info->sampleLocationsCount);
3033 }
3034
3035 state->subpass_sample_locs = vk_alloc(&cmd_buffer->pool->alloc,
3036 sample_locs->postSubpassSampleLocationsCount *
3037 sizeof(state->subpass_sample_locs[0]),
3038 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
3039 if (state->subpass_sample_locs == NULL) {
3040 cmd_buffer->record_result = VK_ERROR_OUT_OF_HOST_MEMORY;
3041 return cmd_buffer->record_result;
3042 }
3043
3044 state->num_subpass_sample_locs = sample_locs->postSubpassSampleLocationsCount;
3045
3046 for (uint32_t i = 0; i < sample_locs->postSubpassSampleLocationsCount; i++) {
3047 const VkSubpassSampleLocationsEXT *subpass_sample_locs_info =
3048 &sample_locs->pPostSubpassSampleLocations[i];
3049 const VkSampleLocationsInfoEXT *sample_locs_info =
3050 &subpass_sample_locs_info->sampleLocationsInfo;
3051
3052 state->subpass_sample_locs[i].subpass_idx =
3053 subpass_sample_locs_info->subpassIndex;
3054 state->subpass_sample_locs[i].sample_location.per_pixel =
3055 sample_locs_info->sampleLocationsPerPixel;
3056 state->subpass_sample_locs[i].sample_location.grid_size =
3057 sample_locs_info->sampleLocationGridSize;
3058 state->subpass_sample_locs[i].sample_location.count =
3059 sample_locs_info->sampleLocationsCount;
3060 typed_memcpy(&state->subpass_sample_locs[i].sample_location.locations[0],
3061 sample_locs_info->pSampleLocations,
3062 sample_locs_info->sampleLocationsCount);
3063 }
3064
3065 return VK_SUCCESS;
3066 }
3067
3068 static VkResult
3069 radv_cmd_state_setup_attachments(struct radv_cmd_buffer *cmd_buffer,
3070 struct radv_render_pass *pass,
3071 const VkRenderPassBeginInfo *info)
3072 {
3073 struct radv_cmd_state *state = &cmd_buffer->state;
3074 const struct VkRenderPassAttachmentBeginInfoKHR *attachment_info = NULL;
3075
3076 if (info) {
3077 attachment_info = vk_find_struct_const(info->pNext,
3078 RENDER_PASS_ATTACHMENT_BEGIN_INFO_KHR);
3079 }
3080
3081
3082 if (pass->attachment_count == 0) {
3083 state->attachments = NULL;
3084 return VK_SUCCESS;
3085 }
3086
3087 state->attachments = vk_alloc(&cmd_buffer->pool->alloc,
3088 pass->attachment_count *
3089 sizeof(state->attachments[0]),
3090 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
3091 if (state->attachments == NULL) {
3092 cmd_buffer->record_result = VK_ERROR_OUT_OF_HOST_MEMORY;
3093 return cmd_buffer->record_result;
3094 }
3095
3096 for (uint32_t i = 0; i < pass->attachment_count; ++i) {
3097 struct radv_render_pass_attachment *att = &pass->attachments[i];
3098 VkImageAspectFlags att_aspects = vk_format_aspects(att->format);
3099 VkImageAspectFlags clear_aspects = 0;
3100
3101 if (att_aspects == VK_IMAGE_ASPECT_COLOR_BIT) {
3102 /* color attachment */
3103 if (att->load_op == VK_ATTACHMENT_LOAD_OP_CLEAR) {
3104 clear_aspects |= VK_IMAGE_ASPECT_COLOR_BIT;
3105 }
3106 } else {
3107 /* depthstencil attachment */
3108 if ((att_aspects & VK_IMAGE_ASPECT_DEPTH_BIT) &&
3109 att->load_op == VK_ATTACHMENT_LOAD_OP_CLEAR) {
3110 clear_aspects |= VK_IMAGE_ASPECT_DEPTH_BIT;
3111 if ((att_aspects & VK_IMAGE_ASPECT_STENCIL_BIT) &&
3112 att->stencil_load_op == VK_ATTACHMENT_LOAD_OP_DONT_CARE)
3113 clear_aspects |= VK_IMAGE_ASPECT_STENCIL_BIT;
3114 }
3115 if ((att_aspects & VK_IMAGE_ASPECT_STENCIL_BIT) &&
3116 att->stencil_load_op == VK_ATTACHMENT_LOAD_OP_CLEAR) {
3117 clear_aspects |= VK_IMAGE_ASPECT_STENCIL_BIT;
3118 }
3119 }
3120
3121 state->attachments[i].pending_clear_aspects = clear_aspects;
3122 state->attachments[i].cleared_views = 0;
3123 if (clear_aspects && info) {
3124 assert(info->clearValueCount > i);
3125 state->attachments[i].clear_value = info->pClearValues[i];
3126 }
3127
3128 state->attachments[i].current_layout = att->initial_layout;
3129 state->attachments[i].sample_location.count = 0;
3130
3131 struct radv_image_view *iview;
3132 if (attachment_info && attachment_info->attachmentCount > i) {
3133 iview = radv_image_view_from_handle(attachment_info->pAttachments[i]);
3134 } else {
3135 iview = state->framebuffer->attachments[i];
3136 }
3137
3138 state->attachments[i].iview = iview;
3139 if (iview->aspect_mask & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) {
3140 radv_initialise_ds_surface(cmd_buffer->device, &state->attachments[i].ds, iview);
3141 } else {
3142 radv_initialise_color_surface(cmd_buffer->device, &state->attachments[i].cb, iview);
3143 }
3144 }
3145
3146 return VK_SUCCESS;
3147 }
3148
3149 VkResult radv_AllocateCommandBuffers(
3150 VkDevice _device,
3151 const VkCommandBufferAllocateInfo *pAllocateInfo,
3152 VkCommandBuffer *pCommandBuffers)
3153 {
3154 RADV_FROM_HANDLE(radv_device, device, _device);
3155 RADV_FROM_HANDLE(radv_cmd_pool, pool, pAllocateInfo->commandPool);
3156
3157 VkResult result = VK_SUCCESS;
3158 uint32_t i;
3159
3160 for (i = 0; i < pAllocateInfo->commandBufferCount; i++) {
3161
3162 if (!list_empty(&pool->free_cmd_buffers)) {
3163 struct radv_cmd_buffer *cmd_buffer = list_first_entry(&pool->free_cmd_buffers, struct radv_cmd_buffer, pool_link);
3164
3165 list_del(&cmd_buffer->pool_link);
3166 list_addtail(&cmd_buffer->pool_link, &pool->cmd_buffers);
3167
3168 result = radv_reset_cmd_buffer(cmd_buffer);
3169 cmd_buffer->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
3170 cmd_buffer->level = pAllocateInfo->level;
3171
3172 pCommandBuffers[i] = radv_cmd_buffer_to_handle(cmd_buffer);
3173 } else {
3174 result = radv_create_cmd_buffer(device, pool, pAllocateInfo->level,
3175 &pCommandBuffers[i]);
3176 }
3177 if (result != VK_SUCCESS)
3178 break;
3179 }
3180
3181 if (result != VK_SUCCESS) {
3182 radv_FreeCommandBuffers(_device, pAllocateInfo->commandPool,
3183 i, pCommandBuffers);
3184
3185 /* From the Vulkan 1.0.66 spec:
3186 *
3187 * "vkAllocateCommandBuffers can be used to create multiple
3188 * command buffers. If the creation of any of those command
3189 * buffers fails, the implementation must destroy all
3190 * successfully created command buffer objects from this
3191 * command, set all entries of the pCommandBuffers array to
3192 * NULL and return the error."
3193 */
3194 memset(pCommandBuffers, 0,
3195 sizeof(*pCommandBuffers) * pAllocateInfo->commandBufferCount);
3196 }
3197
3198 return result;
3199 }
3200
3201 void radv_FreeCommandBuffers(
3202 VkDevice device,
3203 VkCommandPool commandPool,
3204 uint32_t commandBufferCount,
3205 const VkCommandBuffer *pCommandBuffers)
3206 {
3207 for (uint32_t i = 0; i < commandBufferCount; i++) {
3208 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, pCommandBuffers[i]);
3209
3210 if (cmd_buffer) {
3211 if (cmd_buffer->pool) {
3212 list_del(&cmd_buffer->pool_link);
3213 list_addtail(&cmd_buffer->pool_link, &cmd_buffer->pool->free_cmd_buffers);
3214 } else
3215 radv_cmd_buffer_destroy(cmd_buffer);
3216
3217 }
3218 }
3219 }
3220
3221 VkResult radv_ResetCommandBuffer(
3222 VkCommandBuffer commandBuffer,
3223 VkCommandBufferResetFlags flags)
3224 {
3225 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
3226 return radv_reset_cmd_buffer(cmd_buffer);
3227 }
3228
3229 VkResult radv_BeginCommandBuffer(
3230 VkCommandBuffer commandBuffer,
3231 const VkCommandBufferBeginInfo *pBeginInfo)
3232 {
3233 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
3234 VkResult result = VK_SUCCESS;
3235
3236 if (cmd_buffer->status != RADV_CMD_BUFFER_STATUS_INITIAL) {
3237 /* If the command buffer has already been resetted with
3238 * vkResetCommandBuffer, no need to do it again.
3239 */
3240 result = radv_reset_cmd_buffer(cmd_buffer);
3241 if (result != VK_SUCCESS)
3242 return result;
3243 }
3244
3245 memset(&cmd_buffer->state, 0, sizeof(cmd_buffer->state));
3246 cmd_buffer->state.last_primitive_reset_en = -1;
3247 cmd_buffer->state.last_index_type = -1;
3248 cmd_buffer->state.last_num_instances = -1;
3249 cmd_buffer->state.last_vertex_offset = -1;
3250 cmd_buffer->state.last_first_instance = -1;
3251 cmd_buffer->state.predication_type = -1;
3252 cmd_buffer->usage_flags = pBeginInfo->flags;
3253
3254 if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY &&
3255 (pBeginInfo->flags & VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT)) {
3256 assert(pBeginInfo->pInheritanceInfo);
3257 cmd_buffer->state.framebuffer = radv_framebuffer_from_handle(pBeginInfo->pInheritanceInfo->framebuffer);
3258 cmd_buffer->state.pass = radv_render_pass_from_handle(pBeginInfo->pInheritanceInfo->renderPass);
3259
3260 struct radv_subpass *subpass =
3261 &cmd_buffer->state.pass->subpasses[pBeginInfo->pInheritanceInfo->subpass];
3262
3263 if (cmd_buffer->state.framebuffer) {
3264 result = radv_cmd_state_setup_attachments(cmd_buffer, cmd_buffer->state.pass, NULL);
3265 if (result != VK_SUCCESS)
3266 return result;
3267 }
3268
3269 radv_cmd_buffer_set_subpass(cmd_buffer, subpass);
3270 }
3271
3272 if (unlikely(cmd_buffer->device->trace_bo)) {
3273 struct radv_device *device = cmd_buffer->device;
3274
3275 radv_cs_add_buffer(device->ws, cmd_buffer->cs,
3276 device->trace_bo);
3277
3278 radv_cmd_buffer_trace_emit(cmd_buffer);
3279 }
3280
3281 cmd_buffer->status = RADV_CMD_BUFFER_STATUS_RECORDING;
3282
3283 return result;
3284 }
3285
3286 void radv_CmdBindVertexBuffers(
3287 VkCommandBuffer commandBuffer,
3288 uint32_t firstBinding,
3289 uint32_t bindingCount,
3290 const VkBuffer* pBuffers,
3291 const VkDeviceSize* pOffsets)
3292 {
3293 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
3294 struct radv_vertex_binding *vb = cmd_buffer->vertex_bindings;
3295 bool changed = false;
3296
3297 /* We have to defer setting up vertex buffer since we need the buffer
3298 * stride from the pipeline. */
3299
3300 assert(firstBinding + bindingCount <= MAX_VBS);
3301 for (uint32_t i = 0; i < bindingCount; i++) {
3302 uint32_t idx = firstBinding + i;
3303
3304 if (!changed &&
3305 (vb[idx].buffer != radv_buffer_from_handle(pBuffers[i]) ||
3306 vb[idx].offset != pOffsets[i])) {
3307 changed = true;
3308 }
3309
3310 vb[idx].buffer = radv_buffer_from_handle(pBuffers[i]);
3311 vb[idx].offset = pOffsets[i];
3312
3313 radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs,
3314 vb[idx].buffer->bo);
3315 }
3316
3317 if (!changed) {
3318 /* No state changes. */
3319 return;
3320 }
3321
3322 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_VERTEX_BUFFER;
3323 }
3324
3325 static uint32_t
3326 vk_to_index_type(VkIndexType type)
3327 {
3328 switch (type) {
3329 case VK_INDEX_TYPE_UINT8_EXT:
3330 return V_028A7C_VGT_INDEX_8;
3331 case VK_INDEX_TYPE_UINT16:
3332 return V_028A7C_VGT_INDEX_16;
3333 case VK_INDEX_TYPE_UINT32:
3334 return V_028A7C_VGT_INDEX_32;
3335 default:
3336 unreachable("invalid index type");
3337 }
3338 }
3339
3340 static uint32_t
3341 radv_get_vgt_index_size(uint32_t type)
3342 {
3343 switch (type) {
3344 case V_028A7C_VGT_INDEX_8:
3345 return 1;
3346 case V_028A7C_VGT_INDEX_16:
3347 return 2;
3348 case V_028A7C_VGT_INDEX_32:
3349 return 4;
3350 default:
3351 unreachable("invalid index type");
3352 }
3353 }
3354
3355 void radv_CmdBindIndexBuffer(
3356 VkCommandBuffer commandBuffer,
3357 VkBuffer buffer,
3358 VkDeviceSize offset,
3359 VkIndexType indexType)
3360 {
3361 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
3362 RADV_FROM_HANDLE(radv_buffer, index_buffer, buffer);
3363
3364 if (cmd_buffer->state.index_buffer == index_buffer &&
3365 cmd_buffer->state.index_offset == offset &&
3366 cmd_buffer->state.index_type == indexType) {
3367 /* No state changes. */
3368 return;
3369 }
3370
3371 cmd_buffer->state.index_buffer = index_buffer;
3372 cmd_buffer->state.index_offset = offset;
3373 cmd_buffer->state.index_type = vk_to_index_type(indexType);
3374 cmd_buffer->state.index_va = radv_buffer_get_va(index_buffer->bo);
3375 cmd_buffer->state.index_va += index_buffer->offset + offset;
3376
3377 int index_size = radv_get_vgt_index_size(vk_to_index_type(indexType));
3378 cmd_buffer->state.max_index_count = (index_buffer->size - offset) / index_size;
3379 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_INDEX_BUFFER;
3380 radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs, index_buffer->bo);
3381 }
3382
3383
3384 static void
3385 radv_bind_descriptor_set(struct radv_cmd_buffer *cmd_buffer,
3386 VkPipelineBindPoint bind_point,
3387 struct radv_descriptor_set *set, unsigned idx)
3388 {
3389 struct radeon_winsys *ws = cmd_buffer->device->ws;
3390
3391 radv_set_descriptor_set(cmd_buffer, bind_point, set, idx);
3392
3393 assert(set);
3394 assert(!(set->layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
3395
3396 if (!cmd_buffer->device->use_global_bo_list) {
3397 for (unsigned j = 0; j < set->layout->buffer_count; ++j)
3398 if (set->descriptors[j])
3399 radv_cs_add_buffer(ws, cmd_buffer->cs, set->descriptors[j]);
3400 }
3401
3402 if(set->bo)
3403 radv_cs_add_buffer(ws, cmd_buffer->cs, set->bo);
3404 }
3405
3406 void radv_CmdBindDescriptorSets(
3407 VkCommandBuffer commandBuffer,
3408 VkPipelineBindPoint pipelineBindPoint,
3409 VkPipelineLayout _layout,
3410 uint32_t firstSet,
3411 uint32_t descriptorSetCount,
3412 const VkDescriptorSet* pDescriptorSets,
3413 uint32_t dynamicOffsetCount,
3414 const uint32_t* pDynamicOffsets)
3415 {
3416 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
3417 RADV_FROM_HANDLE(radv_pipeline_layout, layout, _layout);
3418 unsigned dyn_idx = 0;
3419
3420 const bool no_dynamic_bounds = cmd_buffer->device->instance->debug_flags & RADV_DEBUG_NO_DYNAMIC_BOUNDS;
3421 struct radv_descriptor_state *descriptors_state =
3422 radv_get_descriptors_state(cmd_buffer, pipelineBindPoint);
3423
3424 for (unsigned i = 0; i < descriptorSetCount; ++i) {
3425 unsigned idx = i + firstSet;
3426 RADV_FROM_HANDLE(radv_descriptor_set, set, pDescriptorSets[i]);
3427
3428 /* If the set is already bound we only need to update the
3429 * (potentially changed) dynamic offsets. */
3430 if (descriptors_state->sets[idx] != set ||
3431 !(descriptors_state->valid & (1u << idx))) {
3432 radv_bind_descriptor_set(cmd_buffer, pipelineBindPoint, set, idx);
3433 }
3434
3435 for(unsigned j = 0; j < set->layout->dynamic_offset_count; ++j, ++dyn_idx) {
3436 unsigned idx = j + layout->set[i + firstSet].dynamic_offset_start;
3437 uint32_t *dst = descriptors_state->dynamic_buffers + idx * 4;
3438 assert(dyn_idx < dynamicOffsetCount);
3439
3440 struct radv_descriptor_range *range = set->dynamic_descriptors + j;
3441 uint64_t va = range->va + pDynamicOffsets[dyn_idx];
3442 dst[0] = va;
3443 dst[1] = S_008F04_BASE_ADDRESS_HI(va >> 32);
3444 dst[2] = no_dynamic_bounds ? 0xffffffffu : range->size;
3445 dst[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3446 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3447 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3448 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3449
3450 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX10) {
3451 dst[3] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3452 S_008F0C_OOB_SELECT(3) |
3453 S_008F0C_RESOURCE_LEVEL(1);
3454 } else {
3455 dst[3] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3456 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3457 }
3458
3459 cmd_buffer->push_constant_stages |=
3460 set->layout->dynamic_shader_stages;
3461 }
3462 }
3463 }
3464
3465 static bool radv_init_push_descriptor_set(struct radv_cmd_buffer *cmd_buffer,
3466 struct radv_descriptor_set *set,
3467 struct radv_descriptor_set_layout *layout,
3468 VkPipelineBindPoint bind_point)
3469 {
3470 struct radv_descriptor_state *descriptors_state =
3471 radv_get_descriptors_state(cmd_buffer, bind_point);
3472 set->size = layout->size;
3473 set->layout = layout;
3474
3475 if (descriptors_state->push_set.capacity < set->size) {
3476 size_t new_size = MAX2(set->size, 1024);
3477 new_size = MAX2(new_size, 2 * descriptors_state->push_set.capacity);
3478 new_size = MIN2(new_size, 96 * MAX_PUSH_DESCRIPTORS);
3479
3480 free(set->mapped_ptr);
3481 set->mapped_ptr = malloc(new_size);
3482
3483 if (!set->mapped_ptr) {
3484 descriptors_state->push_set.capacity = 0;
3485 cmd_buffer->record_result = VK_ERROR_OUT_OF_HOST_MEMORY;
3486 return false;
3487 }
3488
3489 descriptors_state->push_set.capacity = new_size;
3490 }
3491
3492 return true;
3493 }
3494
3495 void radv_meta_push_descriptor_set(
3496 struct radv_cmd_buffer* cmd_buffer,
3497 VkPipelineBindPoint pipelineBindPoint,
3498 VkPipelineLayout _layout,
3499 uint32_t set,
3500 uint32_t descriptorWriteCount,
3501 const VkWriteDescriptorSet* pDescriptorWrites)
3502 {
3503 RADV_FROM_HANDLE(radv_pipeline_layout, layout, _layout);
3504 struct radv_descriptor_set *push_set = &cmd_buffer->meta_push_descriptors;
3505 unsigned bo_offset;
3506
3507 assert(set == 0);
3508 assert(layout->set[set].layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR);
3509
3510 push_set->size = layout->set[set].layout->size;
3511 push_set->layout = layout->set[set].layout;
3512
3513 if (!radv_cmd_buffer_upload_alloc(cmd_buffer, push_set->size, 32,
3514 &bo_offset,
3515 (void**) &push_set->mapped_ptr))
3516 return;
3517
3518 push_set->va = radv_buffer_get_va(cmd_buffer->upload.upload_bo);
3519 push_set->va += bo_offset;
3520
3521 radv_update_descriptor_sets(cmd_buffer->device, cmd_buffer,
3522 radv_descriptor_set_to_handle(push_set),
3523 descriptorWriteCount, pDescriptorWrites, 0, NULL);
3524
3525 radv_set_descriptor_set(cmd_buffer, pipelineBindPoint, push_set, set);
3526 }
3527
3528 void radv_CmdPushDescriptorSetKHR(
3529 VkCommandBuffer commandBuffer,
3530 VkPipelineBindPoint pipelineBindPoint,
3531 VkPipelineLayout _layout,
3532 uint32_t set,
3533 uint32_t descriptorWriteCount,
3534 const VkWriteDescriptorSet* pDescriptorWrites)
3535 {
3536 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
3537 RADV_FROM_HANDLE(radv_pipeline_layout, layout, _layout);
3538 struct radv_descriptor_state *descriptors_state =
3539 radv_get_descriptors_state(cmd_buffer, pipelineBindPoint);
3540 struct radv_descriptor_set *push_set = &descriptors_state->push_set.set;
3541
3542 assert(layout->set[set].layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR);
3543
3544 if (!radv_init_push_descriptor_set(cmd_buffer, push_set,
3545 layout->set[set].layout,
3546 pipelineBindPoint))
3547 return;
3548
3549 /* Check that there are no inline uniform block updates when calling vkCmdPushDescriptorSetKHR()
3550 * because it is invalid, according to Vulkan spec.
3551 */
3552 for (int i = 0; i < descriptorWriteCount; i++) {
3553 ASSERTED const VkWriteDescriptorSet *writeset = &pDescriptorWrites[i];
3554 assert(writeset->descriptorType != VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT);
3555 }
3556
3557 radv_update_descriptor_sets(cmd_buffer->device, cmd_buffer,
3558 radv_descriptor_set_to_handle(push_set),
3559 descriptorWriteCount, pDescriptorWrites, 0, NULL);
3560
3561 radv_set_descriptor_set(cmd_buffer, pipelineBindPoint, push_set, set);
3562 descriptors_state->push_dirty = true;
3563 }
3564
3565 void radv_CmdPushDescriptorSetWithTemplateKHR(
3566 VkCommandBuffer commandBuffer,
3567 VkDescriptorUpdateTemplate descriptorUpdateTemplate,
3568 VkPipelineLayout _layout,
3569 uint32_t set,
3570 const void* pData)
3571 {
3572 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
3573 RADV_FROM_HANDLE(radv_pipeline_layout, layout, _layout);
3574 RADV_FROM_HANDLE(radv_descriptor_update_template, templ, descriptorUpdateTemplate);
3575 struct radv_descriptor_state *descriptors_state =
3576 radv_get_descriptors_state(cmd_buffer, templ->bind_point);
3577 struct radv_descriptor_set *push_set = &descriptors_state->push_set.set;
3578
3579 assert(layout->set[set].layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR);
3580
3581 if (!radv_init_push_descriptor_set(cmd_buffer, push_set,
3582 layout->set[set].layout,
3583 templ->bind_point))
3584 return;
3585
3586 radv_update_descriptor_set_with_template(cmd_buffer->device, cmd_buffer, push_set,
3587 descriptorUpdateTemplate, pData);
3588
3589 radv_set_descriptor_set(cmd_buffer, templ->bind_point, push_set, set);
3590 descriptors_state->push_dirty = true;
3591 }
3592
3593 void radv_CmdPushConstants(VkCommandBuffer commandBuffer,
3594 VkPipelineLayout layout,
3595 VkShaderStageFlags stageFlags,
3596 uint32_t offset,
3597 uint32_t size,
3598 const void* pValues)
3599 {
3600 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
3601 memcpy(cmd_buffer->push_constants + offset, pValues, size);
3602 cmd_buffer->push_constant_stages |= stageFlags;
3603 }
3604
3605 VkResult radv_EndCommandBuffer(
3606 VkCommandBuffer commandBuffer)
3607 {
3608 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
3609
3610 if (cmd_buffer->queue_family_index != RADV_QUEUE_TRANSFER) {
3611 if (cmd_buffer->device->physical_device->rad_info.chip_class == GFX6)
3612 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_PS_PARTIAL_FLUSH | RADV_CMD_FLAG_WB_L2;
3613
3614 /* Make sure to sync all pending active queries at the end of
3615 * command buffer.
3616 */
3617 cmd_buffer->state.flush_bits |= cmd_buffer->active_query_flush_bits;
3618
3619 /* Since NGG streamout uses GDS, we need to make GDS idle when
3620 * we leave the IB, otherwise another process might overwrite
3621 * it while our shaders are busy.
3622 */
3623 if (cmd_buffer->gds_needed)
3624 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_PS_PARTIAL_FLUSH;
3625
3626 si_emit_cache_flush(cmd_buffer);
3627 }
3628
3629 /* Make sure CP DMA is idle at the end of IBs because the kernel
3630 * doesn't wait for it.
3631 */
3632 si_cp_dma_wait_for_idle(cmd_buffer);
3633
3634 vk_free(&cmd_buffer->pool->alloc, cmd_buffer->state.attachments);
3635 vk_free(&cmd_buffer->pool->alloc, cmd_buffer->state.subpass_sample_locs);
3636
3637 if (!cmd_buffer->device->ws->cs_finalize(cmd_buffer->cs))
3638 return vk_error(cmd_buffer->device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY);
3639
3640 cmd_buffer->status = RADV_CMD_BUFFER_STATUS_EXECUTABLE;
3641
3642 return cmd_buffer->record_result;
3643 }
3644
3645 static void
3646 radv_emit_compute_pipeline(struct radv_cmd_buffer *cmd_buffer)
3647 {
3648 struct radv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
3649
3650 if (!pipeline || pipeline == cmd_buffer->state.emitted_compute_pipeline)
3651 return;
3652
3653 assert(!pipeline->ctx_cs.cdw);
3654
3655 cmd_buffer->state.emitted_compute_pipeline = pipeline;
3656
3657 radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, pipeline->cs.cdw);
3658 radeon_emit_array(cmd_buffer->cs, pipeline->cs.buf, pipeline->cs.cdw);
3659
3660 cmd_buffer->compute_scratch_size_needed =
3661 MAX2(cmd_buffer->compute_scratch_size_needed,
3662 pipeline->max_waves * pipeline->scratch_bytes_per_wave);
3663
3664 radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs,
3665 pipeline->shaders[MESA_SHADER_COMPUTE]->bo);
3666
3667 if (unlikely(cmd_buffer->device->trace_bo))
3668 radv_save_pipeline(cmd_buffer, pipeline, RING_COMPUTE);
3669 }
3670
3671 static void radv_mark_descriptor_sets_dirty(struct radv_cmd_buffer *cmd_buffer,
3672 VkPipelineBindPoint bind_point)
3673 {
3674 struct radv_descriptor_state *descriptors_state =
3675 radv_get_descriptors_state(cmd_buffer, bind_point);
3676
3677 descriptors_state->dirty |= descriptors_state->valid;
3678 }
3679
3680 void radv_CmdBindPipeline(
3681 VkCommandBuffer commandBuffer,
3682 VkPipelineBindPoint pipelineBindPoint,
3683 VkPipeline _pipeline)
3684 {
3685 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
3686 RADV_FROM_HANDLE(radv_pipeline, pipeline, _pipeline);
3687
3688 switch (pipelineBindPoint) {
3689 case VK_PIPELINE_BIND_POINT_COMPUTE:
3690 if (cmd_buffer->state.compute_pipeline == pipeline)
3691 return;
3692 radv_mark_descriptor_sets_dirty(cmd_buffer, pipelineBindPoint);
3693
3694 cmd_buffer->state.compute_pipeline = pipeline;
3695 cmd_buffer->push_constant_stages |= VK_SHADER_STAGE_COMPUTE_BIT;
3696 break;
3697 case VK_PIPELINE_BIND_POINT_GRAPHICS:
3698 if (cmd_buffer->state.pipeline == pipeline)
3699 return;
3700 radv_mark_descriptor_sets_dirty(cmd_buffer, pipelineBindPoint);
3701
3702 cmd_buffer->state.pipeline = pipeline;
3703 if (!pipeline)
3704 break;
3705
3706 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_PIPELINE;
3707 cmd_buffer->push_constant_stages |= pipeline->active_stages;
3708
3709 /* the new vertex shader might not have the same user regs */
3710 cmd_buffer->state.last_first_instance = -1;
3711 cmd_buffer->state.last_vertex_offset = -1;
3712
3713 /* Prefetch all pipeline shaders at first draw time. */
3714 cmd_buffer->state.prefetch_L2_mask |= RADV_PREFETCH_SHADERS;
3715
3716 if ((cmd_buffer->device->physical_device->rad_info.family == CHIP_NAVI10 ||
3717 cmd_buffer->device->physical_device->rad_info.family == CHIP_NAVI12 ||
3718 cmd_buffer->device->physical_device->rad_info.family == CHIP_NAVI14) &&
3719 cmd_buffer->state.emitted_pipeline &&
3720 radv_pipeline_has_ngg(cmd_buffer->state.emitted_pipeline) &&
3721 !radv_pipeline_has_ngg(cmd_buffer->state.pipeline)) {
3722 /* Transitioning from NGG to legacy GS requires
3723 * VGT_FLUSH on Navi10-14. VGT_FLUSH is also emitted
3724 * at the beginning of IBs when legacy GS ring pointers
3725 * are set.
3726 */
3727 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_VGT_FLUSH;
3728 }
3729
3730 radv_bind_dynamic_state(cmd_buffer, &pipeline->dynamic_state);
3731 radv_bind_streamout_state(cmd_buffer, pipeline);
3732
3733 if (pipeline->graphics.esgs_ring_size > cmd_buffer->esgs_ring_size_needed)
3734 cmd_buffer->esgs_ring_size_needed = pipeline->graphics.esgs_ring_size;
3735 if (pipeline->graphics.gsvs_ring_size > cmd_buffer->gsvs_ring_size_needed)
3736 cmd_buffer->gsvs_ring_size_needed = pipeline->graphics.gsvs_ring_size;
3737
3738 if (radv_pipeline_has_tess(pipeline))
3739 cmd_buffer->tess_rings_needed = true;
3740 break;
3741 default:
3742 assert(!"invalid bind point");
3743 break;
3744 }
3745 }
3746
3747 void radv_CmdSetViewport(
3748 VkCommandBuffer commandBuffer,
3749 uint32_t firstViewport,
3750 uint32_t viewportCount,
3751 const VkViewport* pViewports)
3752 {
3753 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
3754 struct radv_cmd_state *state = &cmd_buffer->state;
3755 ASSERTED const uint32_t total_count = firstViewport + viewportCount;
3756
3757 assert(firstViewport < MAX_VIEWPORTS);
3758 assert(total_count >= 1 && total_count <= MAX_VIEWPORTS);
3759
3760 if (!memcmp(state->dynamic.viewport.viewports + firstViewport,
3761 pViewports, viewportCount * sizeof(*pViewports))) {
3762 return;
3763 }
3764
3765 memcpy(state->dynamic.viewport.viewports + firstViewport, pViewports,
3766 viewportCount * sizeof(*pViewports));
3767
3768 state->dirty |= RADV_CMD_DIRTY_DYNAMIC_VIEWPORT;
3769 }
3770
3771 void radv_CmdSetScissor(
3772 VkCommandBuffer commandBuffer,
3773 uint32_t firstScissor,
3774 uint32_t scissorCount,
3775 const VkRect2D* pScissors)
3776 {
3777 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
3778 struct radv_cmd_state *state = &cmd_buffer->state;
3779 ASSERTED const uint32_t total_count = firstScissor + scissorCount;
3780
3781 assert(firstScissor < MAX_SCISSORS);
3782 assert(total_count >= 1 && total_count <= MAX_SCISSORS);
3783
3784 if (!memcmp(state->dynamic.scissor.scissors + firstScissor, pScissors,
3785 scissorCount * sizeof(*pScissors))) {
3786 return;
3787 }
3788
3789 memcpy(state->dynamic.scissor.scissors + firstScissor, pScissors,
3790 scissorCount * sizeof(*pScissors));
3791
3792 state->dirty |= RADV_CMD_DIRTY_DYNAMIC_SCISSOR;
3793 }
3794
3795 void radv_CmdSetLineWidth(
3796 VkCommandBuffer commandBuffer,
3797 float lineWidth)
3798 {
3799 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
3800
3801 if (cmd_buffer->state.dynamic.line_width == lineWidth)
3802 return;
3803
3804 cmd_buffer->state.dynamic.line_width = lineWidth;
3805 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_LINE_WIDTH;
3806 }
3807
3808 void radv_CmdSetDepthBias(
3809 VkCommandBuffer commandBuffer,
3810 float depthBiasConstantFactor,
3811 float depthBiasClamp,
3812 float depthBiasSlopeFactor)
3813 {
3814 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
3815 struct radv_cmd_state *state = &cmd_buffer->state;
3816
3817 if (state->dynamic.depth_bias.bias == depthBiasConstantFactor &&
3818 state->dynamic.depth_bias.clamp == depthBiasClamp &&
3819 state->dynamic.depth_bias.slope == depthBiasSlopeFactor) {
3820 return;
3821 }
3822
3823 state->dynamic.depth_bias.bias = depthBiasConstantFactor;
3824 state->dynamic.depth_bias.clamp = depthBiasClamp;
3825 state->dynamic.depth_bias.slope = depthBiasSlopeFactor;
3826
3827 state->dirty |= RADV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS;
3828 }
3829
3830 void radv_CmdSetBlendConstants(
3831 VkCommandBuffer commandBuffer,
3832 const float blendConstants[4])
3833 {
3834 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
3835 struct radv_cmd_state *state = &cmd_buffer->state;
3836
3837 if (!memcmp(state->dynamic.blend_constants, blendConstants, sizeof(float) * 4))
3838 return;
3839
3840 memcpy(state->dynamic.blend_constants, blendConstants, sizeof(float) * 4);
3841
3842 state->dirty |= RADV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS;
3843 }
3844
3845 void radv_CmdSetDepthBounds(
3846 VkCommandBuffer commandBuffer,
3847 float minDepthBounds,
3848 float maxDepthBounds)
3849 {
3850 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
3851 struct radv_cmd_state *state = &cmd_buffer->state;
3852
3853 if (state->dynamic.depth_bounds.min == minDepthBounds &&
3854 state->dynamic.depth_bounds.max == maxDepthBounds) {
3855 return;
3856 }
3857
3858 state->dynamic.depth_bounds.min = minDepthBounds;
3859 state->dynamic.depth_bounds.max = maxDepthBounds;
3860
3861 state->dirty |= RADV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS;
3862 }
3863
3864 void radv_CmdSetStencilCompareMask(
3865 VkCommandBuffer commandBuffer,
3866 VkStencilFaceFlags faceMask,
3867 uint32_t compareMask)
3868 {
3869 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
3870 struct radv_cmd_state *state = &cmd_buffer->state;
3871 bool front_same = state->dynamic.stencil_compare_mask.front == compareMask;
3872 bool back_same = state->dynamic.stencil_compare_mask.back == compareMask;
3873
3874 if ((!(faceMask & VK_STENCIL_FACE_FRONT_BIT) || front_same) &&
3875 (!(faceMask & VK_STENCIL_FACE_BACK_BIT) || back_same)) {
3876 return;
3877 }
3878
3879 if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
3880 state->dynamic.stencil_compare_mask.front = compareMask;
3881 if (faceMask & VK_STENCIL_FACE_BACK_BIT)
3882 state->dynamic.stencil_compare_mask.back = compareMask;
3883
3884 state->dirty |= RADV_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK;
3885 }
3886
3887 void radv_CmdSetStencilWriteMask(
3888 VkCommandBuffer commandBuffer,
3889 VkStencilFaceFlags faceMask,
3890 uint32_t writeMask)
3891 {
3892 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
3893 struct radv_cmd_state *state = &cmd_buffer->state;
3894 bool front_same = state->dynamic.stencil_write_mask.front == writeMask;
3895 bool back_same = state->dynamic.stencil_write_mask.back == writeMask;
3896
3897 if ((!(faceMask & VK_STENCIL_FACE_FRONT_BIT) || front_same) &&
3898 (!(faceMask & VK_STENCIL_FACE_BACK_BIT) || back_same)) {
3899 return;
3900 }
3901
3902 if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
3903 state->dynamic.stencil_write_mask.front = writeMask;
3904 if (faceMask & VK_STENCIL_FACE_BACK_BIT)
3905 state->dynamic.stencil_write_mask.back = writeMask;
3906
3907 state->dirty |= RADV_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK;
3908 }
3909
3910 void radv_CmdSetStencilReference(
3911 VkCommandBuffer commandBuffer,
3912 VkStencilFaceFlags faceMask,
3913 uint32_t reference)
3914 {
3915 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
3916 struct radv_cmd_state *state = &cmd_buffer->state;
3917 bool front_same = state->dynamic.stencil_reference.front == reference;
3918 bool back_same = state->dynamic.stencil_reference.back == reference;
3919
3920 if ((!(faceMask & VK_STENCIL_FACE_FRONT_BIT) || front_same) &&
3921 (!(faceMask & VK_STENCIL_FACE_BACK_BIT) || back_same)) {
3922 return;
3923 }
3924
3925 if (faceMask & VK_STENCIL_FACE_FRONT_BIT)
3926 cmd_buffer->state.dynamic.stencil_reference.front = reference;
3927 if (faceMask & VK_STENCIL_FACE_BACK_BIT)
3928 cmd_buffer->state.dynamic.stencil_reference.back = reference;
3929
3930 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE;
3931 }
3932
3933 void radv_CmdSetDiscardRectangleEXT(
3934 VkCommandBuffer commandBuffer,
3935 uint32_t firstDiscardRectangle,
3936 uint32_t discardRectangleCount,
3937 const VkRect2D* pDiscardRectangles)
3938 {
3939 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
3940 struct radv_cmd_state *state = &cmd_buffer->state;
3941 ASSERTED const uint32_t total_count = firstDiscardRectangle + discardRectangleCount;
3942
3943 assert(firstDiscardRectangle < MAX_DISCARD_RECTANGLES);
3944 assert(total_count >= 1 && total_count <= MAX_DISCARD_RECTANGLES);
3945
3946 if (!memcmp(state->dynamic.discard_rectangle.rectangles + firstDiscardRectangle,
3947 pDiscardRectangles, discardRectangleCount * sizeof(*pDiscardRectangles))) {
3948 return;
3949 }
3950
3951 typed_memcpy(&state->dynamic.discard_rectangle.rectangles[firstDiscardRectangle],
3952 pDiscardRectangles, discardRectangleCount);
3953
3954 state->dirty |= RADV_CMD_DIRTY_DYNAMIC_DISCARD_RECTANGLE;
3955 }
3956
3957 void radv_CmdSetSampleLocationsEXT(
3958 VkCommandBuffer commandBuffer,
3959 const VkSampleLocationsInfoEXT* pSampleLocationsInfo)
3960 {
3961 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
3962 struct radv_cmd_state *state = &cmd_buffer->state;
3963
3964 assert(pSampleLocationsInfo->sampleLocationsCount <= MAX_SAMPLE_LOCATIONS);
3965
3966 state->dynamic.sample_location.per_pixel = pSampleLocationsInfo->sampleLocationsPerPixel;
3967 state->dynamic.sample_location.grid_size = pSampleLocationsInfo->sampleLocationGridSize;
3968 state->dynamic.sample_location.count = pSampleLocationsInfo->sampleLocationsCount;
3969 typed_memcpy(&state->dynamic.sample_location.locations[0],
3970 pSampleLocationsInfo->pSampleLocations,
3971 pSampleLocationsInfo->sampleLocationsCount);
3972
3973 state->dirty |= RADV_CMD_DIRTY_DYNAMIC_SAMPLE_LOCATIONS;
3974 }
3975
3976 void radv_CmdExecuteCommands(
3977 VkCommandBuffer commandBuffer,
3978 uint32_t commandBufferCount,
3979 const VkCommandBuffer* pCmdBuffers)
3980 {
3981 RADV_FROM_HANDLE(radv_cmd_buffer, primary, commandBuffer);
3982
3983 assert(commandBufferCount > 0);
3984
3985 /* Emit pending flushes on primary prior to executing secondary */
3986 si_emit_cache_flush(primary);
3987
3988 for (uint32_t i = 0; i < commandBufferCount; i++) {
3989 RADV_FROM_HANDLE(radv_cmd_buffer, secondary, pCmdBuffers[i]);
3990
3991 primary->scratch_size_needed = MAX2(primary->scratch_size_needed,
3992 secondary->scratch_size_needed);
3993 primary->compute_scratch_size_needed = MAX2(primary->compute_scratch_size_needed,
3994 secondary->compute_scratch_size_needed);
3995
3996 if (secondary->esgs_ring_size_needed > primary->esgs_ring_size_needed)
3997 primary->esgs_ring_size_needed = secondary->esgs_ring_size_needed;
3998 if (secondary->gsvs_ring_size_needed > primary->gsvs_ring_size_needed)
3999 primary->gsvs_ring_size_needed = secondary->gsvs_ring_size_needed;
4000 if (secondary->tess_rings_needed)
4001 primary->tess_rings_needed = true;
4002 if (secondary->sample_positions_needed)
4003 primary->sample_positions_needed = true;
4004
4005 if (!secondary->state.framebuffer &&
4006 (primary->state.dirty & RADV_CMD_DIRTY_FRAMEBUFFER)) {
4007 /* Emit the framebuffer state from primary if secondary
4008 * has been recorded without a framebuffer, otherwise
4009 * fast color/depth clears can't work.
4010 */
4011 radv_emit_framebuffer_state(primary);
4012 }
4013
4014 primary->device->ws->cs_execute_secondary(primary->cs, secondary->cs);
4015
4016
4017 /* When the secondary command buffer is compute only we don't
4018 * need to re-emit the current graphics pipeline.
4019 */
4020 if (secondary->state.emitted_pipeline) {
4021 primary->state.emitted_pipeline =
4022 secondary->state.emitted_pipeline;
4023 }
4024
4025 /* When the secondary command buffer is graphics only we don't
4026 * need to re-emit the current compute pipeline.
4027 */
4028 if (secondary->state.emitted_compute_pipeline) {
4029 primary->state.emitted_compute_pipeline =
4030 secondary->state.emitted_compute_pipeline;
4031 }
4032
4033 /* Only re-emit the draw packets when needed. */
4034 if (secondary->state.last_primitive_reset_en != -1) {
4035 primary->state.last_primitive_reset_en =
4036 secondary->state.last_primitive_reset_en;
4037 }
4038
4039 if (secondary->state.last_primitive_reset_index) {
4040 primary->state.last_primitive_reset_index =
4041 secondary->state.last_primitive_reset_index;
4042 }
4043
4044 if (secondary->state.last_ia_multi_vgt_param) {
4045 primary->state.last_ia_multi_vgt_param =
4046 secondary->state.last_ia_multi_vgt_param;
4047 }
4048
4049 primary->state.last_first_instance = secondary->state.last_first_instance;
4050 primary->state.last_num_instances = secondary->state.last_num_instances;
4051 primary->state.last_vertex_offset = secondary->state.last_vertex_offset;
4052
4053 if (secondary->state.last_index_type != -1) {
4054 primary->state.last_index_type =
4055 secondary->state.last_index_type;
4056 }
4057 }
4058
4059 /* After executing commands from secondary buffers we have to dirty
4060 * some states.
4061 */
4062 primary->state.dirty |= RADV_CMD_DIRTY_PIPELINE |
4063 RADV_CMD_DIRTY_INDEX_BUFFER |
4064 RADV_CMD_DIRTY_DYNAMIC_ALL;
4065 radv_mark_descriptor_sets_dirty(primary, VK_PIPELINE_BIND_POINT_GRAPHICS);
4066 radv_mark_descriptor_sets_dirty(primary, VK_PIPELINE_BIND_POINT_COMPUTE);
4067 }
4068
4069 VkResult radv_CreateCommandPool(
4070 VkDevice _device,
4071 const VkCommandPoolCreateInfo* pCreateInfo,
4072 const VkAllocationCallbacks* pAllocator,
4073 VkCommandPool* pCmdPool)
4074 {
4075 RADV_FROM_HANDLE(radv_device, device, _device);
4076 struct radv_cmd_pool *pool;
4077
4078 pool = vk_alloc2(&device->alloc, pAllocator, sizeof(*pool), 8,
4079 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
4080 if (pool == NULL)
4081 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
4082
4083 if (pAllocator)
4084 pool->alloc = *pAllocator;
4085 else
4086 pool->alloc = device->alloc;
4087
4088 list_inithead(&pool->cmd_buffers);
4089 list_inithead(&pool->free_cmd_buffers);
4090
4091 pool->queue_family_index = pCreateInfo->queueFamilyIndex;
4092
4093 *pCmdPool = radv_cmd_pool_to_handle(pool);
4094
4095 return VK_SUCCESS;
4096
4097 }
4098
4099 void radv_DestroyCommandPool(
4100 VkDevice _device,
4101 VkCommandPool commandPool,
4102 const VkAllocationCallbacks* pAllocator)
4103 {
4104 RADV_FROM_HANDLE(radv_device, device, _device);
4105 RADV_FROM_HANDLE(radv_cmd_pool, pool, commandPool);
4106
4107 if (!pool)
4108 return;
4109
4110 list_for_each_entry_safe(struct radv_cmd_buffer, cmd_buffer,
4111 &pool->cmd_buffers, pool_link) {
4112 radv_cmd_buffer_destroy(cmd_buffer);
4113 }
4114
4115 list_for_each_entry_safe(struct radv_cmd_buffer, cmd_buffer,
4116 &pool->free_cmd_buffers, pool_link) {
4117 radv_cmd_buffer_destroy(cmd_buffer);
4118 }
4119
4120 vk_free2(&device->alloc, pAllocator, pool);
4121 }
4122
4123 VkResult radv_ResetCommandPool(
4124 VkDevice device,
4125 VkCommandPool commandPool,
4126 VkCommandPoolResetFlags flags)
4127 {
4128 RADV_FROM_HANDLE(radv_cmd_pool, pool, commandPool);
4129 VkResult result;
4130
4131 list_for_each_entry(struct radv_cmd_buffer, cmd_buffer,
4132 &pool->cmd_buffers, pool_link) {
4133 result = radv_reset_cmd_buffer(cmd_buffer);
4134 if (result != VK_SUCCESS)
4135 return result;
4136 }
4137
4138 return VK_SUCCESS;
4139 }
4140
4141 void radv_TrimCommandPool(
4142 VkDevice device,
4143 VkCommandPool commandPool,
4144 VkCommandPoolTrimFlags flags)
4145 {
4146 RADV_FROM_HANDLE(radv_cmd_pool, pool, commandPool);
4147
4148 if (!pool)
4149 return;
4150
4151 list_for_each_entry_safe(struct radv_cmd_buffer, cmd_buffer,
4152 &pool->free_cmd_buffers, pool_link) {
4153 radv_cmd_buffer_destroy(cmd_buffer);
4154 }
4155 }
4156
4157 static void
4158 radv_cmd_buffer_begin_subpass(struct radv_cmd_buffer *cmd_buffer,
4159 uint32_t subpass_id)
4160 {
4161 struct radv_cmd_state *state = &cmd_buffer->state;
4162 struct radv_subpass *subpass = &state->pass->subpasses[subpass_id];
4163
4164 ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws,
4165 cmd_buffer->cs, 4096);
4166
4167 radv_subpass_barrier(cmd_buffer, &subpass->start_barrier);
4168
4169 radv_cmd_buffer_set_subpass(cmd_buffer, subpass);
4170
4171 for (uint32_t i = 0; i < subpass->attachment_count; ++i) {
4172 const uint32_t a = subpass->attachments[i].attachment;
4173 if (a == VK_ATTACHMENT_UNUSED)
4174 continue;
4175
4176 radv_handle_subpass_image_transition(cmd_buffer,
4177 subpass->attachments[i],
4178 true);
4179 }
4180
4181 radv_cmd_buffer_clear_subpass(cmd_buffer);
4182
4183 assert(cmd_buffer->cs->cdw <= cdw_max);
4184 }
4185
4186 static void
4187 radv_cmd_buffer_end_subpass(struct radv_cmd_buffer *cmd_buffer)
4188 {
4189 struct radv_cmd_state *state = &cmd_buffer->state;
4190 const struct radv_subpass *subpass = state->subpass;
4191 uint32_t subpass_id = radv_get_subpass_id(cmd_buffer);
4192
4193 radv_cmd_buffer_resolve_subpass(cmd_buffer);
4194
4195 for (uint32_t i = 0; i < subpass->attachment_count; ++i) {
4196 const uint32_t a = subpass->attachments[i].attachment;
4197 if (a == VK_ATTACHMENT_UNUSED)
4198 continue;
4199
4200 if (state->pass->attachments[a].last_subpass_idx != subpass_id)
4201 continue;
4202
4203 VkImageLayout layout = state->pass->attachments[a].final_layout;
4204 struct radv_subpass_attachment att = { a, layout };
4205 radv_handle_subpass_image_transition(cmd_buffer, att, false);
4206 }
4207 }
4208
4209 void radv_CmdBeginRenderPass(
4210 VkCommandBuffer commandBuffer,
4211 const VkRenderPassBeginInfo* pRenderPassBegin,
4212 VkSubpassContents contents)
4213 {
4214 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
4215 RADV_FROM_HANDLE(radv_render_pass, pass, pRenderPassBegin->renderPass);
4216 RADV_FROM_HANDLE(radv_framebuffer, framebuffer, pRenderPassBegin->framebuffer);
4217 VkResult result;
4218
4219 cmd_buffer->state.framebuffer = framebuffer;
4220 cmd_buffer->state.pass = pass;
4221 cmd_buffer->state.render_area = pRenderPassBegin->renderArea;
4222
4223 result = radv_cmd_state_setup_attachments(cmd_buffer, pass, pRenderPassBegin);
4224 if (result != VK_SUCCESS)
4225 return;
4226
4227 result = radv_cmd_state_setup_sample_locations(cmd_buffer, pass, pRenderPassBegin);
4228 if (result != VK_SUCCESS)
4229 return;
4230
4231 radv_cmd_buffer_begin_subpass(cmd_buffer, 0);
4232 }
4233
4234 void radv_CmdBeginRenderPass2KHR(
4235 VkCommandBuffer commandBuffer,
4236 const VkRenderPassBeginInfo* pRenderPassBeginInfo,
4237 const VkSubpassBeginInfoKHR* pSubpassBeginInfo)
4238 {
4239 radv_CmdBeginRenderPass(commandBuffer, pRenderPassBeginInfo,
4240 pSubpassBeginInfo->contents);
4241 }
4242
4243 void radv_CmdNextSubpass(
4244 VkCommandBuffer commandBuffer,
4245 VkSubpassContents contents)
4246 {
4247 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
4248
4249 uint32_t prev_subpass = radv_get_subpass_id(cmd_buffer);
4250 radv_cmd_buffer_end_subpass(cmd_buffer);
4251 radv_cmd_buffer_begin_subpass(cmd_buffer, prev_subpass + 1);
4252 }
4253
4254 void radv_CmdNextSubpass2KHR(
4255 VkCommandBuffer commandBuffer,
4256 const VkSubpassBeginInfoKHR* pSubpassBeginInfo,
4257 const VkSubpassEndInfoKHR* pSubpassEndInfo)
4258 {
4259 radv_CmdNextSubpass(commandBuffer, pSubpassBeginInfo->contents);
4260 }
4261
4262 static void radv_emit_view_index(struct radv_cmd_buffer *cmd_buffer, unsigned index)
4263 {
4264 struct radv_pipeline *pipeline = cmd_buffer->state.pipeline;
4265 for (unsigned stage = 0; stage < MESA_SHADER_STAGES; ++stage) {
4266 if (!radv_get_shader(pipeline, stage))
4267 continue;
4268
4269 struct radv_userdata_info *loc = radv_lookup_user_sgpr(pipeline, stage, AC_UD_VIEW_INDEX);
4270 if (loc->sgpr_idx == -1)
4271 continue;
4272 uint32_t base_reg = pipeline->user_data_0[stage];
4273 radeon_set_sh_reg(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, index);
4274
4275 }
4276 if (radv_pipeline_has_gs_copy_shader(pipeline)) {
4277 struct radv_userdata_info *loc = &pipeline->gs_copy_shader->info.user_sgprs_locs.shader_data[AC_UD_VIEW_INDEX];
4278 if (loc->sgpr_idx != -1) {
4279 uint32_t base_reg = R_00B130_SPI_SHADER_USER_DATA_VS_0;
4280 radeon_set_sh_reg(cmd_buffer->cs, base_reg + loc->sgpr_idx * 4, index);
4281 }
4282 }
4283 }
4284
4285 static void
4286 radv_cs_emit_draw_packet(struct radv_cmd_buffer *cmd_buffer,
4287 uint32_t vertex_count,
4288 bool use_opaque)
4289 {
4290 radeon_emit(cmd_buffer->cs, PKT3(PKT3_DRAW_INDEX_AUTO, 1, cmd_buffer->state.predicating));
4291 radeon_emit(cmd_buffer->cs, vertex_count);
4292 radeon_emit(cmd_buffer->cs, V_0287F0_DI_SRC_SEL_AUTO_INDEX |
4293 S_0287F0_USE_OPAQUE(use_opaque));
4294 }
4295
4296 static void
4297 radv_cs_emit_draw_indexed_packet(struct radv_cmd_buffer *cmd_buffer,
4298 uint64_t index_va,
4299 uint32_t index_count)
4300 {
4301 radeon_emit(cmd_buffer->cs, PKT3(PKT3_DRAW_INDEX_2, 4, cmd_buffer->state.predicating));
4302 radeon_emit(cmd_buffer->cs, cmd_buffer->state.max_index_count);
4303 radeon_emit(cmd_buffer->cs, index_va);
4304 radeon_emit(cmd_buffer->cs, index_va >> 32);
4305 radeon_emit(cmd_buffer->cs, index_count);
4306 radeon_emit(cmd_buffer->cs, V_0287F0_DI_SRC_SEL_DMA);
4307 }
4308
4309 static void
4310 radv_cs_emit_indirect_draw_packet(struct radv_cmd_buffer *cmd_buffer,
4311 bool indexed,
4312 uint32_t draw_count,
4313 uint64_t count_va,
4314 uint32_t stride)
4315 {
4316 struct radeon_cmdbuf *cs = cmd_buffer->cs;
4317 unsigned di_src_sel = indexed ? V_0287F0_DI_SRC_SEL_DMA
4318 : V_0287F0_DI_SRC_SEL_AUTO_INDEX;
4319 bool draw_id_enable = radv_get_shader(cmd_buffer->state.pipeline, MESA_SHADER_VERTEX)->info.vs.needs_draw_id;
4320 uint32_t base_reg = cmd_buffer->state.pipeline->graphics.vtx_base_sgpr;
4321 bool predicating = cmd_buffer->state.predicating;
4322 assert(base_reg);
4323
4324 /* just reset draw state for vertex data */
4325 cmd_buffer->state.last_first_instance = -1;
4326 cmd_buffer->state.last_num_instances = -1;
4327 cmd_buffer->state.last_vertex_offset = -1;
4328
4329 if (draw_count == 1 && !count_va && !draw_id_enable) {
4330 radeon_emit(cs, PKT3(indexed ? PKT3_DRAW_INDEX_INDIRECT :
4331 PKT3_DRAW_INDIRECT, 3, predicating));
4332 radeon_emit(cs, 0);
4333 radeon_emit(cs, (base_reg - SI_SH_REG_OFFSET) >> 2);
4334 radeon_emit(cs, ((base_reg + 4) - SI_SH_REG_OFFSET) >> 2);
4335 radeon_emit(cs, di_src_sel);
4336 } else {
4337 radeon_emit(cs, PKT3(indexed ? PKT3_DRAW_INDEX_INDIRECT_MULTI :
4338 PKT3_DRAW_INDIRECT_MULTI,
4339 8, predicating));
4340 radeon_emit(cs, 0);
4341 radeon_emit(cs, (base_reg - SI_SH_REG_OFFSET) >> 2);
4342 radeon_emit(cs, ((base_reg + 4) - SI_SH_REG_OFFSET) >> 2);
4343 radeon_emit(cs, (((base_reg + 8) - SI_SH_REG_OFFSET) >> 2) |
4344 S_2C3_DRAW_INDEX_ENABLE(draw_id_enable) |
4345 S_2C3_COUNT_INDIRECT_ENABLE(!!count_va));
4346 radeon_emit(cs, draw_count); /* count */
4347 radeon_emit(cs, count_va); /* count_addr */
4348 radeon_emit(cs, count_va >> 32);
4349 radeon_emit(cs, stride); /* stride */
4350 radeon_emit(cs, di_src_sel);
4351 }
4352 }
4353
4354 static void
4355 radv_emit_draw_packets(struct radv_cmd_buffer *cmd_buffer,
4356 const struct radv_draw_info *info)
4357 {
4358 struct radv_cmd_state *state = &cmd_buffer->state;
4359 struct radeon_winsys *ws = cmd_buffer->device->ws;
4360 struct radeon_cmdbuf *cs = cmd_buffer->cs;
4361
4362 if (info->indirect) {
4363 uint64_t va = radv_buffer_get_va(info->indirect->bo);
4364 uint64_t count_va = 0;
4365
4366 va += info->indirect->offset + info->indirect_offset;
4367
4368 radv_cs_add_buffer(ws, cs, info->indirect->bo);
4369
4370 radeon_emit(cs, PKT3(PKT3_SET_BASE, 2, 0));
4371 radeon_emit(cs, 1);
4372 radeon_emit(cs, va);
4373 radeon_emit(cs, va >> 32);
4374
4375 if (info->count_buffer) {
4376 count_va = radv_buffer_get_va(info->count_buffer->bo);
4377 count_va += info->count_buffer->offset +
4378 info->count_buffer_offset;
4379
4380 radv_cs_add_buffer(ws, cs, info->count_buffer->bo);
4381 }
4382
4383 if (!state->subpass->view_mask) {
4384 radv_cs_emit_indirect_draw_packet(cmd_buffer,
4385 info->indexed,
4386 info->count,
4387 count_va,
4388 info->stride);
4389 } else {
4390 unsigned i;
4391 for_each_bit(i, state->subpass->view_mask) {
4392 radv_emit_view_index(cmd_buffer, i);
4393
4394 radv_cs_emit_indirect_draw_packet(cmd_buffer,
4395 info->indexed,
4396 info->count,
4397 count_va,
4398 info->stride);
4399 }
4400 }
4401 } else {
4402 assert(state->pipeline->graphics.vtx_base_sgpr);
4403
4404 if (info->vertex_offset != state->last_vertex_offset ||
4405 info->first_instance != state->last_first_instance) {
4406 radeon_set_sh_reg_seq(cs, state->pipeline->graphics.vtx_base_sgpr,
4407 state->pipeline->graphics.vtx_emit_num);
4408
4409 radeon_emit(cs, info->vertex_offset);
4410 radeon_emit(cs, info->first_instance);
4411 if (state->pipeline->graphics.vtx_emit_num == 3)
4412 radeon_emit(cs, 0);
4413 state->last_first_instance = info->first_instance;
4414 state->last_vertex_offset = info->vertex_offset;
4415 }
4416
4417 if (state->last_num_instances != info->instance_count) {
4418 radeon_emit(cs, PKT3(PKT3_NUM_INSTANCES, 0, false));
4419 radeon_emit(cs, info->instance_count);
4420 state->last_num_instances = info->instance_count;
4421 }
4422
4423 if (info->indexed) {
4424 int index_size = radv_get_vgt_index_size(state->index_type);
4425 uint64_t index_va;
4426
4427 /* Skip draw calls with 0-sized index buffers. They
4428 * cause a hang on some chips, like Navi10-14.
4429 */
4430 if (!cmd_buffer->state.max_index_count)
4431 return;
4432
4433 index_va = state->index_va;
4434 index_va += info->first_index * index_size;
4435
4436 if (!state->subpass->view_mask) {
4437 radv_cs_emit_draw_indexed_packet(cmd_buffer,
4438 index_va,
4439 info->count);
4440 } else {
4441 unsigned i;
4442 for_each_bit(i, state->subpass->view_mask) {
4443 radv_emit_view_index(cmd_buffer, i);
4444
4445 radv_cs_emit_draw_indexed_packet(cmd_buffer,
4446 index_va,
4447 info->count);
4448 }
4449 }
4450 } else {
4451 if (!state->subpass->view_mask) {
4452 radv_cs_emit_draw_packet(cmd_buffer,
4453 info->count,
4454 !!info->strmout_buffer);
4455 } else {
4456 unsigned i;
4457 for_each_bit(i, state->subpass->view_mask) {
4458 radv_emit_view_index(cmd_buffer, i);
4459
4460 radv_cs_emit_draw_packet(cmd_buffer,
4461 info->count,
4462 !!info->strmout_buffer);
4463 }
4464 }
4465 }
4466 }
4467 }
4468
4469 /*
4470 * Vega and raven have a bug which triggers if there are multiple context
4471 * register contexts active at the same time with different scissor values.
4472 *
4473 * There are two possible workarounds:
4474 * 1) Wait for PS_PARTIAL_FLUSH every time the scissor is changed. That way
4475 * there is only ever 1 active set of scissor values at the same time.
4476 *
4477 * 2) Whenever the hardware switches contexts we have to set the scissor
4478 * registers again even if it is a noop. That way the new context gets
4479 * the correct scissor values.
4480 *
4481 * This implements option 2. radv_need_late_scissor_emission needs to
4482 * return true on affected HW if radv_emit_all_graphics_states sets
4483 * any context registers.
4484 */
4485 static bool radv_need_late_scissor_emission(struct radv_cmd_buffer *cmd_buffer,
4486 const struct radv_draw_info *info)
4487 {
4488 struct radv_cmd_state *state = &cmd_buffer->state;
4489
4490 if (!cmd_buffer->device->physical_device->rad_info.has_gfx9_scissor_bug)
4491 return false;
4492
4493 if (cmd_buffer->state.context_roll_without_scissor_emitted || info->strmout_buffer)
4494 return true;
4495
4496 uint32_t used_states = cmd_buffer->state.pipeline->graphics.needed_dynamic_state | ~RADV_CMD_DIRTY_DYNAMIC_ALL;
4497
4498 /* Index, vertex and streamout buffers don't change context regs, and
4499 * pipeline is already handled.
4500 */
4501 used_states &= ~(RADV_CMD_DIRTY_INDEX_BUFFER |
4502 RADV_CMD_DIRTY_VERTEX_BUFFER |
4503 RADV_CMD_DIRTY_STREAMOUT_BUFFER |
4504 RADV_CMD_DIRTY_PIPELINE);
4505
4506 if (cmd_buffer->state.dirty & used_states)
4507 return true;
4508
4509 uint32_t primitive_reset_index =
4510 radv_get_primitive_reset_index(cmd_buffer);
4511
4512 if (info->indexed && state->pipeline->graphics.prim_restart_enable &&
4513 primitive_reset_index != state->last_primitive_reset_index)
4514 return true;
4515
4516 return false;
4517 }
4518
4519 static void
4520 radv_emit_all_graphics_states(struct radv_cmd_buffer *cmd_buffer,
4521 const struct radv_draw_info *info)
4522 {
4523 bool late_scissor_emission;
4524
4525 if ((cmd_buffer->state.dirty & RADV_CMD_DIRTY_FRAMEBUFFER) ||
4526 cmd_buffer->state.emitted_pipeline != cmd_buffer->state.pipeline)
4527 radv_emit_rbplus_state(cmd_buffer);
4528
4529 if (cmd_buffer->state.dirty & RADV_CMD_DIRTY_PIPELINE)
4530 radv_emit_graphics_pipeline(cmd_buffer);
4531
4532 /* This should be before the cmd_buffer->state.dirty is cleared
4533 * (excluding RADV_CMD_DIRTY_PIPELINE) and after
4534 * cmd_buffer->state.context_roll_without_scissor_emitted is set. */
4535 late_scissor_emission =
4536 radv_need_late_scissor_emission(cmd_buffer, info);
4537
4538 if (cmd_buffer->state.dirty & RADV_CMD_DIRTY_FRAMEBUFFER)
4539 radv_emit_framebuffer_state(cmd_buffer);
4540
4541 if (info->indexed) {
4542 if (cmd_buffer->state.dirty & RADV_CMD_DIRTY_INDEX_BUFFER)
4543 radv_emit_index_buffer(cmd_buffer);
4544 } else {
4545 /* On GFX7 and later, non-indexed draws overwrite VGT_INDEX_TYPE,
4546 * so the state must be re-emitted before the next indexed
4547 * draw.
4548 */
4549 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX7) {
4550 cmd_buffer->state.last_index_type = -1;
4551 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_INDEX_BUFFER;
4552 }
4553 }
4554
4555 radv_cmd_buffer_flush_dynamic_state(cmd_buffer);
4556
4557 radv_emit_draw_registers(cmd_buffer, info);
4558
4559 if (late_scissor_emission)
4560 radv_emit_scissor(cmd_buffer);
4561 }
4562
4563 static void
4564 radv_draw(struct radv_cmd_buffer *cmd_buffer,
4565 const struct radv_draw_info *info)
4566 {
4567 struct radeon_info *rad_info =
4568 &cmd_buffer->device->physical_device->rad_info;
4569 bool has_prefetch =
4570 cmd_buffer->device->physical_device->rad_info.chip_class >= GFX7;
4571 bool pipeline_is_dirty =
4572 (cmd_buffer->state.dirty & RADV_CMD_DIRTY_PIPELINE) &&
4573 cmd_buffer->state.pipeline != cmd_buffer->state.emitted_pipeline;
4574
4575 ASSERTED unsigned cdw_max =
4576 radeon_check_space(cmd_buffer->device->ws,
4577 cmd_buffer->cs, 4096);
4578
4579 if (likely(!info->indirect)) {
4580 /* GFX6-GFX7 treat instance_count==0 as instance_count==1. There is
4581 * no workaround for indirect draws, but we can at least skip
4582 * direct draws.
4583 */
4584 if (unlikely(!info->instance_count))
4585 return;
4586
4587 /* Handle count == 0. */
4588 if (unlikely(!info->count && !info->strmout_buffer))
4589 return;
4590 }
4591
4592 /* Use optimal packet order based on whether we need to sync the
4593 * pipeline.
4594 */
4595 if (cmd_buffer->state.flush_bits & (RADV_CMD_FLAG_FLUSH_AND_INV_CB |
4596 RADV_CMD_FLAG_FLUSH_AND_INV_DB |
4597 RADV_CMD_FLAG_PS_PARTIAL_FLUSH |
4598 RADV_CMD_FLAG_CS_PARTIAL_FLUSH)) {
4599 /* If we have to wait for idle, set all states first, so that
4600 * all SET packets are processed in parallel with previous draw
4601 * calls. Then upload descriptors, set shader pointers, and
4602 * draw, and prefetch at the end. This ensures that the time
4603 * the CUs are idle is very short. (there are only SET_SH
4604 * packets between the wait and the draw)
4605 */
4606 radv_emit_all_graphics_states(cmd_buffer, info);
4607 si_emit_cache_flush(cmd_buffer);
4608 /* <-- CUs are idle here --> */
4609
4610 radv_upload_graphics_shader_descriptors(cmd_buffer, pipeline_is_dirty);
4611
4612 radv_emit_draw_packets(cmd_buffer, info);
4613 /* <-- CUs are busy here --> */
4614
4615 /* Start prefetches after the draw has been started. Both will
4616 * run in parallel, but starting the draw first is more
4617 * important.
4618 */
4619 if (has_prefetch && cmd_buffer->state.prefetch_L2_mask) {
4620 radv_emit_prefetch_L2(cmd_buffer,
4621 cmd_buffer->state.pipeline, false);
4622 }
4623 } else {
4624 /* If we don't wait for idle, start prefetches first, then set
4625 * states, and draw at the end.
4626 */
4627 si_emit_cache_flush(cmd_buffer);
4628
4629 if (has_prefetch && cmd_buffer->state.prefetch_L2_mask) {
4630 /* Only prefetch the vertex shader and VBO descriptors
4631 * in order to start the draw as soon as possible.
4632 */
4633 radv_emit_prefetch_L2(cmd_buffer,
4634 cmd_buffer->state.pipeline, true);
4635 }
4636
4637 radv_upload_graphics_shader_descriptors(cmd_buffer, pipeline_is_dirty);
4638
4639 radv_emit_all_graphics_states(cmd_buffer, info);
4640 radv_emit_draw_packets(cmd_buffer, info);
4641
4642 /* Prefetch the remaining shaders after the draw has been
4643 * started.
4644 */
4645 if (has_prefetch && cmd_buffer->state.prefetch_L2_mask) {
4646 radv_emit_prefetch_L2(cmd_buffer,
4647 cmd_buffer->state.pipeline, false);
4648 }
4649 }
4650
4651 /* Workaround for a VGT hang when streamout is enabled.
4652 * It must be done after drawing.
4653 */
4654 if (cmd_buffer->state.streamout.streamout_enabled &&
4655 (rad_info->family == CHIP_HAWAII ||
4656 rad_info->family == CHIP_TONGA ||
4657 rad_info->family == CHIP_FIJI)) {
4658 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_VGT_STREAMOUT_SYNC;
4659 }
4660
4661 assert(cmd_buffer->cs->cdw <= cdw_max);
4662 radv_cmd_buffer_after_draw(cmd_buffer, RADV_CMD_FLAG_PS_PARTIAL_FLUSH);
4663 }
4664
4665 void radv_CmdDraw(
4666 VkCommandBuffer commandBuffer,
4667 uint32_t vertexCount,
4668 uint32_t instanceCount,
4669 uint32_t firstVertex,
4670 uint32_t firstInstance)
4671 {
4672 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
4673 struct radv_draw_info info = {};
4674
4675 info.count = vertexCount;
4676 info.instance_count = instanceCount;
4677 info.first_instance = firstInstance;
4678 info.vertex_offset = firstVertex;
4679
4680 radv_draw(cmd_buffer, &info);
4681 }
4682
4683 void radv_CmdDrawIndexed(
4684 VkCommandBuffer commandBuffer,
4685 uint32_t indexCount,
4686 uint32_t instanceCount,
4687 uint32_t firstIndex,
4688 int32_t vertexOffset,
4689 uint32_t firstInstance)
4690 {
4691 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
4692 struct radv_draw_info info = {};
4693
4694 info.indexed = true;
4695 info.count = indexCount;
4696 info.instance_count = instanceCount;
4697 info.first_index = firstIndex;
4698 info.vertex_offset = vertexOffset;
4699 info.first_instance = firstInstance;
4700
4701 radv_draw(cmd_buffer, &info);
4702 }
4703
4704 void radv_CmdDrawIndirect(
4705 VkCommandBuffer commandBuffer,
4706 VkBuffer _buffer,
4707 VkDeviceSize offset,
4708 uint32_t drawCount,
4709 uint32_t stride)
4710 {
4711 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
4712 RADV_FROM_HANDLE(radv_buffer, buffer, _buffer);
4713 struct radv_draw_info info = {};
4714
4715 info.count = drawCount;
4716 info.indirect = buffer;
4717 info.indirect_offset = offset;
4718 info.stride = stride;
4719
4720 radv_draw(cmd_buffer, &info);
4721 }
4722
4723 void radv_CmdDrawIndexedIndirect(
4724 VkCommandBuffer commandBuffer,
4725 VkBuffer _buffer,
4726 VkDeviceSize offset,
4727 uint32_t drawCount,
4728 uint32_t stride)
4729 {
4730 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
4731 RADV_FROM_HANDLE(radv_buffer, buffer, _buffer);
4732 struct radv_draw_info info = {};
4733
4734 info.indexed = true;
4735 info.count = drawCount;
4736 info.indirect = buffer;
4737 info.indirect_offset = offset;
4738 info.stride = stride;
4739
4740 radv_draw(cmd_buffer, &info);
4741 }
4742
4743 void radv_CmdDrawIndirectCountKHR(
4744 VkCommandBuffer commandBuffer,
4745 VkBuffer _buffer,
4746 VkDeviceSize offset,
4747 VkBuffer _countBuffer,
4748 VkDeviceSize countBufferOffset,
4749 uint32_t maxDrawCount,
4750 uint32_t stride)
4751 {
4752 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
4753 RADV_FROM_HANDLE(radv_buffer, buffer, _buffer);
4754 RADV_FROM_HANDLE(radv_buffer, count_buffer, _countBuffer);
4755 struct radv_draw_info info = {};
4756
4757 info.count = maxDrawCount;
4758 info.indirect = buffer;
4759 info.indirect_offset = offset;
4760 info.count_buffer = count_buffer;
4761 info.count_buffer_offset = countBufferOffset;
4762 info.stride = stride;
4763
4764 radv_draw(cmd_buffer, &info);
4765 }
4766
4767 void radv_CmdDrawIndexedIndirectCountKHR(
4768 VkCommandBuffer commandBuffer,
4769 VkBuffer _buffer,
4770 VkDeviceSize offset,
4771 VkBuffer _countBuffer,
4772 VkDeviceSize countBufferOffset,
4773 uint32_t maxDrawCount,
4774 uint32_t stride)
4775 {
4776 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
4777 RADV_FROM_HANDLE(radv_buffer, buffer, _buffer);
4778 RADV_FROM_HANDLE(radv_buffer, count_buffer, _countBuffer);
4779 struct radv_draw_info info = {};
4780
4781 info.indexed = true;
4782 info.count = maxDrawCount;
4783 info.indirect = buffer;
4784 info.indirect_offset = offset;
4785 info.count_buffer = count_buffer;
4786 info.count_buffer_offset = countBufferOffset;
4787 info.stride = stride;
4788
4789 radv_draw(cmd_buffer, &info);
4790 }
4791
4792 struct radv_dispatch_info {
4793 /**
4794 * Determine the layout of the grid (in block units) to be used.
4795 */
4796 uint32_t blocks[3];
4797
4798 /**
4799 * A starting offset for the grid. If unaligned is set, the offset
4800 * must still be aligned.
4801 */
4802 uint32_t offsets[3];
4803 /**
4804 * Whether it's an unaligned compute dispatch.
4805 */
4806 bool unaligned;
4807
4808 /**
4809 * Indirect compute parameters resource.
4810 */
4811 struct radv_buffer *indirect;
4812 uint64_t indirect_offset;
4813 };
4814
4815 static void
4816 radv_emit_dispatch_packets(struct radv_cmd_buffer *cmd_buffer,
4817 const struct radv_dispatch_info *info)
4818 {
4819 struct radv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
4820 struct radv_shader_variant *compute_shader = pipeline->shaders[MESA_SHADER_COMPUTE];
4821 unsigned dispatch_initiator = cmd_buffer->device->dispatch_initiator;
4822 struct radeon_winsys *ws = cmd_buffer->device->ws;
4823 bool predicating = cmd_buffer->state.predicating;
4824 struct radeon_cmdbuf *cs = cmd_buffer->cs;
4825 struct radv_userdata_info *loc;
4826
4827 loc = radv_lookup_user_sgpr(pipeline, MESA_SHADER_COMPUTE,
4828 AC_UD_CS_GRID_SIZE);
4829
4830 ASSERTED unsigned cdw_max = radeon_check_space(ws, cs, 25);
4831
4832 if (info->indirect) {
4833 uint64_t va = radv_buffer_get_va(info->indirect->bo);
4834
4835 va += info->indirect->offset + info->indirect_offset;
4836
4837 radv_cs_add_buffer(ws, cs, info->indirect->bo);
4838
4839 if (loc->sgpr_idx != -1) {
4840 for (unsigned i = 0; i < 3; ++i) {
4841 radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
4842 radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) |
4843 COPY_DATA_DST_SEL(COPY_DATA_REG));
4844 radeon_emit(cs, (va + 4 * i));
4845 radeon_emit(cs, (va + 4 * i) >> 32);
4846 radeon_emit(cs, ((R_00B900_COMPUTE_USER_DATA_0
4847 + loc->sgpr_idx * 4) >> 2) + i);
4848 radeon_emit(cs, 0);
4849 }
4850 }
4851
4852 if (radv_cmd_buffer_uses_mec(cmd_buffer)) {
4853 radeon_emit(cs, PKT3(PKT3_DISPATCH_INDIRECT, 2, predicating) |
4854 PKT3_SHADER_TYPE_S(1));
4855 radeon_emit(cs, va);
4856 radeon_emit(cs, va >> 32);
4857 radeon_emit(cs, dispatch_initiator);
4858 } else {
4859 radeon_emit(cs, PKT3(PKT3_SET_BASE, 2, 0) |
4860 PKT3_SHADER_TYPE_S(1));
4861 radeon_emit(cs, 1);
4862 radeon_emit(cs, va);
4863 radeon_emit(cs, va >> 32);
4864
4865 radeon_emit(cs, PKT3(PKT3_DISPATCH_INDIRECT, 1, predicating) |
4866 PKT3_SHADER_TYPE_S(1));
4867 radeon_emit(cs, 0);
4868 radeon_emit(cs, dispatch_initiator);
4869 }
4870 } else {
4871 unsigned blocks[3] = { info->blocks[0], info->blocks[1], info->blocks[2] };
4872 unsigned offsets[3] = { info->offsets[0], info->offsets[1], info->offsets[2] };
4873
4874 if (info->unaligned) {
4875 unsigned *cs_block_size = compute_shader->info.cs.block_size;
4876 unsigned remainder[3];
4877
4878 /* If aligned, these should be an entire block size,
4879 * not 0.
4880 */
4881 remainder[0] = blocks[0] + cs_block_size[0] -
4882 align_u32_npot(blocks[0], cs_block_size[0]);
4883 remainder[1] = blocks[1] + cs_block_size[1] -
4884 align_u32_npot(blocks[1], cs_block_size[1]);
4885 remainder[2] = blocks[2] + cs_block_size[2] -
4886 align_u32_npot(blocks[2], cs_block_size[2]);
4887
4888 blocks[0] = round_up_u32(blocks[0], cs_block_size[0]);
4889 blocks[1] = round_up_u32(blocks[1], cs_block_size[1]);
4890 blocks[2] = round_up_u32(blocks[2], cs_block_size[2]);
4891
4892 for(unsigned i = 0; i < 3; ++i) {
4893 assert(offsets[i] % cs_block_size[i] == 0);
4894 offsets[i] /= cs_block_size[i];
4895 }
4896
4897 radeon_set_sh_reg_seq(cs, R_00B81C_COMPUTE_NUM_THREAD_X, 3);
4898 radeon_emit(cs,
4899 S_00B81C_NUM_THREAD_FULL(cs_block_size[0]) |
4900 S_00B81C_NUM_THREAD_PARTIAL(remainder[0]));
4901 radeon_emit(cs,
4902 S_00B81C_NUM_THREAD_FULL(cs_block_size[1]) |
4903 S_00B81C_NUM_THREAD_PARTIAL(remainder[1]));
4904 radeon_emit(cs,
4905 S_00B81C_NUM_THREAD_FULL(cs_block_size[2]) |
4906 S_00B81C_NUM_THREAD_PARTIAL(remainder[2]));
4907
4908 dispatch_initiator |= S_00B800_PARTIAL_TG_EN(1);
4909 }
4910
4911 if (loc->sgpr_idx != -1) {
4912 assert(loc->num_sgprs == 3);
4913
4914 radeon_set_sh_reg_seq(cs, R_00B900_COMPUTE_USER_DATA_0 +
4915 loc->sgpr_idx * 4, 3);
4916 radeon_emit(cs, blocks[0]);
4917 radeon_emit(cs, blocks[1]);
4918 radeon_emit(cs, blocks[2]);
4919 }
4920
4921 if (offsets[0] || offsets[1] || offsets[2]) {
4922 radeon_set_sh_reg_seq(cs, R_00B810_COMPUTE_START_X, 3);
4923 radeon_emit(cs, offsets[0]);
4924 radeon_emit(cs, offsets[1]);
4925 radeon_emit(cs, offsets[2]);
4926
4927 /* The blocks in the packet are not counts but end values. */
4928 for (unsigned i = 0; i < 3; ++i)
4929 blocks[i] += offsets[i];
4930 } else {
4931 dispatch_initiator |= S_00B800_FORCE_START_AT_000(1);
4932 }
4933
4934 radeon_emit(cs, PKT3(PKT3_DISPATCH_DIRECT, 3, predicating) |
4935 PKT3_SHADER_TYPE_S(1));
4936 radeon_emit(cs, blocks[0]);
4937 radeon_emit(cs, blocks[1]);
4938 radeon_emit(cs, blocks[2]);
4939 radeon_emit(cs, dispatch_initiator);
4940 }
4941
4942 assert(cmd_buffer->cs->cdw <= cdw_max);
4943 }
4944
4945 static void
4946 radv_upload_compute_shader_descriptors(struct radv_cmd_buffer *cmd_buffer)
4947 {
4948 radv_flush_descriptors(cmd_buffer, VK_SHADER_STAGE_COMPUTE_BIT);
4949 radv_flush_constants(cmd_buffer, VK_SHADER_STAGE_COMPUTE_BIT);
4950 }
4951
4952 static void
4953 radv_dispatch(struct radv_cmd_buffer *cmd_buffer,
4954 const struct radv_dispatch_info *info)
4955 {
4956 struct radv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
4957 bool has_prefetch =
4958 cmd_buffer->device->physical_device->rad_info.chip_class >= GFX7;
4959 bool pipeline_is_dirty = pipeline &&
4960 pipeline != cmd_buffer->state.emitted_compute_pipeline;
4961
4962 if (cmd_buffer->state.flush_bits & (RADV_CMD_FLAG_FLUSH_AND_INV_CB |
4963 RADV_CMD_FLAG_FLUSH_AND_INV_DB |
4964 RADV_CMD_FLAG_PS_PARTIAL_FLUSH |
4965 RADV_CMD_FLAG_CS_PARTIAL_FLUSH)) {
4966 /* If we have to wait for idle, set all states first, so that
4967 * all SET packets are processed in parallel with previous draw
4968 * calls. Then upload descriptors, set shader pointers, and
4969 * dispatch, and prefetch at the end. This ensures that the
4970 * time the CUs are idle is very short. (there are only SET_SH
4971 * packets between the wait and the draw)
4972 */
4973 radv_emit_compute_pipeline(cmd_buffer);
4974 si_emit_cache_flush(cmd_buffer);
4975 /* <-- CUs are idle here --> */
4976
4977 radv_upload_compute_shader_descriptors(cmd_buffer);
4978
4979 radv_emit_dispatch_packets(cmd_buffer, info);
4980 /* <-- CUs are busy here --> */
4981
4982 /* Start prefetches after the dispatch has been started. Both
4983 * will run in parallel, but starting the dispatch first is
4984 * more important.
4985 */
4986 if (has_prefetch && pipeline_is_dirty) {
4987 radv_emit_shader_prefetch(cmd_buffer,
4988 pipeline->shaders[MESA_SHADER_COMPUTE]);
4989 }
4990 } else {
4991 /* If we don't wait for idle, start prefetches first, then set
4992 * states, and dispatch at the end.
4993 */
4994 si_emit_cache_flush(cmd_buffer);
4995
4996 if (has_prefetch && pipeline_is_dirty) {
4997 radv_emit_shader_prefetch(cmd_buffer,
4998 pipeline->shaders[MESA_SHADER_COMPUTE]);
4999 }
5000
5001 radv_upload_compute_shader_descriptors(cmd_buffer);
5002
5003 radv_emit_compute_pipeline(cmd_buffer);
5004 radv_emit_dispatch_packets(cmd_buffer, info);
5005 }
5006
5007 radv_cmd_buffer_after_draw(cmd_buffer, RADV_CMD_FLAG_CS_PARTIAL_FLUSH);
5008 }
5009
5010 void radv_CmdDispatchBase(
5011 VkCommandBuffer commandBuffer,
5012 uint32_t base_x,
5013 uint32_t base_y,
5014 uint32_t base_z,
5015 uint32_t x,
5016 uint32_t y,
5017 uint32_t z)
5018 {
5019 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
5020 struct radv_dispatch_info info = {};
5021
5022 info.blocks[0] = x;
5023 info.blocks[1] = y;
5024 info.blocks[2] = z;
5025
5026 info.offsets[0] = base_x;
5027 info.offsets[1] = base_y;
5028 info.offsets[2] = base_z;
5029 radv_dispatch(cmd_buffer, &info);
5030 }
5031
5032 void radv_CmdDispatch(
5033 VkCommandBuffer commandBuffer,
5034 uint32_t x,
5035 uint32_t y,
5036 uint32_t z)
5037 {
5038 radv_CmdDispatchBase(commandBuffer, 0, 0, 0, x, y, z);
5039 }
5040
5041 void radv_CmdDispatchIndirect(
5042 VkCommandBuffer commandBuffer,
5043 VkBuffer _buffer,
5044 VkDeviceSize offset)
5045 {
5046 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
5047 RADV_FROM_HANDLE(radv_buffer, buffer, _buffer);
5048 struct radv_dispatch_info info = {};
5049
5050 info.indirect = buffer;
5051 info.indirect_offset = offset;
5052
5053 radv_dispatch(cmd_buffer, &info);
5054 }
5055
5056 void radv_unaligned_dispatch(
5057 struct radv_cmd_buffer *cmd_buffer,
5058 uint32_t x,
5059 uint32_t y,
5060 uint32_t z)
5061 {
5062 struct radv_dispatch_info info = {};
5063
5064 info.blocks[0] = x;
5065 info.blocks[1] = y;
5066 info.blocks[2] = z;
5067 info.unaligned = 1;
5068
5069 radv_dispatch(cmd_buffer, &info);
5070 }
5071
5072 void radv_CmdEndRenderPass(
5073 VkCommandBuffer commandBuffer)
5074 {
5075 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
5076
5077 radv_subpass_barrier(cmd_buffer, &cmd_buffer->state.pass->end_barrier);
5078
5079 radv_cmd_buffer_end_subpass(cmd_buffer);
5080
5081 vk_free(&cmd_buffer->pool->alloc, cmd_buffer->state.attachments);
5082 vk_free(&cmd_buffer->pool->alloc, cmd_buffer->state.subpass_sample_locs);
5083
5084 cmd_buffer->state.pass = NULL;
5085 cmd_buffer->state.subpass = NULL;
5086 cmd_buffer->state.attachments = NULL;
5087 cmd_buffer->state.framebuffer = NULL;
5088 cmd_buffer->state.subpass_sample_locs = NULL;
5089 }
5090
5091 void radv_CmdEndRenderPass2KHR(
5092 VkCommandBuffer commandBuffer,
5093 const VkSubpassEndInfoKHR* pSubpassEndInfo)
5094 {
5095 radv_CmdEndRenderPass(commandBuffer);
5096 }
5097
5098 /*
5099 * For HTILE we have the following interesting clear words:
5100 * 0xfffff30f: Uncompressed, full depth range, for depth+stencil HTILE
5101 * 0xfffc000f: Uncompressed, full depth range, for depth only HTILE.
5102 * 0xfffffff0: Clear depth to 1.0
5103 * 0x00000000: Clear depth to 0.0
5104 */
5105 static void radv_initialize_htile(struct radv_cmd_buffer *cmd_buffer,
5106 struct radv_image *image,
5107 const VkImageSubresourceRange *range,
5108 uint32_t clear_word)
5109 {
5110 assert(range->baseMipLevel == 0);
5111 assert(range->levelCount == 1 || range->levelCount == VK_REMAINING_ARRAY_LAYERS);
5112 VkImageAspectFlags aspects = VK_IMAGE_ASPECT_DEPTH_BIT;
5113 struct radv_cmd_state *state = &cmd_buffer->state;
5114 VkClearDepthStencilValue value = {};
5115
5116 state->flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB |
5117 RADV_CMD_FLAG_FLUSH_AND_INV_DB_META;
5118
5119 state->flush_bits |= radv_clear_htile(cmd_buffer, image, range, clear_word);
5120
5121 state->flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB_META;
5122
5123 if (vk_format_is_stencil(image->vk_format))
5124 aspects |= VK_IMAGE_ASPECT_STENCIL_BIT;
5125
5126 radv_set_ds_clear_metadata(cmd_buffer, image, range, value, aspects);
5127
5128 if (radv_image_is_tc_compat_htile(image)) {
5129 /* Initialize the TC-compat metada value to 0 because by
5130 * default DB_Z_INFO.RANGE_PRECISION is set to 1, and we only
5131 * need have to conditionally update its value when performing
5132 * a fast depth clear.
5133 */
5134 radv_set_tc_compat_zrange_metadata(cmd_buffer, image, range, 0);
5135 }
5136 }
5137
5138 static void radv_handle_depth_image_transition(struct radv_cmd_buffer *cmd_buffer,
5139 struct radv_image *image,
5140 VkImageLayout src_layout,
5141 bool src_render_loop,
5142 VkImageLayout dst_layout,
5143 bool dst_render_loop,
5144 unsigned src_queue_mask,
5145 unsigned dst_queue_mask,
5146 const VkImageSubresourceRange *range,
5147 struct radv_sample_locations_state *sample_locs)
5148 {
5149 if (!radv_image_has_htile(image))
5150 return;
5151
5152 if (src_layout == VK_IMAGE_LAYOUT_UNDEFINED) {
5153 uint32_t clear_value = vk_format_is_stencil(image->vk_format) ? 0xfffff30f : 0xfffc000f;
5154
5155 if (radv_layout_is_htile_compressed(image, dst_layout, dst_render_loop,
5156 dst_queue_mask)) {
5157 clear_value = 0;
5158 }
5159
5160 radv_initialize_htile(cmd_buffer, image, range, clear_value);
5161 } else if (!radv_layout_is_htile_compressed(image, src_layout, src_render_loop, src_queue_mask) &&
5162 radv_layout_is_htile_compressed(image, dst_layout, dst_render_loop, dst_queue_mask)) {
5163 uint32_t clear_value = vk_format_is_stencil(image->vk_format) ? 0xfffff30f : 0xfffc000f;
5164 radv_initialize_htile(cmd_buffer, image, range, clear_value);
5165 } else if (radv_layout_is_htile_compressed(image, src_layout, src_render_loop, src_queue_mask) &&
5166 !radv_layout_is_htile_compressed(image, dst_layout, dst_render_loop, dst_queue_mask)) {
5167 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB |
5168 RADV_CMD_FLAG_FLUSH_AND_INV_DB_META;
5169
5170 radv_decompress_depth_image_inplace(cmd_buffer, image, range,
5171 sample_locs);
5172
5173 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB |
5174 RADV_CMD_FLAG_FLUSH_AND_INV_DB_META;
5175 }
5176 }
5177
5178 static void radv_initialise_cmask(struct radv_cmd_buffer *cmd_buffer,
5179 struct radv_image *image,
5180 const VkImageSubresourceRange *range,
5181 uint32_t value)
5182 {
5183 struct radv_cmd_state *state = &cmd_buffer->state;
5184
5185 state->flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB |
5186 RADV_CMD_FLAG_FLUSH_AND_INV_CB_META;
5187
5188 state->flush_bits |= radv_clear_cmask(cmd_buffer, image, range, value);
5189
5190 state->flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB_META;
5191 }
5192
5193 void radv_initialize_fmask(struct radv_cmd_buffer *cmd_buffer,
5194 struct radv_image *image,
5195 const VkImageSubresourceRange *range)
5196 {
5197 struct radv_cmd_state *state = &cmd_buffer->state;
5198 static const uint32_t fmask_clear_values[4] = {
5199 0x00000000,
5200 0x02020202,
5201 0xE4E4E4E4,
5202 0x76543210
5203 };
5204 uint32_t log2_samples = util_logbase2(image->info.samples);
5205 uint32_t value = fmask_clear_values[log2_samples];
5206
5207 state->flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB |
5208 RADV_CMD_FLAG_FLUSH_AND_INV_CB_META;
5209
5210 state->flush_bits |= radv_clear_fmask(cmd_buffer, image, range, value);
5211
5212 state->flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB_META;
5213 }
5214
5215 void radv_initialize_dcc(struct radv_cmd_buffer *cmd_buffer,
5216 struct radv_image *image,
5217 const VkImageSubresourceRange *range, uint32_t value)
5218 {
5219 struct radv_cmd_state *state = &cmd_buffer->state;
5220 unsigned size = 0;
5221
5222 state->flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB |
5223 RADV_CMD_FLAG_FLUSH_AND_INV_CB_META;
5224
5225 state->flush_bits |= radv_clear_dcc(cmd_buffer, image, range, value);
5226
5227 if (cmd_buffer->device->physical_device->rad_info.chip_class == GFX8) {
5228 /* When DCC is enabled with mipmaps, some levels might not
5229 * support fast clears and we have to initialize them as "fully
5230 * expanded".
5231 */
5232 /* Compute the size of all fast clearable DCC levels. */
5233 for (unsigned i = 0; i < image->planes[0].surface.num_dcc_levels; i++) {
5234 struct legacy_surf_level *surf_level =
5235 &image->planes[0].surface.u.legacy.level[i];
5236 unsigned dcc_fast_clear_size =
5237 surf_level->dcc_slice_fast_clear_size * image->info.array_size;
5238
5239 if (!dcc_fast_clear_size)
5240 break;
5241
5242 size = surf_level->dcc_offset + dcc_fast_clear_size;
5243 }
5244
5245 /* Initialize the mipmap levels without DCC. */
5246 if (size != image->planes[0].surface.dcc_size) {
5247 state->flush_bits |=
5248 radv_fill_buffer(cmd_buffer, image->bo,
5249 image->offset + image->dcc_offset + size,
5250 image->planes[0].surface.dcc_size - size,
5251 0xffffffff);
5252 }
5253 }
5254
5255 state->flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB |
5256 RADV_CMD_FLAG_FLUSH_AND_INV_CB_META;
5257 }
5258
5259 /**
5260 * Initialize DCC/FMASK/CMASK metadata for a color image.
5261 */
5262 static void radv_init_color_image_metadata(struct radv_cmd_buffer *cmd_buffer,
5263 struct radv_image *image,
5264 VkImageLayout src_layout,
5265 bool src_render_loop,
5266 VkImageLayout dst_layout,
5267 bool dst_render_loop,
5268 unsigned src_queue_mask,
5269 unsigned dst_queue_mask,
5270 const VkImageSubresourceRange *range)
5271 {
5272 if (radv_image_has_cmask(image)) {
5273 uint32_t value = 0xffffffffu; /* Fully expanded mode. */
5274
5275 /* TODO: clarify this. */
5276 if (radv_image_has_fmask(image)) {
5277 value = 0xccccccccu;
5278 }
5279
5280 radv_initialise_cmask(cmd_buffer, image, range, value);
5281 }
5282
5283 if (radv_image_has_fmask(image)) {
5284 radv_initialize_fmask(cmd_buffer, image, range);
5285 }
5286
5287 if (radv_dcc_enabled(image, range->baseMipLevel)) {
5288 uint32_t value = 0xffffffffu; /* Fully expanded mode. */
5289 bool need_decompress_pass = false;
5290
5291 if (radv_layout_dcc_compressed(cmd_buffer->device, image, dst_layout,
5292 dst_render_loop,
5293 dst_queue_mask)) {
5294 value = 0x20202020u;
5295 need_decompress_pass = true;
5296 }
5297
5298 radv_initialize_dcc(cmd_buffer, image, range, value);
5299
5300 radv_update_fce_metadata(cmd_buffer, image, range,
5301 need_decompress_pass);
5302 }
5303
5304 if (radv_image_has_cmask(image) ||
5305 radv_dcc_enabled(image, range->baseMipLevel)) {
5306 uint32_t color_values[2] = {};
5307 radv_set_color_clear_metadata(cmd_buffer, image, range,
5308 color_values);
5309 }
5310 }
5311
5312 /**
5313 * Handle color image transitions for DCC/FMASK/CMASK.
5314 */
5315 static void radv_handle_color_image_transition(struct radv_cmd_buffer *cmd_buffer,
5316 struct radv_image *image,
5317 VkImageLayout src_layout,
5318 bool src_render_loop,
5319 VkImageLayout dst_layout,
5320 bool dst_render_loop,
5321 unsigned src_queue_mask,
5322 unsigned dst_queue_mask,
5323 const VkImageSubresourceRange *range)
5324 {
5325 if (src_layout == VK_IMAGE_LAYOUT_UNDEFINED) {
5326 radv_init_color_image_metadata(cmd_buffer, image,
5327 src_layout, src_render_loop,
5328 dst_layout, dst_render_loop,
5329 src_queue_mask, dst_queue_mask,
5330 range);
5331 return;
5332 }
5333
5334 if (radv_dcc_enabled(image, range->baseMipLevel)) {
5335 if (src_layout == VK_IMAGE_LAYOUT_PREINITIALIZED) {
5336 radv_initialize_dcc(cmd_buffer, image, range, 0xffffffffu);
5337 } else if (radv_layout_dcc_compressed(cmd_buffer->device, image, src_layout, src_render_loop, src_queue_mask) &&
5338 !radv_layout_dcc_compressed(cmd_buffer->device, image, dst_layout, dst_render_loop, dst_queue_mask)) {
5339 radv_decompress_dcc(cmd_buffer, image, range);
5340 } else if (radv_layout_can_fast_clear(image, src_layout, src_render_loop, src_queue_mask) &&
5341 !radv_layout_can_fast_clear(image, dst_layout, dst_render_loop, dst_queue_mask)) {
5342 radv_fast_clear_flush_image_inplace(cmd_buffer, image, range);
5343 }
5344 } else if (radv_image_has_cmask(image) || radv_image_has_fmask(image)) {
5345 bool fce_eliminate = false, fmask_expand = false;
5346
5347 if (radv_layout_can_fast_clear(image, src_layout, src_render_loop, src_queue_mask) &&
5348 !radv_layout_can_fast_clear(image, dst_layout, dst_render_loop, dst_queue_mask)) {
5349 fce_eliminate = true;
5350 }
5351
5352 if (radv_image_has_fmask(image)) {
5353 if (src_layout != VK_IMAGE_LAYOUT_GENERAL &&
5354 dst_layout == VK_IMAGE_LAYOUT_GENERAL) {
5355 /* A FMASK decompress is required before doing
5356 * a MSAA decompress using FMASK.
5357 */
5358 fmask_expand = true;
5359 }
5360 }
5361
5362 if (fce_eliminate || fmask_expand)
5363 radv_fast_clear_flush_image_inplace(cmd_buffer, image, range);
5364
5365 if (fmask_expand)
5366 radv_expand_fmask_image_inplace(cmd_buffer, image, range);
5367 }
5368 }
5369
5370 static void radv_handle_image_transition(struct radv_cmd_buffer *cmd_buffer,
5371 struct radv_image *image,
5372 VkImageLayout src_layout,
5373 bool src_render_loop,
5374 VkImageLayout dst_layout,
5375 bool dst_render_loop,
5376 uint32_t src_family,
5377 uint32_t dst_family,
5378 const VkImageSubresourceRange *range,
5379 struct radv_sample_locations_state *sample_locs)
5380 {
5381 if (image->exclusive && src_family != dst_family) {
5382 /* This is an acquire or a release operation and there will be
5383 * a corresponding release/acquire. Do the transition in the
5384 * most flexible queue. */
5385
5386 assert(src_family == cmd_buffer->queue_family_index ||
5387 dst_family == cmd_buffer->queue_family_index);
5388
5389 if (src_family == VK_QUEUE_FAMILY_EXTERNAL ||
5390 src_family == VK_QUEUE_FAMILY_FOREIGN_EXT)
5391 return;
5392
5393 if (cmd_buffer->queue_family_index == RADV_QUEUE_TRANSFER)
5394 return;
5395
5396 if (cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE &&
5397 (src_family == RADV_QUEUE_GENERAL ||
5398 dst_family == RADV_QUEUE_GENERAL))
5399 return;
5400 }
5401
5402 if (src_layout == dst_layout)
5403 return;
5404
5405 unsigned src_queue_mask =
5406 radv_image_queue_family_mask(image, src_family,
5407 cmd_buffer->queue_family_index);
5408 unsigned dst_queue_mask =
5409 radv_image_queue_family_mask(image, dst_family,
5410 cmd_buffer->queue_family_index);
5411
5412 if (vk_format_is_depth(image->vk_format)) {
5413 radv_handle_depth_image_transition(cmd_buffer, image,
5414 src_layout, src_render_loop,
5415 dst_layout, dst_render_loop,
5416 src_queue_mask, dst_queue_mask,
5417 range, sample_locs);
5418 } else {
5419 radv_handle_color_image_transition(cmd_buffer, image,
5420 src_layout, src_render_loop,
5421 dst_layout, dst_render_loop,
5422 src_queue_mask, dst_queue_mask,
5423 range);
5424 }
5425 }
5426
5427 struct radv_barrier_info {
5428 uint32_t eventCount;
5429 const VkEvent *pEvents;
5430 VkPipelineStageFlags srcStageMask;
5431 VkPipelineStageFlags dstStageMask;
5432 };
5433
5434 static void
5435 radv_barrier(struct radv_cmd_buffer *cmd_buffer,
5436 uint32_t memoryBarrierCount,
5437 const VkMemoryBarrier *pMemoryBarriers,
5438 uint32_t bufferMemoryBarrierCount,
5439 const VkBufferMemoryBarrier *pBufferMemoryBarriers,
5440 uint32_t imageMemoryBarrierCount,
5441 const VkImageMemoryBarrier *pImageMemoryBarriers,
5442 const struct radv_barrier_info *info)
5443 {
5444 struct radeon_cmdbuf *cs = cmd_buffer->cs;
5445 enum radv_cmd_flush_bits src_flush_bits = 0;
5446 enum radv_cmd_flush_bits dst_flush_bits = 0;
5447
5448 for (unsigned i = 0; i < info->eventCount; ++i) {
5449 RADV_FROM_HANDLE(radv_event, event, info->pEvents[i]);
5450 uint64_t va = radv_buffer_get_va(event->bo);
5451
5452 radv_cs_add_buffer(cmd_buffer->device->ws, cs, event->bo);
5453
5454 ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, 7);
5455
5456 radv_cp_wait_mem(cs, WAIT_REG_MEM_EQUAL, va, 1, 0xffffffff);
5457 assert(cmd_buffer->cs->cdw <= cdw_max);
5458 }
5459
5460 for (uint32_t i = 0; i < memoryBarrierCount; i++) {
5461 src_flush_bits |= radv_src_access_flush(cmd_buffer, pMemoryBarriers[i].srcAccessMask,
5462 NULL);
5463 dst_flush_bits |= radv_dst_access_flush(cmd_buffer, pMemoryBarriers[i].dstAccessMask,
5464 NULL);
5465 }
5466
5467 for (uint32_t i = 0; i < bufferMemoryBarrierCount; i++) {
5468 src_flush_bits |= radv_src_access_flush(cmd_buffer, pBufferMemoryBarriers[i].srcAccessMask,
5469 NULL);
5470 dst_flush_bits |= radv_dst_access_flush(cmd_buffer, pBufferMemoryBarriers[i].dstAccessMask,
5471 NULL);
5472 }
5473
5474 for (uint32_t i = 0; i < imageMemoryBarrierCount; i++) {
5475 RADV_FROM_HANDLE(radv_image, image, pImageMemoryBarriers[i].image);
5476
5477 src_flush_bits |= radv_src_access_flush(cmd_buffer, pImageMemoryBarriers[i].srcAccessMask,
5478 image);
5479 dst_flush_bits |= radv_dst_access_flush(cmd_buffer, pImageMemoryBarriers[i].dstAccessMask,
5480 image);
5481 }
5482
5483 /* The Vulkan spec 1.1.98 says:
5484 *
5485 * "An execution dependency with only
5486 * VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT in the destination stage mask
5487 * will only prevent that stage from executing in subsequently
5488 * submitted commands. As this stage does not perform any actual
5489 * execution, this is not observable - in effect, it does not delay
5490 * processing of subsequent commands. Similarly an execution dependency
5491 * with only VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT in the source stage mask
5492 * will effectively not wait for any prior commands to complete."
5493 */
5494 if (info->dstStageMask != VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT)
5495 radv_stage_flush(cmd_buffer, info->srcStageMask);
5496 cmd_buffer->state.flush_bits |= src_flush_bits;
5497
5498 for (uint32_t i = 0; i < imageMemoryBarrierCount; i++) {
5499 RADV_FROM_HANDLE(radv_image, image, pImageMemoryBarriers[i].image);
5500
5501 const struct VkSampleLocationsInfoEXT *sample_locs_info =
5502 vk_find_struct_const(pImageMemoryBarriers[i].pNext,
5503 SAMPLE_LOCATIONS_INFO_EXT);
5504 struct radv_sample_locations_state sample_locations = {};
5505
5506 if (sample_locs_info) {
5507 assert(image->flags & VK_IMAGE_CREATE_SAMPLE_LOCATIONS_COMPATIBLE_DEPTH_BIT_EXT);
5508 sample_locations.per_pixel = sample_locs_info->sampleLocationsPerPixel;
5509 sample_locations.grid_size = sample_locs_info->sampleLocationGridSize;
5510 sample_locations.count = sample_locs_info->sampleLocationsCount;
5511 typed_memcpy(&sample_locations.locations[0],
5512 sample_locs_info->pSampleLocations,
5513 sample_locs_info->sampleLocationsCount);
5514 }
5515
5516 radv_handle_image_transition(cmd_buffer, image,
5517 pImageMemoryBarriers[i].oldLayout,
5518 false, /* Outside of a renderpass we are never in a renderloop */
5519 pImageMemoryBarriers[i].newLayout,
5520 false, /* Outside of a renderpass we are never in a renderloop */
5521 pImageMemoryBarriers[i].srcQueueFamilyIndex,
5522 pImageMemoryBarriers[i].dstQueueFamilyIndex,
5523 &pImageMemoryBarriers[i].subresourceRange,
5524 sample_locs_info ? &sample_locations : NULL);
5525 }
5526
5527 /* Make sure CP DMA is idle because the driver might have performed a
5528 * DMA operation for copying or filling buffers/images.
5529 */
5530 if (info->srcStageMask & (VK_PIPELINE_STAGE_TRANSFER_BIT |
5531 VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT))
5532 si_cp_dma_wait_for_idle(cmd_buffer);
5533
5534 cmd_buffer->state.flush_bits |= dst_flush_bits;
5535 }
5536
5537 void radv_CmdPipelineBarrier(
5538 VkCommandBuffer commandBuffer,
5539 VkPipelineStageFlags srcStageMask,
5540 VkPipelineStageFlags destStageMask,
5541 VkBool32 byRegion,
5542 uint32_t memoryBarrierCount,
5543 const VkMemoryBarrier* pMemoryBarriers,
5544 uint32_t bufferMemoryBarrierCount,
5545 const VkBufferMemoryBarrier* pBufferMemoryBarriers,
5546 uint32_t imageMemoryBarrierCount,
5547 const VkImageMemoryBarrier* pImageMemoryBarriers)
5548 {
5549 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
5550 struct radv_barrier_info info;
5551
5552 info.eventCount = 0;
5553 info.pEvents = NULL;
5554 info.srcStageMask = srcStageMask;
5555 info.dstStageMask = destStageMask;
5556
5557 radv_barrier(cmd_buffer, memoryBarrierCount, pMemoryBarriers,
5558 bufferMemoryBarrierCount, pBufferMemoryBarriers,
5559 imageMemoryBarrierCount, pImageMemoryBarriers, &info);
5560 }
5561
5562
5563 static void write_event(struct radv_cmd_buffer *cmd_buffer,
5564 struct radv_event *event,
5565 VkPipelineStageFlags stageMask,
5566 unsigned value)
5567 {
5568 struct radeon_cmdbuf *cs = cmd_buffer->cs;
5569 uint64_t va = radv_buffer_get_va(event->bo);
5570
5571 si_emit_cache_flush(cmd_buffer);
5572
5573 radv_cs_add_buffer(cmd_buffer->device->ws, cs, event->bo);
5574
5575 ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, 21);
5576
5577 /* Flags that only require a top-of-pipe event. */
5578 VkPipelineStageFlags top_of_pipe_flags =
5579 VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT;
5580
5581 /* Flags that only require a post-index-fetch event. */
5582 VkPipelineStageFlags post_index_fetch_flags =
5583 top_of_pipe_flags |
5584 VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT |
5585 VK_PIPELINE_STAGE_VERTEX_INPUT_BIT;
5586
5587 /* Make sure CP DMA is idle because the driver might have performed a
5588 * DMA operation for copying or filling buffers/images.
5589 */
5590 if (stageMask & (VK_PIPELINE_STAGE_TRANSFER_BIT |
5591 VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT))
5592 si_cp_dma_wait_for_idle(cmd_buffer);
5593
5594 /* TODO: Emit EOS events for syncing PS/CS stages. */
5595
5596 if (!(stageMask & ~top_of_pipe_flags)) {
5597 /* Just need to sync the PFP engine. */
5598 radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 3, 0));
5599 radeon_emit(cs, S_370_DST_SEL(V_370_MEM) |
5600 S_370_WR_CONFIRM(1) |
5601 S_370_ENGINE_SEL(V_370_PFP));
5602 radeon_emit(cs, va);
5603 radeon_emit(cs, va >> 32);
5604 radeon_emit(cs, value);
5605 } else if (!(stageMask & ~post_index_fetch_flags)) {
5606 /* Sync ME because PFP reads index and indirect buffers. */
5607 radeon_emit(cs, PKT3(PKT3_WRITE_DATA, 3, 0));
5608 radeon_emit(cs, S_370_DST_SEL(V_370_MEM) |
5609 S_370_WR_CONFIRM(1) |
5610 S_370_ENGINE_SEL(V_370_ME));
5611 radeon_emit(cs, va);
5612 radeon_emit(cs, va >> 32);
5613 radeon_emit(cs, value);
5614 } else {
5615 /* Otherwise, sync all prior GPU work using an EOP event. */
5616 si_cs_emit_write_event_eop(cs,
5617 cmd_buffer->device->physical_device->rad_info.chip_class,
5618 radv_cmd_buffer_uses_mec(cmd_buffer),
5619 V_028A90_BOTTOM_OF_PIPE_TS, 0,
5620 EOP_DST_SEL_MEM,
5621 EOP_DATA_SEL_VALUE_32BIT, va, value,
5622 cmd_buffer->gfx9_eop_bug_va);
5623 }
5624
5625 assert(cmd_buffer->cs->cdw <= cdw_max);
5626 }
5627
5628 void radv_CmdSetEvent(VkCommandBuffer commandBuffer,
5629 VkEvent _event,
5630 VkPipelineStageFlags stageMask)
5631 {
5632 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
5633 RADV_FROM_HANDLE(radv_event, event, _event);
5634
5635 write_event(cmd_buffer, event, stageMask, 1);
5636 }
5637
5638 void radv_CmdResetEvent(VkCommandBuffer commandBuffer,
5639 VkEvent _event,
5640 VkPipelineStageFlags stageMask)
5641 {
5642 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
5643 RADV_FROM_HANDLE(radv_event, event, _event);
5644
5645 write_event(cmd_buffer, event, stageMask, 0);
5646 }
5647
5648 void radv_CmdWaitEvents(VkCommandBuffer commandBuffer,
5649 uint32_t eventCount,
5650 const VkEvent* pEvents,
5651 VkPipelineStageFlags srcStageMask,
5652 VkPipelineStageFlags dstStageMask,
5653 uint32_t memoryBarrierCount,
5654 const VkMemoryBarrier* pMemoryBarriers,
5655 uint32_t bufferMemoryBarrierCount,
5656 const VkBufferMemoryBarrier* pBufferMemoryBarriers,
5657 uint32_t imageMemoryBarrierCount,
5658 const VkImageMemoryBarrier* pImageMemoryBarriers)
5659 {
5660 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
5661 struct radv_barrier_info info;
5662
5663 info.eventCount = eventCount;
5664 info.pEvents = pEvents;
5665 info.srcStageMask = 0;
5666
5667 radv_barrier(cmd_buffer, memoryBarrierCount, pMemoryBarriers,
5668 bufferMemoryBarrierCount, pBufferMemoryBarriers,
5669 imageMemoryBarrierCount, pImageMemoryBarriers, &info);
5670 }
5671
5672
5673 void radv_CmdSetDeviceMask(VkCommandBuffer commandBuffer,
5674 uint32_t deviceMask)
5675 {
5676 /* No-op */
5677 }
5678
5679 /* VK_EXT_conditional_rendering */
5680 void radv_CmdBeginConditionalRenderingEXT(
5681 VkCommandBuffer commandBuffer,
5682 const VkConditionalRenderingBeginInfoEXT* pConditionalRenderingBegin)
5683 {
5684 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
5685 RADV_FROM_HANDLE(radv_buffer, buffer, pConditionalRenderingBegin->buffer);
5686 struct radeon_cmdbuf *cs = cmd_buffer->cs;
5687 bool draw_visible = true;
5688 uint64_t pred_value = 0;
5689 uint64_t va, new_va;
5690 unsigned pred_offset;
5691
5692 va = radv_buffer_get_va(buffer->bo) + pConditionalRenderingBegin->offset;
5693
5694 /* By default, if the 32-bit value at offset in buffer memory is zero,
5695 * then the rendering commands are discarded, otherwise they are
5696 * executed as normal. If the inverted flag is set, all commands are
5697 * discarded if the value is non zero.
5698 */
5699 if (pConditionalRenderingBegin->flags &
5700 VK_CONDITIONAL_RENDERING_INVERTED_BIT_EXT) {
5701 draw_visible = false;
5702 }
5703
5704 si_emit_cache_flush(cmd_buffer);
5705
5706 /* From the Vulkan spec 1.1.107:
5707 *
5708 * "If the 32-bit value at offset in buffer memory is zero, then the
5709 * rendering commands are discarded, otherwise they are executed as
5710 * normal. If the value of the predicate in buffer memory changes while
5711 * conditional rendering is active, the rendering commands may be
5712 * discarded in an implementation-dependent way. Some implementations
5713 * may latch the value of the predicate upon beginning conditional
5714 * rendering while others may read it before every rendering command."
5715 *
5716 * But, the AMD hardware treats the predicate as a 64-bit value which
5717 * means we need a workaround in the driver. Luckily, it's not required
5718 * to support if the value changes when predication is active.
5719 *
5720 * The workaround is as follows:
5721 * 1) allocate a 64-value in the upload BO and initialize it to 0
5722 * 2) copy the 32-bit predicate value to the upload BO
5723 * 3) use the new allocated VA address for predication
5724 *
5725 * Based on the conditionalrender demo, it's faster to do the COPY_DATA
5726 * in ME (+ sync PFP) instead of PFP.
5727 */
5728 radv_cmd_buffer_upload_data(cmd_buffer, 8, 16, &pred_value, &pred_offset);
5729
5730 new_va = radv_buffer_get_va(cmd_buffer->upload.upload_bo) + pred_offset;
5731
5732 radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
5733 radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) |
5734 COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) |
5735 COPY_DATA_WR_CONFIRM);
5736 radeon_emit(cs, va);
5737 radeon_emit(cs, va >> 32);
5738 radeon_emit(cs, new_va);
5739 radeon_emit(cs, new_va >> 32);
5740
5741 radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, 0));
5742 radeon_emit(cs, 0);
5743
5744 /* Enable predication for this command buffer. */
5745 si_emit_set_predication_state(cmd_buffer, draw_visible, new_va);
5746 cmd_buffer->state.predicating = true;
5747
5748 /* Store conditional rendering user info. */
5749 cmd_buffer->state.predication_type = draw_visible;
5750 cmd_buffer->state.predication_va = new_va;
5751 }
5752
5753 void radv_CmdEndConditionalRenderingEXT(
5754 VkCommandBuffer commandBuffer)
5755 {
5756 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
5757
5758 /* Disable predication for this command buffer. */
5759 si_emit_set_predication_state(cmd_buffer, false, 0);
5760 cmd_buffer->state.predicating = false;
5761
5762 /* Reset conditional rendering user info. */
5763 cmd_buffer->state.predication_type = -1;
5764 cmd_buffer->state.predication_va = 0;
5765 }
5766
5767 /* VK_EXT_transform_feedback */
5768 void radv_CmdBindTransformFeedbackBuffersEXT(
5769 VkCommandBuffer commandBuffer,
5770 uint32_t firstBinding,
5771 uint32_t bindingCount,
5772 const VkBuffer* pBuffers,
5773 const VkDeviceSize* pOffsets,
5774 const VkDeviceSize* pSizes)
5775 {
5776 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
5777 struct radv_streamout_binding *sb = cmd_buffer->streamout_bindings;
5778 uint8_t enabled_mask = 0;
5779
5780 assert(firstBinding + bindingCount <= MAX_SO_BUFFERS);
5781 for (uint32_t i = 0; i < bindingCount; i++) {
5782 uint32_t idx = firstBinding + i;
5783
5784 sb[idx].buffer = radv_buffer_from_handle(pBuffers[i]);
5785 sb[idx].offset = pOffsets[i];
5786 sb[idx].size = pSizes[i];
5787
5788 radv_cs_add_buffer(cmd_buffer->device->ws, cmd_buffer->cs,
5789 sb[idx].buffer->bo);
5790
5791 enabled_mask |= 1 << idx;
5792 }
5793
5794 cmd_buffer->state.streamout.enabled_mask |= enabled_mask;
5795
5796 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_STREAMOUT_BUFFER;
5797 }
5798
5799 static void
5800 radv_emit_streamout_enable(struct radv_cmd_buffer *cmd_buffer)
5801 {
5802 struct radv_streamout_state *so = &cmd_buffer->state.streamout;
5803 struct radeon_cmdbuf *cs = cmd_buffer->cs;
5804
5805 radeon_set_context_reg_seq(cs, R_028B94_VGT_STRMOUT_CONFIG, 2);
5806 radeon_emit(cs,
5807 S_028B94_STREAMOUT_0_EN(so->streamout_enabled) |
5808 S_028B94_RAST_STREAM(0) |
5809 S_028B94_STREAMOUT_1_EN(so->streamout_enabled) |
5810 S_028B94_STREAMOUT_2_EN(so->streamout_enabled) |
5811 S_028B94_STREAMOUT_3_EN(so->streamout_enabled));
5812 radeon_emit(cs, so->hw_enabled_mask &
5813 so->enabled_stream_buffers_mask);
5814
5815 cmd_buffer->state.context_roll_without_scissor_emitted = true;
5816 }
5817
5818 static void
5819 radv_set_streamout_enable(struct radv_cmd_buffer *cmd_buffer, bool enable)
5820 {
5821 struct radv_streamout_state *so = &cmd_buffer->state.streamout;
5822 bool old_streamout_enabled = so->streamout_enabled;
5823 uint32_t old_hw_enabled_mask = so->hw_enabled_mask;
5824
5825 so->streamout_enabled = enable;
5826
5827 so->hw_enabled_mask = so->enabled_mask |
5828 (so->enabled_mask << 4) |
5829 (so->enabled_mask << 8) |
5830 (so->enabled_mask << 12);
5831
5832 if (!cmd_buffer->device->physical_device->use_ngg_streamout &&
5833 ((old_streamout_enabled != so->streamout_enabled) ||
5834 (old_hw_enabled_mask != so->hw_enabled_mask)))
5835 radv_emit_streamout_enable(cmd_buffer);
5836
5837 if (cmd_buffer->device->physical_device->use_ngg_streamout)
5838 cmd_buffer->gds_needed = true;
5839 }
5840
5841 static void radv_flush_vgt_streamout(struct radv_cmd_buffer *cmd_buffer)
5842 {
5843 struct radeon_cmdbuf *cs = cmd_buffer->cs;
5844 unsigned reg_strmout_cntl;
5845
5846 /* The register is at different places on different ASICs. */
5847 if (cmd_buffer->device->physical_device->rad_info.chip_class >= GFX7) {
5848 reg_strmout_cntl = R_0300FC_CP_STRMOUT_CNTL;
5849 radeon_set_uconfig_reg(cs, reg_strmout_cntl, 0);
5850 } else {
5851 reg_strmout_cntl = R_0084FC_CP_STRMOUT_CNTL;
5852 radeon_set_config_reg(cs, reg_strmout_cntl, 0);
5853 }
5854
5855 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
5856 radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_SO_VGTSTREAMOUT_FLUSH) | EVENT_INDEX(0));
5857
5858 radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0));
5859 radeon_emit(cs, WAIT_REG_MEM_EQUAL); /* wait until the register is equal to the reference value */
5860 radeon_emit(cs, reg_strmout_cntl >> 2); /* register */
5861 radeon_emit(cs, 0);
5862 radeon_emit(cs, S_0084FC_OFFSET_UPDATE_DONE(1)); /* reference value */
5863 radeon_emit(cs, S_0084FC_OFFSET_UPDATE_DONE(1)); /* mask */
5864 radeon_emit(cs, 4); /* poll interval */
5865 }
5866
5867 static void
5868 radv_emit_streamout_begin(struct radv_cmd_buffer *cmd_buffer,
5869 uint32_t firstCounterBuffer,
5870 uint32_t counterBufferCount,
5871 const VkBuffer *pCounterBuffers,
5872 const VkDeviceSize *pCounterBufferOffsets)
5873
5874 {
5875 struct radv_streamout_binding *sb = cmd_buffer->streamout_bindings;
5876 struct radv_streamout_state *so = &cmd_buffer->state.streamout;
5877 struct radeon_cmdbuf *cs = cmd_buffer->cs;
5878 uint32_t i;
5879
5880 radv_flush_vgt_streamout(cmd_buffer);
5881
5882 assert(firstCounterBuffer + counterBufferCount <= MAX_SO_BUFFERS);
5883 for_each_bit(i, so->enabled_mask) {
5884 int32_t counter_buffer_idx = i - firstCounterBuffer;
5885 if (counter_buffer_idx >= 0 && counter_buffer_idx >= counterBufferCount)
5886 counter_buffer_idx = -1;
5887
5888 /* AMD GCN binds streamout buffers as shader resources.
5889 * VGT only counts primitives and tells the shader through
5890 * SGPRs what to do.
5891 */
5892 radeon_set_context_reg_seq(cs, R_028AD0_VGT_STRMOUT_BUFFER_SIZE_0 + 16*i, 2);
5893 radeon_emit(cs, sb[i].size >> 2); /* BUFFER_SIZE (in DW) */
5894 radeon_emit(cs, so->stride_in_dw[i]); /* VTX_STRIDE (in DW) */
5895
5896 cmd_buffer->state.context_roll_without_scissor_emitted = true;
5897
5898 if (counter_buffer_idx >= 0 && pCounterBuffers && pCounterBuffers[counter_buffer_idx]) {
5899 /* The array of counter buffers is optional. */
5900 RADV_FROM_HANDLE(radv_buffer, buffer, pCounterBuffers[counter_buffer_idx]);
5901 uint64_t va = radv_buffer_get_va(buffer->bo);
5902
5903 va += buffer->offset + pCounterBufferOffsets[counter_buffer_idx];
5904
5905 /* Append */
5906 radeon_emit(cs, PKT3(PKT3_STRMOUT_BUFFER_UPDATE, 4, 0));
5907 radeon_emit(cs, STRMOUT_SELECT_BUFFER(i) |
5908 STRMOUT_DATA_TYPE(1) | /* offset in bytes */
5909 STRMOUT_OFFSET_SOURCE(STRMOUT_OFFSET_FROM_MEM)); /* control */
5910 radeon_emit(cs, 0); /* unused */
5911 radeon_emit(cs, 0); /* unused */
5912 radeon_emit(cs, va); /* src address lo */
5913 radeon_emit(cs, va >> 32); /* src address hi */
5914
5915 radv_cs_add_buffer(cmd_buffer->device->ws, cs, buffer->bo);
5916 } else {
5917 /* Start from the beginning. */
5918 radeon_emit(cs, PKT3(PKT3_STRMOUT_BUFFER_UPDATE, 4, 0));
5919 radeon_emit(cs, STRMOUT_SELECT_BUFFER(i) |
5920 STRMOUT_DATA_TYPE(1) | /* offset in bytes */
5921 STRMOUT_OFFSET_SOURCE(STRMOUT_OFFSET_FROM_PACKET)); /* control */
5922 radeon_emit(cs, 0); /* unused */
5923 radeon_emit(cs, 0); /* unused */
5924 radeon_emit(cs, 0); /* unused */
5925 radeon_emit(cs, 0); /* unused */
5926 }
5927 }
5928
5929 radv_set_streamout_enable(cmd_buffer, true);
5930 }
5931
5932 static void
5933 gfx10_emit_streamout_begin(struct radv_cmd_buffer *cmd_buffer,
5934 uint32_t firstCounterBuffer,
5935 uint32_t counterBufferCount,
5936 const VkBuffer *pCounterBuffers,
5937 const VkDeviceSize *pCounterBufferOffsets)
5938 {
5939 struct radv_streamout_state *so = &cmd_buffer->state.streamout;
5940 unsigned last_target = util_last_bit(so->enabled_mask) - 1;
5941 struct radeon_cmdbuf *cs = cmd_buffer->cs;
5942 uint32_t i;
5943
5944 assert(cmd_buffer->device->physical_device->rad_info.chip_class >= GFX10);
5945 assert(firstCounterBuffer + counterBufferCount <= MAX_SO_BUFFERS);
5946
5947 /* Sync because the next streamout operation will overwrite GDS and we
5948 * have to make sure it's idle.
5949 * TODO: Improve by tracking if there is a streamout operation in
5950 * flight.
5951 */
5952 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_VS_PARTIAL_FLUSH;
5953 si_emit_cache_flush(cmd_buffer);
5954
5955 for_each_bit(i, so->enabled_mask) {
5956 int32_t counter_buffer_idx = i - firstCounterBuffer;
5957 if (counter_buffer_idx >= 0 && counter_buffer_idx >= counterBufferCount)
5958 counter_buffer_idx = -1;
5959
5960 bool append = counter_buffer_idx >= 0 &&
5961 pCounterBuffers && pCounterBuffers[counter_buffer_idx];
5962 uint64_t va = 0;
5963
5964 if (append) {
5965 RADV_FROM_HANDLE(radv_buffer, buffer, pCounterBuffers[counter_buffer_idx]);
5966
5967 va += radv_buffer_get_va(buffer->bo);
5968 va += buffer->offset + pCounterBufferOffsets[counter_buffer_idx];
5969
5970 radv_cs_add_buffer(cmd_buffer->device->ws, cs, buffer->bo);
5971 }
5972
5973 radeon_emit(cs, PKT3(PKT3_DMA_DATA, 5, 0));
5974 radeon_emit(cs, S_411_SRC_SEL(append ? V_411_SRC_ADDR_TC_L2 : V_411_DATA) |
5975 S_411_DST_SEL(V_411_GDS) |
5976 S_411_CP_SYNC(i == last_target));
5977 radeon_emit(cs, va);
5978 radeon_emit(cs, va >> 32);
5979 radeon_emit(cs, 4 * i); /* destination in GDS */
5980 radeon_emit(cs, 0);
5981 radeon_emit(cs, S_414_BYTE_COUNT_GFX9(4) |
5982 S_414_DISABLE_WR_CONFIRM_GFX9(i != last_target));
5983 }
5984
5985 radv_set_streamout_enable(cmd_buffer, true);
5986 }
5987
5988 void radv_CmdBeginTransformFeedbackEXT(
5989 VkCommandBuffer commandBuffer,
5990 uint32_t firstCounterBuffer,
5991 uint32_t counterBufferCount,
5992 const VkBuffer* pCounterBuffers,
5993 const VkDeviceSize* pCounterBufferOffsets)
5994 {
5995 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
5996
5997 if (cmd_buffer->device->physical_device->use_ngg_streamout) {
5998 gfx10_emit_streamout_begin(cmd_buffer,
5999 firstCounterBuffer, counterBufferCount,
6000 pCounterBuffers, pCounterBufferOffsets);
6001 } else {
6002 radv_emit_streamout_begin(cmd_buffer,
6003 firstCounterBuffer, counterBufferCount,
6004 pCounterBuffers, pCounterBufferOffsets);
6005 }
6006 }
6007
6008 static void
6009 radv_emit_streamout_end(struct radv_cmd_buffer *cmd_buffer,
6010 uint32_t firstCounterBuffer,
6011 uint32_t counterBufferCount,
6012 const VkBuffer *pCounterBuffers,
6013 const VkDeviceSize *pCounterBufferOffsets)
6014 {
6015 struct radv_streamout_state *so = &cmd_buffer->state.streamout;
6016 struct radeon_cmdbuf *cs = cmd_buffer->cs;
6017 uint32_t i;
6018
6019 radv_flush_vgt_streamout(cmd_buffer);
6020
6021 assert(firstCounterBuffer + counterBufferCount <= MAX_SO_BUFFERS);
6022 for_each_bit(i, so->enabled_mask) {
6023 int32_t counter_buffer_idx = i - firstCounterBuffer;
6024 if (counter_buffer_idx >= 0 && counter_buffer_idx >= counterBufferCount)
6025 counter_buffer_idx = -1;
6026
6027 if (counter_buffer_idx >= 0 && pCounterBuffers && pCounterBuffers[counter_buffer_idx]) {
6028 /* The array of counters buffer is optional. */
6029 RADV_FROM_HANDLE(radv_buffer, buffer, pCounterBuffers[counter_buffer_idx]);
6030 uint64_t va = radv_buffer_get_va(buffer->bo);
6031
6032 va += buffer->offset + pCounterBufferOffsets[counter_buffer_idx];
6033
6034 radeon_emit(cs, PKT3(PKT3_STRMOUT_BUFFER_UPDATE, 4, 0));
6035 radeon_emit(cs, STRMOUT_SELECT_BUFFER(i) |
6036 STRMOUT_DATA_TYPE(1) | /* offset in bytes */
6037 STRMOUT_OFFSET_SOURCE(STRMOUT_OFFSET_NONE) |
6038 STRMOUT_STORE_BUFFER_FILLED_SIZE); /* control */
6039 radeon_emit(cs, va); /* dst address lo */
6040 radeon_emit(cs, va >> 32); /* dst address hi */
6041 radeon_emit(cs, 0); /* unused */
6042 radeon_emit(cs, 0); /* unused */
6043
6044 radv_cs_add_buffer(cmd_buffer->device->ws, cs, buffer->bo);
6045 }
6046
6047 /* Deactivate transform feedback by zeroing the buffer size.
6048 * The counters (primitives generated, primitives emitted) may
6049 * be enabled even if there is not buffer bound. This ensures
6050 * that the primitives-emitted query won't increment.
6051 */
6052 radeon_set_context_reg(cs, R_028AD0_VGT_STRMOUT_BUFFER_SIZE_0 + 16*i, 0);
6053
6054 cmd_buffer->state.context_roll_without_scissor_emitted = true;
6055 }
6056
6057 radv_set_streamout_enable(cmd_buffer, false);
6058 }
6059
6060 static void
6061 gfx10_emit_streamout_end(struct radv_cmd_buffer *cmd_buffer,
6062 uint32_t firstCounterBuffer,
6063 uint32_t counterBufferCount,
6064 const VkBuffer *pCounterBuffers,
6065 const VkDeviceSize *pCounterBufferOffsets)
6066 {
6067 struct radv_streamout_state *so = &cmd_buffer->state.streamout;
6068 struct radeon_cmdbuf *cs = cmd_buffer->cs;
6069 uint32_t i;
6070
6071 assert(cmd_buffer->device->physical_device->rad_info.chip_class >= GFX10);
6072 assert(firstCounterBuffer + counterBufferCount <= MAX_SO_BUFFERS);
6073
6074 for_each_bit(i, so->enabled_mask) {
6075 int32_t counter_buffer_idx = i - firstCounterBuffer;
6076 if (counter_buffer_idx >= 0 && counter_buffer_idx >= counterBufferCount)
6077 counter_buffer_idx = -1;
6078
6079 if (counter_buffer_idx >= 0 && pCounterBuffers && pCounterBuffers[counter_buffer_idx]) {
6080 /* The array of counters buffer is optional. */
6081 RADV_FROM_HANDLE(radv_buffer, buffer, pCounterBuffers[counter_buffer_idx]);
6082 uint64_t va = radv_buffer_get_va(buffer->bo);
6083
6084 va += buffer->offset + pCounterBufferOffsets[counter_buffer_idx];
6085
6086 si_cs_emit_write_event_eop(cs,
6087 cmd_buffer->device->physical_device->rad_info.chip_class,
6088 radv_cmd_buffer_uses_mec(cmd_buffer),
6089 V_028A90_PS_DONE, 0,
6090 EOP_DST_SEL_TC_L2,
6091 EOP_DATA_SEL_GDS,
6092 va, EOP_DATA_GDS(i, 1), 0);
6093 }
6094 }
6095
6096 radv_set_streamout_enable(cmd_buffer, false);
6097 }
6098
6099 void radv_CmdEndTransformFeedbackEXT(
6100 VkCommandBuffer commandBuffer,
6101 uint32_t firstCounterBuffer,
6102 uint32_t counterBufferCount,
6103 const VkBuffer* pCounterBuffers,
6104 const VkDeviceSize* pCounterBufferOffsets)
6105 {
6106 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
6107
6108 if (cmd_buffer->device->physical_device->use_ngg_streamout) {
6109 gfx10_emit_streamout_end(cmd_buffer,
6110 firstCounterBuffer, counterBufferCount,
6111 pCounterBuffers, pCounterBufferOffsets);
6112 } else {
6113 radv_emit_streamout_end(cmd_buffer,
6114 firstCounterBuffer, counterBufferCount,
6115 pCounterBuffers, pCounterBufferOffsets);
6116 }
6117 }
6118
6119 void radv_CmdDrawIndirectByteCountEXT(
6120 VkCommandBuffer commandBuffer,
6121 uint32_t instanceCount,
6122 uint32_t firstInstance,
6123 VkBuffer _counterBuffer,
6124 VkDeviceSize counterBufferOffset,
6125 uint32_t counterOffset,
6126 uint32_t vertexStride)
6127 {
6128 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
6129 RADV_FROM_HANDLE(radv_buffer, counterBuffer, _counterBuffer);
6130 struct radv_draw_info info = {};
6131
6132 info.instance_count = instanceCount;
6133 info.first_instance = firstInstance;
6134 info.strmout_buffer = counterBuffer;
6135 info.strmout_buffer_offset = counterBufferOffset;
6136 info.stride = vertexStride;
6137
6138 radv_draw(cmd_buffer, &info);
6139 }
6140
6141 /* VK_AMD_buffer_marker */
6142 void radv_CmdWriteBufferMarkerAMD(
6143 VkCommandBuffer commandBuffer,
6144 VkPipelineStageFlagBits pipelineStage,
6145 VkBuffer dstBuffer,
6146 VkDeviceSize dstOffset,
6147 uint32_t marker)
6148 {
6149 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
6150 RADV_FROM_HANDLE(radv_buffer, buffer, dstBuffer);
6151 struct radeon_cmdbuf *cs = cmd_buffer->cs;
6152 uint64_t va = radv_buffer_get_va(buffer->bo) + dstOffset;
6153
6154 si_emit_cache_flush(cmd_buffer);
6155
6156 if (!(pipelineStage & ~VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT)) {
6157 radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
6158 radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) |
6159 COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) |
6160 COPY_DATA_WR_CONFIRM);
6161 radeon_emit(cs, marker);
6162 radeon_emit(cs, 0);
6163 radeon_emit(cs, va);
6164 radeon_emit(cs, va >> 32);
6165 } else {
6166 si_cs_emit_write_event_eop(cs,
6167 cmd_buffer->device->physical_device->rad_info.chip_class,
6168 radv_cmd_buffer_uses_mec(cmd_buffer),
6169 V_028A90_BOTTOM_OF_PIPE_TS, 0,
6170 EOP_DST_SEL_MEM,
6171 EOP_DATA_SEL_VALUE_32BIT,
6172 va, marker,
6173 cmd_buffer->gfx9_eop_bug_va);
6174 }
6175 }