ac: declare an enum for the OOB select field on GFX10
[mesa.git] / src / gallium / drivers / radeonsi / si_descriptors.c
1 /*
2 * Copyright 2013 Advanced Micro Devices, Inc.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * on the rights to use, copy, modify, merge, publish, distribute, sub
9 * license, and/or sell copies of the Software, and to permit persons to whom
10 * the Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
23 */
24
25 /* Resource binding slots and sampler states (each described with 8 or
26 * 4 dwords) are stored in lists in memory which is accessed by shaders
27 * using scalar load instructions.
28 *
29 * This file is responsible for managing such lists. It keeps a copy of all
30 * descriptors in CPU memory and re-uploads a whole list if some slots have
31 * been changed.
32 *
33 * This code is also reponsible for updating shader pointers to those lists.
34 *
35 * Note that CP DMA can't be used for updating the lists, because a GPU hang
36 * could leave the list in a mid-IB state and the next IB would get wrong
37 * descriptors and the whole context would be unusable at that point.
38 * (Note: The register shadowing can't be used due to the same reason)
39 *
40 * Also, uploading descriptors to newly allocated memory doesn't require
41 * a KCACHE flush.
42 *
43 *
44 * Possible scenarios for one 16 dword image+sampler slot:
45 *
46 * | Image | w/ FMASK | Buffer | NULL
47 * [ 0: 3] Image[0:3] | Image[0:3] | Null[0:3] | Null[0:3]
48 * [ 4: 7] Image[4:7] | Image[4:7] | Buffer[0:3] | 0
49 * [ 8:11] Null[0:3] | Fmask[0:3] | Null[0:3] | Null[0:3]
50 * [12:15] Sampler[0:3] | Fmask[4:7] | Sampler[0:3] | Sampler[0:3]
51 *
52 * FMASK implies MSAA, therefore no sampler state.
53 * Sampler states are never unbound except when FMASK is bound.
54 */
55
56 #include "si_pipe.h"
57 #include "sid.h"
58
59 #include "util/hash_table.h"
60 #include "util/u_idalloc.h"
61 #include "util/format/u_format.h"
62 #include "util/u_memory.h"
63 #include "util/u_upload_mgr.h"
64
65
66 /* NULL image and buffer descriptor for textures (alpha = 1) and images
67 * (alpha = 0).
68 *
69 * For images, all fields must be zero except for the swizzle, which
70 * supports arbitrary combinations of 0s and 1s. The texture type must be
71 * any valid type (e.g. 1D). If the texture type isn't set, the hw hangs.
72 *
73 * For buffers, all fields must be zero. If they are not, the hw hangs.
74 *
75 * This is the only reason why the buffer descriptor must be in words [4:7].
76 */
77 static uint32_t null_texture_descriptor[8] = {
78 0,
79 0,
80 0,
81 S_008F1C_DST_SEL_W(V_008F1C_SQ_SEL_1) |
82 S_008F1C_TYPE(V_008F1C_SQ_RSRC_IMG_1D)
83 /* the rest must contain zeros, which is also used by the buffer
84 * descriptor */
85 };
86
87 static uint32_t null_image_descriptor[8] = {
88 0,
89 0,
90 0,
91 S_008F1C_TYPE(V_008F1C_SQ_RSRC_IMG_1D)
92 /* the rest must contain zeros, which is also used by the buffer
93 * descriptor */
94 };
95
96 static uint64_t si_desc_extract_buffer_address(const uint32_t *desc)
97 {
98 uint64_t va = desc[0] |
99 ((uint64_t)G_008F04_BASE_ADDRESS_HI(desc[1]) << 32);
100
101 /* Sign-extend the 48-bit address. */
102 va <<= 16;
103 va = (int64_t)va >> 16;
104 return va;
105 }
106
107 static void si_init_descriptor_list(uint32_t *desc_list,
108 unsigned element_dw_size,
109 unsigned num_elements,
110 const uint32_t *null_descriptor)
111 {
112 int i;
113
114 /* Initialize the array to NULL descriptors if the element size is 8. */
115 if (null_descriptor) {
116 assert(element_dw_size % 8 == 0);
117 for (i = 0; i < num_elements * element_dw_size / 8; i++)
118 memcpy(desc_list + i * 8, null_descriptor, 8 * 4);
119 }
120 }
121
122 static void si_init_descriptors(struct si_descriptors *desc,
123 short shader_userdata_rel_index,
124 unsigned element_dw_size,
125 unsigned num_elements)
126 {
127 desc->list = CALLOC(num_elements, element_dw_size * 4);
128 desc->element_dw_size = element_dw_size;
129 desc->num_elements = num_elements;
130 desc->shader_userdata_offset = shader_userdata_rel_index * 4;
131 desc->slot_index_to_bind_directly = -1;
132 }
133
134 static void si_release_descriptors(struct si_descriptors *desc)
135 {
136 si_resource_reference(&desc->buffer, NULL);
137 FREE(desc->list);
138 }
139
140 static bool si_upload_descriptors(struct si_context *sctx,
141 struct si_descriptors *desc)
142 {
143 unsigned slot_size = desc->element_dw_size * 4;
144 unsigned first_slot_offset = desc->first_active_slot * slot_size;
145 unsigned upload_size = desc->num_active_slots * slot_size;
146
147 /* Skip the upload if no shader is using the descriptors. dirty_mask
148 * will stay dirty and the descriptors will be uploaded when there is
149 * a shader using them.
150 */
151 if (!upload_size)
152 return true;
153
154 /* If there is just one active descriptor, bind it directly. */
155 if ((int)desc->first_active_slot == desc->slot_index_to_bind_directly &&
156 desc->num_active_slots == 1) {
157 uint32_t *descriptor = &desc->list[desc->slot_index_to_bind_directly *
158 desc->element_dw_size];
159
160 /* The buffer is already in the buffer list. */
161 si_resource_reference(&desc->buffer, NULL);
162 desc->gpu_list = NULL;
163 desc->gpu_address = si_desc_extract_buffer_address(descriptor);
164 si_mark_atom_dirty(sctx, &sctx->atoms.s.shader_pointers);
165 return true;
166 }
167
168 uint32_t *ptr;
169 unsigned buffer_offset;
170 u_upload_alloc(sctx->b.const_uploader, first_slot_offset, upload_size,
171 si_optimal_tcc_alignment(sctx, upload_size),
172 &buffer_offset, (struct pipe_resource**)&desc->buffer,
173 (void**)&ptr);
174 if (!desc->buffer) {
175 desc->gpu_address = 0;
176 return false; /* skip the draw call */
177 }
178
179 util_memcpy_cpu_to_le32(ptr, (char*)desc->list + first_slot_offset,
180 upload_size);
181 desc->gpu_list = ptr - first_slot_offset / 4;
182
183 radeon_add_to_buffer_list(sctx, sctx->gfx_cs, desc->buffer,
184 RADEON_USAGE_READ, RADEON_PRIO_DESCRIPTORS);
185
186 /* The shader pointer should point to slot 0. */
187 buffer_offset -= first_slot_offset;
188 desc->gpu_address = desc->buffer->gpu_address + buffer_offset;
189
190 assert(desc->buffer->flags & RADEON_FLAG_32BIT);
191 assert((desc->buffer->gpu_address >> 32) == sctx->screen->info.address32_hi);
192 assert((desc->gpu_address >> 32) == sctx->screen->info.address32_hi);
193
194 si_mark_atom_dirty(sctx, &sctx->atoms.s.shader_pointers);
195 return true;
196 }
197
198 static void
199 si_descriptors_begin_new_cs(struct si_context *sctx, struct si_descriptors *desc)
200 {
201 if (!desc->buffer)
202 return;
203
204 radeon_add_to_buffer_list(sctx, sctx->gfx_cs, desc->buffer,
205 RADEON_USAGE_READ, RADEON_PRIO_DESCRIPTORS);
206 }
207
208 /* SAMPLER VIEWS */
209
210 static inline enum radeon_bo_priority
211 si_get_sampler_view_priority(struct si_resource *res)
212 {
213 if (res->b.b.target == PIPE_BUFFER)
214 return RADEON_PRIO_SAMPLER_BUFFER;
215
216 if (res->b.b.nr_samples > 1)
217 return RADEON_PRIO_SAMPLER_TEXTURE_MSAA;
218
219 return RADEON_PRIO_SAMPLER_TEXTURE;
220 }
221
222 static struct si_descriptors *
223 si_sampler_and_image_descriptors(struct si_context *sctx, unsigned shader)
224 {
225 return &sctx->descriptors[si_sampler_and_image_descriptors_idx(shader)];
226 }
227
228 static void si_release_sampler_views(struct si_samplers *samplers)
229 {
230 int i;
231
232 for (i = 0; i < ARRAY_SIZE(samplers->views); i++) {
233 pipe_sampler_view_reference(&samplers->views[i], NULL);
234 }
235 }
236
237 static void si_sampler_view_add_buffer(struct si_context *sctx,
238 struct pipe_resource *resource,
239 enum radeon_bo_usage usage,
240 bool is_stencil_sampler,
241 bool check_mem)
242 {
243 struct si_texture *tex = (struct si_texture*)resource;
244 enum radeon_bo_priority priority;
245
246 if (!resource)
247 return;
248
249 /* Use the flushed depth texture if direct sampling is unsupported. */
250 if (resource->target != PIPE_BUFFER &&
251 tex->is_depth && !si_can_sample_zs(tex, is_stencil_sampler))
252 tex = tex->flushed_depth_texture;
253
254 priority = si_get_sampler_view_priority(&tex->buffer);
255 radeon_add_to_gfx_buffer_list_check_mem(sctx, &tex->buffer, usage, priority,
256 check_mem);
257
258 if (resource->target == PIPE_BUFFER)
259 return;
260
261 /* Add separate DCC. */
262 if (tex->dcc_separate_buffer) {
263 radeon_add_to_gfx_buffer_list_check_mem(sctx, tex->dcc_separate_buffer,
264 usage, RADEON_PRIO_SEPARATE_META, check_mem);
265 }
266 }
267
268 static void si_sampler_views_begin_new_cs(struct si_context *sctx,
269 struct si_samplers *samplers)
270 {
271 unsigned mask = samplers->enabled_mask;
272
273 /* Add buffers to the CS. */
274 while (mask) {
275 int i = u_bit_scan(&mask);
276 struct si_sampler_view *sview = (struct si_sampler_view *)samplers->views[i];
277
278 si_sampler_view_add_buffer(sctx, sview->base.texture,
279 RADEON_USAGE_READ,
280 sview->is_stencil_sampler, false);
281 }
282 }
283
284 /* Set buffer descriptor fields that can be changed by reallocations. */
285 static void si_set_buf_desc_address(struct si_resource *buf,
286 uint64_t offset, uint32_t *state)
287 {
288 uint64_t va = buf->gpu_address + offset;
289
290 state[0] = va;
291 state[1] &= C_008F04_BASE_ADDRESS_HI;
292 state[1] |= S_008F04_BASE_ADDRESS_HI(va >> 32);
293 }
294
295 /* Set texture descriptor fields that can be changed by reallocations.
296 *
297 * \param tex texture
298 * \param base_level_info information of the level of BASE_ADDRESS
299 * \param base_level the level of BASE_ADDRESS
300 * \param first_level pipe_sampler_view.u.tex.first_level
301 * \param block_width util_format_get_blockwidth()
302 * \param is_stencil select between separate Z & Stencil
303 * \param state descriptor to update
304 */
305 void si_set_mutable_tex_desc_fields(struct si_screen *sscreen,
306 struct si_texture *tex,
307 const struct legacy_surf_level *base_level_info,
308 unsigned base_level, unsigned first_level,
309 unsigned block_width, bool is_stencil,
310 uint32_t *state)
311 {
312 uint64_t va, meta_va = 0;
313
314 if (tex->is_depth && !si_can_sample_zs(tex, is_stencil)) {
315 tex = tex->flushed_depth_texture;
316 is_stencil = false;
317 }
318
319 va = tex->buffer.gpu_address;
320
321 if (sscreen->info.chip_class >= GFX9) {
322 /* Only stencil_offset needs to be added here. */
323 if (is_stencil)
324 va += tex->surface.u.gfx9.stencil_offset;
325 else
326 va += tex->surface.u.gfx9.surf_offset;
327 } else {
328 va += base_level_info->offset;
329 }
330
331 state[0] = va >> 8;
332 state[1] &= C_008F14_BASE_ADDRESS_HI;
333 state[1] |= S_008F14_BASE_ADDRESS_HI(va >> 40);
334
335 /* Only macrotiled modes can set tile swizzle.
336 * GFX9 doesn't use (legacy) base_level_info.
337 */
338 if (sscreen->info.chip_class >= GFX9 ||
339 base_level_info->mode == RADEON_SURF_MODE_2D)
340 state[0] |= tex->surface.tile_swizzle;
341
342 if (sscreen->info.chip_class >= GFX8) {
343 state[6] &= C_008F28_COMPRESSION_EN;
344
345 if (vi_dcc_enabled(tex, first_level)) {
346 meta_va = (!tex->dcc_separate_buffer ? tex->buffer.gpu_address : 0) +
347 tex->surface.dcc_offset;
348
349 if (sscreen->info.chip_class == GFX8) {
350 meta_va += base_level_info->dcc_offset;
351 assert(base_level_info->mode == RADEON_SURF_MODE_2D);
352 }
353
354 unsigned dcc_tile_swizzle = tex->surface.tile_swizzle << 8;
355 dcc_tile_swizzle &= tex->surface.dcc_alignment - 1;
356 meta_va |= dcc_tile_swizzle;
357 } else if (vi_tc_compat_htile_enabled(tex, first_level,
358 is_stencil ? PIPE_MASK_S : PIPE_MASK_Z)) {
359 meta_va = tex->buffer.gpu_address + tex->surface.htile_offset;
360 }
361
362 if (meta_va)
363 state[6] |= S_008F28_COMPRESSION_EN(1);
364 }
365
366 if (sscreen->info.chip_class >= GFX8 && sscreen->info.chip_class <= GFX9)
367 state[7] = meta_va >> 8;
368
369 if (sscreen->info.chip_class >= GFX10) {
370 state[3] &= C_00A00C_SW_MODE;
371
372 if (is_stencil) {
373 state[3] |= S_00A00C_SW_MODE(tex->surface.u.gfx9.stencil.swizzle_mode);
374 } else {
375 state[3] |= S_00A00C_SW_MODE(tex->surface.u.gfx9.surf.swizzle_mode);
376 }
377
378 state[6] &= C_00A018_META_DATA_ADDRESS_LO &
379 C_00A018_META_PIPE_ALIGNED;
380
381 if (meta_va) {
382 struct gfx9_surf_meta_flags meta;
383
384 if (tex->surface.dcc_offset)
385 meta = tex->surface.u.gfx9.dcc;
386 else
387 meta = tex->surface.u.gfx9.htile;
388
389 state[6] |= S_00A018_META_PIPE_ALIGNED(meta.pipe_aligned) |
390 S_00A018_META_DATA_ADDRESS_LO(meta_va >> 8);
391 }
392
393 state[7] = meta_va >> 16;
394 } else if (sscreen->info.chip_class == GFX9) {
395 state[3] &= C_008F1C_SW_MODE;
396 state[4] &= C_008F20_PITCH;
397
398 if (is_stencil) {
399 state[3] |= S_008F1C_SW_MODE(tex->surface.u.gfx9.stencil.swizzle_mode);
400 state[4] |= S_008F20_PITCH(tex->surface.u.gfx9.stencil.epitch);
401 } else {
402 state[3] |= S_008F1C_SW_MODE(tex->surface.u.gfx9.surf.swizzle_mode);
403 state[4] |= S_008F20_PITCH(tex->surface.u.gfx9.surf.epitch);
404 }
405
406 state[5] &= C_008F24_META_DATA_ADDRESS &
407 C_008F24_META_PIPE_ALIGNED &
408 C_008F24_META_RB_ALIGNED;
409 if (meta_va) {
410 struct gfx9_surf_meta_flags meta;
411
412 if (tex->surface.dcc_offset)
413 meta = tex->surface.u.gfx9.dcc;
414 else
415 meta = tex->surface.u.gfx9.htile;
416
417 state[5] |= S_008F24_META_DATA_ADDRESS(meta_va >> 40) |
418 S_008F24_META_PIPE_ALIGNED(meta.pipe_aligned) |
419 S_008F24_META_RB_ALIGNED(meta.rb_aligned);
420 }
421 } else {
422 /* GFX6-GFX8 */
423 unsigned pitch = base_level_info->nblk_x * block_width;
424 unsigned index = si_tile_mode_index(tex, base_level, is_stencil);
425
426 state[3] &= C_008F1C_TILING_INDEX;
427 state[3] |= S_008F1C_TILING_INDEX(index);
428 state[4] &= C_008F20_PITCH;
429 state[4] |= S_008F20_PITCH(pitch - 1);
430 }
431 }
432
433 static void si_set_sampler_state_desc(struct si_sampler_state *sstate,
434 struct si_sampler_view *sview,
435 struct si_texture *tex,
436 uint32_t *desc)
437 {
438 if (sview && sview->is_integer)
439 memcpy(desc, sstate->integer_val, 4*4);
440 else if (tex && tex->upgraded_depth &&
441 (!sview || !sview->is_stencil_sampler))
442 memcpy(desc, sstate->upgraded_depth_val, 4*4);
443 else
444 memcpy(desc, sstate->val, 4*4);
445 }
446
447 static void si_set_sampler_view_desc(struct si_context *sctx,
448 struct si_sampler_view *sview,
449 struct si_sampler_state *sstate,
450 uint32_t *desc)
451 {
452 struct pipe_sampler_view *view = &sview->base;
453 struct si_texture *tex = (struct si_texture *)view->texture;
454 bool is_buffer = tex->buffer.b.b.target == PIPE_BUFFER;
455
456 if (unlikely(!is_buffer && sview->dcc_incompatible)) {
457 if (vi_dcc_enabled(tex, view->u.tex.first_level))
458 if (!si_texture_disable_dcc(sctx, tex))
459 si_decompress_dcc(sctx, tex);
460
461 sview->dcc_incompatible = false;
462 }
463
464 assert(tex); /* views with texture == NULL aren't supported */
465 memcpy(desc, sview->state, 8*4);
466
467 if (is_buffer) {
468 si_set_buf_desc_address(&tex->buffer,
469 sview->base.u.buf.offset,
470 desc + 4);
471 } else {
472 bool is_separate_stencil = tex->db_compatible &&
473 sview->is_stencil_sampler;
474
475 si_set_mutable_tex_desc_fields(sctx->screen, tex,
476 sview->base_level_info,
477 sview->base_level,
478 sview->base.u.tex.first_level,
479 sview->block_width,
480 is_separate_stencil,
481 desc);
482 }
483
484 if (!is_buffer && tex->surface.fmask_size) {
485 memcpy(desc + 8, sview->fmask_state, 8*4);
486 } else {
487 /* Disable FMASK and bind sampler state in [12:15]. */
488 memcpy(desc + 8, null_texture_descriptor, 4*4);
489
490 if (sstate)
491 si_set_sampler_state_desc(sstate, sview,
492 is_buffer ? NULL : tex,
493 desc + 12);
494 }
495 }
496
497 static bool color_needs_decompression(struct si_texture *tex)
498 {
499 return tex->surface.fmask_size ||
500 (tex->dirty_level_mask &&
501 (tex->cmask_buffer || tex->surface.dcc_offset));
502 }
503
504 static bool depth_needs_decompression(struct si_texture *tex)
505 {
506 /* If the depth/stencil texture is TC-compatible, no decompression
507 * will be done. The decompression function will only flush DB caches
508 * to make it coherent with shaders. That's necessary because the driver
509 * doesn't flush DB caches in any other case.
510 */
511 return tex->db_compatible;
512 }
513
514 static void si_set_sampler_view(struct si_context *sctx,
515 unsigned shader,
516 unsigned slot, struct pipe_sampler_view *view,
517 bool disallow_early_out)
518 {
519 struct si_samplers *samplers = &sctx->samplers[shader];
520 struct si_sampler_view *sview = (struct si_sampler_view*)view;
521 struct si_descriptors *descs = si_sampler_and_image_descriptors(sctx, shader);
522 unsigned desc_slot = si_get_sampler_slot(slot);
523 uint32_t *desc = descs->list + desc_slot * 16;
524
525 if (samplers->views[slot] == view && !disallow_early_out)
526 return;
527
528 if (view) {
529 struct si_texture *tex = (struct si_texture *)view->texture;
530
531 si_set_sampler_view_desc(sctx, sview,
532 samplers->sampler_states[slot], desc);
533
534 if (tex->buffer.b.b.target == PIPE_BUFFER) {
535 tex->buffer.bind_history |= PIPE_BIND_SAMPLER_VIEW;
536 samplers->needs_depth_decompress_mask &= ~(1u << slot);
537 samplers->needs_color_decompress_mask &= ~(1u << slot);
538 } else {
539 if (depth_needs_decompression(tex)) {
540 samplers->needs_depth_decompress_mask |= 1u << slot;
541 } else {
542 samplers->needs_depth_decompress_mask &= ~(1u << slot);
543 }
544 if (color_needs_decompression(tex)) {
545 samplers->needs_color_decompress_mask |= 1u << slot;
546 } else {
547 samplers->needs_color_decompress_mask &= ~(1u << slot);
548 }
549
550 if (tex->surface.dcc_offset &&
551 p_atomic_read(&tex->framebuffers_bound))
552 sctx->need_check_render_feedback = true;
553 }
554
555 pipe_sampler_view_reference(&samplers->views[slot], view);
556 samplers->enabled_mask |= 1u << slot;
557
558 /* Since this can flush, it must be done after enabled_mask is
559 * updated. */
560 si_sampler_view_add_buffer(sctx, view->texture,
561 RADEON_USAGE_READ,
562 sview->is_stencil_sampler, true);
563 } else {
564 pipe_sampler_view_reference(&samplers->views[slot], NULL);
565 memcpy(desc, null_texture_descriptor, 8*4);
566 /* Only clear the lower dwords of FMASK. */
567 memcpy(desc + 8, null_texture_descriptor, 4*4);
568 /* Re-set the sampler state if we are transitioning from FMASK. */
569 if (samplers->sampler_states[slot])
570 si_set_sampler_state_desc(samplers->sampler_states[slot], NULL, NULL,
571 desc + 12);
572
573 samplers->enabled_mask &= ~(1u << slot);
574 samplers->needs_depth_decompress_mask &= ~(1u << slot);
575 samplers->needs_color_decompress_mask &= ~(1u << slot);
576 }
577
578 sctx->descriptors_dirty |= 1u << si_sampler_and_image_descriptors_idx(shader);
579 }
580
581 static void si_update_shader_needs_decompress_mask(struct si_context *sctx,
582 unsigned shader)
583 {
584 struct si_samplers *samplers = &sctx->samplers[shader];
585 unsigned shader_bit = 1 << shader;
586
587 if (samplers->needs_depth_decompress_mask ||
588 samplers->needs_color_decompress_mask ||
589 sctx->images[shader].needs_color_decompress_mask)
590 sctx->shader_needs_decompress_mask |= shader_bit;
591 else
592 sctx->shader_needs_decompress_mask &= ~shader_bit;
593 }
594
595 static void si_set_sampler_views(struct pipe_context *ctx,
596 enum pipe_shader_type shader, unsigned start,
597 unsigned count,
598 struct pipe_sampler_view **views)
599 {
600 struct si_context *sctx = (struct si_context *)ctx;
601 int i;
602
603 if (!count || shader >= SI_NUM_SHADERS)
604 return;
605
606 if (views) {
607 for (i = 0; i < count; i++)
608 si_set_sampler_view(sctx, shader, start + i, views[i], false);
609 } else {
610 for (i = 0; i < count; i++)
611 si_set_sampler_view(sctx, shader, start + i, NULL, false);
612 }
613
614 si_update_shader_needs_decompress_mask(sctx, shader);
615 }
616
617 static void
618 si_samplers_update_needs_color_decompress_mask(struct si_samplers *samplers)
619 {
620 unsigned mask = samplers->enabled_mask;
621
622 while (mask) {
623 int i = u_bit_scan(&mask);
624 struct pipe_resource *res = samplers->views[i]->texture;
625
626 if (res && res->target != PIPE_BUFFER) {
627 struct si_texture *tex = (struct si_texture *)res;
628
629 if (color_needs_decompression(tex)) {
630 samplers->needs_color_decompress_mask |= 1u << i;
631 } else {
632 samplers->needs_color_decompress_mask &= ~(1u << i);
633 }
634 }
635 }
636 }
637
638 /* IMAGE VIEWS */
639
640 static void
641 si_release_image_views(struct si_images *images)
642 {
643 unsigned i;
644
645 for (i = 0; i < SI_NUM_IMAGES; ++i) {
646 struct pipe_image_view *view = &images->views[i];
647
648 pipe_resource_reference(&view->resource, NULL);
649 }
650 }
651
652 static void
653 si_image_views_begin_new_cs(struct si_context *sctx, struct si_images *images)
654 {
655 uint mask = images->enabled_mask;
656
657 /* Add buffers to the CS. */
658 while (mask) {
659 int i = u_bit_scan(&mask);
660 struct pipe_image_view *view = &images->views[i];
661
662 assert(view->resource);
663
664 si_sampler_view_add_buffer(sctx, view->resource,
665 RADEON_USAGE_READWRITE, false, false);
666 }
667 }
668
669 static void
670 si_disable_shader_image(struct si_context *ctx, unsigned shader, unsigned slot)
671 {
672 struct si_images *images = &ctx->images[shader];
673
674 if (images->enabled_mask & (1u << slot)) {
675 struct si_descriptors *descs = si_sampler_and_image_descriptors(ctx, shader);
676 unsigned desc_slot = si_get_image_slot(slot);
677
678 pipe_resource_reference(&images->views[slot].resource, NULL);
679 images->needs_color_decompress_mask &= ~(1 << slot);
680
681 memcpy(descs->list + desc_slot*8, null_image_descriptor, 8*4);
682 images->enabled_mask &= ~(1u << slot);
683 ctx->descriptors_dirty |= 1u << si_sampler_and_image_descriptors_idx(shader);
684 }
685 }
686
687 static void
688 si_mark_image_range_valid(const struct pipe_image_view *view)
689 {
690 struct si_resource *res = si_resource(view->resource);
691
692 if (res->b.b.target != PIPE_BUFFER)
693 return;
694
695 util_range_add(&res->b.b, &res->valid_buffer_range,
696 view->u.buf.offset,
697 view->u.buf.offset + view->u.buf.size);
698 }
699
700 static void si_set_shader_image_desc(struct si_context *ctx,
701 const struct pipe_image_view *view,
702 bool skip_decompress,
703 uint32_t *desc, uint32_t *fmask_desc)
704 {
705 struct si_screen *screen = ctx->screen;
706 struct si_resource *res;
707
708 res = si_resource(view->resource);
709
710 if (res->b.b.target == PIPE_BUFFER ||
711 view->shader_access & SI_IMAGE_ACCESS_AS_BUFFER) {
712 if (view->access & PIPE_IMAGE_ACCESS_WRITE)
713 si_mark_image_range_valid(view);
714
715 si_make_buffer_descriptor(screen, res,
716 view->format,
717 view->u.buf.offset,
718 view->u.buf.size, desc);
719 si_set_buf_desc_address(res, view->u.buf.offset, desc + 4);
720 } else {
721 static const unsigned char swizzle[4] = { 0, 1, 2, 3 };
722 struct si_texture *tex = (struct si_texture *)res;
723 unsigned level = view->u.tex.level;
724 unsigned width, height, depth, hw_level;
725 bool uses_dcc = vi_dcc_enabled(tex, level);
726 unsigned access = view->access;
727
728 assert(!tex->is_depth);
729 assert(fmask_desc || tex->surface.fmask_offset == 0);
730
731 if (uses_dcc && !skip_decompress &&
732 (access & PIPE_IMAGE_ACCESS_WRITE ||
733 !vi_dcc_formats_compatible(screen, res->b.b.format, view->format))) {
734 /* If DCC can't be disabled, at least decompress it.
735 * The decompression is relatively cheap if the surface
736 * has been decompressed already.
737 */
738 if (!si_texture_disable_dcc(ctx, tex))
739 si_decompress_dcc(ctx, tex);
740 }
741
742 if (ctx->chip_class >= GFX9) {
743 /* Always set the base address. The swizzle modes don't
744 * allow setting mipmap level offsets as the base.
745 */
746 width = res->b.b.width0;
747 height = res->b.b.height0;
748 depth = res->b.b.depth0;
749 hw_level = level;
750 } else {
751 /* Always force the base level to the selected level.
752 *
753 * This is required for 3D textures, where otherwise
754 * selecting a single slice for non-layered bindings
755 * fails. It doesn't hurt the other targets.
756 */
757 width = u_minify(res->b.b.width0, level);
758 height = u_minify(res->b.b.height0, level);
759 depth = u_minify(res->b.b.depth0, level);
760 hw_level = 0;
761 }
762
763 screen->make_texture_descriptor(screen, tex,
764 false, res->b.b.target,
765 view->format, swizzle,
766 hw_level, hw_level,
767 view->u.tex.first_layer,
768 view->u.tex.last_layer,
769 width, height, depth,
770 desc, fmask_desc);
771 si_set_mutable_tex_desc_fields(screen, tex,
772 &tex->surface.u.legacy.level[level],
773 level, level,
774 util_format_get_blockwidth(view->format),
775 false, desc);
776 }
777 }
778
779 static void si_set_shader_image(struct si_context *ctx,
780 unsigned shader,
781 unsigned slot, const struct pipe_image_view *view,
782 bool skip_decompress)
783 {
784 struct si_images *images = &ctx->images[shader];
785 struct si_descriptors *descs = si_sampler_and_image_descriptors(ctx, shader);
786 struct si_resource *res;
787
788 if (!view || !view->resource) {
789 si_disable_shader_image(ctx, shader, slot);
790 return;
791 }
792
793 res = si_resource(view->resource);
794
795 if (&images->views[slot] != view)
796 util_copy_image_view(&images->views[slot], view);
797
798 si_set_shader_image_desc(ctx, view, skip_decompress,
799 descs->list + si_get_image_slot(slot) * 8,
800 descs->list + si_get_image_slot(slot + SI_NUM_IMAGES) * 8);
801
802 if (res->b.b.target == PIPE_BUFFER ||
803 view->shader_access & SI_IMAGE_ACCESS_AS_BUFFER) {
804 images->needs_color_decompress_mask &= ~(1 << slot);
805 res->bind_history |= PIPE_BIND_SHADER_IMAGE;
806 } else {
807 struct si_texture *tex = (struct si_texture *)res;
808 unsigned level = view->u.tex.level;
809
810 if (color_needs_decompression(tex)) {
811 images->needs_color_decompress_mask |= 1 << slot;
812 } else {
813 images->needs_color_decompress_mask &= ~(1 << slot);
814 }
815
816 if (vi_dcc_enabled(tex, level) &&
817 p_atomic_read(&tex->framebuffers_bound))
818 ctx->need_check_render_feedback = true;
819 }
820
821 images->enabled_mask |= 1u << slot;
822 ctx->descriptors_dirty |= 1u << si_sampler_and_image_descriptors_idx(shader);
823
824 /* Since this can flush, it must be done after enabled_mask is updated. */
825 si_sampler_view_add_buffer(ctx, &res->b.b,
826 (view->access & PIPE_IMAGE_ACCESS_WRITE) ?
827 RADEON_USAGE_READWRITE : RADEON_USAGE_READ,
828 false, true);
829 }
830
831 static void
832 si_set_shader_images(struct pipe_context *pipe,
833 enum pipe_shader_type shader,
834 unsigned start_slot, unsigned count,
835 const struct pipe_image_view *views)
836 {
837 struct si_context *ctx = (struct si_context *)pipe;
838 unsigned i, slot;
839
840 assert(shader < SI_NUM_SHADERS);
841
842 if (!count)
843 return;
844
845 assert(start_slot + count <= SI_NUM_IMAGES);
846
847 if (views) {
848 for (i = 0, slot = start_slot; i < count; ++i, ++slot)
849 si_set_shader_image(ctx, shader, slot, &views[i], false);
850 } else {
851 for (i = 0, slot = start_slot; i < count; ++i, ++slot)
852 si_set_shader_image(ctx, shader, slot, NULL, false);
853 }
854
855 si_update_shader_needs_decompress_mask(ctx, shader);
856 }
857
858 static void
859 si_images_update_needs_color_decompress_mask(struct si_images *images)
860 {
861 unsigned mask = images->enabled_mask;
862
863 while (mask) {
864 int i = u_bit_scan(&mask);
865 struct pipe_resource *res = images->views[i].resource;
866
867 if (res && res->target != PIPE_BUFFER) {
868 struct si_texture *tex = (struct si_texture *)res;
869
870 if (color_needs_decompression(tex)) {
871 images->needs_color_decompress_mask |= 1 << i;
872 } else {
873 images->needs_color_decompress_mask &= ~(1 << i);
874 }
875 }
876 }
877 }
878
879 void si_update_ps_colorbuf0_slot(struct si_context *sctx)
880 {
881 struct si_buffer_resources *buffers = &sctx->rw_buffers;
882 struct si_descriptors *descs = &sctx->descriptors[SI_DESCS_RW_BUFFERS];
883 unsigned slot = SI_PS_IMAGE_COLORBUF0;
884 struct pipe_surface *surf = NULL;
885
886 /* si_texture_disable_dcc can get us here again. */
887 if (sctx->blitter->running)
888 return;
889
890 /* See whether FBFETCH is used and color buffer 0 is set. */
891 if (sctx->ps_shader.cso &&
892 sctx->ps_shader.cso->info.uses_fbfetch &&
893 sctx->framebuffer.state.nr_cbufs &&
894 sctx->framebuffer.state.cbufs[0])
895 surf = sctx->framebuffer.state.cbufs[0];
896
897 /* Return if FBFETCH transitions from disabled to disabled. */
898 if (!buffers->buffers[slot] && !surf)
899 return;
900
901 sctx->ps_uses_fbfetch = surf != NULL;
902 si_update_ps_iter_samples(sctx);
903
904 if (surf) {
905 struct si_texture *tex = (struct si_texture*)surf->texture;
906 struct pipe_image_view view;
907
908 assert(tex);
909 assert(!tex->is_depth);
910
911 /* Disable DCC, because the texture is used as both a sampler
912 * and color buffer.
913 */
914 si_texture_disable_dcc(sctx, tex);
915
916 if (tex->buffer.b.b.nr_samples <= 1 && tex->cmask_buffer) {
917 /* Disable CMASK. */
918 assert(tex->cmask_buffer != &tex->buffer);
919 si_eliminate_fast_color_clear(sctx, tex);
920 si_texture_discard_cmask(sctx->screen, tex);
921 }
922
923 view.resource = surf->texture;
924 view.format = surf->format;
925 view.access = PIPE_IMAGE_ACCESS_READ;
926 view.u.tex.first_layer = surf->u.tex.first_layer;
927 view.u.tex.last_layer = surf->u.tex.last_layer;
928 view.u.tex.level = surf->u.tex.level;
929
930 /* Set the descriptor. */
931 uint32_t *desc = descs->list + slot*4;
932 memset(desc, 0, 16 * 4);
933 si_set_shader_image_desc(sctx, &view, true, desc, desc + 8);
934
935 pipe_resource_reference(&buffers->buffers[slot], &tex->buffer.b.b);
936 radeon_add_to_buffer_list(sctx, sctx->gfx_cs,
937 &tex->buffer, RADEON_USAGE_READ,
938 RADEON_PRIO_SHADER_RW_IMAGE);
939 buffers->enabled_mask |= 1u << slot;
940 } else {
941 /* Clear the descriptor. */
942 memset(descs->list + slot*4, 0, 8*4);
943 pipe_resource_reference(&buffers->buffers[slot], NULL);
944 buffers->enabled_mask &= ~(1u << slot);
945 }
946
947 sctx->descriptors_dirty |= 1u << SI_DESCS_RW_BUFFERS;
948 }
949
950 /* SAMPLER STATES */
951
952 static void si_bind_sampler_states(struct pipe_context *ctx,
953 enum pipe_shader_type shader,
954 unsigned start, unsigned count, void **states)
955 {
956 struct si_context *sctx = (struct si_context *)ctx;
957 struct si_samplers *samplers = &sctx->samplers[shader];
958 struct si_descriptors *desc = si_sampler_and_image_descriptors(sctx, shader);
959 struct si_sampler_state **sstates = (struct si_sampler_state**)states;
960 int i;
961
962 if (!count || shader >= SI_NUM_SHADERS || !sstates)
963 return;
964
965 for (i = 0; i < count; i++) {
966 unsigned slot = start + i;
967 unsigned desc_slot = si_get_sampler_slot(slot);
968
969 if (!sstates[i] ||
970 sstates[i] == samplers->sampler_states[slot])
971 continue;
972
973 #ifndef NDEBUG
974 assert(sstates[i]->magic == SI_SAMPLER_STATE_MAGIC);
975 #endif
976 samplers->sampler_states[slot] = sstates[i];
977
978 /* If FMASK is bound, don't overwrite it.
979 * The sampler state will be set after FMASK is unbound.
980 */
981 struct si_sampler_view *sview =
982 (struct si_sampler_view *)samplers->views[slot];
983
984 struct si_texture *tex = NULL;
985
986 if (sview && sview->base.texture &&
987 sview->base.texture->target != PIPE_BUFFER)
988 tex = (struct si_texture *)sview->base.texture;
989
990 if (tex && tex->surface.fmask_size)
991 continue;
992
993 si_set_sampler_state_desc(sstates[i], sview, tex,
994 desc->list + desc_slot * 16 + 12);
995
996 sctx->descriptors_dirty |= 1u << si_sampler_and_image_descriptors_idx(shader);
997 }
998 }
999
1000 /* BUFFER RESOURCES */
1001
1002 static void si_init_buffer_resources(struct si_buffer_resources *buffers,
1003 struct si_descriptors *descs,
1004 unsigned num_buffers,
1005 short shader_userdata_rel_index,
1006 enum radeon_bo_priority priority,
1007 enum radeon_bo_priority priority_constbuf)
1008 {
1009 buffers->priority = priority;
1010 buffers->priority_constbuf = priority_constbuf;
1011 buffers->buffers = CALLOC(num_buffers, sizeof(struct pipe_resource*));
1012 buffers->offsets = CALLOC(num_buffers, sizeof(buffers->offsets[0]));
1013
1014 si_init_descriptors(descs, shader_userdata_rel_index, 4, num_buffers);
1015 }
1016
1017 static void si_release_buffer_resources(struct si_buffer_resources *buffers,
1018 struct si_descriptors *descs)
1019 {
1020 int i;
1021
1022 for (i = 0; i < descs->num_elements; i++) {
1023 pipe_resource_reference(&buffers->buffers[i], NULL);
1024 }
1025
1026 FREE(buffers->buffers);
1027 FREE(buffers->offsets);
1028 }
1029
1030 static void si_buffer_resources_begin_new_cs(struct si_context *sctx,
1031 struct si_buffer_resources *buffers)
1032 {
1033 unsigned mask = buffers->enabled_mask;
1034
1035 /* Add buffers to the CS. */
1036 while (mask) {
1037 int i = u_bit_scan(&mask);
1038
1039 radeon_add_to_buffer_list(sctx, sctx->gfx_cs,
1040 si_resource(buffers->buffers[i]),
1041 buffers->writable_mask & (1u << i) ? RADEON_USAGE_READWRITE :
1042 RADEON_USAGE_READ,
1043 i < SI_NUM_SHADER_BUFFERS ? buffers->priority :
1044 buffers->priority_constbuf);
1045 }
1046 }
1047
1048 static void si_get_buffer_from_descriptors(struct si_buffer_resources *buffers,
1049 struct si_descriptors *descs,
1050 unsigned idx, struct pipe_resource **buf,
1051 unsigned *offset, unsigned *size)
1052 {
1053 pipe_resource_reference(buf, buffers->buffers[idx]);
1054 if (*buf) {
1055 struct si_resource *res = si_resource(*buf);
1056 const uint32_t *desc = descs->list + idx * 4;
1057 uint64_t va;
1058
1059 *size = desc[2];
1060
1061 assert(G_008F04_STRIDE(desc[1]) == 0);
1062 va = si_desc_extract_buffer_address(desc);
1063
1064 assert(va >= res->gpu_address && va + *size <= res->gpu_address + res->bo_size);
1065 *offset = va - res->gpu_address;
1066 }
1067 }
1068
1069 /* VERTEX BUFFERS */
1070
1071 static void si_vertex_buffers_begin_new_cs(struct si_context *sctx)
1072 {
1073 int count = sctx->vertex_elements ? sctx->vertex_elements->count : 0;
1074 int i;
1075
1076 for (i = 0; i < count; i++) {
1077 int vb = sctx->vertex_elements->vertex_buffer_index[i];
1078
1079 if (vb >= ARRAY_SIZE(sctx->vertex_buffer))
1080 continue;
1081 if (!sctx->vertex_buffer[vb].buffer.resource)
1082 continue;
1083
1084 radeon_add_to_buffer_list(sctx, sctx->gfx_cs,
1085 si_resource(sctx->vertex_buffer[vb].buffer.resource),
1086 RADEON_USAGE_READ, RADEON_PRIO_VERTEX_BUFFER);
1087 }
1088
1089 if (!sctx->vb_descriptors_buffer)
1090 return;
1091 radeon_add_to_buffer_list(sctx, sctx->gfx_cs,
1092 sctx->vb_descriptors_buffer, RADEON_USAGE_READ,
1093 RADEON_PRIO_DESCRIPTORS);
1094 }
1095
1096 bool si_upload_vertex_buffer_descriptors(struct si_context *sctx)
1097 {
1098 struct si_vertex_elements *velems = sctx->vertex_elements;
1099 unsigned i, count;
1100 unsigned desc_list_byte_size;
1101 unsigned first_vb_use_mask;
1102 uint32_t *ptr;
1103
1104 if (!sctx->vertex_buffers_dirty || !velems)
1105 return true;
1106
1107 count = velems->count;
1108
1109 if (!count)
1110 return true;
1111
1112 desc_list_byte_size = velems->desc_list_byte_size;
1113 first_vb_use_mask = velems->first_vb_use_mask;
1114
1115 /* Vertex buffer descriptors are the only ones which are uploaded
1116 * directly through a staging buffer and don't go through
1117 * the fine-grained upload path.
1118 */
1119 u_upload_alloc(sctx->b.const_uploader, 0,
1120 desc_list_byte_size,
1121 si_optimal_tcc_alignment(sctx, desc_list_byte_size),
1122 &sctx->vb_descriptors_offset,
1123 (struct pipe_resource**)&sctx->vb_descriptors_buffer,
1124 (void**)&ptr);
1125 if (!sctx->vb_descriptors_buffer) {
1126 sctx->vb_descriptors_offset = 0;
1127 sctx->vb_descriptors_gpu_list = NULL;
1128 return false;
1129 }
1130
1131 sctx->vb_descriptors_gpu_list = ptr;
1132 radeon_add_to_buffer_list(sctx, sctx->gfx_cs,
1133 sctx->vb_descriptors_buffer, RADEON_USAGE_READ,
1134 RADEON_PRIO_DESCRIPTORS);
1135
1136 assert(count <= SI_MAX_ATTRIBS);
1137
1138 for (i = 0; i < count; i++) {
1139 struct pipe_vertex_buffer *vb;
1140 struct si_resource *buf;
1141 unsigned vbo_index = velems->vertex_buffer_index[i];
1142 uint32_t *desc = &ptr[i*4];
1143
1144 vb = &sctx->vertex_buffer[vbo_index];
1145 buf = si_resource(vb->buffer.resource);
1146 if (!buf) {
1147 memset(desc, 0, 16);
1148 continue;
1149 }
1150
1151 int64_t offset = (int64_t)((int)vb->buffer_offset) +
1152 velems->src_offset[i];
1153
1154 if (offset >= buf->b.b.width0) {
1155 assert(offset < buf->b.b.width0);
1156 memset(desc, 0, 16);
1157 continue;
1158 }
1159
1160 uint64_t va = buf->gpu_address + offset;
1161
1162 int64_t num_records = (int64_t)buf->b.b.width0 - offset;
1163 if (sctx->chip_class != GFX8 && vb->stride) {
1164 /* Round up by rounding down and adding 1 */
1165 num_records = (num_records - velems->format_size[i]) /
1166 vb->stride + 1;
1167 }
1168 assert(num_records >= 0 && num_records <= UINT_MAX);
1169
1170 uint32_t rsrc_word3 = velems->rsrc_word3[i];
1171
1172 /* OOB_SELECT chooses the out-of-bounds check:
1173 * - 1: index >= NUM_RECORDS (Structured)
1174 * - 3: offset >= NUM_RECORDS (Raw)
1175 */
1176 if (sctx->chip_class >= GFX10)
1177 rsrc_word3 |= S_008F0C_OOB_SELECT(vb->stride ? V_008F0C_OOB_SELECT_STRUCTURED : V_008F0C_OOB_SELECT_RAW);
1178
1179 desc[0] = va;
1180 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
1181 S_008F04_STRIDE(vb->stride);
1182 desc[2] = num_records;
1183 desc[3] = rsrc_word3;
1184
1185 if (first_vb_use_mask & (1 << i)) {
1186 radeon_add_to_buffer_list(sctx, sctx->gfx_cs,
1187 si_resource(vb->buffer.resource),
1188 RADEON_USAGE_READ, RADEON_PRIO_VERTEX_BUFFER);
1189 }
1190 }
1191
1192 /* Don't flush the const cache. It would have a very negative effect
1193 * on performance (confirmed by testing). New descriptors are always
1194 * uploaded to a fresh new buffer, so I don't think flushing the const
1195 * cache is needed. */
1196 si_mark_atom_dirty(sctx, &sctx->atoms.s.shader_pointers);
1197 sctx->vertex_buffers_dirty = false;
1198 sctx->vertex_buffer_pointer_dirty = true;
1199 sctx->prefetch_L2_mask |= SI_PREFETCH_VBO_DESCRIPTORS;
1200 return true;
1201 }
1202
1203
1204 /* CONSTANT BUFFERS */
1205
1206 static struct si_descriptors *
1207 si_const_and_shader_buffer_descriptors(struct si_context *sctx, unsigned shader)
1208 {
1209 return &sctx->descriptors[si_const_and_shader_buffer_descriptors_idx(shader)];
1210 }
1211
1212 void si_upload_const_buffer(struct si_context *sctx, struct si_resource **buf,
1213 const uint8_t *ptr, unsigned size, uint32_t *const_offset)
1214 {
1215 void *tmp;
1216
1217 u_upload_alloc(sctx->b.const_uploader, 0, size,
1218 si_optimal_tcc_alignment(sctx, size),
1219 const_offset,
1220 (struct pipe_resource**)buf, &tmp);
1221 if (*buf)
1222 util_memcpy_cpu_to_le32(tmp, ptr, size);
1223 }
1224
1225 static void si_set_constant_buffer(struct si_context *sctx,
1226 struct si_buffer_resources *buffers,
1227 unsigned descriptors_idx,
1228 uint slot, const struct pipe_constant_buffer *input)
1229 {
1230 struct si_descriptors *descs = &sctx->descriptors[descriptors_idx];
1231 assert(slot < descs->num_elements);
1232 pipe_resource_reference(&buffers->buffers[slot], NULL);
1233
1234 /* GFX7 cannot unbind a constant buffer (S_BUFFER_LOAD is buggy
1235 * with a NULL buffer). We need to use a dummy buffer instead. */
1236 if (sctx->chip_class == GFX7 &&
1237 (!input || (!input->buffer && !input->user_buffer)))
1238 input = &sctx->null_const_buf;
1239
1240 if (input && (input->buffer || input->user_buffer)) {
1241 struct pipe_resource *buffer = NULL;
1242 uint64_t va;
1243 unsigned buffer_offset;
1244
1245 /* Upload the user buffer if needed. */
1246 if (input->user_buffer) {
1247 si_upload_const_buffer(sctx,
1248 (struct si_resource**)&buffer, input->user_buffer,
1249 input->buffer_size, &buffer_offset);
1250 if (!buffer) {
1251 /* Just unbind on failure. */
1252 si_set_constant_buffer(sctx, buffers, descriptors_idx, slot, NULL);
1253 return;
1254 }
1255 } else {
1256 pipe_resource_reference(&buffer, input->buffer);
1257 buffer_offset = input->buffer_offset;
1258 }
1259
1260 va = si_resource(buffer)->gpu_address + buffer_offset;
1261
1262 /* Set the descriptor. */
1263 uint32_t *desc = descs->list + slot*4;
1264 desc[0] = va;
1265 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
1266 S_008F04_STRIDE(0);
1267 desc[2] = input->buffer_size;
1268 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1269 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1270 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1271 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
1272
1273 if (sctx->chip_class >= GFX10) {
1274 desc[3] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
1275 S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
1276 S_008F0C_RESOURCE_LEVEL(1);
1277 } else {
1278 desc[3] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1279 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
1280 }
1281
1282 buffers->buffers[slot] = buffer;
1283 buffers->offsets[slot] = buffer_offset;
1284 radeon_add_to_gfx_buffer_list_check_mem(sctx,
1285 si_resource(buffer),
1286 RADEON_USAGE_READ,
1287 buffers->priority_constbuf, true);
1288 buffers->enabled_mask |= 1u << slot;
1289 } else {
1290 /* Clear the descriptor. */
1291 memset(descs->list + slot*4, 0, sizeof(uint32_t) * 4);
1292 buffers->enabled_mask &= ~(1u << slot);
1293 }
1294
1295 sctx->descriptors_dirty |= 1u << descriptors_idx;
1296 }
1297
1298 static void si_pipe_set_constant_buffer(struct pipe_context *ctx,
1299 enum pipe_shader_type shader, uint slot,
1300 const struct pipe_constant_buffer *input)
1301 {
1302 struct si_context *sctx = (struct si_context *)ctx;
1303
1304 if (shader >= SI_NUM_SHADERS)
1305 return;
1306
1307 if (slot == 0 && input && input->buffer &&
1308 !(si_resource(input->buffer)->flags & RADEON_FLAG_32BIT)) {
1309 assert(!"constant buffer 0 must have a 32-bit VM address, use const_uploader");
1310 return;
1311 }
1312
1313 if (input && input->buffer)
1314 si_resource(input->buffer)->bind_history |= PIPE_BIND_CONSTANT_BUFFER;
1315
1316 slot = si_get_constbuf_slot(slot);
1317 si_set_constant_buffer(sctx, &sctx->const_and_shader_buffers[shader],
1318 si_const_and_shader_buffer_descriptors_idx(shader),
1319 slot, input);
1320 }
1321
1322 void si_get_pipe_constant_buffer(struct si_context *sctx, uint shader,
1323 uint slot, struct pipe_constant_buffer *cbuf)
1324 {
1325 cbuf->user_buffer = NULL;
1326 si_get_buffer_from_descriptors(
1327 &sctx->const_and_shader_buffers[shader],
1328 si_const_and_shader_buffer_descriptors(sctx, shader),
1329 si_get_constbuf_slot(slot),
1330 &cbuf->buffer, &cbuf->buffer_offset, &cbuf->buffer_size);
1331 }
1332
1333 /* SHADER BUFFERS */
1334
1335 static void si_set_shader_buffer(struct si_context *sctx,
1336 struct si_buffer_resources *buffers,
1337 unsigned descriptors_idx,
1338 uint slot, const struct pipe_shader_buffer *sbuffer,
1339 bool writable, enum radeon_bo_priority priority)
1340 {
1341 struct si_descriptors *descs = &sctx->descriptors[descriptors_idx];
1342 uint32_t *desc = descs->list + slot * 4;
1343
1344 if (!sbuffer || !sbuffer->buffer) {
1345 pipe_resource_reference(&buffers->buffers[slot], NULL);
1346 memset(desc, 0, sizeof(uint32_t) * 4);
1347 buffers->enabled_mask &= ~(1u << slot);
1348 buffers->writable_mask &= ~(1u << slot);
1349 sctx->descriptors_dirty |= 1u << descriptors_idx;
1350 return;
1351 }
1352
1353 struct si_resource *buf = si_resource(sbuffer->buffer);
1354 uint64_t va = buf->gpu_address + sbuffer->buffer_offset;
1355
1356 desc[0] = va;
1357 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
1358 S_008F04_STRIDE(0);
1359 desc[2] = sbuffer->buffer_size;
1360 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1361 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1362 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1363 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
1364
1365 if (sctx->chip_class >= GFX10) {
1366 desc[3] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
1367 S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
1368 S_008F0C_RESOURCE_LEVEL(1);
1369 } else {
1370 desc[3] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1371 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
1372 }
1373
1374 pipe_resource_reference(&buffers->buffers[slot], &buf->b.b);
1375 buffers->offsets[slot] = sbuffer->buffer_offset;
1376 radeon_add_to_gfx_buffer_list_check_mem(sctx, buf,
1377 writable ? RADEON_USAGE_READWRITE :
1378 RADEON_USAGE_READ,
1379 priority, true);
1380 if (writable)
1381 buffers->writable_mask |= 1u << slot;
1382 else
1383 buffers->writable_mask &= ~(1u << slot);
1384
1385 buffers->enabled_mask |= 1u << slot;
1386 sctx->descriptors_dirty |= 1u << descriptors_idx;
1387
1388 util_range_add(&buf->b.b, &buf->valid_buffer_range, sbuffer->buffer_offset,
1389 sbuffer->buffer_offset + sbuffer->buffer_size);
1390 }
1391
1392 static void si_set_shader_buffers(struct pipe_context *ctx,
1393 enum pipe_shader_type shader,
1394 unsigned start_slot, unsigned count,
1395 const struct pipe_shader_buffer *sbuffers,
1396 unsigned writable_bitmask)
1397 {
1398 struct si_context *sctx = (struct si_context *)ctx;
1399 struct si_buffer_resources *buffers = &sctx->const_and_shader_buffers[shader];
1400 unsigned descriptors_idx = si_const_and_shader_buffer_descriptors_idx(shader);
1401 unsigned i;
1402
1403 assert(start_slot + count <= SI_NUM_SHADER_BUFFERS);
1404
1405 for (i = 0; i < count; ++i) {
1406 const struct pipe_shader_buffer *sbuffer = sbuffers ? &sbuffers[i] : NULL;
1407 unsigned slot = si_get_shaderbuf_slot(start_slot + i);
1408
1409 if (sbuffer && sbuffer->buffer)
1410 si_resource(sbuffer->buffer)->bind_history |= PIPE_BIND_SHADER_BUFFER;
1411
1412 si_set_shader_buffer(sctx, buffers, descriptors_idx, slot, sbuffer,
1413 !!(writable_bitmask & (1u << i)),
1414 buffers->priority);
1415 }
1416 }
1417
1418 void si_get_shader_buffers(struct si_context *sctx,
1419 enum pipe_shader_type shader,
1420 uint start_slot, uint count,
1421 struct pipe_shader_buffer *sbuf)
1422 {
1423 struct si_buffer_resources *buffers = &sctx->const_and_shader_buffers[shader];
1424 struct si_descriptors *descs = si_const_and_shader_buffer_descriptors(sctx, shader);
1425
1426 for (unsigned i = 0; i < count; ++i) {
1427 si_get_buffer_from_descriptors(
1428 buffers, descs,
1429 si_get_shaderbuf_slot(start_slot + i),
1430 &sbuf[i].buffer, &sbuf[i].buffer_offset,
1431 &sbuf[i].buffer_size);
1432 }
1433 }
1434
1435 /* RING BUFFERS */
1436
1437 void si_set_rw_buffer(struct si_context *sctx,
1438 uint slot, const struct pipe_constant_buffer *input)
1439 {
1440 si_set_constant_buffer(sctx, &sctx->rw_buffers, SI_DESCS_RW_BUFFERS,
1441 slot, input);
1442 }
1443
1444 void si_set_rw_shader_buffer(struct si_context *sctx, uint slot,
1445 const struct pipe_shader_buffer *sbuffer)
1446 {
1447 si_set_shader_buffer(sctx, &sctx->rw_buffers, SI_DESCS_RW_BUFFERS,
1448 slot, sbuffer, true, RADEON_PRIO_SHADER_RW_BUFFER);
1449 }
1450
1451 void si_set_ring_buffer(struct si_context *sctx, uint slot,
1452 struct pipe_resource *buffer,
1453 unsigned stride, unsigned num_records,
1454 bool add_tid, bool swizzle,
1455 unsigned element_size, unsigned index_stride, uint64_t offset)
1456 {
1457 struct si_buffer_resources *buffers = &sctx->rw_buffers;
1458 struct si_descriptors *descs = &sctx->descriptors[SI_DESCS_RW_BUFFERS];
1459
1460 /* The stride field in the resource descriptor has 14 bits */
1461 assert(stride < (1 << 14));
1462
1463 assert(slot < descs->num_elements);
1464 pipe_resource_reference(&buffers->buffers[slot], NULL);
1465
1466 if (buffer) {
1467 uint64_t va;
1468
1469 va = si_resource(buffer)->gpu_address + offset;
1470
1471 switch (element_size) {
1472 default:
1473 assert(!"Unsupported ring buffer element size");
1474 case 0:
1475 case 2:
1476 element_size = 0;
1477 break;
1478 case 4:
1479 element_size = 1;
1480 break;
1481 case 8:
1482 element_size = 2;
1483 break;
1484 case 16:
1485 element_size = 3;
1486 break;
1487 }
1488
1489 switch (index_stride) {
1490 default:
1491 assert(!"Unsupported ring buffer index stride");
1492 case 0:
1493 case 8:
1494 index_stride = 0;
1495 break;
1496 case 16:
1497 index_stride = 1;
1498 break;
1499 case 32:
1500 index_stride = 2;
1501 break;
1502 case 64:
1503 index_stride = 3;
1504 break;
1505 }
1506
1507 if (sctx->chip_class >= GFX8 && stride)
1508 num_records *= stride;
1509
1510 /* Set the descriptor. */
1511 uint32_t *desc = descs->list + slot*4;
1512 desc[0] = va;
1513 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
1514 S_008F04_STRIDE(stride) |
1515 S_008F04_SWIZZLE_ENABLE(swizzle);
1516 desc[2] = num_records;
1517 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1518 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1519 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1520 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1521 S_008F0C_INDEX_STRIDE(index_stride) |
1522 S_008F0C_ADD_TID_ENABLE(add_tid);
1523
1524 if (sctx->chip_class >= GFX9)
1525 assert(!swizzle || element_size == 1); /* always 4 bytes on GFX9 */
1526 else
1527 desc[3] |= S_008F0C_ELEMENT_SIZE(element_size);
1528
1529 if (sctx->chip_class >= GFX10) {
1530 desc[3] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
1531 S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) |
1532 S_008F0C_RESOURCE_LEVEL(1);
1533 } else {
1534 desc[3] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1535 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
1536 }
1537
1538 pipe_resource_reference(&buffers->buffers[slot], buffer);
1539 radeon_add_to_buffer_list(sctx, sctx->gfx_cs,
1540 si_resource(buffer),
1541 RADEON_USAGE_READWRITE, buffers->priority);
1542 buffers->enabled_mask |= 1u << slot;
1543 } else {
1544 /* Clear the descriptor. */
1545 memset(descs->list + slot*4, 0, sizeof(uint32_t) * 4);
1546 buffers->enabled_mask &= ~(1u << slot);
1547 }
1548
1549 sctx->descriptors_dirty |= 1u << SI_DESCS_RW_BUFFERS;
1550 }
1551
1552 /* INTERNAL CONST BUFFERS */
1553
1554 static void si_set_polygon_stipple(struct pipe_context *ctx,
1555 const struct pipe_poly_stipple *state)
1556 {
1557 struct si_context *sctx = (struct si_context *)ctx;
1558 struct pipe_constant_buffer cb = {};
1559 unsigned stipple[32];
1560 int i;
1561
1562 for (i = 0; i < 32; i++)
1563 stipple[i] = util_bitreverse(state->stipple[i]);
1564
1565 cb.user_buffer = stipple;
1566 cb.buffer_size = sizeof(stipple);
1567
1568 si_set_rw_buffer(sctx, SI_PS_CONST_POLY_STIPPLE, &cb);
1569 }
1570
1571 /* TEXTURE METADATA ENABLE/DISABLE */
1572
1573 static void
1574 si_resident_handles_update_needs_color_decompress(struct si_context *sctx)
1575 {
1576 util_dynarray_clear(&sctx->resident_tex_needs_color_decompress);
1577 util_dynarray_clear(&sctx->resident_img_needs_color_decompress);
1578
1579 util_dynarray_foreach(&sctx->resident_tex_handles,
1580 struct si_texture_handle *, tex_handle) {
1581 struct pipe_resource *res = (*tex_handle)->view->texture;
1582 struct si_texture *tex;
1583
1584 if (!res || res->target == PIPE_BUFFER)
1585 continue;
1586
1587 tex = (struct si_texture *)res;
1588 if (!color_needs_decompression(tex))
1589 continue;
1590
1591 util_dynarray_append(&sctx->resident_tex_needs_color_decompress,
1592 struct si_texture_handle *, *tex_handle);
1593 }
1594
1595 util_dynarray_foreach(&sctx->resident_img_handles,
1596 struct si_image_handle *, img_handle) {
1597 struct pipe_image_view *view = &(*img_handle)->view;
1598 struct pipe_resource *res = view->resource;
1599 struct si_texture *tex;
1600
1601 if (!res || res->target == PIPE_BUFFER)
1602 continue;
1603
1604 tex = (struct si_texture *)res;
1605 if (!color_needs_decompression(tex))
1606 continue;
1607
1608 util_dynarray_append(&sctx->resident_img_needs_color_decompress,
1609 struct si_image_handle *, *img_handle);
1610 }
1611 }
1612
1613 /* CMASK can be enabled (for fast clear) and disabled (for texture export)
1614 * while the texture is bound, possibly by a different context. In that case,
1615 * call this function to update needs_*_decompress_masks.
1616 */
1617 void si_update_needs_color_decompress_masks(struct si_context *sctx)
1618 {
1619 for (int i = 0; i < SI_NUM_SHADERS; ++i) {
1620 si_samplers_update_needs_color_decompress_mask(&sctx->samplers[i]);
1621 si_images_update_needs_color_decompress_mask(&sctx->images[i]);
1622 si_update_shader_needs_decompress_mask(sctx, i);
1623 }
1624
1625 si_resident_handles_update_needs_color_decompress(sctx);
1626 }
1627
1628 /* BUFFER DISCARD/INVALIDATION */
1629
1630 /* Reset descriptors of buffer resources after \p buf has been invalidated.
1631 * If buf == NULL, reset all descriptors.
1632 */
1633 static void si_reset_buffer_resources(struct si_context *sctx,
1634 struct si_buffer_resources *buffers,
1635 unsigned descriptors_idx,
1636 unsigned slot_mask,
1637 struct pipe_resource *buf,
1638 enum radeon_bo_priority priority)
1639 {
1640 struct si_descriptors *descs = &sctx->descriptors[descriptors_idx];
1641 unsigned mask = buffers->enabled_mask & slot_mask;
1642
1643 while (mask) {
1644 unsigned i = u_bit_scan(&mask);
1645 struct pipe_resource *buffer = buffers->buffers[i];
1646
1647 if (buffer && (!buf || buffer == buf)) {
1648 si_set_buf_desc_address(si_resource(buffer), buffers->offsets[i],
1649 descs->list + i*4);
1650 sctx->descriptors_dirty |= 1u << descriptors_idx;
1651
1652 radeon_add_to_gfx_buffer_list_check_mem(sctx,
1653 si_resource(buffer),
1654 buffers->writable_mask & (1u << i) ?
1655 RADEON_USAGE_READWRITE :
1656 RADEON_USAGE_READ,
1657 priority, true);
1658 }
1659 }
1660 }
1661
1662 /* Update all buffer bindings where the buffer is bound, including
1663 * all resource descriptors. This is invalidate_buffer without
1664 * the invalidation.
1665 *
1666 * If buf == NULL, update all buffer bindings.
1667 */
1668 void si_rebind_buffer(struct si_context *sctx, struct pipe_resource *buf)
1669 {
1670 struct si_resource *buffer = si_resource(buf);
1671 unsigned i, shader;
1672 unsigned num_elems = sctx->vertex_elements ?
1673 sctx->vertex_elements->count : 0;
1674
1675 /* We changed the buffer, now we need to bind it where the old one
1676 * was bound. This consists of 2 things:
1677 * 1) Updating the resource descriptor and dirtying it.
1678 * 2) Adding a relocation to the CS, so that it's usable.
1679 */
1680
1681 /* Vertex buffers. */
1682 if (!buffer) {
1683 if (num_elems)
1684 sctx->vertex_buffers_dirty = true;
1685 } else if (buffer->bind_history & PIPE_BIND_VERTEX_BUFFER) {
1686 for (i = 0; i < num_elems; i++) {
1687 int vb = sctx->vertex_elements->vertex_buffer_index[i];
1688
1689 if (vb >= ARRAY_SIZE(sctx->vertex_buffer))
1690 continue;
1691 if (!sctx->vertex_buffer[vb].buffer.resource)
1692 continue;
1693
1694 if (sctx->vertex_buffer[vb].buffer.resource == buf) {
1695 sctx->vertex_buffers_dirty = true;
1696 break;
1697 }
1698 }
1699 }
1700
1701 /* Streamout buffers. (other internal buffers can't be invalidated) */
1702 if (!buffer || buffer->bind_history & PIPE_BIND_STREAM_OUTPUT) {
1703 for (i = SI_VS_STREAMOUT_BUF0; i <= SI_VS_STREAMOUT_BUF3; i++) {
1704 struct si_buffer_resources *buffers = &sctx->rw_buffers;
1705 struct si_descriptors *descs =
1706 &sctx->descriptors[SI_DESCS_RW_BUFFERS];
1707 struct pipe_resource *buffer = buffers->buffers[i];
1708
1709 if (!buffer || (buf && buffer != buf))
1710 continue;
1711
1712 si_set_buf_desc_address(si_resource(buffer), buffers->offsets[i],
1713 descs->list + i*4);
1714 sctx->descriptors_dirty |= 1u << SI_DESCS_RW_BUFFERS;
1715
1716 radeon_add_to_gfx_buffer_list_check_mem(sctx,
1717 si_resource(buffer),
1718 RADEON_USAGE_WRITE,
1719 RADEON_PRIO_SHADER_RW_BUFFER,
1720 true);
1721
1722 /* Update the streamout state. */
1723 if (sctx->streamout.begin_emitted)
1724 si_emit_streamout_end(sctx);
1725 sctx->streamout.append_bitmask =
1726 sctx->streamout.enabled_mask;
1727 si_streamout_buffers_dirty(sctx);
1728 }
1729 }
1730
1731 /* Constant and shader buffers. */
1732 if (!buffer || buffer->bind_history & PIPE_BIND_CONSTANT_BUFFER) {
1733 for (shader = 0; shader < SI_NUM_SHADERS; shader++)
1734 si_reset_buffer_resources(sctx, &sctx->const_and_shader_buffers[shader],
1735 si_const_and_shader_buffer_descriptors_idx(shader),
1736 u_bit_consecutive(SI_NUM_SHADER_BUFFERS, SI_NUM_CONST_BUFFERS),
1737 buf,
1738 sctx->const_and_shader_buffers[shader].priority_constbuf);
1739 }
1740
1741 if (!buffer || buffer->bind_history & PIPE_BIND_SHADER_BUFFER) {
1742 for (shader = 0; shader < SI_NUM_SHADERS; shader++)
1743 si_reset_buffer_resources(sctx, &sctx->const_and_shader_buffers[shader],
1744 si_const_and_shader_buffer_descriptors_idx(shader),
1745 u_bit_consecutive(0, SI_NUM_SHADER_BUFFERS),
1746 buf,
1747 sctx->const_and_shader_buffers[shader].priority);
1748 }
1749
1750 if (!buffer || buffer->bind_history & PIPE_BIND_SAMPLER_VIEW) {
1751 /* Texture buffers - update bindings. */
1752 for (shader = 0; shader < SI_NUM_SHADERS; shader++) {
1753 struct si_samplers *samplers = &sctx->samplers[shader];
1754 struct si_descriptors *descs =
1755 si_sampler_and_image_descriptors(sctx, shader);
1756 unsigned mask = samplers->enabled_mask;
1757
1758 while (mask) {
1759 unsigned i = u_bit_scan(&mask);
1760 struct pipe_resource *buffer = samplers->views[i]->texture;
1761
1762 if (buffer && buffer->target == PIPE_BUFFER &&
1763 (!buf || buffer == buf)) {
1764 unsigned desc_slot = si_get_sampler_slot(i);
1765
1766 si_set_buf_desc_address(si_resource(buffer),
1767 samplers->views[i]->u.buf.offset,
1768 descs->list + desc_slot * 16 + 4);
1769 sctx->descriptors_dirty |=
1770 1u << si_sampler_and_image_descriptors_idx(shader);
1771
1772 radeon_add_to_gfx_buffer_list_check_mem(
1773 sctx, si_resource(buffer),
1774 RADEON_USAGE_READ,
1775 RADEON_PRIO_SAMPLER_BUFFER, true);
1776 }
1777 }
1778 }
1779 }
1780
1781 /* Shader images */
1782 if (!buffer || buffer->bind_history & PIPE_BIND_SHADER_IMAGE) {
1783 for (shader = 0; shader < SI_NUM_SHADERS; ++shader) {
1784 struct si_images *images = &sctx->images[shader];
1785 struct si_descriptors *descs =
1786 si_sampler_and_image_descriptors(sctx, shader);
1787 unsigned mask = images->enabled_mask;
1788
1789 while (mask) {
1790 unsigned i = u_bit_scan(&mask);
1791 struct pipe_resource *buffer = images->views[i].resource;
1792
1793 if (buffer && buffer->target == PIPE_BUFFER &&
1794 (!buf || buffer == buf)) {
1795 unsigned desc_slot = si_get_image_slot(i);
1796
1797 if (images->views[i].access & PIPE_IMAGE_ACCESS_WRITE)
1798 si_mark_image_range_valid(&images->views[i]);
1799
1800 si_set_buf_desc_address(si_resource(buffer),
1801 images->views[i].u.buf.offset,
1802 descs->list + desc_slot * 8 + 4);
1803 sctx->descriptors_dirty |=
1804 1u << si_sampler_and_image_descriptors_idx(shader);
1805
1806 radeon_add_to_gfx_buffer_list_check_mem(
1807 sctx, si_resource(buffer),
1808 RADEON_USAGE_READWRITE,
1809 RADEON_PRIO_SAMPLER_BUFFER, true);
1810 }
1811 }
1812 }
1813 }
1814
1815 /* Bindless texture handles */
1816 if (!buffer || buffer->texture_handle_allocated) {
1817 struct si_descriptors *descs = &sctx->bindless_descriptors;
1818
1819 util_dynarray_foreach(&sctx->resident_tex_handles,
1820 struct si_texture_handle *, tex_handle) {
1821 struct pipe_sampler_view *view = (*tex_handle)->view;
1822 unsigned desc_slot = (*tex_handle)->desc_slot;
1823 struct pipe_resource *buffer = view->texture;
1824
1825 if (buffer && buffer->target == PIPE_BUFFER &&
1826 (!buf || buffer == buf)) {
1827 si_set_buf_desc_address(si_resource(buffer),
1828 view->u.buf.offset,
1829 descs->list +
1830 desc_slot * 16 + 4);
1831
1832 (*tex_handle)->desc_dirty = true;
1833 sctx->bindless_descriptors_dirty = true;
1834
1835 radeon_add_to_gfx_buffer_list_check_mem(
1836 sctx, si_resource(buffer),
1837 RADEON_USAGE_READ,
1838 RADEON_PRIO_SAMPLER_BUFFER, true);
1839 }
1840 }
1841 }
1842
1843 /* Bindless image handles */
1844 if (!buffer || buffer->image_handle_allocated) {
1845 struct si_descriptors *descs = &sctx->bindless_descriptors;
1846
1847 util_dynarray_foreach(&sctx->resident_img_handles,
1848 struct si_image_handle *, img_handle) {
1849 struct pipe_image_view *view = &(*img_handle)->view;
1850 unsigned desc_slot = (*img_handle)->desc_slot;
1851 struct pipe_resource *buffer = view->resource;
1852
1853 if (buffer && buffer->target == PIPE_BUFFER &&
1854 (!buf || buffer == buf)) {
1855 if (view->access & PIPE_IMAGE_ACCESS_WRITE)
1856 si_mark_image_range_valid(view);
1857
1858 si_set_buf_desc_address(si_resource(buffer),
1859 view->u.buf.offset,
1860 descs->list +
1861 desc_slot * 16 + 4);
1862
1863 (*img_handle)->desc_dirty = true;
1864 sctx->bindless_descriptors_dirty = true;
1865
1866 radeon_add_to_gfx_buffer_list_check_mem(
1867 sctx, si_resource(buffer),
1868 RADEON_USAGE_READWRITE,
1869 RADEON_PRIO_SAMPLER_BUFFER, true);
1870 }
1871 }
1872 }
1873
1874 if (buffer) {
1875 /* Do the same for other contexts. They will invoke this function
1876 * with buffer == NULL.
1877 */
1878 unsigned new_counter = p_atomic_inc_return(&sctx->screen->dirty_buf_counter);
1879
1880 /* Skip the update for the current context, because we have already updated
1881 * the buffer bindings.
1882 */
1883 if (new_counter == sctx->last_dirty_buf_counter + 1)
1884 sctx->last_dirty_buf_counter = new_counter;
1885 }
1886 }
1887
1888 static void si_upload_bindless_descriptor(struct si_context *sctx,
1889 unsigned desc_slot,
1890 unsigned num_dwords)
1891 {
1892 struct si_descriptors *desc = &sctx->bindless_descriptors;
1893 unsigned desc_slot_offset = desc_slot * 16;
1894 uint32_t *data;
1895 uint64_t va;
1896
1897 data = desc->list + desc_slot_offset;
1898 va = desc->gpu_address + desc_slot_offset * 4;
1899
1900 si_cp_write_data(sctx, desc->buffer, va - desc->buffer->gpu_address,
1901 num_dwords * 4, V_370_TC_L2, V_370_ME, data);
1902 }
1903
1904 static void si_upload_bindless_descriptors(struct si_context *sctx)
1905 {
1906 if (!sctx->bindless_descriptors_dirty)
1907 return;
1908
1909 /* Wait for graphics/compute to be idle before updating the resident
1910 * descriptors directly in memory, in case the GPU is using them.
1911 */
1912 sctx->flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
1913 SI_CONTEXT_CS_PARTIAL_FLUSH;
1914 sctx->emit_cache_flush(sctx);
1915
1916 util_dynarray_foreach(&sctx->resident_tex_handles,
1917 struct si_texture_handle *, tex_handle) {
1918 unsigned desc_slot = (*tex_handle)->desc_slot;
1919
1920 if (!(*tex_handle)->desc_dirty)
1921 continue;
1922
1923 si_upload_bindless_descriptor(sctx, desc_slot, 16);
1924 (*tex_handle)->desc_dirty = false;
1925 }
1926
1927 util_dynarray_foreach(&sctx->resident_img_handles,
1928 struct si_image_handle *, img_handle) {
1929 unsigned desc_slot = (*img_handle)->desc_slot;
1930
1931 if (!(*img_handle)->desc_dirty)
1932 continue;
1933
1934 si_upload_bindless_descriptor(sctx, desc_slot, 8);
1935 (*img_handle)->desc_dirty = false;
1936 }
1937
1938 /* Invalidate L1 because it doesn't know that L2 changed. */
1939 sctx->flags |= SI_CONTEXT_INV_SCACHE;
1940 sctx->emit_cache_flush(sctx);
1941
1942 sctx->bindless_descriptors_dirty = false;
1943 }
1944
1945 /* Update mutable image descriptor fields of all resident textures. */
1946 static void si_update_bindless_texture_descriptor(struct si_context *sctx,
1947 struct si_texture_handle *tex_handle)
1948 {
1949 struct si_sampler_view *sview = (struct si_sampler_view *)tex_handle->view;
1950 struct si_descriptors *desc = &sctx->bindless_descriptors;
1951 unsigned desc_slot_offset = tex_handle->desc_slot * 16;
1952 uint32_t desc_list[16];
1953
1954 if (sview->base.texture->target == PIPE_BUFFER)
1955 return;
1956
1957 memcpy(desc_list, desc->list + desc_slot_offset, sizeof(desc_list));
1958 si_set_sampler_view_desc(sctx, sview, &tex_handle->sstate,
1959 desc->list + desc_slot_offset);
1960
1961 if (memcmp(desc_list, desc->list + desc_slot_offset,
1962 sizeof(desc_list))) {
1963 tex_handle->desc_dirty = true;
1964 sctx->bindless_descriptors_dirty = true;
1965 }
1966 }
1967
1968 static void si_update_bindless_image_descriptor(struct si_context *sctx,
1969 struct si_image_handle *img_handle)
1970 {
1971 struct si_descriptors *desc = &sctx->bindless_descriptors;
1972 unsigned desc_slot_offset = img_handle->desc_slot * 16;
1973 struct pipe_image_view *view = &img_handle->view;
1974 struct pipe_resource *res = view->resource;
1975 uint32_t image_desc[16];
1976 unsigned desc_size = (res->nr_samples >= 2 ? 16 : 8) * 4;
1977
1978 if (res->target == PIPE_BUFFER)
1979 return;
1980
1981 memcpy(image_desc, desc->list + desc_slot_offset, desc_size);
1982 si_set_shader_image_desc(sctx, view, true,
1983 desc->list + desc_slot_offset,
1984 desc->list + desc_slot_offset + 8);
1985
1986 if (memcmp(image_desc, desc->list + desc_slot_offset, desc_size)) {
1987 img_handle->desc_dirty = true;
1988 sctx->bindless_descriptors_dirty = true;
1989 }
1990 }
1991
1992 static void si_update_all_resident_texture_descriptors(struct si_context *sctx)
1993 {
1994 util_dynarray_foreach(&sctx->resident_tex_handles,
1995 struct si_texture_handle *, tex_handle) {
1996 si_update_bindless_texture_descriptor(sctx, *tex_handle);
1997 }
1998
1999 util_dynarray_foreach(&sctx->resident_img_handles,
2000 struct si_image_handle *, img_handle) {
2001 si_update_bindless_image_descriptor(sctx, *img_handle);
2002 }
2003
2004 si_upload_bindless_descriptors(sctx);
2005 }
2006
2007 /* Update mutable image descriptor fields of all bound textures. */
2008 void si_update_all_texture_descriptors(struct si_context *sctx)
2009 {
2010 unsigned shader;
2011
2012 for (shader = 0; shader < SI_NUM_SHADERS; shader++) {
2013 struct si_samplers *samplers = &sctx->samplers[shader];
2014 struct si_images *images = &sctx->images[shader];
2015 unsigned mask;
2016
2017 /* Images. */
2018 mask = images->enabled_mask;
2019 while (mask) {
2020 unsigned i = u_bit_scan(&mask);
2021 struct pipe_image_view *view = &images->views[i];
2022
2023 if (!view->resource ||
2024 view->resource->target == PIPE_BUFFER)
2025 continue;
2026
2027 si_set_shader_image(sctx, shader, i, view, true);
2028 }
2029
2030 /* Sampler views. */
2031 mask = samplers->enabled_mask;
2032 while (mask) {
2033 unsigned i = u_bit_scan(&mask);
2034 struct pipe_sampler_view *view = samplers->views[i];
2035
2036 if (!view ||
2037 !view->texture ||
2038 view->texture->target == PIPE_BUFFER)
2039 continue;
2040
2041 si_set_sampler_view(sctx, shader, i,
2042 samplers->views[i], true);
2043 }
2044
2045 si_update_shader_needs_decompress_mask(sctx, shader);
2046 }
2047
2048 si_update_all_resident_texture_descriptors(sctx);
2049 si_update_ps_colorbuf0_slot(sctx);
2050 }
2051
2052 /* SHADER USER DATA */
2053
2054 static void si_mark_shader_pointers_dirty(struct si_context *sctx,
2055 unsigned shader)
2056 {
2057 sctx->shader_pointers_dirty |=
2058 u_bit_consecutive(SI_DESCS_FIRST_SHADER + shader * SI_NUM_SHADER_DESCS,
2059 SI_NUM_SHADER_DESCS);
2060
2061 if (shader == PIPE_SHADER_VERTEX)
2062 sctx->vertex_buffer_pointer_dirty = sctx->vb_descriptors_buffer != NULL;
2063
2064 si_mark_atom_dirty(sctx, &sctx->atoms.s.shader_pointers);
2065 }
2066
2067 static void si_shader_pointers_begin_new_cs(struct si_context *sctx)
2068 {
2069 sctx->shader_pointers_dirty = u_bit_consecutive(0, SI_NUM_DESCS);
2070 sctx->vertex_buffer_pointer_dirty = sctx->vb_descriptors_buffer != NULL;
2071 si_mark_atom_dirty(sctx, &sctx->atoms.s.shader_pointers);
2072 sctx->graphics_bindless_pointer_dirty = sctx->bindless_descriptors.buffer != NULL;
2073 sctx->compute_bindless_pointer_dirty = sctx->bindless_descriptors.buffer != NULL;
2074 }
2075
2076 /* Set a base register address for user data constants in the given shader.
2077 * This assigns a mapping from PIPE_SHADER_* to SPI_SHADER_USER_DATA_*.
2078 */
2079 static void si_set_user_data_base(struct si_context *sctx,
2080 unsigned shader, uint32_t new_base)
2081 {
2082 uint32_t *base = &sctx->shader_pointers.sh_base[shader];
2083
2084 if (*base != new_base) {
2085 *base = new_base;
2086
2087 if (new_base)
2088 si_mark_shader_pointers_dirty(sctx, shader);
2089
2090 /* Any change in enabled shader stages requires re-emitting
2091 * the VS state SGPR, because it contains the clamp_vertex_color
2092 * state, which can be done in VS, TES, and GS.
2093 */
2094 sctx->last_vs_state = ~0;
2095 }
2096 }
2097
2098 /* This must be called when these are changed between enabled and disabled
2099 * - geometry shader
2100 * - tessellation evaluation shader
2101 * - NGG
2102 */
2103 void si_shader_change_notify(struct si_context *sctx)
2104 {
2105 /* VS can be bound as VS, ES, or LS. */
2106 if (sctx->tes_shader.cso) {
2107 if (sctx->chip_class >= GFX10) {
2108 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX,
2109 R_00B430_SPI_SHADER_USER_DATA_HS_0);
2110 } else if (sctx->chip_class == GFX9) {
2111 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX,
2112 R_00B430_SPI_SHADER_USER_DATA_LS_0);
2113 } else {
2114 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX,
2115 R_00B530_SPI_SHADER_USER_DATA_LS_0);
2116 }
2117 } else if (sctx->chip_class >= GFX10) {
2118 if (sctx->ngg || sctx->gs_shader.cso) {
2119 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX,
2120 R_00B230_SPI_SHADER_USER_DATA_GS_0);
2121 } else {
2122 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX,
2123 R_00B130_SPI_SHADER_USER_DATA_VS_0);
2124 }
2125 } else if (sctx->gs_shader.cso) {
2126 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX,
2127 R_00B330_SPI_SHADER_USER_DATA_ES_0);
2128 } else {
2129 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX,
2130 R_00B130_SPI_SHADER_USER_DATA_VS_0);
2131 }
2132
2133 /* TES can be bound as ES, VS, or not bound. */
2134 if (sctx->tes_shader.cso) {
2135 if (sctx->chip_class >= GFX10) {
2136 if (sctx->ngg || sctx->gs_shader.cso) {
2137 si_set_user_data_base(sctx, PIPE_SHADER_TESS_EVAL,
2138 R_00B230_SPI_SHADER_USER_DATA_GS_0);
2139 } else {
2140 si_set_user_data_base(sctx, PIPE_SHADER_TESS_EVAL,
2141 R_00B130_SPI_SHADER_USER_DATA_VS_0);
2142 }
2143 } else if (sctx->gs_shader.cso) {
2144 si_set_user_data_base(sctx, PIPE_SHADER_TESS_EVAL,
2145 R_00B330_SPI_SHADER_USER_DATA_ES_0);
2146 } else {
2147 si_set_user_data_base(sctx, PIPE_SHADER_TESS_EVAL,
2148 R_00B130_SPI_SHADER_USER_DATA_VS_0);
2149 }
2150 } else {
2151 si_set_user_data_base(sctx, PIPE_SHADER_TESS_EVAL, 0);
2152 }
2153 }
2154
2155 static void si_emit_shader_pointer_head(struct radeon_cmdbuf *cs,
2156 unsigned sh_offset,
2157 unsigned pointer_count)
2158 {
2159 radeon_emit(cs, PKT3(PKT3_SET_SH_REG, pointer_count, 0));
2160 radeon_emit(cs, (sh_offset - SI_SH_REG_OFFSET) >> 2);
2161 }
2162
2163 static void si_emit_shader_pointer_body(struct si_screen *sscreen,
2164 struct radeon_cmdbuf *cs,
2165 uint64_t va)
2166 {
2167 radeon_emit(cs, va);
2168
2169 assert(va == 0 || (va >> 32) == sscreen->info.address32_hi);
2170 }
2171
2172 static void si_emit_shader_pointer(struct si_context *sctx,
2173 struct si_descriptors *desc,
2174 unsigned sh_base)
2175 {
2176 struct radeon_cmdbuf *cs = sctx->gfx_cs;
2177 unsigned sh_offset = sh_base + desc->shader_userdata_offset;
2178
2179 si_emit_shader_pointer_head(cs, sh_offset, 1);
2180 si_emit_shader_pointer_body(sctx->screen, cs, desc->gpu_address);
2181 }
2182
2183 static void si_emit_consecutive_shader_pointers(struct si_context *sctx,
2184 unsigned pointer_mask,
2185 unsigned sh_base)
2186 {
2187 if (!sh_base)
2188 return;
2189
2190 struct radeon_cmdbuf *cs = sctx->gfx_cs;
2191 unsigned mask = sctx->shader_pointers_dirty & pointer_mask;
2192
2193 while (mask) {
2194 int start, count;
2195 u_bit_scan_consecutive_range(&mask, &start, &count);
2196
2197 struct si_descriptors *descs = &sctx->descriptors[start];
2198 unsigned sh_offset = sh_base + descs->shader_userdata_offset;
2199
2200 si_emit_shader_pointer_head(cs, sh_offset, count);
2201 for (int i = 0; i < count; i++)
2202 si_emit_shader_pointer_body(sctx->screen, cs,
2203 descs[i].gpu_address);
2204 }
2205 }
2206
2207 static void si_emit_global_shader_pointers(struct si_context *sctx,
2208 struct si_descriptors *descs)
2209 {
2210 if (sctx->chip_class >= GFX10) {
2211 si_emit_shader_pointer(sctx, descs,
2212 R_00B030_SPI_SHADER_USER_DATA_PS_0);
2213 /* HW VS stage only used in non-NGG mode. */
2214 si_emit_shader_pointer(sctx, descs,
2215 R_00B130_SPI_SHADER_USER_DATA_VS_0);
2216 si_emit_shader_pointer(sctx, descs,
2217 R_00B230_SPI_SHADER_USER_DATA_GS_0);
2218 si_emit_shader_pointer(sctx, descs,
2219 R_00B430_SPI_SHADER_USER_DATA_HS_0);
2220 return;
2221 } else if (sctx->chip_class == GFX9) {
2222 /* Broadcast it to all shader stages. */
2223 si_emit_shader_pointer(sctx, descs,
2224 R_00B530_SPI_SHADER_USER_DATA_COMMON_0);
2225 return;
2226 }
2227
2228 si_emit_shader_pointer(sctx, descs,
2229 R_00B030_SPI_SHADER_USER_DATA_PS_0);
2230 si_emit_shader_pointer(sctx, descs,
2231 R_00B130_SPI_SHADER_USER_DATA_VS_0);
2232 si_emit_shader_pointer(sctx, descs,
2233 R_00B330_SPI_SHADER_USER_DATA_ES_0);
2234 si_emit_shader_pointer(sctx, descs,
2235 R_00B230_SPI_SHADER_USER_DATA_GS_0);
2236 si_emit_shader_pointer(sctx, descs,
2237 R_00B430_SPI_SHADER_USER_DATA_HS_0);
2238 si_emit_shader_pointer(sctx, descs,
2239 R_00B530_SPI_SHADER_USER_DATA_LS_0);
2240 }
2241
2242 void si_emit_graphics_shader_pointers(struct si_context *sctx)
2243 {
2244 uint32_t *sh_base = sctx->shader_pointers.sh_base;
2245
2246 if (sctx->shader_pointers_dirty & (1 << SI_DESCS_RW_BUFFERS)) {
2247 si_emit_global_shader_pointers(sctx,
2248 &sctx->descriptors[SI_DESCS_RW_BUFFERS]);
2249 }
2250
2251 si_emit_consecutive_shader_pointers(sctx, SI_DESCS_SHADER_MASK(VERTEX),
2252 sh_base[PIPE_SHADER_VERTEX]);
2253 si_emit_consecutive_shader_pointers(sctx, SI_DESCS_SHADER_MASK(TESS_EVAL),
2254 sh_base[PIPE_SHADER_TESS_EVAL]);
2255 si_emit_consecutive_shader_pointers(sctx, SI_DESCS_SHADER_MASK(FRAGMENT),
2256 sh_base[PIPE_SHADER_FRAGMENT]);
2257 si_emit_consecutive_shader_pointers(sctx, SI_DESCS_SHADER_MASK(TESS_CTRL),
2258 sh_base[PIPE_SHADER_TESS_CTRL]);
2259 si_emit_consecutive_shader_pointers(sctx, SI_DESCS_SHADER_MASK(GEOMETRY),
2260 sh_base[PIPE_SHADER_GEOMETRY]);
2261
2262 sctx->shader_pointers_dirty &=
2263 ~u_bit_consecutive(SI_DESCS_RW_BUFFERS, SI_DESCS_FIRST_COMPUTE);
2264
2265 if (sctx->vertex_buffer_pointer_dirty) {
2266 struct radeon_cmdbuf *cs = sctx->gfx_cs;
2267
2268 /* Find the location of the VB descriptor pointer. */
2269 /* TODO: In the future, the pointer will be packed in unused
2270 * bits of the first 2 VB descriptors. */
2271 unsigned sh_dw_offset = SI_VS_NUM_USER_SGPR;
2272 if (sctx->chip_class >= GFX9) {
2273 if (sctx->tes_shader.cso)
2274 sh_dw_offset = GFX9_TCS_NUM_USER_SGPR;
2275 else if (sctx->gs_shader.cso)
2276 sh_dw_offset = GFX9_VSGS_NUM_USER_SGPR;
2277 }
2278
2279 unsigned sh_offset = sh_base[PIPE_SHADER_VERTEX] + sh_dw_offset * 4;
2280 si_emit_shader_pointer_head(cs, sh_offset, 1);
2281 si_emit_shader_pointer_body(sctx->screen, cs,
2282 sctx->vb_descriptors_buffer->gpu_address +
2283 sctx->vb_descriptors_offset);
2284 sctx->vertex_buffer_pointer_dirty = false;
2285 }
2286
2287 if (sctx->graphics_bindless_pointer_dirty) {
2288 si_emit_global_shader_pointers(sctx,
2289 &sctx->bindless_descriptors);
2290 sctx->graphics_bindless_pointer_dirty = false;
2291 }
2292 }
2293
2294 void si_emit_compute_shader_pointers(struct si_context *sctx)
2295 {
2296 unsigned base = R_00B900_COMPUTE_USER_DATA_0;
2297
2298 si_emit_consecutive_shader_pointers(sctx, SI_DESCS_SHADER_MASK(COMPUTE),
2299 R_00B900_COMPUTE_USER_DATA_0);
2300 sctx->shader_pointers_dirty &= ~SI_DESCS_SHADER_MASK(COMPUTE);
2301
2302 if (sctx->compute_bindless_pointer_dirty) {
2303 si_emit_shader_pointer(sctx, &sctx->bindless_descriptors, base);
2304 sctx->compute_bindless_pointer_dirty = false;
2305 }
2306 }
2307
2308 /* BINDLESS */
2309
2310 static void si_init_bindless_descriptors(struct si_context *sctx,
2311 struct si_descriptors *desc,
2312 short shader_userdata_rel_index,
2313 unsigned num_elements)
2314 {
2315 ASSERTED unsigned desc_slot;
2316
2317 si_init_descriptors(desc, shader_userdata_rel_index, 16, num_elements);
2318 sctx->bindless_descriptors.num_active_slots = num_elements;
2319
2320 /* The first bindless descriptor is stored at slot 1, because 0 is not
2321 * considered to be a valid handle.
2322 */
2323 sctx->num_bindless_descriptors = 1;
2324
2325 /* Track which bindless slots are used (or not). */
2326 util_idalloc_init(&sctx->bindless_used_slots);
2327 util_idalloc_resize(&sctx->bindless_used_slots, num_elements);
2328
2329 /* Reserve slot 0 because it's an invalid handle for bindless. */
2330 desc_slot = util_idalloc_alloc(&sctx->bindless_used_slots);
2331 assert(desc_slot == 0);
2332 }
2333
2334 static void si_release_bindless_descriptors(struct si_context *sctx)
2335 {
2336 si_release_descriptors(&sctx->bindless_descriptors);
2337 util_idalloc_fini(&sctx->bindless_used_slots);
2338 }
2339
2340 static unsigned si_get_first_free_bindless_slot(struct si_context *sctx)
2341 {
2342 struct si_descriptors *desc = &sctx->bindless_descriptors;
2343 unsigned desc_slot;
2344
2345 desc_slot = util_idalloc_alloc(&sctx->bindless_used_slots);
2346 if (desc_slot >= desc->num_elements) {
2347 /* The array of bindless descriptors is full, resize it. */
2348 unsigned slot_size = desc->element_dw_size * 4;
2349 unsigned new_num_elements = desc->num_elements * 2;
2350
2351 desc->list = REALLOC(desc->list, desc->num_elements * slot_size,
2352 new_num_elements * slot_size);
2353 desc->num_elements = new_num_elements;
2354 desc->num_active_slots = new_num_elements;
2355 }
2356
2357 assert(desc_slot);
2358 return desc_slot;
2359 }
2360
2361 static unsigned
2362 si_create_bindless_descriptor(struct si_context *sctx, uint32_t *desc_list,
2363 unsigned size)
2364 {
2365 struct si_descriptors *desc = &sctx->bindless_descriptors;
2366 unsigned desc_slot, desc_slot_offset;
2367
2368 /* Find a free slot. */
2369 desc_slot = si_get_first_free_bindless_slot(sctx);
2370
2371 /* For simplicity, sampler and image bindless descriptors use fixed
2372 * 16-dword slots for now. Image descriptors only need 8-dword but this
2373 * doesn't really matter because no real apps use image handles.
2374 */
2375 desc_slot_offset = desc_slot * 16;
2376
2377 /* Copy the descriptor into the array. */
2378 memcpy(desc->list + desc_slot_offset, desc_list, size);
2379
2380 /* Re-upload the whole array of bindless descriptors into a new buffer.
2381 */
2382 if (!si_upload_descriptors(sctx, desc))
2383 return 0;
2384
2385 /* Make sure to re-emit the shader pointers for all stages. */
2386 sctx->graphics_bindless_pointer_dirty = true;
2387 sctx->compute_bindless_pointer_dirty = true;
2388
2389 return desc_slot;
2390 }
2391
2392 static void si_update_bindless_buffer_descriptor(struct si_context *sctx,
2393 unsigned desc_slot,
2394 struct pipe_resource *resource,
2395 uint64_t offset,
2396 bool *desc_dirty)
2397 {
2398 struct si_descriptors *desc = &sctx->bindless_descriptors;
2399 struct si_resource *buf = si_resource(resource);
2400 unsigned desc_slot_offset = desc_slot * 16;
2401 uint32_t *desc_list = desc->list + desc_slot_offset + 4;
2402 uint64_t old_desc_va;
2403
2404 assert(resource->target == PIPE_BUFFER);
2405
2406 /* Retrieve the old buffer addr from the descriptor. */
2407 old_desc_va = si_desc_extract_buffer_address(desc_list);
2408
2409 if (old_desc_va != buf->gpu_address + offset) {
2410 /* The buffer has been invalidated when the handle wasn't
2411 * resident, update the descriptor and the dirty flag.
2412 */
2413 si_set_buf_desc_address(buf, offset, &desc_list[0]);
2414
2415 *desc_dirty = true;
2416 }
2417 }
2418
2419 static uint64_t si_create_texture_handle(struct pipe_context *ctx,
2420 struct pipe_sampler_view *view,
2421 const struct pipe_sampler_state *state)
2422 {
2423 struct si_sampler_view *sview = (struct si_sampler_view *)view;
2424 struct si_context *sctx = (struct si_context *)ctx;
2425 struct si_texture_handle *tex_handle;
2426 struct si_sampler_state *sstate;
2427 uint32_t desc_list[16];
2428 uint64_t handle;
2429
2430 tex_handle = CALLOC_STRUCT(si_texture_handle);
2431 if (!tex_handle)
2432 return 0;
2433
2434 memset(desc_list, 0, sizeof(desc_list));
2435 si_init_descriptor_list(&desc_list[0], 16, 1, null_texture_descriptor);
2436
2437 sstate = ctx->create_sampler_state(ctx, state);
2438 if (!sstate) {
2439 FREE(tex_handle);
2440 return 0;
2441 }
2442
2443 si_set_sampler_view_desc(sctx, sview, sstate, &desc_list[0]);
2444 memcpy(&tex_handle->sstate, sstate, sizeof(*sstate));
2445 ctx->delete_sampler_state(ctx, sstate);
2446
2447 tex_handle->desc_slot = si_create_bindless_descriptor(sctx, desc_list,
2448 sizeof(desc_list));
2449 if (!tex_handle->desc_slot) {
2450 FREE(tex_handle);
2451 return 0;
2452 }
2453
2454 handle = tex_handle->desc_slot;
2455
2456 if (!_mesa_hash_table_insert(sctx->tex_handles,
2457 (void *)(uintptr_t)handle,
2458 tex_handle)) {
2459 FREE(tex_handle);
2460 return 0;
2461 }
2462
2463 pipe_sampler_view_reference(&tex_handle->view, view);
2464
2465 si_resource(sview->base.texture)->texture_handle_allocated = true;
2466
2467 return handle;
2468 }
2469
2470 static void si_delete_texture_handle(struct pipe_context *ctx, uint64_t handle)
2471 {
2472 struct si_context *sctx = (struct si_context *)ctx;
2473 struct si_texture_handle *tex_handle;
2474 struct hash_entry *entry;
2475
2476 entry = _mesa_hash_table_search(sctx->tex_handles,
2477 (void *)(uintptr_t)handle);
2478 if (!entry)
2479 return;
2480
2481 tex_handle = (struct si_texture_handle *)entry->data;
2482
2483 /* Allow this descriptor slot to be re-used. */
2484 util_idalloc_free(&sctx->bindless_used_slots, tex_handle->desc_slot);
2485
2486 pipe_sampler_view_reference(&tex_handle->view, NULL);
2487 _mesa_hash_table_remove(sctx->tex_handles, entry);
2488 FREE(tex_handle);
2489 }
2490
2491 static void si_make_texture_handle_resident(struct pipe_context *ctx,
2492 uint64_t handle, bool resident)
2493 {
2494 struct si_context *sctx = (struct si_context *)ctx;
2495 struct si_texture_handle *tex_handle;
2496 struct si_sampler_view *sview;
2497 struct hash_entry *entry;
2498
2499 entry = _mesa_hash_table_search(sctx->tex_handles,
2500 (void *)(uintptr_t)handle);
2501 if (!entry)
2502 return;
2503
2504 tex_handle = (struct si_texture_handle *)entry->data;
2505 sview = (struct si_sampler_view *)tex_handle->view;
2506
2507 if (resident) {
2508 if (sview->base.texture->target != PIPE_BUFFER) {
2509 struct si_texture *tex =
2510 (struct si_texture *)sview->base.texture;
2511
2512 if (depth_needs_decompression(tex)) {
2513 util_dynarray_append(
2514 &sctx->resident_tex_needs_depth_decompress,
2515 struct si_texture_handle *,
2516 tex_handle);
2517 }
2518
2519 if (color_needs_decompression(tex)) {
2520 util_dynarray_append(
2521 &sctx->resident_tex_needs_color_decompress,
2522 struct si_texture_handle *,
2523 tex_handle);
2524 }
2525
2526 if (tex->surface.dcc_offset &&
2527 p_atomic_read(&tex->framebuffers_bound))
2528 sctx->need_check_render_feedback = true;
2529
2530 si_update_bindless_texture_descriptor(sctx, tex_handle);
2531 } else {
2532 si_update_bindless_buffer_descriptor(sctx,
2533 tex_handle->desc_slot,
2534 sview->base.texture,
2535 sview->base.u.buf.offset,
2536 &tex_handle->desc_dirty);
2537 }
2538
2539 /* Re-upload the descriptor if it has been updated while it
2540 * wasn't resident.
2541 */
2542 if (tex_handle->desc_dirty)
2543 sctx->bindless_descriptors_dirty = true;
2544
2545 /* Add the texture handle to the per-context list. */
2546 util_dynarray_append(&sctx->resident_tex_handles,
2547 struct si_texture_handle *, tex_handle);
2548
2549 /* Add the buffers to the current CS in case si_begin_new_cs()
2550 * is not going to be called.
2551 */
2552 si_sampler_view_add_buffer(sctx, sview->base.texture,
2553 RADEON_USAGE_READ,
2554 sview->is_stencil_sampler, false);
2555 } else {
2556 /* Remove the texture handle from the per-context list. */
2557 util_dynarray_delete_unordered(&sctx->resident_tex_handles,
2558 struct si_texture_handle *,
2559 tex_handle);
2560
2561 if (sview->base.texture->target != PIPE_BUFFER) {
2562 util_dynarray_delete_unordered(
2563 &sctx->resident_tex_needs_depth_decompress,
2564 struct si_texture_handle *, tex_handle);
2565
2566 util_dynarray_delete_unordered(
2567 &sctx->resident_tex_needs_color_decompress,
2568 struct si_texture_handle *, tex_handle);
2569 }
2570 }
2571 }
2572
2573 static uint64_t si_create_image_handle(struct pipe_context *ctx,
2574 const struct pipe_image_view *view)
2575 {
2576 struct si_context *sctx = (struct si_context *)ctx;
2577 struct si_image_handle *img_handle;
2578 uint32_t desc_list[16];
2579 uint64_t handle;
2580
2581 if (!view || !view->resource)
2582 return 0;
2583
2584 img_handle = CALLOC_STRUCT(si_image_handle);
2585 if (!img_handle)
2586 return 0;
2587
2588 memset(desc_list, 0, sizeof(desc_list));
2589 si_init_descriptor_list(&desc_list[0], 8, 2, null_image_descriptor);
2590
2591 si_set_shader_image_desc(sctx, view, false, &desc_list[0], &desc_list[8]);
2592
2593 img_handle->desc_slot = si_create_bindless_descriptor(sctx, desc_list,
2594 sizeof(desc_list));
2595 if (!img_handle->desc_slot) {
2596 FREE(img_handle);
2597 return 0;
2598 }
2599
2600 handle = img_handle->desc_slot;
2601
2602 if (!_mesa_hash_table_insert(sctx->img_handles,
2603 (void *)(uintptr_t)handle,
2604 img_handle)) {
2605 FREE(img_handle);
2606 return 0;
2607 }
2608
2609 util_copy_image_view(&img_handle->view, view);
2610
2611 si_resource(view->resource)->image_handle_allocated = true;
2612
2613 return handle;
2614 }
2615
2616 static void si_delete_image_handle(struct pipe_context *ctx, uint64_t handle)
2617 {
2618 struct si_context *sctx = (struct si_context *)ctx;
2619 struct si_image_handle *img_handle;
2620 struct hash_entry *entry;
2621
2622 entry = _mesa_hash_table_search(sctx->img_handles,
2623 (void *)(uintptr_t)handle);
2624 if (!entry)
2625 return;
2626
2627 img_handle = (struct si_image_handle *)entry->data;
2628
2629 util_copy_image_view(&img_handle->view, NULL);
2630 _mesa_hash_table_remove(sctx->img_handles, entry);
2631 FREE(img_handle);
2632 }
2633
2634 static void si_make_image_handle_resident(struct pipe_context *ctx,
2635 uint64_t handle, unsigned access,
2636 bool resident)
2637 {
2638 struct si_context *sctx = (struct si_context *)ctx;
2639 struct si_image_handle *img_handle;
2640 struct pipe_image_view *view;
2641 struct si_resource *res;
2642 struct hash_entry *entry;
2643
2644 entry = _mesa_hash_table_search(sctx->img_handles,
2645 (void *)(uintptr_t)handle);
2646 if (!entry)
2647 return;
2648
2649 img_handle = (struct si_image_handle *)entry->data;
2650 view = &img_handle->view;
2651 res = si_resource(view->resource);
2652
2653 if (resident) {
2654 if (res->b.b.target != PIPE_BUFFER) {
2655 struct si_texture *tex = (struct si_texture *)res;
2656 unsigned level = view->u.tex.level;
2657
2658 if (color_needs_decompression(tex)) {
2659 util_dynarray_append(
2660 &sctx->resident_img_needs_color_decompress,
2661 struct si_image_handle *,
2662 img_handle);
2663 }
2664
2665 if (vi_dcc_enabled(tex, level) &&
2666 p_atomic_read(&tex->framebuffers_bound))
2667 sctx->need_check_render_feedback = true;
2668
2669 si_update_bindless_image_descriptor(sctx, img_handle);
2670 } else {
2671 si_update_bindless_buffer_descriptor(sctx,
2672 img_handle->desc_slot,
2673 view->resource,
2674 view->u.buf.offset,
2675 &img_handle->desc_dirty);
2676 }
2677
2678 /* Re-upload the descriptor if it has been updated while it
2679 * wasn't resident.
2680 */
2681 if (img_handle->desc_dirty)
2682 sctx->bindless_descriptors_dirty = true;
2683
2684 /* Add the image handle to the per-context list. */
2685 util_dynarray_append(&sctx->resident_img_handles,
2686 struct si_image_handle *, img_handle);
2687
2688 /* Add the buffers to the current CS in case si_begin_new_cs()
2689 * is not going to be called.
2690 */
2691 si_sampler_view_add_buffer(sctx, view->resource,
2692 (access & PIPE_IMAGE_ACCESS_WRITE) ?
2693 RADEON_USAGE_READWRITE :
2694 RADEON_USAGE_READ, false, false);
2695 } else {
2696 /* Remove the image handle from the per-context list. */
2697 util_dynarray_delete_unordered(&sctx->resident_img_handles,
2698 struct si_image_handle *,
2699 img_handle);
2700
2701 if (res->b.b.target != PIPE_BUFFER) {
2702 util_dynarray_delete_unordered(
2703 &sctx->resident_img_needs_color_decompress,
2704 struct si_image_handle *,
2705 img_handle);
2706 }
2707 }
2708 }
2709
2710 static void si_resident_buffers_add_all_to_bo_list(struct si_context *sctx)
2711 {
2712 unsigned num_resident_tex_handles, num_resident_img_handles;
2713
2714 num_resident_tex_handles = sctx->resident_tex_handles.size /
2715 sizeof(struct si_texture_handle *);
2716 num_resident_img_handles = sctx->resident_img_handles.size /
2717 sizeof(struct si_image_handle *);
2718
2719 /* Add all resident texture handles. */
2720 util_dynarray_foreach(&sctx->resident_tex_handles,
2721 struct si_texture_handle *, tex_handle) {
2722 struct si_sampler_view *sview =
2723 (struct si_sampler_view *)(*tex_handle)->view;
2724
2725 si_sampler_view_add_buffer(sctx, sview->base.texture,
2726 RADEON_USAGE_READ,
2727 sview->is_stencil_sampler, false);
2728 }
2729
2730 /* Add all resident image handles. */
2731 util_dynarray_foreach(&sctx->resident_img_handles,
2732 struct si_image_handle *, img_handle) {
2733 struct pipe_image_view *view = &(*img_handle)->view;
2734
2735 si_sampler_view_add_buffer(sctx, view->resource,
2736 RADEON_USAGE_READWRITE,
2737 false, false);
2738 }
2739
2740 sctx->num_resident_handles += num_resident_tex_handles +
2741 num_resident_img_handles;
2742 assert(sctx->bo_list_add_all_resident_resources);
2743 sctx->bo_list_add_all_resident_resources = false;
2744 }
2745
2746 /* INIT/DEINIT/UPLOAD */
2747
2748 void si_init_all_descriptors(struct si_context *sctx)
2749 {
2750 int i;
2751 unsigned first_shader =
2752 sctx->has_graphics ? 0 : PIPE_SHADER_COMPUTE;
2753
2754 for (i = first_shader; i < SI_NUM_SHADERS; i++) {
2755 bool is_2nd = sctx->chip_class >= GFX9 &&
2756 (i == PIPE_SHADER_TESS_CTRL ||
2757 i == PIPE_SHADER_GEOMETRY);
2758 unsigned num_sampler_slots = SI_NUM_IMAGE_SLOTS / 2 + SI_NUM_SAMPLERS;
2759 unsigned num_buffer_slots = SI_NUM_SHADER_BUFFERS + SI_NUM_CONST_BUFFERS;
2760 int rel_dw_offset;
2761 struct si_descriptors *desc;
2762
2763 if (is_2nd) {
2764 if (i == PIPE_SHADER_TESS_CTRL) {
2765 rel_dw_offset = (R_00B408_SPI_SHADER_USER_DATA_ADDR_LO_HS -
2766 R_00B430_SPI_SHADER_USER_DATA_LS_0) / 4;
2767 } else if (sctx->chip_class >= GFX10) { /* PIPE_SHADER_GEOMETRY */
2768 rel_dw_offset = (R_00B208_SPI_SHADER_USER_DATA_ADDR_LO_GS -
2769 R_00B230_SPI_SHADER_USER_DATA_GS_0) / 4;
2770 } else {
2771 rel_dw_offset = (R_00B208_SPI_SHADER_USER_DATA_ADDR_LO_GS -
2772 R_00B330_SPI_SHADER_USER_DATA_ES_0) / 4;
2773 }
2774 } else {
2775 rel_dw_offset = SI_SGPR_CONST_AND_SHADER_BUFFERS;
2776 }
2777 desc = si_const_and_shader_buffer_descriptors(sctx, i);
2778 si_init_buffer_resources(&sctx->const_and_shader_buffers[i], desc,
2779 num_buffer_slots, rel_dw_offset,
2780 RADEON_PRIO_SHADER_RW_BUFFER,
2781 RADEON_PRIO_CONST_BUFFER);
2782 desc->slot_index_to_bind_directly = si_get_constbuf_slot(0);
2783
2784 if (is_2nd) {
2785 if (i == PIPE_SHADER_TESS_CTRL) {
2786 rel_dw_offset = (R_00B40C_SPI_SHADER_USER_DATA_ADDR_HI_HS -
2787 R_00B430_SPI_SHADER_USER_DATA_LS_0) / 4;
2788 } else if (sctx->chip_class >= GFX10) { /* PIPE_SHADER_GEOMETRY */
2789 rel_dw_offset = (R_00B20C_SPI_SHADER_USER_DATA_ADDR_HI_GS -
2790 R_00B230_SPI_SHADER_USER_DATA_GS_0) / 4;
2791 } else {
2792 rel_dw_offset = (R_00B20C_SPI_SHADER_USER_DATA_ADDR_HI_GS -
2793 R_00B330_SPI_SHADER_USER_DATA_ES_0) / 4;
2794 }
2795 } else {
2796 rel_dw_offset = SI_SGPR_SAMPLERS_AND_IMAGES;
2797 }
2798
2799 desc = si_sampler_and_image_descriptors(sctx, i);
2800 si_init_descriptors(desc, rel_dw_offset, 16, num_sampler_slots);
2801
2802 int j;
2803 for (j = 0; j < SI_NUM_IMAGE_SLOTS; j++)
2804 memcpy(desc->list + j * 8, null_image_descriptor, 8 * 4);
2805 for (; j < SI_NUM_IMAGE_SLOTS + SI_NUM_SAMPLERS * 2; j++)
2806 memcpy(desc->list + j * 8, null_texture_descriptor, 8 * 4);
2807 }
2808
2809 si_init_buffer_resources(&sctx->rw_buffers,
2810 &sctx->descriptors[SI_DESCS_RW_BUFFERS],
2811 SI_NUM_RW_BUFFERS, SI_SGPR_RW_BUFFERS,
2812 /* The second priority is used by
2813 * const buffers in RW buffer slots. */
2814 RADEON_PRIO_SHADER_RINGS, RADEON_PRIO_CONST_BUFFER);
2815 sctx->descriptors[SI_DESCS_RW_BUFFERS].num_active_slots = SI_NUM_RW_BUFFERS;
2816
2817 /* Initialize an array of 1024 bindless descriptors, when the limit is
2818 * reached, just make it larger and re-upload the whole array.
2819 */
2820 si_init_bindless_descriptors(sctx, &sctx->bindless_descriptors,
2821 SI_SGPR_BINDLESS_SAMPLERS_AND_IMAGES,
2822 1024);
2823
2824 sctx->descriptors_dirty = u_bit_consecutive(0, SI_NUM_DESCS);
2825
2826 /* Set pipe_context functions. */
2827 sctx->b.bind_sampler_states = si_bind_sampler_states;
2828 sctx->b.set_shader_images = si_set_shader_images;
2829 sctx->b.set_constant_buffer = si_pipe_set_constant_buffer;
2830 sctx->b.set_shader_buffers = si_set_shader_buffers;
2831 sctx->b.set_sampler_views = si_set_sampler_views;
2832 sctx->b.create_texture_handle = si_create_texture_handle;
2833 sctx->b.delete_texture_handle = si_delete_texture_handle;
2834 sctx->b.make_texture_handle_resident = si_make_texture_handle_resident;
2835 sctx->b.create_image_handle = si_create_image_handle;
2836 sctx->b.delete_image_handle = si_delete_image_handle;
2837 sctx->b.make_image_handle_resident = si_make_image_handle_resident;
2838
2839 if (!sctx->has_graphics)
2840 return;
2841
2842 sctx->b.set_polygon_stipple = si_set_polygon_stipple;
2843
2844 /* Shader user data. */
2845 sctx->atoms.s.shader_pointers.emit = si_emit_graphics_shader_pointers;
2846
2847 /* Set default and immutable mappings. */
2848 if (sctx->ngg) {
2849 assert(sctx->chip_class >= GFX10);
2850 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX, R_00B230_SPI_SHADER_USER_DATA_GS_0);
2851 } else {
2852 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX, R_00B130_SPI_SHADER_USER_DATA_VS_0);
2853 }
2854
2855 if (sctx->chip_class == GFX9) {
2856 si_set_user_data_base(sctx, PIPE_SHADER_TESS_CTRL,
2857 R_00B430_SPI_SHADER_USER_DATA_LS_0);
2858 si_set_user_data_base(sctx, PIPE_SHADER_GEOMETRY,
2859 R_00B330_SPI_SHADER_USER_DATA_ES_0);
2860 } else {
2861 si_set_user_data_base(sctx, PIPE_SHADER_TESS_CTRL,
2862 R_00B430_SPI_SHADER_USER_DATA_HS_0);
2863 si_set_user_data_base(sctx, PIPE_SHADER_GEOMETRY,
2864 R_00B230_SPI_SHADER_USER_DATA_GS_0);
2865 }
2866 si_set_user_data_base(sctx, PIPE_SHADER_FRAGMENT, R_00B030_SPI_SHADER_USER_DATA_PS_0);
2867 }
2868
2869 static bool si_upload_shader_descriptors(struct si_context *sctx, unsigned mask)
2870 {
2871 unsigned dirty = sctx->descriptors_dirty & mask;
2872
2873 /* Assume nothing will go wrong: */
2874 sctx->shader_pointers_dirty |= dirty;
2875
2876 while (dirty) {
2877 unsigned i = u_bit_scan(&dirty);
2878
2879 if (!si_upload_descriptors(sctx, &sctx->descriptors[i]))
2880 return false;
2881 }
2882
2883 sctx->descriptors_dirty &= ~mask;
2884
2885 si_upload_bindless_descriptors(sctx);
2886
2887 return true;
2888 }
2889
2890 bool si_upload_graphics_shader_descriptors(struct si_context *sctx)
2891 {
2892 const unsigned mask = u_bit_consecutive(0, SI_DESCS_FIRST_COMPUTE);
2893 return si_upload_shader_descriptors(sctx, mask);
2894 }
2895
2896 bool si_upload_compute_shader_descriptors(struct si_context *sctx)
2897 {
2898 /* Does not update rw_buffers as that is not needed for compute shaders
2899 * and the input buffer is using the same SGPR's anyway.
2900 */
2901 const unsigned mask = u_bit_consecutive(SI_DESCS_FIRST_COMPUTE,
2902 SI_NUM_DESCS - SI_DESCS_FIRST_COMPUTE);
2903 return si_upload_shader_descriptors(sctx, mask);
2904 }
2905
2906 void si_release_all_descriptors(struct si_context *sctx)
2907 {
2908 int i;
2909
2910 for (i = 0; i < SI_NUM_SHADERS; i++) {
2911 si_release_buffer_resources(&sctx->const_and_shader_buffers[i],
2912 si_const_and_shader_buffer_descriptors(sctx, i));
2913 si_release_sampler_views(&sctx->samplers[i]);
2914 si_release_image_views(&sctx->images[i]);
2915 }
2916 si_release_buffer_resources(&sctx->rw_buffers,
2917 &sctx->descriptors[SI_DESCS_RW_BUFFERS]);
2918 for (i = 0; i < SI_NUM_VERTEX_BUFFERS; i++)
2919 pipe_vertex_buffer_unreference(&sctx->vertex_buffer[i]);
2920
2921 for (i = 0; i < SI_NUM_DESCS; ++i)
2922 si_release_descriptors(&sctx->descriptors[i]);
2923
2924 si_resource_reference(&sctx->vb_descriptors_buffer, NULL);
2925 sctx->vb_descriptors_gpu_list = NULL; /* points into a mapped buffer */
2926
2927 si_release_bindless_descriptors(sctx);
2928 }
2929
2930 void si_gfx_resources_add_all_to_bo_list(struct si_context *sctx)
2931 {
2932 for (unsigned i = 0; i < SI_NUM_GRAPHICS_SHADERS; i++) {
2933 si_buffer_resources_begin_new_cs(sctx, &sctx->const_and_shader_buffers[i]);
2934 si_sampler_views_begin_new_cs(sctx, &sctx->samplers[i]);
2935 si_image_views_begin_new_cs(sctx, &sctx->images[i]);
2936