projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
radeonsi/gfx9: define and set LS-HS user SGPRs
[mesa.git] / src / gallium / drivers / radeonsi / si_descriptors.c
1 /*
2 * Copyright 2013 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 *
23 * Authors:
24 * Marek Olšák <marek.olsak@amd.com>
25 */
26
27 /* Resource binding slots and sampler states (each described with 8 or
28 * 4 dwords) are stored in lists in memory which is accessed by shaders
29 * using scalar load instructions.
30 *
31 * This file is responsible for managing such lists. It keeps a copy of all
32 * descriptors in CPU memory and re-uploads a whole list if some slots have
33 * been changed.
34 *
35 * This code is also reponsible for updating shader pointers to those lists.
36 *
37 * Note that CP DMA can't be used for updating the lists, because a GPU hang
38 * could leave the list in a mid-IB state and the next IB would get wrong
39 * descriptors and the whole context would be unusable at that point.
40 * (Note: The register shadowing can't be used due to the same reason)
41 *
42 * Also, uploading descriptors to newly allocated memory doesn't require
43 * a KCACHE flush.
44 *
45 *
46 * Possible scenarios for one 16 dword image+sampler slot:
47 *
48 * | Image | w/ FMASK | Buffer | NULL
49 * [ 0: 3] Image[0:3] | Image[0:3] | Null[0:3] | Null[0:3]
50 * [ 4: 7] Image[4:7] | Image[4:7] | Buffer[0:3] | 0
51 * [ 8:11] Null[0:3] | Fmask[0:3] | Null[0:3] | Null[0:3]
52 * [12:15] Sampler[0:3] | Fmask[4:7] | Sampler[0:3] | Sampler[0:3]
53 *
54 * FMASK implies MSAA, therefore no sampler state.
55 * Sampler states are never unbound except when FMASK is bound.
56 */
57
58 #include "radeon/r600_cs.h"
59 #include "si_pipe.h"
60 #include "sid.h"
61 #include "gfx9d.h"
62
63 #include "util/u_format.h"
64 #include "util/u_memory.h"
65 #include "util/u_upload_mgr.h"
66
67
68 /* NULL image and buffer descriptor for textures (alpha = 1) and images
69 * (alpha = 0).
70 *
71 * For images, all fields must be zero except for the swizzle, which
72 * supports arbitrary combinations of 0s and 1s. The texture type must be
73 * any valid type (e.g. 1D). If the texture type isn't set, the hw hangs.
74 *
75 * For buffers, all fields must be zero. If they are not, the hw hangs.
76 *
77 * This is the only reason why the buffer descriptor must be in words [4:7].
78 */
79 static uint32_t null_texture_descriptor[8] = {
80 0,
81 0,
82 0,
83 S_008F1C_DST_SEL_W(V_008F1C_SQ_SEL_1) |
84 S_008F1C_TYPE(V_008F1C_SQ_RSRC_IMG_1D)
85 /* the rest must contain zeros, which is also used by the buffer
86 * descriptor */
87 };
88
89 static uint32_t null_image_descriptor[8] = {
90 0,
91 0,
92 0,
93 S_008F1C_TYPE(V_008F1C_SQ_RSRC_IMG_1D)
94 /* the rest must contain zeros, which is also used by the buffer
95 * descriptor */
96 };
97
98 static void si_init_descriptors(struct si_descriptors *desc,
99 unsigned shader_userdata_index,
100 unsigned element_dw_size,
101 unsigned num_elements,
102 const uint32_t *null_descriptor,
103 unsigned *ce_offset)
104 {
105 int i;
106
107 assert(num_elements <= sizeof(desc->dirty_mask)*8);
108
109 desc->list = CALLOC(num_elements, element_dw_size * 4);
110 desc->element_dw_size = element_dw_size;
111 desc->num_elements = num_elements;
112 desc->dirty_mask = num_elements == 32 ? ~0u : (1u << num_elements) - 1;
113 desc->shader_userdata_offset = shader_userdata_index * 4;
114
115 if (ce_offset) {
116 desc->uses_ce = true;
117 desc->ce_offset = *ce_offset;
118
119 /* make sure that ce_offset stays 32 byte aligned */
120 *ce_offset += align(element_dw_size * num_elements * 4, 32);
121 }
122
123 /* Initialize the array to NULL descriptors if the element size is 8. */
124 if (null_descriptor) {
125 assert(element_dw_size % 8 == 0);
126 for (i = 0; i < num_elements * element_dw_size / 8; i++)
127 memcpy(desc->list + i * 8, null_descriptor,
128 8 * 4);
129 }
130 }
131
132 static void si_release_descriptors(struct si_descriptors *desc)
133 {
134 r600_resource_reference(&desc->buffer, NULL);
135 FREE(desc->list);
136 }
137
138 static bool si_ce_upload(struct si_context *sctx, unsigned ce_offset, unsigned size,
139 unsigned *out_offset, struct r600_resource **out_buf) {
140 uint64_t va;
141
142 u_suballocator_alloc(sctx->ce_suballocator, size,
143 sctx->screen->b.info.tcc_cache_line_size,
144 out_offset, (struct pipe_resource**)out_buf);
145 if (!out_buf)
146 return false;
147
148 va = (*out_buf)->gpu_address + *out_offset;
149
150 radeon_emit(sctx->ce_ib, PKT3(PKT3_DUMP_CONST_RAM, 3, 0));
151 radeon_emit(sctx->ce_ib, ce_offset);
152 radeon_emit(sctx->ce_ib, size / 4);
153 radeon_emit(sctx->ce_ib, va);
154 radeon_emit(sctx->ce_ib, va >> 32);
155
156 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx, *out_buf,
157 RADEON_USAGE_READWRITE, RADEON_PRIO_DESCRIPTORS);
158
159 sctx->ce_need_synchronization = true;
160 return true;
161 }
162
163 static void si_ce_reinitialize_descriptors(struct si_context *sctx,
164 struct si_descriptors *desc)
165 {
166 if (desc->buffer) {
167 struct r600_resource *buffer = (struct r600_resource*)desc->buffer;
168 unsigned list_size = desc->num_elements * desc->element_dw_size * 4;
169 uint64_t va = buffer->gpu_address + desc->buffer_offset;
170 struct radeon_winsys_cs *ib = sctx->ce_preamble_ib;
171
172 if (!ib)
173 ib = sctx->ce_ib;
174
175 list_size = align(list_size, 32);
176
177 radeon_emit(ib, PKT3(PKT3_LOAD_CONST_RAM, 3, 0));
178 radeon_emit(ib, va);
179 radeon_emit(ib, va >> 32);
180 radeon_emit(ib, list_size / 4);
181 radeon_emit(ib, desc->ce_offset);
182
183 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx, desc->buffer,
184 RADEON_USAGE_READ, RADEON_PRIO_DESCRIPTORS);
185 }
186 desc->ce_ram_dirty = false;
187 }
188
189 void si_ce_reinitialize_all_descriptors(struct si_context *sctx)
190 {
191 int i;
192
193 for (i = 0; i < SI_NUM_DESCS; ++i)
194 si_ce_reinitialize_descriptors(sctx, &sctx->descriptors[i]);
195 }
196
197 void si_ce_enable_loads(struct radeon_winsys_cs *ib)
198 {
199 radeon_emit(ib, PKT3(PKT3_CONTEXT_CONTROL, 1, 0));
200 radeon_emit(ib, CONTEXT_CONTROL_LOAD_ENABLE(1) |
201 CONTEXT_CONTROL_LOAD_CE_RAM(1));
202 radeon_emit(ib, CONTEXT_CONTROL_SHADOW_ENABLE(1));
203 }
204
205 static bool si_upload_descriptors(struct si_context *sctx,
206 struct si_descriptors *desc,
207 struct r600_atom * atom)
208 {
209 unsigned list_size = desc->num_elements * desc->element_dw_size * 4;
210
211 if (!desc->dirty_mask)
212 return true;
213
214 if (sctx->ce_ib && desc->uses_ce) {
215 uint32_t const* list = (uint32_t const*)desc->list;
216
217 if (desc->ce_ram_dirty)
218 si_ce_reinitialize_descriptors(sctx, desc);
219
220 while(desc->dirty_mask) {
221 int begin, count;
222 u_bit_scan_consecutive_range(&desc->dirty_mask, &begin,
223 &count);
224
225 begin *= desc->element_dw_size;
226 count *= desc->element_dw_size;
227
228 radeon_emit(sctx->ce_ib,
229 PKT3(PKT3_WRITE_CONST_RAM, count, 0));
230 radeon_emit(sctx->ce_ib, desc->ce_offset + begin * 4);
231 radeon_emit_array(sctx->ce_ib, list + begin, count);
232 }
233
234 if (!si_ce_upload(sctx, desc->ce_offset, list_size,
235 &desc->buffer_offset, &desc->buffer))
236 return false;
237 } else {
238 void *ptr;
239
240 u_upload_alloc(sctx->b.b.const_uploader, 0, list_size,
241 sctx->screen->b.info.tcc_cache_line_size,
242 &desc->buffer_offset,
243 (struct pipe_resource**)&desc->buffer, &ptr);
244 if (!desc->buffer)
245 return false; /* skip the draw call */
246
247 util_memcpy_cpu_to_le32(ptr, desc->list, list_size);
248 desc->gpu_list = ptr;
249
250 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx, desc->buffer,
251 RADEON_USAGE_READ, RADEON_PRIO_DESCRIPTORS);
252 }
253 desc->dirty_mask = 0;
254
255 if (atom)
256 si_mark_atom_dirty(sctx, atom);
257
258 return true;
259 }
260
261 static void
262 si_descriptors_begin_new_cs(struct si_context *sctx, struct si_descriptors *desc)
263 {
264 desc->ce_ram_dirty = true;
265
266 if (!desc->buffer)
267 return;
268
269 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx, desc->buffer,
270 RADEON_USAGE_READ, RADEON_PRIO_DESCRIPTORS);
271 }
272
273 /* SAMPLER VIEWS */
274
275 static unsigned
276 si_sampler_descriptors_idx(unsigned shader)
277 {
278 return SI_DESCS_FIRST_SHADER + shader * SI_NUM_SHADER_DESCS +
279 SI_SHADER_DESCS_SAMPLERS;
280 }
281
282 static struct si_descriptors *
283 si_sampler_descriptors(struct si_context *sctx, unsigned shader)
284 {
285 return &sctx->descriptors[si_sampler_descriptors_idx(shader)];
286 }
287
288 static void si_release_sampler_views(struct si_sampler_views *views)
289 {
290 int i;
291
292 for (i = 0; i < ARRAY_SIZE(views->views); i++) {
293 pipe_sampler_view_reference(&views->views[i], NULL);
294 }
295 }
296
297 static void si_sampler_view_add_buffer(struct si_context *sctx,
298 struct pipe_resource *resource,
299 enum radeon_bo_usage usage,
300 bool is_stencil_sampler,
301 bool check_mem)
302 {
303 struct r600_resource *rres;
304 struct r600_texture *rtex;
305 enum radeon_bo_priority priority;
306
307 if (!resource)
308 return;
309
310 if (resource->target != PIPE_BUFFER) {
311 struct r600_texture *tex = (struct r600_texture*)resource;
312
313 if (tex->is_depth && !r600_can_sample_zs(tex, is_stencil_sampler))
314 resource = &tex->flushed_depth_texture->resource.b.b;
315 }
316
317 rres = (struct r600_resource*)resource;
318 priority = r600_get_sampler_view_priority(rres);
319
320 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
321 rres, usage, priority,
322 check_mem);
323
324 if (resource->target == PIPE_BUFFER)
325 return;
326
327 /* Now add separate DCC or HTILE. */
328 rtex = (struct r600_texture*)resource;
329 if (rtex->dcc_separate_buffer) {
330 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
331 rtex->dcc_separate_buffer, usage,
332 RADEON_PRIO_DCC, check_mem);
333 }
334
335 if (rtex->htile_buffer &&
336 rtex->tc_compatible_htile &&
337 !is_stencil_sampler) {
338 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
339 rtex->htile_buffer, usage,
340 RADEON_PRIO_HTILE, check_mem);
341 }
342 }
343
344 static void si_sampler_views_begin_new_cs(struct si_context *sctx,
345 struct si_sampler_views *views)
346 {
347 unsigned mask = views->enabled_mask;
348
349 /* Add buffers to the CS. */
350 while (mask) {
351 int i = u_bit_scan(&mask);
352 struct si_sampler_view *sview = (struct si_sampler_view *)views->views[i];
353
354 si_sampler_view_add_buffer(sctx, sview->base.texture,
355 RADEON_USAGE_READ,
356 sview->is_stencil_sampler, false);
357 }
358 }
359
360 /* Set buffer descriptor fields that can be changed by reallocations. */
361 static void si_set_buf_desc_address(struct r600_resource *buf,
362 uint64_t offset, uint32_t *state)
363 {
364 uint64_t va = buf->gpu_address + offset;
365
366 state[0] = va;
367 state[1] &= C_008F04_BASE_ADDRESS_HI;
368 state[1] |= S_008F04_BASE_ADDRESS_HI(va >> 32);
369 }
370
371 /* Set texture descriptor fields that can be changed by reallocations.
372 *
373 * \param tex texture
374 * \param base_level_info information of the level of BASE_ADDRESS
375 * \param base_level the level of BASE_ADDRESS
376 * \param first_level pipe_sampler_view.u.tex.first_level
377 * \param block_width util_format_get_blockwidth()
378 * \param is_stencil select between separate Z & Stencil
379 * \param state descriptor to update
380 */
381 void si_set_mutable_tex_desc_fields(struct si_screen *sscreen,
382 struct r600_texture *tex,
383 const struct legacy_surf_level *base_level_info,
384 unsigned base_level, unsigned first_level,
385 unsigned block_width, bool is_stencil,
386 uint32_t *state)
387 {
388 uint64_t va, meta_va = 0;
389
390 if (tex->is_depth && !r600_can_sample_zs(tex, is_stencil)) {
391 tex = tex->flushed_depth_texture;
392 is_stencil = false;
393 }
394
395 va = tex->resource.gpu_address;
396
397 if (sscreen->b.chip_class >= GFX9) {
398 /* Only stencil_offset needs to be added here. */
399 if (is_stencil)
400 va += tex->surface.u.gfx9.stencil_offset;
401 else
402 va += tex->surface.u.gfx9.surf_offset;
403 } else {
404 va += base_level_info->offset;
405 }
406
407 state[0] = va >> 8;
408 state[1] &= C_008F14_BASE_ADDRESS_HI;
409 state[1] |= S_008F14_BASE_ADDRESS_HI(va >> 40);
410
411 if (sscreen->b.chip_class >= VI) {
412 state[6] &= C_008F28_COMPRESSION_EN;
413 state[7] = 0;
414
415 if (vi_dcc_enabled(tex, first_level)) {
416 meta_va = (!tex->dcc_separate_buffer ? tex->resource.gpu_address : 0) +
417 tex->dcc_offset;
418
419 if (sscreen->b.chip_class <= VI)
420 meta_va += base_level_info->dcc_offset;
421 } else if (tex->tc_compatible_htile && !is_stencil) {
422 meta_va = tex->htile_buffer->gpu_address;
423 }
424
425 if (meta_va) {
426 state[6] |= S_008F28_COMPRESSION_EN(1);
427 state[7] = meta_va >> 8;
428 }
429 }
430
431 if (sscreen->b.chip_class >= GFX9) {
432 state[3] &= C_008F1C_SW_MODE;
433 state[4] &= C_008F20_PITCH_GFX9;
434
435 if (is_stencil) {
436 state[3] |= S_008F1C_SW_MODE(tex->surface.u.gfx9.stencil.swizzle_mode);
437 state[4] |= S_008F20_PITCH_GFX9(tex->surface.u.gfx9.stencil.epitch);
438 } else {
439 state[3] |= S_008F1C_SW_MODE(tex->surface.u.gfx9.surf.swizzle_mode);
440 state[4] |= S_008F20_PITCH_GFX9(tex->surface.u.gfx9.surf.epitch);
441 }
442
443 state[5] &= C_008F24_META_DATA_ADDRESS &
444 C_008F24_META_PIPE_ALIGNED &
445 C_008F24_META_RB_ALIGNED;
446 if (meta_va) {
447 struct gfx9_surf_meta_flags meta;
448
449 if (tex->dcc_offset)
450 meta = tex->surface.u.gfx9.dcc;
451 else
452 meta = tex->surface.u.gfx9.htile;
453
454 state[5] |= S_008F24_META_DATA_ADDRESS(meta_va >> 40) |
455 S_008F24_META_PIPE_ALIGNED(meta.pipe_aligned) |
456 S_008F24_META_RB_ALIGNED(meta.rb_aligned);
457 }
458 } else {
459 /* SI-CI-VI */
460 unsigned pitch = base_level_info->nblk_x * block_width;
461 unsigned index = si_tile_mode_index(tex, base_level, is_stencil);
462
463 state[3] &= C_008F1C_TILING_INDEX;
464 state[3] |= S_008F1C_TILING_INDEX(index);
465 state[4] &= C_008F20_PITCH_GFX6;
466 state[4] |= S_008F20_PITCH_GFX6(pitch - 1);
467 }
468 }
469
470 static void si_set_sampler_view(struct si_context *sctx,
471 unsigned shader,
472 unsigned slot, struct pipe_sampler_view *view,
473 bool disallow_early_out)
474 {
475 struct si_sampler_views *views = &sctx->samplers[shader].views;
476 struct si_sampler_view *rview = (struct si_sampler_view*)view;
477 struct si_descriptors *descs = si_sampler_descriptors(sctx, shader);
478 uint32_t *desc = descs->list + slot * 16;
479
480 if (views->views[slot] == view && !disallow_early_out)
481 return;
482
483 if (view) {
484 struct r600_texture *rtex = (struct r600_texture *)view->texture;
485 bool is_buffer = rtex->resource.b.b.target == PIPE_BUFFER;
486
487 if (unlikely(!is_buffer && rview->dcc_incompatible)) {
488 if (vi_dcc_enabled(rtex, view->u.tex.first_level))
489 if (!r600_texture_disable_dcc(&sctx->b, rtex))
490 sctx->b.decompress_dcc(&sctx->b.b, rtex);
491
492 rview->dcc_incompatible = false;
493 }
494
495 assert(rtex); /* views with texture == NULL aren't supported */
496 pipe_sampler_view_reference(&views->views[slot], view);
497 memcpy(desc, rview->state, 8*4);
498
499 if (is_buffer) {
500 rtex->resource.bind_history |= PIPE_BIND_SAMPLER_VIEW;
501
502 si_set_buf_desc_address(&rtex->resource,
503 view->u.buf.offset,
504 desc + 4);
505 } else {
506 bool is_separate_stencil =
507 rtex->db_compatible &&
508 rview->is_stencil_sampler;
509
510 si_set_mutable_tex_desc_fields(sctx->screen, rtex,
511 rview->base_level_info,
512 rview->base_level,
513 rview->base.u.tex.first_level,
514 rview->block_width,
515 is_separate_stencil,
516 desc);
517 }
518
519 if (!is_buffer && rtex->fmask.size) {
520 memcpy(desc + 8,
521 rview->fmask_state, 8*4);
522 } else {
523 /* Disable FMASK and bind sampler state in [12:15]. */
524 memcpy(desc + 8,
525 null_texture_descriptor, 4*4);
526
527 if (views->sampler_states[slot])
528 memcpy(desc + 12,
529 views->sampler_states[slot]->val, 4*4);
530 }
531
532 views->enabled_mask |= 1u << slot;
533
534 /* Since this can flush, it must be done after enabled_mask is
535 * updated. */
536 si_sampler_view_add_buffer(sctx, view->texture,
537 RADEON_USAGE_READ,
538 rview->is_stencil_sampler, true);
539 } else {
540 pipe_sampler_view_reference(&views->views[slot], NULL);
541 memcpy(desc, null_texture_descriptor, 8*4);
542 /* Only clear the lower dwords of FMASK. */
543 memcpy(desc + 8, null_texture_descriptor, 4*4);
544 /* Re-set the sampler state if we are transitioning from FMASK. */
545 if (views->sampler_states[slot])
546 memcpy(desc + 12,
547 views->sampler_states[slot]->val, 4*4);
548
549 views->enabled_mask &= ~(1u << slot);
550 }
551
552 descs->dirty_mask |= 1u << slot;
553 sctx->descriptors_dirty |= 1u << si_sampler_descriptors_idx(shader);
554 }
555
556 static bool is_compressed_colortex(struct r600_texture *rtex)
557 {
558 return rtex->fmask.size ||
559 (rtex->dirty_level_mask &&
560 (rtex->cmask.size || rtex->dcc_offset));
561 }
562
563 static bool depth_needs_decompression(struct r600_texture *rtex,
564 struct si_sampler_view *sview)
565 {
566 return rtex->db_compatible &&
567 (!rtex->tc_compatible_htile || sview->is_stencil_sampler);
568 }
569
570 static void si_update_compressed_tex_shader_mask(struct si_context *sctx,
571 unsigned shader)
572 {
573 struct si_textures_info *samplers = &sctx->samplers[shader];
574 unsigned shader_bit = 1 << shader;
575
576 if (samplers->depth_texture_mask ||
577 samplers->compressed_colortex_mask ||
578 sctx->images[shader].compressed_colortex_mask)
579 sctx->compressed_tex_shader_mask |= shader_bit;
580 else
581 sctx->compressed_tex_shader_mask &= ~shader_bit;
582 }
583
584 static void si_set_sampler_views(struct pipe_context *ctx,
585 enum pipe_shader_type shader, unsigned start,
586 unsigned count,
587 struct pipe_sampler_view **views)
588 {
589 struct si_context *sctx = (struct si_context *)ctx;
590 struct si_textures_info *samplers = &sctx->samplers[shader];
591 int i;
592
593 if (!count || shader >= SI_NUM_SHADERS)
594 return;
595
596 for (i = 0; i < count; i++) {
597 unsigned slot = start + i;
598
599 if (!views || !views[i]) {
600 samplers->depth_texture_mask &= ~(1u << slot);
601 samplers->compressed_colortex_mask &= ~(1u << slot);
602 si_set_sampler_view(sctx, shader, slot, NULL, false);
603 continue;
604 }
605
606 si_set_sampler_view(sctx, shader, slot, views[i], false);
607
608 if (views[i]->texture && views[i]->texture->target != PIPE_BUFFER) {
609 struct r600_texture *rtex =
610 (struct r600_texture*)views[i]->texture;
611 struct si_sampler_view *rview = (struct si_sampler_view *)views[i];
612
613 if (depth_needs_decompression(rtex, rview)) {
614 samplers->depth_texture_mask |= 1u << slot;
615 } else {
616 samplers->depth_texture_mask &= ~(1u << slot);
617 }
618 if (is_compressed_colortex(rtex)) {
619 samplers->compressed_colortex_mask |= 1u << slot;
620 } else {
621 samplers->compressed_colortex_mask &= ~(1u << slot);
622 }
623
624 if (rtex->dcc_offset &&
625 p_atomic_read(&rtex->framebuffers_bound))
626 sctx->need_check_render_feedback = true;
627 } else {
628 samplers->depth_texture_mask &= ~(1u << slot);
629 samplers->compressed_colortex_mask &= ~(1u << slot);
630 }
631 }
632
633 si_update_compressed_tex_shader_mask(sctx, shader);
634 }
635
636 static void
637 si_samplers_update_compressed_colortex_mask(struct si_textures_info *samplers)
638 {
639 unsigned mask = samplers->views.enabled_mask;
640
641 while (mask) {
642 int i = u_bit_scan(&mask);
643 struct pipe_resource *res = samplers->views.views[i]->texture;
644
645 if (res && res->target != PIPE_BUFFER) {
646 struct r600_texture *rtex = (struct r600_texture *)res;
647
648 if (is_compressed_colortex(rtex)) {
649 samplers->compressed_colortex_mask |= 1u << i;
650 } else {
651 samplers->compressed_colortex_mask &= ~(1u << i);
652 }
653 }
654 }
655 }
656
657 /* IMAGE VIEWS */
658
659 static unsigned
660 si_image_descriptors_idx(unsigned shader)
661 {
662 return SI_DESCS_FIRST_SHADER + shader * SI_NUM_SHADER_DESCS +
663 SI_SHADER_DESCS_IMAGES;
664 }
665
666 static struct si_descriptors*
667 si_image_descriptors(struct si_context *sctx, unsigned shader)
668 {
669 return &sctx->descriptors[si_image_descriptors_idx(shader)];
670 }
671
672 static void
673 si_release_image_views(struct si_images_info *images)
674 {
675 unsigned i;
676
677 for (i = 0; i < SI_NUM_IMAGES; ++i) {
678 struct pipe_image_view *view = &images->views[i];
679
680 pipe_resource_reference(&view->resource, NULL);
681 }
682 }
683
684 static void
685 si_image_views_begin_new_cs(struct si_context *sctx, struct si_images_info *images)
686 {
687 uint mask = images->enabled_mask;
688
689 /* Add buffers to the CS. */
690 while (mask) {
691 int i = u_bit_scan(&mask);
692 struct pipe_image_view *view = &images->views[i];
693
694 assert(view->resource);
695
696 si_sampler_view_add_buffer(sctx, view->resource,
697 RADEON_USAGE_READWRITE, false, false);
698 }
699 }
700
701 static void
702 si_disable_shader_image(struct si_context *ctx, unsigned shader, unsigned slot)
703 {
704 struct si_images_info *images = &ctx->images[shader];
705
706 if (images->enabled_mask & (1u << slot)) {
707 struct si_descriptors *descs = si_image_descriptors(ctx, shader);
708
709 pipe_resource_reference(&images->views[slot].resource, NULL);
710 images->compressed_colortex_mask &= ~(1 << slot);
711
712 memcpy(descs->list + slot*8, null_image_descriptor, 8*4);
713 images->enabled_mask &= ~(1u << slot);
714 descs->dirty_mask |= 1u << slot;
715 ctx->descriptors_dirty |= 1u << si_image_descriptors_idx(shader);
716 }
717 }
718
719 static void
720 si_mark_image_range_valid(const struct pipe_image_view *view)
721 {
722 struct r600_resource *res = (struct r600_resource *)view->resource;
723
724 assert(res && res->b.b.target == PIPE_BUFFER);
725
726 util_range_add(&res->valid_buffer_range,
727 view->u.buf.offset,
728 view->u.buf.offset + view->u.buf.size);
729 }
730
731 static void si_set_shader_image(struct si_context *ctx,
732 unsigned shader,
733 unsigned slot, const struct pipe_image_view *view,
734 bool skip_decompress)
735 {
736 struct si_screen *screen = ctx->screen;
737 struct si_images_info *images = &ctx->images[shader];
738 struct si_descriptors *descs = si_image_descriptors(ctx, shader);
739 struct r600_resource *res;
740 uint32_t *desc = descs->list + slot * 8;
741
742 if (!view || !view->resource) {
743 si_disable_shader_image(ctx, shader, slot);
744 return;
745 }
746
747 res = (struct r600_resource *)view->resource;
748
749 if (&images->views[slot] != view)
750 util_copy_image_view(&images->views[slot], view);
751
752 if (res->b.b.target == PIPE_BUFFER) {
753 if (view->access & PIPE_IMAGE_ACCESS_WRITE)
754 si_mark_image_range_valid(view);
755
756 si_make_buffer_descriptor(screen, res,
757 view->format,
758 view->u.buf.offset,
759 view->u.buf.size, desc);
760 si_set_buf_desc_address(res, view->u.buf.offset, desc + 4);
761
762 images->compressed_colortex_mask &= ~(1 << slot);
763 res->bind_history |= PIPE_BIND_SHADER_IMAGE;
764 } else {
765 static const unsigned char swizzle[4] = { 0, 1, 2, 3 };
766 struct r600_texture *tex = (struct r600_texture *)res;
767 unsigned level = view->u.tex.level;
768 unsigned width, height, depth, hw_level;
769 bool uses_dcc = vi_dcc_enabled(tex, level);
770
771 assert(!tex->is_depth);
772 assert(tex->fmask.size == 0);
773
774 if (uses_dcc && !skip_decompress &&
775 (view->access & PIPE_IMAGE_ACCESS_WRITE ||
776 !vi_dcc_formats_compatible(res->b.b.format, view->format))) {
777 /* If DCC can't be disabled, at least decompress it.
778 * The decompression is relatively cheap if the surface
779 * has been decompressed already.
780 */
781 if (r600_texture_disable_dcc(&ctx->b, tex))
782 uses_dcc = false;
783 else
784 ctx->b.decompress_dcc(&ctx->b.b, tex);
785 }
786
787 if (is_compressed_colortex(tex)) {
788 images->compressed_colortex_mask |= 1 << slot;
789 } else {
790 images->compressed_colortex_mask &= ~(1 << slot);
791 }
792
793 if (uses_dcc &&
794 p_atomic_read(&tex->framebuffers_bound))
795 ctx->need_check_render_feedback = true;
796
797 if (ctx->b.chip_class >= GFX9) {
798 /* Always set the base address. The swizzle modes don't
799 * allow setting mipmap level offsets as the base.
800 */
801 width = res->b.b.width0;
802 height = res->b.b.height0;
803 depth = res->b.b.depth0;
804 hw_level = level;
805 } else {
806 /* Always force the base level to the selected level.
807 *
808 * This is required for 3D textures, where otherwise
809 * selecting a single slice for non-layered bindings
810 * fails. It doesn't hurt the other targets.
811 */
812 width = u_minify(res->b.b.width0, level);
813 height = u_minify(res->b.b.height0, level);
814 depth = u_minify(res->b.b.depth0, level);
815 hw_level = 0;
816 }
817
818 si_make_texture_descriptor(screen, tex,
819 false, res->b.b.target,
820 view->format, swizzle,
821 hw_level, hw_level,
822 view->u.tex.first_layer,
823 view->u.tex.last_layer,
824 width, height, depth,
825 desc, NULL);
826 si_set_mutable_tex_desc_fields(screen, tex,
827 &tex->surface.u.legacy.level[level],
828 level, level,
829 util_format_get_blockwidth(view->format),
830 false, desc);
831 }
832
833 images->enabled_mask |= 1u << slot;
834 descs->dirty_mask |= 1u << slot;
835 ctx->descriptors_dirty |= 1u << si_image_descriptors_idx(shader);
836
837 /* Since this can flush, it must be done after enabled_mask is updated. */
838 si_sampler_view_add_buffer(ctx, &res->b.b,
839 RADEON_USAGE_READWRITE, false, true);
840 }
841
842 static void
843 si_set_shader_images(struct pipe_context *pipe,
844 enum pipe_shader_type shader,
845 unsigned start_slot, unsigned count,
846 const struct pipe_image_view *views)
847 {
848 struct si_context *ctx = (struct si_context *)pipe;
849 unsigned i, slot;
850
851 assert(shader < SI_NUM_SHADERS);
852
853 if (!count)
854 return;
855
856 assert(start_slot + count <= SI_NUM_IMAGES);
857
858 if (views) {
859 for (i = 0, slot = start_slot; i < count; ++i, ++slot)
860 si_set_shader_image(ctx, shader, slot, &views[i], false);
861 } else {
862 for (i = 0, slot = start_slot; i < count; ++i, ++slot)
863 si_set_shader_image(ctx, shader, slot, NULL, false);
864 }
865
866 si_update_compressed_tex_shader_mask(ctx, shader);
867 }
868
869 static void
870 si_images_update_compressed_colortex_mask(struct si_images_info *images)
871 {
872 unsigned mask = images->enabled_mask;
873
874 while (mask) {
875 int i = u_bit_scan(&mask);
876 struct pipe_resource *res = images->views[i].resource;
877
878 if (res && res->target != PIPE_BUFFER) {
879 struct r600_texture *rtex = (struct r600_texture *)res;
880
881 if (is_compressed_colortex(rtex)) {
882 images->compressed_colortex_mask |= 1 << i;
883 } else {
884 images->compressed_colortex_mask &= ~(1 << i);
885 }
886 }
887 }
888 }
889
890 /* SAMPLER STATES */
891
892 static void si_bind_sampler_states(struct pipe_context *ctx,
893 enum pipe_shader_type shader,
894 unsigned start, unsigned count, void **states)
895 {
896 struct si_context *sctx = (struct si_context *)ctx;
897 struct si_textures_info *samplers = &sctx->samplers[shader];
898 struct si_descriptors *desc = si_sampler_descriptors(sctx, shader);
899 struct si_sampler_state **sstates = (struct si_sampler_state**)states;
900 int i;
901
902 if (!count || shader >= SI_NUM_SHADERS)
903 return;
904
905 for (i = 0; i < count; i++) {
906 unsigned slot = start + i;
907
908 if (!sstates[i] ||
909 sstates[i] == samplers->views.sampler_states[slot])
910 continue;
911
912 #ifdef DEBUG
913 assert(sstates[i]->magic == SI_SAMPLER_STATE_MAGIC);
914 #endif
915 samplers->views.sampler_states[slot] = sstates[i];
916
917 /* If FMASK is bound, don't overwrite it.
918 * The sampler state will be set after FMASK is unbound.
919 */
920 if (samplers->views.views[slot] &&
921 samplers->views.views[slot]->texture &&
922 samplers->views.views[slot]->texture->target != PIPE_BUFFER &&
923 ((struct r600_texture*)samplers->views.views[slot]->texture)->fmask.size)
924 continue;
925
926 memcpy(desc->list + slot * 16 + 12, sstates[i]->val, 4*4);
927 desc->dirty_mask |= 1u << slot;
928 sctx->descriptors_dirty |= 1u << si_sampler_descriptors_idx(shader);
929 }
930 }
931
932 /* BUFFER RESOURCES */
933
934 static void si_init_buffer_resources(struct si_buffer_resources *buffers,
935 struct si_descriptors *descs,
936 unsigned num_buffers,
937 unsigned shader_userdata_index,
938 enum radeon_bo_usage shader_usage,
939 enum radeon_bo_priority priority,
940 unsigned *ce_offset)
941 {
942 buffers->shader_usage = shader_usage;
943 buffers->priority = priority;
944 buffers->buffers = CALLOC(num_buffers, sizeof(struct pipe_resource*));
945
946 si_init_descriptors(descs, shader_userdata_index, 4,
947 num_buffers, NULL, ce_offset);
948 }
949
950 static void si_release_buffer_resources(struct si_buffer_resources *buffers,
951 struct si_descriptors *descs)
952 {
953 int i;
954
955 for (i = 0; i < descs->num_elements; i++) {
956 pipe_resource_reference(&buffers->buffers[i], NULL);
957 }
958
959 FREE(buffers->buffers);
960 }
961
962 static void si_buffer_resources_begin_new_cs(struct si_context *sctx,
963 struct si_buffer_resources *buffers)
964 {
965 unsigned mask = buffers->enabled_mask;
966
967 /* Add buffers to the CS. */
968 while (mask) {
969 int i = u_bit_scan(&mask);
970
971 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
972 (struct r600_resource*)buffers->buffers[i],
973 buffers->shader_usage, buffers->priority);
974 }
975 }
976
977 static void si_get_buffer_from_descriptors(struct si_buffer_resources *buffers,
978 struct si_descriptors *descs,
979 unsigned idx, struct pipe_resource **buf,
980 unsigned *offset, unsigned *size)
981 {
982 pipe_resource_reference(buf, buffers->buffers[idx]);
983 if (*buf) {
984 struct r600_resource *res = r600_resource(*buf);
985 const uint32_t *desc = descs->list + idx * 4;
986 uint64_t va;
987
988 *size = desc[2];
989
990 assert(G_008F04_STRIDE(desc[1]) == 0);
991 va = ((uint64_t)desc[1] << 32) | desc[0];
992
993 assert(va >= res->gpu_address && va + *size <= res->gpu_address + res->bo_size);
994 *offset = va - res->gpu_address;
995 }
996 }
997
998 /* VERTEX BUFFERS */
999
1000 static void si_vertex_buffers_begin_new_cs(struct si_context *sctx)
1001 {
1002 struct si_descriptors *desc = &sctx->vertex_buffers;
1003 int count = sctx->vertex_elements ? sctx->vertex_elements->count : 0;
1004 int i;
1005
1006 for (i = 0; i < count; i++) {
1007 int vb = sctx->vertex_elements->elements[i].vertex_buffer_index;
1008
1009 if (vb >= ARRAY_SIZE(sctx->vertex_buffer))
1010 continue;
1011 if (!sctx->vertex_buffer[vb].buffer)
1012 continue;
1013
1014 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
1015 (struct r600_resource*)sctx->vertex_buffer[vb].buffer,
1016 RADEON_USAGE_READ, RADEON_PRIO_VERTEX_BUFFER);
1017 }
1018
1019 if (!desc->buffer)
1020 return;
1021 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
1022 desc->buffer, RADEON_USAGE_READ,
1023 RADEON_PRIO_DESCRIPTORS);
1024 }
1025
1026 bool si_upload_vertex_buffer_descriptors(struct si_context *sctx)
1027 {
1028 struct si_vertex_element *velems = sctx->vertex_elements;
1029 struct si_descriptors *desc = &sctx->vertex_buffers;
1030 unsigned i, count;
1031 unsigned desc_list_byte_size;
1032 unsigned first_vb_use_mask;
1033 uint64_t va;
1034 uint32_t *ptr;
1035
1036 if (!sctx->vertex_buffers_dirty || !velems)
1037 return true;
1038
1039 count = velems->count;
1040
1041 if (!count)
1042 return true;
1043
1044 desc_list_byte_size = velems->desc_list_byte_size;
1045 first_vb_use_mask = velems->first_vb_use_mask;
1046
1047 /* Vertex buffer descriptors are the only ones which are uploaded
1048 * directly through a staging buffer and don't go through
1049 * the fine-grained upload path.
1050 */
1051 u_upload_alloc(sctx->b.b.const_uploader, 0,
1052 desc_list_byte_size,
1053 si_optimal_tcc_alignment(sctx, desc_list_byte_size),
1054 &desc->buffer_offset,
1055 (struct pipe_resource**)&desc->buffer, (void**)&ptr);
1056 if (!desc->buffer)
1057 return false;
1058
1059 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
1060 desc->buffer, RADEON_USAGE_READ,
1061 RADEON_PRIO_DESCRIPTORS);
1062
1063 assert(count <= SI_MAX_ATTRIBS);
1064
1065 for (i = 0; i < count; i++) {
1066 struct pipe_vertex_element *ve = &velems->elements[i];
1067 struct pipe_vertex_buffer *vb;
1068 struct r600_resource *rbuffer;
1069 unsigned offset;
1070 unsigned vbo_index = ve->vertex_buffer_index;
1071 uint32_t *desc = &ptr[i*4];
1072
1073 vb = &sctx->vertex_buffer[vbo_index];
1074 rbuffer = (struct r600_resource*)vb->buffer;
1075 if (!rbuffer) {
1076 memset(desc, 0, 16);
1077 continue;
1078 }
1079
1080 offset = vb->buffer_offset + ve->src_offset;
1081 va = rbuffer->gpu_address + offset;
1082
1083 /* Fill in T# buffer resource description */
1084 desc[0] = va;
1085 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
1086 S_008F04_STRIDE(vb->stride);
1087
1088 if (sctx->b.chip_class != VI && vb->stride) {
1089 /* Round up by rounding down and adding 1 */
1090 desc[2] = (vb->buffer->width0 - offset -
1091 velems->format_size[i]) /
1092 vb->stride + 1;
1093 } else {
1094 desc[2] = vb->buffer->width0 - offset;
1095 }
1096
1097 desc[3] = velems->rsrc_word3[i];
1098
1099 if (first_vb_use_mask & (1 << i)) {
1100 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
1101 (struct r600_resource*)vb->buffer,
1102 RADEON_USAGE_READ, RADEON_PRIO_VERTEX_BUFFER);
1103 }
1104 }
1105
1106 /* Don't flush the const cache. It would have a very negative effect
1107 * on performance (confirmed by testing). New descriptors are always
1108 * uploaded to a fresh new buffer, so I don't think flushing the const
1109 * cache is needed. */
1110 si_mark_atom_dirty(sctx, &sctx->shader_userdata.atom);
1111 if (sctx->b.chip_class >= CIK)
1112 si_mark_atom_dirty(sctx, &sctx->prefetch_L2);
1113 sctx->vertex_buffers_dirty = false;
1114 sctx->vertex_buffer_pointer_dirty = true;
1115 return true;
1116 }
1117
1118
1119 /* CONSTANT BUFFERS */
1120
1121 static unsigned
1122 si_const_buffer_descriptors_idx(unsigned shader)
1123 {
1124 return SI_DESCS_FIRST_SHADER + shader * SI_NUM_SHADER_DESCS +
1125 SI_SHADER_DESCS_CONST_BUFFERS;
1126 }
1127
1128 static struct si_descriptors *
1129 si_const_buffer_descriptors(struct si_context *sctx, unsigned shader)
1130 {
1131 return &sctx->descriptors[si_const_buffer_descriptors_idx(shader)];
1132 }
1133
1134 void si_upload_const_buffer(struct si_context *sctx, struct r600_resource **rbuffer,
1135 const uint8_t *ptr, unsigned size, uint32_t *const_offset)
1136 {
1137 void *tmp;
1138
1139 u_upload_alloc(sctx->b.b.const_uploader, 0, size,
1140 si_optimal_tcc_alignment(sctx, size),
1141 const_offset,
1142 (struct pipe_resource**)rbuffer, &tmp);
1143 if (*rbuffer)
1144 util_memcpy_cpu_to_le32(tmp, ptr, size);
1145 }
1146
1147 static void si_set_constant_buffer(struct si_context *sctx,
1148 struct si_buffer_resources *buffers,
1149 unsigned descriptors_idx,
1150 uint slot, const struct pipe_constant_buffer *input)
1151 {
1152 struct si_descriptors *descs = &sctx->descriptors[descriptors_idx];
1153 assert(slot < descs->num_elements);
1154 pipe_resource_reference(&buffers->buffers[slot], NULL);
1155
1156 /* CIK cannot unbind a constant buffer (S_BUFFER_LOAD is buggy
1157 * with a NULL buffer). We need to use a dummy buffer instead. */
1158 if (sctx->b.chip_class == CIK &&
1159 (!input || (!input->buffer && !input->user_buffer)))
1160 input = &sctx->null_const_buf;
1161
1162 if (input && (input->buffer || input->user_buffer)) {
1163 struct pipe_resource *buffer = NULL;
1164 uint64_t va;
1165
1166 /* Upload the user buffer if needed. */
1167 if (input->user_buffer) {
1168 unsigned buffer_offset;
1169
1170 si_upload_const_buffer(sctx,
1171 (struct r600_resource**)&buffer, input->user_buffer,
1172 input->buffer_size, &buffer_offset);
1173 if (!buffer) {
1174 /* Just unbind on failure. */
1175 si_set_constant_buffer(sctx, buffers, descriptors_idx, slot, NULL);
1176 return;
1177 }
1178 va = r600_resource(buffer)->gpu_address + buffer_offset;
1179 } else {
1180 pipe_resource_reference(&buffer, input->buffer);
1181 va = r600_resource(buffer)->gpu_address + input->buffer_offset;
1182 /* Only track usage for non-user buffers. */
1183 r600_resource(buffer)->bind_history |= PIPE_BIND_CONSTANT_BUFFER;
1184 }
1185
1186 /* Set the descriptor. */
1187 uint32_t *desc = descs->list + slot*4;
1188 desc[0] = va;
1189 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
1190 S_008F04_STRIDE(0);
1191 desc[2] = input->buffer_size;
1192 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1193 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1194 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1195 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1196 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1197 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
1198
1199 buffers->buffers[slot] = buffer;
1200 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
1201 (struct r600_resource*)buffer,
1202 buffers->shader_usage,
1203 buffers->priority, true);
1204 buffers->enabled_mask |= 1u << slot;
1205 } else {
1206 /* Clear the descriptor. */
1207 memset(descs->list + slot*4, 0, sizeof(uint32_t) * 4);
1208 buffers->enabled_mask &= ~(1u << slot);
1209 }
1210
1211 descs->dirty_mask |= 1u << slot;
1212 sctx->descriptors_dirty |= 1u << descriptors_idx;
1213 }
1214
1215 void si_set_rw_buffer(struct si_context *sctx,
1216 uint slot, const struct pipe_constant_buffer *input)
1217 {
1218 si_set_constant_buffer(sctx, &sctx->rw_buffers,
1219 SI_DESCS_RW_BUFFERS, slot, input);
1220 }
1221
1222 static void si_pipe_set_constant_buffer(struct pipe_context *ctx,
1223 enum pipe_shader_type shader, uint slot,
1224 const struct pipe_constant_buffer *input)
1225 {
1226 struct si_context *sctx = (struct si_context *)ctx;
1227
1228 if (shader >= SI_NUM_SHADERS)
1229 return;
1230
1231 si_set_constant_buffer(sctx, &sctx->const_buffers[shader],
1232 si_const_buffer_descriptors_idx(shader),
1233 slot, input);
1234 }
1235
1236 void si_get_pipe_constant_buffer(struct si_context *sctx, uint shader,
1237 uint slot, struct pipe_constant_buffer *cbuf)
1238 {
1239 cbuf->user_buffer = NULL;
1240 si_get_buffer_from_descriptors(
1241 &sctx->const_buffers[shader],
1242 si_const_buffer_descriptors(sctx, shader),
1243 slot, &cbuf->buffer, &cbuf->buffer_offset, &cbuf->buffer_size);
1244 }
1245
1246 /* SHADER BUFFERS */
1247
1248 static unsigned
1249 si_shader_buffer_descriptors_idx(enum pipe_shader_type shader)
1250 {
1251 return SI_DESCS_FIRST_SHADER + shader * SI_NUM_SHADER_DESCS +
1252 SI_SHADER_DESCS_SHADER_BUFFERS;
1253 }
1254
1255 static struct si_descriptors *
1256 si_shader_buffer_descriptors(struct si_context *sctx,
1257 enum pipe_shader_type shader)
1258 {
1259 return &sctx->descriptors[si_shader_buffer_descriptors_idx(shader)];
1260 }
1261
1262 static void si_set_shader_buffers(struct pipe_context *ctx,
1263 enum pipe_shader_type shader,
1264 unsigned start_slot, unsigned count,
1265 const struct pipe_shader_buffer *sbuffers)
1266 {
1267 struct si_context *sctx = (struct si_context *)ctx;
1268 struct si_buffer_resources *buffers = &sctx->shader_buffers[shader];
1269 struct si_descriptors *descs = si_shader_buffer_descriptors(sctx, shader);
1270 unsigned i;
1271
1272 assert(start_slot + count <= SI_NUM_SHADER_BUFFERS);
1273
1274 for (i = 0; i < count; ++i) {
1275 const struct pipe_shader_buffer *sbuffer = sbuffers ? &sbuffers[i] : NULL;
1276 struct r600_resource *buf;
1277 unsigned slot = start_slot + i;
1278 uint32_t *desc = descs->list + slot * 4;
1279 uint64_t va;
1280
1281 if (!sbuffer || !sbuffer->buffer) {
1282 pipe_resource_reference(&buffers->buffers[slot], NULL);
1283 memset(desc, 0, sizeof(uint32_t) * 4);
1284 buffers->enabled_mask &= ~(1u << slot);
1285 descs->dirty_mask |= 1u << slot;
1286 sctx->descriptors_dirty |=
1287 1u << si_shader_buffer_descriptors_idx(shader);
1288 continue;
1289 }
1290
1291 buf = (struct r600_resource *)sbuffer->buffer;
1292 va = buf->gpu_address + sbuffer->buffer_offset;
1293
1294 desc[0] = va;
1295 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
1296 S_008F04_STRIDE(0);
1297 desc[2] = sbuffer->buffer_size;
1298 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1299 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1300 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1301 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1302 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1303 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
1304
1305 pipe_resource_reference(&buffers->buffers[slot], &buf->b.b);
1306 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx, buf,
1307 buffers->shader_usage,
1308 buffers->priority, true);
1309 buf->bind_history |= PIPE_BIND_SHADER_BUFFER;
1310
1311 buffers->enabled_mask |= 1u << slot;
1312 descs->dirty_mask |= 1u << slot;
1313 sctx->descriptors_dirty |=
1314 1u << si_shader_buffer_descriptors_idx(shader);
1315
1316 util_range_add(&buf->valid_buffer_range, sbuffer->buffer_offset,
1317 sbuffer->buffer_offset + sbuffer->buffer_size);
1318 }
1319 }
1320
1321 void si_get_shader_buffers(struct si_context *sctx,
1322 enum pipe_shader_type shader,
1323 uint start_slot, uint count,
1324 struct pipe_shader_buffer *sbuf)
1325 {
1326 struct si_buffer_resources *buffers = &sctx->shader_buffers[shader];
1327 struct si_descriptors *descs = si_shader_buffer_descriptors(sctx, shader);
1328
1329 for (unsigned i = 0; i < count; ++i) {
1330 si_get_buffer_from_descriptors(
1331 buffers, descs, start_slot + i,
1332 &sbuf[i].buffer, &sbuf[i].buffer_offset,
1333 &sbuf[i].buffer_size);
1334 }
1335 }
1336
1337 /* RING BUFFERS */
1338
1339 void si_set_ring_buffer(struct pipe_context *ctx, uint slot,
1340 struct pipe_resource *buffer,
1341 unsigned stride, unsigned num_records,
1342 bool add_tid, bool swizzle,
1343 unsigned element_size, unsigned index_stride, uint64_t offset)
1344 {
1345 struct si_context *sctx = (struct si_context *)ctx;
1346 struct si_buffer_resources *buffers = &sctx->rw_buffers;
1347 struct si_descriptors *descs = &sctx->descriptors[SI_DESCS_RW_BUFFERS];
1348
1349 /* The stride field in the resource descriptor has 14 bits */
1350 assert(stride < (1 << 14));
1351
1352 assert(slot < descs->num_elements);
1353 pipe_resource_reference(&buffers->buffers[slot], NULL);
1354
1355 if (buffer) {
1356 uint64_t va;
1357
1358 va = r600_resource(buffer)->gpu_address + offset;
1359
1360 switch (element_size) {
1361 default:
1362 assert(!"Unsupported ring buffer element size");
1363 case 0:
1364 case 2:
1365 element_size = 0;
1366 break;
1367 case 4:
1368 element_size = 1;
1369 break;
1370 case 8:
1371 element_size = 2;
1372 break;
1373 case 16:
1374 element_size = 3;
1375 break;
1376 }
1377
1378 switch (index_stride) {
1379 default:
1380 assert(!"Unsupported ring buffer index stride");
1381 case 0:
1382 case 8:
1383 index_stride = 0;
1384 break;
1385 case 16:
1386 index_stride = 1;
1387 break;
1388 case 32:
1389 index_stride = 2;
1390 break;
1391 case 64:
1392 index_stride = 3;
1393 break;
1394 }
1395
1396 if (sctx->b.chip_class >= VI && stride)
1397 num_records *= stride;
1398
1399 /* Set the descriptor. */
1400 uint32_t *desc = descs->list + slot*4;
1401 desc[0] = va;
1402 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
1403 S_008F04_STRIDE(stride) |
1404 S_008F04_SWIZZLE_ENABLE(swizzle);
1405 desc[2] = num_records;
1406 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1407 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1408 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1409 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1410 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1411 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) |
1412 S_008F0C_INDEX_STRIDE(index_stride) |
1413 S_008F0C_ADD_TID_ENABLE(add_tid);
1414
1415 if (sctx->b.chip_class >= GFX9)
1416 assert(!swizzle || element_size == 1); /* always 4 bytes on GFX9 */
1417 else
1418 desc[3] |= S_008F0C_ELEMENT_SIZE(element_size);
1419
1420 pipe_resource_reference(&buffers->buffers[slot], buffer);
1421 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
1422 (struct r600_resource*)buffer,
1423 buffers->shader_usage, buffers->priority);
1424 buffers->enabled_mask |= 1u << slot;
1425 } else {
1426 /* Clear the descriptor. */
1427 memset(descs->list + slot*4, 0, sizeof(uint32_t) * 4);
1428 buffers->enabled_mask &= ~(1u << slot);
1429 }
1430
1431 descs->dirty_mask |= 1u << slot;
1432 sctx->descriptors_dirty |= 1u << SI_DESCS_RW_BUFFERS;
1433 }
1434
1435 /* STREAMOUT BUFFERS */
1436
1437 static void si_set_streamout_targets(struct pipe_context *ctx,
1438 unsigned num_targets,
1439 struct pipe_stream_output_target **targets,
1440 const unsigned *offsets)
1441 {
1442 struct si_context *sctx = (struct si_context *)ctx;
1443 struct si_buffer_resources *buffers = &sctx->rw_buffers;
1444 struct si_descriptors *descs = &sctx->descriptors[SI_DESCS_RW_BUFFERS];
1445 unsigned old_num_targets = sctx->b.streamout.num_targets;
1446 unsigned i, bufidx;
1447
1448 /* We are going to unbind the buffers. Mark which caches need to be flushed. */
1449 if (sctx->b.streamout.num_targets && sctx->b.streamout.begin_emitted) {
1450 /* Since streamout uses vector writes which go through TC L2
1451 * and most other clients can use TC L2 as well, we don't need
1452 * to flush it.
1453 *
1454 * The only cases which requires flushing it is VGT DMA index
1455 * fetching (on <= CIK) and indirect draw data, which are rare
1456 * cases. Thus, flag the TC L2 dirtiness in the resource and
1457 * handle it at draw call time.
1458 */
1459 for (i = 0; i < sctx->b.streamout.num_targets; i++)
1460 if (sctx->b.streamout.targets[i])
1461 r600_resource(sctx->b.streamout.targets[i]->b.buffer)->TC_L2_dirty = true;
1462
1463 /* Invalidate the scalar cache in case a streamout buffer is
1464 * going to be used as a constant buffer.
1465 *
1466 * Invalidate TC L1, because streamout bypasses it (done by
1467 * setting GLC=1 in the store instruction), but it can contain
1468 * outdated data of streamout buffers.
1469 *
1470 * VS_PARTIAL_FLUSH is required if the buffers are going to be
1471 * used as an input immediately.
1472 */
1473 sctx->b.flags |= SI_CONTEXT_INV_SMEM_L1 |
1474 SI_CONTEXT_INV_VMEM_L1 |
1475 SI_CONTEXT_VS_PARTIAL_FLUSH;
1476 }
1477
1478 /* All readers of the streamout targets need to be finished before we can
1479 * start writing to the targets.
1480 */
1481 if (num_targets)
1482 sctx->b.flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
1483 SI_CONTEXT_CS_PARTIAL_FLUSH;
1484
1485 /* Streamout buffers must be bound in 2 places:
1486 * 1) in VGT by setting the VGT_STRMOUT registers
1487 * 2) as shader resources
1488 */
1489
1490 /* Set the VGT regs. */
1491 r600_set_streamout_targets(ctx, num_targets, targets, offsets);
1492
1493 /* Set the shader resources.*/
1494 for (i = 0; i < num_targets; i++) {
1495 bufidx = SI_VS_STREAMOUT_BUF0 + i;
1496
1497 if (targets[i]) {
1498 struct pipe_resource *buffer = targets[i]->buffer;
1499 uint64_t va = r600_resource(buffer)->gpu_address;
1500
1501 /* Set the descriptor.
1502 *
1503 * On VI, the format must be non-INVALID, otherwise
1504 * the buffer will be considered not bound and store
1505 * instructions will be no-ops.
1506 */
1507 uint32_t *desc = descs->list + bufidx*4;
1508 desc[0] = va;
1509 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32);
1510 desc[2] = 0xffffffff;
1511 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1512 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1513 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1514 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1515 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
1516
1517 /* Set the resource. */
1518 pipe_resource_reference(&buffers->buffers[bufidx],
1519 buffer);
1520 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
1521 (struct r600_resource*)buffer,
1522 buffers->shader_usage,
1523 RADEON_PRIO_SHADER_RW_BUFFER,
1524 true);
1525 r600_resource(buffer)->bind_history |= PIPE_BIND_STREAM_OUTPUT;
1526
1527 buffers->enabled_mask |= 1u << bufidx;
1528 } else {
1529 /* Clear the descriptor and unset the resource. */
1530 memset(descs->list + bufidx*4, 0,
1531 sizeof(uint32_t) * 4);
1532 pipe_resource_reference(&buffers->buffers[bufidx],
1533 NULL);
1534 buffers->enabled_mask &= ~(1u << bufidx);
1535 }
1536 descs->dirty_mask |= 1u << bufidx;
1537 }
1538 for (; i < old_num_targets; i++) {
1539 bufidx = SI_VS_STREAMOUT_BUF0 + i;
1540 /* Clear the descriptor and unset the resource. */
1541 memset(descs->list + bufidx*4, 0, sizeof(uint32_t) * 4);
1542 pipe_resource_reference(&buffers->buffers[bufidx], NULL);
1543 buffers->enabled_mask &= ~(1u << bufidx);
1544 descs->dirty_mask |= 1u << bufidx;
1545 }
1546
1547 sctx->descriptors_dirty |= 1u << SI_DESCS_RW_BUFFERS;
1548 }
1549
1550 static void si_desc_reset_buffer_offset(struct pipe_context *ctx,
1551 uint32_t *desc, uint64_t old_buf_va,
1552 struct pipe_resource *new_buf)
1553 {
1554 /* Retrieve the buffer offset from the descriptor. */
1555 uint64_t old_desc_va =
1556 desc[0] | ((uint64_t)G_008F04_BASE_ADDRESS_HI(desc[1]) << 32);
1557
1558 assert(old_buf_va <= old_desc_va);
1559 uint64_t offset_within_buffer = old_desc_va - old_buf_va;
1560
1561 /* Update the descriptor. */
1562 si_set_buf_desc_address(r600_resource(new_buf), offset_within_buffer,
1563 desc);
1564 }
1565
1566 /* INTERNAL CONST BUFFERS */
1567
1568 static void si_set_polygon_stipple(struct pipe_context *ctx,
1569 const struct pipe_poly_stipple *state)
1570 {
1571 struct si_context *sctx = (struct si_context *)ctx;
1572 struct pipe_constant_buffer cb = {};
1573 unsigned stipple[32];
1574 int i;
1575
1576 for (i = 0; i < 32; i++)
1577 stipple[i] = util_bitreverse(state->stipple[i]);
1578
1579 cb.user_buffer = stipple;
1580 cb.buffer_size = sizeof(stipple);
1581
1582 si_set_rw_buffer(sctx, SI_PS_CONST_POLY_STIPPLE, &cb);
1583 }
1584
1585 /* TEXTURE METADATA ENABLE/DISABLE */
1586
1587 /* CMASK can be enabled (for fast clear) and disabled (for texture export)
1588 * while the texture is bound, possibly by a different context. In that case,
1589 * call this function to update compressed_colortex_masks.
1590 */
1591 void si_update_compressed_colortex_masks(struct si_context *sctx)
1592 {
1593 for (int i = 0; i < SI_NUM_SHADERS; ++i) {
1594 si_samplers_update_compressed_colortex_mask(&sctx->samplers[i]);
1595 si_images_update_compressed_colortex_mask(&sctx->images[i]);
1596 si_update_compressed_tex_shader_mask(sctx, i);
1597 }
1598 }
1599
1600 /* BUFFER DISCARD/INVALIDATION */
1601
1602 /** Reset descriptors of buffer resources after \p buf has been invalidated. */
1603 static void si_reset_buffer_resources(struct si_context *sctx,
1604 struct si_buffer_resources *buffers,
1605 unsigned descriptors_idx,
1606 struct pipe_resource *buf,
1607 uint64_t old_va)
1608 {
1609 struct si_descriptors *descs = &sctx->descriptors[descriptors_idx];
1610 unsigned mask = buffers->enabled_mask;
1611
1612 while (mask) {
1613 unsigned i = u_bit_scan(&mask);
1614 if (buffers->buffers[i] == buf) {
1615 si_desc_reset_buffer_offset(&sctx->b.b,
1616 descs->list + i*4,
1617 old_va, buf);
1618 descs->dirty_mask |= 1u << i;
1619 sctx->descriptors_dirty |= 1u << descriptors_idx;
1620
1621 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
1622 (struct r600_resource *)buf,
1623 buffers->shader_usage,
1624 buffers->priority, true);
1625 }
1626 }
1627 }
1628
1629 /* Reallocate a buffer a update all resource bindings where the buffer is
1630 * bound.
1631 *
1632 * This is used to avoid CPU-GPU synchronizations, because it makes the buffer
1633 * idle by discarding its contents. Apps usually tell us when to do this using
1634 * map_buffer flags, for example.
1635 */
1636 static void si_invalidate_buffer(struct pipe_context *ctx, struct pipe_resource *buf)
1637 {
1638 struct si_context *sctx = (struct si_context*)ctx;
1639 struct r600_resource *rbuffer = r600_resource(buf);
1640 unsigned i, shader;
1641 uint64_t old_va = rbuffer->gpu_address;
1642 unsigned num_elems = sctx->vertex_elements ?
1643 sctx->vertex_elements->count : 0;
1644
1645 /* Reallocate the buffer in the same pipe_resource. */
1646 r600_alloc_resource(&sctx->screen->b, rbuffer);
1647
1648 /* We changed the buffer, now we need to bind it where the old one
1649 * was bound. This consists of 2 things:
1650 * 1) Updating the resource descriptor and dirtying it.
1651 * 2) Adding a relocation to the CS, so that it's usable.
1652 */
1653
1654 /* Vertex buffers. */
1655 if (rbuffer->bind_history & PIPE_BIND_VERTEX_BUFFER) {
1656 for (i = 0; i < num_elems; i++) {
1657 int vb = sctx->vertex_elements->elements[i].vertex_buffer_index;
1658
1659 if (vb >= ARRAY_SIZE(sctx->vertex_buffer))
1660 continue;
1661 if (!sctx->vertex_buffer[vb].buffer)
1662 continue;
1663
1664 if (sctx->vertex_buffer[vb].buffer == buf) {
1665 sctx->vertex_buffers_dirty = true;
1666 break;
1667 }
1668 }
1669 }
1670
1671 /* Streamout buffers. (other internal buffers can't be invalidated) */
1672 if (rbuffer->bind_history & PIPE_BIND_STREAM_OUTPUT) {
1673 for (i = SI_VS_STREAMOUT_BUF0; i <= SI_VS_STREAMOUT_BUF3; i++) {
1674 struct si_buffer_resources *buffers = &sctx->rw_buffers;
1675 struct si_descriptors *descs =
1676 &sctx->descriptors[SI_DESCS_RW_BUFFERS];
1677
1678 if (buffers->buffers[i] != buf)
1679 continue;
1680
1681 si_desc_reset_buffer_offset(ctx, descs->list + i*4,
1682 old_va, buf);
1683 descs->dirty_mask |= 1u << i;
1684 sctx->descriptors_dirty |= 1u << SI_DESCS_RW_BUFFERS;
1685
1686 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
1687 rbuffer, buffers->shader_usage,
1688 RADEON_PRIO_SHADER_RW_BUFFER,
1689 true);
1690
1691 /* Update the streamout state. */
1692 if (sctx->b.streamout.begin_emitted)
1693 r600_emit_streamout_end(&sctx->b);
1694 sctx->b.streamout.append_bitmask =
1695 sctx->b.streamout.enabled_mask;
1696 r600_streamout_buffers_dirty(&sctx->b);
1697 }
1698 }
1699
1700 /* Constant and shader buffers. */
1701 if (rbuffer->bind_history & PIPE_BIND_CONSTANT_BUFFER) {
1702 for (shader = 0; shader < SI_NUM_SHADERS; shader++)
1703 si_reset_buffer_resources(sctx, &sctx->const_buffers[shader],
1704 si_const_buffer_descriptors_idx(shader),
1705 buf, old_va);
1706 }
1707
1708 if (rbuffer->bind_history & PIPE_BIND_SHADER_BUFFER) {
1709 for (shader = 0; shader < SI_NUM_SHADERS; shader++)
1710 si_reset_buffer_resources(sctx, &sctx->shader_buffers[shader],
1711 si_shader_buffer_descriptors_idx(shader),
1712 buf, old_va);
1713 }
1714
1715 if (rbuffer->bind_history & PIPE_BIND_SAMPLER_VIEW) {
1716 /* Texture buffers - update bindings. */
1717 for (shader = 0; shader < SI_NUM_SHADERS; shader++) {
1718 struct si_sampler_views *views = &sctx->samplers[shader].views;
1719 struct si_descriptors *descs =
1720 si_sampler_descriptors(sctx, shader);
1721 unsigned mask = views->enabled_mask;
1722
1723 while (mask) {
1724 unsigned i = u_bit_scan(&mask);
1725 if (views->views[i]->texture == buf) {
1726 si_desc_reset_buffer_offset(ctx,
1727 descs->list +
1728 i * 16 + 4,
1729 old_va, buf);
1730 descs->dirty_mask |= 1u << i;
1731 sctx->descriptors_dirty |=
1732 1u << si_sampler_descriptors_idx(shader);
1733
1734 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
1735 rbuffer, RADEON_USAGE_READ,
1736 RADEON_PRIO_SAMPLER_BUFFER,
1737 true);
1738 }
1739 }
1740 }
1741 }
1742
1743 /* Shader images */
1744 if (rbuffer->bind_history & PIPE_BIND_SHADER_IMAGE) {
1745 for (shader = 0; shader < SI_NUM_SHADERS; ++shader) {
1746 struct si_images_info *images = &sctx->images[shader];
1747 struct si_descriptors *descs =
1748 si_image_descriptors(sctx, shader);
1749 unsigned mask = images->enabled_mask;
1750
1751 while (mask) {
1752 unsigned i = u_bit_scan(&mask);
1753
1754 if (images->views[i].resource == buf) {
1755 if (images->views[i].access & PIPE_IMAGE_ACCESS_WRITE)
1756 si_mark_image_range_valid(&images->views[i]);
1757
1758 si_desc_reset_buffer_offset(
1759 ctx, descs->list + i * 8 + 4,
1760 old_va, buf);
1761 descs->dirty_mask |= 1u << i;
1762 sctx->descriptors_dirty |=
1763 1u << si_image_descriptors_idx(shader);
1764
1765 radeon_add_to_buffer_list_check_mem(
1766 &sctx->b, &sctx->b.gfx, rbuffer,
1767 RADEON_USAGE_READWRITE,
1768 RADEON_PRIO_SAMPLER_BUFFER, true);
1769 }
1770 }
1771 }
1772 }
1773 }
1774
1775 /* Update mutable image descriptor fields of all bound textures. */
1776 void si_update_all_texture_descriptors(struct si_context *sctx)
1777 {
1778 unsigned shader;
1779
1780 for (shader = 0; shader < SI_NUM_SHADERS; shader++) {
1781 struct si_sampler_views *samplers = &sctx->samplers[shader].views;
1782 struct si_images_info *images = &sctx->images[shader];
1783 unsigned mask;
1784
1785 /* Images. */
1786 mask = images->enabled_mask;
1787 while (mask) {
1788 unsigned i = u_bit_scan(&mask);
1789 struct pipe_image_view *view = &images->views[i];
1790
1791 if (!view->resource ||
1792 view->resource->target == PIPE_BUFFER)
1793 continue;
1794
1795 si_set_shader_image(sctx, shader, i, view, true);
1796 }
1797
1798 /* Sampler views. */
1799 mask = samplers->enabled_mask;
1800 while (mask) {
1801 unsigned i = u_bit_scan(&mask);
1802 struct pipe_sampler_view *view = samplers->views[i];
1803
1804 if (!view ||
1805 !view->texture ||
1806 view->texture->target == PIPE_BUFFER)
1807 continue;
1808
1809 si_set_sampler_view(sctx, shader, i,
1810 samplers->views[i], true);
1811 }
1812
1813 si_update_compressed_tex_shader_mask(sctx, shader);
1814 }
1815 }
1816
1817 /* SHADER USER DATA */
1818
1819 static void si_mark_shader_pointers_dirty(struct si_context *sctx,
1820 unsigned shader)
1821 {
1822 sctx->shader_pointers_dirty |=
1823 u_bit_consecutive(SI_DESCS_FIRST_SHADER + shader * SI_NUM_SHADER_DESCS,
1824 SI_NUM_SHADER_DESCS);
1825
1826 if (shader == PIPE_SHADER_VERTEX)
1827 sctx->vertex_buffer_pointer_dirty = sctx->vertex_buffers.buffer != NULL;
1828
1829 si_mark_atom_dirty(sctx, &sctx->shader_userdata.atom);
1830 }
1831
1832 static void si_shader_userdata_begin_new_cs(struct si_context *sctx)
1833 {
1834 sctx->shader_pointers_dirty = u_bit_consecutive(0, SI_NUM_DESCS);
1835 sctx->vertex_buffer_pointer_dirty = sctx->vertex_buffers.buffer != NULL;
1836 si_mark_atom_dirty(sctx, &sctx->shader_userdata.atom);
1837 }
1838
1839 /* Set a base register address for user data constants in the given shader.
1840 * This assigns a mapping from PIPE_SHADER_* to SPI_SHADER_USER_DATA_*.
1841 */
1842 static void si_set_user_data_base(struct si_context *sctx,
1843 unsigned shader, uint32_t new_base)
1844 {
1845 uint32_t *base = &sctx->shader_userdata.sh_base[shader];
1846
1847 if (*base != new_base) {
1848 *base = new_base;
1849
1850 if (new_base) {
1851 si_mark_shader_pointers_dirty(sctx, shader);
1852
1853 if (shader == PIPE_SHADER_VERTEX)
1854 sctx->last_vs_state = ~0;
1855 }
1856 }
1857 }
1858
1859 /* This must be called when these shaders are changed from non-NULL to NULL
1860 * and vice versa:
1861 * - geometry shader
1862 * - tessellation control shader
1863 * - tessellation evaluation shader
1864 */
1865 void si_shader_change_notify(struct si_context *sctx)
1866 {
1867 /* VS can be bound as VS, ES, or LS. */
1868 if (sctx->tes_shader.cso) {
1869 if (sctx->b.chip_class >= GFX9) {
1870 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX,
1871 R_00B430_SPI_SHADER_USER_DATA_LS_0);
1872 } else {
1873 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX,
1874 R_00B530_SPI_SHADER_USER_DATA_LS_0);
1875 }
1876 } else if (sctx->gs_shader.cso) {
1877 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX,
1878 R_00B330_SPI_SHADER_USER_DATA_ES_0);
1879 } else {
1880 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX,
1881 R_00B130_SPI_SHADER_USER_DATA_VS_0);
1882 }
1883
1884 /* TES can be bound as ES, VS, or not bound. */
1885 if (sctx->tes_shader.cso) {
1886 if (sctx->gs_shader.cso)
1887 si_set_user_data_base(sctx, PIPE_SHADER_TESS_EVAL,
1888 R_00B330_SPI_SHADER_USER_DATA_ES_0);
1889 else
1890 si_set_user_data_base(sctx, PIPE_SHADER_TESS_EVAL,
1891 R_00B130_SPI_SHADER_USER_DATA_VS_0);
1892 } else {
1893 si_set_user_data_base(sctx, PIPE_SHADER_TESS_EVAL, 0);
1894 }
1895 }
1896
1897 static void si_emit_shader_pointer(struct si_context *sctx,
1898 struct si_descriptors *desc,
1899 unsigned sh_base)
1900 {
1901 struct radeon_winsys_cs *cs = sctx->b.gfx.cs;
1902 uint64_t va;
1903
1904 assert(desc->buffer);
1905
1906 va = desc->buffer->gpu_address +
1907 desc->buffer_offset;
1908
1909 radeon_emit(cs, PKT3(PKT3_SET_SH_REG, 2, 0));
1910 radeon_emit(cs, (sh_base + desc->shader_userdata_offset - SI_SH_REG_OFFSET) >> 2);
1911 radeon_emit(cs, va);
1912 radeon_emit(cs, va >> 32);
1913 }
1914
1915 void si_emit_graphics_shader_userdata(struct si_context *sctx,
1916 struct r600_atom *atom)
1917 {
1918 unsigned mask;
1919 uint32_t *sh_base = sctx->shader_userdata.sh_base;
1920 struct si_descriptors *descs;
1921
1922 descs = &sctx->descriptors[SI_DESCS_RW_BUFFERS];
1923
1924 if (sctx->shader_pointers_dirty & (1 << SI_DESCS_RW_BUFFERS)) {
1925 si_emit_shader_pointer(sctx, descs,
1926 R_00B030_SPI_SHADER_USER_DATA_PS_0);
1927 si_emit_shader_pointer(sctx, descs,
1928 R_00B130_SPI_SHADER_USER_DATA_VS_0);
1929 si_emit_shader_pointer(sctx, descs,
1930 R_00B330_SPI_SHADER_USER_DATA_ES_0);
1931
1932 /* GFX9 merged LS-HS and ES-GS. Only set RW_BUFFERS for ES and LS. */
1933 if (sctx->b.chip_class >= GFX9) {
1934 si_emit_shader_pointer(sctx, descs,
1935 R_00B430_SPI_SHADER_USER_DATA_LS_0);
1936 } else {
1937 si_emit_shader_pointer(sctx, descs,
1938 R_00B230_SPI_SHADER_USER_DATA_GS_0);
1939 si_emit_shader_pointer(sctx, descs,
1940 R_00B430_SPI_SHADER_USER_DATA_HS_0);
1941 }
1942 }
1943
1944 mask = sctx->shader_pointers_dirty &
1945 u_bit_consecutive(SI_DESCS_FIRST_SHADER,
1946 SI_DESCS_FIRST_COMPUTE - SI_DESCS_FIRST_SHADER);
1947
1948 while (mask) {
1949 unsigned i = u_bit_scan(&mask);
1950 unsigned shader = (i - SI_DESCS_FIRST_SHADER) / SI_NUM_SHADER_DESCS;
1951 unsigned base = sh_base[shader];
1952
1953 if (base)
1954 si_emit_shader_pointer(sctx, descs + i, base);
1955 }
1956 sctx->shader_pointers_dirty &=
1957 ~u_bit_consecutive(SI_DESCS_RW_BUFFERS, SI_DESCS_FIRST_COMPUTE);
1958
1959 if (sctx->vertex_buffer_pointer_dirty) {
1960 si_emit_shader_pointer(sctx, &sctx->vertex_buffers,
1961 sh_base[PIPE_SHADER_VERTEX]);
1962 sctx->vertex_buffer_pointer_dirty = false;
1963 }
1964 }
1965
1966 void si_emit_compute_shader_userdata(struct si_context *sctx)
1967 {
1968 unsigned base = R_00B900_COMPUTE_USER_DATA_0;
1969 struct si_descriptors *descs = sctx->descriptors;
1970 unsigned compute_mask =
1971 u_bit_consecutive(SI_DESCS_FIRST_COMPUTE, SI_NUM_SHADER_DESCS);
1972 unsigned mask = sctx->shader_pointers_dirty & compute_mask;
1973
1974 while (mask) {
1975 unsigned i = u_bit_scan(&mask);
1976
1977 si_emit_shader_pointer(sctx, descs + i, base);
1978 }
1979 sctx->shader_pointers_dirty &= ~compute_mask;
1980 }
1981
1982 /* INIT/DEINIT/UPLOAD */
1983
1984 void si_init_all_descriptors(struct si_context *sctx)
1985 {
1986 int i;
1987 unsigned ce_offset = 0;
1988
1989 STATIC_ASSERT(GFX9_SGPR_TCS_CONST_BUFFERS % 2 == 0);
1990
1991 for (i = 0; i < SI_NUM_SHADERS; i++) {
1992 bool gfx9_tcs = sctx->b.chip_class == GFX9 &&
1993 i == PIPE_SHADER_TESS_CTRL;
1994 /* GFX9 has only 4KB of CE, while previous chips had 32KB.
1995 * Rarely used descriptors don't use CE RAM.
1996 */
1997 bool big_ce = sctx->b.chip_class <= VI;
1998 bool images_use_ce = big_ce;
1999 bool shaderbufs_use_ce = big_ce ||
2000 i == PIPE_SHADER_COMPUTE;
2001 bool samplers_use_ce = big_ce ||
2002 i == PIPE_SHADER_FRAGMENT;
2003
2004 si_init_buffer_resources(&sctx->const_buffers[i],
2005 si_const_buffer_descriptors(sctx, i),
2006 SI_NUM_CONST_BUFFERS,
2007 gfx9_tcs ? GFX9_SGPR_TCS_CONST_BUFFERS :
2008 SI_SGPR_CONST_BUFFERS,
2009 RADEON_USAGE_READ, RADEON_PRIO_CONST_BUFFER,
2010 &ce_offset);
2011 si_init_buffer_resources(&sctx->shader_buffers[i],
2012 si_shader_buffer_descriptors(sctx, i),
2013 SI_NUM_SHADER_BUFFERS,
2014 gfx9_tcs ? GFX9_SGPR_TCS_SHADER_BUFFERS :
2015 SI_SGPR_SHADER_BUFFERS,
2016 RADEON_USAGE_READWRITE, RADEON_PRIO_SHADER_RW_BUFFER,
2017 shaderbufs_use_ce ? &ce_offset : NULL);
2018
2019 si_init_descriptors(si_sampler_descriptors(sctx, i),
2020 gfx9_tcs ? GFX9_SGPR_TCS_SAMPLERS :
2021 SI_SGPR_SAMPLERS,
2022 16, SI_NUM_SAMPLERS,
2023 null_texture_descriptor,
2024 samplers_use_ce ? &ce_offset : NULL);
2025
2026 si_init_descriptors(si_image_descriptors(sctx, i),
2027 gfx9_tcs ? GFX9_SGPR_TCS_IMAGES :
2028 SI_SGPR_IMAGES,
2029 8, SI_NUM_IMAGES,
2030 null_image_descriptor,
2031 images_use_ce ? &ce_offset : NULL);
2032 }
2033
2034 si_init_buffer_resources(&sctx->rw_buffers,
2035 &sctx->descriptors[SI_DESCS_RW_BUFFERS],
2036 SI_NUM_RW_BUFFERS, SI_SGPR_RW_BUFFERS,
2037 RADEON_USAGE_READWRITE, RADEON_PRIO_SHADER_RINGS,
2038 &ce_offset);
2039 si_init_descriptors(&sctx->vertex_buffers, SI_SGPR_VERTEX_BUFFERS,
2040 4, SI_NUM_VERTEX_BUFFERS, NULL, NULL);
2041
2042 sctx->descriptors_dirty = u_bit_consecutive(0, SI_NUM_DESCS);
2043
2044 if (sctx->b.chip_class >= GFX9)
2045 assert(ce_offset <= 4096);
2046 else
2047 assert(ce_offset <= 32768);
2048
2049 /* Set pipe_context functions. */
2050 sctx->b.b.bind_sampler_states = si_bind_sampler_states;
2051 sctx->b.b.set_shader_images = si_set_shader_images;
2052 sctx->b.b.set_constant_buffer = si_pipe_set_constant_buffer;
2053 sctx->b.b.set_polygon_stipple = si_set_polygon_stipple;
2054 sctx->b.b.set_shader_buffers = si_set_shader_buffers;
2055 sctx->b.b.set_sampler_views = si_set_sampler_views;
2056 sctx->b.b.set_stream_output_targets = si_set_streamout_targets;
2057 sctx->b.invalidate_buffer = si_invalidate_buffer;
2058
2059 /* Shader user data. */
2060 si_init_atom(sctx, &sctx->shader_userdata.atom, &sctx->atoms.s.shader_userdata,
2061 si_emit_graphics_shader_userdata);
2062
2063 /* Set default and immutable mappings. */
2064 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX, R_00B130_SPI_SHADER_USER_DATA_VS_0);
2065
2066 if (sctx->b.chip_class >= GFX9) {
2067 si_set_user_data_base(sctx, PIPE_SHADER_TESS_CTRL,
2068 R_00B430_SPI_SHADER_USER_DATA_LS_0);
2069 si_set_user_data_base(sctx, PIPE_SHADER_GEOMETRY,
2070 R_00B330_SPI_SHADER_USER_DATA_ES_0);
2071 } else {
2072 si_set_user_data_base(sctx, PIPE_SHADER_TESS_CTRL,
2073 R_00B430_SPI_SHADER_USER_DATA_HS_0);
2074 si_set_user_data_base(sctx, PIPE_SHADER_GEOMETRY,
2075 R_00B230_SPI_SHADER_USER_DATA_GS_0);
2076 }
2077 si_set_user_data_base(sctx, PIPE_SHADER_FRAGMENT, R_00B030_SPI_SHADER_USER_DATA_PS_0);
2078 }
2079
2080 bool si_upload_graphics_shader_descriptors(struct si_context *sctx)
2081 {
2082 const unsigned mask = u_bit_consecutive(0, SI_DESCS_FIRST_COMPUTE);
2083 unsigned dirty = sctx->descriptors_dirty & mask;
2084
2085 /* Assume nothing will go wrong: */
2086 sctx->shader_pointers_dirty |= dirty;
2087
2088 while (dirty) {
2089 unsigned i = u_bit_scan(&dirty);
2090
2091 if (!si_upload_descriptors(sctx, &sctx->descriptors[i],
2092 &sctx->shader_userdata.atom))
2093 return false;
2094 }
2095
2096 sctx->descriptors_dirty &= ~mask;
2097 return true;
2098 }
2099
2100 bool si_upload_compute_shader_descriptors(struct si_context *sctx)
2101 {
2102 /* Does not update rw_buffers as that is not needed for compute shaders
2103 * and the input buffer is using the same SGPR's anyway.
2104 */
2105 const unsigned mask = u_bit_consecutive(SI_DESCS_FIRST_COMPUTE,
2106 SI_NUM_DESCS - SI_DESCS_FIRST_COMPUTE);
2107 unsigned dirty = sctx->descriptors_dirty & mask;
2108
2109 /* Assume nothing will go wrong: */
2110 sctx->shader_pointers_dirty |= dirty;
2111
2112 while (dirty) {
2113 unsigned i = u_bit_scan(&dirty);
2114
2115 if (!si_upload_descriptors(sctx, &sctx->descriptors[i], NULL))
2116 return false;
2117 }
2118
2119 sctx->descriptors_dirty &= ~mask;
2120
2121 return true;
2122 }
2123
2124 void si_release_all_descriptors(struct si_context *sctx)
2125 {
2126 int i;
2127
2128 for (i = 0; i < SI_NUM_SHADERS; i++) {
2129 si_release_buffer_resources(&sctx->const_buffers[i],
2130 si_const_buffer_descriptors(sctx, i));
2131 si_release_buffer_resources(&sctx->shader_buffers[i],
2132 si_shader_buffer_descriptors(sctx, i));
2133 si_release_sampler_views(&sctx->samplers[i].views);
2134 si_release_image_views(&sctx->images[i]);
2135 }
2136 si_release_buffer_resources(&sctx->rw_buffers,
2137 &sctx->descriptors[SI_DESCS_RW_BUFFERS]);
2138
2139 for (i = 0; i < SI_NUM_DESCS; ++i)
2140 si_release_descriptors(&sctx->descriptors[i]);
2141 si_release_descriptors(&sctx->vertex_buffers);
2142 }
2143
2144 void si_all_descriptors_begin_new_cs(struct si_context *sctx)
2145 {
2146 int i;
2147
2148 for (i = 0; i < SI_NUM_SHADERS; i++) {
2149 si_buffer_resources_begin_new_cs(sctx, &sctx->const_buffers[i]);
2150 si_buffer_resources_begin_new_cs(sctx, &sctx->shader_buffers[i]);
2151 si_sampler_views_begin_new_cs(sctx, &sctx->samplers[i].views);
2152 si_image_views_begin_new_cs(sctx, &sctx->images[i]);
2153 }
2154 si_buffer_resources_begin_new_cs(sctx, &sctx->rw_buffers);
2155 si_vertex_buffers_begin_new_cs(sctx);
2156
2157 for (i = 0; i < SI_NUM_DESCS; ++i)
2158 si_descriptors_begin_new_cs(sctx, &sctx->descriptors[i]);
2159
2160 si_shader_userdata_begin_new_cs(sctx);
2161 }