projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
radeonsi: use a clever alignment for descriptor uploads
[mesa.git] / src / gallium / drivers / radeonsi / si_descriptors.c
1 /*
2 * Copyright 2013 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 *
23 * Authors:
24 * Marek Olšák <marek.olsak@amd.com>
25 */
26
27 /* Resource binding slots and sampler states (each described with 8 or
28 * 4 dwords) are stored in lists in memory which is accessed by shaders
29 * using scalar load instructions.
30 *
31 * This file is responsible for managing such lists. It keeps a copy of all
32 * descriptors in CPU memory and re-uploads a whole list if some slots have
33 * been changed.
34 *
35 * This code is also reponsible for updating shader pointers to those lists.
36 *
37 * Note that CP DMA can't be used for updating the lists, because a GPU hang
38 * could leave the list in a mid-IB state and the next IB would get wrong
39 * descriptors and the whole context would be unusable at that point.
40 * (Note: The register shadowing can't be used due to the same reason)
41 *
42 * Also, uploading descriptors to newly allocated memory doesn't require
43 * a KCACHE flush.
44 *
45 *
46 * Possible scenarios for one 16 dword image+sampler slot:
47 *
48 * | Image | w/ FMASK | Buffer | NULL
49 * [ 0: 3] Image[0:3] | Image[0:3] | Null[0:3] | Null[0:3]
50 * [ 4: 7] Image[4:7] | Image[4:7] | Buffer[0:3] | 0
51 * [ 8:11] Null[0:3] | Fmask[0:3] | Null[0:3] | Null[0:3]
52 * [12:15] Sampler[0:3] | Fmask[4:7] | Sampler[0:3] | Sampler[0:3]
53 *
54 * FMASK implies MSAA, therefore no sampler state.
55 * Sampler states are never unbound except when FMASK is bound.
56 */
57
58 #include "radeon/r600_cs.h"
59 #include "si_pipe.h"
60 #include "sid.h"
61
62 #include "util/u_format.h"
63 #include "util/u_memory.h"
64 #include "util/u_upload_mgr.h"
65
66
67 /* NULL image and buffer descriptor for textures (alpha = 1) and images
68 * (alpha = 0).
69 *
70 * For images, all fields must be zero except for the swizzle, which
71 * supports arbitrary combinations of 0s and 1s. The texture type must be
72 * any valid type (e.g. 1D). If the texture type isn't set, the hw hangs.
73 *
74 * For buffers, all fields must be zero. If they are not, the hw hangs.
75 *
76 * This is the only reason why the buffer descriptor must be in words [4:7].
77 */
78 static uint32_t null_texture_descriptor[8] = {
79 0,
80 0,
81 0,
82 S_008F1C_DST_SEL_W(V_008F1C_SQ_SEL_1) |
83 S_008F1C_TYPE(V_008F1C_SQ_RSRC_IMG_1D)
84 /* the rest must contain zeros, which is also used by the buffer
85 * descriptor */
86 };
87
88 static uint32_t null_image_descriptor[8] = {
89 0,
90 0,
91 0,
92 S_008F1C_TYPE(V_008F1C_SQ_RSRC_IMG_1D)
93 /* the rest must contain zeros, which is also used by the buffer
94 * descriptor */
95 };
96
97 static void si_init_descriptors(struct si_descriptors *desc,
98 unsigned shader_userdata_index,
99 unsigned element_dw_size,
100 unsigned num_elements,
101 const uint32_t *null_descriptor,
102 unsigned *ce_offset)
103 {
104 int i;
105
106 assert(num_elements <= sizeof(desc->dirty_mask)*8);
107
108 desc->list = CALLOC(num_elements, element_dw_size * 4);
109 desc->element_dw_size = element_dw_size;
110 desc->num_elements = num_elements;
111 desc->dirty_mask = num_elements == 32 ? ~0u : (1u << num_elements) - 1;
112 desc->shader_userdata_offset = shader_userdata_index * 4;
113
114 if (ce_offset) {
115 desc->ce_offset = *ce_offset;
116
117 /* make sure that ce_offset stays 32 byte aligned */
118 *ce_offset += align(element_dw_size * num_elements * 4, 32);
119 }
120
121 /* Initialize the array to NULL descriptors if the element size is 8. */
122 if (null_descriptor) {
123 assert(element_dw_size % 8 == 0);
124 for (i = 0; i < num_elements * element_dw_size / 8; i++)
125 memcpy(desc->list + i * 8, null_descriptor,
126 8 * 4);
127 }
128 }
129
130 static void si_release_descriptors(struct si_descriptors *desc)
131 {
132 r600_resource_reference(&desc->buffer, NULL);
133 FREE(desc->list);
134 }
135
136 static bool si_ce_upload(struct si_context *sctx, unsigned ce_offset, unsigned size,
137 unsigned *out_offset, struct r600_resource **out_buf) {
138 uint64_t va;
139
140 u_suballocator_alloc(sctx->ce_suballocator, size, 64, out_offset,
141 (struct pipe_resource**)out_buf);
142 if (!out_buf)
143 return false;
144
145 va = (*out_buf)->gpu_address + *out_offset;
146
147 radeon_emit(sctx->ce_ib, PKT3(PKT3_DUMP_CONST_RAM, 3, 0));
148 radeon_emit(sctx->ce_ib, ce_offset);
149 radeon_emit(sctx->ce_ib, size / 4);
150 radeon_emit(sctx->ce_ib, va);
151 radeon_emit(sctx->ce_ib, va >> 32);
152
153 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx, *out_buf,
154 RADEON_USAGE_READWRITE, RADEON_PRIO_DESCRIPTORS);
155
156 sctx->ce_need_synchronization = true;
157 return true;
158 }
159
160 static void si_ce_reinitialize_descriptors(struct si_context *sctx,
161 struct si_descriptors *desc)
162 {
163 if (desc->buffer) {
164 struct r600_resource *buffer = (struct r600_resource*)desc->buffer;
165 unsigned list_size = desc->num_elements * desc->element_dw_size * 4;
166 uint64_t va = buffer->gpu_address + desc->buffer_offset;
167 struct radeon_winsys_cs *ib = sctx->ce_preamble_ib;
168
169 if (!ib)
170 ib = sctx->ce_ib;
171
172 list_size = align(list_size, 32);
173
174 radeon_emit(ib, PKT3(PKT3_LOAD_CONST_RAM, 3, 0));
175 radeon_emit(ib, va);
176 radeon_emit(ib, va >> 32);
177 radeon_emit(ib, list_size / 4);
178 radeon_emit(ib, desc->ce_offset);
179
180 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx, desc->buffer,
181 RADEON_USAGE_READ, RADEON_PRIO_DESCRIPTORS);
182 }
183 desc->ce_ram_dirty = false;
184 }
185
186 void si_ce_reinitialize_all_descriptors(struct si_context *sctx)
187 {
188 int i;
189
190 for (i = 0; i < SI_NUM_DESCS; ++i)
191 si_ce_reinitialize_descriptors(sctx, &sctx->descriptors[i]);
192 }
193
194 void si_ce_enable_loads(struct radeon_winsys_cs *ib)
195 {
196 radeon_emit(ib, PKT3(PKT3_CONTEXT_CONTROL, 1, 0));
197 radeon_emit(ib, CONTEXT_CONTROL_LOAD_ENABLE(1) |
198 CONTEXT_CONTROL_LOAD_CE_RAM(1));
199 radeon_emit(ib, CONTEXT_CONTROL_SHADOW_ENABLE(1));
200 }
201
202 static bool si_upload_descriptors(struct si_context *sctx,
203 struct si_descriptors *desc,
204 struct r600_atom * atom)
205 {
206 unsigned list_size = desc->num_elements * desc->element_dw_size * 4;
207
208 if (!desc->dirty_mask)
209 return true;
210
211 if (sctx->ce_ib) {
212 uint32_t const* list = (uint32_t const*)desc->list;
213
214 if (desc->ce_ram_dirty)
215 si_ce_reinitialize_descriptors(sctx, desc);
216
217 while(desc->dirty_mask) {
218 int begin, count;
219 u_bit_scan_consecutive_range(&desc->dirty_mask, &begin,
220 &count);
221
222 begin *= desc->element_dw_size;
223 count *= desc->element_dw_size;
224
225 radeon_emit(sctx->ce_ib,
226 PKT3(PKT3_WRITE_CONST_RAM, count, 0));
227 radeon_emit(sctx->ce_ib, desc->ce_offset + begin * 4);
228 radeon_emit_array(sctx->ce_ib, list + begin, count);
229 }
230
231 if (!si_ce_upload(sctx, desc->ce_offset, list_size,
232 &desc->buffer_offset, &desc->buffer))
233 return false;
234 } else {
235 void *ptr;
236
237 u_upload_alloc(sctx->b.b.stream_uploader, 0, list_size,
238 sctx->screen->b.info.tcc_cache_line_size,
239 &desc->buffer_offset,
240 (struct pipe_resource**)&desc->buffer, &ptr);
241 if (!desc->buffer)
242 return false; /* skip the draw call */
243
244 util_memcpy_cpu_to_le32(ptr, desc->list, list_size);
245 desc->gpu_list = ptr;
246
247 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx, desc->buffer,
248 RADEON_USAGE_READ, RADEON_PRIO_DESCRIPTORS);
249 }
250 desc->dirty_mask = 0;
251
252 if (atom)
253 si_mark_atom_dirty(sctx, atom);
254
255 return true;
256 }
257
258 static void
259 si_descriptors_begin_new_cs(struct si_context *sctx, struct si_descriptors *desc)
260 {
261 desc->ce_ram_dirty = true;
262
263 if (!desc->buffer)
264 return;
265
266 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx, desc->buffer,
267 RADEON_USAGE_READ, RADEON_PRIO_DESCRIPTORS);
268 }
269
270 /* SAMPLER VIEWS */
271
272 static unsigned
273 si_sampler_descriptors_idx(unsigned shader)
274 {
275 return SI_DESCS_FIRST_SHADER + shader * SI_NUM_SHADER_DESCS +
276 SI_SHADER_DESCS_SAMPLERS;
277 }
278
279 static struct si_descriptors *
280 si_sampler_descriptors(struct si_context *sctx, unsigned shader)
281 {
282 return &sctx->descriptors[si_sampler_descriptors_idx(shader)];
283 }
284
285 static void si_release_sampler_views(struct si_sampler_views *views)
286 {
287 int i;
288
289 for (i = 0; i < ARRAY_SIZE(views->views); i++) {
290 pipe_sampler_view_reference(&views->views[i], NULL);
291 }
292 }
293
294 static void si_sampler_view_add_buffer(struct si_context *sctx,
295 struct pipe_resource *resource,
296 enum radeon_bo_usage usage,
297 bool is_stencil_sampler,
298 bool check_mem)
299 {
300 struct r600_resource *rres;
301 struct r600_texture *rtex;
302 enum radeon_bo_priority priority;
303
304 if (!resource)
305 return;
306
307 if (resource->target != PIPE_BUFFER) {
308 struct r600_texture *tex = (struct r600_texture*)resource;
309
310 if (tex->is_depth && !r600_can_sample_zs(tex, is_stencil_sampler))
311 resource = &tex->flushed_depth_texture->resource.b.b;
312 }
313
314 rres = (struct r600_resource*)resource;
315 priority = r600_get_sampler_view_priority(rres);
316
317 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
318 rres, usage, priority,
319 check_mem);
320
321 if (resource->target == PIPE_BUFFER)
322 return;
323
324 /* Now add separate DCC or HTILE. */
325 rtex = (struct r600_texture*)resource;
326 if (rtex->dcc_separate_buffer) {
327 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
328 rtex->dcc_separate_buffer, usage,
329 RADEON_PRIO_DCC, check_mem);
330 }
331
332 if (rtex->htile_buffer &&
333 rtex->tc_compatible_htile &&
334 !is_stencil_sampler) {
335 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
336 rtex->htile_buffer, usage,
337 RADEON_PRIO_HTILE, check_mem);
338 }
339 }
340
341 static void si_sampler_views_begin_new_cs(struct si_context *sctx,
342 struct si_sampler_views *views)
343 {
344 unsigned mask = views->enabled_mask;
345
346 /* Add buffers to the CS. */
347 while (mask) {
348 int i = u_bit_scan(&mask);
349 struct si_sampler_view *sview = (struct si_sampler_view *)views->views[i];
350
351 si_sampler_view_add_buffer(sctx, sview->base.texture,
352 RADEON_USAGE_READ,
353 sview->is_stencil_sampler, false);
354 }
355 }
356
357 /* Set buffer descriptor fields that can be changed by reallocations. */
358 static void si_set_buf_desc_address(struct r600_resource *buf,
359 uint64_t offset, uint32_t *state)
360 {
361 uint64_t va = buf->gpu_address + offset;
362
363 state[0] = va;
364 state[1] &= C_008F04_BASE_ADDRESS_HI;
365 state[1] |= S_008F04_BASE_ADDRESS_HI(va >> 32);
366 }
367
368 /* Set texture descriptor fields that can be changed by reallocations.
369 *
370 * \param tex texture
371 * \param base_level_info information of the level of BASE_ADDRESS
372 * \param base_level the level of BASE_ADDRESS
373 * \param first_level pipe_sampler_view.u.tex.first_level
374 * \param block_width util_format_get_blockwidth()
375 * \param is_stencil select between separate Z & Stencil
376 * \param state descriptor to update
377 */
378 void si_set_mutable_tex_desc_fields(struct r600_texture *tex,
379 const struct radeon_surf_level *base_level_info,
380 unsigned base_level, unsigned first_level,
381 unsigned block_width, bool is_stencil,
382 uint32_t *state)
383 {
384 uint64_t va;
385 unsigned pitch = base_level_info->nblk_x * block_width;
386
387 if (tex->is_depth && !r600_can_sample_zs(tex, is_stencil)) {
388 tex = tex->flushed_depth_texture;
389 is_stencil = false;
390 }
391
392 va = tex->resource.gpu_address + base_level_info->offset;
393
394 state[1] &= C_008F14_BASE_ADDRESS_HI;
395 state[3] &= C_008F1C_TILING_INDEX;
396 state[4] &= C_008F20_PITCH;
397 state[6] &= C_008F28_COMPRESSION_EN;
398
399 state[0] = va >> 8;
400 state[1] |= S_008F14_BASE_ADDRESS_HI(va >> 40);
401 state[3] |= S_008F1C_TILING_INDEX(si_tile_mode_index(tex, base_level,
402 is_stencil));
403 state[4] |= S_008F20_PITCH(pitch - 1);
404
405 if (tex->dcc_offset && first_level < tex->surface.num_dcc_levels) {
406 state[6] |= S_008F28_COMPRESSION_EN(1);
407 state[7] = ((!tex->dcc_separate_buffer ? tex->resource.gpu_address : 0) +
408 tex->dcc_offset +
409 base_level_info->dcc_offset) >> 8;
410 } else if (tex->tc_compatible_htile) {
411 state[6] |= S_008F28_COMPRESSION_EN(1);
412 state[7] = tex->htile_buffer->gpu_address >> 8;
413 }
414 }
415
416 static void si_set_sampler_view(struct si_context *sctx,
417 unsigned shader,
418 unsigned slot, struct pipe_sampler_view *view,
419 bool disallow_early_out)
420 {
421 struct si_sampler_views *views = &sctx->samplers[shader].views;
422 struct si_sampler_view *rview = (struct si_sampler_view*)view;
423 struct si_descriptors *descs = si_sampler_descriptors(sctx, shader);
424 uint32_t *desc = descs->list + slot * 16;
425
426 if (views->views[slot] == view && !disallow_early_out)
427 return;
428
429 if (view) {
430 struct r600_texture *rtex = (struct r600_texture *)view->texture;
431
432 assert(rtex); /* views with texture == NULL aren't supported */
433 pipe_sampler_view_reference(&views->views[slot], view);
434 memcpy(desc, rview->state, 8*4);
435
436 if (rtex->resource.b.b.target == PIPE_BUFFER) {
437 rtex->resource.bind_history |= PIPE_BIND_SAMPLER_VIEW;
438
439 si_set_buf_desc_address(&rtex->resource,
440 view->u.buf.offset,
441 desc + 4);
442 } else {
443 bool is_separate_stencil =
444 rtex->db_compatible &&
445 rview->is_stencil_sampler;
446
447 si_set_mutable_tex_desc_fields(rtex,
448 rview->base_level_info,
449 rview->base_level,
450 rview->base.u.tex.first_level,
451 rview->block_width,
452 is_separate_stencil,
453 desc);
454 }
455
456 if (rtex->resource.b.b.target != PIPE_BUFFER &&
457 rtex->fmask.size) {
458 memcpy(desc + 8,
459 rview->fmask_state, 8*4);
460 } else {
461 /* Disable FMASK and bind sampler state in [12:15]. */
462 memcpy(desc + 8,
463 null_texture_descriptor, 4*4);
464
465 if (views->sampler_states[slot])
466 memcpy(desc + 12,
467 views->sampler_states[slot]->val, 4*4);
468 }
469
470 views->enabled_mask |= 1u << slot;
471
472 /* Since this can flush, it must be done after enabled_mask is
473 * updated. */
474 si_sampler_view_add_buffer(sctx, view->texture,
475 RADEON_USAGE_READ,
476 rview->is_stencil_sampler, true);
477 } else {
478 pipe_sampler_view_reference(&views->views[slot], NULL);
479 memcpy(desc, null_texture_descriptor, 8*4);
480 /* Only clear the lower dwords of FMASK. */
481 memcpy(desc + 8, null_texture_descriptor, 4*4);
482 /* Re-set the sampler state if we are transitioning from FMASK. */
483 if (views->sampler_states[slot])
484 memcpy(desc + 12,
485 views->sampler_states[slot]->val, 4*4);
486
487 views->enabled_mask &= ~(1u << slot);
488 }
489
490 descs->dirty_mask |= 1u << slot;
491 sctx->descriptors_dirty |= 1u << si_sampler_descriptors_idx(shader);
492 }
493
494 static bool is_compressed_colortex(struct r600_texture *rtex)
495 {
496 return rtex->cmask.size || rtex->fmask.size ||
497 (rtex->dcc_offset && rtex->dirty_level_mask);
498 }
499
500 static void si_update_compressed_tex_shader_mask(struct si_context *sctx,
501 unsigned shader)
502 {
503 struct si_textures_info *samplers = &sctx->samplers[shader];
504 unsigned shader_bit = 1 << shader;
505
506 if (samplers->depth_texture_mask ||
507 samplers->compressed_colortex_mask ||
508 sctx->images[shader].compressed_colortex_mask)
509 sctx->compressed_tex_shader_mask |= shader_bit;
510 else
511 sctx->compressed_tex_shader_mask &= ~shader_bit;
512 }
513
514 static void si_set_sampler_views(struct pipe_context *ctx,
515 enum pipe_shader_type shader, unsigned start,
516 unsigned count,
517 struct pipe_sampler_view **views)
518 {
519 struct si_context *sctx = (struct si_context *)ctx;
520 struct si_textures_info *samplers = &sctx->samplers[shader];
521 int i;
522
523 if (!count || shader >= SI_NUM_SHADERS)
524 return;
525
526 for (i = 0; i < count; i++) {
527 unsigned slot = start + i;
528
529 if (!views || !views[i]) {
530 samplers->depth_texture_mask &= ~(1u << slot);
531 samplers->compressed_colortex_mask &= ~(1u << slot);
532 si_set_sampler_view(sctx, shader, slot, NULL, false);
533 continue;
534 }
535
536 si_set_sampler_view(sctx, shader, slot, views[i], false);
537
538 if (views[i]->texture && views[i]->texture->target != PIPE_BUFFER) {
539 struct r600_texture *rtex =
540 (struct r600_texture*)views[i]->texture;
541 struct si_sampler_view *rview = (struct si_sampler_view *)views[i];
542
543 if (rtex->db_compatible &&
544 (!rtex->tc_compatible_htile || rview->is_stencil_sampler)) {
545 samplers->depth_texture_mask |= 1u << slot;
546 } else {
547 samplers->depth_texture_mask &= ~(1u << slot);
548 }
549 if (is_compressed_colortex(rtex)) {
550 samplers->compressed_colortex_mask |= 1u << slot;
551 } else {
552 samplers->compressed_colortex_mask &= ~(1u << slot);
553 }
554
555 if (rtex->dcc_offset &&
556 p_atomic_read(&rtex->framebuffers_bound))
557 sctx->need_check_render_feedback = true;
558 } else {
559 samplers->depth_texture_mask &= ~(1u << slot);
560 samplers->compressed_colortex_mask &= ~(1u << slot);
561 }
562 }
563
564 si_update_compressed_tex_shader_mask(sctx, shader);
565 }
566
567 static void
568 si_samplers_update_compressed_colortex_mask(struct si_textures_info *samplers)
569 {
570 unsigned mask = samplers->views.enabled_mask;
571
572 while (mask) {
573 int i = u_bit_scan(&mask);
574 struct pipe_resource *res = samplers->views.views[i]->texture;
575
576 if (res && res->target != PIPE_BUFFER) {
577 struct r600_texture *rtex = (struct r600_texture *)res;
578
579 if (is_compressed_colortex(rtex)) {
580 samplers->compressed_colortex_mask |= 1u << i;
581 } else {
582 samplers->compressed_colortex_mask &= ~(1u << i);
583 }
584 }
585 }
586 }
587
588 /* IMAGE VIEWS */
589
590 static unsigned
591 si_image_descriptors_idx(unsigned shader)
592 {
593 return SI_DESCS_FIRST_SHADER + shader * SI_NUM_SHADER_DESCS +
594 SI_SHADER_DESCS_IMAGES;
595 }
596
597 static struct si_descriptors*
598 si_image_descriptors(struct si_context *sctx, unsigned shader)
599 {
600 return &sctx->descriptors[si_image_descriptors_idx(shader)];
601 }
602
603 static void
604 si_release_image_views(struct si_images_info *images)
605 {
606 unsigned i;
607
608 for (i = 0; i < SI_NUM_IMAGES; ++i) {
609 struct pipe_image_view *view = &images->views[i];
610
611 pipe_resource_reference(&view->resource, NULL);
612 }
613 }
614
615 static void
616 si_image_views_begin_new_cs(struct si_context *sctx, struct si_images_info *images)
617 {
618 uint mask = images->enabled_mask;
619
620 /* Add buffers to the CS. */
621 while (mask) {
622 int i = u_bit_scan(&mask);
623 struct pipe_image_view *view = &images->views[i];
624
625 assert(view->resource);
626
627 si_sampler_view_add_buffer(sctx, view->resource,
628 RADEON_USAGE_READWRITE, false, false);
629 }
630 }
631
632 static void
633 si_disable_shader_image(struct si_context *ctx, unsigned shader, unsigned slot)
634 {
635 struct si_images_info *images = &ctx->images[shader];
636
637 if (images->enabled_mask & (1u << slot)) {
638 struct si_descriptors *descs = si_image_descriptors(ctx, shader);
639
640 pipe_resource_reference(&images->views[slot].resource, NULL);
641 images->compressed_colortex_mask &= ~(1 << slot);
642
643 memcpy(descs->list + slot*8, null_image_descriptor, 8*4);
644 images->enabled_mask &= ~(1u << slot);
645 descs->dirty_mask |= 1u << slot;
646 ctx->descriptors_dirty |= 1u << si_image_descriptors_idx(shader);
647 }
648 }
649
650 static void
651 si_mark_image_range_valid(const struct pipe_image_view *view)
652 {
653 struct r600_resource *res = (struct r600_resource *)view->resource;
654
655 assert(res && res->b.b.target == PIPE_BUFFER);
656
657 util_range_add(&res->valid_buffer_range,
658 view->u.buf.offset,
659 view->u.buf.offset + view->u.buf.size);
660 }
661
662 static void si_set_shader_image(struct si_context *ctx,
663 unsigned shader,
664 unsigned slot, const struct pipe_image_view *view,
665 bool skip_decompress)
666 {
667 struct si_screen *screen = ctx->screen;
668 struct si_images_info *images = &ctx->images[shader];
669 struct si_descriptors *descs = si_image_descriptors(ctx, shader);
670 struct r600_resource *res;
671 uint32_t *desc = descs->list + slot * 8;
672
673 if (!view || !view->resource) {
674 si_disable_shader_image(ctx, shader, slot);
675 return;
676 }
677
678 res = (struct r600_resource *)view->resource;
679
680 if (&images->views[slot] != view)
681 util_copy_image_view(&images->views[slot], view);
682
683 if (res->b.b.target == PIPE_BUFFER) {
684 if (view->access & PIPE_IMAGE_ACCESS_WRITE)
685 si_mark_image_range_valid(view);
686
687 si_make_buffer_descriptor(screen, res,
688 view->format,
689 view->u.buf.offset,
690 view->u.buf.size,
691 descs->list + slot * 8);
692 si_set_buf_desc_address(res, view->u.buf.offset, desc + 4);
693
694 images->compressed_colortex_mask &= ~(1 << slot);
695 res->bind_history |= PIPE_BIND_SHADER_IMAGE;
696 } else {
697 static const unsigned char swizzle[4] = { 0, 1, 2, 3 };
698 struct r600_texture *tex = (struct r600_texture *)res;
699 unsigned level = view->u.tex.level;
700 unsigned width, height, depth;
701 bool uses_dcc = tex->dcc_offset &&
702 level < tex->surface.num_dcc_levels;
703
704 assert(!tex->is_depth);
705 assert(tex->fmask.size == 0);
706
707 if (uses_dcc && !skip_decompress &&
708 (view->access & PIPE_IMAGE_ACCESS_WRITE ||
709 !vi_dcc_formats_compatible(res->b.b.format, view->format))) {
710 /* If DCC can't be disabled, at least decompress it.
711 * The decompression is relatively cheap if the surface
712 * has been decompressed already.
713 */
714 if (r600_texture_disable_dcc(&ctx->b, tex))
715 uses_dcc = false;
716 else
717 ctx->b.decompress_dcc(&ctx->b.b, tex);
718 }
719
720 if (is_compressed_colortex(tex)) {
721 images->compressed_colortex_mask |= 1 << slot;
722 } else {
723 images->compressed_colortex_mask &= ~(1 << slot);
724 }
725
726 if (uses_dcc &&
727 p_atomic_read(&tex->framebuffers_bound))
728 ctx->need_check_render_feedback = true;
729
730 /* Always force the base level to the selected level.
731 *
732 * This is required for 3D textures, where otherwise
733 * selecting a single slice for non-layered bindings
734 * fails. It doesn't hurt the other targets.
735 */
736 width = u_minify(res->b.b.width0, level);
737 height = u_minify(res->b.b.height0, level);
738 depth = u_minify(res->b.b.depth0, level);
739
740 si_make_texture_descriptor(screen, tex,
741 false, res->b.b.target,
742 view->format, swizzle,
743 0, 0,
744 view->u.tex.first_layer,
745 view->u.tex.last_layer,
746 width, height, depth,
747 desc, NULL);
748 si_set_mutable_tex_desc_fields(tex, &tex->surface.level[level],
749 level, level,
750 util_format_get_blockwidth(view->format),
751 false, desc);
752 }
753
754 images->enabled_mask |= 1u << slot;
755 descs->dirty_mask |= 1u << slot;
756 ctx->descriptors_dirty |= 1u << si_image_descriptors_idx(shader);
757
758 /* Since this can flush, it must be done after enabled_mask is updated. */
759 si_sampler_view_add_buffer(ctx, &res->b.b,
760 RADEON_USAGE_READWRITE, false, true);
761 }
762
763 static void
764 si_set_shader_images(struct pipe_context *pipe,
765 enum pipe_shader_type shader,
766 unsigned start_slot, unsigned count,
767 const struct pipe_image_view *views)
768 {
769 struct si_context *ctx = (struct si_context *)pipe;
770 unsigned i, slot;
771
772 assert(shader < SI_NUM_SHADERS);
773
774 if (!count)
775 return;
776
777 assert(start_slot + count <= SI_NUM_IMAGES);
778
779 if (views) {
780 for (i = 0, slot = start_slot; i < count; ++i, ++slot)
781 si_set_shader_image(ctx, shader, slot, &views[i], false);
782 } else {
783 for (i = 0, slot = start_slot; i < count; ++i, ++slot)
784 si_set_shader_image(ctx, shader, slot, NULL, false);
785 }
786
787 si_update_compressed_tex_shader_mask(ctx, shader);
788 }
789
790 static void
791 si_images_update_compressed_colortex_mask(struct si_images_info *images)
792 {
793 unsigned mask = images->enabled_mask;
794
795 while (mask) {
796 int i = u_bit_scan(&mask);
797 struct pipe_resource *res = images->views[i].resource;
798
799 if (res && res->target != PIPE_BUFFER) {
800 struct r600_texture *rtex = (struct r600_texture *)res;
801
802 if (is_compressed_colortex(rtex)) {
803 images->compressed_colortex_mask |= 1 << i;
804 } else {
805 images->compressed_colortex_mask &= ~(1 << i);
806 }
807 }
808 }
809 }
810
811 /* SAMPLER STATES */
812
813 static void si_bind_sampler_states(struct pipe_context *ctx,
814 enum pipe_shader_type shader,
815 unsigned start, unsigned count, void **states)
816 {
817 struct si_context *sctx = (struct si_context *)ctx;
818 struct si_textures_info *samplers = &sctx->samplers[shader];
819 struct si_descriptors *desc = si_sampler_descriptors(sctx, shader);
820 struct si_sampler_state **sstates = (struct si_sampler_state**)states;
821 int i;
822
823 if (!count || shader >= SI_NUM_SHADERS)
824 return;
825
826 for (i = 0; i < count; i++) {
827 unsigned slot = start + i;
828
829 if (!sstates[i] ||
830 sstates[i] == samplers->views.sampler_states[slot])
831 continue;
832
833 #ifdef DEBUG
834 assert(sstates[i]->magic == SI_SAMPLER_STATE_MAGIC);
835 #endif
836 samplers->views.sampler_states[slot] = sstates[i];
837
838 /* If FMASK is bound, don't overwrite it.
839 * The sampler state will be set after FMASK is unbound.
840 */
841 if (samplers->views.views[slot] &&
842 samplers->views.views[slot]->texture &&
843 samplers->views.views[slot]->texture->target != PIPE_BUFFER &&
844 ((struct r600_texture*)samplers->views.views[slot]->texture)->fmask.size)
845 continue;
846
847 memcpy(desc->list + slot * 16 + 12, sstates[i]->val, 4*4);
848 desc->dirty_mask |= 1u << slot;
849 sctx->descriptors_dirty |= 1u << si_sampler_descriptors_idx(shader);
850 }
851 }
852
853 /* BUFFER RESOURCES */
854
855 static void si_init_buffer_resources(struct si_buffer_resources *buffers,
856 struct si_descriptors *descs,
857 unsigned num_buffers,
858 unsigned shader_userdata_index,
859 enum radeon_bo_usage shader_usage,
860 enum radeon_bo_priority priority,
861 unsigned *ce_offset)
862 {
863 buffers->shader_usage = shader_usage;
864 buffers->priority = priority;
865 buffers->buffers = CALLOC(num_buffers, sizeof(struct pipe_resource*));
866
867 si_init_descriptors(descs, shader_userdata_index, 4,
868 num_buffers, NULL, ce_offset);
869 }
870
871 static void si_release_buffer_resources(struct si_buffer_resources *buffers,
872 struct si_descriptors *descs)
873 {
874 int i;
875
876 for (i = 0; i < descs->num_elements; i++) {
877 pipe_resource_reference(&buffers->buffers[i], NULL);
878 }
879
880 FREE(buffers->buffers);
881 }
882
883 static void si_buffer_resources_begin_new_cs(struct si_context *sctx,
884 struct si_buffer_resources *buffers)
885 {
886 unsigned mask = buffers->enabled_mask;
887
888 /* Add buffers to the CS. */
889 while (mask) {
890 int i = u_bit_scan(&mask);
891
892 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
893 (struct r600_resource*)buffers->buffers[i],
894 buffers->shader_usage, buffers->priority);
895 }
896 }
897
898 static void si_get_buffer_from_descriptors(struct si_buffer_resources *buffers,
899 struct si_descriptors *descs,
900 unsigned idx, struct pipe_resource **buf,
901 unsigned *offset, unsigned *size)
902 {
903 pipe_resource_reference(buf, buffers->buffers[idx]);
904 if (*buf) {
905 struct r600_resource *res = r600_resource(*buf);
906 const uint32_t *desc = descs->list + idx * 4;
907 uint64_t va;
908
909 *size = desc[2];
910
911 assert(G_008F04_STRIDE(desc[1]) == 0);
912 va = ((uint64_t)desc[1] << 32) | desc[0];
913
914 assert(va >= res->gpu_address && va + *size <= res->gpu_address + res->bo_size);
915 *offset = va - res->gpu_address;
916 }
917 }
918
919 /* VERTEX BUFFERS */
920
921 static void si_vertex_buffers_begin_new_cs(struct si_context *sctx)
922 {
923 struct si_descriptors *desc = &sctx->vertex_buffers;
924 int count = sctx->vertex_elements ? sctx->vertex_elements->count : 0;
925 int i;
926
927 for (i = 0; i < count; i++) {
928 int vb = sctx->vertex_elements->elements[i].vertex_buffer_index;
929
930 if (vb >= ARRAY_SIZE(sctx->vertex_buffer))
931 continue;
932 if (!sctx->vertex_buffer[vb].buffer)
933 continue;
934
935 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
936 (struct r600_resource*)sctx->vertex_buffer[vb].buffer,
937 RADEON_USAGE_READ, RADEON_PRIO_VERTEX_BUFFER);
938 }
939
940 if (!desc->buffer)
941 return;
942 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
943 desc->buffer, RADEON_USAGE_READ,
944 RADEON_PRIO_DESCRIPTORS);
945 }
946
947 bool si_upload_vertex_buffer_descriptors(struct si_context *sctx)
948 {
949 struct si_vertex_element *velems = sctx->vertex_elements;
950 struct si_descriptors *desc = &sctx->vertex_buffers;
951 unsigned i, count = velems->count;
952 unsigned desc_list_byte_size = velems->desc_list_byte_size;
953 uint64_t va;
954 uint32_t *ptr;
955
956 if (!sctx->vertex_buffers_dirty || !count || !velems)
957 return true;
958
959 unsigned first_vb_use_mask = velems->first_vb_use_mask;
960
961 /* Vertex buffer descriptors are the only ones which are uploaded
962 * directly through a staging buffer and don't go through
963 * the fine-grained upload path.
964 */
965 u_upload_alloc(sctx->b.b.stream_uploader, 0,
966 desc_list_byte_size,
967 si_optimal_tcc_alignment(sctx, desc_list_byte_size),
968 &desc->buffer_offset,
969 (struct pipe_resource**)&desc->buffer, (void**)&ptr);
970 if (!desc->buffer)
971 return false;
972
973 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
974 desc->buffer, RADEON_USAGE_READ,
975 RADEON_PRIO_DESCRIPTORS);
976
977 assert(count <= SI_MAX_ATTRIBS);
978
979 for (i = 0; i < count; i++) {
980 struct pipe_vertex_element *ve = &velems->elements[i];
981 struct pipe_vertex_buffer *vb;
982 struct r600_resource *rbuffer;
983 unsigned offset;
984 unsigned vbo_index = ve->vertex_buffer_index;
985 uint32_t *desc = &ptr[i*4];
986
987 vb = &sctx->vertex_buffer[vbo_index];
988 rbuffer = (struct r600_resource*)vb->buffer;
989 if (!rbuffer) {
990 memset(desc, 0, 16);
991 continue;
992 }
993
994 offset = vb->buffer_offset + ve->src_offset;
995 va = rbuffer->gpu_address + offset;
996
997 /* Fill in T# buffer resource description */
998 desc[0] = va;
999 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
1000 S_008F04_STRIDE(vb->stride);
1001
1002 if (sctx->b.chip_class <= CIK && vb->stride) {
1003 /* Round up by rounding down and adding 1 */
1004 desc[2] = (vb->buffer->width0 - offset -
1005 velems->format_size[i]) /
1006 vb->stride + 1;
1007 } else {
1008 desc[2] = vb->buffer->width0 - offset;
1009 }
1010
1011 desc[3] = velems->rsrc_word3[i];
1012
1013 if (first_vb_use_mask & (1 << i)) {
1014 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
1015 (struct r600_resource*)vb->buffer,
1016 RADEON_USAGE_READ, RADEON_PRIO_VERTEX_BUFFER);
1017 }
1018 }
1019
1020 /* Don't flush the const cache. It would have a very negative effect
1021 * on performance (confirmed by testing). New descriptors are always
1022 * uploaded to a fresh new buffer, so I don't think flushing the const
1023 * cache is needed. */
1024 si_mark_atom_dirty(sctx, &sctx->shader_userdata.atom);
1025 if (sctx->b.chip_class >= CIK)
1026 si_mark_atom_dirty(sctx, &sctx->prefetch_L2);
1027 sctx->vertex_buffers_dirty = false;
1028 sctx->vertex_buffer_pointer_dirty = true;
1029 return true;
1030 }
1031
1032
1033 /* CONSTANT BUFFERS */
1034
1035 static unsigned
1036 si_const_buffer_descriptors_idx(unsigned shader)
1037 {
1038 return SI_DESCS_FIRST_SHADER + shader * SI_NUM_SHADER_DESCS +
1039 SI_SHADER_DESCS_CONST_BUFFERS;
1040 }
1041
1042 static struct si_descriptors *
1043 si_const_buffer_descriptors(struct si_context *sctx, unsigned shader)
1044 {
1045 return &sctx->descriptors[si_const_buffer_descriptors_idx(shader)];
1046 }
1047
1048 void si_upload_const_buffer(struct si_context *sctx, struct r600_resource **rbuffer,
1049 const uint8_t *ptr, unsigned size, uint32_t *const_offset)
1050 {
1051 void *tmp;
1052
1053 u_upload_alloc(sctx->b.b.stream_uploader, 0, size,
1054 si_optimal_tcc_alignment(sctx, size),
1055 const_offset,
1056 (struct pipe_resource**)rbuffer, &tmp);
1057 if (*rbuffer)
1058 util_memcpy_cpu_to_le32(tmp, ptr, size);
1059 }
1060
1061 static void si_set_constant_buffer(struct si_context *sctx,
1062 struct si_buffer_resources *buffers,
1063 unsigned descriptors_idx,
1064 uint slot, const struct pipe_constant_buffer *input)
1065 {
1066 struct si_descriptors *descs = &sctx->descriptors[descriptors_idx];
1067 assert(slot < descs->num_elements);
1068 pipe_resource_reference(&buffers->buffers[slot], NULL);
1069
1070 /* CIK cannot unbind a constant buffer (S_BUFFER_LOAD is buggy
1071 * with a NULL buffer). We need to use a dummy buffer instead. */
1072 if (sctx->b.chip_class == CIK &&
1073 (!input || (!input->buffer && !input->user_buffer)))
1074 input = &sctx->null_const_buf;
1075
1076 if (input && (input->buffer || input->user_buffer)) {
1077 struct pipe_resource *buffer = NULL;
1078 uint64_t va;
1079
1080 /* Upload the user buffer if needed. */
1081 if (input->user_buffer) {
1082 unsigned buffer_offset;
1083
1084 si_upload_const_buffer(sctx,
1085 (struct r600_resource**)&buffer, input->user_buffer,
1086 input->buffer_size, &buffer_offset);
1087 if (!buffer) {
1088 /* Just unbind on failure. */
1089 si_set_constant_buffer(sctx, buffers, descriptors_idx, slot, NULL);
1090 return;
1091 }
1092 va = r600_resource(buffer)->gpu_address + buffer_offset;
1093 } else {
1094 pipe_resource_reference(&buffer, input->buffer);
1095 va = r600_resource(buffer)->gpu_address + input->buffer_offset;
1096 /* Only track usage for non-user buffers. */
1097 r600_resource(buffer)->bind_history |= PIPE_BIND_CONSTANT_BUFFER;
1098 }
1099
1100 /* Set the descriptor. */
1101 uint32_t *desc = descs->list + slot*4;
1102 desc[0] = va;
1103 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
1104 S_008F04_STRIDE(0);
1105 desc[2] = input->buffer_size;
1106 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1107 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1108 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1109 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1110 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1111 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
1112
1113 buffers->buffers[slot] = buffer;
1114 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
1115 (struct r600_resource*)buffer,
1116 buffers->shader_usage,
1117 buffers->priority, true);
1118 buffers->enabled_mask |= 1u << slot;
1119 } else {
1120 /* Clear the descriptor. */
1121 memset(descs->list + slot*4, 0, sizeof(uint32_t) * 4);
1122 buffers->enabled_mask &= ~(1u << slot);
1123 }
1124
1125 descs->dirty_mask |= 1u << slot;
1126 sctx->descriptors_dirty |= 1u << descriptors_idx;
1127 }
1128
1129 void si_set_rw_buffer(struct si_context *sctx,
1130 uint slot, const struct pipe_constant_buffer *input)
1131 {
1132 si_set_constant_buffer(sctx, &sctx->rw_buffers,
1133 SI_DESCS_RW_BUFFERS, slot, input);
1134 }
1135
1136 static void si_pipe_set_constant_buffer(struct pipe_context *ctx,
1137 uint shader, uint slot,
1138 const struct pipe_constant_buffer *input)
1139 {
1140 struct si_context *sctx = (struct si_context *)ctx;
1141
1142 if (shader >= SI_NUM_SHADERS)
1143 return;
1144
1145 si_set_constant_buffer(sctx, &sctx->const_buffers[shader],
1146 si_const_buffer_descriptors_idx(shader),
1147 slot, input);
1148 }
1149
1150 void si_get_pipe_constant_buffer(struct si_context *sctx, uint shader,
1151 uint slot, struct pipe_constant_buffer *cbuf)
1152 {
1153 cbuf->user_buffer = NULL;
1154 si_get_buffer_from_descriptors(
1155 &sctx->const_buffers[shader],
1156 si_const_buffer_descriptors(sctx, shader),
1157 slot, &cbuf->buffer, &cbuf->buffer_offset, &cbuf->buffer_size);
1158 }
1159
1160 /* SHADER BUFFERS */
1161
1162 static unsigned
1163 si_shader_buffer_descriptors_idx(enum pipe_shader_type shader)
1164 {
1165 return SI_DESCS_FIRST_SHADER + shader * SI_NUM_SHADER_DESCS +
1166 SI_SHADER_DESCS_SHADER_BUFFERS;
1167 }
1168
1169 static struct si_descriptors *
1170 si_shader_buffer_descriptors(struct si_context *sctx,
1171 enum pipe_shader_type shader)
1172 {
1173 return &sctx->descriptors[si_shader_buffer_descriptors_idx(shader)];
1174 }
1175
1176 static void si_set_shader_buffers(struct pipe_context *ctx,
1177 enum pipe_shader_type shader,
1178 unsigned start_slot, unsigned count,
1179 const struct pipe_shader_buffer *sbuffers)
1180 {
1181 struct si_context *sctx = (struct si_context *)ctx;
1182 struct si_buffer_resources *buffers = &sctx->shader_buffers[shader];
1183 struct si_descriptors *descs = si_shader_buffer_descriptors(sctx, shader);
1184 unsigned i;
1185
1186 assert(start_slot + count <= SI_NUM_SHADER_BUFFERS);
1187
1188 for (i = 0; i < count; ++i) {
1189 const struct pipe_shader_buffer *sbuffer = sbuffers ? &sbuffers[i] : NULL;
1190 struct r600_resource *buf;
1191 unsigned slot = start_slot + i;
1192 uint32_t *desc = descs->list + slot * 4;
1193 uint64_t va;
1194
1195 if (!sbuffer || !sbuffer->buffer) {
1196 pipe_resource_reference(&buffers->buffers[slot], NULL);
1197 memset(desc, 0, sizeof(uint32_t) * 4);
1198 buffers->enabled_mask &= ~(1u << slot);
1199 descs->dirty_mask |= 1u << slot;
1200 sctx->descriptors_dirty |=
1201 1u << si_shader_buffer_descriptors_idx(shader);
1202 continue;
1203 }
1204
1205 buf = (struct r600_resource *)sbuffer->buffer;
1206 va = buf->gpu_address + sbuffer->buffer_offset;
1207
1208 desc[0] = va;
1209 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
1210 S_008F04_STRIDE(0);
1211 desc[2] = sbuffer->buffer_size;
1212 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1213 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1214 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1215 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1216 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1217 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
1218
1219 pipe_resource_reference(&buffers->buffers[slot], &buf->b.b);
1220 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx, buf,
1221 buffers->shader_usage,
1222 buffers->priority, true);
1223 buf->bind_history |= PIPE_BIND_SHADER_BUFFER;
1224
1225 buffers->enabled_mask |= 1u << slot;
1226 descs->dirty_mask |= 1u << slot;
1227 sctx->descriptors_dirty |=
1228 1u << si_shader_buffer_descriptors_idx(shader);
1229 }
1230 }
1231
1232 void si_get_shader_buffers(struct si_context *sctx, uint shader,
1233 uint start_slot, uint count,
1234 struct pipe_shader_buffer *sbuf)
1235 {
1236 struct si_buffer_resources *buffers = &sctx->shader_buffers[shader];
1237 struct si_descriptors *descs = si_shader_buffer_descriptors(sctx, shader);
1238
1239 for (unsigned i = 0; i < count; ++i) {
1240 si_get_buffer_from_descriptors(
1241 buffers, descs, start_slot + i,
1242 &sbuf[i].buffer, &sbuf[i].buffer_offset,
1243 &sbuf[i].buffer_size);
1244 }
1245 }
1246
1247 /* RING BUFFERS */
1248
1249 void si_set_ring_buffer(struct pipe_context *ctx, uint slot,
1250 struct pipe_resource *buffer,
1251 unsigned stride, unsigned num_records,
1252 bool add_tid, bool swizzle,
1253 unsigned element_size, unsigned index_stride, uint64_t offset)
1254 {
1255 struct si_context *sctx = (struct si_context *)ctx;
1256 struct si_buffer_resources *buffers = &sctx->rw_buffers;
1257 struct si_descriptors *descs = &sctx->descriptors[SI_DESCS_RW_BUFFERS];
1258
1259 /* The stride field in the resource descriptor has 14 bits */
1260 assert(stride < (1 << 14));
1261
1262 assert(slot < descs->num_elements);
1263 pipe_resource_reference(&buffers->buffers[slot], NULL);
1264
1265 if (buffer) {
1266 uint64_t va;
1267
1268 va = r600_resource(buffer)->gpu_address + offset;
1269
1270 switch (element_size) {
1271 default:
1272 assert(!"Unsupported ring buffer element size");
1273 case 0:
1274 case 2:
1275 element_size = 0;
1276 break;
1277 case 4:
1278 element_size = 1;
1279 break;
1280 case 8:
1281 element_size = 2;
1282 break;
1283 case 16:
1284 element_size = 3;
1285 break;
1286 }
1287
1288 switch (index_stride) {
1289 default:
1290 assert(!"Unsupported ring buffer index stride");
1291 case 0:
1292 case 8:
1293 index_stride = 0;
1294 break;
1295 case 16:
1296 index_stride = 1;
1297 break;
1298 case 32:
1299 index_stride = 2;
1300 break;
1301 case 64:
1302 index_stride = 3;
1303 break;
1304 }
1305
1306 if (sctx->b.chip_class >= VI && stride)
1307 num_records *= stride;
1308
1309 /* Set the descriptor. */
1310 uint32_t *desc = descs->list + slot*4;
1311 desc[0] = va;
1312 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
1313 S_008F04_STRIDE(stride) |
1314 S_008F04_SWIZZLE_ENABLE(swizzle);
1315 desc[2] = num_records;
1316 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1317 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1318 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1319 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1320 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1321 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) |
1322 S_008F0C_ELEMENT_SIZE(element_size) |
1323 S_008F0C_INDEX_STRIDE(index_stride) |
1324 S_008F0C_ADD_TID_ENABLE(add_tid);
1325
1326 pipe_resource_reference(&buffers->buffers[slot], buffer);
1327 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
1328 (struct r600_resource*)buffer,
1329 buffers->shader_usage, buffers->priority);
1330 buffers->enabled_mask |= 1u << slot;
1331 } else {
1332 /* Clear the descriptor. */
1333 memset(descs->list + slot*4, 0, sizeof(uint32_t) * 4);
1334 buffers->enabled_mask &= ~(1u << slot);
1335 }
1336
1337 descs->dirty_mask |= 1u << slot;
1338 sctx->descriptors_dirty |= 1u << SI_DESCS_RW_BUFFERS;
1339 }
1340
1341 /* STREAMOUT BUFFERS */
1342
1343 static void si_set_streamout_targets(struct pipe_context *ctx,
1344 unsigned num_targets,
1345 struct pipe_stream_output_target **targets,
1346 const unsigned *offsets)
1347 {
1348 struct si_context *sctx = (struct si_context *)ctx;
1349 struct si_buffer_resources *buffers = &sctx->rw_buffers;
1350 struct si_descriptors *descs = &sctx->descriptors[SI_DESCS_RW_BUFFERS];
1351 unsigned old_num_targets = sctx->b.streamout.num_targets;
1352 unsigned i, bufidx;
1353
1354 /* We are going to unbind the buffers. Mark which caches need to be flushed. */
1355 if (sctx->b.streamout.num_targets && sctx->b.streamout.begin_emitted) {
1356 /* Since streamout uses vector writes which go through TC L2
1357 * and most other clients can use TC L2 as well, we don't need
1358 * to flush it.
1359 *
1360 * The only cases which requires flushing it is VGT DMA index
1361 * fetching (on <= CIK) and indirect draw data, which are rare
1362 * cases. Thus, flag the TC L2 dirtiness in the resource and
1363 * handle it at draw call time.
1364 */
1365 for (i = 0; i < sctx->b.streamout.num_targets; i++)
1366 if (sctx->b.streamout.targets[i])
1367 r600_resource(sctx->b.streamout.targets[i]->b.buffer)->TC_L2_dirty = true;
1368
1369 /* Invalidate the scalar cache in case a streamout buffer is
1370 * going to be used as a constant buffer.
1371 *
1372 * Invalidate TC L1, because streamout bypasses it (done by
1373 * setting GLC=1 in the store instruction), but it can contain
1374 * outdated data of streamout buffers.
1375 *
1376 * VS_PARTIAL_FLUSH is required if the buffers are going to be
1377 * used as an input immediately.
1378 */
1379 sctx->b.flags |= SI_CONTEXT_INV_SMEM_L1 |
1380 SI_CONTEXT_INV_VMEM_L1 |
1381 SI_CONTEXT_VS_PARTIAL_FLUSH;
1382 }
1383
1384 /* All readers of the streamout targets need to be finished before we can
1385 * start writing to the targets.
1386 */
1387 if (num_targets)
1388 sctx->b.flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
1389 SI_CONTEXT_CS_PARTIAL_FLUSH;
1390
1391 /* Streamout buffers must be bound in 2 places:
1392 * 1) in VGT by setting the VGT_STRMOUT registers
1393 * 2) as shader resources
1394 */
1395
1396 /* Set the VGT regs. */
1397 r600_set_streamout_targets(ctx, num_targets, targets, offsets);
1398
1399 /* Set the shader resources.*/
1400 for (i = 0; i < num_targets; i++) {
1401 bufidx = SI_VS_STREAMOUT_BUF0 + i;
1402
1403 if (targets[i]) {
1404 struct pipe_resource *buffer = targets[i]->buffer;
1405 uint64_t va = r600_resource(buffer)->gpu_address;
1406
1407 /* Set the descriptor.
1408 *
1409 * On VI, the format must be non-INVALID, otherwise
1410 * the buffer will be considered not bound and store
1411 * instructions will be no-ops.
1412 */
1413 uint32_t *desc = descs->list + bufidx*4;
1414 desc[0] = va;
1415 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32);
1416 desc[2] = 0xffffffff;
1417 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1418 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1419 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1420 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1421 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
1422
1423 /* Set the resource. */
1424 pipe_resource_reference(&buffers->buffers[bufidx],
1425 buffer);
1426 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
1427 (struct r600_resource*)buffer,
1428 buffers->shader_usage,
1429 RADEON_PRIO_SHADER_RW_BUFFER,
1430 true);
1431 r600_resource(buffer)->bind_history |= PIPE_BIND_STREAM_OUTPUT;
1432
1433 buffers->enabled_mask |= 1u << bufidx;
1434 } else {
1435 /* Clear the descriptor and unset the resource. */
1436 memset(descs->list + bufidx*4, 0,
1437 sizeof(uint32_t) * 4);
1438 pipe_resource_reference(&buffers->buffers[bufidx],
1439 NULL);
1440 buffers->enabled_mask &= ~(1u << bufidx);
1441 }
1442 descs->dirty_mask |= 1u << bufidx;
1443 }
1444 for (; i < old_num_targets; i++) {
1445 bufidx = SI_VS_STREAMOUT_BUF0 + i;
1446 /* Clear the descriptor and unset the resource. */
1447 memset(descs->list + bufidx*4, 0, sizeof(uint32_t) * 4);
1448 pipe_resource_reference(&buffers->buffers[bufidx], NULL);
1449 buffers->enabled_mask &= ~(1u << bufidx);
1450 descs->dirty_mask |= 1u << bufidx;
1451 }
1452
1453 sctx->descriptors_dirty |= 1u << SI_DESCS_RW_BUFFERS;
1454 }
1455
1456 static void si_desc_reset_buffer_offset(struct pipe_context *ctx,
1457 uint32_t *desc, uint64_t old_buf_va,
1458 struct pipe_resource *new_buf)
1459 {
1460 /* Retrieve the buffer offset from the descriptor. */
1461 uint64_t old_desc_va =
1462 desc[0] | ((uint64_t)G_008F04_BASE_ADDRESS_HI(desc[1]) << 32);
1463
1464 assert(old_buf_va <= old_desc_va);
1465 uint64_t offset_within_buffer = old_desc_va - old_buf_va;
1466
1467 /* Update the descriptor. */
1468 si_set_buf_desc_address(r600_resource(new_buf), offset_within_buffer,
1469 desc);
1470 }
1471
1472 /* INTERNAL CONST BUFFERS */
1473
1474 static void si_set_polygon_stipple(struct pipe_context *ctx,
1475 const struct pipe_poly_stipple *state)
1476 {
1477 struct si_context *sctx = (struct si_context *)ctx;
1478 struct pipe_constant_buffer cb = {};
1479 unsigned stipple[32];
1480 int i;
1481
1482 for (i = 0; i < 32; i++)
1483 stipple[i] = util_bitreverse(state->stipple[i]);
1484
1485 cb.user_buffer = stipple;
1486 cb.buffer_size = sizeof(stipple);
1487
1488 si_set_rw_buffer(sctx, SI_PS_CONST_POLY_STIPPLE, &cb);
1489 }
1490
1491 /* TEXTURE METADATA ENABLE/DISABLE */
1492
1493 /* CMASK can be enabled (for fast clear) and disabled (for texture export)
1494 * while the texture is bound, possibly by a different context. In that case,
1495 * call this function to update compressed_colortex_masks.
1496 */
1497 void si_update_compressed_colortex_masks(struct si_context *sctx)
1498 {
1499 for (int i = 0; i < SI_NUM_SHADERS; ++i) {
1500 si_samplers_update_compressed_colortex_mask(&sctx->samplers[i]);
1501 si_images_update_compressed_colortex_mask(&sctx->images[i]);
1502 si_update_compressed_tex_shader_mask(sctx, i);
1503 }
1504 }
1505
1506 /* BUFFER DISCARD/INVALIDATION */
1507
1508 /** Reset descriptors of buffer resources after \p buf has been invalidated. */
1509 static void si_reset_buffer_resources(struct si_context *sctx,
1510 struct si_buffer_resources *buffers,
1511 unsigned descriptors_idx,
1512 struct pipe_resource *buf,
1513 uint64_t old_va)
1514 {
1515 struct si_descriptors *descs = &sctx->descriptors[descriptors_idx];
1516 unsigned mask = buffers->enabled_mask;
1517
1518 while (mask) {
1519 unsigned i = u_bit_scan(&mask);
1520 if (buffers->buffers[i] == buf) {
1521 si_desc_reset_buffer_offset(&sctx->b.b,
1522 descs->list + i*4,
1523 old_va, buf);
1524 descs->dirty_mask |= 1u << i;
1525 sctx->descriptors_dirty |= 1u << descriptors_idx;
1526
1527 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
1528 (struct r600_resource *)buf,
1529 buffers->shader_usage,
1530 buffers->priority, true);
1531 }
1532 }
1533 }
1534
1535 /* Reallocate a buffer a update all resource bindings where the buffer is
1536 * bound.
1537 *
1538 * This is used to avoid CPU-GPU synchronizations, because it makes the buffer
1539 * idle by discarding its contents. Apps usually tell us when to do this using
1540 * map_buffer flags, for example.
1541 */
1542 static void si_invalidate_buffer(struct pipe_context *ctx, struct pipe_resource *buf)
1543 {
1544 struct si_context *sctx = (struct si_context*)ctx;
1545 struct r600_resource *rbuffer = r600_resource(buf);
1546 unsigned i, shader;
1547 uint64_t old_va = rbuffer->gpu_address;
1548 unsigned num_elems = sctx->vertex_elements ?
1549 sctx->vertex_elements->count : 0;
1550
1551 /* Reallocate the buffer in the same pipe_resource. */
1552 r600_alloc_resource(&sctx->screen->b, rbuffer);
1553
1554 /* We changed the buffer, now we need to bind it where the old one
1555 * was bound. This consists of 2 things:
1556 * 1) Updating the resource descriptor and dirtying it.
1557 * 2) Adding a relocation to the CS, so that it's usable.
1558 */
1559
1560 /* Vertex buffers. */
1561 if (rbuffer->bind_history & PIPE_BIND_VERTEX_BUFFER) {
1562 for (i = 0; i < num_elems; i++) {
1563 int vb = sctx->vertex_elements->elements[i].vertex_buffer_index;
1564
1565 if (vb >= ARRAY_SIZE(sctx->vertex_buffer))
1566 continue;
1567 if (!sctx->vertex_buffer[vb].buffer)
1568 continue;
1569
1570 if (sctx->vertex_buffer[vb].buffer == buf) {
1571 sctx->vertex_buffers_dirty = true;
1572 break;
1573 }
1574 }
1575 }
1576
1577 /* Streamout buffers. (other internal buffers can't be invalidated) */
1578 if (rbuffer->bind_history & PIPE_BIND_STREAM_OUTPUT) {
1579 for (i = SI_VS_STREAMOUT_BUF0; i <= SI_VS_STREAMOUT_BUF3; i++) {
1580 struct si_buffer_resources *buffers = &sctx->rw_buffers;
1581 struct si_descriptors *descs =
1582 &sctx->descriptors[SI_DESCS_RW_BUFFERS];
1583
1584 if (buffers->buffers[i] != buf)
1585 continue;
1586
1587 si_desc_reset_buffer_offset(ctx, descs->list + i*4,
1588 old_va, buf);
1589 descs->dirty_mask |= 1u << i;
1590 sctx->descriptors_dirty |= 1u << SI_DESCS_RW_BUFFERS;
1591
1592 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
1593 rbuffer, buffers->shader_usage,
1594 RADEON_PRIO_SHADER_RW_BUFFER,
1595 true);
1596
1597 /* Update the streamout state. */
1598 if (sctx->b.streamout.begin_emitted)
1599 r600_emit_streamout_end(&sctx->b);
1600 sctx->b.streamout.append_bitmask =
1601 sctx->b.streamout.enabled_mask;
1602 r600_streamout_buffers_dirty(&sctx->b);
1603 }
1604 }
1605
1606 /* Constant and shader buffers. */
1607 if (rbuffer->bind_history & PIPE_BIND_CONSTANT_BUFFER) {
1608 for (shader = 0; shader < SI_NUM_SHADERS; shader++)
1609 si_reset_buffer_resources(sctx, &sctx->const_buffers[shader],
1610 si_const_buffer_descriptors_idx(shader),
1611 buf, old_va);
1612 }
1613
1614 if (rbuffer->bind_history & PIPE_BIND_SHADER_BUFFER) {
1615 for (shader = 0; shader < SI_NUM_SHADERS; shader++)
1616 si_reset_buffer_resources(sctx, &sctx->shader_buffers[shader],
1617 si_shader_buffer_descriptors_idx(shader),
1618 buf, old_va);
1619 }
1620
1621 if (rbuffer->bind_history & PIPE_BIND_SAMPLER_VIEW) {
1622 /* Texture buffers - update bindings. */
1623 for (shader = 0; shader < SI_NUM_SHADERS; shader++) {
1624 struct si_sampler_views *views = &sctx->samplers[shader].views;
1625 struct si_descriptors *descs =
1626 si_sampler_descriptors(sctx, shader);
1627 unsigned mask = views->enabled_mask;
1628
1629 while (mask) {
1630 unsigned i = u_bit_scan(&mask);
1631 if (views->views[i]->texture == buf) {
1632 si_desc_reset_buffer_offset(ctx,
1633 descs->list +
1634 i * 16 + 4,
1635 old_va, buf);
1636 descs->dirty_mask |= 1u << i;
1637 sctx->descriptors_dirty |=
1638 1u << si_sampler_descriptors_idx(shader);
1639
1640 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
1641 rbuffer, RADEON_USAGE_READ,
1642 RADEON_PRIO_SAMPLER_BUFFER,
1643 true);
1644 }
1645 }
1646 }
1647 }
1648
1649 /* Shader images */
1650 if (rbuffer->bind_history & PIPE_BIND_SHADER_IMAGE) {
1651 for (shader = 0; shader < SI_NUM_SHADERS; ++shader) {
1652 struct si_images_info *images = &sctx->images[shader];
1653 struct si_descriptors *descs =
1654 si_image_descriptors(sctx, shader);
1655 unsigned mask = images->enabled_mask;
1656
1657 while (mask) {
1658 unsigned i = u_bit_scan(&mask);
1659
1660 if (images->views[i].resource == buf) {
1661 if (images->views[i].access & PIPE_IMAGE_ACCESS_WRITE)
1662 si_mark_image_range_valid(&images->views[i]);
1663
1664 si_desc_reset_buffer_offset(
1665 ctx, descs->list + i * 8 + 4,
1666 old_va, buf);
1667 descs->dirty_mask |= 1u << i;
1668 sctx->descriptors_dirty |=
1669 1u << si_image_descriptors_idx(shader);
1670
1671 radeon_add_to_buffer_list_check_mem(
1672 &sctx->b, &sctx->b.gfx, rbuffer,
1673 RADEON_USAGE_READWRITE,
1674 RADEON_PRIO_SAMPLER_BUFFER, true);
1675 }
1676 }
1677 }
1678 }
1679 }
1680
1681 /* Update mutable image descriptor fields of all bound textures. */
1682 void si_update_all_texture_descriptors(struct si_context *sctx)
1683 {
1684 unsigned shader;
1685
1686 for (shader = 0; shader < SI_NUM_SHADERS; shader++) {
1687 struct si_sampler_views *samplers = &sctx->samplers[shader].views;
1688 struct si_images_info *images = &sctx->images[shader];
1689 unsigned mask;
1690
1691 /* Images. */
1692 mask = images->enabled_mask;
1693 while (mask) {
1694 unsigned i = u_bit_scan(&mask);
1695 struct pipe_image_view *view = &images->views[i];
1696
1697 if (!view->resource ||
1698 view->resource->target == PIPE_BUFFER)
1699 continue;
1700
1701 si_set_shader_image(sctx, shader, i, view, true);
1702 }
1703
1704 /* Sampler views. */
1705 mask = samplers->enabled_mask;
1706 while (mask) {
1707 unsigned i = u_bit_scan(&mask);
1708 struct pipe_sampler_view *view = samplers->views[i];
1709
1710 if (!view ||
1711 !view->texture ||
1712 view->texture->target == PIPE_BUFFER)
1713 continue;
1714
1715 si_set_sampler_view(sctx, shader, i,
1716 samplers->views[i], true);
1717 }
1718
1719 si_update_compressed_tex_shader_mask(sctx, shader);
1720 }
1721 }
1722
1723 /* SHADER USER DATA */
1724
1725 static void si_mark_shader_pointers_dirty(struct si_context *sctx,
1726 unsigned shader)
1727 {
1728 sctx->shader_pointers_dirty |=
1729 u_bit_consecutive(SI_DESCS_FIRST_SHADER + shader * SI_NUM_SHADER_DESCS,
1730 SI_NUM_SHADER_DESCS);
1731
1732 if (shader == PIPE_SHADER_VERTEX)
1733 sctx->vertex_buffer_pointer_dirty = sctx->vertex_buffers.buffer != NULL;
1734
1735 si_mark_atom_dirty(sctx, &sctx->shader_userdata.atom);
1736 }
1737
1738 static void si_shader_userdata_begin_new_cs(struct si_context *sctx)
1739 {
1740 sctx->shader_pointers_dirty = u_bit_consecutive(0, SI_NUM_DESCS);
1741 sctx->vertex_buffer_pointer_dirty = sctx->vertex_buffers.buffer != NULL;
1742 si_mark_atom_dirty(sctx, &sctx->shader_userdata.atom);
1743 }
1744
1745 /* Set a base register address for user data constants in the given shader.
1746 * This assigns a mapping from PIPE_SHADER_* to SPI_SHADER_USER_DATA_*.
1747 */
1748 static void si_set_user_data_base(struct si_context *sctx,
1749 unsigned shader, uint32_t new_base)
1750 {
1751 uint32_t *base = &sctx->shader_userdata.sh_base[shader];
1752
1753 if (*base != new_base) {
1754 *base = new_base;
1755
1756 if (new_base)
1757 si_mark_shader_pointers_dirty(sctx, shader);
1758 }
1759 }
1760
1761 /* This must be called when these shaders are changed from non-NULL to NULL
1762 * and vice versa:
1763 * - geometry shader
1764 * - tessellation control shader
1765 * - tessellation evaluation shader
1766 */
1767 void si_shader_change_notify(struct si_context *sctx)
1768 {
1769 /* VS can be bound as VS, ES, or LS. */
1770 if (sctx->tes_shader.cso)
1771 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX,
1772 R_00B530_SPI_SHADER_USER_DATA_LS_0);
1773 else if (sctx->gs_shader.cso)
1774 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX,
1775 R_00B330_SPI_SHADER_USER_DATA_ES_0);
1776 else
1777 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX,
1778 R_00B130_SPI_SHADER_USER_DATA_VS_0);
1779
1780 /* TES can be bound as ES, VS, or not bound. */
1781 if (sctx->tes_shader.cso) {
1782 if (sctx->gs_shader.cso)
1783 si_set_user_data_base(sctx, PIPE_SHADER_TESS_EVAL,
1784 R_00B330_SPI_SHADER_USER_DATA_ES_0);
1785 else
1786 si_set_user_data_base(sctx, PIPE_SHADER_TESS_EVAL,
1787 R_00B130_SPI_SHADER_USER_DATA_VS_0);
1788 } else {
1789 si_set_user_data_base(sctx, PIPE_SHADER_TESS_EVAL, 0);
1790 }
1791 }
1792
1793 static void si_emit_shader_pointer(struct si_context *sctx,
1794 struct si_descriptors *desc,
1795 unsigned sh_base)
1796 {
1797 struct radeon_winsys_cs *cs = sctx->b.gfx.cs;
1798 uint64_t va;
1799
1800 assert(desc->buffer);
1801
1802 va = desc->buffer->gpu_address +
1803 desc->buffer_offset;
1804
1805 radeon_emit(cs, PKT3(PKT3_SET_SH_REG, 2, 0));
1806 radeon_emit(cs, (sh_base + desc->shader_userdata_offset - SI_SH_REG_OFFSET) >> 2);
1807 radeon_emit(cs, va);
1808 radeon_emit(cs, va >> 32);
1809 }
1810
1811 void si_emit_graphics_shader_userdata(struct si_context *sctx,
1812 struct r600_atom *atom)
1813 {
1814 unsigned mask;
1815 uint32_t *sh_base = sctx->shader_userdata.sh_base;
1816 struct si_descriptors *descs;
1817
1818 descs = &sctx->descriptors[SI_DESCS_RW_BUFFERS];
1819
1820 if (sctx->shader_pointers_dirty & (1 << SI_DESCS_RW_BUFFERS)) {
1821 si_emit_shader_pointer(sctx, descs,
1822 R_00B030_SPI_SHADER_USER_DATA_PS_0);
1823 si_emit_shader_pointer(sctx, descs,
1824 R_00B130_SPI_SHADER_USER_DATA_VS_0);
1825 si_emit_shader_pointer(sctx, descs,
1826 R_00B230_SPI_SHADER_USER_DATA_GS_0);
1827 si_emit_shader_pointer(sctx, descs,
1828 R_00B330_SPI_SHADER_USER_DATA_ES_0);
1829 si_emit_shader_pointer(sctx, descs,
1830 R_00B430_SPI_SHADER_USER_DATA_HS_0);
1831 }
1832
1833 mask = sctx->shader_pointers_dirty &
1834 u_bit_consecutive(SI_DESCS_FIRST_SHADER,
1835 SI_DESCS_FIRST_COMPUTE - SI_DESCS_FIRST_SHADER);
1836
1837 while (mask) {
1838 unsigned i = u_bit_scan(&mask);
1839 unsigned shader = (i - SI_DESCS_FIRST_SHADER) / SI_NUM_SHADER_DESCS;
1840 unsigned base = sh_base[shader];
1841
1842 if (base)
1843 si_emit_shader_pointer(sctx, descs + i, base);
1844 }
1845 sctx->shader_pointers_dirty &=
1846 ~u_bit_consecutive(SI_DESCS_RW_BUFFERS, SI_DESCS_FIRST_COMPUTE);
1847
1848 if (sctx->vertex_buffer_pointer_dirty) {
1849 si_emit_shader_pointer(sctx, &sctx->vertex_buffers,
1850 sh_base[PIPE_SHADER_VERTEX]);
1851 sctx->vertex_buffer_pointer_dirty = false;
1852 }
1853 }
1854
1855 void si_emit_compute_shader_userdata(struct si_context *sctx)
1856 {
1857 unsigned base = R_00B900_COMPUTE_USER_DATA_0;
1858 struct si_descriptors *descs = sctx->descriptors;
1859 unsigned compute_mask =
1860 u_bit_consecutive(SI_DESCS_FIRST_COMPUTE, SI_NUM_SHADER_DESCS);
1861 unsigned mask = sctx->shader_pointers_dirty & compute_mask;
1862
1863 while (mask) {
1864 unsigned i = u_bit_scan(&mask);
1865
1866 si_emit_shader_pointer(sctx, descs + i, base);
1867 }
1868 sctx->shader_pointers_dirty &= ~compute_mask;
1869 }
1870
1871 /* INIT/DEINIT/UPLOAD */
1872
1873 void si_init_all_descriptors(struct si_context *sctx)
1874 {
1875 int i;
1876 unsigned ce_offset = 0;
1877
1878 for (i = 0; i < SI_NUM_SHADERS; i++) {
1879 si_init_buffer_resources(&sctx->const_buffers[i],
1880 si_const_buffer_descriptors(sctx, i),
1881 SI_NUM_CONST_BUFFERS, SI_SGPR_CONST_BUFFERS,
1882 RADEON_USAGE_READ, RADEON_PRIO_CONST_BUFFER,
1883 &ce_offset);
1884 si_init_buffer_resources(&sctx->shader_buffers[i],
1885 si_shader_buffer_descriptors(sctx, i),
1886 SI_NUM_SHADER_BUFFERS, SI_SGPR_SHADER_BUFFERS,
1887 RADEON_USAGE_READWRITE, RADEON_PRIO_SHADER_RW_BUFFER,
1888 &ce_offset);
1889
1890 si_init_descriptors(si_sampler_descriptors(sctx, i),
1891 SI_SGPR_SAMPLERS, 16, SI_NUM_SAMPLERS,
1892 null_texture_descriptor, &ce_offset);
1893
1894 si_init_descriptors(si_image_descriptors(sctx, i),
1895 SI_SGPR_IMAGES, 8, SI_NUM_IMAGES,
1896 null_image_descriptor, &ce_offset);
1897 }
1898
1899 si_init_buffer_resources(&sctx->rw_buffers,
1900 &sctx->descriptors[SI_DESCS_RW_BUFFERS],
1901 SI_NUM_RW_BUFFERS, SI_SGPR_RW_BUFFERS,
1902 RADEON_USAGE_READWRITE, RADEON_PRIO_SHADER_RINGS,
1903 &ce_offset);
1904 si_init_descriptors(&sctx->vertex_buffers, SI_SGPR_VERTEX_BUFFERS,
1905 4, SI_NUM_VERTEX_BUFFERS, NULL, NULL);
1906
1907 sctx->descriptors_dirty = u_bit_consecutive(0, SI_NUM_DESCS);
1908
1909 assert(ce_offset <= 32768);
1910
1911 /* Set pipe_context functions. */
1912 sctx->b.b.bind_sampler_states = si_bind_sampler_states;
1913 sctx->b.b.set_shader_images = si_set_shader_images;
1914 sctx->b.b.set_constant_buffer = si_pipe_set_constant_buffer;
1915 sctx->b.b.set_polygon_stipple = si_set_polygon_stipple;
1916 sctx->b.b.set_shader_buffers = si_set_shader_buffers;
1917 sctx->b.b.set_sampler_views = si_set_sampler_views;
1918 sctx->b.b.set_stream_output_targets = si_set_streamout_targets;
1919 sctx->b.invalidate_buffer = si_invalidate_buffer;
1920
1921 /* Shader user data. */
1922 si_init_atom(sctx, &sctx->shader_userdata.atom, &sctx->atoms.s.shader_userdata,
1923 si_emit_graphics_shader_userdata);
1924
1925 /* Set default and immutable mappings. */
1926 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX, R_00B130_SPI_SHADER_USER_DATA_VS_0);
1927 si_set_user_data_base(sctx, PIPE_SHADER_TESS_CTRL, R_00B430_SPI_SHADER_USER_DATA_HS_0);
1928 si_set_user_data_base(sctx, PIPE_SHADER_GEOMETRY, R_00B230_SPI_SHADER_USER_DATA_GS_0);
1929 si_set_user_data_base(sctx, PIPE_SHADER_FRAGMENT, R_00B030_SPI_SHADER_USER_DATA_PS_0);
1930 }
1931
1932 bool si_upload_graphics_shader_descriptors(struct si_context *sctx)
1933 {
1934 const unsigned mask = u_bit_consecutive(0, SI_DESCS_FIRST_COMPUTE);
1935 unsigned dirty = sctx->descriptors_dirty & mask;
1936
1937 /* Assume nothing will go wrong: */
1938 sctx->shader_pointers_dirty |= dirty;
1939
1940 while (dirty) {
1941 unsigned i = u_bit_scan(&dirty);
1942
1943 if (!si_upload_descriptors(sctx, &sctx->descriptors[i],
1944 &sctx->shader_userdata.atom))
1945 return false;
1946 }
1947
1948 sctx->descriptors_dirty &= ~mask;
1949 return true;
1950 }
1951
1952 bool si_upload_compute_shader_descriptors(struct si_context *sctx)
1953 {
1954 /* Does not update rw_buffers as that is not needed for compute shaders
1955 * and the input buffer is using the same SGPR's anyway.
1956 */
1957 const unsigned mask = u_bit_consecutive(SI_DESCS_FIRST_COMPUTE,
1958 SI_NUM_DESCS - SI_DESCS_FIRST_COMPUTE);
1959 unsigned dirty = sctx->descriptors_dirty & mask;
1960
1961 /* Assume nothing will go wrong: */
1962 sctx->shader_pointers_dirty |= dirty;
1963
1964 while (dirty) {
1965 unsigned i = u_bit_scan(&dirty);
1966
1967 if (!si_upload_descriptors(sctx, &sctx->descriptors[i], NULL))
1968 return false;
1969 }
1970
1971 sctx->descriptors_dirty &= ~mask;
1972
1973 return true;
1974 }
1975
1976 void si_release_all_descriptors(struct si_context *sctx)
1977 {
1978 int i;
1979
1980 for (i = 0; i < SI_NUM_SHADERS; i++) {
1981 si_release_buffer_resources(&sctx->const_buffers[i],
1982 si_const_buffer_descriptors(sctx, i));
1983 si_release_buffer_resources(&sctx->shader_buffers[i],
1984 si_shader_buffer_descriptors(sctx, i));
1985 si_release_sampler_views(&sctx->samplers[i].views);
1986 si_release_image_views(&sctx->images[i]);
1987 }
1988 si_release_buffer_resources(&sctx->rw_buffers,
1989 &sctx->descriptors[SI_DESCS_RW_BUFFERS]);
1990
1991 for (i = 0; i < SI_NUM_DESCS; ++i)
1992 si_release_descriptors(&sctx->descriptors[i]);
1993 si_release_descriptors(&sctx->vertex_buffers);
1994 }
1995
1996 void si_all_descriptors_begin_new_cs(struct si_context *sctx)
1997 {
1998 int i;
1999
2000 for (i = 0; i < SI_NUM_SHADERS; i++) {
2001 si_buffer_resources_begin_new_cs(sctx, &sctx->const_buffers[i]);
2002 si_buffer_resources_begin_new_cs(sctx, &sctx->shader_buffers[i]);
2003 si_sampler_views_begin_new_cs(sctx, &sctx->samplers[i].views);
2004 si_image_views_begin_new_cs(sctx, &sctx->images[i]);
2005 }
2006 si_buffer_resources_begin_new_cs(sctx, &sctx->rw_buffers);
2007 si_vertex_buffers_begin_new_cs(sctx);
2008
2009 for (i = 0; i < SI_NUM_DESCS; ++i)
2010 si_descriptors_begin_new_cs(sctx, &sctx->descriptors[i]);
2011
2012 si_shader_userdata_begin_new_cs(sctx);
2013 }