projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
amd: resolve remaining definition conflicts with gfx9d.h
[mesa.git] / src / gallium / drivers / radeonsi / si_descriptors.c
1 /*
2 * Copyright 2013 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 *
23 * Authors:
24 * Marek Olšák <marek.olsak@amd.com>
25 */
26
27 /* Resource binding slots and sampler states (each described with 8 or
28 * 4 dwords) are stored in lists in memory which is accessed by shaders
29 * using scalar load instructions.
30 *
31 * This file is responsible for managing such lists. It keeps a copy of all
32 * descriptors in CPU memory and re-uploads a whole list if some slots have
33 * been changed.
34 *
35 * This code is also reponsible for updating shader pointers to those lists.
36 *
37 * Note that CP DMA can't be used for updating the lists, because a GPU hang
38 * could leave the list in a mid-IB state and the next IB would get wrong
39 * descriptors and the whole context would be unusable at that point.
40 * (Note: The register shadowing can't be used due to the same reason)
41 *
42 * Also, uploading descriptors to newly allocated memory doesn't require
43 * a KCACHE flush.
44 *
45 *
46 * Possible scenarios for one 16 dword image+sampler slot:
47 *
48 * | Image | w/ FMASK | Buffer | NULL
49 * [ 0: 3] Image[0:3] | Image[0:3] | Null[0:3] | Null[0:3]
50 * [ 4: 7] Image[4:7] | Image[4:7] | Buffer[0:3] | 0
51 * [ 8:11] Null[0:3] | Fmask[0:3] | Null[0:3] | Null[0:3]
52 * [12:15] Sampler[0:3] | Fmask[4:7] | Sampler[0:3] | Sampler[0:3]
53 *
54 * FMASK implies MSAA, therefore no sampler state.
55 * Sampler states are never unbound except when FMASK is bound.
56 */
57
58 #include "radeon/r600_cs.h"
59 #include "si_pipe.h"
60 #include "sid.h"
61
62 #include "util/u_format.h"
63 #include "util/u_memory.h"
64 #include "util/u_upload_mgr.h"
65
66
67 /* NULL image and buffer descriptor for textures (alpha = 1) and images
68 * (alpha = 0).
69 *
70 * For images, all fields must be zero except for the swizzle, which
71 * supports arbitrary combinations of 0s and 1s. The texture type must be
72 * any valid type (e.g. 1D). If the texture type isn't set, the hw hangs.
73 *
74 * For buffers, all fields must be zero. If they are not, the hw hangs.
75 *
76 * This is the only reason why the buffer descriptor must be in words [4:7].
77 */
78 static uint32_t null_texture_descriptor[8] = {
79 0,
80 0,
81 0,
82 S_008F1C_DST_SEL_W(V_008F1C_SQ_SEL_1) |
83 S_008F1C_TYPE(V_008F1C_SQ_RSRC_IMG_1D)
84 /* the rest must contain zeros, which is also used by the buffer
85 * descriptor */
86 };
87
88 static uint32_t null_image_descriptor[8] = {
89 0,
90 0,
91 0,
92 S_008F1C_TYPE(V_008F1C_SQ_RSRC_IMG_1D)
93 /* the rest must contain zeros, which is also used by the buffer
94 * descriptor */
95 };
96
97 static void si_init_descriptors(struct si_descriptors *desc,
98 unsigned shader_userdata_index,
99 unsigned element_dw_size,
100 unsigned num_elements,
101 const uint32_t *null_descriptor,
102 unsigned *ce_offset)
103 {
104 int i;
105
106 assert(num_elements <= sizeof(desc->dirty_mask)*8);
107
108 desc->list = CALLOC(num_elements, element_dw_size * 4);
109 desc->element_dw_size = element_dw_size;
110 desc->num_elements = num_elements;
111 desc->dirty_mask = num_elements == 32 ? ~0u : (1u << num_elements) - 1;
112 desc->shader_userdata_offset = shader_userdata_index * 4;
113
114 if (ce_offset) {
115 desc->ce_offset = *ce_offset;
116
117 /* make sure that ce_offset stays 32 byte aligned */
118 *ce_offset += align(element_dw_size * num_elements * 4, 32);
119 }
120
121 /* Initialize the array to NULL descriptors if the element size is 8. */
122 if (null_descriptor) {
123 assert(element_dw_size % 8 == 0);
124 for (i = 0; i < num_elements * element_dw_size / 8; i++)
125 memcpy(desc->list + i * 8, null_descriptor,
126 8 * 4);
127 }
128 }
129
130 static void si_release_descriptors(struct si_descriptors *desc)
131 {
132 r600_resource_reference(&desc->buffer, NULL);
133 FREE(desc->list);
134 }
135
136 static bool si_ce_upload(struct si_context *sctx, unsigned ce_offset, unsigned size,
137 unsigned *out_offset, struct r600_resource **out_buf) {
138 uint64_t va;
139
140 u_suballocator_alloc(sctx->ce_suballocator, size,
141 sctx->screen->b.info.tcc_cache_line_size,
142 out_offset, (struct pipe_resource**)out_buf);
143 if (!out_buf)
144 return false;
145
146 va = (*out_buf)->gpu_address + *out_offset;
147
148 radeon_emit(sctx->ce_ib, PKT3(PKT3_DUMP_CONST_RAM, 3, 0));
149 radeon_emit(sctx->ce_ib, ce_offset);
150 radeon_emit(sctx->ce_ib, size / 4);
151 radeon_emit(sctx->ce_ib, va);
152 radeon_emit(sctx->ce_ib, va >> 32);
153
154 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx, *out_buf,
155 RADEON_USAGE_READWRITE, RADEON_PRIO_DESCRIPTORS);
156
157 sctx->ce_need_synchronization = true;
158 return true;
159 }
160
161 static void si_ce_reinitialize_descriptors(struct si_context *sctx,
162 struct si_descriptors *desc)
163 {
164 if (desc->buffer) {
165 struct r600_resource *buffer = (struct r600_resource*)desc->buffer;
166 unsigned list_size = desc->num_elements * desc->element_dw_size * 4;
167 uint64_t va = buffer->gpu_address + desc->buffer_offset;
168 struct radeon_winsys_cs *ib = sctx->ce_preamble_ib;
169
170 if (!ib)
171 ib = sctx->ce_ib;
172
173 list_size = align(list_size, 32);
174
175 radeon_emit(ib, PKT3(PKT3_LOAD_CONST_RAM, 3, 0));
176 radeon_emit(ib, va);
177 radeon_emit(ib, va >> 32);
178 radeon_emit(ib, list_size / 4);
179 radeon_emit(ib, desc->ce_offset);
180
181 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx, desc->buffer,
182 RADEON_USAGE_READ, RADEON_PRIO_DESCRIPTORS);
183 }
184 desc->ce_ram_dirty = false;
185 }
186
187 void si_ce_reinitialize_all_descriptors(struct si_context *sctx)
188 {
189 int i;
190
191 for (i = 0; i < SI_NUM_DESCS; ++i)
192 si_ce_reinitialize_descriptors(sctx, &sctx->descriptors[i]);
193 }
194
195 void si_ce_enable_loads(struct radeon_winsys_cs *ib)
196 {
197 radeon_emit(ib, PKT3(PKT3_CONTEXT_CONTROL, 1, 0));
198 radeon_emit(ib, CONTEXT_CONTROL_LOAD_ENABLE(1) |
199 CONTEXT_CONTROL_LOAD_CE_RAM(1));
200 radeon_emit(ib, CONTEXT_CONTROL_SHADOW_ENABLE(1));
201 }
202
203 static bool si_upload_descriptors(struct si_context *sctx,
204 struct si_descriptors *desc,
205 struct r600_atom * atom)
206 {
207 unsigned list_size = desc->num_elements * desc->element_dw_size * 4;
208
209 if (!desc->dirty_mask)
210 return true;
211
212 if (sctx->ce_ib) {
213 uint32_t const* list = (uint32_t const*)desc->list;
214
215 if (desc->ce_ram_dirty)
216 si_ce_reinitialize_descriptors(sctx, desc);
217
218 while(desc->dirty_mask) {
219 int begin, count;
220 u_bit_scan_consecutive_range(&desc->dirty_mask, &begin,
221 &count);
222
223 begin *= desc->element_dw_size;
224 count *= desc->element_dw_size;
225
226 radeon_emit(sctx->ce_ib,
227 PKT3(PKT3_WRITE_CONST_RAM, count, 0));
228 radeon_emit(sctx->ce_ib, desc->ce_offset + begin * 4);
229 radeon_emit_array(sctx->ce_ib, list + begin, count);
230 }
231
232 if (!si_ce_upload(sctx, desc->ce_offset, list_size,
233 &desc->buffer_offset, &desc->buffer))
234 return false;
235 } else {
236 void *ptr;
237
238 u_upload_alloc(sctx->b.b.const_uploader, 0, list_size,
239 sctx->screen->b.info.tcc_cache_line_size,
240 &desc->buffer_offset,
241 (struct pipe_resource**)&desc->buffer, &ptr);
242 if (!desc->buffer)
243 return false; /* skip the draw call */
244
245 util_memcpy_cpu_to_le32(ptr, desc->list, list_size);
246 desc->gpu_list = ptr;
247
248 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx, desc->buffer,
249 RADEON_USAGE_READ, RADEON_PRIO_DESCRIPTORS);
250 }
251 desc->dirty_mask = 0;
252
253 if (atom)
254 si_mark_atom_dirty(sctx, atom);
255
256 return true;
257 }
258
259 static void
260 si_descriptors_begin_new_cs(struct si_context *sctx, struct si_descriptors *desc)
261 {
262 desc->ce_ram_dirty = true;
263
264 if (!desc->buffer)
265 return;
266
267 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx, desc->buffer,
268 RADEON_USAGE_READ, RADEON_PRIO_DESCRIPTORS);
269 }
270
271 /* SAMPLER VIEWS */
272
273 static unsigned
274 si_sampler_descriptors_idx(unsigned shader)
275 {
276 return SI_DESCS_FIRST_SHADER + shader * SI_NUM_SHADER_DESCS +
277 SI_SHADER_DESCS_SAMPLERS;
278 }
279
280 static struct si_descriptors *
281 si_sampler_descriptors(struct si_context *sctx, unsigned shader)
282 {
283 return &sctx->descriptors[si_sampler_descriptors_idx(shader)];
284 }
285
286 static void si_release_sampler_views(struct si_sampler_views *views)
287 {
288 int i;
289
290 for (i = 0; i < ARRAY_SIZE(views->views); i++) {
291 pipe_sampler_view_reference(&views->views[i], NULL);
292 }
293 }
294
295 static void si_sampler_view_add_buffer(struct si_context *sctx,
296 struct pipe_resource *resource,
297 enum radeon_bo_usage usage,
298 bool is_stencil_sampler,
299 bool check_mem)
300 {
301 struct r600_resource *rres;
302 struct r600_texture *rtex;
303 enum radeon_bo_priority priority;
304
305 if (!resource)
306 return;
307
308 if (resource->target != PIPE_BUFFER) {
309 struct r600_texture *tex = (struct r600_texture*)resource;
310
311 if (tex->is_depth && !r600_can_sample_zs(tex, is_stencil_sampler))
312 resource = &tex->flushed_depth_texture->resource.b.b;
313 }
314
315 rres = (struct r600_resource*)resource;
316 priority = r600_get_sampler_view_priority(rres);
317
318 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
319 rres, usage, priority,
320 check_mem);
321
322 if (resource->target == PIPE_BUFFER)
323 return;
324
325 /* Now add separate DCC or HTILE. */
326 rtex = (struct r600_texture*)resource;
327 if (rtex->dcc_separate_buffer) {
328 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
329 rtex->dcc_separate_buffer, usage,
330 RADEON_PRIO_DCC, check_mem);
331 }
332
333 if (rtex->htile_buffer &&
334 rtex->tc_compatible_htile &&
335 !is_stencil_sampler) {
336 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
337 rtex->htile_buffer, usage,
338 RADEON_PRIO_HTILE, check_mem);
339 }
340 }
341
342 static void si_sampler_views_begin_new_cs(struct si_context *sctx,
343 struct si_sampler_views *views)
344 {
345 unsigned mask = views->enabled_mask;
346
347 /* Add buffers to the CS. */
348 while (mask) {
349 int i = u_bit_scan(&mask);
350 struct si_sampler_view *sview = (struct si_sampler_view *)views->views[i];
351
352 si_sampler_view_add_buffer(sctx, sview->base.texture,
353 RADEON_USAGE_READ,
354 sview->is_stencil_sampler, false);
355 }
356 }
357
358 /* Set buffer descriptor fields that can be changed by reallocations. */
359 static void si_set_buf_desc_address(struct r600_resource *buf,
360 uint64_t offset, uint32_t *state)
361 {
362 uint64_t va = buf->gpu_address + offset;
363
364 state[0] = va;
365 state[1] &= C_008F04_BASE_ADDRESS_HI;
366 state[1] |= S_008F04_BASE_ADDRESS_HI(va >> 32);
367 }
368
369 /* Set texture descriptor fields that can be changed by reallocations.
370 *
371 * \param tex texture
372 * \param base_level_info information of the level of BASE_ADDRESS
373 * \param base_level the level of BASE_ADDRESS
374 * \param first_level pipe_sampler_view.u.tex.first_level
375 * \param block_width util_format_get_blockwidth()
376 * \param is_stencil select between separate Z & Stencil
377 * \param state descriptor to update
378 */
379 void si_set_mutable_tex_desc_fields(struct r600_texture *tex,
380 const struct radeon_surf_level *base_level_info,
381 unsigned base_level, unsigned first_level,
382 unsigned block_width, bool is_stencil,
383 uint32_t *state)
384 {
385 uint64_t va;
386 unsigned pitch = base_level_info->nblk_x * block_width;
387
388 if (tex->is_depth && !r600_can_sample_zs(tex, is_stencil)) {
389 tex = tex->flushed_depth_texture;
390 is_stencil = false;
391 }
392
393 va = tex->resource.gpu_address + base_level_info->offset;
394
395 state[1] &= C_008F14_BASE_ADDRESS_HI;
396 state[3] &= C_008F1C_TILING_INDEX;
397 state[4] &= C_008F20_PITCH_GFX6;
398 state[6] &= C_008F28_COMPRESSION_EN;
399
400 state[0] = va >> 8;
401 state[1] |= S_008F14_BASE_ADDRESS_HI(va >> 40);
402 state[3] |= S_008F1C_TILING_INDEX(si_tile_mode_index(tex, base_level,
403 is_stencil));
404 state[4] |= S_008F20_PITCH_GFX6(pitch - 1);
405
406 if (tex->dcc_offset && first_level < tex->surface.num_dcc_levels) {
407 state[6] |= S_008F28_COMPRESSION_EN(1);
408 state[7] = ((!tex->dcc_separate_buffer ? tex->resource.gpu_address : 0) +
409 tex->dcc_offset +
410 base_level_info->dcc_offset) >> 8;
411 } else if (tex->tc_compatible_htile) {
412 state[6] |= S_008F28_COMPRESSION_EN(1);
413 state[7] = tex->htile_buffer->gpu_address >> 8;
414 }
415 }
416
417 static void si_set_sampler_view(struct si_context *sctx,
418 unsigned shader,
419 unsigned slot, struct pipe_sampler_view *view,
420 bool disallow_early_out)
421 {
422 struct si_sampler_views *views = &sctx->samplers[shader].views;
423 struct si_sampler_view *rview = (struct si_sampler_view*)view;
424 struct si_descriptors *descs = si_sampler_descriptors(sctx, shader);
425 uint32_t *desc = descs->list + slot * 16;
426
427 if (views->views[slot] == view && !disallow_early_out)
428 return;
429
430 if (view) {
431 struct r600_texture *rtex = (struct r600_texture *)view->texture;
432
433 assert(rtex); /* views with texture == NULL aren't supported */
434 pipe_sampler_view_reference(&views->views[slot], view);
435 memcpy(desc, rview->state, 8*4);
436
437 if (rtex->resource.b.b.target == PIPE_BUFFER) {
438 rtex->resource.bind_history |= PIPE_BIND_SAMPLER_VIEW;
439
440 si_set_buf_desc_address(&rtex->resource,
441 view->u.buf.offset,
442 desc + 4);
443 } else {
444 bool is_separate_stencil =
445 rtex->db_compatible &&
446 rview->is_stencil_sampler;
447
448 si_set_mutable_tex_desc_fields(rtex,
449 rview->base_level_info,
450 rview->base_level,
451 rview->base.u.tex.first_level,
452 rview->block_width,
453 is_separate_stencil,
454 desc);
455 }
456
457 if (rtex->resource.b.b.target != PIPE_BUFFER &&
458 rtex->fmask.size) {
459 memcpy(desc + 8,
460 rview->fmask_state, 8*4);
461 } else {
462 /* Disable FMASK and bind sampler state in [12:15]. */
463 memcpy(desc + 8,
464 null_texture_descriptor, 4*4);
465
466 if (views->sampler_states[slot])
467 memcpy(desc + 12,
468 views->sampler_states[slot]->val, 4*4);
469 }
470
471 views->enabled_mask |= 1u << slot;
472
473 /* Since this can flush, it must be done after enabled_mask is
474 * updated. */
475 si_sampler_view_add_buffer(sctx, view->texture,
476 RADEON_USAGE_READ,
477 rview->is_stencil_sampler, true);
478 } else {
479 pipe_sampler_view_reference(&views->views[slot], NULL);
480 memcpy(desc, null_texture_descriptor, 8*4);
481 /* Only clear the lower dwords of FMASK. */
482 memcpy(desc + 8, null_texture_descriptor, 4*4);
483 /* Re-set the sampler state if we are transitioning from FMASK. */
484 if (views->sampler_states[slot])
485 memcpy(desc + 12,
486 views->sampler_states[slot]->val, 4*4);
487
488 views->enabled_mask &= ~(1u << slot);
489 }
490
491 descs->dirty_mask |= 1u << slot;
492 sctx->descriptors_dirty |= 1u << si_sampler_descriptors_idx(shader);
493 }
494
495 static bool is_compressed_colortex(struct r600_texture *rtex)
496 {
497 return rtex->cmask.size || rtex->fmask.size ||
498 (rtex->dcc_offset && rtex->dirty_level_mask);
499 }
500
501 static void si_update_compressed_tex_shader_mask(struct si_context *sctx,
502 unsigned shader)
503 {
504 struct si_textures_info *samplers = &sctx->samplers[shader];
505 unsigned shader_bit = 1 << shader;
506
507 if (samplers->depth_texture_mask ||
508 samplers->compressed_colortex_mask ||
509 sctx->images[shader].compressed_colortex_mask)
510 sctx->compressed_tex_shader_mask |= shader_bit;
511 else
512 sctx->compressed_tex_shader_mask &= ~shader_bit;
513 }
514
515 static void si_set_sampler_views(struct pipe_context *ctx,
516 enum pipe_shader_type shader, unsigned start,
517 unsigned count,
518 struct pipe_sampler_view **views)
519 {
520 struct si_context *sctx = (struct si_context *)ctx;
521 struct si_textures_info *samplers = &sctx->samplers[shader];
522 int i;
523
524 if (!count || shader >= SI_NUM_SHADERS)
525 return;
526
527 for (i = 0; i < count; i++) {
528 unsigned slot = start + i;
529
530 if (!views || !views[i]) {
531 samplers->depth_texture_mask &= ~(1u << slot);
532 samplers->compressed_colortex_mask &= ~(1u << slot);
533 si_set_sampler_view(sctx, shader, slot, NULL, false);
534 continue;
535 }
536
537 si_set_sampler_view(sctx, shader, slot, views[i], false);
538
539 if (views[i]->texture && views[i]->texture->target != PIPE_BUFFER) {
540 struct r600_texture *rtex =
541 (struct r600_texture*)views[i]->texture;
542 struct si_sampler_view *rview = (struct si_sampler_view *)views[i];
543
544 if (rtex->db_compatible &&
545 (!rtex->tc_compatible_htile || rview->is_stencil_sampler)) {
546 samplers->depth_texture_mask |= 1u << slot;
547 } else {
548 samplers->depth_texture_mask &= ~(1u << slot);
549 }
550 if (is_compressed_colortex(rtex)) {
551 samplers->compressed_colortex_mask |= 1u << slot;
552 } else {
553 samplers->compressed_colortex_mask &= ~(1u << slot);
554 }
555
556 if (rtex->dcc_offset &&
557 p_atomic_read(&rtex->framebuffers_bound))
558 sctx->need_check_render_feedback = true;
559 } else {
560 samplers->depth_texture_mask &= ~(1u << slot);
561 samplers->compressed_colortex_mask &= ~(1u << slot);
562 }
563 }
564
565 si_update_compressed_tex_shader_mask(sctx, shader);
566 }
567
568 static void
569 si_samplers_update_compressed_colortex_mask(struct si_textures_info *samplers)
570 {
571 unsigned mask = samplers->views.enabled_mask;
572
573 while (mask) {
574 int i = u_bit_scan(&mask);
575 struct pipe_resource *res = samplers->views.views[i]->texture;
576
577 if (res && res->target != PIPE_BUFFER) {
578 struct r600_texture *rtex = (struct r600_texture *)res;
579
580 if (is_compressed_colortex(rtex)) {
581 samplers->compressed_colortex_mask |= 1u << i;
582 } else {
583 samplers->compressed_colortex_mask &= ~(1u << i);
584 }
585 }
586 }
587 }
588
589 /* IMAGE VIEWS */
590
591 static unsigned
592 si_image_descriptors_idx(unsigned shader)
593 {
594 return SI_DESCS_FIRST_SHADER + shader * SI_NUM_SHADER_DESCS +
595 SI_SHADER_DESCS_IMAGES;
596 }
597
598 static struct si_descriptors*
599 si_image_descriptors(struct si_context *sctx, unsigned shader)
600 {
601 return &sctx->descriptors[si_image_descriptors_idx(shader)];
602 }
603
604 static void
605 si_release_image_views(struct si_images_info *images)
606 {
607 unsigned i;
608
609 for (i = 0; i < SI_NUM_IMAGES; ++i) {
610 struct pipe_image_view *view = &images->views[i];
611
612 pipe_resource_reference(&view->resource, NULL);
613 }
614 }
615
616 static void
617 si_image_views_begin_new_cs(struct si_context *sctx, struct si_images_info *images)
618 {
619 uint mask = images->enabled_mask;
620
621 /* Add buffers to the CS. */
622 while (mask) {
623 int i = u_bit_scan(&mask);
624 struct pipe_image_view *view = &images->views[i];
625
626 assert(view->resource);
627
628 si_sampler_view_add_buffer(sctx, view->resource,
629 RADEON_USAGE_READWRITE, false, false);
630 }
631 }
632
633 static void
634 si_disable_shader_image(struct si_context *ctx, unsigned shader, unsigned slot)
635 {
636 struct si_images_info *images = &ctx->images[shader];
637
638 if (images->enabled_mask & (1u << slot)) {
639 struct si_descriptors *descs = si_image_descriptors(ctx, shader);
640
641 pipe_resource_reference(&images->views[slot].resource, NULL);
642 images->compressed_colortex_mask &= ~(1 << slot);
643
644 memcpy(descs->list + slot*8, null_image_descriptor, 8*4);
645 images->enabled_mask &= ~(1u << slot);
646 descs->dirty_mask |= 1u << slot;
647 ctx->descriptors_dirty |= 1u << si_image_descriptors_idx(shader);
648 }
649 }
650
651 static void
652 si_mark_image_range_valid(const struct pipe_image_view *view)
653 {
654 struct r600_resource *res = (struct r600_resource *)view->resource;
655
656 assert(res && res->b.b.target == PIPE_BUFFER);
657
658 util_range_add(&res->valid_buffer_range,
659 view->u.buf.offset,
660 view->u.buf.offset + view->u.buf.size);
661 }
662
663 static void si_set_shader_image(struct si_context *ctx,
664 unsigned shader,
665 unsigned slot, const struct pipe_image_view *view,
666 bool skip_decompress)
667 {
668 struct si_screen *screen = ctx->screen;
669 struct si_images_info *images = &ctx->images[shader];
670 struct si_descriptors *descs = si_image_descriptors(ctx, shader);
671 struct r600_resource *res;
672 uint32_t *desc = descs->list + slot * 8;
673
674 if (!view || !view->resource) {
675 si_disable_shader_image(ctx, shader, slot);
676 return;
677 }
678
679 res = (struct r600_resource *)view->resource;
680
681 if (&images->views[slot] != view)
682 util_copy_image_view(&images->views[slot], view);
683
684 if (res->b.b.target == PIPE_BUFFER) {
685 if (view->access & PIPE_IMAGE_ACCESS_WRITE)
686 si_mark_image_range_valid(view);
687
688 si_make_buffer_descriptor(screen, res,
689 view->format,
690 view->u.buf.offset,
691 view->u.buf.size,
692 descs->list + slot * 8);
693 si_set_buf_desc_address(res, view->u.buf.offset, desc + 4);
694
695 images->compressed_colortex_mask &= ~(1 << slot);
696 res->bind_history |= PIPE_BIND_SHADER_IMAGE;
697 } else {
698 static const unsigned char swizzle[4] = { 0, 1, 2, 3 };
699 struct r600_texture *tex = (struct r600_texture *)res;
700 unsigned level = view->u.tex.level;
701 unsigned width, height, depth;
702 bool uses_dcc = tex->dcc_offset &&
703 level < tex->surface.num_dcc_levels;
704
705 assert(!tex->is_depth);
706 assert(tex->fmask.size == 0);
707
708 if (uses_dcc && !skip_decompress &&
709 (view->access & PIPE_IMAGE_ACCESS_WRITE ||
710 !vi_dcc_formats_compatible(res->b.b.format, view->format))) {
711 /* If DCC can't be disabled, at least decompress it.
712 * The decompression is relatively cheap if the surface
713 * has been decompressed already.
714 */
715 if (r600_texture_disable_dcc(&ctx->b, tex))
716 uses_dcc = false;
717 else
718 ctx->b.decompress_dcc(&ctx->b.b, tex);
719 }
720
721 if (is_compressed_colortex(tex)) {
722 images->compressed_colortex_mask |= 1 << slot;
723 } else {
724 images->compressed_colortex_mask &= ~(1 << slot);
725 }
726
727 if (uses_dcc &&
728 p_atomic_read(&tex->framebuffers_bound))
729 ctx->need_check_render_feedback = true;
730
731 /* Always force the base level to the selected level.
732 *
733 * This is required for 3D textures, where otherwise
734 * selecting a single slice for non-layered bindings
735 * fails. It doesn't hurt the other targets.
736 */
737 width = u_minify(res->b.b.width0, level);
738 height = u_minify(res->b.b.height0, level);
739 depth = u_minify(res->b.b.depth0, level);
740
741 si_make_texture_descriptor(screen, tex,
742 false, res->b.b.target,
743 view->format, swizzle,
744 0, 0,
745 view->u.tex.first_layer,
746 view->u.tex.last_layer,
747 width, height, depth,
748 desc, NULL);
749 si_set_mutable_tex_desc_fields(tex, &tex->surface.level[level],
750 level, level,
751 util_format_get_blockwidth(view->format),
752 false, desc);
753 }
754
755 images->enabled_mask |= 1u << slot;
756 descs->dirty_mask |= 1u << slot;
757 ctx->descriptors_dirty |= 1u << si_image_descriptors_idx(shader);
758
759 /* Since this can flush, it must be done after enabled_mask is updated. */
760 si_sampler_view_add_buffer(ctx, &res->b.b,
761 RADEON_USAGE_READWRITE, false, true);
762 }
763
764 static void
765 si_set_shader_images(struct pipe_context *pipe,
766 enum pipe_shader_type shader,
767 unsigned start_slot, unsigned count,
768 const struct pipe_image_view *views)
769 {
770 struct si_context *ctx = (struct si_context *)pipe;
771 unsigned i, slot;
772
773 assert(shader < SI_NUM_SHADERS);
774
775 if (!count)
776 return;
777
778 assert(start_slot + count <= SI_NUM_IMAGES);
779
780 if (views) {
781 for (i = 0, slot = start_slot; i < count; ++i, ++slot)
782 si_set_shader_image(ctx, shader, slot, &views[i], false);
783 } else {
784 for (i = 0, slot = start_slot; i < count; ++i, ++slot)
785 si_set_shader_image(ctx, shader, slot, NULL, false);
786 }
787
788 si_update_compressed_tex_shader_mask(ctx, shader);
789 }
790
791 static void
792 si_images_update_compressed_colortex_mask(struct si_images_info *images)
793 {
794 unsigned mask = images->enabled_mask;
795
796 while (mask) {
797 int i = u_bit_scan(&mask);
798 struct pipe_resource *res = images->views[i].resource;
799
800 if (res && res->target != PIPE_BUFFER) {
801 struct r600_texture *rtex = (struct r600_texture *)res;
802
803 if (is_compressed_colortex(rtex)) {
804 images->compressed_colortex_mask |= 1 << i;
805 } else {
806 images->compressed_colortex_mask &= ~(1 << i);
807 }
808 }
809 }
810 }
811
812 /* SAMPLER STATES */
813
814 static void si_bind_sampler_states(struct pipe_context *ctx,
815 enum pipe_shader_type shader,
816 unsigned start, unsigned count, void **states)
817 {
818 struct si_context *sctx = (struct si_context *)ctx;
819 struct si_textures_info *samplers = &sctx->samplers[shader];
820 struct si_descriptors *desc = si_sampler_descriptors(sctx, shader);
821 struct si_sampler_state **sstates = (struct si_sampler_state**)states;
822 int i;
823
824 if (!count || shader >= SI_NUM_SHADERS)
825 return;
826
827 for (i = 0; i < count; i++) {
828 unsigned slot = start + i;
829
830 if (!sstates[i] ||
831 sstates[i] == samplers->views.sampler_states[slot])
832 continue;
833
834 #ifdef DEBUG
835 assert(sstates[i]->magic == SI_SAMPLER_STATE_MAGIC);
836 #endif
837 samplers->views.sampler_states[slot] = sstates[i];
838
839 /* If FMASK is bound, don't overwrite it.
840 * The sampler state will be set after FMASK is unbound.
841 */
842 if (samplers->views.views[slot] &&
843 samplers->views.views[slot]->texture &&
844 samplers->views.views[slot]->texture->target != PIPE_BUFFER &&
845 ((struct r600_texture*)samplers->views.views[slot]->texture)->fmask.size)
846 continue;
847
848 memcpy(desc->list + slot * 16 + 12, sstates[i]->val, 4*4);
849 desc->dirty_mask |= 1u << slot;
850 sctx->descriptors_dirty |= 1u << si_sampler_descriptors_idx(shader);
851 }
852 }
853
854 /* BUFFER RESOURCES */
855
856 static void si_init_buffer_resources(struct si_buffer_resources *buffers,
857 struct si_descriptors *descs,
858 unsigned num_buffers,
859 unsigned shader_userdata_index,
860 enum radeon_bo_usage shader_usage,
861 enum radeon_bo_priority priority,
862 unsigned *ce_offset)
863 {
864 buffers->shader_usage = shader_usage;
865 buffers->priority = priority;
866 buffers->buffers = CALLOC(num_buffers, sizeof(struct pipe_resource*));
867
868 si_init_descriptors(descs, shader_userdata_index, 4,
869 num_buffers, NULL, ce_offset);
870 }
871
872 static void si_release_buffer_resources(struct si_buffer_resources *buffers,
873 struct si_descriptors *descs)
874 {
875 int i;
876
877 for (i = 0; i < descs->num_elements; i++) {
878 pipe_resource_reference(&buffers->buffers[i], NULL);
879 }
880
881 FREE(buffers->buffers);
882 }
883
884 static void si_buffer_resources_begin_new_cs(struct si_context *sctx,
885 struct si_buffer_resources *buffers)
886 {
887 unsigned mask = buffers->enabled_mask;
888
889 /* Add buffers to the CS. */
890 while (mask) {
891 int i = u_bit_scan(&mask);
892
893 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
894 (struct r600_resource*)buffers->buffers[i],
895 buffers->shader_usage, buffers->priority);
896 }
897 }
898
899 static void si_get_buffer_from_descriptors(struct si_buffer_resources *buffers,
900 struct si_descriptors *descs,
901 unsigned idx, struct pipe_resource **buf,
902 unsigned *offset, unsigned *size)
903 {
904 pipe_resource_reference(buf, buffers->buffers[idx]);
905 if (*buf) {
906 struct r600_resource *res = r600_resource(*buf);
907 const uint32_t *desc = descs->list + idx * 4;
908 uint64_t va;
909
910 *size = desc[2];
911
912 assert(G_008F04_STRIDE(desc[1]) == 0);
913 va = ((uint64_t)desc[1] << 32) | desc[0];
914
915 assert(va >= res->gpu_address && va + *size <= res->gpu_address + res->bo_size);
916 *offset = va - res->gpu_address;
917 }
918 }
919
920 /* VERTEX BUFFERS */
921
922 static void si_vertex_buffers_begin_new_cs(struct si_context *sctx)
923 {
924 struct si_descriptors *desc = &sctx->vertex_buffers;
925 int count = sctx->vertex_elements ? sctx->vertex_elements->count : 0;
926 int i;
927
928 for (i = 0; i < count; i++) {
929 int vb = sctx->vertex_elements->elements[i].vertex_buffer_index;
930
931 if (vb >= ARRAY_SIZE(sctx->vertex_buffer))
932 continue;
933 if (!sctx->vertex_buffer[vb].buffer)
934 continue;
935
936 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
937 (struct r600_resource*)sctx->vertex_buffer[vb].buffer,
938 RADEON_USAGE_READ, RADEON_PRIO_VERTEX_BUFFER);
939 }
940
941 if (!desc->buffer)
942 return;
943 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
944 desc->buffer, RADEON_USAGE_READ,
945 RADEON_PRIO_DESCRIPTORS);
946 }
947
948 bool si_upload_vertex_buffer_descriptors(struct si_context *sctx)
949 {
950 struct si_vertex_element *velems = sctx->vertex_elements;
951 struct si_descriptors *desc = &sctx->vertex_buffers;
952 unsigned i, count;
953 unsigned desc_list_byte_size;
954 unsigned first_vb_use_mask;
955 uint64_t va;
956 uint32_t *ptr;
957
958 if (!sctx->vertex_buffers_dirty || !velems)
959 return true;
960
961 count = velems->count;
962
963 if (!count)
964 return true;
965
966 desc_list_byte_size = velems->desc_list_byte_size;
967 first_vb_use_mask = velems->first_vb_use_mask;
968
969 /* Vertex buffer descriptors are the only ones which are uploaded
970 * directly through a staging buffer and don't go through
971 * the fine-grained upload path.
972 */
973 u_upload_alloc(sctx->b.b.const_uploader, 0,
974 desc_list_byte_size,
975 si_optimal_tcc_alignment(sctx, desc_list_byte_size),
976 &desc->buffer_offset,
977 (struct pipe_resource**)&desc->buffer, (void**)&ptr);
978 if (!desc->buffer)
979 return false;
980
981 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
982 desc->buffer, RADEON_USAGE_READ,
983 RADEON_PRIO_DESCRIPTORS);
984
985 assert(count <= SI_MAX_ATTRIBS);
986
987 for (i = 0; i < count; i++) {
988 struct pipe_vertex_element *ve = &velems->elements[i];
989 struct pipe_vertex_buffer *vb;
990 struct r600_resource *rbuffer;
991 unsigned offset;
992 unsigned vbo_index = ve->vertex_buffer_index;
993 uint32_t *desc = &ptr[i*4];
994
995 vb = &sctx->vertex_buffer[vbo_index];
996 rbuffer = (struct r600_resource*)vb->buffer;
997 if (!rbuffer) {
998 memset(desc, 0, 16);
999 continue;
1000 }
1001
1002 offset = vb->buffer_offset + ve->src_offset;
1003 va = rbuffer->gpu_address + offset;
1004
1005 /* Fill in T# buffer resource description */
1006 desc[0] = va;
1007 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
1008 S_008F04_STRIDE(vb->stride);
1009
1010 if (sctx->b.chip_class <= CIK && vb->stride) {
1011 /* Round up by rounding down and adding 1 */
1012 desc[2] = (vb->buffer->width0 - offset -
1013 velems->format_size[i]) /
1014 vb->stride + 1;
1015 } else {
1016 desc[2] = vb->buffer->width0 - offset;
1017 }
1018
1019 desc[3] = velems->rsrc_word3[i];
1020
1021 if (first_vb_use_mask & (1 << i)) {
1022 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
1023 (struct r600_resource*)vb->buffer,
1024 RADEON_USAGE_READ, RADEON_PRIO_VERTEX_BUFFER);
1025 }
1026 }
1027
1028 /* Don't flush the const cache. It would have a very negative effect
1029 * on performance (confirmed by testing). New descriptors are always
1030 * uploaded to a fresh new buffer, so I don't think flushing the const
1031 * cache is needed. */
1032 si_mark_atom_dirty(sctx, &sctx->shader_userdata.atom);
1033 if (sctx->b.chip_class >= CIK)
1034 si_mark_atom_dirty(sctx, &sctx->prefetch_L2);
1035 sctx->vertex_buffers_dirty = false;
1036 sctx->vertex_buffer_pointer_dirty = true;
1037 return true;
1038 }
1039
1040
1041 /* CONSTANT BUFFERS */
1042
1043 static unsigned
1044 si_const_buffer_descriptors_idx(unsigned shader)
1045 {
1046 return SI_DESCS_FIRST_SHADER + shader * SI_NUM_SHADER_DESCS +
1047 SI_SHADER_DESCS_CONST_BUFFERS;
1048 }
1049
1050 static struct si_descriptors *
1051 si_const_buffer_descriptors(struct si_context *sctx, unsigned shader)
1052 {
1053 return &sctx->descriptors[si_const_buffer_descriptors_idx(shader)];
1054 }
1055
1056 void si_upload_const_buffer(struct si_context *sctx, struct r600_resource **rbuffer,
1057 const uint8_t *ptr, unsigned size, uint32_t *const_offset)
1058 {
1059 void *tmp;
1060
1061 u_upload_alloc(sctx->b.b.const_uploader, 0, size,
1062 si_optimal_tcc_alignment(sctx, size),
1063 const_offset,
1064 (struct pipe_resource**)rbuffer, &tmp);
1065 if (*rbuffer)
1066 util_memcpy_cpu_to_le32(tmp, ptr, size);
1067 }
1068
1069 static void si_set_constant_buffer(struct si_context *sctx,
1070 struct si_buffer_resources *buffers,
1071 unsigned descriptors_idx,
1072 uint slot, const struct pipe_constant_buffer *input)
1073 {
1074 struct si_descriptors *descs = &sctx->descriptors[descriptors_idx];
1075 assert(slot < descs->num_elements);
1076 pipe_resource_reference(&buffers->buffers[slot], NULL);
1077
1078 /* CIK cannot unbind a constant buffer (S_BUFFER_LOAD is buggy
1079 * with a NULL buffer). We need to use a dummy buffer instead. */
1080 if (sctx->b.chip_class == CIK &&
1081 (!input || (!input->buffer && !input->user_buffer)))
1082 input = &sctx->null_const_buf;
1083
1084 if (input && (input->buffer || input->user_buffer)) {
1085 struct pipe_resource *buffer = NULL;
1086 uint64_t va;
1087
1088 /* Upload the user buffer if needed. */
1089 if (input->user_buffer) {
1090 unsigned buffer_offset;
1091
1092 si_upload_const_buffer(sctx,
1093 (struct r600_resource**)&buffer, input->user_buffer,
1094 input->buffer_size, &buffer_offset);
1095 if (!buffer) {
1096 /* Just unbind on failure. */
1097 si_set_constant_buffer(sctx, buffers, descriptors_idx, slot, NULL);
1098 return;
1099 }
1100 va = r600_resource(buffer)->gpu_address + buffer_offset;
1101 } else {
1102 pipe_resource_reference(&buffer, input->buffer);
1103 va = r600_resource(buffer)->gpu_address + input->buffer_offset;
1104 /* Only track usage for non-user buffers. */
1105 r600_resource(buffer)->bind_history |= PIPE_BIND_CONSTANT_BUFFER;
1106 }
1107
1108 /* Set the descriptor. */
1109 uint32_t *desc = descs->list + slot*4;
1110 desc[0] = va;
1111 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
1112 S_008F04_STRIDE(0);
1113 desc[2] = input->buffer_size;
1114 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1115 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1116 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1117 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1118 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1119 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
1120
1121 buffers->buffers[slot] = buffer;
1122 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
1123 (struct r600_resource*)buffer,
1124 buffers->shader_usage,
1125 buffers->priority, true);
1126 buffers->enabled_mask |= 1u << slot;
1127 } else {
1128 /* Clear the descriptor. */
1129 memset(descs->list + slot*4, 0, sizeof(uint32_t) * 4);
1130 buffers->enabled_mask &= ~(1u << slot);
1131 }
1132
1133 descs->dirty_mask |= 1u << slot;
1134 sctx->descriptors_dirty |= 1u << descriptors_idx;
1135 }
1136
1137 void si_set_rw_buffer(struct si_context *sctx,
1138 uint slot, const struct pipe_constant_buffer *input)
1139 {
1140 si_set_constant_buffer(sctx, &sctx->rw_buffers,
1141 SI_DESCS_RW_BUFFERS, slot, input);
1142 }
1143
1144 static void si_pipe_set_constant_buffer(struct pipe_context *ctx,
1145 enum pipe_shader_type shader, uint slot,
1146 const struct pipe_constant_buffer *input)
1147 {
1148 struct si_context *sctx = (struct si_context *)ctx;
1149
1150 if (shader >= SI_NUM_SHADERS)
1151 return;
1152
1153 si_set_constant_buffer(sctx, &sctx->const_buffers[shader],
1154 si_const_buffer_descriptors_idx(shader),
1155 slot, input);
1156 }
1157
1158 void si_get_pipe_constant_buffer(struct si_context *sctx, uint shader,
1159 uint slot, struct pipe_constant_buffer *cbuf)
1160 {
1161 cbuf->user_buffer = NULL;
1162 si_get_buffer_from_descriptors(
1163 &sctx->const_buffers[shader],
1164 si_const_buffer_descriptors(sctx, shader),
1165 slot, &cbuf->buffer, &cbuf->buffer_offset, &cbuf->buffer_size);
1166 }
1167
1168 /* SHADER BUFFERS */
1169
1170 static unsigned
1171 si_shader_buffer_descriptors_idx(enum pipe_shader_type shader)
1172 {
1173 return SI_DESCS_FIRST_SHADER + shader * SI_NUM_SHADER_DESCS +
1174 SI_SHADER_DESCS_SHADER_BUFFERS;
1175 }
1176
1177 static struct si_descriptors *
1178 si_shader_buffer_descriptors(struct si_context *sctx,
1179 enum pipe_shader_type shader)
1180 {
1181 return &sctx->descriptors[si_shader_buffer_descriptors_idx(shader)];
1182 }
1183
1184 static void si_set_shader_buffers(struct pipe_context *ctx,
1185 enum pipe_shader_type shader,
1186 unsigned start_slot, unsigned count,
1187 const struct pipe_shader_buffer *sbuffers)
1188 {
1189 struct si_context *sctx = (struct si_context *)ctx;
1190 struct si_buffer_resources *buffers = &sctx->shader_buffers[shader];
1191 struct si_descriptors *descs = si_shader_buffer_descriptors(sctx, shader);
1192 unsigned i;
1193
1194 assert(start_slot + count <= SI_NUM_SHADER_BUFFERS);
1195
1196 for (i = 0; i < count; ++i) {
1197 const struct pipe_shader_buffer *sbuffer = sbuffers ? &sbuffers[i] : NULL;
1198 struct r600_resource *buf;
1199 unsigned slot = start_slot + i;
1200 uint32_t *desc = descs->list + slot * 4;
1201 uint64_t va;
1202
1203 if (!sbuffer || !sbuffer->buffer) {
1204 pipe_resource_reference(&buffers->buffers[slot], NULL);
1205 memset(desc, 0, sizeof(uint32_t) * 4);
1206 buffers->enabled_mask &= ~(1u << slot);
1207 descs->dirty_mask |= 1u << slot;
1208 sctx->descriptors_dirty |=
1209 1u << si_shader_buffer_descriptors_idx(shader);
1210 continue;
1211 }
1212
1213 buf = (struct r600_resource *)sbuffer->buffer;
1214 va = buf->gpu_address + sbuffer->buffer_offset;
1215
1216 desc[0] = va;
1217 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
1218 S_008F04_STRIDE(0);
1219 desc[2] = sbuffer->buffer_size;
1220 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1221 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1222 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1223 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1224 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1225 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
1226
1227 pipe_resource_reference(&buffers->buffers[slot], &buf->b.b);
1228 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx, buf,
1229 buffers->shader_usage,
1230 buffers->priority, true);
1231 buf->bind_history |= PIPE_BIND_SHADER_BUFFER;
1232
1233 buffers->enabled_mask |= 1u << slot;
1234 descs->dirty_mask |= 1u << slot;
1235 sctx->descriptors_dirty |=
1236 1u << si_shader_buffer_descriptors_idx(shader);
1237
1238 util_range_add(&buf->valid_buffer_range, sbuffer->buffer_offset,
1239 sbuffer->buffer_offset + sbuffer->buffer_size);
1240 }
1241 }
1242
1243 void si_get_shader_buffers(struct si_context *sctx,
1244 enum pipe_shader_type shader,
1245 uint start_slot, uint count,
1246 struct pipe_shader_buffer *sbuf)
1247 {
1248 struct si_buffer_resources *buffers = &sctx->shader_buffers[shader];
1249 struct si_descriptors *descs = si_shader_buffer_descriptors(sctx, shader);
1250
1251 for (unsigned i = 0; i < count; ++i) {
1252 si_get_buffer_from_descriptors(
1253 buffers, descs, start_slot + i,
1254 &sbuf[i].buffer, &sbuf[i].buffer_offset,
1255 &sbuf[i].buffer_size);
1256 }
1257 }
1258
1259 /* RING BUFFERS */
1260
1261 void si_set_ring_buffer(struct pipe_context *ctx, uint slot,
1262 struct pipe_resource *buffer,
1263 unsigned stride, unsigned num_records,
1264 bool add_tid, bool swizzle,
1265 unsigned element_size, unsigned index_stride, uint64_t offset)
1266 {
1267 struct si_context *sctx = (struct si_context *)ctx;
1268 struct si_buffer_resources *buffers = &sctx->rw_buffers;
1269 struct si_descriptors *descs = &sctx->descriptors[SI_DESCS_RW_BUFFERS];
1270
1271 /* The stride field in the resource descriptor has 14 bits */
1272 assert(stride < (1 << 14));
1273
1274 assert(slot < descs->num_elements);
1275 pipe_resource_reference(&buffers->buffers[slot], NULL);
1276
1277 if (buffer) {
1278 uint64_t va;
1279
1280 va = r600_resource(buffer)->gpu_address + offset;
1281
1282 switch (element_size) {
1283 default:
1284 assert(!"Unsupported ring buffer element size");
1285 case 0:
1286 case 2:
1287 element_size = 0;
1288 break;
1289 case 4:
1290 element_size = 1;
1291 break;
1292 case 8:
1293 element_size = 2;
1294 break;
1295 case 16:
1296 element_size = 3;
1297 break;
1298 }
1299
1300 switch (index_stride) {
1301 default:
1302 assert(!"Unsupported ring buffer index stride");
1303 case 0:
1304 case 8:
1305 index_stride = 0;
1306 break;
1307 case 16:
1308 index_stride = 1;
1309 break;
1310 case 32:
1311 index_stride = 2;
1312 break;
1313 case 64:
1314 index_stride = 3;
1315 break;
1316 }
1317
1318 if (sctx->b.chip_class >= VI && stride)
1319 num_records *= stride;
1320
1321 /* Set the descriptor. */
1322 uint32_t *desc = descs->list + slot*4;
1323 desc[0] = va;
1324 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
1325 S_008F04_STRIDE(stride) |
1326 S_008F04_SWIZZLE_ENABLE(swizzle);
1327 desc[2] = num_records;
1328 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1329 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1330 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1331 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1332 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1333 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) |
1334 S_008F0C_ELEMENT_SIZE(element_size) |
1335 S_008F0C_INDEX_STRIDE(index_stride) |
1336 S_008F0C_ADD_TID_ENABLE(add_tid);
1337
1338 pipe_resource_reference(&buffers->buffers[slot], buffer);
1339 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
1340 (struct r600_resource*)buffer,
1341 buffers->shader_usage, buffers->priority);
1342 buffers->enabled_mask |= 1u << slot;
1343 } else {
1344 /* Clear the descriptor. */
1345 memset(descs->list + slot*4, 0, sizeof(uint32_t) * 4);
1346 buffers->enabled_mask &= ~(1u << slot);
1347 }
1348
1349 descs->dirty_mask |= 1u << slot;
1350 sctx->descriptors_dirty |= 1u << SI_DESCS_RW_BUFFERS;
1351 }
1352
1353 /* STREAMOUT BUFFERS */
1354
1355 static void si_set_streamout_targets(struct pipe_context *ctx,
1356 unsigned num_targets,
1357 struct pipe_stream_output_target **targets,
1358 const unsigned *offsets)
1359 {
1360 struct si_context *sctx = (struct si_context *)ctx;
1361 struct si_buffer_resources *buffers = &sctx->rw_buffers;
1362 struct si_descriptors *descs = &sctx->descriptors[SI_DESCS_RW_BUFFERS];
1363 unsigned old_num_targets = sctx->b.streamout.num_targets;
1364 unsigned i, bufidx;
1365
1366 /* We are going to unbind the buffers. Mark which caches need to be flushed. */
1367 if (sctx->b.streamout.num_targets && sctx->b.streamout.begin_emitted) {
1368 /* Since streamout uses vector writes which go through TC L2
1369 * and most other clients can use TC L2 as well, we don't need
1370 * to flush it.
1371 *
1372 * The only cases which requires flushing it is VGT DMA index
1373 * fetching (on <= CIK) and indirect draw data, which are rare
1374 * cases. Thus, flag the TC L2 dirtiness in the resource and
1375 * handle it at draw call time.
1376 */
1377 for (i = 0; i < sctx->b.streamout.num_targets; i++)
1378 if (sctx->b.streamout.targets[i])
1379 r600_resource(sctx->b.streamout.targets[i]->b.buffer)->TC_L2_dirty = true;
1380
1381 /* Invalidate the scalar cache in case a streamout buffer is
1382 * going to be used as a constant buffer.
1383 *
1384 * Invalidate TC L1, because streamout bypasses it (done by
1385 * setting GLC=1 in the store instruction), but it can contain
1386 * outdated data of streamout buffers.
1387 *
1388 * VS_PARTIAL_FLUSH is required if the buffers are going to be
1389 * used as an input immediately.
1390 */
1391 sctx->b.flags |= SI_CONTEXT_INV_SMEM_L1 |
1392 SI_CONTEXT_INV_VMEM_L1 |
1393 SI_CONTEXT_VS_PARTIAL_FLUSH;
1394 }
1395
1396 /* All readers of the streamout targets need to be finished before we can
1397 * start writing to the targets.
1398 */
1399 if (num_targets)
1400 sctx->b.flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
1401 SI_CONTEXT_CS_PARTIAL_FLUSH;
1402
1403 /* Streamout buffers must be bound in 2 places:
1404 * 1) in VGT by setting the VGT_STRMOUT registers
1405 * 2) as shader resources
1406 */
1407
1408 /* Set the VGT regs. */
1409 r600_set_streamout_targets(ctx, num_targets, targets, offsets);
1410
1411 /* Set the shader resources.*/
1412 for (i = 0; i < num_targets; i++) {
1413 bufidx = SI_VS_STREAMOUT_BUF0 + i;
1414
1415 if (targets[i]) {
1416 struct pipe_resource *buffer = targets[i]->buffer;
1417 uint64_t va = r600_resource(buffer)->gpu_address;
1418
1419 /* Set the descriptor.
1420 *
1421 * On VI, the format must be non-INVALID, otherwise
1422 * the buffer will be considered not bound and store
1423 * instructions will be no-ops.
1424 */
1425 uint32_t *desc = descs->list + bufidx*4;
1426 desc[0] = va;
1427 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32);
1428 desc[2] = 0xffffffff;
1429 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1430 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1431 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1432 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1433 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
1434
1435 /* Set the resource. */
1436 pipe_resource_reference(&buffers->buffers[bufidx],
1437 buffer);
1438 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
1439 (struct r600_resource*)buffer,
1440 buffers->shader_usage,
1441 RADEON_PRIO_SHADER_RW_BUFFER,
1442 true);
1443 r600_resource(buffer)->bind_history |= PIPE_BIND_STREAM_OUTPUT;
1444
1445 buffers->enabled_mask |= 1u << bufidx;
1446 } else {
1447 /* Clear the descriptor and unset the resource. */
1448 memset(descs->list + bufidx*4, 0,
1449 sizeof(uint32_t) * 4);
1450 pipe_resource_reference(&buffers->buffers[bufidx],
1451 NULL);
1452 buffers->enabled_mask &= ~(1u << bufidx);
1453 }
1454 descs->dirty_mask |= 1u << bufidx;
1455 }
1456 for (; i < old_num_targets; i++) {
1457 bufidx = SI_VS_STREAMOUT_BUF0 + i;
1458 /* Clear the descriptor and unset the resource. */
1459 memset(descs->list + bufidx*4, 0, sizeof(uint32_t) * 4);
1460 pipe_resource_reference(&buffers->buffers[bufidx], NULL);
1461 buffers->enabled_mask &= ~(1u << bufidx);
1462 descs->dirty_mask |= 1u << bufidx;
1463 }
1464
1465 sctx->descriptors_dirty |= 1u << SI_DESCS_RW_BUFFERS;
1466 }
1467
1468 static void si_desc_reset_buffer_offset(struct pipe_context *ctx,
1469 uint32_t *desc, uint64_t old_buf_va,
1470 struct pipe_resource *new_buf)
1471 {
1472 /* Retrieve the buffer offset from the descriptor. */
1473 uint64_t old_desc_va =
1474 desc[0] | ((uint64_t)G_008F04_BASE_ADDRESS_HI(desc[1]) << 32);
1475
1476 assert(old_buf_va <= old_desc_va);
1477 uint64_t offset_within_buffer = old_desc_va - old_buf_va;
1478
1479 /* Update the descriptor. */
1480 si_set_buf_desc_address(r600_resource(new_buf), offset_within_buffer,
1481 desc);
1482 }
1483
1484 /* INTERNAL CONST BUFFERS */
1485
1486 static void si_set_polygon_stipple(struct pipe_context *ctx,
1487 const struct pipe_poly_stipple *state)
1488 {
1489 struct si_context *sctx = (struct si_context *)ctx;
1490 struct pipe_constant_buffer cb = {};
1491 unsigned stipple[32];
1492 int i;
1493
1494 for (i = 0; i < 32; i++)
1495 stipple[i] = util_bitreverse(state->stipple[i]);
1496
1497 cb.user_buffer = stipple;
1498 cb.buffer_size = sizeof(stipple);
1499
1500 si_set_rw_buffer(sctx, SI_PS_CONST_POLY_STIPPLE, &cb);
1501 }
1502
1503 /* TEXTURE METADATA ENABLE/DISABLE */
1504
1505 /* CMASK can be enabled (for fast clear) and disabled (for texture export)
1506 * while the texture is bound, possibly by a different context. In that case,
1507 * call this function to update compressed_colortex_masks.
1508 */
1509 void si_update_compressed_colortex_masks(struct si_context *sctx)
1510 {
1511 for (int i = 0; i < SI_NUM_SHADERS; ++i) {
1512 si_samplers_update_compressed_colortex_mask(&sctx->samplers[i]);
1513 si_images_update_compressed_colortex_mask(&sctx->images[i]);
1514 si_update_compressed_tex_shader_mask(sctx, i);
1515 }
1516 }
1517
1518 /* BUFFER DISCARD/INVALIDATION */
1519
1520 /** Reset descriptors of buffer resources after \p buf has been invalidated. */
1521 static void si_reset_buffer_resources(struct si_context *sctx,
1522 struct si_buffer_resources *buffers,
1523 unsigned descriptors_idx,
1524 struct pipe_resource *buf,
1525 uint64_t old_va)
1526 {
1527 struct si_descriptors *descs = &sctx->descriptors[descriptors_idx];
1528 unsigned mask = buffers->enabled_mask;
1529
1530 while (mask) {
1531 unsigned i = u_bit_scan(&mask);
1532 if (buffers->buffers[i] == buf) {
1533 si_desc_reset_buffer_offset(&sctx->b.b,
1534 descs->list + i*4,
1535 old_va, buf);
1536 descs->dirty_mask |= 1u << i;
1537 sctx->descriptors_dirty |= 1u << descriptors_idx;
1538
1539 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
1540 (struct r600_resource *)buf,
1541 buffers->shader_usage,
1542 buffers->priority, true);
1543 }
1544 }
1545 }
1546
1547 /* Reallocate a buffer a update all resource bindings where the buffer is
1548 * bound.
1549 *
1550 * This is used to avoid CPU-GPU synchronizations, because it makes the buffer
1551 * idle by discarding its contents. Apps usually tell us when to do this using
1552 * map_buffer flags, for example.
1553 */
1554 static void si_invalidate_buffer(struct pipe_context *ctx, struct pipe_resource *buf)
1555 {
1556 struct si_context *sctx = (struct si_context*)ctx;
1557 struct r600_resource *rbuffer = r600_resource(buf);
1558 unsigned i, shader;
1559 uint64_t old_va = rbuffer->gpu_address;
1560 unsigned num_elems = sctx->vertex_elements ?
1561 sctx->vertex_elements->count : 0;
1562
1563 /* Reallocate the buffer in the same pipe_resource. */
1564 r600_alloc_resource(&sctx->screen->b, rbuffer);
1565
1566 /* We changed the buffer, now we need to bind it where the old one
1567 * was bound. This consists of 2 things:
1568 * 1) Updating the resource descriptor and dirtying it.
1569 * 2) Adding a relocation to the CS, so that it's usable.
1570 */
1571
1572 /* Vertex buffers. */
1573 if (rbuffer->bind_history & PIPE_BIND_VERTEX_BUFFER) {
1574 for (i = 0; i < num_elems; i++) {
1575 int vb = sctx->vertex_elements->elements[i].vertex_buffer_index;
1576
1577 if (vb >= ARRAY_SIZE(sctx->vertex_buffer))
1578 continue;
1579 if (!sctx->vertex_buffer[vb].buffer)
1580 continue;
1581
1582 if (sctx->vertex_buffer[vb].buffer == buf) {
1583 sctx->vertex_buffers_dirty = true;
1584 break;
1585 }
1586 }
1587 }
1588
1589 /* Streamout buffers. (other internal buffers can't be invalidated) */
1590 if (rbuffer->bind_history & PIPE_BIND_STREAM_OUTPUT) {
1591 for (i = SI_VS_STREAMOUT_BUF0; i <= SI_VS_STREAMOUT_BUF3; i++) {
1592 struct si_buffer_resources *buffers = &sctx->rw_buffers;
1593 struct si_descriptors *descs =
1594 &sctx->descriptors[SI_DESCS_RW_BUFFERS];
1595
1596 if (buffers->buffers[i] != buf)
1597 continue;
1598
1599 si_desc_reset_buffer_offset(ctx, descs->list + i*4,
1600 old_va, buf);
1601 descs->dirty_mask |= 1u << i;
1602 sctx->descriptors_dirty |= 1u << SI_DESCS_RW_BUFFERS;
1603
1604 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
1605 rbuffer, buffers->shader_usage,
1606 RADEON_PRIO_SHADER_RW_BUFFER,
1607 true);
1608
1609 /* Update the streamout state. */
1610 if (sctx->b.streamout.begin_emitted)
1611 r600_emit_streamout_end(&sctx->b);
1612 sctx->b.streamout.append_bitmask =
1613 sctx->b.streamout.enabled_mask;
1614 r600_streamout_buffers_dirty(&sctx->b);
1615 }
1616 }
1617
1618 /* Constant and shader buffers. */
1619 if (rbuffer->bind_history & PIPE_BIND_CONSTANT_BUFFER) {
1620 for (shader = 0; shader < SI_NUM_SHADERS; shader++)
1621 si_reset_buffer_resources(sctx, &sctx->const_buffers[shader],
1622 si_const_buffer_descriptors_idx(shader),
1623 buf, old_va);
1624 }
1625
1626 if (rbuffer->bind_history & PIPE_BIND_SHADER_BUFFER) {
1627 for (shader = 0; shader < SI_NUM_SHADERS; shader++)
1628 si_reset_buffer_resources(sctx, &sctx->shader_buffers[shader],
1629 si_shader_buffer_descriptors_idx(shader),
1630 buf, old_va);
1631 }
1632
1633 if (rbuffer->bind_history & PIPE_BIND_SAMPLER_VIEW) {
1634 /* Texture buffers - update bindings. */
1635 for (shader = 0; shader < SI_NUM_SHADERS; shader++) {
1636 struct si_sampler_views *views = &sctx->samplers[shader].views;
1637 struct si_descriptors *descs =
1638 si_sampler_descriptors(sctx, shader);
1639 unsigned mask = views->enabled_mask;
1640
1641 while (mask) {
1642 unsigned i = u_bit_scan(&mask);
1643 if (views->views[i]->texture == buf) {
1644 si_desc_reset_buffer_offset(ctx,
1645 descs->list +
1646 i * 16 + 4,
1647 old_va, buf);
1648 descs->dirty_mask |= 1u << i;
1649 sctx->descriptors_dirty |=
1650 1u << si_sampler_descriptors_idx(shader);
1651
1652 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
1653 rbuffer, RADEON_USAGE_READ,
1654 RADEON_PRIO_SAMPLER_BUFFER,
1655 true);
1656 }
1657 }
1658 }
1659 }
1660
1661 /* Shader images */
1662 if (rbuffer->bind_history & PIPE_BIND_SHADER_IMAGE) {
1663 for (shader = 0; shader < SI_NUM_SHADERS; ++shader) {
1664 struct si_images_info *images = &sctx->images[shader];
1665 struct si_descriptors *descs =
1666 si_image_descriptors(sctx, shader);
1667 unsigned mask = images->enabled_mask;
1668
1669 while (mask) {
1670 unsigned i = u_bit_scan(&mask);
1671
1672 if (images->views[i].resource == buf) {
1673 if (images->views[i].access & PIPE_IMAGE_ACCESS_WRITE)
1674 si_mark_image_range_valid(&images->views[i]);
1675
1676 si_desc_reset_buffer_offset(
1677 ctx, descs->list + i * 8 + 4,
1678 old_va, buf);
1679 descs->dirty_mask |= 1u << i;
1680 sctx->descriptors_dirty |=
1681 1u << si_image_descriptors_idx(shader);
1682
1683 radeon_add_to_buffer_list_check_mem(
1684 &sctx->b, &sctx->b.gfx, rbuffer,
1685 RADEON_USAGE_READWRITE,
1686 RADEON_PRIO_SAMPLER_BUFFER, true);
1687 }
1688 }
1689 }
1690 }
1691 }
1692
1693 /* Update mutable image descriptor fields of all bound textures. */
1694 void si_update_all_texture_descriptors(struct si_context *sctx)
1695 {
1696 unsigned shader;
1697
1698 for (shader = 0; shader < SI_NUM_SHADERS; shader++) {
1699 struct si_sampler_views *samplers = &sctx->samplers[shader].views;
1700 struct si_images_info *images = &sctx->images[shader];
1701 unsigned mask;
1702
1703 /* Images. */
1704 mask = images->enabled_mask;
1705 while (mask) {
1706 unsigned i = u_bit_scan(&mask);
1707 struct pipe_image_view *view = &images->views[i];
1708
1709 if (!view->resource ||
1710 view->resource->target == PIPE_BUFFER)
1711 continue;
1712
1713 si_set_shader_image(sctx, shader, i, view, true);
1714 }
1715
1716 /* Sampler views. */
1717 mask = samplers->enabled_mask;
1718 while (mask) {
1719 unsigned i = u_bit_scan(&mask);
1720 struct pipe_sampler_view *view = samplers->views[i];
1721
1722 if (!view ||
1723 !view->texture ||
1724 view->texture->target == PIPE_BUFFER)
1725 continue;
1726
1727 si_set_sampler_view(sctx, shader, i,
1728 samplers->views[i], true);
1729 }
1730
1731 si_update_compressed_tex_shader_mask(sctx, shader);
1732 }
1733 }
1734
1735 /* SHADER USER DATA */
1736
1737 static void si_mark_shader_pointers_dirty(struct si_context *sctx,
1738 unsigned shader)
1739 {
1740 sctx->shader_pointers_dirty |=
1741 u_bit_consecutive(SI_DESCS_FIRST_SHADER + shader * SI_NUM_SHADER_DESCS,
1742 SI_NUM_SHADER_DESCS);
1743
1744 if (shader == PIPE_SHADER_VERTEX)
1745 sctx->vertex_buffer_pointer_dirty = sctx->vertex_buffers.buffer != NULL;
1746
1747 si_mark_atom_dirty(sctx, &sctx->shader_userdata.atom);
1748 }
1749
1750 static void si_shader_userdata_begin_new_cs(struct si_context *sctx)
1751 {
1752 sctx->shader_pointers_dirty = u_bit_consecutive(0, SI_NUM_DESCS);
1753 sctx->vertex_buffer_pointer_dirty = sctx->vertex_buffers.buffer != NULL;
1754 si_mark_atom_dirty(sctx, &sctx->shader_userdata.atom);
1755 }
1756
1757 /* Set a base register address for user data constants in the given shader.
1758 * This assigns a mapping from PIPE_SHADER_* to SPI_SHADER_USER_DATA_*.
1759 */
1760 static void si_set_user_data_base(struct si_context *sctx,
1761 unsigned shader, uint32_t new_base)
1762 {
1763 uint32_t *base = &sctx->shader_userdata.sh_base[shader];
1764
1765 if (*base != new_base) {
1766 *base = new_base;
1767
1768 if (new_base)
1769 si_mark_shader_pointers_dirty(sctx, shader);
1770 }
1771 }
1772
1773 /* This must be called when these shaders are changed from non-NULL to NULL
1774 * and vice versa:
1775 * - geometry shader
1776 * - tessellation control shader
1777 * - tessellation evaluation shader
1778 */
1779 void si_shader_change_notify(struct si_context *sctx)
1780 {
1781 /* VS can be bound as VS, ES, or LS. */
1782 if (sctx->tes_shader.cso)
1783 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX,
1784 R_00B530_SPI_SHADER_USER_DATA_LS_0);
1785 else if (sctx->gs_shader.cso)
1786 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX,
1787 R_00B330_SPI_SHADER_USER_DATA_ES_0);
1788 else
1789 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX,
1790 R_00B130_SPI_SHADER_USER_DATA_VS_0);
1791
1792 /* TES can be bound as ES, VS, or not bound. */
1793 if (sctx->tes_shader.cso) {
1794 if (sctx->gs_shader.cso)
1795 si_set_user_data_base(sctx, PIPE_SHADER_TESS_EVAL,
1796 R_00B330_SPI_SHADER_USER_DATA_ES_0);
1797 else
1798 si_set_user_data_base(sctx, PIPE_SHADER_TESS_EVAL,
1799 R_00B130_SPI_SHADER_USER_DATA_VS_0);
1800 } else {
1801 si_set_user_data_base(sctx, PIPE_SHADER_TESS_EVAL, 0);
1802 }
1803 }
1804
1805 static void si_emit_shader_pointer(struct si_context *sctx,
1806 struct si_descriptors *desc,
1807 unsigned sh_base)
1808 {
1809 struct radeon_winsys_cs *cs = sctx->b.gfx.cs;
1810 uint64_t va;
1811
1812 assert(desc->buffer);
1813
1814 va = desc->buffer->gpu_address +
1815 desc->buffer_offset;
1816
1817 radeon_emit(cs, PKT3(PKT3_SET_SH_REG, 2, 0));
1818 radeon_emit(cs, (sh_base + desc->shader_userdata_offset - SI_SH_REG_OFFSET) >> 2);
1819 radeon_emit(cs, va);
1820 radeon_emit(cs, va >> 32);
1821 }
1822
1823 void si_emit_graphics_shader_userdata(struct si_context *sctx,
1824 struct r600_atom *atom)
1825 {
1826 unsigned mask;
1827 uint32_t *sh_base = sctx->shader_userdata.sh_base;
1828 struct si_descriptors *descs;
1829
1830 descs = &sctx->descriptors[SI_DESCS_RW_BUFFERS];
1831
1832 if (sctx->shader_pointers_dirty & (1 << SI_DESCS_RW_BUFFERS)) {
1833 si_emit_shader_pointer(sctx, descs,
1834 R_00B030_SPI_SHADER_USER_DATA_PS_0);
1835 si_emit_shader_pointer(sctx, descs,
1836 R_00B130_SPI_SHADER_USER_DATA_VS_0);
1837 si_emit_shader_pointer(sctx, descs,
1838 R_00B230_SPI_SHADER_USER_DATA_GS_0);
1839 si_emit_shader_pointer(sctx, descs,
1840 R_00B330_SPI_SHADER_USER_DATA_ES_0);
1841 si_emit_shader_pointer(sctx, descs,
1842 R_00B430_SPI_SHADER_USER_DATA_HS_0);
1843 }
1844
1845 mask = sctx->shader_pointers_dirty &
1846 u_bit_consecutive(SI_DESCS_FIRST_SHADER,
1847 SI_DESCS_FIRST_COMPUTE - SI_DESCS_FIRST_SHADER);
1848
1849 while (mask) {
1850 unsigned i = u_bit_scan(&mask);
1851 unsigned shader = (i - SI_DESCS_FIRST_SHADER) / SI_NUM_SHADER_DESCS;
1852 unsigned base = sh_base[shader];
1853
1854 if (base)
1855 si_emit_shader_pointer(sctx, descs + i, base);
1856 }
1857 sctx->shader_pointers_dirty &=
1858 ~u_bit_consecutive(SI_DESCS_RW_BUFFERS, SI_DESCS_FIRST_COMPUTE);
1859
1860 if (sctx->vertex_buffer_pointer_dirty) {
1861 si_emit_shader_pointer(sctx, &sctx->vertex_buffers,
1862 sh_base[PIPE_SHADER_VERTEX]);
1863 sctx->vertex_buffer_pointer_dirty = false;
1864 }
1865 }
1866
1867 void si_emit_compute_shader_userdata(struct si_context *sctx)
1868 {
1869 unsigned base = R_00B900_COMPUTE_USER_DATA_0;
1870 struct si_descriptors *descs = sctx->descriptors;
1871 unsigned compute_mask =
1872 u_bit_consecutive(SI_DESCS_FIRST_COMPUTE, SI_NUM_SHADER_DESCS);
1873 unsigned mask = sctx->shader_pointers_dirty & compute_mask;
1874
1875 while (mask) {
1876 unsigned i = u_bit_scan(&mask);
1877
1878 si_emit_shader_pointer(sctx, descs + i, base);
1879 }
1880 sctx->shader_pointers_dirty &= ~compute_mask;
1881 }
1882
1883 /* INIT/DEINIT/UPLOAD */
1884
1885 void si_init_all_descriptors(struct si_context *sctx)
1886 {
1887 int i;
1888 unsigned ce_offset = 0;
1889
1890 for (i = 0; i < SI_NUM_SHADERS; i++) {
1891 si_init_buffer_resources(&sctx->const_buffers[i],
1892 si_const_buffer_descriptors(sctx, i),
1893 SI_NUM_CONST_BUFFERS, SI_SGPR_CONST_BUFFERS,
1894 RADEON_USAGE_READ, RADEON_PRIO_CONST_BUFFER,
1895 &ce_offset);
1896 si_init_buffer_resources(&sctx->shader_buffers[i],
1897 si_shader_buffer_descriptors(sctx, i),
1898 SI_NUM_SHADER_BUFFERS, SI_SGPR_SHADER_BUFFERS,
1899 RADEON_USAGE_READWRITE, RADEON_PRIO_SHADER_RW_BUFFER,
1900 &ce_offset);
1901
1902 si_init_descriptors(si_sampler_descriptors(sctx, i),
1903 SI_SGPR_SAMPLERS, 16, SI_NUM_SAMPLERS,
1904 null_texture_descriptor, &ce_offset);
1905
1906 si_init_descriptors(si_image_descriptors(sctx, i),
1907 SI_SGPR_IMAGES, 8, SI_NUM_IMAGES,
1908 null_image_descriptor, &ce_offset);
1909 }
1910
1911 si_init_buffer_resources(&sctx->rw_buffers,
1912 &sctx->descriptors[SI_DESCS_RW_BUFFERS],
1913 SI_NUM_RW_BUFFERS, SI_SGPR_RW_BUFFERS,
1914 RADEON_USAGE_READWRITE, RADEON_PRIO_SHADER_RINGS,
1915 &ce_offset);
1916 si_init_descriptors(&sctx->vertex_buffers, SI_SGPR_VERTEX_BUFFERS,
1917 4, SI_NUM_VERTEX_BUFFERS, NULL, NULL);
1918
1919 sctx->descriptors_dirty = u_bit_consecutive(0, SI_NUM_DESCS);
1920
1921 assert(ce_offset <= 32768);
1922
1923 /* Set pipe_context functions. */
1924 sctx->b.b.bind_sampler_states = si_bind_sampler_states;
1925 sctx->b.b.set_shader_images = si_set_shader_images;
1926 sctx->b.b.set_constant_buffer = si_pipe_set_constant_buffer;
1927 sctx->b.b.set_polygon_stipple = si_set_polygon_stipple;
1928 sctx->b.b.set_shader_buffers = si_set_shader_buffers;
1929 sctx->b.b.set_sampler_views = si_set_sampler_views;
1930 sctx->b.b.set_stream_output_targets = si_set_streamout_targets;
1931 sctx->b.invalidate_buffer = si_invalidate_buffer;
1932
1933 /* Shader user data. */
1934 si_init_atom(sctx, &sctx->shader_userdata.atom, &sctx->atoms.s.shader_userdata,
1935 si_emit_graphics_shader_userdata);
1936
1937 /* Set default and immutable mappings. */
1938 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX, R_00B130_SPI_SHADER_USER_DATA_VS_0);
1939 si_set_user_data_base(sctx, PIPE_SHADER_TESS_CTRL, R_00B430_SPI_SHADER_USER_DATA_HS_0);
1940 si_set_user_data_base(sctx, PIPE_SHADER_GEOMETRY, R_00B230_SPI_SHADER_USER_DATA_GS_0);
1941 si_set_user_data_base(sctx, PIPE_SHADER_FRAGMENT, R_00B030_SPI_SHADER_USER_DATA_PS_0);
1942 }
1943
1944 bool si_upload_graphics_shader_descriptors(struct si_context *sctx)
1945 {
1946 const unsigned mask = u_bit_consecutive(0, SI_DESCS_FIRST_COMPUTE);
1947 unsigned dirty = sctx->descriptors_dirty & mask;
1948
1949 /* Assume nothing will go wrong: */
1950 sctx->shader_pointers_dirty |= dirty;
1951
1952 while (dirty) {
1953 unsigned i = u_bit_scan(&dirty);
1954
1955 if (!si_upload_descriptors(sctx, &sctx->descriptors[i],
1956 &sctx->shader_userdata.atom))
1957 return false;
1958 }
1959
1960 sctx->descriptors_dirty &= ~mask;
1961 return true;
1962 }
1963
1964 bool si_upload_compute_shader_descriptors(struct si_context *sctx)
1965 {
1966 /* Does not update rw_buffers as that is not needed for compute shaders
1967 * and the input buffer is using the same SGPR's anyway.
1968 */
1969 const unsigned mask = u_bit_consecutive(SI_DESCS_FIRST_COMPUTE,
1970 SI_NUM_DESCS - SI_DESCS_FIRST_COMPUTE);
1971 unsigned dirty = sctx->descriptors_dirty & mask;
1972
1973 /* Assume nothing will go wrong: */
1974 sctx->shader_pointers_dirty |= dirty;
1975
1976 while (dirty) {
1977 unsigned i = u_bit_scan(&dirty);
1978
1979 if (!si_upload_descriptors(sctx, &sctx->descriptors[i], NULL))
1980 return false;
1981 }
1982
1983 sctx->descriptors_dirty &= ~mask;
1984
1985 return true;
1986 }
1987
1988 void si_release_all_descriptors(struct si_context *sctx)
1989 {
1990 int i;
1991
1992 for (i = 0; i < SI_NUM_SHADERS; i++) {
1993 si_release_buffer_resources(&sctx->const_buffers[i],
1994 si_const_buffer_descriptors(sctx, i));
1995 si_release_buffer_resources(&sctx->shader_buffers[i],
1996 si_shader_buffer_descriptors(sctx, i));
1997 si_release_sampler_views(&sctx->samplers[i].views);
1998 si_release_image_views(&sctx->images[i]);
1999 }
2000 si_release_buffer_resources(&sctx->rw_buffers,
2001 &sctx->descriptors[SI_DESCS_RW_BUFFERS]);
2002
2003 for (i = 0; i < SI_NUM_DESCS; ++i)
2004 si_release_descriptors(&sctx->descriptors[i]);
2005 si_release_descriptors(&sctx->vertex_buffers);
2006 }
2007
2008 void si_all_descriptors_begin_new_cs(struct si_context *sctx)
2009 {
2010 int i;
2011
2012 for (i = 0; i < SI_NUM_SHADERS; i++) {
2013 si_buffer_resources_begin_new_cs(sctx, &sctx->const_buffers[i]);
2014 si_buffer_resources_begin_new_cs(sctx, &sctx->shader_buffers[i]);
2015 si_sampler_views_begin_new_cs(sctx, &sctx->samplers[i].views);
2016 si_image_views_begin_new_cs(sctx, &sctx->images[i]);
2017 }
2018 si_buffer_resources_begin_new_cs(sctx, &sctx->rw_buffers);
2019 si_vertex_buffers_begin_new_cs(sctx);
2020
2021 for (i = 0; i < SI_NUM_DESCS; ++i)
2022 si_descriptors_begin_new_cs(sctx, &sctx->descriptors[i]);
2023
2024 si_shader_userdata_begin_new_cs(sctx);
2025 }