projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
radeonsi: don't invoke DCC decompression in update_all_texture_descriptors
[mesa.git] / src / gallium / drivers / radeonsi / si_descriptors.c
1 /*
2 * Copyright 2013 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 *
23 * Authors:
24 * Marek Olšák <marek.olsak@amd.com>
25 */
26
27 /* Resource binding slots and sampler states (each described with 8 or
28 * 4 dwords) are stored in lists in memory which is accessed by shaders
29 * using scalar load instructions.
30 *
31 * This file is responsible for managing such lists. It keeps a copy of all
32 * descriptors in CPU memory and re-uploads a whole list if some slots have
33 * been changed.
34 *
35 * This code is also reponsible for updating shader pointers to those lists.
36 *
37 * Note that CP DMA can't be used for updating the lists, because a GPU hang
38 * could leave the list in a mid-IB state and the next IB would get wrong
39 * descriptors and the whole context would be unusable at that point.
40 * (Note: The register shadowing can't be used due to the same reason)
41 *
42 * Also, uploading descriptors to newly allocated memory doesn't require
43 * a KCACHE flush.
44 *
45 *
46 * Possible scenarios for one 16 dword image+sampler slot:
47 *
48 * | Image | w/ FMASK | Buffer | NULL
49 * [ 0: 3] Image[0:3] | Image[0:3] | Null[0:3] | Null[0:3]
50 * [ 4: 7] Image[4:7] | Image[4:7] | Buffer[0:3] | 0
51 * [ 8:11] Null[0:3] | Fmask[0:3] | Null[0:3] | Null[0:3]
52 * [12:15] Sampler[0:3] | Fmask[4:7] | Sampler[0:3] | Sampler[0:3]
53 *
54 * FMASK implies MSAA, therefore no sampler state.
55 * Sampler states are never unbound except when FMASK is bound.
56 */
57
58 #include "radeon/r600_cs.h"
59 #include "si_pipe.h"
60 #include "sid.h"
61
62 #include "util/u_format.h"
63 #include "util/u_memory.h"
64 #include "util/u_upload_mgr.h"
65
66
67 /* NULL image and buffer descriptor for textures (alpha = 1) and images
68 * (alpha = 0).
69 *
70 * For images, all fields must be zero except for the swizzle, which
71 * supports arbitrary combinations of 0s and 1s. The texture type must be
72 * any valid type (e.g. 1D). If the texture type isn't set, the hw hangs.
73 *
74 * For buffers, all fields must be zero. If they are not, the hw hangs.
75 *
76 * This is the only reason why the buffer descriptor must be in words [4:7].
77 */
78 static uint32_t null_texture_descriptor[8] = {
79 0,
80 0,
81 0,
82 S_008F1C_DST_SEL_W(V_008F1C_SQ_SEL_1) |
83 S_008F1C_TYPE(V_008F1C_SQ_RSRC_IMG_1D)
84 /* the rest must contain zeros, which is also used by the buffer
85 * descriptor */
86 };
87
88 static uint32_t null_image_descriptor[8] = {
89 0,
90 0,
91 0,
92 S_008F1C_TYPE(V_008F1C_SQ_RSRC_IMG_1D)
93 /* the rest must contain zeros, which is also used by the buffer
94 * descriptor */
95 };
96
97 static void si_init_descriptors(struct si_descriptors *desc,
98 unsigned shader_userdata_index,
99 unsigned element_dw_size,
100 unsigned num_elements,
101 const uint32_t *null_descriptor,
102 unsigned *ce_offset)
103 {
104 int i;
105
106 assert(num_elements <= sizeof(desc->dirty_mask)*8);
107
108 desc->list = CALLOC(num_elements, element_dw_size * 4);
109 desc->element_dw_size = element_dw_size;
110 desc->num_elements = num_elements;
111 desc->dirty_mask = num_elements == 32 ? ~0u : (1u << num_elements) - 1;
112 desc->shader_userdata_offset = shader_userdata_index * 4;
113
114 if (ce_offset) {
115 desc->ce_offset = *ce_offset;
116
117 /* make sure that ce_offset stays 32 byte aligned */
118 *ce_offset += align(element_dw_size * num_elements * 4, 32);
119 }
120
121 /* Initialize the array to NULL descriptors if the element size is 8. */
122 if (null_descriptor) {
123 assert(element_dw_size % 8 == 0);
124 for (i = 0; i < num_elements * element_dw_size / 8; i++)
125 memcpy(desc->list + i * 8, null_descriptor,
126 8 * 4);
127 }
128 }
129
130 static void si_release_descriptors(struct si_descriptors *desc)
131 {
132 r600_resource_reference(&desc->buffer, NULL);
133 FREE(desc->list);
134 }
135
136 static bool si_ce_upload(struct si_context *sctx, unsigned ce_offset, unsigned size,
137 unsigned *out_offset, struct r600_resource **out_buf) {
138 uint64_t va;
139
140 u_suballocator_alloc(sctx->ce_suballocator, size, 64, out_offset,
141 (struct pipe_resource**)out_buf);
142 if (!out_buf)
143 return false;
144
145 va = (*out_buf)->gpu_address + *out_offset;
146
147 radeon_emit(sctx->ce_ib, PKT3(PKT3_DUMP_CONST_RAM, 3, 0));
148 radeon_emit(sctx->ce_ib, ce_offset);
149 radeon_emit(sctx->ce_ib, size / 4);
150 radeon_emit(sctx->ce_ib, va);
151 radeon_emit(sctx->ce_ib, va >> 32);
152
153 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx, *out_buf,
154 RADEON_USAGE_READWRITE, RADEON_PRIO_DESCRIPTORS);
155
156 sctx->ce_need_synchronization = true;
157 return true;
158 }
159
160 static void si_ce_reinitialize_descriptors(struct si_context *sctx,
161 struct si_descriptors *desc)
162 {
163 if (desc->buffer) {
164 struct r600_resource *buffer = (struct r600_resource*)desc->buffer;
165 unsigned list_size = desc->num_elements * desc->element_dw_size * 4;
166 uint64_t va = buffer->gpu_address + desc->buffer_offset;
167 struct radeon_winsys_cs *ib = sctx->ce_preamble_ib;
168
169 if (!ib)
170 ib = sctx->ce_ib;
171
172 list_size = align(list_size, 32);
173
174 radeon_emit(ib, PKT3(PKT3_LOAD_CONST_RAM, 3, 0));
175 radeon_emit(ib, va);
176 radeon_emit(ib, va >> 32);
177 radeon_emit(ib, list_size / 4);
178 radeon_emit(ib, desc->ce_offset);
179
180 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx, desc->buffer,
181 RADEON_USAGE_READ, RADEON_PRIO_DESCRIPTORS);
182 }
183 desc->ce_ram_dirty = false;
184 }
185
186 void si_ce_reinitialize_all_descriptors(struct si_context *sctx)
187 {
188 int i;
189
190 for (i = 0; i < SI_NUM_DESCS; ++i)
191 si_ce_reinitialize_descriptors(sctx, &sctx->descriptors[i]);
192 }
193
194 void si_ce_enable_loads(struct radeon_winsys_cs *ib)
195 {
196 radeon_emit(ib, PKT3(PKT3_CONTEXT_CONTROL, 1, 0));
197 radeon_emit(ib, CONTEXT_CONTROL_LOAD_ENABLE(1) |
198 CONTEXT_CONTROL_LOAD_CE_RAM(1));
199 radeon_emit(ib, CONTEXT_CONTROL_SHADOW_ENABLE(1));
200 }
201
202 static bool si_upload_descriptors(struct si_context *sctx,
203 struct si_descriptors *desc,
204 struct r600_atom * atom)
205 {
206 unsigned list_size = desc->num_elements * desc->element_dw_size * 4;
207
208 if (!desc->dirty_mask)
209 return true;
210
211 if (sctx->ce_ib) {
212 uint32_t const* list = (uint32_t const*)desc->list;
213
214 if (desc->ce_ram_dirty)
215 si_ce_reinitialize_descriptors(sctx, desc);
216
217 while(desc->dirty_mask) {
218 int begin, count;
219 u_bit_scan_consecutive_range(&desc->dirty_mask, &begin,
220 &count);
221
222 begin *= desc->element_dw_size;
223 count *= desc->element_dw_size;
224
225 radeon_emit(sctx->ce_ib,
226 PKT3(PKT3_WRITE_CONST_RAM, count, 0));
227 radeon_emit(sctx->ce_ib, desc->ce_offset + begin * 4);
228 radeon_emit_array(sctx->ce_ib, list + begin, count);
229 }
230
231 if (!si_ce_upload(sctx, desc->ce_offset, list_size,
232 &desc->buffer_offset, &desc->buffer))
233 return false;
234 } else {
235 void *ptr;
236
237 u_upload_alloc(sctx->b.uploader, 0, list_size, 256,
238 &desc->buffer_offset,
239 (struct pipe_resource**)&desc->buffer, &ptr);
240 if (!desc->buffer)
241 return false; /* skip the draw call */
242
243 util_memcpy_cpu_to_le32(ptr, desc->list, list_size);
244 desc->gpu_list = ptr;
245
246 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx, desc->buffer,
247 RADEON_USAGE_READ, RADEON_PRIO_DESCRIPTORS);
248 }
249 desc->dirty_mask = 0;
250
251 if (atom)
252 si_mark_atom_dirty(sctx, atom);
253
254 return true;
255 }
256
257 static void
258 si_descriptors_begin_new_cs(struct si_context *sctx, struct si_descriptors *desc)
259 {
260 desc->ce_ram_dirty = true;
261
262 if (!desc->buffer)
263 return;
264
265 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx, desc->buffer,
266 RADEON_USAGE_READ, RADEON_PRIO_DESCRIPTORS);
267 }
268
269 /* SAMPLER VIEWS */
270
271 static unsigned
272 si_sampler_descriptors_idx(unsigned shader)
273 {
274 return SI_DESCS_FIRST_SHADER + shader * SI_NUM_SHADER_DESCS +
275 SI_SHADER_DESCS_SAMPLERS;
276 }
277
278 static struct si_descriptors *
279 si_sampler_descriptors(struct si_context *sctx, unsigned shader)
280 {
281 return &sctx->descriptors[si_sampler_descriptors_idx(shader)];
282 }
283
284 static void si_release_sampler_views(struct si_sampler_views *views)
285 {
286 int i;
287
288 for (i = 0; i < ARRAY_SIZE(views->views); i++) {
289 pipe_sampler_view_reference(&views->views[i], NULL);
290 }
291 }
292
293 static void si_sampler_view_add_buffer(struct si_context *sctx,
294 struct pipe_resource *resource,
295 enum radeon_bo_usage usage,
296 bool is_stencil_sampler,
297 bool check_mem)
298 {
299 struct r600_resource *rres;
300 struct r600_texture *rtex;
301 enum radeon_bo_priority priority;
302
303 if (!resource)
304 return;
305
306 if (resource->target != PIPE_BUFFER) {
307 struct r600_texture *tex = (struct r600_texture*)resource;
308
309 if (tex->is_depth && !r600_can_sample_zs(tex, is_stencil_sampler))
310 resource = &tex->flushed_depth_texture->resource.b.b;
311 }
312
313 rres = (struct r600_resource*)resource;
314 priority = r600_get_sampler_view_priority(rres);
315
316 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
317 rres, usage, priority,
318 check_mem);
319
320 if (resource->target == PIPE_BUFFER)
321 return;
322
323 /* Now add separate DCC or HTILE. */
324 rtex = (struct r600_texture*)resource;
325 if (rtex->dcc_separate_buffer) {
326 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
327 rtex->dcc_separate_buffer, usage,
328 RADEON_PRIO_DCC, check_mem);
329 }
330
331 if (rtex->htile_buffer &&
332 rtex->tc_compatible_htile &&
333 !is_stencil_sampler) {
334 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
335 rtex->htile_buffer, usage,
336 RADEON_PRIO_HTILE, check_mem);
337 }
338 }
339
340 static void si_sampler_views_begin_new_cs(struct si_context *sctx,
341 struct si_sampler_views *views)
342 {
343 unsigned mask = views->enabled_mask;
344
345 /* Add buffers to the CS. */
346 while (mask) {
347 int i = u_bit_scan(&mask);
348 struct si_sampler_view *sview = (struct si_sampler_view *)views->views[i];
349
350 si_sampler_view_add_buffer(sctx, sview->base.texture,
351 RADEON_USAGE_READ,
352 sview->is_stencil_sampler, false);
353 }
354 }
355
356 /* Set buffer descriptor fields that can be changed by reallocations. */
357 static void si_set_buf_desc_address(struct r600_resource *buf,
358 uint64_t offset, uint32_t *state)
359 {
360 uint64_t va = buf->gpu_address + offset;
361
362 state[0] = va;
363 state[1] &= C_008F04_BASE_ADDRESS_HI;
364 state[1] |= S_008F04_BASE_ADDRESS_HI(va >> 32);
365 }
366
367 /* Set texture descriptor fields that can be changed by reallocations.
368 *
369 * \param tex texture
370 * \param base_level_info information of the level of BASE_ADDRESS
371 * \param base_level the level of BASE_ADDRESS
372 * \param first_level pipe_sampler_view.u.tex.first_level
373 * \param block_width util_format_get_blockwidth()
374 * \param is_stencil select between separate Z & Stencil
375 * \param state descriptor to update
376 */
377 void si_set_mutable_tex_desc_fields(struct r600_texture *tex,
378 const struct radeon_surf_level *base_level_info,
379 unsigned base_level, unsigned first_level,
380 unsigned block_width, bool is_stencil,
381 uint32_t *state)
382 {
383 uint64_t va;
384 unsigned pitch = base_level_info->nblk_x * block_width;
385
386 if (tex->is_depth && !r600_can_sample_zs(tex, is_stencil)) {
387 tex = tex->flushed_depth_texture;
388 is_stencil = false;
389 }
390
391 va = tex->resource.gpu_address + base_level_info->offset;
392
393 state[1] &= C_008F14_BASE_ADDRESS_HI;
394 state[3] &= C_008F1C_TILING_INDEX;
395 state[4] &= C_008F20_PITCH;
396 state[6] &= C_008F28_COMPRESSION_EN;
397
398 state[0] = va >> 8;
399 state[1] |= S_008F14_BASE_ADDRESS_HI(va >> 40);
400 state[3] |= S_008F1C_TILING_INDEX(si_tile_mode_index(tex, base_level,
401 is_stencil));
402 state[4] |= S_008F20_PITCH(pitch - 1);
403
404 if (tex->dcc_offset && first_level < tex->surface.num_dcc_levels) {
405 state[6] |= S_008F28_COMPRESSION_EN(1);
406 state[7] = ((!tex->dcc_separate_buffer ? tex->resource.gpu_address : 0) +
407 tex->dcc_offset +
408 base_level_info->dcc_offset) >> 8;
409 } else if (tex->tc_compatible_htile) {
410 state[6] |= S_008F28_COMPRESSION_EN(1);
411 state[7] = tex->htile_buffer->gpu_address >> 8;
412 }
413 }
414
415 static void si_set_sampler_view(struct si_context *sctx,
416 unsigned shader,
417 unsigned slot, struct pipe_sampler_view *view,
418 bool disallow_early_out)
419 {
420 struct si_sampler_views *views = &sctx->samplers[shader].views;
421 struct si_sampler_view *rview = (struct si_sampler_view*)view;
422 struct si_descriptors *descs = si_sampler_descriptors(sctx, shader);
423 uint32_t *desc = descs->list + slot * 16;
424
425 if (views->views[slot] == view && !disallow_early_out)
426 return;
427
428 if (view) {
429 struct r600_texture *rtex = (struct r600_texture *)view->texture;
430
431 assert(rtex); /* views with texture == NULL aren't supported */
432 pipe_sampler_view_reference(&views->views[slot], view);
433 memcpy(desc, rview->state, 8*4);
434
435 if (rtex->resource.b.b.target == PIPE_BUFFER) {
436 rtex->resource.bind_history |= PIPE_BIND_SAMPLER_VIEW;
437
438 si_set_buf_desc_address(&rtex->resource,
439 view->u.buf.offset,
440 desc + 4);
441 } else {
442 bool is_separate_stencil =
443 rtex->db_compatible &&
444 rview->is_stencil_sampler;
445
446 si_set_mutable_tex_desc_fields(rtex,
447 rview->base_level_info,
448 rview->base_level,
449 rview->base.u.tex.first_level,
450 rview->block_width,
451 is_separate_stencil,
452 desc);
453 }
454
455 if (rtex->resource.b.b.target != PIPE_BUFFER &&
456 rtex->fmask.size) {
457 memcpy(desc + 8,
458 rview->fmask_state, 8*4);
459 } else {
460 /* Disable FMASK and bind sampler state in [12:15]. */
461 memcpy(desc + 8,
462 null_texture_descriptor, 4*4);
463
464 if (views->sampler_states[slot])
465 memcpy(desc + 12,
466 views->sampler_states[slot]->val, 4*4);
467 }
468
469 views->enabled_mask |= 1u << slot;
470
471 /* Since this can flush, it must be done after enabled_mask is
472 * updated. */
473 si_sampler_view_add_buffer(sctx, view->texture,
474 RADEON_USAGE_READ,
475 rview->is_stencil_sampler, true);
476 } else {
477 pipe_sampler_view_reference(&views->views[slot], NULL);
478 memcpy(desc, null_texture_descriptor, 8*4);
479 /* Only clear the lower dwords of FMASK. */
480 memcpy(desc + 8, null_texture_descriptor, 4*4);
481 /* Re-set the sampler state if we are transitioning from FMASK. */
482 if (views->sampler_states[slot])
483 memcpy(desc + 12,
484 views->sampler_states[slot]->val, 4*4);
485
486 views->enabled_mask &= ~(1u << slot);
487 }
488
489 descs->dirty_mask |= 1u << slot;
490 sctx->descriptors_dirty |= 1u << si_sampler_descriptors_idx(shader);
491 }
492
493 static bool is_compressed_colortex(struct r600_texture *rtex)
494 {
495 return rtex->cmask.size || rtex->fmask.size ||
496 (rtex->dcc_offset && rtex->dirty_level_mask);
497 }
498
499 static void si_update_compressed_tex_shader_mask(struct si_context *sctx,
500 unsigned shader)
501 {
502 struct si_textures_info *samplers = &sctx->samplers[shader];
503 unsigned shader_bit = 1 << shader;
504
505 if (samplers->depth_texture_mask ||
506 samplers->compressed_colortex_mask ||
507 sctx->images[shader].compressed_colortex_mask)
508 sctx->compressed_tex_shader_mask |= shader_bit;
509 else
510 sctx->compressed_tex_shader_mask &= ~shader_bit;
511 }
512
513 static void si_set_sampler_views(struct pipe_context *ctx,
514 enum pipe_shader_type shader, unsigned start,
515 unsigned count,
516 struct pipe_sampler_view **views)
517 {
518 struct si_context *sctx = (struct si_context *)ctx;
519 struct si_textures_info *samplers = &sctx->samplers[shader];
520 int i;
521
522 if (!count || shader >= SI_NUM_SHADERS)
523 return;
524
525 for (i = 0; i < count; i++) {
526 unsigned slot = start + i;
527
528 if (!views || !views[i]) {
529 samplers->depth_texture_mask &= ~(1u << slot);
530 samplers->compressed_colortex_mask &= ~(1u << slot);
531 si_set_sampler_view(sctx, shader, slot, NULL, false);
532 continue;
533 }
534
535 si_set_sampler_view(sctx, shader, slot, views[i], false);
536
537 if (views[i]->texture && views[i]->texture->target != PIPE_BUFFER) {
538 struct r600_texture *rtex =
539 (struct r600_texture*)views[i]->texture;
540 struct si_sampler_view *rview = (struct si_sampler_view *)views[i];
541
542 if (rtex->db_compatible &&
543 (!rtex->tc_compatible_htile || rview->is_stencil_sampler)) {
544 samplers->depth_texture_mask |= 1u << slot;
545 } else {
546 samplers->depth_texture_mask &= ~(1u << slot);
547 }
548 if (is_compressed_colortex(rtex)) {
549 samplers->compressed_colortex_mask |= 1u << slot;
550 } else {
551 samplers->compressed_colortex_mask &= ~(1u << slot);
552 }
553
554 if (rtex->dcc_offset &&
555 p_atomic_read(&rtex->framebuffers_bound))
556 sctx->need_check_render_feedback = true;
557 } else {
558 samplers->depth_texture_mask &= ~(1u << slot);
559 samplers->compressed_colortex_mask &= ~(1u << slot);
560 }
561 }
562
563 si_update_compressed_tex_shader_mask(sctx, shader);
564 }
565
566 static void
567 si_samplers_update_compressed_colortex_mask(struct si_textures_info *samplers)
568 {
569 unsigned mask = samplers->views.enabled_mask;
570
571 while (mask) {
572 int i = u_bit_scan(&mask);
573 struct pipe_resource *res = samplers->views.views[i]->texture;
574
575 if (res && res->target != PIPE_BUFFER) {
576 struct r600_texture *rtex = (struct r600_texture *)res;
577
578 if (is_compressed_colortex(rtex)) {
579 samplers->compressed_colortex_mask |= 1u << i;
580 } else {
581 samplers->compressed_colortex_mask &= ~(1u << i);
582 }
583 }
584 }
585 }
586
587 /* IMAGE VIEWS */
588
589 static unsigned
590 si_image_descriptors_idx(unsigned shader)
591 {
592 return SI_DESCS_FIRST_SHADER + shader * SI_NUM_SHADER_DESCS +
593 SI_SHADER_DESCS_IMAGES;
594 }
595
596 static struct si_descriptors*
597 si_image_descriptors(struct si_context *sctx, unsigned shader)
598 {
599 return &sctx->descriptors[si_image_descriptors_idx(shader)];
600 }
601
602 static void
603 si_release_image_views(struct si_images_info *images)
604 {
605 unsigned i;
606
607 for (i = 0; i < SI_NUM_IMAGES; ++i) {
608 struct pipe_image_view *view = &images->views[i];
609
610 pipe_resource_reference(&view->resource, NULL);
611 }
612 }
613
614 static void
615 si_image_views_begin_new_cs(struct si_context *sctx, struct si_images_info *images)
616 {
617 uint mask = images->enabled_mask;
618
619 /* Add buffers to the CS. */
620 while (mask) {
621 int i = u_bit_scan(&mask);
622 struct pipe_image_view *view = &images->views[i];
623
624 assert(view->resource);
625
626 si_sampler_view_add_buffer(sctx, view->resource,
627 RADEON_USAGE_READWRITE, false, false);
628 }
629 }
630
631 static void
632 si_disable_shader_image(struct si_context *ctx, unsigned shader, unsigned slot)
633 {
634 struct si_images_info *images = &ctx->images[shader];
635
636 if (images->enabled_mask & (1u << slot)) {
637 struct si_descriptors *descs = si_image_descriptors(ctx, shader);
638
639 pipe_resource_reference(&images->views[slot].resource, NULL);
640 images->compressed_colortex_mask &= ~(1 << slot);
641
642 memcpy(descs->list + slot*8, null_image_descriptor, 8*4);
643 images->enabled_mask &= ~(1u << slot);
644 descs->dirty_mask |= 1u << slot;
645 ctx->descriptors_dirty |= 1u << si_image_descriptors_idx(shader);
646 }
647 }
648
649 static void
650 si_mark_image_range_valid(const struct pipe_image_view *view)
651 {
652 struct r600_resource *res = (struct r600_resource *)view->resource;
653
654 assert(res && res->b.b.target == PIPE_BUFFER);
655
656 util_range_add(&res->valid_buffer_range,
657 view->u.buf.offset,
658 view->u.buf.offset + view->u.buf.size);
659 }
660
661 static void si_set_shader_image(struct si_context *ctx,
662 unsigned shader,
663 unsigned slot, const struct pipe_image_view *view,
664 bool skip_decompress)
665 {
666 struct si_screen *screen = ctx->screen;
667 struct si_images_info *images = &ctx->images[shader];
668 struct si_descriptors *descs = si_image_descriptors(ctx, shader);
669 struct r600_resource *res;
670 uint32_t *desc = descs->list + slot * 8;
671
672 if (!view || !view->resource) {
673 si_disable_shader_image(ctx, shader, slot);
674 return;
675 }
676
677 res = (struct r600_resource *)view->resource;
678
679 if (&images->views[slot] != view)
680 util_copy_image_view(&images->views[slot], view);
681
682 if (res->b.b.target == PIPE_BUFFER) {
683 if (view->access & PIPE_IMAGE_ACCESS_WRITE)
684 si_mark_image_range_valid(view);
685
686 si_make_buffer_descriptor(screen, res,
687 view->format,
688 view->u.buf.offset,
689 view->u.buf.size,
690 descs->list + slot * 8);
691 si_set_buf_desc_address(res, view->u.buf.offset, desc + 4);
692
693 images->compressed_colortex_mask &= ~(1 << slot);
694 res->bind_history |= PIPE_BIND_SHADER_IMAGE;
695 } else {
696 static const unsigned char swizzle[4] = { 0, 1, 2, 3 };
697 struct r600_texture *tex = (struct r600_texture *)res;
698 unsigned level = view->u.tex.level;
699 unsigned width, height, depth;
700 bool uses_dcc = tex->dcc_offset &&
701 level < tex->surface.num_dcc_levels;
702
703 assert(!tex->is_depth);
704 assert(tex->fmask.size == 0);
705
706 if (uses_dcc && !skip_decompress &&
707 (view->access & PIPE_IMAGE_ACCESS_WRITE ||
708 !vi_dcc_formats_compatible(res->b.b.format, view->format))) {
709 /* If DCC can't be disabled, at least decompress it.
710 * The decompression is relatively cheap if the surface
711 * has been decompressed already.
712 */
713 if (r600_texture_disable_dcc(&ctx->b, tex))
714 uses_dcc = false;
715 else
716 ctx->b.decompress_dcc(&ctx->b.b, tex);
717 }
718
719 if (is_compressed_colortex(tex)) {
720 images->compressed_colortex_mask |= 1 << slot;
721 } else {
722 images->compressed_colortex_mask &= ~(1 << slot);
723 }
724
725 if (uses_dcc &&
726 p_atomic_read(&tex->framebuffers_bound))
727 ctx->need_check_render_feedback = true;
728
729 /* Always force the base level to the selected level.
730 *
731 * This is required for 3D textures, where otherwise
732 * selecting a single slice for non-layered bindings
733 * fails. It doesn't hurt the other targets.
734 */
735 width = u_minify(res->b.b.width0, level);
736 height = u_minify(res->b.b.height0, level);
737 depth = u_minify(res->b.b.depth0, level);
738
739 si_make_texture_descriptor(screen, tex,
740 false, res->b.b.target,
741 view->format, swizzle,
742 0, 0,
743 view->u.tex.first_layer,
744 view->u.tex.last_layer,
745 width, height, depth,
746 desc, NULL);
747 si_set_mutable_tex_desc_fields(tex, &tex->surface.level[level],
748 level, level,
749 util_format_get_blockwidth(view->format),
750 false, desc);
751 }
752
753 images->enabled_mask |= 1u << slot;
754 descs->dirty_mask |= 1u << slot;
755 ctx->descriptors_dirty |= 1u << si_image_descriptors_idx(shader);
756
757 /* Since this can flush, it must be done after enabled_mask is updated. */
758 si_sampler_view_add_buffer(ctx, &res->b.b,
759 RADEON_USAGE_READWRITE, false, true);
760 }
761
762 static void
763 si_set_shader_images(struct pipe_context *pipe,
764 enum pipe_shader_type shader,
765 unsigned start_slot, unsigned count,
766 const struct pipe_image_view *views)
767 {
768 struct si_context *ctx = (struct si_context *)pipe;
769 unsigned i, slot;
770
771 assert(shader < SI_NUM_SHADERS);
772
773 if (!count)
774 return;
775
776 assert(start_slot + count <= SI_NUM_IMAGES);
777
778 if (views) {
779 for (i = 0, slot = start_slot; i < count; ++i, ++slot)
780 si_set_shader_image(ctx, shader, slot, &views[i], false);
781 } else {
782 for (i = 0, slot = start_slot; i < count; ++i, ++slot)
783 si_set_shader_image(ctx, shader, slot, NULL, false);
784 }
785
786 si_update_compressed_tex_shader_mask(ctx, shader);
787 }
788
789 static void
790 si_images_update_compressed_colortex_mask(struct si_images_info *images)
791 {
792 unsigned mask = images->enabled_mask;
793
794 while (mask) {
795 int i = u_bit_scan(&mask);
796 struct pipe_resource *res = images->views[i].resource;
797
798 if (res && res->target != PIPE_BUFFER) {
799 struct r600_texture *rtex = (struct r600_texture *)res;
800
801 if (is_compressed_colortex(rtex)) {
802 images->compressed_colortex_mask |= 1 << i;
803 } else {
804 images->compressed_colortex_mask &= ~(1 << i);
805 }
806 }
807 }
808 }
809
810 /* SAMPLER STATES */
811
812 static void si_bind_sampler_states(struct pipe_context *ctx,
813 enum pipe_shader_type shader,
814 unsigned start, unsigned count, void **states)
815 {
816 struct si_context *sctx = (struct si_context *)ctx;
817 struct si_textures_info *samplers = &sctx->samplers[shader];
818 struct si_descriptors *desc = si_sampler_descriptors(sctx, shader);
819 struct si_sampler_state **sstates = (struct si_sampler_state**)states;
820 int i;
821
822 if (!count || shader >= SI_NUM_SHADERS)
823 return;
824
825 for (i = 0; i < count; i++) {
826 unsigned slot = start + i;
827
828 if (!sstates[i] ||
829 sstates[i] == samplers->views.sampler_states[slot])
830 continue;
831
832 #ifdef DEBUG
833 assert(sstates[i]->magic == SI_SAMPLER_STATE_MAGIC);
834 #endif
835 samplers->views.sampler_states[slot] = sstates[i];
836
837 /* If FMASK is bound, don't overwrite it.
838 * The sampler state will be set after FMASK is unbound.
839 */
840 if (samplers->views.views[slot] &&
841 samplers->views.views[slot]->texture &&
842 samplers->views.views[slot]->texture->target != PIPE_BUFFER &&
843 ((struct r600_texture*)samplers->views.views[slot]->texture)->fmask.size)
844 continue;
845
846 memcpy(desc->list + slot * 16 + 12, sstates[i]->val, 4*4);
847 desc->dirty_mask |= 1u << slot;
848 sctx->descriptors_dirty |= 1u << si_sampler_descriptors_idx(shader);
849 }
850 }
851
852 /* BUFFER RESOURCES */
853
854 static void si_init_buffer_resources(struct si_buffer_resources *buffers,
855 struct si_descriptors *descs,
856 unsigned num_buffers,
857 unsigned shader_userdata_index,
858 enum radeon_bo_usage shader_usage,
859 enum radeon_bo_priority priority,
860 unsigned *ce_offset)
861 {
862 buffers->shader_usage = shader_usage;
863 buffers->priority = priority;
864 buffers->buffers = CALLOC(num_buffers, sizeof(struct pipe_resource*));
865
866 si_init_descriptors(descs, shader_userdata_index, 4,
867 num_buffers, NULL, ce_offset);
868 }
869
870 static void si_release_buffer_resources(struct si_buffer_resources *buffers,
871 struct si_descriptors *descs)
872 {
873 int i;
874
875 for (i = 0; i < descs->num_elements; i++) {
876 pipe_resource_reference(&buffers->buffers[i], NULL);
877 }
878
879 FREE(buffers->buffers);
880 }
881
882 static void si_buffer_resources_begin_new_cs(struct si_context *sctx,
883 struct si_buffer_resources *buffers)
884 {
885 unsigned mask = buffers->enabled_mask;
886
887 /* Add buffers to the CS. */
888 while (mask) {
889 int i = u_bit_scan(&mask);
890
891 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
892 (struct r600_resource*)buffers->buffers[i],
893 buffers->shader_usage, buffers->priority);
894 }
895 }
896
897 static void si_get_buffer_from_descriptors(struct si_buffer_resources *buffers,
898 struct si_descriptors *descs,
899 unsigned idx, struct pipe_resource **buf,
900 unsigned *offset, unsigned *size)
901 {
902 pipe_resource_reference(buf, buffers->buffers[idx]);
903 if (*buf) {
904 struct r600_resource *res = r600_resource(*buf);
905 const uint32_t *desc = descs->list + idx * 4;
906 uint64_t va;
907
908 *size = desc[2];
909
910 assert(G_008F04_STRIDE(desc[1]) == 0);
911 va = ((uint64_t)desc[1] << 32) | desc[0];
912
913 assert(va >= res->gpu_address && va + *size <= res->gpu_address + res->bo_size);
914 *offset = va - res->gpu_address;
915 }
916 }
917
918 /* VERTEX BUFFERS */
919
920 static void si_vertex_buffers_begin_new_cs(struct si_context *sctx)
921 {
922 struct si_descriptors *desc = &sctx->vertex_buffers;
923 int count = sctx->vertex_elements ? sctx->vertex_elements->count : 0;
924 int i;
925
926 for (i = 0; i < count; i++) {
927 int vb = sctx->vertex_elements->elements[i].vertex_buffer_index;
928
929 if (vb >= ARRAY_SIZE(sctx->vertex_buffer))
930 continue;
931 if (!sctx->vertex_buffer[vb].buffer)
932 continue;
933
934 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
935 (struct r600_resource*)sctx->vertex_buffer[vb].buffer,
936 RADEON_USAGE_READ, RADEON_PRIO_VERTEX_BUFFER);
937 }
938
939 if (!desc->buffer)
940 return;
941 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
942 desc->buffer, RADEON_USAGE_READ,
943 RADEON_PRIO_DESCRIPTORS);
944 }
945
946 bool si_upload_vertex_buffer_descriptors(struct si_context *sctx)
947 {
948 struct si_vertex_element *velems = sctx->vertex_elements;
949 struct si_descriptors *desc = &sctx->vertex_buffers;
950 unsigned i, count = velems->count;
951 uint64_t va;
952 uint32_t *ptr;
953
954 if (!sctx->vertex_buffers_dirty || !count || !velems)
955 return true;
956
957 unsigned fix_size3 = velems->fix_size3;
958 unsigned first_vb_use_mask = velems->first_vb_use_mask;
959
960 /* Vertex buffer descriptors are the only ones which are uploaded
961 * directly through a staging buffer and don't go through
962 * the fine-grained upload path.
963 */
964 u_upload_alloc(sctx->b.uploader, 0, count * 16, 256, &desc->buffer_offset,
965 (struct pipe_resource**)&desc->buffer, (void**)&ptr);
966 if (!desc->buffer)
967 return false;
968
969 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
970 desc->buffer, RADEON_USAGE_READ,
971 RADEON_PRIO_DESCRIPTORS);
972
973 assert(count <= SI_NUM_VERTEX_BUFFERS);
974
975 for (i = 0; i < count; i++) {
976 struct pipe_vertex_element *ve = &velems->elements[i];
977 struct pipe_vertex_buffer *vb;
978 struct r600_resource *rbuffer;
979 unsigned offset;
980 unsigned vbo_index = ve->vertex_buffer_index;
981 uint32_t *desc = &ptr[i*4];
982
983 vb = &sctx->vertex_buffer[vbo_index];
984 rbuffer = (struct r600_resource*)vb->buffer;
985 if (!rbuffer) {
986 memset(desc, 0, 16);
987 continue;
988 }
989
990 offset = vb->buffer_offset + ve->src_offset;
991 va = rbuffer->gpu_address + offset;
992
993 /* Fill in T# buffer resource description */
994 desc[0] = va;
995 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
996 S_008F04_STRIDE(vb->stride);
997
998 if (sctx->b.chip_class <= CIK && vb->stride) {
999 /* Round up by rounding down and adding 1 */
1000 desc[2] = (vb->buffer->width0 - offset -
1001 velems->format_size[i]) /
1002 vb->stride + 1;
1003 } else {
1004 uint32_t size3;
1005
1006 desc[2] = vb->buffer->width0 - offset;
1007
1008 /* For attributes of size 3 with byte or short
1009 * components, we use a 4-component data format.
1010 *
1011 * As a consequence, we have to round the buffer size
1012 * up so that the hardware sees four components as
1013 * being inside the buffer if and only if the first
1014 * three components are in the buffer.
1015 *
1016 * Since the offset and stride are guaranteed to be
1017 * 4-byte aligned, this alignment will never cross the
1018 * winsys buffer boundary.
1019 */
1020 size3 = (fix_size3 >> (2 * i)) & 3;
1021 if (vb->stride && size3) {
1022 assert(offset % 4 == 0 && vb->stride % 4 == 0);
1023 assert(size3 <= 2);
1024 desc[2] = align(desc[2], size3 * 2);
1025 }
1026 }
1027
1028 desc[3] = velems->rsrc_word3[i];
1029
1030 if (first_vb_use_mask & (1 << i)) {
1031 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
1032 (struct r600_resource*)vb->buffer,
1033 RADEON_USAGE_READ, RADEON_PRIO_VERTEX_BUFFER);
1034 }
1035 }
1036
1037 /* Don't flush the const cache. It would have a very negative effect
1038 * on performance (confirmed by testing). New descriptors are always
1039 * uploaded to a fresh new buffer, so I don't think flushing the const
1040 * cache is needed. */
1041 si_mark_atom_dirty(sctx, &sctx->shader_userdata.atom);
1042 if (sctx->b.chip_class >= CIK)
1043 si_mark_atom_dirty(sctx, &sctx->prefetch_L2);
1044 sctx->vertex_buffers_dirty = false;
1045 sctx->vertex_buffer_pointer_dirty = true;
1046 return true;
1047 }
1048
1049
1050 /* CONSTANT BUFFERS */
1051
1052 static unsigned
1053 si_const_buffer_descriptors_idx(unsigned shader)
1054 {
1055 return SI_DESCS_FIRST_SHADER + shader * SI_NUM_SHADER_DESCS +
1056 SI_SHADER_DESCS_CONST_BUFFERS;
1057 }
1058
1059 static struct si_descriptors *
1060 si_const_buffer_descriptors(struct si_context *sctx, unsigned shader)
1061 {
1062 return &sctx->descriptors[si_const_buffer_descriptors_idx(shader)];
1063 }
1064
1065 void si_upload_const_buffer(struct si_context *sctx, struct r600_resource **rbuffer,
1066 const uint8_t *ptr, unsigned size, uint32_t *const_offset)
1067 {
1068 void *tmp;
1069
1070 u_upload_alloc(sctx->b.uploader, 0, size, 256, const_offset,
1071 (struct pipe_resource**)rbuffer, &tmp);
1072 if (*rbuffer)
1073 util_memcpy_cpu_to_le32(tmp, ptr, size);
1074 }
1075
1076 static void si_set_constant_buffer(struct si_context *sctx,
1077 struct si_buffer_resources *buffers,
1078 unsigned descriptors_idx,
1079 uint slot, const struct pipe_constant_buffer *input)
1080 {
1081 struct si_descriptors *descs = &sctx->descriptors[descriptors_idx];
1082 assert(slot < descs->num_elements);
1083 pipe_resource_reference(&buffers->buffers[slot], NULL);
1084
1085 /* CIK cannot unbind a constant buffer (S_BUFFER_LOAD is buggy
1086 * with a NULL buffer). We need to use a dummy buffer instead. */
1087 if (sctx->b.chip_class == CIK &&
1088 (!input || (!input->buffer && !input->user_buffer)))
1089 input = &sctx->null_const_buf;
1090
1091 if (input && (input->buffer || input->user_buffer)) {
1092 struct pipe_resource *buffer = NULL;
1093 uint64_t va;
1094
1095 /* Upload the user buffer if needed. */
1096 if (input->user_buffer) {
1097 unsigned buffer_offset;
1098
1099 si_upload_const_buffer(sctx,
1100 (struct r600_resource**)&buffer, input->user_buffer,
1101 input->buffer_size, &buffer_offset);
1102 if (!buffer) {
1103 /* Just unbind on failure. */
1104 si_set_constant_buffer(sctx, buffers, descriptors_idx, slot, NULL);
1105 return;
1106 }
1107 va = r600_resource(buffer)->gpu_address + buffer_offset;
1108 } else {
1109 pipe_resource_reference(&buffer, input->buffer);
1110 va = r600_resource(buffer)->gpu_address + input->buffer_offset;
1111 /* Only track usage for non-user buffers. */
1112 r600_resource(buffer)->bind_history |= PIPE_BIND_CONSTANT_BUFFER;
1113 }
1114
1115 /* Set the descriptor. */
1116 uint32_t *desc = descs->list + slot*4;
1117 desc[0] = va;
1118 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
1119 S_008F04_STRIDE(0);
1120 desc[2] = input->buffer_size;
1121 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1122 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1123 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1124 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1125 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1126 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
1127
1128 buffers->buffers[slot] = buffer;
1129 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
1130 (struct r600_resource*)buffer,
1131 buffers->shader_usage,
1132 buffers->priority, true);
1133 buffers->enabled_mask |= 1u << slot;
1134 } else {
1135 /* Clear the descriptor. */
1136 memset(descs->list + slot*4, 0, sizeof(uint32_t) * 4);
1137 buffers->enabled_mask &= ~(1u << slot);
1138 }
1139
1140 descs->dirty_mask |= 1u << slot;
1141 sctx->descriptors_dirty |= 1u << descriptors_idx;
1142 }
1143
1144 void si_set_rw_buffer(struct si_context *sctx,
1145 uint slot, const struct pipe_constant_buffer *input)
1146 {
1147 si_set_constant_buffer(sctx, &sctx->rw_buffers,
1148 SI_DESCS_RW_BUFFERS, slot, input);
1149 }
1150
1151 static void si_pipe_set_constant_buffer(struct pipe_context *ctx,
1152 uint shader, uint slot,
1153 const struct pipe_constant_buffer *input)
1154 {
1155 struct si_context *sctx = (struct si_context *)ctx;
1156
1157 if (shader >= SI_NUM_SHADERS)
1158 return;
1159
1160 si_set_constant_buffer(sctx, &sctx->const_buffers[shader],
1161 si_const_buffer_descriptors_idx(shader),
1162 slot, input);
1163 }
1164
1165 void si_get_pipe_constant_buffer(struct si_context *sctx, uint shader,
1166 uint slot, struct pipe_constant_buffer *cbuf)
1167 {
1168 cbuf->user_buffer = NULL;
1169 si_get_buffer_from_descriptors(
1170 &sctx->const_buffers[shader],
1171 si_const_buffer_descriptors(sctx, shader),
1172 slot, &cbuf->buffer, &cbuf->buffer_offset, &cbuf->buffer_size);
1173 }
1174
1175 /* SHADER BUFFERS */
1176
1177 static unsigned
1178 si_shader_buffer_descriptors_idx(enum pipe_shader_type shader)
1179 {
1180 return SI_DESCS_FIRST_SHADER + shader * SI_NUM_SHADER_DESCS +
1181 SI_SHADER_DESCS_SHADER_BUFFERS;
1182 }
1183
1184 static struct si_descriptors *
1185 si_shader_buffer_descriptors(struct si_context *sctx,
1186 enum pipe_shader_type shader)
1187 {
1188 return &sctx->descriptors[si_shader_buffer_descriptors_idx(shader)];
1189 }
1190
1191 static void si_set_shader_buffers(struct pipe_context *ctx,
1192 enum pipe_shader_type shader,
1193 unsigned start_slot, unsigned count,
1194 const struct pipe_shader_buffer *sbuffers)
1195 {
1196 struct si_context *sctx = (struct si_context *)ctx;
1197 struct si_buffer_resources *buffers = &sctx->shader_buffers[shader];
1198 struct si_descriptors *descs = si_shader_buffer_descriptors(sctx, shader);
1199 unsigned i;
1200
1201 assert(start_slot + count <= SI_NUM_SHADER_BUFFERS);
1202
1203 for (i = 0; i < count; ++i) {
1204 const struct pipe_shader_buffer *sbuffer = sbuffers ? &sbuffers[i] : NULL;
1205 struct r600_resource *buf;
1206 unsigned slot = start_slot + i;
1207 uint32_t *desc = descs->list + slot * 4;
1208 uint64_t va;
1209
1210 if (!sbuffer || !sbuffer->buffer) {
1211 pipe_resource_reference(&buffers->buffers[slot], NULL);
1212 memset(desc, 0, sizeof(uint32_t) * 4);
1213 buffers->enabled_mask &= ~(1u << slot);
1214 descs->dirty_mask |= 1u << slot;
1215 sctx->descriptors_dirty |=
1216 1u << si_shader_buffer_descriptors_idx(shader);
1217 continue;
1218 }
1219
1220 buf = (struct r600_resource *)sbuffer->buffer;
1221 va = buf->gpu_address + sbuffer->buffer_offset;
1222
1223 desc[0] = va;
1224 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
1225 S_008F04_STRIDE(0);
1226 desc[2] = sbuffer->buffer_size;
1227 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1228 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1229 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1230 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1231 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1232 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
1233
1234 pipe_resource_reference(&buffers->buffers[slot], &buf->b.b);
1235 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx, buf,
1236 buffers->shader_usage,
1237 buffers->priority, true);
1238 buf->bind_history |= PIPE_BIND_SHADER_BUFFER;
1239
1240 buffers->enabled_mask |= 1u << slot;
1241 descs->dirty_mask |= 1u << slot;
1242 sctx->descriptors_dirty |=
1243 1u << si_shader_buffer_descriptors_idx(shader);
1244 }
1245 }
1246
1247 void si_get_shader_buffers(struct si_context *sctx, uint shader,
1248 uint start_slot, uint count,
1249 struct pipe_shader_buffer *sbuf)
1250 {
1251 struct si_buffer_resources *buffers = &sctx->shader_buffers[shader];
1252 struct si_descriptors *descs = si_shader_buffer_descriptors(sctx, shader);
1253
1254 for (unsigned i = 0; i < count; ++i) {
1255 si_get_buffer_from_descriptors(
1256 buffers, descs, start_slot + i,
1257 &sbuf[i].buffer, &sbuf[i].buffer_offset,
1258 &sbuf[i].buffer_size);
1259 }
1260 }
1261
1262 /* RING BUFFERS */
1263
1264 void si_set_ring_buffer(struct pipe_context *ctx, uint slot,
1265 struct pipe_resource *buffer,
1266 unsigned stride, unsigned num_records,
1267 bool add_tid, bool swizzle,
1268 unsigned element_size, unsigned index_stride, uint64_t offset)
1269 {
1270 struct si_context *sctx = (struct si_context *)ctx;
1271 struct si_buffer_resources *buffers = &sctx->rw_buffers;
1272 struct si_descriptors *descs = &sctx->descriptors[SI_DESCS_RW_BUFFERS];
1273
1274 /* The stride field in the resource descriptor has 14 bits */
1275 assert(stride < (1 << 14));
1276
1277 assert(slot < descs->num_elements);
1278 pipe_resource_reference(&buffers->buffers[slot], NULL);
1279
1280 if (buffer) {
1281 uint64_t va;
1282
1283 va = r600_resource(buffer)->gpu_address + offset;
1284
1285 switch (element_size) {
1286 default:
1287 assert(!"Unsupported ring buffer element size");
1288 case 0:
1289 case 2:
1290 element_size = 0;
1291 break;
1292 case 4:
1293 element_size = 1;
1294 break;
1295 case 8:
1296 element_size = 2;
1297 break;
1298 case 16:
1299 element_size = 3;
1300 break;
1301 }
1302
1303 switch (index_stride) {
1304 default:
1305 assert(!"Unsupported ring buffer index stride");
1306 case 0:
1307 case 8:
1308 index_stride = 0;
1309 break;
1310 case 16:
1311 index_stride = 1;
1312 break;
1313 case 32:
1314 index_stride = 2;
1315 break;
1316 case 64:
1317 index_stride = 3;
1318 break;
1319 }
1320
1321 if (sctx->b.chip_class >= VI && stride)
1322 num_records *= stride;
1323
1324 /* Set the descriptor. */
1325 uint32_t *desc = descs->list + slot*4;
1326 desc[0] = va;
1327 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
1328 S_008F04_STRIDE(stride) |
1329 S_008F04_SWIZZLE_ENABLE(swizzle);
1330 desc[2] = num_records;
1331 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1332 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1333 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1334 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1335 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1336 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) |
1337 S_008F0C_ELEMENT_SIZE(element_size) |
1338 S_008F0C_INDEX_STRIDE(index_stride) |
1339 S_008F0C_ADD_TID_ENABLE(add_tid);
1340
1341 pipe_resource_reference(&buffers->buffers[slot], buffer);
1342 radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
1343 (struct r600_resource*)buffer,
1344 buffers->shader_usage, buffers->priority);
1345 buffers->enabled_mask |= 1u << slot;
1346 } else {
1347 /* Clear the descriptor. */
1348 memset(descs->list + slot*4, 0, sizeof(uint32_t) * 4);
1349 buffers->enabled_mask &= ~(1u << slot);
1350 }
1351
1352 descs->dirty_mask |= 1u << slot;
1353 sctx->descriptors_dirty |= 1u << SI_DESCS_RW_BUFFERS;
1354 }
1355
1356 /* STREAMOUT BUFFERS */
1357
1358 static void si_set_streamout_targets(struct pipe_context *ctx,
1359 unsigned num_targets,
1360 struct pipe_stream_output_target **targets,
1361 const unsigned *offsets)
1362 {
1363 struct si_context *sctx = (struct si_context *)ctx;
1364 struct si_buffer_resources *buffers = &sctx->rw_buffers;
1365 struct si_descriptors *descs = &sctx->descriptors[SI_DESCS_RW_BUFFERS];
1366 unsigned old_num_targets = sctx->b.streamout.num_targets;
1367 unsigned i, bufidx;
1368
1369 /* We are going to unbind the buffers. Mark which caches need to be flushed. */
1370 if (sctx->b.streamout.num_targets && sctx->b.streamout.begin_emitted) {
1371 /* Since streamout uses vector writes which go through TC L2
1372 * and most other clients can use TC L2 as well, we don't need
1373 * to flush it.
1374 *
1375 * The only cases which requires flushing it is VGT DMA index
1376 * fetching (on <= CIK) and indirect draw data, which are rare
1377 * cases. Thus, flag the TC L2 dirtiness in the resource and
1378 * handle it at draw call time.
1379 */
1380 for (i = 0; i < sctx->b.streamout.num_targets; i++)
1381 if (sctx->b.streamout.targets[i])
1382 r600_resource(sctx->b.streamout.targets[i]->b.buffer)->TC_L2_dirty = true;
1383
1384 /* Invalidate the scalar cache in case a streamout buffer is
1385 * going to be used as a constant buffer.
1386 *
1387 * Invalidate TC L1, because streamout bypasses it (done by
1388 * setting GLC=1 in the store instruction), but it can contain
1389 * outdated data of streamout buffers.
1390 *
1391 * VS_PARTIAL_FLUSH is required if the buffers are going to be
1392 * used as an input immediately.
1393 */
1394 sctx->b.flags |= SI_CONTEXT_INV_SMEM_L1 |
1395 SI_CONTEXT_INV_VMEM_L1 |
1396 SI_CONTEXT_VS_PARTIAL_FLUSH;
1397 }
1398
1399 /* All readers of the streamout targets need to be finished before we can
1400 * start writing to the targets.
1401 */
1402 if (num_targets)
1403 sctx->b.flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
1404 SI_CONTEXT_CS_PARTIAL_FLUSH;
1405
1406 /* Streamout buffers must be bound in 2 places:
1407 * 1) in VGT by setting the VGT_STRMOUT registers
1408 * 2) as shader resources
1409 */
1410
1411 /* Set the VGT regs. */
1412 r600_set_streamout_targets(ctx, num_targets, targets, offsets);
1413
1414 /* Set the shader resources.*/
1415 for (i = 0; i < num_targets; i++) {
1416 bufidx = SI_VS_STREAMOUT_BUF0 + i;
1417
1418 if (targets[i]) {
1419 struct pipe_resource *buffer = targets[i]->buffer;
1420 uint64_t va = r600_resource(buffer)->gpu_address;
1421
1422 /* Set the descriptor.
1423 *
1424 * On VI, the format must be non-INVALID, otherwise
1425 * the buffer will be considered not bound and store
1426 * instructions will be no-ops.
1427 */
1428 uint32_t *desc = descs->list + bufidx*4;
1429 desc[0] = va;
1430 desc[1] = S_008F04_BASE_ADDRESS_HI(va >> 32);
1431 desc[2] = 0xffffffff;
1432 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1433 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1434 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1435 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1436 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
1437
1438 /* Set the resource. */
1439 pipe_resource_reference(&buffers->buffers[bufidx],
1440 buffer);
1441 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
1442 (struct r600_resource*)buffer,
1443 buffers->shader_usage,
1444 RADEON_PRIO_SHADER_RW_BUFFER,
1445 true);
1446 r600_resource(buffer)->bind_history |= PIPE_BIND_STREAM_OUTPUT;
1447
1448 buffers->enabled_mask |= 1u << bufidx;
1449 } else {
1450 /* Clear the descriptor and unset the resource. */
1451 memset(descs->list + bufidx*4, 0,
1452 sizeof(uint32_t) * 4);
1453 pipe_resource_reference(&buffers->buffers[bufidx],
1454 NULL);
1455 buffers->enabled_mask &= ~(1u << bufidx);
1456 }
1457 descs->dirty_mask |= 1u << bufidx;
1458 }
1459 for (; i < old_num_targets; i++) {
1460 bufidx = SI_VS_STREAMOUT_BUF0 + i;
1461 /* Clear the descriptor and unset the resource. */
1462 memset(descs->list + bufidx*4, 0, sizeof(uint32_t) * 4);
1463 pipe_resource_reference(&buffers->buffers[bufidx], NULL);
1464 buffers->enabled_mask &= ~(1u << bufidx);
1465 descs->dirty_mask |= 1u << bufidx;
1466 }
1467
1468 sctx->descriptors_dirty |= 1u << SI_DESCS_RW_BUFFERS;
1469 }
1470
1471 static void si_desc_reset_buffer_offset(struct pipe_context *ctx,
1472 uint32_t *desc, uint64_t old_buf_va,
1473 struct pipe_resource *new_buf)
1474 {
1475 /* Retrieve the buffer offset from the descriptor. */
1476 uint64_t old_desc_va =
1477 desc[0] | ((uint64_t)G_008F04_BASE_ADDRESS_HI(desc[1]) << 32);
1478
1479 assert(old_buf_va <= old_desc_va);
1480 uint64_t offset_within_buffer = old_desc_va - old_buf_va;
1481
1482 /* Update the descriptor. */
1483 si_set_buf_desc_address(r600_resource(new_buf), offset_within_buffer,
1484 desc);
1485 }
1486
1487 /* INTERNAL CONST BUFFERS */
1488
1489 static void si_set_polygon_stipple(struct pipe_context *ctx,
1490 const struct pipe_poly_stipple *state)
1491 {
1492 struct si_context *sctx = (struct si_context *)ctx;
1493 struct pipe_constant_buffer cb = {};
1494 unsigned stipple[32];
1495 int i;
1496
1497 for (i = 0; i < 32; i++)
1498 stipple[i] = util_bitreverse(state->stipple[i]);
1499
1500 cb.user_buffer = stipple;
1501 cb.buffer_size = sizeof(stipple);
1502
1503 si_set_rw_buffer(sctx, SI_PS_CONST_POLY_STIPPLE, &cb);
1504 }
1505
1506 /* TEXTURE METADATA ENABLE/DISABLE */
1507
1508 /* CMASK can be enabled (for fast clear) and disabled (for texture export)
1509 * while the texture is bound, possibly by a different context. In that case,
1510 * call this function to update compressed_colortex_masks.
1511 */
1512 void si_update_compressed_colortex_masks(struct si_context *sctx)
1513 {
1514 for (int i = 0; i < SI_NUM_SHADERS; ++i) {
1515 si_samplers_update_compressed_colortex_mask(&sctx->samplers[i]);
1516 si_images_update_compressed_colortex_mask(&sctx->images[i]);
1517 si_update_compressed_tex_shader_mask(sctx, i);
1518 }
1519 }
1520
1521 /* BUFFER DISCARD/INVALIDATION */
1522
1523 /** Reset descriptors of buffer resources after \p buf has been invalidated. */
1524 static void si_reset_buffer_resources(struct si_context *sctx,
1525 struct si_buffer_resources *buffers,
1526 unsigned descriptors_idx,
1527 struct pipe_resource *buf,
1528 uint64_t old_va)
1529 {
1530 struct si_descriptors *descs = &sctx->descriptors[descriptors_idx];
1531 unsigned mask = buffers->enabled_mask;
1532
1533 while (mask) {
1534 unsigned i = u_bit_scan(&mask);
1535 if (buffers->buffers[i] == buf) {
1536 si_desc_reset_buffer_offset(&sctx->b.b,
1537 descs->list + i*4,
1538 old_va, buf);
1539 descs->dirty_mask |= 1u << i;
1540 sctx->descriptors_dirty |= 1u << descriptors_idx;
1541
1542 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
1543 (struct r600_resource *)buf,
1544 buffers->shader_usage,
1545 buffers->priority, true);
1546 }
1547 }
1548 }
1549
1550 /* Reallocate a buffer a update all resource bindings where the buffer is
1551 * bound.
1552 *
1553 * This is used to avoid CPU-GPU synchronizations, because it makes the buffer
1554 * idle by discarding its contents. Apps usually tell us when to do this using
1555 * map_buffer flags, for example.
1556 */
1557 static void si_invalidate_buffer(struct pipe_context *ctx, struct pipe_resource *buf)
1558 {
1559 struct si_context *sctx = (struct si_context*)ctx;
1560 struct r600_resource *rbuffer = r600_resource(buf);
1561 unsigned i, shader;
1562 uint64_t old_va = rbuffer->gpu_address;
1563 unsigned num_elems = sctx->vertex_elements ?
1564 sctx->vertex_elements->count : 0;
1565
1566 /* Reallocate the buffer in the same pipe_resource. */
1567 r600_alloc_resource(&sctx->screen->b, rbuffer);
1568
1569 /* We changed the buffer, now we need to bind it where the old one
1570 * was bound. This consists of 2 things:
1571 * 1) Updating the resource descriptor and dirtying it.
1572 * 2) Adding a relocation to the CS, so that it's usable.
1573 */
1574
1575 /* Vertex buffers. */
1576 if (rbuffer->bind_history & PIPE_BIND_VERTEX_BUFFER) {
1577 for (i = 0; i < num_elems; i++) {
1578 int vb = sctx->vertex_elements->elements[i].vertex_buffer_index;
1579
1580 if (vb >= ARRAY_SIZE(sctx->vertex_buffer))
1581 continue;
1582 if (!sctx->vertex_buffer[vb].buffer)
1583 continue;
1584
1585 if (sctx->vertex_buffer[vb].buffer == buf) {
1586 sctx->vertex_buffers_dirty = true;
1587 break;
1588 }
1589 }
1590 }
1591
1592 /* Streamout buffers. (other internal buffers can't be invalidated) */
1593 if (rbuffer->bind_history & PIPE_BIND_STREAM_OUTPUT) {
1594 for (i = SI_VS_STREAMOUT_BUF0; i <= SI_VS_STREAMOUT_BUF3; i++) {
1595 struct si_buffer_resources *buffers = &sctx->rw_buffers;
1596 struct si_descriptors *descs =
1597 &sctx->descriptors[SI_DESCS_RW_BUFFERS];
1598
1599 if (buffers->buffers[i] != buf)
1600 continue;
1601
1602 si_desc_reset_buffer_offset(ctx, descs->list + i*4,
1603 old_va, buf);
1604 descs->dirty_mask |= 1u << i;
1605 sctx->descriptors_dirty |= 1u << SI_DESCS_RW_BUFFERS;
1606
1607 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
1608 rbuffer, buffers->shader_usage,
1609 RADEON_PRIO_SHADER_RW_BUFFER,
1610 true);
1611
1612 /* Update the streamout state. */
1613 if (sctx->b.streamout.begin_emitted)
1614 r600_emit_streamout_end(&sctx->b);
1615 sctx->b.streamout.append_bitmask =
1616 sctx->b.streamout.enabled_mask;
1617 r600_streamout_buffers_dirty(&sctx->b);
1618 }
1619 }
1620
1621 /* Constant and shader buffers. */
1622 if (rbuffer->bind_history & PIPE_BIND_CONSTANT_BUFFER) {
1623 for (shader = 0; shader < SI_NUM_SHADERS; shader++)
1624 si_reset_buffer_resources(sctx, &sctx->const_buffers[shader],
1625 si_const_buffer_descriptors_idx(shader),
1626 buf, old_va);
1627 }
1628
1629 if (rbuffer->bind_history & PIPE_BIND_SHADER_BUFFER) {
1630 for (shader = 0; shader < SI_NUM_SHADERS; shader++)
1631 si_reset_buffer_resources(sctx, &sctx->shader_buffers[shader],
1632 si_shader_buffer_descriptors_idx(shader),
1633 buf, old_va);
1634 }
1635
1636 if (rbuffer->bind_history & PIPE_BIND_SAMPLER_VIEW) {
1637 /* Texture buffers - update bindings. */
1638 for (shader = 0; shader < SI_NUM_SHADERS; shader++) {
1639 struct si_sampler_views *views = &sctx->samplers[shader].views;
1640 struct si_descriptors *descs =
1641 si_sampler_descriptors(sctx, shader);
1642 unsigned mask = views->enabled_mask;
1643
1644 while (mask) {
1645 unsigned i = u_bit_scan(&mask);
1646 if (views->views[i]->texture == buf) {
1647 si_desc_reset_buffer_offset(ctx,
1648 descs->list +
1649 i * 16 + 4,
1650 old_va, buf);
1651 descs->dirty_mask |= 1u << i;
1652 sctx->descriptors_dirty |=
1653 1u << si_sampler_descriptors_idx(shader);
1654
1655 radeon_add_to_buffer_list_check_mem(&sctx->b, &sctx->b.gfx,
1656 rbuffer, RADEON_USAGE_READ,
1657 RADEON_PRIO_SAMPLER_BUFFER,
1658 true);
1659 }
1660 }
1661 }
1662 }
1663
1664 /* Shader images */
1665 if (rbuffer->bind_history & PIPE_BIND_SHADER_IMAGE) {
1666 for (shader = 0; shader < SI_NUM_SHADERS; ++shader) {
1667 struct si_images_info *images = &sctx->images[shader];
1668 struct si_descriptors *descs =
1669 si_image_descriptors(sctx, shader);
1670 unsigned mask = images->enabled_mask;
1671
1672 while (mask) {
1673 unsigned i = u_bit_scan(&mask);
1674
1675 if (images->views[i].resource == buf) {
1676 if (images->views[i].access & PIPE_IMAGE_ACCESS_WRITE)
1677 si_mark_image_range_valid(&images->views[i]);
1678
1679 si_desc_reset_buffer_offset(
1680 ctx, descs->list + i * 8 + 4,
1681 old_va, buf);
1682 descs->dirty_mask |= 1u << i;
1683 sctx->descriptors_dirty |=
1684 1u << si_image_descriptors_idx(shader);
1685
1686 radeon_add_to_buffer_list_check_mem(
1687 &sctx->b, &sctx->b.gfx, rbuffer,
1688 RADEON_USAGE_READWRITE,
1689 RADEON_PRIO_SAMPLER_BUFFER, true);
1690 }
1691 }
1692 }
1693 }
1694 }
1695
1696 /* Update mutable image descriptor fields of all bound textures. */
1697 void si_update_all_texture_descriptors(struct si_context *sctx)
1698 {
1699 unsigned shader;
1700
1701 for (shader = 0; shader < SI_NUM_SHADERS; shader++) {
1702 struct si_sampler_views *samplers = &sctx->samplers[shader].views;
1703 struct si_images_info *images = &sctx->images[shader];
1704 unsigned mask;
1705
1706 /* Images. */
1707 mask = images->enabled_mask;
1708 while (mask) {
1709 unsigned i = u_bit_scan(&mask);
1710 struct pipe_image_view *view = &images->views[i];
1711
1712 if (!view->resource ||
1713 view->resource->target == PIPE_BUFFER)
1714 continue;
1715
1716 si_set_shader_image(sctx, shader, i, view, true);
1717 }
1718
1719 /* Sampler views. */
1720 mask = samplers->enabled_mask;
1721 while (mask) {
1722 unsigned i = u_bit_scan(&mask);
1723 struct pipe_sampler_view *view = samplers->views[i];
1724
1725 if (!view ||
1726 !view->texture ||
1727 view->texture->target == PIPE_BUFFER)
1728 continue;
1729
1730 si_set_sampler_view(sctx, shader, i,
1731 samplers->views[i], true);
1732 }
1733
1734 si_update_compressed_tex_shader_mask(sctx, shader);
1735 }
1736 }
1737
1738 /* SHADER USER DATA */
1739
1740 static void si_mark_shader_pointers_dirty(struct si_context *sctx,
1741 unsigned shader)
1742 {
1743 sctx->shader_pointers_dirty |=
1744 u_bit_consecutive(SI_DESCS_FIRST_SHADER + shader * SI_NUM_SHADER_DESCS,
1745 SI_NUM_SHADER_DESCS);
1746
1747 if (shader == PIPE_SHADER_VERTEX)
1748 sctx->vertex_buffer_pointer_dirty = sctx->vertex_buffers.buffer != NULL;
1749
1750 si_mark_atom_dirty(sctx, &sctx->shader_userdata.atom);
1751 }
1752
1753 static void si_shader_userdata_begin_new_cs(struct si_context *sctx)
1754 {
1755 sctx->shader_pointers_dirty = u_bit_consecutive(0, SI_NUM_DESCS);
1756 sctx->vertex_buffer_pointer_dirty = sctx->vertex_buffers.buffer != NULL;
1757 si_mark_atom_dirty(sctx, &sctx->shader_userdata.atom);
1758 }
1759
1760 /* Set a base register address for user data constants in the given shader.
1761 * This assigns a mapping from PIPE_SHADER_* to SPI_SHADER_USER_DATA_*.
1762 */
1763 static void si_set_user_data_base(struct si_context *sctx,
1764 unsigned shader, uint32_t new_base)
1765 {
1766 uint32_t *base = &sctx->shader_userdata.sh_base[shader];
1767
1768 if (*base != new_base) {
1769 *base = new_base;
1770
1771 if (new_base)
1772 si_mark_shader_pointers_dirty(sctx, shader);
1773 }
1774 }
1775
1776 /* This must be called when these shaders are changed from non-NULL to NULL
1777 * and vice versa:
1778 * - geometry shader
1779 * - tessellation control shader
1780 * - tessellation evaluation shader
1781 */
1782 void si_shader_change_notify(struct si_context *sctx)
1783 {
1784 /* VS can be bound as VS, ES, or LS. */
1785 if (sctx->tes_shader.cso)
1786 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX,
1787 R_00B530_SPI_SHADER_USER_DATA_LS_0);
1788 else if (sctx->gs_shader.cso)
1789 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX,
1790 R_00B330_SPI_SHADER_USER_DATA_ES_0);
1791 else
1792 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX,
1793 R_00B130_SPI_SHADER_USER_DATA_VS_0);
1794
1795 /* TES can be bound as ES, VS, or not bound. */
1796 if (sctx->tes_shader.cso) {
1797 if (sctx->gs_shader.cso)
1798 si_set_user_data_base(sctx, PIPE_SHADER_TESS_EVAL,
1799 R_00B330_SPI_SHADER_USER_DATA_ES_0);
1800 else
1801 si_set_user_data_base(sctx, PIPE_SHADER_TESS_EVAL,
1802 R_00B130_SPI_SHADER_USER_DATA_VS_0);
1803 } else {
1804 si_set_user_data_base(sctx, PIPE_SHADER_TESS_EVAL, 0);
1805 }
1806 }
1807
1808 static void si_emit_shader_pointer(struct si_context *sctx,
1809 struct si_descriptors *desc,
1810 unsigned sh_base)
1811 {
1812 struct radeon_winsys_cs *cs = sctx->b.gfx.cs;
1813 uint64_t va;
1814
1815 assert(desc->buffer);
1816
1817 va = desc->buffer->gpu_address +
1818 desc->buffer_offset;
1819
1820 radeon_emit(cs, PKT3(PKT3_SET_SH_REG, 2, 0));
1821 radeon_emit(cs, (sh_base + desc->shader_userdata_offset - SI_SH_REG_OFFSET) >> 2);
1822 radeon_emit(cs, va);
1823 radeon_emit(cs, va >> 32);
1824 }
1825
1826 void si_emit_graphics_shader_userdata(struct si_context *sctx,
1827 struct r600_atom *atom)
1828 {
1829 unsigned mask;
1830 uint32_t *sh_base = sctx->shader_userdata.sh_base;
1831 struct si_descriptors *descs;
1832
1833 descs = &sctx->descriptors[SI_DESCS_RW_BUFFERS];
1834
1835 if (sctx->shader_pointers_dirty & (1 << SI_DESCS_RW_BUFFERS)) {
1836 si_emit_shader_pointer(sctx, descs,
1837 R_00B030_SPI_SHADER_USER_DATA_PS_0);
1838 si_emit_shader_pointer(sctx, descs,
1839 R_00B130_SPI_SHADER_USER_DATA_VS_0);
1840 si_emit_shader_pointer(sctx, descs,
1841 R_00B230_SPI_SHADER_USER_DATA_GS_0);
1842 si_emit_shader_pointer(sctx, descs,
1843 R_00B330_SPI_SHADER_USER_DATA_ES_0);
1844 si_emit_shader_pointer(sctx, descs,
1845 R_00B430_SPI_SHADER_USER_DATA_HS_0);
1846 }
1847
1848 mask = sctx->shader_pointers_dirty &
1849 u_bit_consecutive(SI_DESCS_FIRST_SHADER,
1850 SI_DESCS_FIRST_COMPUTE - SI_DESCS_FIRST_SHADER);
1851
1852 while (mask) {
1853 unsigned i = u_bit_scan(&mask);
1854 unsigned shader = (i - SI_DESCS_FIRST_SHADER) / SI_NUM_SHADER_DESCS;
1855 unsigned base = sh_base[shader];
1856
1857 if (base)
1858 si_emit_shader_pointer(sctx, descs + i, base);
1859 }
1860 sctx->shader_pointers_dirty &=
1861 ~u_bit_consecutive(SI_DESCS_RW_BUFFERS, SI_DESCS_FIRST_COMPUTE);
1862
1863 if (sctx->vertex_buffer_pointer_dirty) {
1864 si_emit_shader_pointer(sctx, &sctx->vertex_buffers,
1865 sh_base[PIPE_SHADER_VERTEX]);
1866 sctx->vertex_buffer_pointer_dirty = false;
1867 }
1868 }
1869
1870 void si_emit_compute_shader_userdata(struct si_context *sctx)
1871 {
1872 unsigned base = R_00B900_COMPUTE_USER_DATA_0;
1873 struct si_descriptors *descs = sctx->descriptors;
1874 unsigned compute_mask =
1875 u_bit_consecutive(SI_DESCS_FIRST_COMPUTE, SI_NUM_SHADER_DESCS);
1876 unsigned mask = sctx->shader_pointers_dirty & compute_mask;
1877
1878 while (mask) {
1879 unsigned i = u_bit_scan(&mask);
1880
1881 si_emit_shader_pointer(sctx, descs + i, base);
1882 }
1883 sctx->shader_pointers_dirty &= ~compute_mask;
1884 }
1885
1886 /* INIT/DEINIT/UPLOAD */
1887
1888 void si_init_all_descriptors(struct si_context *sctx)
1889 {
1890 int i;
1891 unsigned ce_offset = 0;
1892
1893 for (i = 0; i < SI_NUM_SHADERS; i++) {
1894 si_init_buffer_resources(&sctx->const_buffers[i],
1895 si_const_buffer_descriptors(sctx, i),
1896 SI_NUM_CONST_BUFFERS, SI_SGPR_CONST_BUFFERS,
1897 RADEON_USAGE_READ, RADEON_PRIO_CONST_BUFFER,
1898 &ce_offset);
1899 si_init_buffer_resources(&sctx->shader_buffers[i],
1900 si_shader_buffer_descriptors(sctx, i),
1901 SI_NUM_SHADER_BUFFERS, SI_SGPR_SHADER_BUFFERS,
1902 RADEON_USAGE_READWRITE, RADEON_PRIO_SHADER_RW_BUFFER,
1903 &ce_offset);
1904
1905 si_init_descriptors(si_sampler_descriptors(sctx, i),
1906 SI_SGPR_SAMPLERS, 16, SI_NUM_SAMPLERS,
1907 null_texture_descriptor, &ce_offset);
1908
1909 si_init_descriptors(si_image_descriptors(sctx, i),
1910 SI_SGPR_IMAGES, 8, SI_NUM_IMAGES,
1911 null_image_descriptor, &ce_offset);
1912 }
1913
1914 si_init_buffer_resources(&sctx->rw_buffers,
1915 &sctx->descriptors[SI_DESCS_RW_BUFFERS],
1916 SI_NUM_RW_BUFFERS, SI_SGPR_RW_BUFFERS,
1917 RADEON_USAGE_READWRITE, RADEON_PRIO_SHADER_RINGS,
1918 &ce_offset);
1919 si_init_descriptors(&sctx->vertex_buffers, SI_SGPR_VERTEX_BUFFERS,
1920 4, SI_NUM_VERTEX_BUFFERS, NULL, NULL);
1921
1922 sctx->descriptors_dirty = u_bit_consecutive(0, SI_NUM_DESCS);
1923
1924 assert(ce_offset <= 32768);
1925
1926 /* Set pipe_context functions. */
1927 sctx->b.b.bind_sampler_states = si_bind_sampler_states;
1928 sctx->b.b.set_shader_images = si_set_shader_images;
1929 sctx->b.b.set_constant_buffer = si_pipe_set_constant_buffer;
1930 sctx->b.b.set_polygon_stipple = si_set_polygon_stipple;
1931 sctx->b.b.set_shader_buffers = si_set_shader_buffers;
1932 sctx->b.b.set_sampler_views = si_set_sampler_views;
1933 sctx->b.b.set_stream_output_targets = si_set_streamout_targets;
1934 sctx->b.invalidate_buffer = si_invalidate_buffer;
1935
1936 /* Shader user data. */
1937 si_init_atom(sctx, &sctx->shader_userdata.atom, &sctx->atoms.s.shader_userdata,
1938 si_emit_graphics_shader_userdata);
1939
1940 /* Set default and immutable mappings. */
1941 si_set_user_data_base(sctx, PIPE_SHADER_VERTEX, R_00B130_SPI_SHADER_USER_DATA_VS_0);
1942 si_set_user_data_base(sctx, PIPE_SHADER_TESS_CTRL, R_00B430_SPI_SHADER_USER_DATA_HS_0);
1943 si_set_user_data_base(sctx, PIPE_SHADER_GEOMETRY, R_00B230_SPI_SHADER_USER_DATA_GS_0);
1944 si_set_user_data_base(sctx, PIPE_SHADER_FRAGMENT, R_00B030_SPI_SHADER_USER_DATA_PS_0);
1945 }
1946
1947 bool si_upload_graphics_shader_descriptors(struct si_context *sctx)
1948 {
1949 const unsigned mask = u_bit_consecutive(0, SI_DESCS_FIRST_COMPUTE);
1950 unsigned dirty = sctx->descriptors_dirty & mask;
1951
1952 /* Assume nothing will go wrong: */
1953 sctx->shader_pointers_dirty |= dirty;
1954
1955 while (dirty) {
1956 unsigned i = u_bit_scan(&dirty);
1957
1958 if (!si_upload_descriptors(sctx, &sctx->descriptors[i],
1959 &sctx->shader_userdata.atom))
1960 return false;
1961 }
1962
1963 sctx->descriptors_dirty &= ~mask;
1964 return true;
1965 }
1966
1967 bool si_upload_compute_shader_descriptors(struct si_context *sctx)
1968 {
1969 /* Does not update rw_buffers as that is not needed for compute shaders
1970 * and the input buffer is using the same SGPR's anyway.
1971 */
1972 const unsigned mask = u_bit_consecutive(SI_DESCS_FIRST_COMPUTE,
1973 SI_NUM_DESCS - SI_DESCS_FIRST_COMPUTE);
1974 unsigned dirty = sctx->descriptors_dirty & mask;
1975
1976 /* Assume nothing will go wrong: */
1977 sctx->shader_pointers_dirty |= dirty;
1978
1979 while (dirty) {
1980 unsigned i = u_bit_scan(&dirty);
1981
1982 if (!si_upload_descriptors(sctx, &sctx->descriptors[i], NULL))
1983 return false;
1984 }
1985
1986 sctx->descriptors_dirty &= ~mask;
1987
1988 return true;
1989 }
1990
1991 void si_release_all_descriptors(struct si_context *sctx)
1992 {
1993 int i;
1994
1995 for (i = 0; i < SI_NUM_SHADERS; i++) {
1996 si_release_buffer_resources(&sctx->const_buffers[i],
1997 si_const_buffer_descriptors(sctx, i));
1998 si_release_buffer_resources(&sctx->shader_buffers[i],
1999 si_shader_buffer_descriptors(sctx, i));
2000 si_release_sampler_views(&sctx->samplers[i].views);
2001 si_release_image_views(&sctx->images[i]);
2002 }
2003 si_release_buffer_resources(&sctx->rw_buffers,
2004 &sctx->descriptors[SI_DESCS_RW_BUFFERS]);
2005
2006 for (i = 0; i < SI_NUM_DESCS; ++i)
2007 si_release_descriptors(&sctx->descriptors[i]);
2008 si_release_descriptors(&sctx->vertex_buffers);
2009 }
2010
2011 void si_all_descriptors_begin_new_cs(struct si_context *sctx)
2012 {
2013 int i;
2014
2015 for (i = 0; i < SI_NUM_SHADERS; i++) {
2016 si_buffer_resources_begin_new_cs(sctx, &sctx->const_buffers[i]);
2017 si_buffer_resources_begin_new_cs(sctx, &sctx->shader_buffers[i]);
2018 si_sampler_views_begin_new_cs(sctx, &sctx->samplers[i].views);
2019 si_image_views_begin_new_cs(sctx, &sctx->images[i]);
2020 }
2021 si_buffer_resources_begin_new_cs(sctx, &sctx->rw_buffers);
2022 si_vertex_buffers_begin_new_cs(sctx);
2023
2024 for (i = 0; i < SI_NUM_DESCS; ++i)
2025 si_descriptors_begin_new_cs(sctx, &sctx->descriptors[i]);
2026
2027 si_shader_userdata_begin_new_cs(sctx);
2028 }