projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
radeonsi: release saved resources in si_compute_clear_render_target
[mesa.git] / src / gallium / drivers / radeonsi / si_compute_blit.c
1 /*
2 * Copyright 2018 Advanced Micro Devices, Inc.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * on the rights to use, copy, modify, merge, publish, distribute, sub
9 * license, and/or sell copies of the Software, and to permit persons to whom
10 * the Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
23 *
24 */
25
26 #include "si_pipe.h"
27 #include "util/format/u_format.h"
28 #include "util/format_srgb.h"
29
30 /* Note: Compute shaders always use SI_COMPUTE_DST_CACHE_POLICY for dst
31 * and L2_STREAM for src.
32 */
33 static enum si_cache_policy get_cache_policy(struct si_context *sctx,
34 enum si_coherency coher,
35 uint64_t size)
36 {
37 if ((sctx->chip_class >= GFX9 && (coher == SI_COHERENCY_CB_META ||
38 coher == SI_COHERENCY_CP)) ||
39 (sctx->chip_class >= GFX7 && coher == SI_COHERENCY_SHADER))
40 return size <= 256 * 1024 ? L2_LRU : L2_STREAM;
41
42 return L2_BYPASS;
43 }
44
45 unsigned si_get_flush_flags(struct si_context *sctx, enum si_coherency coher,
46 enum si_cache_policy cache_policy)
47 {
48 switch (coher) {
49 default:
50 case SI_COHERENCY_NONE:
51 case SI_COHERENCY_CP:
52 return 0;
53 case SI_COHERENCY_SHADER:
54 return SI_CONTEXT_INV_SCACHE |
55 SI_CONTEXT_INV_VCACHE |
56 (cache_policy == L2_BYPASS ? SI_CONTEXT_INV_L2 : 0);
57 case SI_COHERENCY_CB_META:
58 return SI_CONTEXT_FLUSH_AND_INV_CB;
59 }
60 }
61
62 static void si_compute_internal_begin(struct si_context *sctx)
63 {
64 sctx->flags &= ~SI_CONTEXT_START_PIPELINE_STATS;
65 sctx->flags |= SI_CONTEXT_STOP_PIPELINE_STATS;
66 sctx->render_cond_force_off = true;
67 }
68
69 static void si_compute_internal_end(struct si_context *sctx)
70 {
71 sctx->flags &= ~SI_CONTEXT_STOP_PIPELINE_STATS;
72 sctx->flags |= SI_CONTEXT_START_PIPELINE_STATS;
73 sctx->render_cond_force_off = false;
74 }
75
76 static void si_compute_clear_12bytes_buffer(struct si_context *sctx,
77 struct pipe_resource *dst,
78 unsigned dst_offset,
79 unsigned size,
80 const uint32_t *clear_value,
81 enum si_coherency coher)
82 {
83 struct pipe_context *ctx = &sctx->b;
84
85 assert(dst_offset % 4 == 0);
86 assert(size % 4 == 0);
87 unsigned size_12 = DIV_ROUND_UP(size, 12);
88
89 unsigned data[4] = {0};
90 memcpy(data, clear_value, 12);
91
92 si_compute_internal_begin(sctx);
93
94 sctx->flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
95 SI_CONTEXT_CS_PARTIAL_FLUSH |
96 si_get_flush_flags(sctx, coher, SI_COMPUTE_DST_CACHE_POLICY);
97
98 struct pipe_shader_buffer saved_sb = {0};
99 si_get_shader_buffers(sctx, PIPE_SHADER_COMPUTE, 0, 1, &saved_sb);
100
101 unsigned saved_writable_mask = 0;
102 if (sctx->const_and_shader_buffers[PIPE_SHADER_COMPUTE].writable_mask &
103 (1u << si_get_shaderbuf_slot(0)))
104 saved_writable_mask = 1;
105
106 struct pipe_constant_buffer saved_cb = {};
107 si_get_pipe_constant_buffer(sctx, PIPE_SHADER_COMPUTE, 0, &saved_cb);
108
109 void *saved_cs = sctx->cs_shader_state.program;
110
111 struct pipe_constant_buffer cb = {};
112 cb.buffer_size = sizeof(data);
113 cb.user_buffer = data;
114 ctx->set_constant_buffer(ctx, PIPE_SHADER_COMPUTE, 0, &cb);
115
116 struct pipe_shader_buffer sb = {0};
117 sb.buffer = dst;
118 sb.buffer_offset = dst_offset;
119 sb.buffer_size = size;
120
121 ctx->set_shader_buffers(ctx, PIPE_SHADER_COMPUTE, 0, 1, &sb, 0x1);
122
123 struct pipe_grid_info info = {0};
124
125 if (!sctx->cs_clear_12bytes_buffer)
126 sctx->cs_clear_12bytes_buffer =
127 si_clear_12bytes_buffer_shader(ctx);
128 ctx->bind_compute_state(ctx, sctx->cs_clear_12bytes_buffer);
129 info.block[0] = 64;
130 info.last_block[0] = size_12 % 64;
131 info.block[1] = 1;
132 info.block[2] = 1;
133 info.grid[0] = DIV_ROUND_UP(size_12, 64);
134 info.grid[1] = 1;
135 info.grid[2] = 1;
136
137 ctx->launch_grid(ctx, &info);
138
139 ctx->bind_compute_state(ctx, saved_cs);
140 ctx->set_shader_buffers(ctx, PIPE_SHADER_COMPUTE, 0, 1, &saved_sb, saved_writable_mask);
141 ctx->set_constant_buffer(ctx, PIPE_SHADER_COMPUTE, 0, &saved_cb);
142
143 si_compute_internal_end(sctx);
144 }
145
146 static void si_compute_do_clear_or_copy(struct si_context *sctx,
147 struct pipe_resource *dst,
148 unsigned dst_offset,
149 struct pipe_resource *src,
150 unsigned src_offset,
151 unsigned size,
152 const uint32_t *clear_value,
153 unsigned clear_value_size,
154 enum si_coherency coher)
155 {
156 struct pipe_context *ctx = &sctx->b;
157
158 assert(src_offset % 4 == 0);
159 assert(dst_offset % 4 == 0);
160 assert(size % 4 == 0);
161
162 assert(dst->target != PIPE_BUFFER || dst_offset + size <= dst->width0);
163 assert(!src || src_offset + size <= src->width0);
164
165 si_compute_internal_begin(sctx);
166 sctx->flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
167 SI_CONTEXT_CS_PARTIAL_FLUSH |
168 si_get_flush_flags(sctx, coher, SI_COMPUTE_DST_CACHE_POLICY);
169
170 /* Save states. */
171 void *saved_cs = sctx->cs_shader_state.program;
172 struct pipe_shader_buffer saved_sb[2] = {};
173 si_get_shader_buffers(sctx, PIPE_SHADER_COMPUTE, 0, src ? 2 : 1, saved_sb);
174
175 unsigned saved_writable_mask = 0;
176 for (unsigned i = 0; i < (src ? 2 : 1); i++) {
177 if (sctx->const_and_shader_buffers[PIPE_SHADER_COMPUTE].writable_mask &
178 (1u << si_get_shaderbuf_slot(i)))
179 saved_writable_mask |= 1 << i;
180 }
181
182 /* The memory accesses are coalesced, meaning that the 1st instruction writes
183 * the 1st contiguous block of data for the whole wave, the 2nd instruction
184 * writes the 2nd contiguous block of data, etc.
185 */
186 unsigned dwords_per_thread = src ? SI_COMPUTE_COPY_DW_PER_THREAD :
187 SI_COMPUTE_CLEAR_DW_PER_THREAD;
188 unsigned instructions_per_thread = MAX2(1, dwords_per_thread / 4);
189 unsigned dwords_per_instruction = dwords_per_thread / instructions_per_thread;
190 unsigned wave_size = sctx->screen->compute_wave_size;
191 unsigned dwords_per_wave = dwords_per_thread * wave_size;
192
193 unsigned num_dwords = size / 4;
194 unsigned num_instructions = DIV_ROUND_UP(num_dwords, dwords_per_instruction);
195
196 struct pipe_grid_info info = {};
197 info.block[0] = MIN2(wave_size, num_instructions);
198 info.block[1] = 1;
199 info.block[2] = 1;
200 info.grid[0] = DIV_ROUND_UP(num_dwords, dwords_per_wave);
201 info.grid[1] = 1;
202 info.grid[2] = 1;
203
204 struct pipe_shader_buffer sb[2] = {};
205 sb[0].buffer = dst;
206 sb[0].buffer_offset = dst_offset;
207 sb[0].buffer_size = size;
208
209 bool shader_dst_stream_policy = SI_COMPUTE_DST_CACHE_POLICY != L2_LRU;
210
211 if (src) {
212 sb[1].buffer = src;
213 sb[1].buffer_offset = src_offset;
214 sb[1].buffer_size = size;
215
216 ctx->set_shader_buffers(ctx, PIPE_SHADER_COMPUTE, 0, 2, sb, 0x1);
217
218 if (!sctx->cs_copy_buffer) {
219 sctx->cs_copy_buffer = si_create_dma_compute_shader(&sctx->b,
220 SI_COMPUTE_COPY_DW_PER_THREAD,
221 shader_dst_stream_policy, true);
222 }
223 ctx->bind_compute_state(ctx, sctx->cs_copy_buffer);
224 } else {
225 assert(clear_value_size >= 4 &&
226 clear_value_size <= 16 &&
227 util_is_power_of_two_or_zero(clear_value_size));
228
229 for (unsigned i = 0; i < 4; i++)
230 sctx->cs_user_data[i] = clear_value[i % (clear_value_size / 4)];
231
232 ctx->set_shader_buffers(ctx, PIPE_SHADER_COMPUTE, 0, 1, sb, 0x1);
233
234 if (!sctx->cs_clear_buffer) {
235 sctx->cs_clear_buffer = si_create_dma_compute_shader(&sctx->b,
236 SI_COMPUTE_CLEAR_DW_PER_THREAD,
237 shader_dst_stream_policy, false);
238 }
239 ctx->bind_compute_state(ctx, sctx->cs_clear_buffer);
240 }
241
242 ctx->launch_grid(ctx, &info);
243
244 enum si_cache_policy cache_policy = get_cache_policy(sctx, coher, size);
245 sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH |
246 (cache_policy == L2_BYPASS ? SI_CONTEXT_WB_L2 : 0);
247
248 if (cache_policy != L2_BYPASS)
249 si_resource(dst)->TC_L2_dirty = true;
250
251 /* Restore states. */
252 ctx->bind_compute_state(ctx, saved_cs);
253 ctx->set_shader_buffers(ctx, PIPE_SHADER_COMPUTE, 0, src ? 2 : 1, saved_sb,
254 saved_writable_mask);
255 si_compute_internal_end(sctx);
256 }
257
258 void si_clear_buffer(struct si_context *sctx, struct pipe_resource *dst,
259 uint64_t offset, uint64_t size, uint32_t *clear_value,
260 uint32_t clear_value_size, enum si_coherency coher,
261 bool force_cpdma)
262 {
263 if (!size)
264 return;
265
266 ASSERTED unsigned clear_alignment = MIN2(clear_value_size, 4);
267
268 assert(clear_value_size != 3 && clear_value_size != 6); /* 12 is allowed. */
269 assert(offset % clear_alignment == 0);
270 assert(size % clear_alignment == 0);
271 assert(size < (UINT_MAX & ~0xf)); /* TODO: test 64-bit sizes in all codepaths */
272
273 /* Reduce a large clear value size if possible. */
274 if (clear_value_size > 4) {
275 bool clear_dword_duplicated = true;
276
277 /* See if we can lower large fills to dword fills. */
278 for (unsigned i = 1; i < clear_value_size / 4; i++) {
279 if (clear_value[0] != clear_value[i]) {
280 clear_dword_duplicated = false;
281 break;
282 }
283 }
284 if (clear_dword_duplicated)
285 clear_value_size = 4;
286 }
287
288 /* Expand a small clear value size. */
289 uint32_t tmp_clear_value;
290 if (clear_value_size <= 2) {
291 if (clear_value_size == 1) {
292 tmp_clear_value = *(uint8_t*)clear_value;
293 tmp_clear_value |= (tmp_clear_value << 8) |
294 (tmp_clear_value << 16) |
295 (tmp_clear_value << 24);
296 } else {
297 tmp_clear_value = *(uint16_t*)clear_value;
298 tmp_clear_value |= tmp_clear_value << 16;
299 }
300 clear_value = &tmp_clear_value;
301 clear_value_size = 4;
302 }
303
304 if (clear_value_size == 12) {
305 si_compute_clear_12bytes_buffer(sctx, dst, offset, size, clear_value, coher);
306 return;
307 }
308
309 uint64_t aligned_size = size & ~3ull;
310 if (aligned_size >= 4) {
311 /* Before GFX9, CP DMA was very slow when clearing GTT, so never
312 * use CP DMA clears on those chips, because we can't be certain
313 * about buffer placements.
314 */
315 if (clear_value_size > 4 ||
316 (!force_cpdma &&
317 clear_value_size == 4 &&
318 offset % 4 == 0 &&
319 (size > 32*1024 || sctx->chip_class <= GFX8))) {
320 si_compute_do_clear_or_copy(sctx, dst, offset, NULL, 0,
321 aligned_size, clear_value,
322 clear_value_size, coher);
323 } else {
324 assert(clear_value_size == 4);
325 si_cp_dma_clear_buffer(sctx, sctx->gfx_cs, dst, offset,
326 aligned_size, *clear_value, 0, coher,
327 get_cache_policy(sctx, coher, size));
328 }
329
330 offset += aligned_size;
331 size -= aligned_size;
332 }
333
334 /* Handle non-dword alignment. */
335 if (size) {
336 assert(dst);
337 assert(dst->target == PIPE_BUFFER);
338 assert(size < 4);
339
340 pipe_buffer_write(&sctx->b, dst, offset, size, clear_value);
341 }
342 }
343
344 static void si_pipe_clear_buffer(struct pipe_context *ctx,
345 struct pipe_resource *dst,
346 unsigned offset, unsigned size,
347 const void *clear_value,
348 int clear_value_size)
349 {
350 si_clear_buffer((struct si_context*)ctx, dst, offset, size, (uint32_t*)clear_value,
351 clear_value_size, SI_COHERENCY_SHADER, false);
352 }
353
354 void si_copy_buffer(struct si_context *sctx,
355 struct pipe_resource *dst, struct pipe_resource *src,
356 uint64_t dst_offset, uint64_t src_offset, unsigned size)
357 {
358 if (!size)
359 return;
360
361 enum si_coherency coher = SI_COHERENCY_SHADER;
362 enum si_cache_policy cache_policy = get_cache_policy(sctx, coher, size);
363
364 /* Only use compute for VRAM copies on dGPUs. */
365 if (sctx->screen->info.has_dedicated_vram &&
366 si_resource(dst)->domains & RADEON_DOMAIN_VRAM &&
367 si_resource(src)->domains & RADEON_DOMAIN_VRAM &&
368 size > 32 * 1024 &&
369 dst_offset % 4 == 0 && src_offset % 4 == 0 && size % 4 == 0) {
370 si_compute_do_clear_or_copy(sctx, dst, dst_offset, src, src_offset,
371 size, NULL, 0, coher);
372 } else {
373 si_cp_dma_copy_buffer(sctx, dst, src, dst_offset, src_offset, size,
374 0, coher, cache_policy);
375 }
376 }
377
378 void si_compute_copy_image(struct si_context *sctx,
379 struct pipe_resource *dst,
380 unsigned dst_level,
381 struct pipe_resource *src,
382 unsigned src_level,
383 unsigned dstx, unsigned dsty, unsigned dstz,
384 const struct pipe_box *src_box)
385 {
386 struct pipe_context *ctx = &sctx->b;
387 unsigned width = src_box->width;
388 unsigned height = src_box->height;
389 unsigned depth = src_box->depth;
390
391 unsigned data[] = {src_box->x, src_box->y, src_box->z, 0, dstx, dsty, dstz, 0};
392
393 if (width == 0 || height == 0)
394 return;
395
396 si_compute_internal_begin(sctx);
397 sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH |
398 si_get_flush_flags(sctx, SI_COHERENCY_SHADER, L2_STREAM);
399
400 /* src and dst have the same number of samples. */
401 si_make_CB_shader_coherent(sctx, src->nr_samples, true,
402 /* Only src can have DCC.*/
403 ((struct si_texture*)src)->surface.u.gfx9.dcc.pipe_aligned);
404
405 struct pipe_constant_buffer saved_cb = {};
406 si_get_pipe_constant_buffer(sctx, PIPE_SHADER_COMPUTE, 0, &saved_cb);
407
408 struct si_images *images = &sctx->images[PIPE_SHADER_COMPUTE];
409 struct pipe_image_view saved_image[2] = {0};
410 util_copy_image_view(&saved_image[0], &images->views[0]);
411 util_copy_image_view(&saved_image[1], &images->views[1]);
412
413 void *saved_cs = sctx->cs_shader_state.program;
414
415 struct pipe_constant_buffer cb = {};
416 cb.buffer_size = sizeof(data);
417 cb.user_buffer = data;
418 ctx->set_constant_buffer(ctx, PIPE_SHADER_COMPUTE, 0, &cb);
419
420 struct pipe_image_view image[2] = {0};
421 image[0].resource = src;
422 image[0].shader_access = image[0].access = PIPE_IMAGE_ACCESS_READ;
423 image[0].format = util_format_linear(src->format);
424 image[0].u.tex.level = src_level;
425 image[0].u.tex.first_layer = 0;
426 image[0].u.tex.last_layer =
427 src->target == PIPE_TEXTURE_3D ? u_minify(src->depth0, src_level) - 1
428 : (unsigned)(src->array_size - 1);
429 image[1].resource = dst;
430 image[1].shader_access = image[1].access = PIPE_IMAGE_ACCESS_WRITE;
431 image[1].format = util_format_linear(dst->format);
432 image[1].u.tex.level = dst_level;
433 image[1].u.tex.first_layer = 0;
434 image[1].u.tex.last_layer =
435 dst->target == PIPE_TEXTURE_3D ? u_minify(dst->depth0, dst_level) - 1
436 : (unsigned)(dst->array_size - 1);
437
438 if (src->format == PIPE_FORMAT_R9G9B9E5_FLOAT)
439 image[0].format = image[1].format = PIPE_FORMAT_R32_UINT;
440
441 /* SNORM8 blitting has precision issues on some chips. Use the SINT
442 * equivalent instead, which doesn't force DCC decompression.
443 * Note that some chips avoid this issue by using SDMA.
444 */
445 if (util_format_is_snorm8(dst->format)) {
446 image[0].format = image[1].format =
447 util_format_snorm8_to_sint8(dst->format);
448 }
449
450 ctx->set_shader_images(ctx, PIPE_SHADER_COMPUTE, 0, 2, image);
451
452 struct pipe_grid_info info = {0};
453
454 if (dst->target == PIPE_TEXTURE_1D_ARRAY && src->target == PIPE_TEXTURE_1D_ARRAY) {
455 if (!sctx->cs_copy_image_1d_array)
456 sctx->cs_copy_image_1d_array =
457 si_create_copy_image_compute_shader_1d_array(ctx);
458 ctx->bind_compute_state(ctx, sctx->cs_copy_image_1d_array);
459 info.block[0] = 64;
460 info.last_block[0] = width % 64;
461 info.block[1] = 1;
462 info.block[2] = 1;
463 info.grid[0] = DIV_ROUND_UP(width, 64);
464 info.grid[1] = depth;
465 info.grid[2] = 1;
466 } else {
467 if (!sctx->cs_copy_image)
468 sctx->cs_copy_image = si_create_copy_image_compute_shader(ctx);
469 ctx->bind_compute_state(ctx, sctx->cs_copy_image);
470 info.block[0] = 8;
471 info.last_block[0] = width % 8;
472 info.block[1] = 8;
473 info.last_block[1] = height % 8;
474 info.block[2] = 1;
475 info.grid[0] = DIV_ROUND_UP(width, 8);
476 info.grid[1] = DIV_ROUND_UP(height, 8);
477 info.grid[2] = depth;
478 }
479
480 ctx->launch_grid(ctx, &info);
481
482 sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH |
483 (sctx->chip_class <= GFX8 ? SI_CONTEXT_WB_L2 : 0) |
484 si_get_flush_flags(sctx, SI_COHERENCY_SHADER, L2_STREAM);
485 ctx->bind_compute_state(ctx, saved_cs);
486 ctx->set_shader_images(ctx, PIPE_SHADER_COMPUTE, 0, 2, saved_image);
487 ctx->set_constant_buffer(ctx, PIPE_SHADER_COMPUTE, 0, &saved_cb);
488 si_compute_internal_end(sctx);
489 }
490
491 void si_retile_dcc(struct si_context *sctx, struct si_texture *tex)
492 {
493 struct pipe_context *ctx = &sctx->b;
494
495 sctx->flags |= SI_CONTEXT_PS_PARTIAL_FLUSH |
496 SI_CONTEXT_CS_PARTIAL_FLUSH |
497 si_get_flush_flags(sctx, SI_COHERENCY_CB_META, L2_LRU) |
498 si_get_flush_flags(sctx, SI_COHERENCY_SHADER, L2_LRU);
499 sctx->emit_cache_flush(sctx);
500
501 /* Save states. */
502 void *saved_cs = sctx->cs_shader_state.program;
503 struct pipe_image_view saved_img[3] = {};
504
505 for (unsigned i = 0; i < 3; i++) {
506 util_copy_image_view(&saved_img[i],
507 &sctx->images[PIPE_SHADER_COMPUTE].views[i]);
508 }
509
510 /* Set images. */
511 bool use_uint16 = tex->surface.u.gfx9.dcc_retile_use_uint16;
512 unsigned num_elements = tex->surface.u.gfx9.dcc_retile_num_elements;
513 struct pipe_image_view img[3];
514
515 assert(tex->surface.dcc_retile_map_offset && tex->surface.dcc_retile_map_offset <= UINT_MAX);
516 assert(tex->surface.dcc_offset && tex->surface.dcc_offset <= UINT_MAX);
517 assert(tex->surface.display_dcc_offset && tex->surface.display_dcc_offset <= UINT_MAX);
518
519 for (unsigned i = 0; i < 3; i++) {
520 img[i].resource = &tex->buffer.b.b;
521 img[i].access = i == 2 ? PIPE_IMAGE_ACCESS_WRITE : PIPE_IMAGE_ACCESS_READ;
522 img[i].shader_access = SI_IMAGE_ACCESS_AS_BUFFER;
523 }
524
525 img[0].format = use_uint16 ? PIPE_FORMAT_R16G16B16A16_UINT :
526 PIPE_FORMAT_R32G32B32A32_UINT;
527 img[0].u.buf.offset = tex->surface.dcc_retile_map_offset;
528 img[0].u.buf.size = num_elements * (use_uint16 ? 2 : 4);
529
530 img[1].format = PIPE_FORMAT_R8_UINT;
531 img[1].u.buf.offset = tex->surface.dcc_offset;
532 img[1].u.buf.size = tex->surface.dcc_size;
533
534 img[2].format = PIPE_FORMAT_R8_UINT;
535 img[2].u.buf.offset = tex->surface.display_dcc_offset;
536 img[2].u.buf.size = tex->surface.u.gfx9.display_dcc_size;
537
538 ctx->set_shader_images(ctx, PIPE_SHADER_COMPUTE, 0, 3, img);
539
540 /* Bind the compute shader. */
541 if (!sctx->cs_dcc_retile)
542 sctx->cs_dcc_retile = si_create_dcc_retile_cs(ctx);
543 ctx->bind_compute_state(ctx, sctx->cs_dcc_retile);
544
545 /* Dispatch compute. */
546 /* img[0] has 4 channels per element containing 2 pairs of DCC offsets. */
547 unsigned num_threads = num_elements / 4;
548
549 struct pipe_grid_info info = {};
550 info.block[0] = 64;
551 info.block[1] = 1;
552 info.block[2] = 1;
553 info.grid[0] = DIV_ROUND_UP(num_threads, 64); /* includes the partial block */
554 info.grid[1] = 1;
555 info.grid[2] = 1;
556 info.last_block[0] = num_threads % 64;
557
558 ctx->launch_grid(ctx, &info);
559
560 /* Don't flush caches or wait. The driver will wait at the end of this IB,
561 * and L2 will be flushed by the kernel fence.
562 */
563
564 /* Restore states. */
565 ctx->bind_compute_state(ctx, saved_cs);
566 ctx->set_shader_images(ctx, PIPE_SHADER_COMPUTE, 0, 3, saved_img);
567
568 for (unsigned i = 0; i < 3; i++) {
569 pipe_resource_reference(&saved_img[i].resource, NULL);
570 }
571 }
572
573 /* Expand FMASK to make it identity, so that image stores can ignore it. */
574 void si_compute_expand_fmask(struct pipe_context *ctx, struct pipe_resource *tex)
575 {
576 struct si_context *sctx = (struct si_context *)ctx;
577 bool is_array = tex->target == PIPE_TEXTURE_2D_ARRAY;
578 unsigned log_fragments = util_logbase2(tex->nr_storage_samples);
579 unsigned log_samples = util_logbase2(tex->nr_samples);
580 assert(tex->nr_samples >= 2);
581
582 /* EQAA FMASK expansion is unimplemented. */
583 if (tex->nr_samples != tex->nr_storage_samples)
584 return;
585
586 si_compute_internal_begin(sctx);
587
588 /* Flush caches and sync engines. */
589 sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH |
590 si_get_flush_flags(sctx, SI_COHERENCY_SHADER, L2_STREAM);
591 si_make_CB_shader_coherent(sctx, tex->nr_samples, true,
592 true /* DCC is not possible with image stores */);
593
594 /* Save states. */
595 void *saved_cs = sctx->cs_shader_state.program;
596 struct pipe_image_view saved_image = {0};
597 util_copy_image_view(&saved_image, &sctx->images[PIPE_SHADER_COMPUTE].views[0]);
598
599 /* Bind the image. */
600 struct pipe_image_view image = {0};
601 image.resource = tex;
602 /* Don't set WRITE so as not to trigger FMASK expansion, causing
603 * an infinite loop. */
604 image.shader_access = image.access = PIPE_IMAGE_ACCESS_READ;
605 image.format = util_format_linear(tex->format);
606 if (is_array)
607 image.u.tex.last_layer = tex->array_size - 1;
608
609 ctx->set_shader_images(ctx, PIPE_SHADER_COMPUTE, 0, 1, &image);
610
611 /* Bind the shader. */
612 void **shader = &sctx->cs_fmask_expand[log_samples - 1][is_array];
613 if (!*shader)
614 *shader = si_create_fmask_expand_cs(ctx, tex->nr_samples, is_array);
615 ctx->bind_compute_state(ctx, *shader);
616
617 /* Dispatch compute. */
618 struct pipe_grid_info info = {0};
619 info.block[0] = 8;
620 info.last_block[0] = tex->width0 % 8;
621 info.block[1] = 8;
622 info.last_block[1] = tex->height0 % 8;
623 info.block[2] = 1;
624 info.grid[0] = DIV_ROUND_UP(tex->width0, 8);
625 info.grid[1] = DIV_ROUND_UP(tex->height0, 8);
626 info.grid[2] = is_array ? tex->array_size : 1;
627
628 ctx->launch_grid(ctx, &info);
629
630 /* Flush caches and sync engines. */
631 sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH |
632 (sctx->chip_class <= GFX8 ? SI_CONTEXT_WB_L2 : 0) |
633 si_get_flush_flags(sctx, SI_COHERENCY_SHADER, L2_STREAM);
634
635 /* Restore previous states. */
636 ctx->bind_compute_state(ctx, saved_cs);
637 ctx->set_shader_images(ctx, PIPE_SHADER_COMPUTE, 0, 1, &saved_image);
638 si_compute_internal_end(sctx);
639 pipe_resource_reference(&saved_image.resource, NULL);
640
641 /* Array of fully expanded FMASK values, arranged by [log2(fragments)][log2(samples)-1]. */
642 #define INVALID 0 /* never used */
643 static const uint64_t fmask_expand_values[][4] = {
644 /* samples */
645 /* 2 (8 bpp) 4 (8 bpp) 8 (8-32bpp) 16 (16-64bpp) fragments */
646 {0x02020202, 0x0E0E0E0E, 0xFEFEFEFE, 0xFFFEFFFE}, /* 1 */
647 {0x02020202, 0xA4A4A4A4, 0xAAA4AAA4, 0xAAAAAAA4}, /* 2 */
648 {INVALID, 0xE4E4E4E4, 0x44443210, 0x4444444444443210}, /* 4 */
649 {INVALID, INVALID, 0x76543210, 0x8888888876543210}, /* 8 */
650 };
651
652 /* Clear FMASK to identity. */
653 struct si_texture *stex = (struct si_texture*)tex;
654 si_clear_buffer(sctx, tex, stex->surface.fmask_offset, stex->surface.fmask_size,
655 (uint32_t*)&fmask_expand_values[log_fragments][log_samples - 1],
656 4, SI_COHERENCY_SHADER, false);
657 }
658
659 void si_init_compute_blit_functions(struct si_context *sctx)
660 {
661 sctx->b.clear_buffer = si_pipe_clear_buffer;
662 }
663
664 /* Clear a region of a color surface to a constant value. */
665 void si_compute_clear_render_target(struct pipe_context *ctx,
666 struct pipe_surface *dstsurf,
667 const union pipe_color_union *color,
668 unsigned dstx, unsigned dsty,
669 unsigned width, unsigned height,
670 bool render_condition_enabled)
671 {
672 struct si_context *sctx = (struct si_context *)ctx;
673 unsigned num_layers = dstsurf->u.tex.last_layer - dstsurf->u.tex.first_layer + 1;
674 unsigned data[4 + sizeof(color->ui)] = {dstx, dsty, dstsurf->u.tex.first_layer, 0};
675
676 if (width == 0 || height == 0)
677 return;
678
679 if (util_format_is_srgb(dstsurf->format)) {
680 union pipe_color_union color_srgb;
681 for (int i = 0; i < 3; i++)
682 color_srgb.f[i] = util_format_linear_to_srgb_float(color->f[i]);
683 color_srgb.f[3] = color->f[3];
684 memcpy(data + 4, color_srgb.ui, sizeof(color->ui));
685 } else {
686 memcpy(data + 4, color->ui, sizeof(color->ui));
687 }
688
689 si_compute_internal_begin(sctx);
690 sctx->render_cond_force_off = !render_condition_enabled;
691
692 sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH |
693 si_get_flush_flags(sctx, SI_COHERENCY_SHADER, L2_STREAM);
694 si_make_CB_shader_coherent(sctx, dstsurf->texture->nr_samples, true,
695 true /* DCC is not possible with image stores */);
696
697 struct pipe_constant_buffer saved_cb = {};
698 si_get_pipe_constant_buffer(sctx, PIPE_SHADER_COMPUTE, 0, &saved_cb);
699
700 struct si_images *images = &sctx->images[PIPE_SHADER_COMPUTE];
701 struct pipe_image_view saved_image = {0};
702 util_copy_image_view(&saved_image, &images->views[0]);
703
704 void *saved_cs = sctx->cs_shader_state.program;
705
706 struct pipe_constant_buffer cb = {};
707 cb.buffer_size = sizeof(data);
708 cb.user_buffer = data;
709 ctx->set_constant_buffer(ctx, PIPE_SHADER_COMPUTE, 0, &cb);
710
711 struct pipe_image_view image = {0};
712 image.resource = dstsurf->texture;
713 image.shader_access = image.access = PIPE_IMAGE_ACCESS_WRITE;
714 image.format = util_format_linear(dstsurf->format);
715 image.u.tex.level = dstsurf->u.tex.level;
716 image.u.tex.first_layer = 0; /* 3D images ignore first_layer (BASE_ARRAY) */
717 image.u.tex.last_layer = dstsurf->u.tex.last_layer;
718
719 ctx->set_shader_images(ctx, PIPE_SHADER_COMPUTE, 0, 1, &image);
720
721 struct pipe_grid_info info = {0};
722
723 if (dstsurf->texture->target != PIPE_TEXTURE_1D_ARRAY) {
724 if (!sctx->cs_clear_render_target)
725 sctx->cs_clear_render_target = si_clear_render_target_shader(ctx);
726 ctx->bind_compute_state(ctx, sctx->cs_clear_render_target);
727 info.block[0] = 8;
728 info.last_block[0] = width % 8;
729 info.block[1] = 8;
730 info.last_block[1] = height % 8;
731 info.block[2] = 1;
732 info.grid[0] = DIV_ROUND_UP(width, 8);
733 info.grid[1] = DIV_ROUND_UP(height, 8);
734 info.grid[2] = num_layers;
735 } else {
736 if (!sctx->cs_clear_render_target_1d_array)
737 sctx->cs_clear_render_target_1d_array =
738 si_clear_render_target_shader_1d_array(ctx);
739 ctx->bind_compute_state(ctx, sctx->cs_clear_render_target_1d_array);
740 info.block[0] = 64;
741 info.last_block[0] = width % 64;
742 info.block[1] = 1;
743 info.block[2] = 1;
744 info.grid[0] = DIV_ROUND_UP(width, 64);
745 info.grid[1] = num_layers;
746 info.grid[2] = 1;
747 }
748
749 ctx->launch_grid(ctx, &info);
750
751 sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH |
752 (sctx->chip_class <= GFX8 ? SI_CONTEXT_WB_L2 : 0) |
753 si_get_flush_flags(sctx, SI_COHERENCY_SHADER, L2_STREAM);
754 ctx->bind_compute_state(ctx, saved_cs);
755 ctx->set_shader_images(ctx, PIPE_SHADER_COMPUTE, 0, 1, &saved_image);
756 ctx->set_constant_buffer(ctx, PIPE_SHADER_COMPUTE, 0, &saved_cb);
757 si_compute_internal_end(sctx);
758 pipe_resource_reference(&saved_image.resource, NULL);
759 pipe_resource_reference(&saved_cb.buffer, NULL);
760 }