875341bf06e8a4ae5bf1dae942224a7694a52a43
[mesa.git] / src / gallium / drivers / radeonsi / si_texture.c
1 /*
2 * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
3 * Copyright 2018 Advanced Micro Devices, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * on the rights to use, copy, modify, merge, publish, distribute, sub
10 * license, and/or sell copies of the Software, and to permit persons to whom
11 * the Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
15 * Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
21 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
22 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
23 * USE OR OTHER DEALINGS IN THE SOFTWARE.
24 */
25
26 #include "drm-uapi/drm_fourcc.h"
27 #include "si_pipe.h"
28 #include "si_query.h"
29 #include "sid.h"
30 #include "frontend/drm_driver.h"
31 #include "util/format/u_format.h"
32 #include "util/os_time.h"
33 #include "util/u_log.h"
34 #include "util/u_memory.h"
35 #include "util/u_pack_color.h"
36 #include "util/u_resource.h"
37 #include "util/u_surface.h"
38 #include "util/u_transfer.h"
39
40 #include <errno.h>
41 #include <inttypes.h>
42
43 #include "amd/addrlib/inc/addrinterface.h"
44
45 static enum radeon_surf_mode si_choose_tiling(struct si_screen *sscreen,
46 const struct pipe_resource *templ,
47 bool tc_compatible_htile);
48
49 bool si_prepare_for_dma_blit(struct si_context *sctx, struct si_texture *dst, unsigned dst_level,
50 unsigned dstx, unsigned dsty, unsigned dstz, struct si_texture *src,
51 unsigned src_level, const struct pipe_box *src_box)
52 {
53 if (!sctx->sdma_cs)
54 return false;
55
56 if (dst->surface.bpe != src->surface.bpe)
57 return false;
58
59 /* MSAA: Blits don't exist in the real world. */
60 if (src->buffer.b.b.nr_samples > 1 || dst->buffer.b.b.nr_samples > 1)
61 return false;
62
63 /* Depth-stencil surfaces:
64 * When dst is linear, the DB->CB copy preserves HTILE.
65 * When dst is tiled, the 3D path must be used to update HTILE.
66 */
67 if (src->is_depth || dst->is_depth)
68 return false;
69
70 /* DCC as:
71 * src: Use the 3D path. DCC decompression is expensive.
72 * dst: Use the 3D path to compress the pixels with DCC.
73 */
74 if (vi_dcc_enabled(src, src_level) || vi_dcc_enabled(dst, dst_level))
75 return false;
76
77 /* TMZ: mixing encrypted and non-encrypted buffer in a single command
78 * doesn't seem supported.
79 */
80 if ((src->buffer.flags & RADEON_FLAG_ENCRYPTED) !=
81 (dst->buffer.flags & RADEON_FLAG_ENCRYPTED))
82 return false;
83
84 /* CMASK as:
85 * src: Both texture and SDMA paths need decompression. Use SDMA.
86 * dst: If overwriting the whole texture, discard CMASK and use
87 * SDMA. Otherwise, use the 3D path.
88 */
89 if (dst->cmask_buffer && dst->dirty_level_mask & (1 << dst_level)) {
90 /* The CMASK clear is only enabled for the first level. */
91 assert(dst_level == 0);
92 if (!util_texrange_covers_whole_level(&dst->buffer.b.b, dst_level, dstx, dsty, dstz,
93 src_box->width, src_box->height, src_box->depth))
94 return false;
95
96 si_texture_discard_cmask(sctx->screen, dst);
97 }
98
99 /* All requirements are met. Prepare textures for SDMA. */
100 if (src->cmask_buffer && src->dirty_level_mask & (1 << src_level))
101 sctx->b.flush_resource(&sctx->b, &src->buffer.b.b);
102
103 assert(!(src->dirty_level_mask & (1 << src_level)));
104 assert(!(dst->dirty_level_mask & (1 << dst_level)));
105
106 return true;
107 }
108
109 /* Same as resource_copy_region, except that both upsampling and downsampling are allowed. */
110 static void si_copy_region_with_blit(struct pipe_context *pipe, struct pipe_resource *dst,
111 unsigned dst_level, unsigned dstx, unsigned dsty,
112 unsigned dstz, struct pipe_resource *src, unsigned src_level,
113 const struct pipe_box *src_box)
114 {
115 struct pipe_blit_info blit;
116
117 memset(&blit, 0, sizeof(blit));
118 blit.src.resource = src;
119 blit.src.format = src->format;
120 blit.src.level = src_level;
121 blit.src.box = *src_box;
122 blit.dst.resource = dst;
123 blit.dst.format = dst->format;
124 blit.dst.level = dst_level;
125 blit.dst.box.x = dstx;
126 blit.dst.box.y = dsty;
127 blit.dst.box.z = dstz;
128 blit.dst.box.width = src_box->width;
129 blit.dst.box.height = src_box->height;
130 blit.dst.box.depth = src_box->depth;
131 blit.mask = util_format_get_mask(dst->format);
132 blit.filter = PIPE_TEX_FILTER_NEAREST;
133
134 if (blit.mask) {
135 pipe->blit(pipe, &blit);
136 }
137 }
138
139 /* Copy from a full GPU texture to a transfer's staging one. */
140 static void si_copy_to_staging_texture(struct pipe_context *ctx, struct si_transfer *stransfer)
141 {
142 struct si_context *sctx = (struct si_context *)ctx;
143 struct pipe_transfer *transfer = (struct pipe_transfer *)stransfer;
144 struct pipe_resource *dst = &stransfer->staging->b.b;
145 struct pipe_resource *src = transfer->resource;
146
147 if (src->nr_samples > 1 || ((struct si_texture *)src)->is_depth) {
148 si_copy_region_with_blit(ctx, dst, 0, 0, 0, 0, src, transfer->level, &transfer->box);
149 return;
150 }
151
152 sctx->dma_copy(ctx, dst, 0, 0, 0, 0, src, transfer->level, &transfer->box);
153 }
154
155 /* Copy from a transfer's staging texture to a full GPU one. */
156 static void si_copy_from_staging_texture(struct pipe_context *ctx, struct si_transfer *stransfer)
157 {
158 struct si_context *sctx = (struct si_context *)ctx;
159 struct pipe_transfer *transfer = (struct pipe_transfer *)stransfer;
160 struct pipe_resource *dst = transfer->resource;
161 struct pipe_resource *src = &stransfer->staging->b.b;
162 struct pipe_box sbox;
163
164 u_box_3d(0, 0, 0, transfer->box.width, transfer->box.height, transfer->box.depth, &sbox);
165
166 if (dst->nr_samples > 1 || ((struct si_texture *)dst)->is_depth) {
167 si_copy_region_with_blit(ctx, dst, transfer->level, transfer->box.x, transfer->box.y,
168 transfer->box.z, src, 0, &sbox);
169 return;
170 }
171
172 if (util_format_is_compressed(dst->format)) {
173 sbox.width = util_format_get_nblocksx(dst->format, sbox.width);
174 sbox.height = util_format_get_nblocksx(dst->format, sbox.height);
175 }
176
177 sctx->dma_copy(ctx, dst, transfer->level, transfer->box.x, transfer->box.y, transfer->box.z, src,
178 0, &sbox);
179 }
180
181 static unsigned si_texture_get_offset(struct si_screen *sscreen, struct si_texture *tex,
182 unsigned level, const struct pipe_box *box, unsigned *stride,
183 unsigned *layer_stride)
184 {
185 if (sscreen->info.chip_class >= GFX9) {
186 *stride = tex->surface.u.gfx9.surf_pitch * tex->surface.bpe;
187 *layer_stride = tex->surface.u.gfx9.surf_slice_size;
188
189 if (!box)
190 return 0;
191
192 /* Each texture is an array of slices. Each slice is an array
193 * of mipmap levels. */
194 return tex->surface.u.gfx9.surf_offset + box->z * tex->surface.u.gfx9.surf_slice_size +
195 tex->surface.u.gfx9.offset[level] +
196 (box->y / tex->surface.blk_h * tex->surface.u.gfx9.surf_pitch +
197 box->x / tex->surface.blk_w) *
198 tex->surface.bpe;
199 } else {
200 *stride = tex->surface.u.legacy.level[level].nblk_x * tex->surface.bpe;
201 assert((uint64_t)tex->surface.u.legacy.level[level].slice_size_dw * 4 <= UINT_MAX);
202 *layer_stride = (uint64_t)tex->surface.u.legacy.level[level].slice_size_dw * 4;
203
204 if (!box)
205 return tex->surface.u.legacy.level[level].offset;
206
207 /* Each texture is an array of mipmap levels. Each level is
208 * an array of slices. */
209 return tex->surface.u.legacy.level[level].offset +
210 box->z * (uint64_t)tex->surface.u.legacy.level[level].slice_size_dw * 4 +
211 (box->y / tex->surface.blk_h * tex->surface.u.legacy.level[level].nblk_x +
212 box->x / tex->surface.blk_w) *
213 tex->surface.bpe;
214 }
215 }
216
217 static int si_init_surface(struct si_screen *sscreen, struct radeon_surf *surface,
218 const struct pipe_resource *ptex, enum radeon_surf_mode array_mode,
219 bool is_imported, bool is_scanout, bool is_flushed_depth,
220 bool tc_compatible_htile)
221 {
222 const struct util_format_description *desc = util_format_description(ptex->format);
223 bool is_depth, is_stencil;
224 int r;
225 unsigned bpe, flags = 0;
226
227 is_depth = util_format_has_depth(desc);
228 is_stencil = util_format_has_stencil(desc);
229
230 if (!is_flushed_depth && ptex->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) {
231 bpe = 4; /* stencil is allocated separately */
232 } else {
233 bpe = util_format_get_blocksize(ptex->format);
234 assert(util_is_power_of_two_or_zero(bpe));
235 }
236
237 if (!is_flushed_depth && is_depth) {
238 flags |= RADEON_SURF_ZBUFFER;
239
240 if (sscreen->debug_flags & DBG(NO_HYPERZ)) {
241 flags |= RADEON_SURF_NO_HTILE;
242 } else if (tc_compatible_htile &&
243 (sscreen->info.chip_class >= GFX9 || array_mode == RADEON_SURF_MODE_2D)) {
244 /* TC-compatible HTILE only supports Z32_FLOAT.
245 * GFX9 also supports Z16_UNORM.
246 * On GFX8, promote Z16 to Z32. DB->CB copies will convert
247 * the format for transfers.
248 */
249 if (sscreen->info.chip_class == GFX8)
250 bpe = 4;
251
252 flags |= RADEON_SURF_TC_COMPATIBLE_HTILE;
253 }
254
255 if (is_stencil)
256 flags |= RADEON_SURF_SBUFFER;
257 }
258
259 if (sscreen->info.chip_class >= GFX8 &&
260 (ptex->flags & SI_RESOURCE_FLAG_DISABLE_DCC ||
261 (sscreen->info.chip_class < GFX10_3 &&
262 ptex->format == PIPE_FORMAT_R9G9B9E5_FLOAT) ||
263 (ptex->nr_samples >= 2 && !sscreen->dcc_msaa_allowed)))
264 flags |= RADEON_SURF_DISABLE_DCC;
265
266 /* Stoney: 128bpp MSAA textures randomly fail piglit tests with DCC. */
267 if (sscreen->info.family == CHIP_STONEY && bpe == 16 && ptex->nr_samples >= 2)
268 flags |= RADEON_SURF_DISABLE_DCC;
269
270 /* GFX8: DCC clear for 4x and 8x MSAA array textures unimplemented. */
271 if (sscreen->info.chip_class == GFX8 && ptex->nr_storage_samples >= 4 && ptex->array_size > 1)
272 flags |= RADEON_SURF_DISABLE_DCC;
273
274 /* GFX9: DCC clear for 4x and 8x MSAA textures unimplemented. */
275 if (sscreen->info.chip_class == GFX9 &&
276 (ptex->nr_storage_samples >= 4 ||
277 (sscreen->info.family == CHIP_RAVEN && ptex->nr_storage_samples >= 2 && bpe < 4)))
278 flags |= RADEON_SURF_DISABLE_DCC;
279
280 /* TODO: GFX10: DCC causes corruption with MSAA. */
281 if (sscreen->info.chip_class >= GFX10 && ptex->nr_storage_samples >= 2)
282 flags |= RADEON_SURF_DISABLE_DCC;
283
284 /* Shared textures must always set up DCC.
285 * If it's not present, it will be disabled by
286 * si_get_opaque_metadata later.
287 */
288 if (!is_imported && (sscreen->debug_flags & DBG(NO_DCC)))
289 flags |= RADEON_SURF_DISABLE_DCC;
290
291 if (is_scanout) {
292 /* This should catch bugs in gallium users setting incorrect flags. */
293 assert(ptex->nr_samples <= 1 && ptex->array_size == 1 && ptex->depth0 == 1 &&
294 ptex->last_level == 0 && !(flags & RADEON_SURF_Z_OR_SBUFFER));
295
296 flags |= RADEON_SURF_SCANOUT;
297 }
298
299 if (ptex->bind & PIPE_BIND_SHARED)
300 flags |= RADEON_SURF_SHAREABLE;
301 if (is_imported)
302 flags |= RADEON_SURF_IMPORTED | RADEON_SURF_SHAREABLE;
303 if (sscreen->debug_flags & DBG(NO_FMASK))
304 flags |= RADEON_SURF_NO_FMASK;
305
306 if (sscreen->info.chip_class == GFX9 && (ptex->flags & SI_RESOURCE_FLAG_FORCE_MICRO_TILE_MODE)) {
307 flags |= RADEON_SURF_FORCE_MICRO_TILE_MODE;
308 surface->micro_tile_mode = SI_RESOURCE_FLAG_MICRO_TILE_MODE_GET(ptex->flags);
309 }
310
311 if (ptex->flags & SI_RESOURCE_FLAG_FORCE_MSAA_TILING) {
312 flags |= RADEON_SURF_FORCE_SWIZZLE_MODE;
313
314 if (sscreen->info.chip_class >= GFX10)
315 surface->u.gfx9.surf.swizzle_mode = ADDR_SW_64KB_R_X;
316 }
317
318 r = sscreen->ws->surface_init(sscreen->ws, ptex, flags, bpe, array_mode, surface);
319 if (r) {
320 return r;
321 }
322
323 return 0;
324 }
325
326 void si_eliminate_fast_color_clear(struct si_context *sctx, struct si_texture *tex,
327 bool *ctx_flushed)
328 {
329 struct si_screen *sscreen = sctx->screen;
330 struct pipe_context *ctx = &sctx->b;
331
332 if (ctx == sscreen->aux_context)
333 simple_mtx_lock(&sscreen->aux_context_lock);
334
335 unsigned n = sctx->num_decompress_calls;
336 ctx->flush_resource(ctx, &tex->buffer.b.b);
337
338 /* Flush only if any fast clear elimination took place. */
339 bool flushed = false;
340 if (n != sctx->num_decompress_calls)
341 {
342 ctx->flush(ctx, NULL, 0);
343 flushed = true;
344 }
345 if (ctx_flushed)
346 *ctx_flushed = flushed;
347
348 if (ctx == sscreen->aux_context)
349 simple_mtx_unlock(&sscreen->aux_context_lock);
350 }
351
352 void si_texture_discard_cmask(struct si_screen *sscreen, struct si_texture *tex)
353 {
354 if (!tex->cmask_buffer)
355 return;
356
357 assert(tex->buffer.b.b.nr_samples <= 1);
358
359 /* Disable CMASK. */
360 tex->cmask_base_address_reg = tex->buffer.gpu_address >> 8;
361 tex->dirty_level_mask = 0;
362
363 tex->cb_color_info &= ~S_028C70_FAST_CLEAR(1);
364
365 if (tex->cmask_buffer != &tex->buffer)
366 si_resource_reference(&tex->cmask_buffer, NULL);
367
368 tex->cmask_buffer = NULL;
369
370 /* Notify all contexts about the change. */
371 p_atomic_inc(&sscreen->dirty_tex_counter);
372 p_atomic_inc(&sscreen->compressed_colortex_counter);
373 }
374
375 static bool si_can_disable_dcc(struct si_texture *tex)
376 {
377 /* We can't disable DCC if it can be written by another process. */
378 return tex->surface.dcc_offset &&
379 (!tex->buffer.b.is_shared ||
380 !(tex->buffer.external_usage & PIPE_HANDLE_USAGE_FRAMEBUFFER_WRITE));
381 }
382
383 static bool si_texture_discard_dcc(struct si_screen *sscreen, struct si_texture *tex)
384 {
385 if (!si_can_disable_dcc(tex))
386 return false;
387
388 assert(tex->dcc_separate_buffer == NULL);
389
390 /* Disable DCC. */
391 ac_surface_zero_dcc_fields(&tex->surface);
392
393 /* Notify all contexts about the change. */
394 p_atomic_inc(&sscreen->dirty_tex_counter);
395 return true;
396 }
397
398 /**
399 * Disable DCC for the texture. (first decompress, then discard metadata).
400 *
401 * There is unresolved multi-context synchronization issue between
402 * screen::aux_context and the current context. If applications do this with
403 * multiple contexts, it's already undefined behavior for them and we don't
404 * have to worry about that. The scenario is:
405 *
406 * If context 1 disables DCC and context 2 has queued commands that write
407 * to the texture via CB with DCC enabled, and the order of operations is
408 * as follows:
409 * context 2 queues draw calls rendering to the texture, but doesn't flush
410 * context 1 disables DCC and flushes
411 * context 1 & 2 reset descriptors and FB state
412 * context 2 flushes (new compressed tiles written by the draw calls)
413 * context 1 & 2 read garbage, because DCC is disabled, yet there are
414 * compressed tiled
415 *
416 * \param sctx the current context if you have one, or sscreen->aux_context
417 * if you don't.
418 */
419 bool si_texture_disable_dcc(struct si_context *sctx, struct si_texture *tex)
420 {
421 struct si_screen *sscreen = sctx->screen;
422
423 if (!sctx->has_graphics)
424 return si_texture_discard_dcc(sscreen, tex);
425
426 if (!si_can_disable_dcc(tex))
427 return false;
428
429 if (&sctx->b == sscreen->aux_context)
430 simple_mtx_lock(&sscreen->aux_context_lock);
431
432 /* Decompress DCC. */
433 si_decompress_dcc(sctx, tex);
434 sctx->b.flush(&sctx->b, NULL, 0);
435
436 if (&sctx->b == sscreen->aux_context)
437 simple_mtx_unlock(&sscreen->aux_context_lock);
438
439 return si_texture_discard_dcc(sscreen, tex);
440 }
441
442 static void si_reallocate_texture_inplace(struct si_context *sctx, struct si_texture *tex,
443 unsigned new_bind_flag, bool invalidate_storage)
444 {
445 struct pipe_screen *screen = sctx->b.screen;
446 struct si_texture *new_tex;
447 struct pipe_resource templ = tex->buffer.b.b;
448 unsigned i;
449
450 templ.bind |= new_bind_flag;
451
452 if (tex->buffer.b.is_shared || tex->num_planes > 1)
453 return;
454
455 if (new_bind_flag == PIPE_BIND_LINEAR) {
456 if (tex->surface.is_linear)
457 return;
458
459 /* This fails with MSAA, depth, and compressed textures. */
460 if (si_choose_tiling(sctx->screen, &templ, false) != RADEON_SURF_MODE_LINEAR_ALIGNED)
461 return;
462 }
463
464 new_tex = (struct si_texture *)screen->resource_create(screen, &templ);
465 if (!new_tex)
466 return;
467
468 /* Copy the pixels to the new texture. */
469 if (!invalidate_storage) {
470 for (i = 0; i <= templ.last_level; i++) {
471 struct pipe_box box;
472
473 u_box_3d(0, 0, 0, u_minify(templ.width0, i), u_minify(templ.height0, i),
474 util_num_layers(&templ, i), &box);
475
476 sctx->dma_copy(&sctx->b, &new_tex->buffer.b.b, i, 0, 0, 0, &tex->buffer.b.b, i, &box);
477 }
478 }
479
480 if (new_bind_flag == PIPE_BIND_LINEAR) {
481 si_texture_discard_cmask(sctx->screen, tex);
482 si_texture_discard_dcc(sctx->screen, tex);
483 }
484
485 /* Replace the structure fields of tex. */
486 tex->buffer.b.b.bind = templ.bind;
487 pb_reference(&tex->buffer.buf, new_tex->buffer.buf);
488 tex->buffer.gpu_address = new_tex->buffer.gpu_address;
489 tex->buffer.vram_usage = new_tex->buffer.vram_usage;
490 tex->buffer.gart_usage = new_tex->buffer.gart_usage;
491 tex->buffer.bo_size = new_tex->buffer.bo_size;
492 tex->buffer.bo_alignment = new_tex->buffer.bo_alignment;
493 tex->buffer.domains = new_tex->buffer.domains;
494 tex->buffer.flags = new_tex->buffer.flags;
495
496 tex->surface = new_tex->surface;
497 si_texture_reference(&tex->flushed_depth_texture, new_tex->flushed_depth_texture);
498
499 tex->surface.fmask_offset = new_tex->surface.fmask_offset;
500 tex->surface.cmask_offset = new_tex->surface.cmask_offset;
501 tex->cmask_base_address_reg = new_tex->cmask_base_address_reg;
502
503 if (tex->cmask_buffer == &tex->buffer)
504 tex->cmask_buffer = NULL;
505 else
506 si_resource_reference(&tex->cmask_buffer, NULL);
507
508 if (new_tex->cmask_buffer == &new_tex->buffer)
509 tex->cmask_buffer = &tex->buffer;
510 else
511 si_resource_reference(&tex->cmask_buffer, new_tex->cmask_buffer);
512
513 tex->surface.dcc_offset = new_tex->surface.dcc_offset;
514 tex->cb_color_info = new_tex->cb_color_info;
515 memcpy(tex->color_clear_value, new_tex->color_clear_value, sizeof(tex->color_clear_value));
516 tex->last_msaa_resolve_target_micro_mode = new_tex->last_msaa_resolve_target_micro_mode;
517
518 tex->surface.htile_offset = new_tex->surface.htile_offset;
519 tex->depth_clear_value = new_tex->depth_clear_value;
520 tex->dirty_level_mask = new_tex->dirty_level_mask;
521 tex->stencil_dirty_level_mask = new_tex->stencil_dirty_level_mask;
522 tex->db_render_format = new_tex->db_render_format;
523 tex->stencil_clear_value = new_tex->stencil_clear_value;
524 tex->tc_compatible_htile = new_tex->tc_compatible_htile;
525 tex->depth_cleared = new_tex->depth_cleared;
526 tex->stencil_cleared = new_tex->stencil_cleared;
527 tex->upgraded_depth = new_tex->upgraded_depth;
528 tex->db_compatible = new_tex->db_compatible;
529 tex->can_sample_z = new_tex->can_sample_z;
530 tex->can_sample_s = new_tex->can_sample_s;
531
532 tex->separate_dcc_dirty = new_tex->separate_dcc_dirty;
533 tex->displayable_dcc_dirty = new_tex->displayable_dcc_dirty;
534 tex->dcc_gather_statistics = new_tex->dcc_gather_statistics;
535 si_resource_reference(&tex->dcc_separate_buffer, new_tex->dcc_separate_buffer);
536 si_resource_reference(&tex->last_dcc_separate_buffer, new_tex->last_dcc_separate_buffer);
537
538 if (new_bind_flag == PIPE_BIND_LINEAR) {
539 assert(!tex->surface.htile_offset);
540 assert(!tex->cmask_buffer);
541 assert(!tex->surface.fmask_size);
542 assert(!tex->surface.dcc_offset);
543 assert(!tex->is_depth);
544 }
545
546 si_texture_reference(&new_tex, NULL);
547
548 p_atomic_inc(&sctx->screen->dirty_tex_counter);
549 }
550
551 static void si_set_tex_bo_metadata(struct si_screen *sscreen, struct si_texture *tex)
552 {
553 struct pipe_resource *res = &tex->buffer.b.b;
554 struct radeon_bo_metadata md;
555
556 memset(&md, 0, sizeof(md));
557
558 assert(tex->dcc_separate_buffer == NULL);
559 assert(tex->surface.fmask_size == 0);
560
561 static const unsigned char swizzle[] = {PIPE_SWIZZLE_X, PIPE_SWIZZLE_Y, PIPE_SWIZZLE_Z,
562 PIPE_SWIZZLE_W};
563 bool is_array = util_texture_is_array(res->target);
564 uint32_t desc[8];
565
566 sscreen->make_texture_descriptor(sscreen, tex, true, res->target, res->format, swizzle, 0,
567 res->last_level, 0, is_array ? res->array_size - 1 : 0,
568 res->width0, res->height0, res->depth0, desc, NULL);
569 si_set_mutable_tex_desc_fields(sscreen, tex, &tex->surface.u.legacy.level[0], 0, 0,
570 tex->surface.blk_w, false, false, desc);
571
572 ac_surface_get_umd_metadata(&sscreen->info, &tex->surface,
573 tex->buffer.b.b.last_level + 1,
574 desc, &md.size_metadata, md.metadata);
575 sscreen->ws->buffer_set_metadata(tex->buffer.buf, &md, &tex->surface);
576 }
577
578 static bool si_has_displayable_dcc(struct si_texture *tex)
579 {
580 struct si_screen *sscreen = (struct si_screen *)tex->buffer.b.b.screen;
581
582 if (sscreen->info.chip_class <= GFX8)
583 return false;
584
585 return tex->surface.is_displayable && tex->surface.dcc_offset;
586 }
587
588 static bool si_resource_get_param(struct pipe_screen *screen, struct pipe_context *context,
589 struct pipe_resource *resource, unsigned plane, unsigned layer,
590 enum pipe_resource_param param, unsigned handle_usage,
591 uint64_t *value)
592 {
593 for (unsigned i = 0; i < plane; i++)
594 resource = resource->next;
595
596 struct si_screen *sscreen = (struct si_screen *)screen;
597 struct si_texture *tex = (struct si_texture *)resource;
598 struct winsys_handle whandle;
599
600 switch (param) {
601 case PIPE_RESOURCE_PARAM_NPLANES:
602 *value = resource->target == PIPE_BUFFER ? 1 : tex->num_planes;
603 return true;
604
605 case PIPE_RESOURCE_PARAM_STRIDE:
606 if (resource->target == PIPE_BUFFER)
607 *value = 0;
608 else if (sscreen->info.chip_class >= GFX9)
609 *value = tex->surface.u.gfx9.surf_pitch * tex->surface.bpe;
610 else
611 *value = tex->surface.u.legacy.level[0].nblk_x * tex->surface.bpe;
612 return true;
613
614 case PIPE_RESOURCE_PARAM_OFFSET:
615 if (resource->target == PIPE_BUFFER)
616 *value = 0;
617 else if (sscreen->info.chip_class >= GFX9)
618 *value = tex->surface.u.gfx9.surf_offset + layer * tex->surface.u.gfx9.surf_slice_size;
619 else
620 *value = tex->surface.u.legacy.level[0].offset +
621 layer * (uint64_t)tex->surface.u.legacy.level[0].slice_size_dw * 4;
622 return true;
623
624 case PIPE_RESOURCE_PARAM_MODIFIER:
625 *value = DRM_FORMAT_MOD_INVALID;
626 return true;
627
628 case PIPE_RESOURCE_PARAM_HANDLE_TYPE_SHARED:
629 case PIPE_RESOURCE_PARAM_HANDLE_TYPE_KMS:
630 case PIPE_RESOURCE_PARAM_HANDLE_TYPE_FD:
631 memset(&whandle, 0, sizeof(whandle));
632
633 if (param == PIPE_RESOURCE_PARAM_HANDLE_TYPE_SHARED)
634 whandle.type = WINSYS_HANDLE_TYPE_SHARED;
635 else if (param == PIPE_RESOURCE_PARAM_HANDLE_TYPE_KMS)
636 whandle.type = WINSYS_HANDLE_TYPE_KMS;
637 else if (param == PIPE_RESOURCE_PARAM_HANDLE_TYPE_FD)
638 whandle.type = WINSYS_HANDLE_TYPE_FD;
639
640 if (!screen->resource_get_handle(screen, context, resource, &whandle, handle_usage))
641 return false;
642
643 *value = whandle.handle;
644 return true;
645 }
646 return false;
647 }
648
649 static void si_texture_get_info(struct pipe_screen *screen, struct pipe_resource *resource,
650 unsigned *pstride, unsigned *poffset)
651 {
652 uint64_t value;
653
654 if (pstride) {
655 si_resource_get_param(screen, NULL, resource, 0, 0, PIPE_RESOURCE_PARAM_STRIDE, 0, &value);
656 *pstride = value;
657 }
658
659 if (poffset) {
660 si_resource_get_param(screen, NULL, resource, 0, 0, PIPE_RESOURCE_PARAM_OFFSET, 0, &value);
661 *poffset = value;
662 }
663 }
664
665 static bool si_texture_get_handle(struct pipe_screen *screen, struct pipe_context *ctx,
666 struct pipe_resource *resource, struct winsys_handle *whandle,
667 unsigned usage)
668 {
669 struct si_screen *sscreen = (struct si_screen *)screen;
670 struct si_context *sctx;
671 struct si_resource *res = si_resource(resource);
672 struct si_texture *tex = (struct si_texture *)resource;
673 bool update_metadata = false;
674 unsigned stride, offset, slice_size;
675 bool flush = false;
676
677 ctx = threaded_context_unwrap_sync(ctx);
678 sctx = (struct si_context *)(ctx ? ctx : sscreen->aux_context);
679
680 if (resource->target != PIPE_BUFFER) {
681 /* Individual planes are chained pipe_resource instances. */
682 for (unsigned i = 0; i < whandle->plane; i++) {
683 resource = resource->next;
684 res = si_resource(resource);
685 tex = (struct si_texture *)resource;
686 }
687
688 /* This is not supported now, but it might be required for OpenCL
689 * interop in the future.
690 */
691 if (resource->nr_samples > 1 || tex->is_depth)
692 return false;
693
694 /* Move a suballocated texture into a non-suballocated allocation. */
695 if (sscreen->ws->buffer_is_suballocated(res->buf) || tex->surface.tile_swizzle ||
696 (tex->buffer.flags & RADEON_FLAG_NO_INTERPROCESS_SHARING &&
697 sscreen->info.has_local_buffers)) {
698 assert(!res->b.is_shared);
699 si_reallocate_texture_inplace(sctx, tex, PIPE_BIND_SHARED, false);
700 flush = true;
701 assert(res->b.b.bind & PIPE_BIND_SHARED);
702 assert(res->flags & RADEON_FLAG_NO_SUBALLOC);
703 assert(!(res->flags & RADEON_FLAG_NO_INTERPROCESS_SHARING));
704 assert(tex->surface.tile_swizzle == 0);
705 }
706
707 /* Since shader image stores don't support DCC on GFX8,
708 * disable it for external clients that want write
709 * access.
710 */
711 if ((usage & PIPE_HANDLE_USAGE_SHADER_WRITE && tex->surface.dcc_offset) ||
712 /* Displayable DCC requires an explicit flush. */
713 (!(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH) && si_has_displayable_dcc(tex))) {
714 if (si_texture_disable_dcc(sctx, tex)) {
715 update_metadata = true;
716 /* si_texture_disable_dcc flushes the context */
717 flush = false;
718 }
719 }
720
721 if (!(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH) &&
722 (tex->cmask_buffer || tex->surface.dcc_offset)) {
723 /* Eliminate fast clear (both CMASK and DCC) */
724 bool flushed;
725 si_eliminate_fast_color_clear(sctx, tex, &flushed);
726 /* eliminate_fast_color_clear sometimes flushes the context */
727 if (flushed)
728 flush = false;
729
730 /* Disable CMASK if flush_resource isn't going
731 * to be called.
732 */
733 if (tex->cmask_buffer)
734 si_texture_discard_cmask(sscreen, tex);
735 }
736
737 /* Set metadata. */
738 if ((!res->b.is_shared || update_metadata) && whandle->offset == 0)
739 si_set_tex_bo_metadata(sscreen, tex);
740
741 if (sscreen->info.chip_class >= GFX9) {
742 slice_size = tex->surface.u.gfx9.surf_slice_size;
743 } else {
744 slice_size = (uint64_t)tex->surface.u.legacy.level[0].slice_size_dw * 4;
745 }
746 } else {
747 /* Buffer exports are for the OpenCL interop. */
748 /* Move a suballocated buffer into a non-suballocated allocation. */
749 if (sscreen->ws->buffer_is_suballocated(res->buf) ||
750 /* A DMABUF export always fails if the BO is local. */
751 (tex->buffer.flags & RADEON_FLAG_NO_INTERPROCESS_SHARING &&
752 sscreen->info.has_local_buffers)) {
753 assert(!res->b.is_shared);
754
755 /* Allocate a new buffer with PIPE_BIND_SHARED. */
756 struct pipe_resource templ = res->b.b;
757 templ.bind |= PIPE_BIND_SHARED;
758
759 struct pipe_resource *newb = screen->resource_create(screen, &templ);
760 if (!newb)
761 return false;
762
763 /* Copy the old buffer contents to the new one. */
764 struct pipe_box box;
765 u_box_1d(0, newb->width0, &box);
766 sctx->b.resource_copy_region(&sctx->b, newb, 0, 0, 0, 0, &res->b.b, 0, &box);
767 flush = true;
768 /* Move the new buffer storage to the old pipe_resource. */
769 si_replace_buffer_storage(&sctx->b, &res->b.b, newb);
770 pipe_resource_reference(&newb, NULL);
771
772 assert(res->b.b.bind & PIPE_BIND_SHARED);
773 assert(res->flags & RADEON_FLAG_NO_SUBALLOC);
774 }
775
776 /* Buffers */
777 slice_size = 0;
778 }
779
780 si_texture_get_info(screen, resource, &stride, &offset);
781
782 if (flush)
783 sctx->b.flush(&sctx->b, NULL, 0);
784
785 if (res->b.is_shared) {
786 /* USAGE_EXPLICIT_FLUSH must be cleared if at least one user
787 * doesn't set it.
788 */
789 res->external_usage |= usage & ~PIPE_HANDLE_USAGE_EXPLICIT_FLUSH;
790 if (!(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH))
791 res->external_usage &= ~PIPE_HANDLE_USAGE_EXPLICIT_FLUSH;
792 } else {
793 res->b.is_shared = true;
794 res->external_usage = usage;
795 }
796
797 whandle->stride = stride;
798 whandle->offset = offset + slice_size * whandle->layer;
799
800 return sscreen->ws->buffer_get_handle(sscreen->ws, res->buf, whandle);
801 }
802
803 static void si_texture_destroy(struct pipe_screen *screen, struct pipe_resource *ptex)
804 {
805 struct si_texture *tex = (struct si_texture *)ptex;
806 struct si_resource *resource = &tex->buffer;
807
808 si_texture_reference(&tex->flushed_depth_texture, NULL);
809
810 if (tex->cmask_buffer != &tex->buffer) {
811 si_resource_reference(&tex->cmask_buffer, NULL);
812 }
813 pb_reference(&resource->buf, NULL);
814 si_resource_reference(&tex->dcc_separate_buffer, NULL);
815 si_resource_reference(&tex->last_dcc_separate_buffer, NULL);
816 FREE(tex);
817 }
818
819 static const struct u_resource_vtbl si_texture_vtbl;
820
821 void si_print_texture_info(struct si_screen *sscreen, struct si_texture *tex,
822 struct u_log_context *log)
823 {
824 int i;
825
826 /* Common parameters. */
827 u_log_printf(log,
828 " Info: npix_x=%u, npix_y=%u, npix_z=%u, blk_w=%u, "
829 "blk_h=%u, array_size=%u, last_level=%u, "
830 "bpe=%u, nsamples=%u, flags=0x%x, %s\n",
831 tex->buffer.b.b.width0, tex->buffer.b.b.height0, tex->buffer.b.b.depth0,
832 tex->surface.blk_w, tex->surface.blk_h, tex->buffer.b.b.array_size,
833 tex->buffer.b.b.last_level, tex->surface.bpe, tex->buffer.b.b.nr_samples,
834 tex->surface.flags, util_format_short_name(tex->buffer.b.b.format));
835
836 if (sscreen->info.chip_class >= GFX9) {
837 u_log_printf(log,
838 " Surf: size=%" PRIu64 ", slice_size=%" PRIu64 ", "
839 "alignment=%u, swmode=%u, epitch=%u, pitch=%u\n",
840 tex->surface.surf_size, tex->surface.u.gfx9.surf_slice_size,
841 tex->surface.surf_alignment, tex->surface.u.gfx9.surf.swizzle_mode,
842 tex->surface.u.gfx9.surf.epitch, tex->surface.u.gfx9.surf_pitch);
843
844 if (tex->surface.fmask_offset) {
845 u_log_printf(log,
846 " FMASK: offset=%" PRIu64 ", size=%" PRIu64 ", "
847 "alignment=%u, swmode=%u, epitch=%u\n",
848 tex->surface.fmask_offset, tex->surface.fmask_size,
849 tex->surface.fmask_alignment, tex->surface.u.gfx9.fmask.swizzle_mode,
850 tex->surface.u.gfx9.fmask.epitch);
851 }
852
853 if (tex->cmask_buffer) {
854 u_log_printf(log,
855 " CMask: offset=%" PRIu64 ", size=%u, "
856 "alignment=%u\n",
857 tex->surface.cmask_offset, tex->surface.cmask_size,
858 tex->surface.cmask_alignment);
859 }
860
861 if (tex->surface.htile_offset) {
862 u_log_printf(log,
863 " HTile: offset=%" PRIu64 ", size=%u, alignment=%u\n",
864 tex->surface.htile_offset, tex->surface.htile_size,
865 tex->surface.htile_alignment);
866 }
867
868 if (tex->surface.dcc_offset) {
869 u_log_printf(log,
870 " DCC: offset=%" PRIu64 ", size=%u, "
871 "alignment=%u, pitch_max=%u, num_dcc_levels=%u\n",
872 tex->surface.dcc_offset, tex->surface.dcc_size, tex->surface.dcc_alignment,
873 tex->surface.u.gfx9.display_dcc_pitch_max, tex->surface.num_dcc_levels);
874 }
875
876 if (tex->surface.u.gfx9.stencil_offset) {
877 u_log_printf(log, " Stencil: offset=%" PRIu64 ", swmode=%u, epitch=%u\n",
878 tex->surface.u.gfx9.stencil_offset, tex->surface.u.gfx9.stencil.swizzle_mode,
879 tex->surface.u.gfx9.stencil.epitch);
880 }
881 return;
882 }
883
884 u_log_printf(log,
885 " Layout: size=%" PRIu64 ", alignment=%u, bankw=%u, "
886 "bankh=%u, nbanks=%u, mtilea=%u, tilesplit=%u, pipeconfig=%u, scanout=%u\n",
887 tex->surface.surf_size, tex->surface.surf_alignment, tex->surface.u.legacy.bankw,
888 tex->surface.u.legacy.bankh, tex->surface.u.legacy.num_banks,
889 tex->surface.u.legacy.mtilea, tex->surface.u.legacy.tile_split,
890 tex->surface.u.legacy.pipe_config, (tex->surface.flags & RADEON_SURF_SCANOUT) != 0);
891
892 if (tex->surface.fmask_offset)
893 u_log_printf(
894 log,
895 " FMask: offset=%" PRIu64 ", size=%" PRIu64 ", alignment=%u, pitch_in_pixels=%u, "
896 "bankh=%u, slice_tile_max=%u, tile_mode_index=%u\n",
897 tex->surface.fmask_offset, tex->surface.fmask_size, tex->surface.fmask_alignment,
898 tex->surface.u.legacy.fmask.pitch_in_pixels, tex->surface.u.legacy.fmask.bankh,
899 tex->surface.u.legacy.fmask.slice_tile_max, tex->surface.u.legacy.fmask.tiling_index);
900
901 if (tex->cmask_buffer)
902 u_log_printf(log,
903 " CMask: offset=%" PRIu64 ", size=%u, alignment=%u, "
904 "slice_tile_max=%u\n",
905 tex->surface.cmask_offset, tex->surface.cmask_size, tex->surface.cmask_alignment,
906 tex->surface.u.legacy.cmask_slice_tile_max);
907
908 if (tex->surface.htile_offset)
909 u_log_printf(log,
910 " HTile: offset=%" PRIu64 ", size=%u, "
911 "alignment=%u, TC_compatible = %u\n",
912 tex->surface.htile_offset, tex->surface.htile_size, tex->surface.htile_alignment,
913 tex->tc_compatible_htile);
914
915 if (tex->surface.dcc_offset) {
916 u_log_printf(log, " DCC: offset=%" PRIu64 ", size=%u, alignment=%u\n",
917 tex->surface.dcc_offset, tex->surface.dcc_size, tex->surface.dcc_alignment);
918 for (i = 0; i <= tex->buffer.b.b.last_level; i++)
919 u_log_printf(log,
920 " DCCLevel[%i]: enabled=%u, offset=%u, "
921 "fast_clear_size=%u\n",
922 i, i < tex->surface.num_dcc_levels, tex->surface.u.legacy.level[i].dcc_offset,
923 tex->surface.u.legacy.level[i].dcc_fast_clear_size);
924 }
925
926 for (i = 0; i <= tex->buffer.b.b.last_level; i++)
927 u_log_printf(log,
928 " Level[%i]: offset=%" PRIu64 ", slice_size=%" PRIu64 ", "
929 "npix_x=%u, npix_y=%u, npix_z=%u, nblk_x=%u, nblk_y=%u, "
930 "mode=%u, tiling_index = %u\n",
931 i, tex->surface.u.legacy.level[i].offset,
932 (uint64_t)tex->surface.u.legacy.level[i].slice_size_dw * 4,
933 u_minify(tex->buffer.b.b.width0, i), u_minify(tex->buffer.b.b.height0, i),
934 u_minify(tex->buffer.b.b.depth0, i), tex->surface.u.legacy.level[i].nblk_x,
935 tex->surface.u.legacy.level[i].nblk_y, tex->surface.u.legacy.level[i].mode,
936 tex->surface.u.legacy.tiling_index[i]);
937
938 if (tex->surface.has_stencil) {
939 u_log_printf(log, " StencilLayout: tilesplit=%u\n",
940 tex->surface.u.legacy.stencil_tile_split);
941 for (i = 0; i <= tex->buffer.b.b.last_level; i++) {
942 u_log_printf(log,
943 " StencilLevel[%i]: offset=%" PRIu64 ", "
944 "slice_size=%" PRIu64 ", npix_x=%u, "
945 "npix_y=%u, npix_z=%u, nblk_x=%u, nblk_y=%u, "
946 "mode=%u, tiling_index = %u\n",
947 i, tex->surface.u.legacy.stencil_level[i].offset,
948 (uint64_t)tex->surface.u.legacy.stencil_level[i].slice_size_dw * 4,
949 u_minify(tex->buffer.b.b.width0, i), u_minify(tex->buffer.b.b.height0, i),
950 u_minify(tex->buffer.b.b.depth0, i),
951 tex->surface.u.legacy.stencil_level[i].nblk_x,
952 tex->surface.u.legacy.stencil_level[i].nblk_y,
953 tex->surface.u.legacy.stencil_level[i].mode,
954 tex->surface.u.legacy.stencil_tiling_index[i]);
955 }
956 }
957 }
958
959 /**
960 * Common function for si_texture_create and si_texture_from_handle.
961 *
962 * \param screen screen
963 * \param base resource template
964 * \param surface radeon_surf
965 * \param plane0 if a non-zero plane is being created, this is the first plane
966 * \param imported_buf from si_texture_from_handle
967 * \param offset offset for non-zero planes or imported buffers
968 * \param alloc_size the size to allocate if plane0 != NULL
969 * \param alignment alignment for the allocation
970 */
971 static struct si_texture *si_texture_create_object(struct pipe_screen *screen,
972 const struct pipe_resource *base,
973 const struct radeon_surf *surface,
974 const struct si_texture *plane0,
975 struct pb_buffer *imported_buf,
976 uint64_t offset, unsigned pitch_in_bytes,
977 uint64_t alloc_size, unsigned alignment)
978 {
979 struct si_texture *tex;
980 struct si_resource *resource;
981 struct si_screen *sscreen = (struct si_screen *)screen;
982
983 tex = CALLOC_STRUCT(si_texture);
984 if (!tex)
985 goto error;
986
987 resource = &tex->buffer;
988 resource->b.b = *base;
989 resource->b.vtbl = &si_texture_vtbl;
990 pipe_reference_init(&resource->b.b.reference, 1);
991 resource->b.b.screen = screen;
992
993 /* don't include stencil-only formats which we don't support for rendering */
994 tex->is_depth = util_format_has_depth(util_format_description(tex->buffer.b.b.format));
995 tex->surface = *surface;
996
997 /* On GFX8, HTILE uses different tiling depending on the TC_COMPATIBLE_HTILE
998 * setting, so we have to enable it if we enabled it at allocation.
999 *
1000 * GFX9 and later use the same tiling for both, so TC-compatible HTILE can be
1001 * enabled on demand.
1002 */
1003 tex->tc_compatible_htile = sscreen->info.chip_class == GFX8 &&
1004 tex->surface.flags & RADEON_SURF_TC_COMPATIBLE_HTILE;
1005
1006 /* TC-compatible HTILE:
1007 * - GFX8 only supports Z32_FLOAT.
1008 * - GFX9 only supports Z32_FLOAT and Z16_UNORM. */
1009 if (tex->surface.flags & RADEON_SURF_TC_COMPATIBLE_HTILE) {
1010 if (sscreen->info.chip_class >= GFX9 && base->format == PIPE_FORMAT_Z16_UNORM)
1011 tex->db_render_format = base->format;
1012 else {
1013 tex->db_render_format = PIPE_FORMAT_Z32_FLOAT;
1014 tex->upgraded_depth = base->format != PIPE_FORMAT_Z32_FLOAT &&
1015 base->format != PIPE_FORMAT_Z32_FLOAT_S8X24_UINT;
1016 }
1017 } else {
1018 tex->db_render_format = base->format;
1019 }
1020
1021 /* Applies to GCN. */
1022 tex->last_msaa_resolve_target_micro_mode = tex->surface.micro_tile_mode;
1023
1024 /* Disable separate DCC at the beginning. DRI2 doesn't reuse buffers
1025 * between frames, so the only thing that can enable separate DCC
1026 * with DRI2 is multiple slow clears within a frame.
1027 */
1028 tex->ps_draw_ratio = 0;
1029
1030 ac_surface_override_offset_stride(&sscreen->info, &tex->surface,
1031 tex->buffer.b.b.last_level + 1,
1032 offset, pitch_in_bytes / tex->surface.bpe);
1033
1034 if (tex->is_depth) {
1035 if (sscreen->info.chip_class >= GFX9) {
1036 tex->can_sample_z = true;
1037 tex->can_sample_s = true;
1038
1039 /* Stencil texturing with HTILE doesn't work
1040 * with mipmapping on Navi10-14. */
1041 if (sscreen->info.chip_class == GFX10 && base->last_level > 0)
1042 tex->htile_stencil_disabled = true;
1043 } else {
1044 tex->can_sample_z = !tex->surface.u.legacy.depth_adjusted;
1045 tex->can_sample_s = !tex->surface.u.legacy.stencil_adjusted;
1046 }
1047
1048 tex->db_compatible = surface->flags & RADEON_SURF_ZBUFFER;
1049 } else {
1050 if (tex->surface.cmask_offset) {
1051 tex->cb_color_info |= S_028C70_FAST_CLEAR(1);
1052 tex->cmask_buffer = &tex->buffer;
1053 }
1054 }
1055
1056 if (plane0) {
1057 /* The buffer is shared with the first plane. */
1058 resource->bo_size = plane0->buffer.bo_size;
1059 resource->bo_alignment = plane0->buffer.bo_alignment;
1060 resource->flags = plane0->buffer.flags;
1061 resource->domains = plane0->buffer.domains;
1062 resource->vram_usage = plane0->buffer.vram_usage;
1063 resource->gart_usage = plane0->buffer.gart_usage;
1064
1065 pb_reference(&resource->buf, plane0->buffer.buf);
1066 resource->gpu_address = plane0->buffer.gpu_address;
1067 } else if (!(surface->flags & RADEON_SURF_IMPORTED)) {
1068 /* Create the backing buffer. */
1069 si_init_resource_fields(sscreen, resource, alloc_size, alignment);
1070
1071 if (!si_alloc_resource(sscreen, resource))
1072 goto error;
1073 } else {
1074 resource->buf = imported_buf;
1075 resource->gpu_address = sscreen->ws->buffer_get_virtual_address(resource->buf);
1076 resource->bo_size = imported_buf->size;
1077 resource->bo_alignment = imported_buf->alignment;
1078 resource->domains = sscreen->ws->buffer_get_initial_domain(resource->buf);
1079 if (resource->domains & RADEON_DOMAIN_VRAM)
1080 resource->vram_usage = resource->bo_size;
1081 else if (resource->domains & RADEON_DOMAIN_GTT)
1082 resource->gart_usage = resource->bo_size;
1083 if (sscreen->ws->buffer_get_flags)
1084 resource->flags = sscreen->ws->buffer_get_flags(resource->buf);
1085 }
1086
1087 if (tex->cmask_buffer) {
1088 /* Initialize the cmask to 0xCC (= compressed state). */
1089 si_screen_clear_buffer(sscreen, &tex->cmask_buffer->b.b, tex->surface.cmask_offset,
1090 tex->surface.cmask_size, 0xCCCCCCCC);
1091 }
1092 if (tex->surface.htile_offset) {
1093 uint32_t clear_value = 0;
1094
1095 if (sscreen->info.chip_class >= GFX9 || tex->tc_compatible_htile)
1096 clear_value = 0x0000030F;
1097
1098 si_screen_clear_buffer(sscreen, &tex->buffer.b.b, tex->surface.htile_offset,
1099 tex->surface.htile_size, clear_value);
1100 }
1101
1102 /* Initialize DCC only if the texture is not being imported. */
1103 if (!(surface->flags & RADEON_SURF_IMPORTED) && tex->surface.dcc_offset) {
1104 /* Clear DCC to black for all tiles with DCC enabled.
1105 *
1106 * This fixes corruption in 3DMark Slingshot Extreme, which
1107 * uses uninitialized textures, causing corruption.
1108 */
1109 if (tex->surface.num_dcc_levels == tex->buffer.b.b.last_level + 1 &&
1110 tex->buffer.b.b.nr_samples <= 2) {
1111 /* Simple case - all tiles have DCC enabled. */
1112 si_screen_clear_buffer(sscreen, &tex->buffer.b.b, tex->surface.dcc_offset,
1113 tex->surface.dcc_size, DCC_CLEAR_COLOR_0000);
1114 } else if (sscreen->info.chip_class >= GFX9) {
1115 /* Clear to uncompressed. Clearing this to black is complicated. */
1116 si_screen_clear_buffer(sscreen, &tex->buffer.b.b, tex->surface.dcc_offset,
1117 tex->surface.dcc_size, DCC_UNCOMPRESSED);
1118 } else {
1119 /* GFX8: Initialize mipmap levels and multisamples separately. */
1120 if (tex->buffer.b.b.nr_samples >= 2) {
1121 /* Clearing this to black is complicated. */
1122 si_screen_clear_buffer(sscreen, &tex->buffer.b.b, tex->surface.dcc_offset,
1123 tex->surface.dcc_size, DCC_UNCOMPRESSED);
1124 } else {
1125 /* Clear the enabled mipmap levels to black. */
1126 unsigned size = 0;
1127
1128 for (unsigned i = 0; i < tex->surface.num_dcc_levels; i++) {
1129 if (!tex->surface.u.legacy.level[i].dcc_fast_clear_size)
1130 break;
1131
1132 size = tex->surface.u.legacy.level[i].dcc_offset +
1133 tex->surface.u.legacy.level[i].dcc_fast_clear_size;
1134 }
1135
1136 /* Mipmap levels with DCC. */
1137 if (size) {
1138 si_screen_clear_buffer(sscreen, &tex->buffer.b.b, tex->surface.dcc_offset, size,
1139 DCC_CLEAR_COLOR_0000);
1140 }
1141 /* Mipmap levels without DCC. */
1142 if (size != tex->surface.dcc_size) {
1143 si_screen_clear_buffer(sscreen, &tex->buffer.b.b, tex->surface.dcc_offset + size,
1144 tex->surface.dcc_size - size, DCC_UNCOMPRESSED);
1145 }
1146 }
1147 }
1148
1149 /* Initialize displayable DCC that requires the retile blit. */
1150 if (tex->surface.dcc_retile_map_offset) {
1151 /* Uninitialized DCC can hang the display hw.
1152 * Clear to white to indicate that. */
1153 si_screen_clear_buffer(sscreen, &tex->buffer.b.b, tex->surface.display_dcc_offset,
1154 tex->surface.u.gfx9.display_dcc_size, DCC_CLEAR_COLOR_1111);
1155
1156 /* Upload the DCC retile map.
1157 * Use a staging buffer for the upload, because
1158 * the buffer backing the texture is unmappable.
1159 */
1160 bool use_uint16 = tex->surface.u.gfx9.dcc_retile_use_uint16;
1161 unsigned num_elements = tex->surface.u.gfx9.dcc_retile_num_elements;
1162 unsigned dcc_retile_map_size = num_elements * (use_uint16 ? 2 : 4);
1163 struct si_resource *buf = si_aligned_buffer_create(screen, 0, PIPE_USAGE_STREAM,
1164 dcc_retile_map_size,
1165 sscreen->info.tcc_cache_line_size);
1166 void *map = sscreen->ws->buffer_map(buf->buf, NULL, PIPE_TRANSFER_WRITE);
1167
1168 /* Upload the retile map into the staging buffer. */
1169 memcpy(map, tex->surface.u.gfx9.dcc_retile_map, dcc_retile_map_size);
1170
1171 /* Copy the staging buffer to the buffer backing the texture. */
1172 struct si_context *sctx = (struct si_context *)sscreen->aux_context;
1173
1174 assert(tex->surface.dcc_retile_map_offset <= UINT_MAX);
1175 simple_mtx_lock(&sscreen->aux_context_lock);
1176 si_sdma_copy_buffer(sctx, &tex->buffer.b.b, &buf->b.b, tex->surface.dcc_retile_map_offset,
1177 0, buf->b.b.width0);
1178 sscreen->aux_context->flush(sscreen->aux_context, NULL, 0);
1179 simple_mtx_unlock(&sscreen->aux_context_lock);
1180
1181 si_resource_reference(&buf, NULL);
1182 }
1183 }
1184
1185 /* Initialize the CMASK base register value. */
1186 tex->cmask_base_address_reg = (tex->buffer.gpu_address + tex->surface.cmask_offset) >> 8;
1187
1188 if (sscreen->debug_flags & DBG(VM)) {
1189 fprintf(stderr,
1190 "VM start=0x%" PRIX64 " end=0x%" PRIX64
1191 " | Texture %ix%ix%i, %i levels, %i samples, %s\n",
1192 tex->buffer.gpu_address, tex->buffer.gpu_address + tex->buffer.buf->size,
1193 base->width0, base->height0, util_num_layers(base, 0), base->last_level + 1,
1194 base->nr_samples ? base->nr_samples : 1, util_format_short_name(base->format));
1195 }
1196
1197 if (sscreen->debug_flags & DBG(TEX)) {
1198 puts("Texture:");
1199 struct u_log_context log;
1200 u_log_context_init(&log);
1201 si_print_texture_info(sscreen, tex, &log);
1202 u_log_new_page_print(&log, stdout);
1203 fflush(stdout);
1204 u_log_context_destroy(&log);
1205 }
1206
1207 return tex;
1208
1209 error:
1210 FREE(tex);
1211 return NULL;
1212 }
1213
1214 static enum radeon_surf_mode si_choose_tiling(struct si_screen *sscreen,
1215 const struct pipe_resource *templ,
1216 bool tc_compatible_htile)
1217 {
1218 const struct util_format_description *desc = util_format_description(templ->format);
1219 bool force_tiling = templ->flags & SI_RESOURCE_FLAG_FORCE_MSAA_TILING;
1220 bool is_depth_stencil = util_format_is_depth_or_stencil(templ->format) &&
1221 !(templ->flags & SI_RESOURCE_FLAG_FLUSHED_DEPTH);
1222
1223 /* MSAA resources must be 2D tiled. */
1224 if (templ->nr_samples > 1)
1225 return RADEON_SURF_MODE_2D;
1226
1227 /* Transfer resources should be linear. */
1228 if (templ->flags & SI_RESOURCE_FLAG_FORCE_LINEAR)
1229 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1230
1231 /* Avoid Z/S decompress blits by forcing TC-compatible HTILE on GFX8,
1232 * which requires 2D tiling.
1233 */
1234 if (sscreen->info.chip_class == GFX8 && tc_compatible_htile)
1235 return RADEON_SURF_MODE_2D;
1236
1237 /* Handle common candidates for the linear mode.
1238 * Compressed textures and DB surfaces must always be tiled.
1239 */
1240 if (!force_tiling && !is_depth_stencil && !util_format_is_compressed(templ->format)) {
1241 if (sscreen->debug_flags & DBG(NO_TILING))
1242 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1243
1244 /* Tiling doesn't work with the 422 (SUBSAMPLED) formats. */
1245 if (desc->layout == UTIL_FORMAT_LAYOUT_SUBSAMPLED)
1246 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1247
1248 /* Cursors are linear on AMD GCN.
1249 * (XXX double-check, maybe also use RADEON_SURF_SCANOUT) */
1250 if (templ->bind & PIPE_BIND_CURSOR)
1251 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1252
1253 if (templ->bind & PIPE_BIND_LINEAR)
1254 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1255
1256 /* Textures with a very small height are recommended to be linear. */
1257 if (templ->target == PIPE_TEXTURE_1D || templ->target == PIPE_TEXTURE_1D_ARRAY ||
1258 /* Only very thin and long 2D textures should benefit from
1259 * linear_aligned. */
1260 (templ->width0 > 8 && templ->height0 <= 2))
1261 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1262
1263 /* Textures likely to be mapped often. */
1264 if (templ->usage == PIPE_USAGE_STAGING || templ->usage == PIPE_USAGE_STREAM)
1265 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1266 }
1267
1268 /* Make small textures 1D tiled. */
1269 if (templ->width0 <= 16 || templ->height0 <= 16 || (sscreen->debug_flags & DBG(NO_2D_TILING)))
1270 return RADEON_SURF_MODE_1D;
1271
1272 /* The allocator will switch to 1D if needed. */
1273 return RADEON_SURF_MODE_2D;
1274 }
1275
1276 struct pipe_resource *si_texture_create(struct pipe_screen *screen,
1277 const struct pipe_resource *templ)
1278 {
1279 struct si_screen *sscreen = (struct si_screen *)screen;
1280 bool is_zs = util_format_is_depth_or_stencil(templ->format);
1281
1282 if (templ->nr_samples >= 2) {
1283 /* This is hackish (overwriting the const pipe_resource template),
1284 * but should be harmless and gallium frontends can also see
1285 * the overriden number of samples in the created pipe_resource.
1286 */
1287 if (is_zs && sscreen->eqaa_force_z_samples) {
1288 ((struct pipe_resource *)templ)->nr_samples =
1289 ((struct pipe_resource *)templ)->nr_storage_samples = sscreen->eqaa_force_z_samples;
1290 } else if (!is_zs && sscreen->eqaa_force_color_samples) {
1291 ((struct pipe_resource *)templ)->nr_samples = sscreen->eqaa_force_coverage_samples;
1292 ((struct pipe_resource *)templ)->nr_storage_samples = sscreen->eqaa_force_color_samples;
1293 }
1294 }
1295
1296 bool is_flushed_depth = templ->flags & SI_RESOURCE_FLAG_FLUSHED_DEPTH ||
1297 templ->flags & SI_RESOURCE_FLAG_FORCE_LINEAR;
1298 bool tc_compatible_htile =
1299 sscreen->info.chip_class >= GFX8 &&
1300 /* There are issues with TC-compatible HTILE on Tonga (and
1301 * Iceland is the same design), and documented bug workarounds
1302 * don't help. For example, this fails:
1303 * piglit/bin/tex-miplevel-selection 'texture()' 2DShadow -auto
1304 */
1305 sscreen->info.family != CHIP_TONGA && sscreen->info.family != CHIP_ICELAND &&
1306 (templ->flags & PIPE_RESOURCE_FLAG_TEXTURING_MORE_LIKELY) &&
1307 !(sscreen->debug_flags & DBG(NO_HYPERZ)) && !is_flushed_depth &&
1308 templ->nr_samples <= 1 && /* TC-compat HTILE is less efficient with MSAA */
1309 is_zs;
1310 enum radeon_surf_mode tile_mode = si_choose_tiling(sscreen, templ, tc_compatible_htile);
1311
1312 /* This allocates textures with multiple planes like NV12 in 1 buffer. */
1313 enum
1314 {
1315 SI_TEXTURE_MAX_PLANES = 3
1316 };
1317 struct radeon_surf surface[SI_TEXTURE_MAX_PLANES] = {};
1318 struct pipe_resource plane_templ[SI_TEXTURE_MAX_PLANES];
1319 uint64_t plane_offset[SI_TEXTURE_MAX_PLANES] = {};
1320 uint64_t total_size = 0;
1321 unsigned max_alignment = 0;
1322 unsigned num_planes = util_format_get_num_planes(templ->format);
1323 assert(num_planes <= SI_TEXTURE_MAX_PLANES);
1324
1325 /* Compute texture or plane layouts and offsets. */
1326 for (unsigned i = 0; i < num_planes; i++) {
1327 plane_templ[i] = *templ;
1328 plane_templ[i].format = util_format_get_plane_format(templ->format, i);
1329 plane_templ[i].width0 = util_format_get_plane_width(templ->format, i, templ->width0);
1330 plane_templ[i].height0 = util_format_get_plane_height(templ->format, i, templ->height0);
1331
1332 /* Multi-plane allocations need PIPE_BIND_SHARED, because we can't
1333 * reallocate the storage to add PIPE_BIND_SHARED, because it's
1334 * shared by 3 pipe_resources.
1335 */
1336 if (num_planes > 1)
1337 plane_templ[i].bind |= PIPE_BIND_SHARED;
1338
1339 if (si_init_surface(sscreen, &surface[i], &plane_templ[i], tile_mode, false,
1340 plane_templ[i].bind & PIPE_BIND_SCANOUT, is_flushed_depth,
1341 tc_compatible_htile))
1342 return NULL;
1343
1344 plane_offset[i] = align64(total_size, surface[i].surf_alignment);
1345 total_size = plane_offset[i] + surface[i].total_size;
1346 max_alignment = MAX2(max_alignment, surface[i].surf_alignment);
1347 }
1348
1349 struct si_texture *plane0 = NULL, *last_plane = NULL;
1350
1351 for (unsigned i = 0; i < num_planes; i++) {
1352 struct si_texture *tex =
1353 si_texture_create_object(screen, &plane_templ[i], &surface[i], plane0, NULL,
1354 plane_offset[i], 0, total_size, max_alignment);
1355 if (!tex) {
1356 si_texture_reference(&plane0, NULL);
1357 return NULL;
1358 }
1359
1360 tex->plane_index = i;
1361 tex->num_planes = num_planes;
1362
1363 if (!plane0) {
1364 plane0 = last_plane = tex;
1365 } else {
1366 last_plane->buffer.b.b.next = &tex->buffer.b.b;
1367 last_plane = tex;
1368 }
1369 }
1370
1371 return (struct pipe_resource *)plane0;
1372 }
1373
1374 static struct pipe_resource *si_texture_from_winsys_buffer(struct si_screen *sscreen,
1375 const struct pipe_resource *templ,
1376 struct pb_buffer *buf, unsigned stride,
1377 uint64_t offset, unsigned usage,
1378 bool dedicated)
1379 {
1380 struct radeon_surf surface = {};
1381 struct radeon_bo_metadata metadata = {};
1382 struct si_texture *tex;
1383 int r;
1384
1385 /* Ignore metadata for non-zero planes. */
1386 if (offset != 0)
1387 dedicated = false;
1388
1389 if (dedicated) {
1390 sscreen->ws->buffer_get_metadata(buf, &metadata, &surface);
1391 } else {
1392 /**
1393 * The bo metadata is unset for un-dedicated images. So we fall
1394 * back to linear. See answer to question 5 of the
1395 * VK_KHX_external_memory spec for some details.
1396 *
1397 * It is possible that this case isn't going to work if the
1398 * surface pitch isn't correctly aligned by default.
1399 *
1400 * In order to support it correctly we require multi-image
1401 * metadata to be syncrhonized between radv and radeonsi. The
1402 * semantics of associating multiple image metadata to a memory
1403 * object on the vulkan export side are not concretely defined
1404 * either.
1405 *
1406 * All the use cases we are aware of at the moment for memory
1407 * objects use dedicated allocations. So lets keep the initial
1408 * implementation simple.
1409 *
1410 * A possible alternative is to attempt to reconstruct the
1411 * tiling information when the TexParameter TEXTURE_TILING_EXT
1412 * is set.
1413 */
1414 metadata.mode = RADEON_SURF_MODE_LINEAR_ALIGNED;
1415 }
1416
1417 r = si_init_surface(sscreen, &surface, templ, metadata.mode, true,
1418 surface.flags & RADEON_SURF_SCANOUT, false, false);
1419 if (r)
1420 return NULL;
1421
1422 tex = si_texture_create_object(&sscreen->b, templ, &surface, NULL, buf,
1423 offset, stride, 0, 0);
1424 if (!tex)
1425 return NULL;
1426
1427 tex->buffer.b.is_shared = true;
1428 tex->buffer.external_usage = usage;
1429 tex->num_planes = 1;
1430
1431 /* Account for multiple planes with lowered yuv import. */
1432 struct pipe_resource *next_plane = tex->buffer.b.b.next;
1433 while(next_plane) {
1434 struct si_texture *next_tex = (struct si_texture *)next_plane;
1435 ++next_tex->num_planes;
1436 ++tex->num_planes;
1437 next_plane = next_plane->next;
1438 }
1439
1440 if (!ac_surface_set_umd_metadata(&sscreen->info, &tex->surface,
1441 tex->buffer.b.b.nr_storage_samples,
1442 tex->buffer.b.b.last_level + 1,
1443 metadata.size_metadata,
1444 metadata.metadata)) {
1445 si_texture_reference(&tex, NULL);
1446 return NULL;
1447 }
1448
1449 /* Displayable DCC requires an explicit flush. */
1450 if (dedicated && offset == 0 && !(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH) &&
1451 si_has_displayable_dcc(tex)) {
1452 /* TODO: do we need to decompress DCC? */
1453 if (si_texture_discard_dcc(sscreen, tex)) {
1454 /* Update BO metadata after disabling DCC. */
1455 si_set_tex_bo_metadata(sscreen, tex);
1456 }
1457 }
1458
1459 assert(tex->surface.tile_swizzle == 0);
1460 return &tex->buffer.b.b;
1461 }
1462
1463 static struct pipe_resource *si_texture_from_handle(struct pipe_screen *screen,
1464 const struct pipe_resource *templ,
1465 struct winsys_handle *whandle, unsigned usage)
1466 {
1467 struct si_screen *sscreen = (struct si_screen *)screen;
1468 struct pb_buffer *buf = NULL;
1469
1470 /* Support only 2D textures without mipmaps */
1471 if ((templ->target != PIPE_TEXTURE_2D && templ->target != PIPE_TEXTURE_RECT &&
1472 templ->target != PIPE_TEXTURE_2D_ARRAY) ||
1473 templ->last_level != 0)
1474 return NULL;
1475
1476 buf = sscreen->ws->buffer_from_handle(sscreen->ws, whandle, sscreen->info.max_alignment);
1477 if (!buf)
1478 return NULL;
1479
1480 return si_texture_from_winsys_buffer(sscreen, templ, buf, whandle->stride, whandle->offset,
1481 usage, true);
1482 }
1483
1484 bool si_init_flushed_depth_texture(struct pipe_context *ctx, struct pipe_resource *texture)
1485 {
1486 struct si_texture *tex = (struct si_texture *)texture;
1487 struct pipe_resource resource;
1488 enum pipe_format pipe_format = texture->format;
1489
1490 assert(!tex->flushed_depth_texture);
1491
1492 if (!tex->can_sample_z && tex->can_sample_s) {
1493 switch (pipe_format) {
1494 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
1495 /* Save memory by not allocating the S plane. */
1496 pipe_format = PIPE_FORMAT_Z32_FLOAT;
1497 break;
1498 case PIPE_FORMAT_Z24_UNORM_S8_UINT:
1499 case PIPE_FORMAT_S8_UINT_Z24_UNORM:
1500 /* Save memory bandwidth by not copying the
1501 * stencil part during flush.
1502 *
1503 * This potentially increases memory bandwidth
1504 * if an application uses both Z and S texturing
1505 * simultaneously (a flushed Z24S8 texture
1506 * would be stored compactly), but how often
1507 * does that really happen?
1508 */
1509 pipe_format = PIPE_FORMAT_Z24X8_UNORM;
1510 break;
1511 default:;
1512 }
1513 } else if (!tex->can_sample_s && tex->can_sample_z) {
1514 assert(util_format_has_stencil(util_format_description(pipe_format)));
1515
1516 /* DB->CB copies to an 8bpp surface don't work. */
1517 pipe_format = PIPE_FORMAT_X24S8_UINT;
1518 }
1519
1520 memset(&resource, 0, sizeof(resource));
1521 resource.target = texture->target;
1522 resource.format = pipe_format;
1523 resource.width0 = texture->width0;
1524 resource.height0 = texture->height0;
1525 resource.depth0 = texture->depth0;
1526 resource.array_size = texture->array_size;
1527 resource.last_level = texture->last_level;
1528 resource.nr_samples = texture->nr_samples;
1529 resource.usage = PIPE_USAGE_DEFAULT;
1530 resource.bind = texture->bind & ~PIPE_BIND_DEPTH_STENCIL;
1531 resource.flags = texture->flags | SI_RESOURCE_FLAG_FLUSHED_DEPTH;
1532
1533 tex->flushed_depth_texture =
1534 (struct si_texture *)ctx->screen->resource_create(ctx->screen, &resource);
1535 if (!tex->flushed_depth_texture) {
1536 PRINT_ERR("failed to create temporary texture to hold flushed depth\n");
1537 return false;
1538 }
1539 return true;
1540 }
1541
1542 /**
1543 * Initialize the pipe_resource descriptor to be of the same size as the box,
1544 * which is supposed to hold a subregion of the texture "orig" at the given
1545 * mipmap level.
1546 */
1547 static void si_init_temp_resource_from_box(struct pipe_resource *res, struct pipe_resource *orig,
1548 const struct pipe_box *box, unsigned level,
1549 unsigned usage, unsigned flags)
1550 {
1551 memset(res, 0, sizeof(*res));
1552 res->format = orig->format;
1553 res->width0 = box->width;
1554 res->height0 = box->height;
1555 res->depth0 = 1;
1556 res->array_size = 1;
1557 res->usage = usage;
1558 res->flags = flags;
1559
1560 if (flags & SI_RESOURCE_FLAG_FORCE_LINEAR && util_format_is_compressed(orig->format)) {
1561 /* Transfer resources are allocated with linear tiling, which is
1562 * not supported for compressed formats.
1563 */
1564 unsigned blocksize = util_format_get_blocksize(orig->format);
1565
1566 if (blocksize == 8) {
1567 res->format = PIPE_FORMAT_R16G16B16A16_UINT;
1568 } else {
1569 assert(blocksize == 16);
1570 res->format = PIPE_FORMAT_R32G32B32A32_UINT;
1571 }
1572
1573 res->width0 = util_format_get_nblocksx(orig->format, box->width);
1574 res->height0 = util_format_get_nblocksy(orig->format, box->height);
1575 }
1576
1577 /* We must set the correct texture target and dimensions for a 3D box. */
1578 if (box->depth > 1 && util_max_layer(orig, level) > 0) {
1579 res->target = PIPE_TEXTURE_2D_ARRAY;
1580 res->array_size = box->depth;
1581 } else {
1582 res->target = PIPE_TEXTURE_2D;
1583 }
1584 }
1585
1586 static bool si_can_invalidate_texture(struct si_screen *sscreen, struct si_texture *tex,
1587 unsigned transfer_usage, const struct pipe_box *box)
1588 {
1589 return !tex->buffer.b.is_shared && !(tex->surface.flags & RADEON_SURF_IMPORTED) &&
1590 !(transfer_usage & PIPE_TRANSFER_READ) && tex->buffer.b.b.last_level == 0 &&
1591 util_texrange_covers_whole_level(&tex->buffer.b.b, 0, box->x, box->y, box->z, box->width,
1592 box->height, box->depth);
1593 }
1594
1595 static void si_texture_invalidate_storage(struct si_context *sctx, struct si_texture *tex)
1596 {
1597 struct si_screen *sscreen = sctx->screen;
1598
1599 /* There is no point in discarding depth and tiled buffers. */
1600 assert(!tex->is_depth);
1601 assert(tex->surface.is_linear);
1602
1603 /* Reallocate the buffer in the same pipe_resource. */
1604 si_alloc_resource(sscreen, &tex->buffer);
1605
1606 /* Initialize the CMASK base address (needed even without CMASK). */
1607 tex->cmask_base_address_reg = (tex->buffer.gpu_address + tex->surface.cmask_offset) >> 8;
1608
1609 p_atomic_inc(&sscreen->dirty_tex_counter);
1610
1611 sctx->num_alloc_tex_transfer_bytes += tex->surface.total_size;
1612 }
1613
1614 static void *si_texture_transfer_map(struct pipe_context *ctx, struct pipe_resource *texture,
1615 unsigned level, unsigned usage, const struct pipe_box *box,
1616 struct pipe_transfer **ptransfer)
1617 {
1618 struct si_context *sctx = (struct si_context *)ctx;
1619 struct si_texture *tex = (struct si_texture *)texture;
1620 struct si_transfer *trans;
1621 struct si_resource *buf;
1622 unsigned offset = 0;
1623 char *map;
1624 bool use_staging_texture = false;
1625
1626 assert(!(texture->flags & SI_RESOURCE_FLAG_FORCE_LINEAR));
1627 assert(box->width && box->height && box->depth);
1628
1629 /* If we are uploading into FP16 or R11G11B10_FLOAT via a blit, CB clobbers NaNs,
1630 * so in order to preserve them exactly, we have to use the compute blit.
1631 * The compute blit is used only when the destination doesn't have DCC, so
1632 * disable it here, which is kinda a hack.
1633 *
1634 * This makes KHR-GL45.texture_view.view_classes pass on gfx9.
1635 * gfx10 has the same issue, but the test doesn't use a large enough texture
1636 * to enable DCC and fail, so it always passes.
1637 */
1638 const struct util_format_description *desc = util_format_description(texture->format);
1639 if (vi_dcc_enabled(tex, level) &&
1640 desc->channel[0].type == UTIL_FORMAT_TYPE_FLOAT &&
1641 desc->channel[0].size < 32)
1642 si_texture_disable_dcc(sctx, tex);
1643
1644 if (tex->is_depth) {
1645 /* Depth textures use staging unconditionally. */
1646 use_staging_texture = true;
1647 } else {
1648 /* Degrade the tile mode if we get too many transfers on APUs.
1649 * On dGPUs, the staging texture is always faster.
1650 * Only count uploads that are at least 4x4 pixels large.
1651 */
1652 if (!sctx->screen->info.has_dedicated_vram && level == 0 && box->width >= 4 &&
1653 box->height >= 4 && p_atomic_inc_return(&tex->num_level0_transfers) == 10) {
1654 bool can_invalidate = si_can_invalidate_texture(sctx->screen, tex, usage, box);
1655
1656 si_reallocate_texture_inplace(sctx, tex, PIPE_BIND_LINEAR, can_invalidate);
1657 }
1658
1659 /* Tiled textures need to be converted into a linear texture for CPU
1660 * access. The staging texture is always linear and is placed in GART.
1661 *
1662 * Reading from VRAM or GTT WC is slow, always use the staging
1663 * texture in this case.
1664 *
1665 * Use the staging texture for uploads if the underlying BO
1666 * is busy.
1667 */
1668 if (!tex->surface.is_linear || (tex->buffer.flags & RADEON_FLAG_ENCRYPTED))
1669 use_staging_texture = true;
1670 else if (usage & PIPE_TRANSFER_READ)
1671 use_staging_texture =
1672 tex->buffer.domains & RADEON_DOMAIN_VRAM || tex->buffer.flags & RADEON_FLAG_GTT_WC;
1673 /* Write & linear only: */
1674 else if (si_rings_is_buffer_referenced(sctx, tex->buffer.buf, RADEON_USAGE_READWRITE) ||
1675 !sctx->ws->buffer_wait(tex->buffer.buf, 0, RADEON_USAGE_READWRITE)) {
1676 /* It's busy. */
1677 if (si_can_invalidate_texture(sctx->screen, tex, usage, box))
1678 si_texture_invalidate_storage(sctx, tex);
1679 else
1680 use_staging_texture = true;
1681 }
1682 }
1683
1684 trans = CALLOC_STRUCT(si_transfer);
1685 if (!trans)
1686 return NULL;
1687 pipe_resource_reference(&trans->b.b.resource, texture);
1688 trans->b.b.level = level;
1689 trans->b.b.usage = usage;
1690 trans->b.b.box = *box;
1691
1692 if (use_staging_texture) {
1693 struct pipe_resource resource;
1694 struct si_texture *staging;
1695 unsigned bo_usage = usage & PIPE_TRANSFER_READ ? PIPE_USAGE_STAGING : PIPE_USAGE_STREAM;
1696 unsigned bo_flags = SI_RESOURCE_FLAG_FORCE_LINEAR;
1697
1698 /* The pixel shader has a bad access pattern for linear textures.
1699 * If a pixel shader is used to blit to/from staging, don't disable caches.
1700 *
1701 * MSAA, depth/stencil textures, and compressed textures use the pixel shader
1702 * to blit.
1703 */
1704 if (texture->nr_samples <= 1 &&
1705 !tex->is_depth &&
1706 !util_format_is_compressed(texture->format) &&
1707 /* Texture uploads with DCC use the pixel shader to blit */
1708 (!(usage & PIPE_TRANSFER_WRITE) || !vi_dcc_enabled(tex, level)))
1709 bo_flags |= SI_RESOURCE_FLAG_UNCACHED;
1710
1711 si_init_temp_resource_from_box(&resource, texture, box, level, bo_usage,
1712 bo_flags);
1713
1714 /* Since depth-stencil textures don't support linear tiling,
1715 * blit from ZS to color and vice versa. u_blitter will do
1716 * the packing for these formats.
1717 */
1718 if (tex->is_depth)
1719 resource.format = util_blitter_get_color_format_for_zs(resource.format);
1720
1721 /* Create the temporary texture. */
1722 staging = (struct si_texture *)ctx->screen->resource_create(ctx->screen, &resource);
1723 if (!staging) {
1724 PRINT_ERR("failed to create temporary texture to hold untiled copy\n");
1725 goto fail_trans;
1726 }
1727 trans->staging = &staging->buffer;
1728
1729 /* Just get the strides. */
1730 si_texture_get_offset(sctx->screen, staging, 0, NULL, &trans->b.b.stride,
1731 &trans->b.b.layer_stride);
1732
1733 if (usage & PIPE_TRANSFER_READ)
1734 si_copy_to_staging_texture(ctx, trans);
1735 else
1736 usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
1737
1738 buf = trans->staging;
1739 } else {
1740 /* the resource is mapped directly */
1741 offset = si_texture_get_offset(sctx->screen, tex, level, box, &trans->b.b.stride,
1742 &trans->b.b.layer_stride);
1743 buf = &tex->buffer;
1744 }
1745
1746 /* Always unmap texture CPU mappings on 32-bit architectures, so that
1747 * we don't run out of the CPU address space.
1748 */
1749 if (sizeof(void *) == 4)
1750 usage |= RADEON_TRANSFER_TEMPORARY;
1751
1752 if (!(map = si_buffer_map_sync_with_rings(sctx, buf, usage)))
1753 goto fail_trans;
1754
1755 *ptransfer = &trans->b.b;
1756 return map + offset;
1757
1758 fail_trans:
1759 si_resource_reference(&trans->staging, NULL);
1760 pipe_resource_reference(&trans->b.b.resource, NULL);
1761 FREE(trans);
1762 return NULL;
1763 }
1764
1765 static void si_texture_transfer_unmap(struct pipe_context *ctx, struct pipe_transfer *transfer)
1766 {
1767 struct si_context *sctx = (struct si_context *)ctx;
1768 struct si_transfer *stransfer = (struct si_transfer *)transfer;
1769 struct pipe_resource *texture = transfer->resource;
1770 struct si_texture *tex = (struct si_texture *)texture;
1771
1772 /* Always unmap texture CPU mappings on 32-bit architectures, so that
1773 * we don't run out of the CPU address space.
1774 */
1775 if (sizeof(void *) == 4) {
1776 struct si_resource *buf = stransfer->staging ? stransfer->staging : &tex->buffer;
1777
1778 sctx->ws->buffer_unmap(buf->buf);
1779 }
1780
1781 if ((transfer->usage & PIPE_TRANSFER_WRITE) && stransfer->staging)
1782 si_copy_from_staging_texture(ctx, stransfer);
1783
1784 if (stransfer->staging) {
1785 sctx->num_alloc_tex_transfer_bytes += stransfer->staging->buf->size;
1786 si_resource_reference(&stransfer->staging, NULL);
1787 }
1788
1789 /* Heuristic for {upload, draw, upload, draw, ..}:
1790 *
1791 * Flush the gfx IB if we've allocated too much texture storage.
1792 *
1793 * The idea is that we don't want to build IBs that use too much
1794 * memory and put pressure on the kernel memory manager and we also
1795 * want to make temporary and invalidated buffers go idle ASAP to
1796 * decrease the total memory usage or make them reusable. The memory
1797 * usage will be slightly higher than given here because of the buffer
1798 * cache in the winsys.
1799 *
1800 * The result is that the kernel memory manager is never a bottleneck.
1801 */
1802 if (sctx->num_alloc_tex_transfer_bytes > sctx->screen->info.gart_size / 4) {
1803 si_flush_gfx_cs(sctx, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW, NULL);
1804 sctx->num_alloc_tex_transfer_bytes = 0;
1805 }
1806
1807 pipe_resource_reference(&transfer->resource, NULL);
1808 FREE(transfer);
1809 }
1810
1811 static const struct u_resource_vtbl si_texture_vtbl = {
1812 NULL, /* get_handle */
1813 si_texture_destroy, /* resource_destroy */
1814 si_texture_transfer_map, /* transfer_map */
1815 u_default_transfer_flush_region, /* transfer_flush_region */
1816 si_texture_transfer_unmap, /* transfer_unmap */
1817 };
1818
1819 /* Return if it's allowed to reinterpret one format as another with DCC enabled.
1820 */
1821 bool vi_dcc_formats_compatible(struct si_screen *sscreen, enum pipe_format format1,
1822 enum pipe_format format2)
1823 {
1824 const struct util_format_description *desc1, *desc2;
1825
1826 /* No format change - exit early. */
1827 if (format1 == format2)
1828 return true;
1829
1830 format1 = si_simplify_cb_format(format1);
1831 format2 = si_simplify_cb_format(format2);
1832
1833 /* Check again after format adjustments. */
1834 if (format1 == format2)
1835 return true;
1836
1837 desc1 = util_format_description(format1);
1838 desc2 = util_format_description(format2);
1839
1840 if (desc1->layout != UTIL_FORMAT_LAYOUT_PLAIN || desc2->layout != UTIL_FORMAT_LAYOUT_PLAIN)
1841 return false;
1842
1843 /* Float and non-float are totally incompatible. */
1844 if ((desc1->channel[0].type == UTIL_FORMAT_TYPE_FLOAT) !=
1845 (desc2->channel[0].type == UTIL_FORMAT_TYPE_FLOAT))
1846 return false;
1847
1848 /* Channel sizes must match across DCC formats.
1849 * Comparing just the first 2 channels should be enough.
1850 */
1851 if (desc1->channel[0].size != desc2->channel[0].size ||
1852 (desc1->nr_channels >= 2 && desc1->channel[1].size != desc2->channel[1].size))
1853 return false;
1854
1855 /* Everything below is not needed if the driver never uses the DCC
1856 * clear code with the value of 1.
1857 */
1858
1859 /* If the clear values are all 1 or all 0, this constraint can be
1860 * ignored. */
1861 if (vi_alpha_is_on_msb(sscreen, format1) != vi_alpha_is_on_msb(sscreen, format2))
1862 return false;
1863
1864 /* Channel types must match if the clear value of 1 is used.
1865 * The type categories are only float, signed, unsigned.
1866 * NORM and INT are always compatible.
1867 */
1868 if (desc1->channel[0].type != desc2->channel[0].type ||
1869 (desc1->nr_channels >= 2 && desc1->channel[1].type != desc2->channel[1].type))
1870 return false;
1871
1872 return true;
1873 }
1874
1875 bool vi_dcc_formats_are_incompatible(struct pipe_resource *tex, unsigned level,
1876 enum pipe_format view_format)
1877 {
1878 struct si_texture *stex = (struct si_texture *)tex;
1879
1880 return vi_dcc_enabled(stex, level) &&
1881 !vi_dcc_formats_compatible((struct si_screen *)tex->screen, tex->format, view_format);
1882 }
1883
1884 /* This can't be merged with the above function, because
1885 * vi_dcc_formats_compatible should be called only when DCC is enabled. */
1886 void vi_disable_dcc_if_incompatible_format(struct si_context *sctx, struct pipe_resource *tex,
1887 unsigned level, enum pipe_format view_format)
1888 {
1889 struct si_texture *stex = (struct si_texture *)tex;
1890
1891 if (vi_dcc_formats_are_incompatible(tex, level, view_format))
1892 if (!si_texture_disable_dcc(sctx, stex))
1893 si_decompress_dcc(sctx, stex);
1894 }
1895
1896 struct pipe_surface *si_create_surface_custom(struct pipe_context *pipe,
1897 struct pipe_resource *texture,
1898 const struct pipe_surface *templ, unsigned width0,
1899 unsigned height0, unsigned width, unsigned height)
1900 {
1901 struct si_surface *surface = CALLOC_STRUCT(si_surface);
1902
1903 if (!surface)
1904 return NULL;
1905
1906 assert(templ->u.tex.first_layer <= util_max_layer(texture, templ->u.tex.level));
1907 assert(templ->u.tex.last_layer <= util_max_layer(texture, templ->u.tex.level));
1908
1909 pipe_reference_init(&surface->base.reference, 1);
1910 pipe_resource_reference(&surface->base.texture, texture);
1911 surface->base.context = pipe;
1912 surface->base.format = templ->format;
1913 surface->base.width = width;
1914 surface->base.height = height;
1915 surface->base.u = templ->u;
1916
1917 surface->width0 = width0;
1918 surface->height0 = height0;
1919
1920 surface->dcc_incompatible =
1921 texture->target != PIPE_BUFFER &&
1922 vi_dcc_formats_are_incompatible(texture, templ->u.tex.level, templ->format);
1923 return &surface->base;
1924 }
1925
1926 static struct pipe_surface *si_create_surface(struct pipe_context *pipe, struct pipe_resource *tex,
1927 const struct pipe_surface *templ)
1928 {
1929 unsigned level = templ->u.tex.level;
1930 unsigned width = u_minify(tex->width0, level);
1931 unsigned height = u_minify(tex->height0, level);
1932 unsigned width0 = tex->width0;
1933 unsigned height0 = tex->height0;
1934
1935 if (tex->target != PIPE_BUFFER && templ->format != tex->format) {
1936 const struct util_format_description *tex_desc = util_format_description(tex->format);
1937 const struct util_format_description *templ_desc = util_format_description(templ->format);
1938
1939 assert(tex_desc->block.bits == templ_desc->block.bits);
1940
1941 /* Adjust size of surface if and only if the block width or
1942 * height is changed. */
1943 if (tex_desc->block.width != templ_desc->block.width ||
1944 tex_desc->block.height != templ_desc->block.height) {
1945 unsigned nblks_x = util_format_get_nblocksx(tex->format, width);
1946 unsigned nblks_y = util_format_get_nblocksy(tex->format, height);
1947
1948 width = nblks_x * templ_desc->block.width;
1949 height = nblks_y * templ_desc->block.height;
1950
1951 width0 = util_format_get_nblocksx(tex->format, width0);
1952 height0 = util_format_get_nblocksy(tex->format, height0);
1953 }
1954 }
1955
1956 return si_create_surface_custom(pipe, tex, templ, width0, height0, width, height);
1957 }
1958
1959 static void si_surface_destroy(struct pipe_context *pipe, struct pipe_surface *surface)
1960 {
1961 pipe_resource_reference(&surface->texture, NULL);
1962 FREE(surface);
1963 }
1964
1965 unsigned si_translate_colorswap(enum pipe_format format, bool do_endian_swap)
1966 {
1967 const struct util_format_description *desc = util_format_description(format);
1968
1969 #define HAS_SWIZZLE(chan, swz) (desc->swizzle[chan] == PIPE_SWIZZLE_##swz)
1970
1971 if (format == PIPE_FORMAT_R11G11B10_FLOAT) /* isn't plain */
1972 return V_028C70_SWAP_STD;
1973
1974 if (desc->layout != UTIL_FORMAT_LAYOUT_PLAIN)
1975 return ~0U;
1976
1977 switch (desc->nr_channels) {
1978 case 1:
1979 if (HAS_SWIZZLE(0, X))
1980 return V_028C70_SWAP_STD; /* X___ */
1981 else if (HAS_SWIZZLE(3, X))
1982 return V_028C70_SWAP_ALT_REV; /* ___X */
1983 break;
1984 case 2:
1985 if ((HAS_SWIZZLE(0, X) && HAS_SWIZZLE(1, Y)) || (HAS_SWIZZLE(0, X) && HAS_SWIZZLE(1, NONE)) ||
1986 (HAS_SWIZZLE(0, NONE) && HAS_SWIZZLE(1, Y)))
1987 return V_028C70_SWAP_STD; /* XY__ */
1988 else if ((HAS_SWIZZLE(0, Y) && HAS_SWIZZLE(1, X)) ||
1989 (HAS_SWIZZLE(0, Y) && HAS_SWIZZLE(1, NONE)) ||
1990 (HAS_SWIZZLE(0, NONE) && HAS_SWIZZLE(1, X)))
1991 /* YX__ */
1992 return (do_endian_swap ? V_028C70_SWAP_STD : V_028C70_SWAP_STD_REV);
1993 else if (HAS_SWIZZLE(0, X) && HAS_SWIZZLE(3, Y))
1994 return V_028C70_SWAP_ALT; /* X__Y */
1995 else if (HAS_SWIZZLE(0, Y) && HAS_SWIZZLE(3, X))
1996 return V_028C70_SWAP_ALT_REV; /* Y__X */
1997 break;
1998 case 3:
1999 if (HAS_SWIZZLE(0, X))
2000 return (do_endian_swap ? V_028C70_SWAP_STD_REV : V_028C70_SWAP_STD);
2001 else if (HAS_SWIZZLE(0, Z))
2002 return V_028C70_SWAP_STD_REV; /* ZYX */
2003 break;
2004 case 4:
2005 /* check the middle channels, the 1st and 4th channel can be NONE */
2006 if (HAS_SWIZZLE(1, Y) && HAS_SWIZZLE(2, Z)) {
2007 return V_028C70_SWAP_STD; /* XYZW */
2008 } else if (HAS_SWIZZLE(1, Z) && HAS_SWIZZLE(2, Y)) {
2009 return V_028C70_SWAP_STD_REV; /* WZYX */
2010 } else if (HAS_SWIZZLE(1, Y) && HAS_SWIZZLE(2, X)) {
2011 return V_028C70_SWAP_ALT; /* ZYXW */
2012 } else if (HAS_SWIZZLE(1, Z) && HAS_SWIZZLE(2, W)) {
2013 /* YZWX */
2014 if (desc->is_array)
2015 return V_028C70_SWAP_ALT_REV;
2016 else
2017 return (do_endian_swap ? V_028C70_SWAP_ALT : V_028C70_SWAP_ALT_REV);
2018 }
2019 break;
2020 }
2021 return ~0U;
2022 }
2023
2024 /* PIPELINE_STAT-BASED DCC ENABLEMENT FOR DISPLAYABLE SURFACES */
2025
2026 static void vi_dcc_clean_up_context_slot(struct si_context *sctx, int slot)
2027 {
2028 int i;
2029
2030 if (sctx->dcc_stats[slot].query_active)
2031 vi_separate_dcc_stop_query(sctx, sctx->dcc_stats[slot].tex);
2032
2033 for (i = 0; i < ARRAY_SIZE(sctx->dcc_stats[slot].ps_stats); i++)
2034 if (sctx->dcc_stats[slot].ps_stats[i]) {
2035 sctx->b.destroy_query(&sctx->b, sctx->dcc_stats[slot].ps_stats[i]);
2036 sctx->dcc_stats[slot].ps_stats[i] = NULL;
2037 }
2038
2039 si_texture_reference(&sctx->dcc_stats[slot].tex, NULL);
2040 }
2041
2042 /**
2043 * Return the per-context slot where DCC statistics queries for the texture live.
2044 */
2045 static unsigned vi_get_context_dcc_stats_index(struct si_context *sctx, struct si_texture *tex)
2046 {
2047 int i, empty_slot = -1;
2048
2049 /* Remove zombie textures (textures kept alive by this array only). */
2050 for (i = 0; i < ARRAY_SIZE(sctx->dcc_stats); i++)
2051 if (sctx->dcc_stats[i].tex && sctx->dcc_stats[i].tex->buffer.b.b.reference.count == 1)
2052 vi_dcc_clean_up_context_slot(sctx, i);
2053
2054 /* Find the texture. */
2055 for (i = 0; i < ARRAY_SIZE(sctx->dcc_stats); i++) {
2056 /* Return if found. */
2057 if (sctx->dcc_stats[i].tex == tex) {
2058 sctx->dcc_stats[i].last_use_timestamp = os_time_get();
2059 return i;
2060 }
2061
2062 /* Record the first seen empty slot. */
2063 if (empty_slot == -1 && !sctx->dcc_stats[i].tex)
2064 empty_slot = i;
2065 }
2066
2067 /* Not found. Remove the oldest member to make space in the array. */
2068 if (empty_slot == -1) {
2069 int oldest_slot = 0;
2070
2071 /* Find the oldest slot. */
2072 for (i = 1; i < ARRAY_SIZE(sctx->dcc_stats); i++)
2073 if (sctx->dcc_stats[oldest_slot].last_use_timestamp >
2074 sctx->dcc_stats[i].last_use_timestamp)
2075 oldest_slot = i;
2076
2077 /* Clean up the oldest slot. */
2078 vi_dcc_clean_up_context_slot(sctx, oldest_slot);
2079 empty_slot = oldest_slot;
2080 }
2081
2082 /* Add the texture to the new slot. */
2083 si_texture_reference(&sctx->dcc_stats[empty_slot].tex, tex);
2084 sctx->dcc_stats[empty_slot].last_use_timestamp = os_time_get();
2085 return empty_slot;
2086 }
2087
2088 static struct pipe_query *vi_create_resuming_pipestats_query(struct si_context *sctx)
2089 {
2090 struct si_query_hw *query =
2091 (struct si_query_hw *)sctx->b.create_query(&sctx->b, PIPE_QUERY_PIPELINE_STATISTICS, 0);
2092
2093 query->flags |= SI_QUERY_HW_FLAG_BEGIN_RESUMES;
2094 return (struct pipe_query *)query;
2095 }
2096
2097 /**
2098 * Called when binding a color buffer.
2099 */
2100 void vi_separate_dcc_start_query(struct si_context *sctx, struct si_texture *tex)
2101 {
2102 unsigned i = vi_get_context_dcc_stats_index(sctx, tex);
2103
2104 assert(!sctx->dcc_stats[i].query_active);
2105
2106 if (!sctx->dcc_stats[i].ps_stats[0])
2107 sctx->dcc_stats[i].ps_stats[0] = vi_create_resuming_pipestats_query(sctx);
2108
2109 /* begin or resume the query */
2110 sctx->b.begin_query(&sctx->b, sctx->dcc_stats[i].ps_stats[0]);
2111 sctx->dcc_stats[i].query_active = true;
2112 }
2113
2114 /**
2115 * Called when unbinding a color buffer.
2116 */
2117 void vi_separate_dcc_stop_query(struct si_context *sctx, struct si_texture *tex)
2118 {
2119 unsigned i = vi_get_context_dcc_stats_index(sctx, tex);
2120
2121 assert(sctx->dcc_stats[i].query_active);
2122 assert(sctx->dcc_stats[i].ps_stats[0]);
2123
2124 /* pause or end the query */
2125 sctx->b.end_query(&sctx->b, sctx->dcc_stats[i].ps_stats[0]);
2126 sctx->dcc_stats[i].query_active = false;
2127 }
2128
2129 static bool vi_should_enable_separate_dcc(struct si_texture *tex)
2130 {
2131 /* The minimum number of fullscreen draws per frame that is required
2132 * to enable DCC. */
2133 return tex->ps_draw_ratio + tex->num_slow_clears >= 5;
2134 }
2135
2136 /* Called by fast clear. */
2137 void vi_separate_dcc_try_enable(struct si_context *sctx, struct si_texture *tex)
2138 {
2139 /* The intent is to use this with shared displayable back buffers,
2140 * but it's not strictly limited only to them.
2141 */
2142 if (!tex->buffer.b.is_shared ||
2143 !(tex->buffer.external_usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH) ||
2144 tex->buffer.b.b.target != PIPE_TEXTURE_2D || tex->buffer.b.b.last_level > 0 ||
2145 !tex->surface.dcc_size || sctx->screen->debug_flags & DBG(NO_DCC) ||
2146 sctx->screen->debug_flags & DBG(NO_DCC_FB))
2147 return;
2148
2149 assert(sctx->chip_class >= GFX8);
2150
2151 if (tex->surface.dcc_offset)
2152 return; /* already enabled */
2153
2154 /* Enable the DCC stat gathering. */
2155 if (!tex->dcc_gather_statistics) {
2156 tex->dcc_gather_statistics = true;
2157 vi_separate_dcc_start_query(sctx, tex);
2158 }
2159
2160 if (!vi_should_enable_separate_dcc(tex))
2161 return; /* stats show that DCC decompression is too expensive */
2162
2163 assert(tex->surface.num_dcc_levels);
2164 assert(!tex->dcc_separate_buffer);
2165
2166 si_texture_discard_cmask(sctx->screen, tex);
2167
2168 /* Get a DCC buffer. */
2169 if (tex->last_dcc_separate_buffer) {
2170 assert(tex->dcc_gather_statistics);
2171 assert(!tex->dcc_separate_buffer);
2172 tex->dcc_separate_buffer = tex->last_dcc_separate_buffer;
2173 tex->last_dcc_separate_buffer = NULL;
2174 } else {
2175 tex->dcc_separate_buffer =
2176 si_aligned_buffer_create(sctx->b.screen, SI_RESOURCE_FLAG_UNMAPPABLE, PIPE_USAGE_DEFAULT,
2177 tex->surface.dcc_size, tex->surface.dcc_alignment);
2178 if (!tex->dcc_separate_buffer)
2179 return;
2180 }
2181
2182 /* dcc_offset is the absolute GPUVM address. */
2183 tex->surface.dcc_offset = tex->dcc_separate_buffer->gpu_address;
2184
2185 /* no need to flag anything since this is called by fast clear that
2186 * flags framebuffer state
2187 */
2188 }
2189
2190 /**
2191 * Called by pipe_context::flush_resource, the place where DCC decompression
2192 * takes place.
2193 */
2194 void vi_separate_dcc_process_and_reset_stats(struct pipe_context *ctx, struct si_texture *tex)
2195 {
2196 struct si_context *sctx = (struct si_context *)ctx;
2197 struct pipe_query *tmp;
2198 unsigned i = vi_get_context_dcc_stats_index(sctx, tex);
2199 bool query_active = sctx->dcc_stats[i].query_active;
2200 bool disable = false;
2201
2202 if (sctx->dcc_stats[i].ps_stats[2]) {
2203 union pipe_query_result result;
2204
2205 /* Read the results. */
2206 struct pipe_query *query = sctx->dcc_stats[i].ps_stats[2];
2207 ctx->get_query_result(ctx, query, true, &result);
2208 si_query_buffer_reset(sctx, &((struct si_query_hw *)query)->buffer);
2209
2210 /* Compute the approximate number of fullscreen draws. */
2211 tex->ps_draw_ratio = result.pipeline_statistics.ps_invocations /
2212 (tex->buffer.b.b.width0 * tex->buffer.b.b.height0);
2213 sctx->last_tex_ps_draw_ratio = tex->ps_draw_ratio;
2214
2215 disable = tex->dcc_separate_buffer && !vi_should_enable_separate_dcc(tex);
2216 }
2217
2218 tex->num_slow_clears = 0;
2219
2220 /* stop the statistics query for ps_stats[0] */
2221 if (query_active)
2222 vi_separate_dcc_stop_query(sctx, tex);
2223
2224 /* Move the queries in the queue by one. */
2225 tmp = sctx->dcc_stats[i].ps_stats[2];
2226 sctx->dcc_stats[i].ps_stats[2] = sctx->dcc_stats[i].ps_stats[1];
2227 sctx->dcc_stats[i].ps_stats[1] = sctx->dcc_stats[i].ps_stats[0];
2228 sctx->dcc_stats[i].ps_stats[0] = tmp;
2229
2230 /* create and start a new query as ps_stats[0] */
2231 if (query_active)
2232 vi_separate_dcc_start_query(sctx, tex);
2233
2234 if (disable) {
2235 assert(!tex->last_dcc_separate_buffer);
2236 tex->last_dcc_separate_buffer = tex->dcc_separate_buffer;
2237 tex->dcc_separate_buffer = NULL;
2238 tex->surface.dcc_offset = 0;
2239 /* no need to flag anything since this is called after
2240 * decompression that re-sets framebuffer state
2241 */
2242 }
2243 }
2244
2245 static struct pipe_memory_object *
2246 si_memobj_from_handle(struct pipe_screen *screen, struct winsys_handle *whandle, bool dedicated)
2247 {
2248 struct si_screen *sscreen = (struct si_screen *)screen;
2249 struct si_memory_object *memobj = CALLOC_STRUCT(si_memory_object);
2250 struct pb_buffer *buf = NULL;
2251
2252 if (!memobj)
2253 return NULL;
2254
2255 buf = sscreen->ws->buffer_from_handle(sscreen->ws, whandle, sscreen->info.max_alignment);
2256 if (!buf) {
2257 free(memobj);
2258 return NULL;
2259 }
2260
2261 memobj->b.dedicated = dedicated;
2262 memobj->buf = buf;
2263 memobj->stride = whandle->stride;
2264
2265 return (struct pipe_memory_object *)memobj;
2266 }
2267
2268 static void si_memobj_destroy(struct pipe_screen *screen, struct pipe_memory_object *_memobj)
2269 {
2270 struct si_memory_object *memobj = (struct si_memory_object *)_memobj;
2271
2272 pb_reference(&memobj->buf, NULL);
2273 free(memobj);
2274 }
2275
2276 static struct pipe_resource *si_texture_from_memobj(struct pipe_screen *screen,
2277 const struct pipe_resource *templ,
2278 struct pipe_memory_object *_memobj,
2279 uint64_t offset)
2280 {
2281 struct si_screen *sscreen = (struct si_screen *)screen;
2282 struct si_memory_object *memobj = (struct si_memory_object *)_memobj;
2283 struct pipe_resource *tex = si_texture_from_winsys_buffer(
2284 sscreen, templ, memobj->buf, memobj->stride, offset,
2285 PIPE_HANDLE_USAGE_FRAMEBUFFER_WRITE | PIPE_HANDLE_USAGE_SHADER_WRITE, memobj->b.dedicated);
2286 if (!tex)
2287 return NULL;
2288
2289 /* si_texture_from_winsys_buffer doesn't increment refcount of
2290 * memobj->buf, so increment it here.
2291 */
2292 struct pb_buffer *buf = NULL;
2293 pb_reference(&buf, memobj->buf);
2294 return tex;
2295 }
2296
2297 static bool si_check_resource_capability(struct pipe_screen *screen, struct pipe_resource *resource,
2298 unsigned bind)
2299 {
2300 struct si_texture *tex = (struct si_texture *)resource;
2301
2302 /* Buffers only support the linear flag. */
2303 if (resource->target == PIPE_BUFFER)
2304 return (bind & ~PIPE_BIND_LINEAR) == 0;
2305
2306 if (bind & PIPE_BIND_LINEAR && !tex->surface.is_linear)
2307 return false;
2308
2309 if (bind & PIPE_BIND_SCANOUT && !tex->surface.is_displayable)
2310 return false;
2311
2312 /* TODO: PIPE_BIND_CURSOR - do we care? */
2313 return true;
2314 }
2315
2316 void si_init_screen_texture_functions(struct si_screen *sscreen)
2317 {
2318 sscreen->b.resource_from_handle = si_texture_from_handle;
2319 sscreen->b.resource_get_handle = si_texture_get_handle;
2320 sscreen->b.resource_get_param = si_resource_get_param;
2321 sscreen->b.resource_get_info = si_texture_get_info;
2322 sscreen->b.resource_from_memobj = si_texture_from_memobj;
2323 sscreen->b.memobj_create_from_handle = si_memobj_from_handle;
2324 sscreen->b.memobj_destroy = si_memobj_destroy;
2325 sscreen->b.check_resource_capability = si_check_resource_capability;
2326 }
2327
2328 void si_init_context_texture_functions(struct si_context *sctx)
2329 {
2330 sctx->b.create_surface = si_create_surface;
2331 sctx->b.surface_destroy = si_surface_destroy;
2332 }