projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
gallium/radeon: reallocate suballocated buffers when exported
[mesa.git] / src / gallium / drivers / radeon / r600_texture.c
1 /*
2 * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 *
23 * Authors:
24 * Jerome Glisse
25 * Corbin Simpson
26 */
27 #include "r600_pipe_common.h"
28 #include "r600_cs.h"
29 #include "r600_query.h"
30 #include "util/u_format.h"
31 #include "util/u_memory.h"
32 #include "util/u_pack_color.h"
33 #include "util/u_surface.h"
34 #include "os/os_time.h"
35 #include <errno.h>
36 #include <inttypes.h>
37
38 static void r600_texture_discard_cmask(struct r600_common_screen *rscreen,
39 struct r600_texture *rtex);
40 static enum radeon_surf_mode
41 r600_choose_tiling(struct r600_common_screen *rscreen,
42 const struct pipe_resource *templ);
43
44
45 bool r600_prepare_for_dma_blit(struct r600_common_context *rctx,
46 struct r600_texture *rdst,
47 unsigned dst_level, unsigned dstx,
48 unsigned dsty, unsigned dstz,
49 struct r600_texture *rsrc,
50 unsigned src_level,
51 const struct pipe_box *src_box)
52 {
53 if (!rctx->dma.cs)
54 return false;
55
56 if (rdst->surface.bpe != rsrc->surface.bpe)
57 return false;
58
59 /* MSAA: Blits don't exist in the real world. */
60 if (rsrc->resource.b.b.nr_samples > 1 ||
61 rdst->resource.b.b.nr_samples > 1)
62 return false;
63
64 /* Depth-stencil surfaces:
65 * When dst is linear, the DB->CB copy preserves HTILE.
66 * When dst is tiled, the 3D path must be used to update HTILE.
67 */
68 if (rsrc->is_depth || rdst->is_depth)
69 return false;
70
71 /* DCC as:
72 * src: Use the 3D path. DCC decompression is expensive.
73 * dst: Use the 3D path to compress the pixels with DCC.
74 */
75 if (vi_dcc_enabled(rsrc, src_level) ||
76 vi_dcc_enabled(rdst, dst_level))
77 return false;
78
79 /* CMASK as:
80 * src: Both texture and SDMA paths need decompression. Use SDMA.
81 * dst: If overwriting the whole texture, discard CMASK and use
82 * SDMA. Otherwise, use the 3D path.
83 */
84 if (rdst->cmask.size && rdst->dirty_level_mask & (1 << dst_level)) {
85 /* The CMASK clear is only enabled for the first level. */
86 assert(dst_level == 0);
87 if (!util_texrange_covers_whole_level(&rdst->resource.b.b, dst_level,
88 dstx, dsty, dstz, src_box->width,
89 src_box->height, src_box->depth))
90 return false;
91
92 r600_texture_discard_cmask(rctx->screen, rdst);
93 }
94
95 /* All requirements are met. Prepare textures for SDMA. */
96 if (rsrc->cmask.size && rsrc->dirty_level_mask & (1 << src_level))
97 rctx->b.flush_resource(&rctx->b, &rsrc->resource.b.b);
98
99 assert(!(rsrc->dirty_level_mask & (1 << src_level)));
100 assert(!(rdst->dirty_level_mask & (1 << dst_level)));
101
102 return true;
103 }
104
105 /* Same as resource_copy_region, except that both upsampling and downsampling are allowed. */
106 static void r600_copy_region_with_blit(struct pipe_context *pipe,
107 struct pipe_resource *dst,
108 unsigned dst_level,
109 unsigned dstx, unsigned dsty, unsigned dstz,
110 struct pipe_resource *src,
111 unsigned src_level,
112 const struct pipe_box *src_box)
113 {
114 struct pipe_blit_info blit;
115
116 memset(&blit, 0, sizeof(blit));
117 blit.src.resource = src;
118 blit.src.format = src->format;
119 blit.src.level = src_level;
120 blit.src.box = *src_box;
121 blit.dst.resource = dst;
122 blit.dst.format = dst->format;
123 blit.dst.level = dst_level;
124 blit.dst.box.x = dstx;
125 blit.dst.box.y = dsty;
126 blit.dst.box.z = dstz;
127 blit.dst.box.width = src_box->width;
128 blit.dst.box.height = src_box->height;
129 blit.dst.box.depth = src_box->depth;
130 blit.mask = util_format_get_mask(src->format) &
131 util_format_get_mask(dst->format);
132 blit.filter = PIPE_TEX_FILTER_NEAREST;
133
134 if (blit.mask) {
135 pipe->blit(pipe, &blit);
136 }
137 }
138
139 /* Copy from a full GPU texture to a transfer's staging one. */
140 static void r600_copy_to_staging_texture(struct pipe_context *ctx, struct r600_transfer *rtransfer)
141 {
142 struct r600_common_context *rctx = (struct r600_common_context*)ctx;
143 struct pipe_transfer *transfer = (struct pipe_transfer*)rtransfer;
144 struct pipe_resource *dst = &rtransfer->staging->b.b;
145 struct pipe_resource *src = transfer->resource;
146
147 if (src->nr_samples > 1) {
148 r600_copy_region_with_blit(ctx, dst, 0, 0, 0, 0,
149 src, transfer->level, &transfer->box);
150 return;
151 }
152
153 rctx->dma_copy(ctx, dst, 0, 0, 0, 0, src, transfer->level,
154 &transfer->box);
155 }
156
157 /* Copy from a transfer's staging texture to a full GPU one. */
158 static void r600_copy_from_staging_texture(struct pipe_context *ctx, struct r600_transfer *rtransfer)
159 {
160 struct r600_common_context *rctx = (struct r600_common_context*)ctx;
161 struct pipe_transfer *transfer = (struct pipe_transfer*)rtransfer;
162 struct pipe_resource *dst = transfer->resource;
163 struct pipe_resource *src = &rtransfer->staging->b.b;
164 struct pipe_box sbox;
165
166 u_box_3d(0, 0, 0, transfer->box.width, transfer->box.height, transfer->box.depth, &sbox);
167
168 if (dst->nr_samples > 1) {
169 r600_copy_region_with_blit(ctx, dst, transfer->level,
170 transfer->box.x, transfer->box.y, transfer->box.z,
171 src, 0, &sbox);
172 return;
173 }
174
175 rctx->dma_copy(ctx, dst, transfer->level,
176 transfer->box.x, transfer->box.y, transfer->box.z,
177 src, 0, &sbox);
178 }
179
180 static unsigned r600_texture_get_offset(struct r600_common_screen *rscreen,
181 struct r600_texture *rtex, unsigned level,
182 const struct pipe_box *box,
183 unsigned *stride,
184 unsigned *layer_stride)
185 {
186 if (rscreen->chip_class >= GFX9) {
187 *stride = rtex->surface.u.gfx9.surf_pitch * rtex->surface.bpe;
188 *layer_stride = rtex->surface.u.gfx9.surf_slice_size;
189
190 if (!box)
191 return 0;
192
193 /* Each texture is an array of slices. Each slice is an array
194 * of mipmap levels. */
195 return box->z * rtex->surface.u.gfx9.surf_slice_size +
196 rtex->surface.u.gfx9.offset[level] +
197 (box->y / rtex->surface.blk_h *
198 rtex->surface.u.gfx9.surf_pitch +
199 box->x / rtex->surface.blk_w) * rtex->surface.bpe;
200 } else {
201 *stride = rtex->surface.u.legacy.level[level].nblk_x *
202 rtex->surface.bpe;
203 *layer_stride = rtex->surface.u.legacy.level[level].slice_size;
204
205 if (!box)
206 return rtex->surface.u.legacy.level[level].offset;
207
208 /* Each texture is an array of mipmap levels. Each level is
209 * an array of slices. */
210 return rtex->surface.u.legacy.level[level].offset +
211 box->z * rtex->surface.u.legacy.level[level].slice_size +
212 (box->y / rtex->surface.blk_h *
213 rtex->surface.u.legacy.level[level].nblk_x +
214 box->x / rtex->surface.blk_w) * rtex->surface.bpe;
215 }
216 }
217
218 static int r600_init_surface(struct r600_common_screen *rscreen,
219 struct radeon_surf *surface,
220 const struct pipe_resource *ptex,
221 enum radeon_surf_mode array_mode,
222 unsigned pitch_in_bytes_override,
223 unsigned offset,
224 bool is_imported,
225 bool is_scanout,
226 bool is_flushed_depth,
227 bool tc_compatible_htile)
228 {
229 const struct util_format_description *desc =
230 util_format_description(ptex->format);
231 bool is_depth, is_stencil;
232 int r;
233 unsigned i, bpe, flags = 0;
234
235 is_depth = util_format_has_depth(desc);
236 is_stencil = util_format_has_stencil(desc);
237
238 if (rscreen->chip_class >= EVERGREEN && !is_flushed_depth &&
239 ptex->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) {
240 bpe = 4; /* stencil is allocated separately on evergreen */
241 } else {
242 bpe = util_format_get_blocksize(ptex->format);
243 assert(util_is_power_of_two(bpe));
244 }
245
246 if (!is_flushed_depth && is_depth) {
247 flags |= RADEON_SURF_ZBUFFER;
248
249 if (tc_compatible_htile &&
250 (rscreen->chip_class >= GFX9 ||
251 array_mode == RADEON_SURF_MODE_2D)) {
252 /* TC-compatible HTILE only supports Z32_FLOAT.
253 * GFX9 also supports Z16_UNORM.
254 * On VI, promote Z16 to Z32. DB->CB copies will convert
255 * the format for transfers.
256 */
257 if (rscreen->chip_class == VI)
258 bpe = 4;
259
260 flags |= RADEON_SURF_TC_COMPATIBLE_HTILE;
261 }
262
263 if (is_stencil)
264 flags |= RADEON_SURF_SBUFFER;
265 }
266
267 if (rscreen->chip_class >= VI &&
268 (ptex->flags & R600_RESOURCE_FLAG_DISABLE_DCC ||
269 ptex->format == PIPE_FORMAT_R9G9B9E5_FLOAT))
270 flags |= RADEON_SURF_DISABLE_DCC;
271
272 if (ptex->bind & PIPE_BIND_SCANOUT || is_scanout) {
273 /* This should catch bugs in gallium users setting incorrect flags. */
274 assert(ptex->nr_samples <= 1 &&
275 ptex->array_size == 1 &&
276 ptex->depth0 == 1 &&
277 ptex->last_level == 0 &&
278 !(flags & RADEON_SURF_Z_OR_SBUFFER));
279
280 flags |= RADEON_SURF_SCANOUT;
281 }
282
283 if (is_imported)
284 flags |= RADEON_SURF_IMPORTED;
285 if (!(ptex->flags & R600_RESOURCE_FLAG_FORCE_TILING))
286 flags |= RADEON_SURF_OPTIMIZE_FOR_SPACE;
287
288 r = rscreen->ws->surface_init(rscreen->ws, ptex, flags, bpe,
289 array_mode, surface);
290 if (r) {
291 return r;
292 }
293
294 if (rscreen->chip_class >= GFX9) {
295 assert(!pitch_in_bytes_override ||
296 pitch_in_bytes_override == surface->u.gfx9.surf_pitch * bpe);
297 surface->u.gfx9.surf_offset = offset;
298 } else {
299 if (pitch_in_bytes_override &&
300 pitch_in_bytes_override != surface->u.legacy.level[0].nblk_x * bpe) {
301 /* old ddx on evergreen over estimate alignment for 1d, only 1 level
302 * for those
303 */
304 surface->u.legacy.level[0].nblk_x = pitch_in_bytes_override / bpe;
305 surface->u.legacy.level[0].slice_size = pitch_in_bytes_override *
306 surface->u.legacy.level[0].nblk_y;
307 }
308
309 if (offset) {
310 for (i = 0; i < ARRAY_SIZE(surface->u.legacy.level); ++i)
311 surface->u.legacy.level[i].offset += offset;
312 }
313 }
314 return 0;
315 }
316
317 static void r600_texture_init_metadata(struct r600_common_screen *rscreen,
318 struct r600_texture *rtex,
319 struct radeon_bo_metadata *metadata)
320 {
321 struct radeon_surf *surface = &rtex->surface;
322
323 memset(metadata, 0, sizeof(*metadata));
324
325 if (rscreen->chip_class >= GFX9) {
326 metadata->u.gfx9.swizzle_mode = surface->u.gfx9.surf.swizzle_mode;
327 } else {
328 metadata->u.legacy.microtile = surface->u.legacy.level[0].mode >= RADEON_SURF_MODE_1D ?
329 RADEON_LAYOUT_TILED : RADEON_LAYOUT_LINEAR;
330 metadata->u.legacy.macrotile = surface->u.legacy.level[0].mode >= RADEON_SURF_MODE_2D ?
331 RADEON_LAYOUT_TILED : RADEON_LAYOUT_LINEAR;
332 metadata->u.legacy.pipe_config = surface->u.legacy.pipe_config;
333 metadata->u.legacy.bankw = surface->u.legacy.bankw;
334 metadata->u.legacy.bankh = surface->u.legacy.bankh;
335 metadata->u.legacy.tile_split = surface->u.legacy.tile_split;
336 metadata->u.legacy.mtilea = surface->u.legacy.mtilea;
337 metadata->u.legacy.num_banks = surface->u.legacy.num_banks;
338 metadata->u.legacy.stride = surface->u.legacy.level[0].nblk_x * surface->bpe;
339 metadata->u.legacy.scanout = (surface->flags & RADEON_SURF_SCANOUT) != 0;
340 }
341 }
342
343 static void r600_eliminate_fast_color_clear(struct r600_common_context *rctx,
344 struct r600_texture *rtex)
345 {
346 struct r600_common_screen *rscreen = rctx->screen;
347 struct pipe_context *ctx = &rctx->b;
348
349 if (ctx == rscreen->aux_context)
350 mtx_lock(&rscreen->aux_context_lock);
351
352 ctx->flush_resource(ctx, &rtex->resource.b.b);
353 ctx->flush(ctx, NULL, 0);
354
355 if (ctx == rscreen->aux_context)
356 mtx_unlock(&rscreen->aux_context_lock);
357 }
358
359 static void r600_texture_discard_cmask(struct r600_common_screen *rscreen,
360 struct r600_texture *rtex)
361 {
362 if (!rtex->cmask.size)
363 return;
364
365 assert(rtex->resource.b.b.nr_samples <= 1);
366
367 /* Disable CMASK. */
368 memset(&rtex->cmask, 0, sizeof(rtex->cmask));
369 rtex->cmask.base_address_reg = rtex->resource.gpu_address >> 8;
370 rtex->dirty_level_mask = 0;
371
372 if (rscreen->chip_class >= SI)
373 rtex->cb_color_info &= ~SI_S_028C70_FAST_CLEAR(1);
374 else
375 rtex->cb_color_info &= ~EG_S_028C70_FAST_CLEAR(1);
376
377 if (rtex->cmask_buffer != &rtex->resource)
378 r600_resource_reference(&rtex->cmask_buffer, NULL);
379
380 /* Notify all contexts about the change. */
381 p_atomic_inc(&rscreen->dirty_tex_counter);
382 p_atomic_inc(&rscreen->compressed_colortex_counter);
383 }
384
385 static bool r600_can_disable_dcc(struct r600_texture *rtex)
386 {
387 /* We can't disable DCC if it can be written by another process. */
388 return rtex->dcc_offset &&
389 (!rtex->resource.b.is_shared ||
390 !(rtex->resource.external_usage & PIPE_HANDLE_USAGE_WRITE));
391 }
392
393 static bool r600_texture_discard_dcc(struct r600_common_screen *rscreen,
394 struct r600_texture *rtex)
395 {
396 if (!r600_can_disable_dcc(rtex))
397 return false;
398
399 assert(rtex->dcc_separate_buffer == NULL);
400
401 /* Disable DCC. */
402 rtex->dcc_offset = 0;
403
404 /* Notify all contexts about the change. */
405 p_atomic_inc(&rscreen->dirty_tex_counter);
406 return true;
407 }
408
409 /**
410 * Disable DCC for the texture. (first decompress, then discard metadata).
411 *
412 * There is unresolved multi-context synchronization issue between
413 * screen::aux_context and the current context. If applications do this with
414 * multiple contexts, it's already undefined behavior for them and we don't
415 * have to worry about that. The scenario is:
416 *
417 * If context 1 disables DCC and context 2 has queued commands that write
418 * to the texture via CB with DCC enabled, and the order of operations is
419 * as follows:
420 * context 2 queues draw calls rendering to the texture, but doesn't flush
421 * context 1 disables DCC and flushes
422 * context 1 & 2 reset descriptors and FB state
423 * context 2 flushes (new compressed tiles written by the draw calls)
424 * context 1 & 2 read garbage, because DCC is disabled, yet there are
425 * compressed tiled
426 *
427 * \param rctx the current context if you have one, or rscreen->aux_context
428 * if you don't.
429 */
430 bool r600_texture_disable_dcc(struct r600_common_context *rctx,
431 struct r600_texture *rtex)
432 {
433 struct r600_common_screen *rscreen = rctx->screen;
434
435 if (!r600_can_disable_dcc(rtex))
436 return false;
437
438 if (&rctx->b == rscreen->aux_context)
439 mtx_lock(&rscreen->aux_context_lock);
440
441 /* Decompress DCC. */
442 rctx->decompress_dcc(&rctx->b, rtex);
443 rctx->b.flush(&rctx->b, NULL, 0);
444
445 if (&rctx->b == rscreen->aux_context)
446 mtx_unlock(&rscreen->aux_context_lock);
447
448 return r600_texture_discard_dcc(rscreen, rtex);
449 }
450
451 static void r600_reallocate_texture_inplace(struct r600_common_context *rctx,
452 struct r600_texture *rtex,
453 unsigned new_bind_flag,
454 bool invalidate_storage)
455 {
456 struct pipe_screen *screen = rctx->b.screen;
457 struct r600_texture *new_tex;
458 struct pipe_resource templ = rtex->resource.b.b;
459 unsigned i;
460
461 templ.bind |= new_bind_flag;
462
463 /* r600g doesn't react to dirty_tex_descriptor_counter */
464 if (rctx->chip_class < SI)
465 return;
466
467 if (rtex->resource.b.is_shared)
468 return;
469
470 if (new_bind_flag == PIPE_BIND_LINEAR) {
471 if (rtex->surface.is_linear)
472 return;
473
474 /* This fails with MSAA, depth, and compressed textures. */
475 if (r600_choose_tiling(rctx->screen, &templ) !=
476 RADEON_SURF_MODE_LINEAR_ALIGNED)
477 return;
478 }
479
480 new_tex = (struct r600_texture*)screen->resource_create(screen, &templ);
481 if (!new_tex)
482 return;
483
484 /* Copy the pixels to the new texture. */
485 if (!invalidate_storage) {
486 for (i = 0; i <= templ.last_level; i++) {
487 struct pipe_box box;
488
489 u_box_3d(0, 0, 0,
490 u_minify(templ.width0, i), u_minify(templ.height0, i),
491 util_max_layer(&templ, i) + 1, &box);
492
493 rctx->dma_copy(&rctx->b, &new_tex->resource.b.b, i, 0, 0, 0,
494 &rtex->resource.b.b, i, &box);
495 }
496 }
497
498 if (new_bind_flag == PIPE_BIND_LINEAR) {
499 r600_texture_discard_cmask(rctx->screen, rtex);
500 r600_texture_discard_dcc(rctx->screen, rtex);
501 }
502
503 /* Replace the structure fields of rtex. */
504 rtex->resource.b.b.bind = templ.bind;
505 pb_reference(&rtex->resource.buf, new_tex->resource.buf);
506 rtex->resource.gpu_address = new_tex->resource.gpu_address;
507 rtex->resource.vram_usage = new_tex->resource.vram_usage;
508 rtex->resource.gart_usage = new_tex->resource.gart_usage;
509 rtex->resource.bo_size = new_tex->resource.bo_size;
510 rtex->resource.bo_alignment = new_tex->resource.bo_alignment;
511 rtex->resource.domains = new_tex->resource.domains;
512 rtex->resource.flags = new_tex->resource.flags;
513 rtex->size = new_tex->size;
514 rtex->db_render_format = new_tex->db_render_format;
515 rtex->db_compatible = new_tex->db_compatible;
516 rtex->can_sample_z = new_tex->can_sample_z;
517 rtex->can_sample_s = new_tex->can_sample_s;
518 rtex->surface = new_tex->surface;
519 rtex->fmask = new_tex->fmask;
520 rtex->cmask = new_tex->cmask;
521 rtex->cb_color_info = new_tex->cb_color_info;
522 rtex->last_msaa_resolve_target_micro_mode = new_tex->last_msaa_resolve_target_micro_mode;
523 rtex->htile_offset = new_tex->htile_offset;
524 rtex->tc_compatible_htile = new_tex->tc_compatible_htile;
525 rtex->depth_cleared = new_tex->depth_cleared;
526 rtex->stencil_cleared = new_tex->stencil_cleared;
527 rtex->non_disp_tiling = new_tex->non_disp_tiling;
528 rtex->dcc_gather_statistics = new_tex->dcc_gather_statistics;
529 rtex->framebuffers_bound = new_tex->framebuffers_bound;
530
531 if (new_bind_flag == PIPE_BIND_LINEAR) {
532 assert(!rtex->htile_offset);
533 assert(!rtex->cmask.size);
534 assert(!rtex->fmask.size);
535 assert(!rtex->dcc_offset);
536 assert(!rtex->is_depth);
537 }
538
539 r600_texture_reference(&new_tex, NULL);
540
541 p_atomic_inc(&rctx->screen->dirty_tex_counter);
542 }
543
544 static boolean r600_texture_get_handle(struct pipe_screen* screen,
545 struct pipe_context *ctx,
546 struct pipe_resource *resource,
547 struct winsys_handle *whandle,
548 unsigned usage)
549 {
550 struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
551 struct r600_common_context *rctx;
552 struct r600_resource *res = (struct r600_resource*)resource;
553 struct r600_texture *rtex = (struct r600_texture*)resource;
554 struct radeon_bo_metadata metadata;
555 bool update_metadata = false;
556 unsigned stride, offset, slice_size;
557
558 ctx = threaded_context_unwrap_sync(ctx);
559 rctx = (struct r600_common_context*)(ctx ? ctx : rscreen->aux_context);
560
561 if (resource->target != PIPE_BUFFER) {
562 /* This is not supported now, but it might be required for OpenCL
563 * interop in the future.
564 */
565 if (resource->nr_samples > 1 || rtex->is_depth)
566 return false;
567
568 /* Move a suballocated texture into a non-suballocated allocation. */
569 if (rscreen->ws->buffer_is_suballocated(res->buf)) {
570 assert(!res->b.is_shared);
571 r600_reallocate_texture_inplace(rctx, rtex,
572 PIPE_BIND_SHARED, false);
573 rctx->b.flush(&rctx->b, NULL, 0);
574 assert(res->b.b.bind & PIPE_BIND_SHARED);
575 assert(res->flags & RADEON_FLAG_NO_SUBALLOC);
576 }
577
578 /* Since shader image stores don't support DCC on VI,
579 * disable it for external clients that want write
580 * access.
581 */
582 if (usage & PIPE_HANDLE_USAGE_WRITE && rtex->dcc_offset) {
583 if (r600_texture_disable_dcc(rctx, rtex))
584 update_metadata = true;
585 }
586
587 if (!(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH) &&
588 (rtex->cmask.size || rtex->dcc_offset)) {
589 /* Eliminate fast clear (both CMASK and DCC) */
590 r600_eliminate_fast_color_clear(rctx, rtex);
591
592 /* Disable CMASK if flush_resource isn't going
593 * to be called.
594 */
595 if (rtex->cmask.size)
596 r600_texture_discard_cmask(rscreen, rtex);
597 }
598
599 /* Set metadata. */
600 if (!res->b.is_shared || update_metadata) {
601 r600_texture_init_metadata(rscreen, rtex, &metadata);
602 if (rscreen->query_opaque_metadata)
603 rscreen->query_opaque_metadata(rscreen, rtex,
604 &metadata);
605
606 rscreen->ws->buffer_set_metadata(res->buf, &metadata);
607 }
608
609 if (rscreen->chip_class >= GFX9) {
610 offset = rtex->surface.u.gfx9.surf_offset;
611 stride = rtex->surface.u.gfx9.surf_pitch *
612 rtex->surface.bpe;
613 slice_size = rtex->surface.u.gfx9.surf_slice_size;
614 } else {
615 offset = rtex->surface.u.legacy.level[0].offset;
616 stride = rtex->surface.u.legacy.level[0].nblk_x *
617 rtex->surface.bpe;
618 slice_size = rtex->surface.u.legacy.level[0].slice_size;
619 }
620 } else {
621 /* Move a suballocated buffer into a non-suballocated allocation. */
622 if (rscreen->ws->buffer_is_suballocated(res->buf)) {
623 assert(!res->b.is_shared);
624
625 /* Allocate a new buffer with PIPE_BIND_SHARED. */
626 struct pipe_resource templ = res->b.b;
627 templ.bind |= PIPE_BIND_SHARED;
628
629 struct pipe_resource *newb =
630 screen->resource_create(screen, &templ);
631 if (!newb)
632 return false;
633
634 /* Copy the old buffer contents to the new one. */
635 struct pipe_box box;
636 u_box_1d(0, newb->width0, &box);
637 rctx->b.resource_copy_region(&rctx->b, newb, 0, 0, 0, 0,
638 &res->b.b, 0, &box);
639 /* Move the new buffer storage to the old pipe_resource. */
640 r600_replace_buffer_storage(&rctx->b, &res->b.b, newb);
641 pipe_resource_reference(&newb, NULL);
642
643 assert(res->b.b.bind & PIPE_BIND_SHARED);
644 assert(res->flags & RADEON_FLAG_NO_SUBALLOC);
645 }
646
647 /* Buffers */
648 offset = 0;
649 stride = 0;
650 slice_size = 0;
651 }
652
653 if (res->b.is_shared) {
654 /* USAGE_EXPLICIT_FLUSH must be cleared if at least one user
655 * doesn't set it.
656 */
657 res->external_usage |= usage & ~PIPE_HANDLE_USAGE_EXPLICIT_FLUSH;
658 if (!(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH))
659 res->external_usage &= ~PIPE_HANDLE_USAGE_EXPLICIT_FLUSH;
660 } else {
661 res->b.is_shared = true;
662 res->external_usage = usage;
663 }
664
665 return rscreen->ws->buffer_get_handle(res->buf, stride, offset,
666 slice_size, whandle);
667 }
668
669 static void r600_texture_destroy(struct pipe_screen *screen,
670 struct pipe_resource *ptex)
671 {
672 struct r600_texture *rtex = (struct r600_texture*)ptex;
673 struct r600_resource *resource = &rtex->resource;
674
675 r600_texture_reference(&rtex->flushed_depth_texture, NULL);
676
677 if (rtex->cmask_buffer != &rtex->resource) {
678 r600_resource_reference(&rtex->cmask_buffer, NULL);
679 }
680 pb_reference(&resource->buf, NULL);
681 r600_resource_reference(&rtex->dcc_separate_buffer, NULL);
682 r600_resource_reference(&rtex->last_dcc_separate_buffer, NULL);
683 FREE(rtex);
684 }
685
686 static const struct u_resource_vtbl r600_texture_vtbl;
687
688 /* The number of samples can be specified independently of the texture. */
689 void r600_texture_get_fmask_info(struct r600_common_screen *rscreen,
690 struct r600_texture *rtex,
691 unsigned nr_samples,
692 struct r600_fmask_info *out)
693 {
694 /* FMASK is allocated like an ordinary texture. */
695 struct pipe_resource templ = rtex->resource.b.b;
696 struct radeon_surf fmask = {};
697 unsigned flags, bpe;
698
699 memset(out, 0, sizeof(*out));
700
701 if (rscreen->chip_class >= GFX9) {
702 out->alignment = rtex->surface.u.gfx9.fmask_alignment;
703 out->size = rtex->surface.u.gfx9.fmask_size;
704 return;
705 }
706
707 templ.nr_samples = 1;
708 flags = rtex->surface.flags | RADEON_SURF_FMASK;
709
710 if (rscreen->chip_class <= CAYMAN) {
711 /* Use the same parameters and tile mode. */
712 fmask.u.legacy.bankw = rtex->surface.u.legacy.bankw;
713 fmask.u.legacy.bankh = rtex->surface.u.legacy.bankh;
714 fmask.u.legacy.mtilea = rtex->surface.u.legacy.mtilea;
715 fmask.u.legacy.tile_split = rtex->surface.u.legacy.tile_split;
716
717 if (nr_samples <= 4)
718 fmask.u.legacy.bankh = 4;
719 }
720
721 switch (nr_samples) {
722 case 2:
723 case 4:
724 bpe = 1;
725 break;
726 case 8:
727 bpe = 4;
728 break;
729 default:
730 R600_ERR("Invalid sample count for FMASK allocation.\n");
731 return;
732 }
733
734 /* Overallocate FMASK on R600-R700 to fix colorbuffer corruption.
735 * This can be fixed by writing a separate FMASK allocator specifically
736 * for R600-R700 asics. */
737 if (rscreen->chip_class <= R700) {
738 bpe *= 2;
739 }
740
741 if (rscreen->ws->surface_init(rscreen->ws, &templ, flags, bpe,
742 RADEON_SURF_MODE_2D, &fmask)) {
743 R600_ERR("Got error in surface_init while allocating FMASK.\n");
744 return;
745 }
746
747 assert(fmask.u.legacy.level[0].mode == RADEON_SURF_MODE_2D);
748
749 out->slice_tile_max = (fmask.u.legacy.level[0].nblk_x * fmask.u.legacy.level[0].nblk_y) / 64;
750 if (out->slice_tile_max)
751 out->slice_tile_max -= 1;
752
753 out->tile_mode_index = fmask.u.legacy.tiling_index[0];
754 out->pitch_in_pixels = fmask.u.legacy.level[0].nblk_x;
755 out->bank_height = fmask.u.legacy.bankh;
756 out->alignment = MAX2(256, fmask.surf_alignment);
757 out->size = fmask.surf_size;
758 }
759
760 static void r600_texture_allocate_fmask(struct r600_common_screen *rscreen,
761 struct r600_texture *rtex)
762 {
763 r600_texture_get_fmask_info(rscreen, rtex,
764 rtex->resource.b.b.nr_samples, &rtex->fmask);
765
766 rtex->fmask.offset = align64(rtex->size, rtex->fmask.alignment);
767 rtex->size = rtex->fmask.offset + rtex->fmask.size;
768 }
769
770 void r600_texture_get_cmask_info(struct r600_common_screen *rscreen,
771 struct r600_texture *rtex,
772 struct r600_cmask_info *out)
773 {
774 unsigned cmask_tile_width = 8;
775 unsigned cmask_tile_height = 8;
776 unsigned cmask_tile_elements = cmask_tile_width * cmask_tile_height;
777 unsigned element_bits = 4;
778 unsigned cmask_cache_bits = 1024;
779 unsigned num_pipes = rscreen->info.num_tile_pipes;
780 unsigned pipe_interleave_bytes = rscreen->info.pipe_interleave_bytes;
781
782 unsigned elements_per_macro_tile = (cmask_cache_bits / element_bits) * num_pipes;
783 unsigned pixels_per_macro_tile = elements_per_macro_tile * cmask_tile_elements;
784 unsigned sqrt_pixels_per_macro_tile = sqrt(pixels_per_macro_tile);
785 unsigned macro_tile_width = util_next_power_of_two(sqrt_pixels_per_macro_tile);
786 unsigned macro_tile_height = pixels_per_macro_tile / macro_tile_width;
787
788 unsigned pitch_elements = align(rtex->resource.b.b.width0, macro_tile_width);
789 unsigned height = align(rtex->resource.b.b.height0, macro_tile_height);
790
791 unsigned base_align = num_pipes * pipe_interleave_bytes;
792 unsigned slice_bytes =
793 ((pitch_elements * height * element_bits + 7) / 8) / cmask_tile_elements;
794
795 assert(macro_tile_width % 128 == 0);
796 assert(macro_tile_height % 128 == 0);
797
798 out->slice_tile_max = ((pitch_elements * height) / (128*128)) - 1;
799 out->alignment = MAX2(256, base_align);
800 out->size = (util_max_layer(&rtex->resource.b.b, 0) + 1) *
801 align(slice_bytes, base_align);
802 }
803
804 static void si_texture_get_cmask_info(struct r600_common_screen *rscreen,
805 struct r600_texture *rtex,
806 struct r600_cmask_info *out)
807 {
808 unsigned pipe_interleave_bytes = rscreen->info.pipe_interleave_bytes;
809 unsigned num_pipes = rscreen->info.num_tile_pipes;
810 unsigned cl_width, cl_height;
811
812 if (rscreen->chip_class >= GFX9) {
813 out->alignment = rtex->surface.u.gfx9.cmask_alignment;
814 out->size = rtex->surface.u.gfx9.cmask_size;
815 return;
816 }
817
818 switch (num_pipes) {
819 case 2:
820 cl_width = 32;
821 cl_height = 16;
822 break;
823 case 4:
824 cl_width = 32;
825 cl_height = 32;
826 break;
827 case 8:
828 cl_width = 64;
829 cl_height = 32;
830 break;
831 case 16: /* Hawaii */
832 cl_width = 64;
833 cl_height = 64;
834 break;
835 default:
836 assert(0);
837 return;
838 }
839
840 unsigned base_align = num_pipes * pipe_interleave_bytes;
841
842 unsigned width = align(rtex->resource.b.b.width0, cl_width*8);
843 unsigned height = align(rtex->resource.b.b.height0, cl_height*8);
844 unsigned slice_elements = (width * height) / (8*8);
845
846 /* Each element of CMASK is a nibble. */
847 unsigned slice_bytes = slice_elements / 2;
848
849 out->slice_tile_max = (width * height) / (128*128);
850 if (out->slice_tile_max)
851 out->slice_tile_max -= 1;
852
853 out->alignment = MAX2(256, base_align);
854 out->size = (util_max_layer(&rtex->resource.b.b, 0) + 1) *
855 align(slice_bytes, base_align);
856 }
857
858 static void r600_texture_allocate_cmask(struct r600_common_screen *rscreen,
859 struct r600_texture *rtex)
860 {
861 if (rscreen->chip_class >= SI) {
862 si_texture_get_cmask_info(rscreen, rtex, &rtex->cmask);
863 } else {
864 r600_texture_get_cmask_info(rscreen, rtex, &rtex->cmask);
865 }
866
867 rtex->cmask.offset = align64(rtex->size, rtex->cmask.alignment);
868 rtex->size = rtex->cmask.offset + rtex->cmask.size;
869
870 if (rscreen->chip_class >= SI)
871 rtex->cb_color_info |= SI_S_028C70_FAST_CLEAR(1);
872 else
873 rtex->cb_color_info |= EG_S_028C70_FAST_CLEAR(1);
874 }
875
876 static void r600_texture_alloc_cmask_separate(struct r600_common_screen *rscreen,
877 struct r600_texture *rtex)
878 {
879 if (rtex->cmask_buffer)
880 return;
881
882 assert(rtex->cmask.size == 0);
883
884 if (rscreen->chip_class >= SI) {
885 si_texture_get_cmask_info(rscreen, rtex, &rtex->cmask);
886 } else {
887 r600_texture_get_cmask_info(rscreen, rtex, &rtex->cmask);
888 }
889
890 rtex->cmask_buffer = (struct r600_resource *)
891 r600_aligned_buffer_create(&rscreen->b,
892 R600_RESOURCE_FLAG_UNMAPPABLE,
893 PIPE_USAGE_DEFAULT,
894 rtex->cmask.size,
895 rtex->cmask.alignment);
896 if (rtex->cmask_buffer == NULL) {
897 rtex->cmask.size = 0;
898 return;
899 }
900
901 /* update colorbuffer state bits */
902 rtex->cmask.base_address_reg = rtex->cmask_buffer->gpu_address >> 8;
903
904 if (rscreen->chip_class >= SI)
905 rtex->cb_color_info |= SI_S_028C70_FAST_CLEAR(1);
906 else
907 rtex->cb_color_info |= EG_S_028C70_FAST_CLEAR(1);
908
909 p_atomic_inc(&rscreen->compressed_colortex_counter);
910 }
911
912 static void r600_texture_get_htile_size(struct r600_common_screen *rscreen,
913 struct r600_texture *rtex)
914 {
915 unsigned cl_width, cl_height, width, height;
916 unsigned slice_elements, slice_bytes, pipe_interleave_bytes, base_align;
917 unsigned num_pipes = rscreen->info.num_tile_pipes;
918
919 assert(rscreen->chip_class <= VI);
920
921 rtex->surface.htile_size = 0;
922
923 if (rscreen->chip_class <= EVERGREEN &&
924 rscreen->info.drm_major == 2 && rscreen->info.drm_minor < 26)
925 return;
926
927 /* HW bug on R6xx. */
928 if (rscreen->chip_class == R600 &&
929 (rtex->resource.b.b.width0 > 7680 ||
930 rtex->resource.b.b.height0 > 7680))
931 return;
932
933 /* HTILE is broken with 1D tiling on old kernels and CIK. */
934 if (rscreen->chip_class >= CIK &&
935 rtex->surface.u.legacy.level[0].mode == RADEON_SURF_MODE_1D &&
936 rscreen->info.drm_major == 2 && rscreen->info.drm_minor < 38)
937 return;
938
939 /* Overalign HTILE on P2 configs to work around GPU hangs in
940 * piglit/depthstencil-render-miplevels 585.
941 *
942 * This has been confirmed to help Kabini & Stoney, where the hangs
943 * are always reproducible. I think I have seen the test hang
944 * on Carrizo too, though it was very rare there.
945 */
946 if (rscreen->chip_class >= CIK && num_pipes < 4)
947 num_pipes = 4;
948
949 switch (num_pipes) {
950 case 1:
951 cl_width = 32;
952 cl_height = 16;
953 break;
954 case 2:
955 cl_width = 32;
956 cl_height = 32;
957 break;
958 case 4:
959 cl_width = 64;
960 cl_height = 32;
961 break;
962 case 8:
963 cl_width = 64;
964 cl_height = 64;
965 break;
966 case 16:
967 cl_width = 128;
968 cl_height = 64;
969 break;
970 default:
971 assert(0);
972 return;
973 }
974
975 width = align(rtex->resource.b.b.width0, cl_width * 8);
976 height = align(rtex->resource.b.b.height0, cl_height * 8);
977
978 slice_elements = (width * height) / (8 * 8);
979 slice_bytes = slice_elements * 4;
980
981 pipe_interleave_bytes = rscreen->info.pipe_interleave_bytes;
982 base_align = num_pipes * pipe_interleave_bytes;
983
984 rtex->surface.htile_alignment = base_align;
985 rtex->surface.htile_size =
986 (util_max_layer(&rtex->resource.b.b, 0) + 1) *
987 align(slice_bytes, base_align);
988 }
989
990 static void r600_texture_allocate_htile(struct r600_common_screen *rscreen,
991 struct r600_texture *rtex)
992 {
993 if (rscreen->chip_class <= VI && !rtex->tc_compatible_htile)
994 r600_texture_get_htile_size(rscreen, rtex);
995
996 if (!rtex->surface.htile_size)
997 return;
998
999 rtex->htile_offset = align(rtex->size, rtex->surface.htile_alignment);
1000 rtex->size = rtex->htile_offset + rtex->surface.htile_size;
1001 }
1002
1003 void r600_print_texture_info(struct r600_common_screen *rscreen,
1004 struct r600_texture *rtex, FILE *f)
1005 {
1006 int i;
1007
1008 /* Common parameters. */
1009 fprintf(f, " Info: npix_x=%u, npix_y=%u, npix_z=%u, blk_w=%u, "
1010 "blk_h=%u, array_size=%u, last_level=%u, "
1011 "bpe=%u, nsamples=%u, flags=0x%x, %s\n",
1012 rtex->resource.b.b.width0, rtex->resource.b.b.height0,
1013 rtex->resource.b.b.depth0, rtex->surface.blk_w,
1014 rtex->surface.blk_h,
1015 rtex->resource.b.b.array_size, rtex->resource.b.b.last_level,
1016 rtex->surface.bpe, rtex->resource.b.b.nr_samples,
1017 rtex->surface.flags, util_format_short_name(rtex->resource.b.b.format));
1018
1019 if (rscreen->chip_class >= GFX9) {
1020 fprintf(f, " Surf: size=%"PRIu64", slice_size=%"PRIu64", "
1021 "alignment=%u, swmode=%u, epitch=%u, pitch=%u\n",
1022 rtex->surface.surf_size,
1023 rtex->surface.u.gfx9.surf_slice_size,
1024 rtex->surface.surf_alignment,
1025 rtex->surface.u.gfx9.surf.swizzle_mode,
1026 rtex->surface.u.gfx9.surf.epitch,
1027 rtex->surface.u.gfx9.surf_pitch);
1028
1029 if (rtex->fmask.size) {
1030 fprintf(f, " FMASK: offset=%"PRIu64", size=%"PRIu64", "
1031 "alignment=%u, swmode=%u, epitch=%u\n",
1032 rtex->fmask.offset,
1033 rtex->surface.u.gfx9.fmask_size,
1034 rtex->surface.u.gfx9.fmask_alignment,
1035 rtex->surface.u.gfx9.fmask.swizzle_mode,
1036 rtex->surface.u.gfx9.fmask.epitch);
1037 }
1038
1039 if (rtex->cmask.size) {
1040 fprintf(f, " CMask: offset=%"PRIu64", size=%"PRIu64", "
1041 "alignment=%u, rb_aligned=%u, pipe_aligned=%u\n",
1042 rtex->cmask.offset,
1043 rtex->surface.u.gfx9.cmask_size,
1044 rtex->surface.u.gfx9.cmask_alignment,
1045 rtex->surface.u.gfx9.cmask.rb_aligned,
1046 rtex->surface.u.gfx9.cmask.pipe_aligned);
1047 }
1048
1049 if (rtex->htile_offset) {
1050 fprintf(f, " HTile: offset=%"PRIu64", size=%"PRIu64", alignment=%u, "
1051 "rb_aligned=%u, pipe_aligned=%u\n",
1052 rtex->htile_offset,
1053 rtex->surface.htile_size,
1054 rtex->surface.htile_alignment,
1055 rtex->surface.u.gfx9.htile.rb_aligned,
1056 rtex->surface.u.gfx9.htile.pipe_aligned);
1057 }
1058
1059 if (rtex->dcc_offset) {
1060 fprintf(f, " DCC: offset=%"PRIu64", size=%"PRIu64", "
1061 "alignment=%u, pitch_max=%u, num_dcc_levels=%u\n",
1062 rtex->dcc_offset, rtex->surface.dcc_size,
1063 rtex->surface.dcc_alignment,
1064 rtex->surface.u.gfx9.dcc_pitch_max,
1065 rtex->surface.num_dcc_levels);
1066 }
1067
1068 if (rtex->surface.u.gfx9.stencil_offset) {
1069 fprintf(f, " Stencil: offset=%"PRIu64", swmode=%u, epitch=%u\n",
1070 rtex->surface.u.gfx9.stencil_offset,
1071 rtex->surface.u.gfx9.stencil.swizzle_mode,
1072 rtex->surface.u.gfx9.stencil.epitch);
1073 }
1074 return;
1075 }
1076
1077 fprintf(f, " Layout: size=%"PRIu64", alignment=%u, bankw=%u, "
1078 "bankh=%u, nbanks=%u, mtilea=%u, tilesplit=%u, pipeconfig=%u, scanout=%u\n",
1079 rtex->surface.surf_size, rtex->surface.surf_alignment, rtex->surface.u.legacy.bankw,
1080 rtex->surface.u.legacy.bankh, rtex->surface.u.legacy.num_banks, rtex->surface.u.legacy.mtilea,
1081 rtex->surface.u.legacy.tile_split, rtex->surface.u.legacy.pipe_config,
1082 (rtex->surface.flags & RADEON_SURF_SCANOUT) != 0);
1083
1084 if (rtex->fmask.size)
1085 fprintf(f, " FMask: offset=%"PRIu64", size=%"PRIu64", alignment=%u, pitch_in_pixels=%u, "
1086 "bankh=%u, slice_tile_max=%u, tile_mode_index=%u\n",
1087 rtex->fmask.offset, rtex->fmask.size, rtex->fmask.alignment,
1088 rtex->fmask.pitch_in_pixels, rtex->fmask.bank_height,
1089 rtex->fmask.slice_tile_max, rtex->fmask.tile_mode_index);
1090
1091 if (rtex->cmask.size)
1092 fprintf(f, " CMask: offset=%"PRIu64", size=%"PRIu64", alignment=%u, "
1093 "slice_tile_max=%u\n",
1094 rtex->cmask.offset, rtex->cmask.size, rtex->cmask.alignment,
1095 rtex->cmask.slice_tile_max);
1096
1097 if (rtex->htile_offset)
1098 fprintf(f, " HTile: offset=%"PRIu64", size=%"PRIu64", "
1099 "alignment=%u, TC_compatible = %u\n",
1100 rtex->htile_offset, rtex->surface.htile_size,
1101 rtex->surface.htile_alignment,
1102 rtex->tc_compatible_htile);
1103
1104 if (rtex->dcc_offset) {
1105 fprintf(f, " DCC: offset=%"PRIu64", size=%"PRIu64", alignment=%u\n",
1106 rtex->dcc_offset, rtex->surface.dcc_size,
1107 rtex->surface.dcc_alignment);
1108 for (i = 0; i <= rtex->resource.b.b.last_level; i++)
1109 fprintf(f, " DCCLevel[%i]: enabled=%u, offset=%"PRIu64", "
1110 "fast_clear_size=%"PRIu64"\n",
1111 i, i < rtex->surface.num_dcc_levels,
1112 rtex->surface.u.legacy.level[i].dcc_offset,
1113 rtex->surface.u.legacy.level[i].dcc_fast_clear_size);
1114 }
1115
1116 for (i = 0; i <= rtex->resource.b.b.last_level; i++)
1117 fprintf(f, " Level[%i]: offset=%"PRIu64", slice_size=%"PRIu64", "
1118 "npix_x=%u, npix_y=%u, npix_z=%u, nblk_x=%u, nblk_y=%u, "
1119 "mode=%u, tiling_index = %u\n",
1120 i, rtex->surface.u.legacy.level[i].offset,
1121 rtex->surface.u.legacy.level[i].slice_size,
1122 u_minify(rtex->resource.b.b.width0, i),
1123 u_minify(rtex->resource.b.b.height0, i),
1124 u_minify(rtex->resource.b.b.depth0, i),
1125 rtex->surface.u.legacy.level[i].nblk_x,
1126 rtex->surface.u.legacy.level[i].nblk_y,
1127 rtex->surface.u.legacy.level[i].mode,
1128 rtex->surface.u.legacy.tiling_index[i]);
1129
1130 if (rtex->surface.flags & RADEON_SURF_SBUFFER) {
1131 fprintf(f, " StencilLayout: tilesplit=%u\n",
1132 rtex->surface.u.legacy.stencil_tile_split);
1133 for (i = 0; i <= rtex->resource.b.b.last_level; i++) {
1134 fprintf(f, " StencilLevel[%i]: offset=%"PRIu64", "
1135 "slice_size=%"PRIu64", npix_x=%u, "
1136 "npix_y=%u, npix_z=%u, nblk_x=%u, nblk_y=%u, "
1137 "mode=%u, tiling_index = %u\n",
1138 i, rtex->surface.u.legacy.stencil_level[i].offset,
1139 rtex->surface.u.legacy.stencil_level[i].slice_size,
1140 u_minify(rtex->resource.b.b.width0, i),
1141 u_minify(rtex->resource.b.b.height0, i),
1142 u_minify(rtex->resource.b.b.depth0, i),
1143 rtex->surface.u.legacy.stencil_level[i].nblk_x,
1144 rtex->surface.u.legacy.stencil_level[i].nblk_y,
1145 rtex->surface.u.legacy.stencil_level[i].mode,
1146 rtex->surface.u.legacy.stencil_tiling_index[i]);
1147 }
1148 }
1149 }
1150
1151 /* Common processing for r600_texture_create and r600_texture_from_handle */
1152 static struct r600_texture *
1153 r600_texture_create_object(struct pipe_screen *screen,
1154 const struct pipe_resource *base,
1155 struct pb_buffer *buf,
1156 struct radeon_surf *surface)
1157 {
1158 struct r600_texture *rtex;
1159 struct r600_resource *resource;
1160 struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
1161
1162 rtex = CALLOC_STRUCT(r600_texture);
1163 if (!rtex)
1164 return NULL;
1165
1166 resource = &rtex->resource;
1167 resource->b.b = *base;
1168 resource->b.b.next = NULL;
1169 resource->b.vtbl = &r600_texture_vtbl;
1170 pipe_reference_init(&resource->b.b.reference, 1);
1171 resource->b.b.screen = screen;
1172
1173 /* don't include stencil-only formats which we don't support for rendering */
1174 rtex->is_depth = util_format_has_depth(util_format_description(rtex->resource.b.b.format));
1175
1176 rtex->surface = *surface;
1177 rtex->size = rtex->surface.surf_size;
1178
1179 rtex->tc_compatible_htile = rtex->surface.htile_size != 0 &&
1180 (rtex->surface.flags &
1181 RADEON_SURF_TC_COMPATIBLE_HTILE);
1182
1183 /* TC-compatible HTILE:
1184 * - VI only supports Z32_FLOAT.
1185 * - GFX9 only supports Z32_FLOAT and Z16_UNORM. */
1186 if (rtex->tc_compatible_htile) {
1187 if (rscreen->chip_class >= GFX9 &&
1188 base->format == PIPE_FORMAT_Z16_UNORM)
1189 rtex->db_render_format = base->format;
1190 else
1191 rtex->db_render_format = PIPE_FORMAT_Z32_FLOAT;
1192 } else {
1193 rtex->db_render_format = base->format;
1194 }
1195
1196 /* Tiled depth textures utilize the non-displayable tile order.
1197 * This must be done after r600_setup_surface.
1198 * Applies to R600-Cayman. */
1199 rtex->non_disp_tiling = rtex->is_depth && rtex->surface.u.legacy.level[0].mode >= RADEON_SURF_MODE_1D;
1200 /* Applies to GCN. */
1201 rtex->last_msaa_resolve_target_micro_mode = rtex->surface.micro_tile_mode;
1202
1203 /* Disable separate DCC at the beginning. DRI2 doesn't reuse buffers
1204 * between frames, so the only thing that can enable separate DCC
1205 * with DRI2 is multiple slow clears within a frame.
1206 */
1207 rtex->ps_draw_ratio = 0;
1208
1209 if (rtex->is_depth) {
1210 if (base->flags & (R600_RESOURCE_FLAG_TRANSFER |
1211 R600_RESOURCE_FLAG_FLUSHED_DEPTH) ||
1212 rscreen->chip_class >= EVERGREEN) {
1213 if (rscreen->chip_class >= GFX9) {
1214 rtex->can_sample_z = true;
1215 rtex->can_sample_s = true;
1216 } else {
1217 rtex->can_sample_z = !rtex->surface.u.legacy.depth_adjusted;
1218 rtex->can_sample_s = !rtex->surface.u.legacy.stencil_adjusted;
1219 }
1220 } else {
1221 if (rtex->resource.b.b.nr_samples <= 1 &&
1222 (rtex->resource.b.b.format == PIPE_FORMAT_Z16_UNORM ||
1223 rtex->resource.b.b.format == PIPE_FORMAT_Z32_FLOAT))
1224 rtex->can_sample_z = true;
1225 }
1226
1227 if (!(base->flags & (R600_RESOURCE_FLAG_TRANSFER |
1228 R600_RESOURCE_FLAG_FLUSHED_DEPTH))) {
1229 rtex->db_compatible = true;
1230
1231 if (!(rscreen->debug_flags & DBG_NO_HYPERZ))
1232 r600_texture_allocate_htile(rscreen, rtex);
1233 }
1234 } else {
1235 if (base->nr_samples > 1) {
1236 if (!buf) {
1237 r600_texture_allocate_fmask(rscreen, rtex);
1238 r600_texture_allocate_cmask(rscreen, rtex);
1239 rtex->cmask_buffer = &rtex->resource;
1240 }
1241 if (!rtex->fmask.size || !rtex->cmask.size) {
1242 FREE(rtex);
1243 return NULL;
1244 }
1245 }
1246
1247 /* Shared textures must always set up DCC here.
1248 * If it's not present, it will be disabled by
1249 * apply_opaque_metadata later.
1250 */
1251 if (rtex->surface.dcc_size &&
1252 (buf || !(rscreen->debug_flags & DBG_NO_DCC)) &&
1253 !(rtex->surface.flags & RADEON_SURF_SCANOUT)) {
1254 /* Reserve space for the DCC buffer. */
1255 rtex->dcc_offset = align64(rtex->size, rtex->surface.dcc_alignment);
1256 rtex->size = rtex->dcc_offset + rtex->surface.dcc_size;
1257 }
1258 }
1259
1260 /* Now create the backing buffer. */
1261 if (!buf) {
1262 r600_init_resource_fields(rscreen, resource, rtex->size,
1263 rtex->surface.surf_alignment);
1264
1265 /* Displayable surfaces are not suballocated. */
1266 if (resource->b.b.bind & PIPE_BIND_SCANOUT)
1267 resource->flags |= RADEON_FLAG_NO_SUBALLOC;
1268
1269 if (!r600_alloc_resource(rscreen, resource)) {
1270 FREE(rtex);
1271 return NULL;
1272 }
1273 } else {
1274 resource->buf = buf;
1275 resource->gpu_address = rscreen->ws->buffer_get_virtual_address(resource->buf);
1276 resource->bo_size = buf->size;
1277 resource->bo_alignment = buf->alignment;
1278 resource->domains = rscreen->ws->buffer_get_initial_domain(resource->buf);
1279 if (resource->domains & RADEON_DOMAIN_VRAM)
1280 resource->vram_usage = buf->size;
1281 else if (resource->domains & RADEON_DOMAIN_GTT)
1282 resource->gart_usage = buf->size;
1283 }
1284
1285 if (rtex->cmask.size) {
1286 /* Initialize the cmask to 0xCC (= compressed state). */
1287 r600_screen_clear_buffer(rscreen, &rtex->cmask_buffer->b.b,
1288 rtex->cmask.offset, rtex->cmask.size,
1289 0xCCCCCCCC);
1290 }
1291 if (rtex->htile_offset) {
1292 uint32_t clear_value = 0;
1293
1294 if (rscreen->chip_class >= GFX9 || rtex->tc_compatible_htile)
1295 clear_value = 0x0000030F;
1296
1297 r600_screen_clear_buffer(rscreen, &rtex->resource.b.b,
1298 rtex->htile_offset,
1299 rtex->surface.htile_size,
1300 clear_value);
1301 }
1302
1303 /* Initialize DCC only if the texture is not being imported. */
1304 if (!buf && rtex->dcc_offset) {
1305 r600_screen_clear_buffer(rscreen, &rtex->resource.b.b,
1306 rtex->dcc_offset,
1307 rtex->surface.dcc_size,
1308 0xFFFFFFFF);
1309 }
1310
1311 /* Initialize the CMASK base register value. */
1312 rtex->cmask.base_address_reg =
1313 (rtex->resource.gpu_address + rtex->cmask.offset) >> 8;
1314
1315 if (rscreen->debug_flags & DBG_VM) {
1316 fprintf(stderr, "VM start=0x%"PRIX64" end=0x%"PRIX64" | Texture %ix%ix%i, %i levels, %i samples, %s\n",
1317 rtex->resource.gpu_address,
1318 rtex->resource.gpu_address + rtex->resource.buf->size,
1319 base->width0, base->height0, util_max_layer(base, 0)+1, base->last_level+1,
1320 base->nr_samples ? base->nr_samples : 1, util_format_short_name(base->format));
1321 }
1322
1323 if (rscreen->debug_flags & DBG_TEX) {
1324 puts("Texture:");
1325 r600_print_texture_info(rscreen, rtex, stdout);
1326 fflush(stdout);
1327 }
1328
1329 return rtex;
1330 }
1331
1332 static enum radeon_surf_mode
1333 r600_choose_tiling(struct r600_common_screen *rscreen,
1334 const struct pipe_resource *templ)
1335 {
1336 const struct util_format_description *desc = util_format_description(templ->format);
1337 bool force_tiling = templ->flags & R600_RESOURCE_FLAG_FORCE_TILING;
1338 bool is_depth_stencil = util_format_is_depth_or_stencil(templ->format) &&
1339 !(templ->flags & R600_RESOURCE_FLAG_FLUSHED_DEPTH);
1340
1341 /* MSAA resources must be 2D tiled. */
1342 if (templ->nr_samples > 1)
1343 return RADEON_SURF_MODE_2D;
1344
1345 /* Transfer resources should be linear. */
1346 if (templ->flags & R600_RESOURCE_FLAG_TRANSFER)
1347 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1348
1349 /* Avoid Z/S decompress blits by forcing TC-compatible HTILE on VI,
1350 * which requires 2D tiling.
1351 */
1352 if (rscreen->chip_class == VI &&
1353 is_depth_stencil &&
1354 (templ->flags & PIPE_RESOURCE_FLAG_TEXTURING_MORE_LIKELY))
1355 return RADEON_SURF_MODE_2D;
1356
1357 /* r600g: force tiling on TEXTURE_2D and TEXTURE_3D compute resources. */
1358 if (rscreen->chip_class >= R600 && rscreen->chip_class <= CAYMAN &&
1359 (templ->bind & PIPE_BIND_COMPUTE_RESOURCE) &&
1360 (templ->target == PIPE_TEXTURE_2D ||
1361 templ->target == PIPE_TEXTURE_3D))
1362 force_tiling = true;
1363
1364 /* Handle common candidates for the linear mode.
1365 * Compressed textures and DB surfaces must always be tiled.
1366 */
1367 if (!force_tiling &&
1368 !is_depth_stencil &&
1369 !util_format_is_compressed(templ->format)) {
1370 if (rscreen->debug_flags & DBG_NO_TILING)
1371 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1372
1373 /* Tiling doesn't work with the 422 (SUBSAMPLED) formats on R600+. */
1374 if (desc->layout == UTIL_FORMAT_LAYOUT_SUBSAMPLED)
1375 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1376
1377 /* Cursors are linear on SI.
1378 * (XXX double-check, maybe also use RADEON_SURF_SCANOUT) */
1379 if (rscreen->chip_class >= SI &&
1380 (templ->bind & PIPE_BIND_CURSOR))
1381 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1382
1383 if (templ->bind & PIPE_BIND_LINEAR)
1384 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1385
1386 /* Textures with a very small height are recommended to be linear. */
1387 if (templ->target == PIPE_TEXTURE_1D ||
1388 templ->target == PIPE_TEXTURE_1D_ARRAY ||
1389 /* Only very thin and long 2D textures should benefit from
1390 * linear_aligned. */
1391 (templ->width0 > 8 && templ->height0 <= 2))
1392 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1393
1394 /* Textures likely to be mapped often. */
1395 if (templ->usage == PIPE_USAGE_STAGING ||
1396 templ->usage == PIPE_USAGE_STREAM)
1397 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1398 }
1399
1400 /* Make small textures 1D tiled. */
1401 if (templ->width0 <= 16 || templ->height0 <= 16 ||
1402 (rscreen->debug_flags & DBG_NO_2D_TILING))
1403 return RADEON_SURF_MODE_1D;
1404
1405 /* The allocator will switch to 1D if needed. */
1406 return RADEON_SURF_MODE_2D;
1407 }
1408
1409 struct pipe_resource *r600_texture_create(struct pipe_screen *screen,
1410 const struct pipe_resource *templ)
1411 {
1412 struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
1413 struct radeon_surf surface = {0};
1414 bool is_flushed_depth = templ->flags & R600_RESOURCE_FLAG_FLUSHED_DEPTH;
1415 bool tc_compatible_htile =
1416 rscreen->chip_class >= VI &&
1417 (templ->flags & PIPE_RESOURCE_FLAG_TEXTURING_MORE_LIKELY) &&
1418 !(rscreen->debug_flags & DBG_NO_HYPERZ) &&
1419 !is_flushed_depth &&
1420 templ->nr_samples <= 1 && /* TC-compat HTILE is less efficient with MSAA */
1421 util_format_is_depth_or_stencil(templ->format);
1422
1423 int r;
1424
1425 r = r600_init_surface(rscreen, &surface, templ,
1426 r600_choose_tiling(rscreen, templ), 0, 0,
1427 false, false, is_flushed_depth,
1428 tc_compatible_htile);
1429 if (r) {
1430 return NULL;
1431 }
1432
1433 return (struct pipe_resource *)
1434 r600_texture_create_object(screen, templ, NULL, &surface);
1435 }
1436
1437 static struct pipe_resource *r600_texture_from_handle(struct pipe_screen *screen,
1438 const struct pipe_resource *templ,
1439 struct winsys_handle *whandle,
1440 unsigned usage)
1441 {
1442 struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
1443 struct pb_buffer *buf = NULL;
1444 unsigned stride = 0, offset = 0;
1445 unsigned array_mode;
1446 struct radeon_surf surface;
1447 int r;
1448 struct radeon_bo_metadata metadata = {};
1449 struct r600_texture *rtex;
1450 bool is_scanout;
1451
1452 /* Support only 2D textures without mipmaps */
1453 if ((templ->target != PIPE_TEXTURE_2D && templ->target != PIPE_TEXTURE_RECT) ||
1454 templ->depth0 != 1 || templ->last_level != 0)
1455 return NULL;
1456
1457 buf = rscreen->ws->buffer_from_handle(rscreen->ws, whandle, &stride, &offset);
1458 if (!buf)
1459 return NULL;
1460
1461 rscreen->ws->buffer_get_metadata(buf, &metadata);
1462
1463 if (rscreen->chip_class >= GFX9) {
1464 if (metadata.u.gfx9.swizzle_mode > 0)
1465 array_mode = RADEON_SURF_MODE_2D;
1466 else
1467 array_mode = RADEON_SURF_MODE_LINEAR_ALIGNED;
1468
1469 is_scanout = metadata.u.gfx9.swizzle_mode == 0 ||
1470 metadata.u.gfx9.swizzle_mode % 4 == 2;
1471 } else {
1472 surface.u.legacy.pipe_config = metadata.u.legacy.pipe_config;
1473 surface.u.legacy.bankw = metadata.u.legacy.bankw;
1474 surface.u.legacy.bankh = metadata.u.legacy.bankh;
1475 surface.u.legacy.tile_split = metadata.u.legacy.tile_split;
1476 surface.u.legacy.mtilea = metadata.u.legacy.mtilea;
1477 surface.u.legacy.num_banks = metadata.u.legacy.num_banks;
1478
1479 if (metadata.u.legacy.macrotile == RADEON_LAYOUT_TILED)
1480 array_mode = RADEON_SURF_MODE_2D;
1481 else if (metadata.u.legacy.microtile == RADEON_LAYOUT_TILED)
1482 array_mode = RADEON_SURF_MODE_1D;
1483 else
1484 array_mode = RADEON_SURF_MODE_LINEAR_ALIGNED;
1485
1486 is_scanout = metadata.u.legacy.scanout;
1487 }
1488
1489 r = r600_init_surface(rscreen, &surface, templ, array_mode, stride,
1490 offset, true, is_scanout, false, false);
1491 if (r) {
1492 return NULL;
1493 }
1494
1495 rtex = r600_texture_create_object(screen, templ, buf, &surface);
1496 if (!rtex)
1497 return NULL;
1498
1499 rtex->resource.b.is_shared = true;
1500 rtex->resource.external_usage = usage;
1501
1502 if (rscreen->apply_opaque_metadata)
1503 rscreen->apply_opaque_metadata(rscreen, rtex, &metadata);
1504
1505 /* Validate that addrlib arrived at the same surface parameters. */
1506 if (rscreen->chip_class >= GFX9) {
1507 assert(metadata.u.gfx9.swizzle_mode == surface.u.gfx9.surf.swizzle_mode);
1508 }
1509
1510 return &rtex->resource.b.b;
1511 }
1512
1513 bool r600_init_flushed_depth_texture(struct pipe_context *ctx,
1514 struct pipe_resource *texture,
1515 struct r600_texture **staging)
1516 {
1517 struct r600_texture *rtex = (struct r600_texture*)texture;
1518 struct pipe_resource resource;
1519 struct r600_texture **flushed_depth_texture = staging ?
1520 staging : &rtex->flushed_depth_texture;
1521 enum pipe_format pipe_format = texture->format;
1522
1523 if (!staging) {
1524 if (rtex->flushed_depth_texture)
1525 return true; /* it's ready */
1526
1527 if (!rtex->can_sample_z && rtex->can_sample_s) {
1528 switch (pipe_format) {
1529 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
1530 /* Save memory by not allocating the S plane. */
1531 pipe_format = PIPE_FORMAT_Z32_FLOAT;
1532 break;
1533 case PIPE_FORMAT_Z24_UNORM_S8_UINT:
1534 case PIPE_FORMAT_S8_UINT_Z24_UNORM:
1535 /* Save memory bandwidth by not copying the
1536 * stencil part during flush.
1537 *
1538 * This potentially increases memory bandwidth
1539 * if an application uses both Z and S texturing
1540 * simultaneously (a flushed Z24S8 texture
1541 * would be stored compactly), but how often
1542 * does that really happen?
1543 */
1544 pipe_format = PIPE_FORMAT_Z24X8_UNORM;
1545 break;
1546 default:;
1547 }
1548 } else if (!rtex->can_sample_s && rtex->can_sample_z) {
1549 assert(util_format_has_stencil(util_format_description(pipe_format)));
1550
1551 /* DB->CB copies to an 8bpp surface don't work. */
1552 pipe_format = PIPE_FORMAT_X24S8_UINT;
1553 }
1554 }
1555
1556 memset(&resource, 0, sizeof(resource));
1557 resource.target = texture->target;
1558 resource.format = pipe_format;
1559 resource.width0 = texture->width0;
1560 resource.height0 = texture->height0;
1561 resource.depth0 = texture->depth0;
1562 resource.array_size = texture->array_size;
1563 resource.last_level = texture->last_level;
1564 resource.nr_samples = texture->nr_samples;
1565 resource.usage = staging ? PIPE_USAGE_STAGING : PIPE_USAGE_DEFAULT;
1566 resource.bind = texture->bind & ~PIPE_BIND_DEPTH_STENCIL;
1567 resource.flags = texture->flags | R600_RESOURCE_FLAG_FLUSHED_DEPTH;
1568
1569 if (staging)
1570 resource.flags |= R600_RESOURCE_FLAG_TRANSFER;
1571
1572 *flushed_depth_texture = (struct r600_texture *)ctx->screen->resource_create(ctx->screen, &resource);
1573 if (*flushed_depth_texture == NULL) {
1574 R600_ERR("failed to create temporary texture to hold flushed depth\n");
1575 return false;
1576 }
1577
1578 (*flushed_depth_texture)->non_disp_tiling = false;
1579 return true;
1580 }
1581
1582 /**
1583 * Initialize the pipe_resource descriptor to be of the same size as the box,
1584 * which is supposed to hold a subregion of the texture "orig" at the given
1585 * mipmap level.
1586 */
1587 static void r600_init_temp_resource_from_box(struct pipe_resource *res,
1588 struct pipe_resource *orig,
1589 const struct pipe_box *box,
1590 unsigned level, unsigned flags)
1591 {
1592 memset(res, 0, sizeof(*res));
1593 res->format = orig->format;
1594 res->width0 = box->width;
1595 res->height0 = box->height;
1596 res->depth0 = 1;
1597 res->array_size = 1;
1598 res->usage = flags & R600_RESOURCE_FLAG_TRANSFER ? PIPE_USAGE_STAGING : PIPE_USAGE_DEFAULT;
1599 res->flags = flags;
1600
1601 /* We must set the correct texture target and dimensions for a 3D box. */
1602 if (box->depth > 1 && util_max_layer(orig, level) > 0) {
1603 res->target = PIPE_TEXTURE_2D_ARRAY;
1604 res->array_size = box->depth;
1605 } else {
1606 res->target = PIPE_TEXTURE_2D;
1607 }
1608 }
1609
1610 static bool r600_can_invalidate_texture(struct r600_common_screen *rscreen,
1611 struct r600_texture *rtex,
1612 unsigned transfer_usage,
1613 const struct pipe_box *box)
1614 {
1615 /* r600g doesn't react to dirty_tex_descriptor_counter */
1616 return rscreen->chip_class >= SI &&
1617 !rtex->resource.b.is_shared &&
1618 !(transfer_usage & PIPE_TRANSFER_READ) &&
1619 rtex->resource.b.b.last_level == 0 &&
1620 util_texrange_covers_whole_level(&rtex->resource.b.b, 0,
1621 box->x, box->y, box->z,
1622 box->width, box->height,
1623 box->depth);
1624 }
1625
1626 static void r600_texture_invalidate_storage(struct r600_common_context *rctx,
1627 struct r600_texture *rtex)
1628 {
1629 struct r600_common_screen *rscreen = rctx->screen;
1630
1631 /* There is no point in discarding depth and tiled buffers. */
1632 assert(!rtex->is_depth);
1633 assert(rtex->surface.is_linear);
1634
1635 /* Reallocate the buffer in the same pipe_resource. */
1636 r600_alloc_resource(rscreen, &rtex->resource);
1637
1638 /* Initialize the CMASK base address (needed even without CMASK). */
1639 rtex->cmask.base_address_reg =
1640 (rtex->resource.gpu_address + rtex->cmask.offset) >> 8;
1641
1642 p_atomic_inc(&rscreen->dirty_tex_counter);
1643
1644 rctx->num_alloc_tex_transfer_bytes += rtex->size;
1645 }
1646
1647 static void *r600_texture_transfer_map(struct pipe_context *ctx,
1648 struct pipe_resource *texture,
1649 unsigned level,
1650 unsigned usage,
1651 const struct pipe_box *box,
1652 struct pipe_transfer **ptransfer)
1653 {
1654 struct r600_common_context *rctx = (struct r600_common_context*)ctx;
1655 struct r600_texture *rtex = (struct r600_texture*)texture;
1656 struct r600_transfer *trans;
1657 struct r600_resource *buf;
1658 unsigned offset = 0;
1659 char *map;
1660 bool use_staging_texture = false;
1661
1662 assert(!(texture->flags & R600_RESOURCE_FLAG_TRANSFER));
1663 assert(box->width && box->height && box->depth);
1664
1665 /* Depth textures use staging unconditionally. */
1666 if (!rtex->is_depth) {
1667 /* Degrade the tile mode if we get too many transfers on APUs.
1668 * On dGPUs, the staging texture is always faster.
1669 * Only count uploads that are at least 4x4 pixels large.
1670 */
1671 if (!rctx->screen->info.has_dedicated_vram &&
1672 level == 0 &&
1673 box->width >= 4 && box->height >= 4 &&
1674 p_atomic_inc_return(&rtex->num_level0_transfers) == 10) {
1675 bool can_invalidate =
1676 r600_can_invalidate_texture(rctx->screen, rtex,
1677 usage, box);
1678
1679 r600_reallocate_texture_inplace(rctx, rtex,
1680 PIPE_BIND_LINEAR,
1681 can_invalidate);
1682 }
1683
1684 /* Tiled textures need to be converted into a linear texture for CPU
1685 * access. The staging texture is always linear and is placed in GART.
1686 *
1687 * Reading from VRAM or GTT WC is slow, always use the staging
1688 * texture in this case.
1689 *
1690 * Use the staging texture for uploads if the underlying BO
1691 * is busy.
1692 */
1693 if (!rtex->surface.is_linear)
1694 use_staging_texture = true;
1695 else if (usage & PIPE_TRANSFER_READ)
1696 use_staging_texture =
1697 rtex->resource.domains & RADEON_DOMAIN_VRAM ||
1698 rtex->resource.flags & RADEON_FLAG_GTT_WC;
1699 /* Write & linear only: */
1700 else if (r600_rings_is_buffer_referenced(rctx, rtex->resource.buf,
1701 RADEON_USAGE_READWRITE) ||
1702 !rctx->ws->buffer_wait(rtex->resource.buf, 0,
1703 RADEON_USAGE_READWRITE)) {
1704 /* It's busy. */
1705 if (r600_can_invalidate_texture(rctx->screen, rtex,
1706 usage, box))
1707 r600_texture_invalidate_storage(rctx, rtex);
1708 else
1709 use_staging_texture = true;
1710 }
1711 }
1712
1713 trans = CALLOC_STRUCT(r600_transfer);
1714 if (!trans)
1715 return NULL;
1716 pipe_resource_reference(&trans->b.b.resource, texture);
1717 trans->b.b.level = level;
1718 trans->b.b.usage = usage;
1719 trans->b.b.box = *box;
1720
1721 if (rtex->is_depth) {
1722 struct r600_texture *staging_depth;
1723
1724 if (rtex->resource.b.b.nr_samples > 1) {
1725 /* MSAA depth buffers need to be converted to single sample buffers.
1726 *
1727 * Mapping MSAA depth buffers can occur if ReadPixels is called
1728 * with a multisample GLX visual.
1729 *
1730 * First downsample the depth buffer to a temporary texture,
1731 * then decompress the temporary one to staging.
1732 *
1733 * Only the region being mapped is transfered.
1734 */
1735 struct pipe_resource resource;
1736
1737 r600_init_temp_resource_from_box(&resource, texture, box, level, 0);
1738
1739 if (!r600_init_flushed_depth_texture(ctx, &resource, &staging_depth)) {
1740 R600_ERR("failed to create temporary texture to hold untiled copy\n");
1741 FREE(trans);
1742 return NULL;
1743 }
1744
1745 if (usage & PIPE_TRANSFER_READ) {
1746 struct pipe_resource *temp = ctx->screen->resource_create(ctx->screen, &resource);
1747 if (!temp) {
1748 R600_ERR("failed to create a temporary depth texture\n");
1749 FREE(trans);
1750 return NULL;
1751 }
1752
1753 r600_copy_region_with_blit(ctx, temp, 0, 0, 0, 0, texture, level, box);
1754 rctx->blit_decompress_depth(ctx, (struct r600_texture*)temp, staging_depth,
1755 0, 0, 0, box->depth, 0, 0);
1756 pipe_resource_reference(&temp, NULL);
1757 }
1758
1759 /* Just get the strides. */
1760 r600_texture_get_offset(rctx->screen, staging_depth, level, NULL,
1761 &trans->b.b.stride,
1762 &trans->b.b.layer_stride);
1763 } else {
1764 /* XXX: only readback the rectangle which is being mapped? */
1765 /* XXX: when discard is true, no need to read back from depth texture */
1766 if (!r600_init_flushed_depth_texture(ctx, texture, &staging_depth)) {
1767 R600_ERR("failed to create temporary texture to hold untiled copy\n");
1768 FREE(trans);
1769 return NULL;
1770 }
1771
1772 rctx->blit_decompress_depth(ctx, rtex, staging_depth,
1773 level, level,
1774 box->z, box->z + box->depth - 1,
1775 0, 0);
1776
1777 offset = r600_texture_get_offset(rctx->screen, staging_depth,
1778 level, box,
1779 &trans->b.b.stride,
1780 &trans->b.b.layer_stride);
1781 }
1782
1783 trans->staging = (struct r600_resource*)staging_depth;
1784 buf = trans->staging;
1785 } else if (use_staging_texture) {
1786 struct pipe_resource resource;
1787 struct r600_texture *staging;
1788
1789 r600_init_temp_resource_from_box(&resource, texture, box, level,
1790 R600_RESOURCE_FLAG_TRANSFER);
1791 resource.usage = (usage & PIPE_TRANSFER_READ) ?
1792 PIPE_USAGE_STAGING : PIPE_USAGE_STREAM;
1793
1794 /* Create the temporary texture. */
1795 staging = (struct r600_texture*)ctx->screen->resource_create(ctx->screen, &resource);
1796 if (!staging) {
1797 R600_ERR("failed to create temporary texture to hold untiled copy\n");
1798 FREE(trans);
1799 return NULL;
1800 }
1801 trans->staging = &staging->resource;
1802
1803 /* Just get the strides. */
1804 r600_texture_get_offset(rctx->screen, staging, 0, NULL,
1805 &trans->b.b.stride,
1806 &trans->b.b.layer_stride);
1807
1808 if (usage & PIPE_TRANSFER_READ)
1809 r600_copy_to_staging_texture(ctx, trans);
1810 else
1811 usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
1812
1813 buf = trans->staging;
1814 } else {
1815 /* the resource is mapped directly */
1816 offset = r600_texture_get_offset(rctx->screen, rtex, level, box,
1817 &trans->b.b.stride,
1818 &trans->b.b.layer_stride);
1819 buf = &rtex->resource;
1820 }
1821
1822 if (!(map = r600_buffer_map_sync_with_rings(rctx, buf, usage))) {
1823 r600_resource_reference(&trans->staging, NULL);
1824 FREE(trans);
1825 return NULL;
1826 }
1827
1828 *ptransfer = &trans->b.b;
1829 return map + offset;
1830 }
1831
1832 static void r600_texture_transfer_unmap(struct pipe_context *ctx,
1833 struct pipe_transfer* transfer)
1834 {
1835 struct r600_common_context *rctx = (struct r600_common_context*)ctx;
1836 struct r600_transfer *rtransfer = (struct r600_transfer*)transfer;
1837 struct pipe_resource *texture = transfer->resource;
1838 struct r600_texture *rtex = (struct r600_texture*)texture;
1839
1840 if ((transfer->usage & PIPE_TRANSFER_WRITE) && rtransfer->staging) {
1841 if (rtex->is_depth && rtex->resource.b.b.nr_samples <= 1) {
1842 ctx->resource_copy_region(ctx, texture, transfer->level,
1843 transfer->box.x, transfer->box.y, transfer->box.z,
1844 &rtransfer->staging->b.b, transfer->level,
1845 &transfer->box);
1846 } else {
1847 r600_copy_from_staging_texture(ctx, rtransfer);
1848 }
1849 }
1850
1851 if (rtransfer->staging) {
1852 rctx->num_alloc_tex_transfer_bytes += rtransfer->staging->buf->size;
1853 r600_resource_reference(&rtransfer->staging, NULL);
1854 }
1855
1856 /* Heuristic for {upload, draw, upload, draw, ..}:
1857 *
1858 * Flush the gfx IB if we've allocated too much texture storage.
1859 *
1860 * The idea is that we don't want to build IBs that use too much
1861 * memory and put pressure on the kernel memory manager and we also
1862 * want to make temporary and invalidated buffers go idle ASAP to
1863 * decrease the total memory usage or make them reusable. The memory
1864 * usage will be slightly higher than given here because of the buffer
1865 * cache in the winsys.
1866 *
1867 * The result is that the kernel memory manager is never a bottleneck.
1868 */
1869 if (rctx->num_alloc_tex_transfer_bytes > rctx->screen->info.gart_size / 4) {
1870 rctx->gfx.flush(rctx, RADEON_FLUSH_ASYNC, NULL);
1871 rctx->num_alloc_tex_transfer_bytes = 0;
1872 }
1873
1874 pipe_resource_reference(&transfer->resource, NULL);
1875 FREE(transfer);
1876 }
1877
1878 static const struct u_resource_vtbl r600_texture_vtbl =
1879 {
1880 NULL, /* get_handle */
1881 r600_texture_destroy, /* resource_destroy */
1882 r600_texture_transfer_map, /* transfer_map */
1883 u_default_transfer_flush_region, /* transfer_flush_region */
1884 r600_texture_transfer_unmap, /* transfer_unmap */
1885 };
1886
1887 /* DCC channel type categories within which formats can be reinterpreted
1888 * while keeping the same DCC encoding. The swizzle must also match. */
1889 enum dcc_channel_type {
1890 dcc_channel_float32,
1891 dcc_channel_uint32,
1892 dcc_channel_sint32,
1893 dcc_channel_float16,
1894 dcc_channel_uint16,
1895 dcc_channel_sint16,
1896 dcc_channel_uint_10_10_10_2,
1897 dcc_channel_uint8,
1898 dcc_channel_sint8,
1899 dcc_channel_incompatible,
1900 };
1901
1902 /* Return the type of DCC encoding. */
1903 static enum dcc_channel_type
1904 vi_get_dcc_channel_type(const struct util_format_description *desc)
1905 {
1906 int i;
1907
1908 /* Find the first non-void channel. */
1909 for (i = 0; i < desc->nr_channels; i++)
1910 if (desc->channel[i].type != UTIL_FORMAT_TYPE_VOID)
1911 break;
1912 if (i == desc->nr_channels)
1913 return dcc_channel_incompatible;
1914
1915 switch (desc->channel[i].size) {
1916 case 32:
1917 if (desc->channel[i].type == UTIL_FORMAT_TYPE_FLOAT)
1918 return dcc_channel_float32;
1919 if (desc->channel[i].type == UTIL_FORMAT_TYPE_UNSIGNED)
1920 return dcc_channel_uint32;
1921 return dcc_channel_sint32;
1922 case 16:
1923 if (desc->channel[i].type == UTIL_FORMAT_TYPE_FLOAT)
1924 return dcc_channel_float16;
1925 if (desc->channel[i].type == UTIL_FORMAT_TYPE_UNSIGNED)
1926 return dcc_channel_uint16;
1927 return dcc_channel_sint16;
1928 case 10:
1929 return dcc_channel_uint_10_10_10_2;
1930 case 8:
1931 if (desc->channel[i].type == UTIL_FORMAT_TYPE_UNSIGNED)
1932 return dcc_channel_uint8;
1933 return dcc_channel_sint8;
1934 default:
1935 return dcc_channel_incompatible;
1936 }
1937 }
1938
1939 /* Return if it's allowed to reinterpret one format as another with DCC enabled. */
1940 bool vi_dcc_formats_compatible(enum pipe_format format1,
1941 enum pipe_format format2)
1942 {
1943 const struct util_format_description *desc1, *desc2;
1944 enum dcc_channel_type type1, type2;
1945 int i;
1946
1947 if (format1 == format2)
1948 return true;
1949
1950 desc1 = util_format_description(format1);
1951 desc2 = util_format_description(format2);
1952
1953 if (desc1->nr_channels != desc2->nr_channels)
1954 return false;
1955
1956 /* Swizzles must be the same. */
1957 for (i = 0; i < desc1->nr_channels; i++)
1958 if (desc1->swizzle[i] <= PIPE_SWIZZLE_W &&
1959 desc2->swizzle[i] <= PIPE_SWIZZLE_W &&
1960 desc1->swizzle[i] != desc2->swizzle[i])
1961 return false;
1962
1963 type1 = vi_get_dcc_channel_type(desc1);
1964 type2 = vi_get_dcc_channel_type(desc2);
1965
1966 return type1 != dcc_channel_incompatible &&
1967 type2 != dcc_channel_incompatible &&
1968 type1 == type2;
1969 }
1970
1971 bool vi_dcc_formats_are_incompatible(struct pipe_resource *tex,
1972 unsigned level,
1973 enum pipe_format view_format)
1974 {
1975 struct r600_texture *rtex = (struct r600_texture *)tex;
1976
1977 return vi_dcc_enabled(rtex, level) &&
1978 !vi_dcc_formats_compatible(tex->format, view_format);
1979 }
1980
1981 /* This can't be merged with the above function, because
1982 * vi_dcc_formats_compatible should be called only when DCC is enabled. */
1983 void vi_disable_dcc_if_incompatible_format(struct r600_common_context *rctx,
1984 struct pipe_resource *tex,
1985 unsigned level,
1986 enum pipe_format view_format)
1987 {
1988 struct r600_texture *rtex = (struct r600_texture *)tex;
1989
1990 if (vi_dcc_enabled(rtex, level) &&
1991 !vi_dcc_formats_compatible(tex->format, view_format))
1992 if (!r600_texture_disable_dcc(rctx, (struct r600_texture*)tex))
1993 rctx->decompress_dcc(&rctx->b, rtex);
1994 }
1995
1996 struct pipe_surface *r600_create_surface_custom(struct pipe_context *pipe,
1997 struct pipe_resource *texture,
1998 const struct pipe_surface *templ,
1999 unsigned width0, unsigned height0,
2000 unsigned width, unsigned height)
2001 {
2002 struct r600_surface *surface = CALLOC_STRUCT(r600_surface);
2003
2004 if (!surface)
2005 return NULL;
2006
2007 assert(templ->u.tex.first_layer <= util_max_layer(texture, templ->u.tex.level));
2008 assert(templ->u.tex.last_layer <= util_max_layer(texture, templ->u.tex.level));
2009
2010 pipe_reference_init(&surface->base.reference, 1);
2011 pipe_resource_reference(&surface->base.texture, texture);
2012 surface->base.context = pipe;
2013 surface->base.format = templ->format;
2014 surface->base.width = width;
2015 surface->base.height = height;
2016 surface->base.u = templ->u;
2017
2018 surface->width0 = width0;
2019 surface->height0 = height0;
2020
2021 surface->dcc_incompatible =
2022 texture->target != PIPE_BUFFER &&
2023 vi_dcc_formats_are_incompatible(texture, templ->u.tex.level,
2024 templ->format);
2025 return &surface->base;
2026 }
2027
2028 static struct pipe_surface *r600_create_surface(struct pipe_context *pipe,
2029 struct pipe_resource *tex,
2030 const struct pipe_surface *templ)
2031 {
2032 unsigned level = templ->u.tex.level;
2033 unsigned width = u_minify(tex->width0, level);
2034 unsigned height = u_minify(tex->height0, level);
2035 unsigned width0 = tex->width0;
2036 unsigned height0 = tex->height0;
2037
2038 if (tex->target != PIPE_BUFFER && templ->format != tex->format) {
2039 const struct util_format_description *tex_desc
2040 = util_format_description(tex->format);
2041 const struct util_format_description *templ_desc
2042 = util_format_description(templ->format);
2043
2044 assert(tex_desc->block.bits == templ_desc->block.bits);
2045
2046 /* Adjust size of surface if and only if the block width or
2047 * height is changed. */
2048 if (tex_desc->block.width != templ_desc->block.width ||
2049 tex_desc->block.height != templ_desc->block.height) {
2050 unsigned nblks_x = util_format_get_nblocksx(tex->format, width);
2051 unsigned nblks_y = util_format_get_nblocksy(tex->format, height);
2052
2053 width = nblks_x * templ_desc->block.width;
2054 height = nblks_y * templ_desc->block.height;
2055
2056 width0 = util_format_get_nblocksx(tex->format, width0);
2057 height0 = util_format_get_nblocksy(tex->format, height0);
2058 }
2059 }
2060
2061 return r600_create_surface_custom(pipe, tex, templ,
2062 width0, height0,
2063 width, height);
2064 }
2065
2066 static void r600_surface_destroy(struct pipe_context *pipe,
2067 struct pipe_surface *surface)
2068 {
2069 struct r600_surface *surf = (struct r600_surface*)surface;
2070 r600_resource_reference(&surf->cb_buffer_fmask, NULL);
2071 r600_resource_reference(&surf->cb_buffer_cmask, NULL);
2072 pipe_resource_reference(&surface->texture, NULL);
2073 FREE(surface);
2074 }
2075
2076 static void r600_clear_texture(struct pipe_context *pipe,
2077 struct pipe_resource *tex,
2078 unsigned level,
2079 const struct pipe_box *box,
2080 const void *data)
2081 {
2082 struct pipe_screen *screen = pipe->screen;
2083 struct r600_texture *rtex = (struct r600_texture*)tex;
2084 struct pipe_surface tmpl = {{0}};
2085 struct pipe_surface *sf;
2086 const struct util_format_description *desc =
2087 util_format_description(tex->format);
2088
2089 tmpl.format = tex->format;
2090 tmpl.u.tex.first_layer = box->z;
2091 tmpl.u.tex.last_layer = box->z + box->depth - 1;
2092 tmpl.u.tex.level = level;
2093 sf = pipe->create_surface(pipe, tex, &tmpl);
2094 if (!sf)
2095 return;
2096
2097 if (rtex->is_depth) {
2098 unsigned clear;
2099 float depth;
2100 uint8_t stencil = 0;
2101
2102 /* Depth is always present. */
2103 clear = PIPE_CLEAR_DEPTH;
2104 desc->unpack_z_float(&depth, 0, data, 0, 1, 1);
2105
2106 if (rtex->surface.flags & RADEON_SURF_SBUFFER) {
2107 clear |= PIPE_CLEAR_STENCIL;
2108 desc->unpack_s_8uint(&stencil, 0, data, 0, 1, 1);
2109 }
2110
2111 pipe->clear_depth_stencil(pipe, sf, clear, depth, stencil,
2112 box->x, box->y,
2113 box->width, box->height, false);
2114 } else {
2115 union pipe_color_union color;
2116
2117 /* pipe_color_union requires the full vec4 representation. */
2118 if (util_format_is_pure_uint(tex->format))
2119 desc->unpack_rgba_uint(color.ui, 0, data, 0, 1, 1);
2120 else if (util_format_is_pure_sint(tex->format))
2121 desc->unpack_rgba_sint(color.i, 0, data, 0, 1, 1);
2122 else
2123 desc->unpack_rgba_float(color.f, 0, data, 0, 1, 1);
2124
2125 if (screen->is_format_supported(screen, tex->format,
2126 tex->target, 0,
2127 PIPE_BIND_RENDER_TARGET)) {
2128 pipe->clear_render_target(pipe, sf, &color,
2129 box->x, box->y,
2130 box->width, box->height, false);
2131 } else {
2132 /* Software fallback - just for R9G9B9E5_FLOAT */
2133 util_clear_render_target(pipe, sf, &color,
2134 box->x, box->y,
2135 box->width, box->height);
2136 }
2137 }
2138 pipe_surface_reference(&sf, NULL);
2139 }
2140
2141 unsigned r600_translate_colorswap(enum pipe_format format, bool do_endian_swap)
2142 {
2143 const struct util_format_description *desc = util_format_description(format);
2144
2145 #define HAS_SWIZZLE(chan,swz) (desc->swizzle[chan] == PIPE_SWIZZLE_##swz)
2146
2147 if (format == PIPE_FORMAT_R11G11B10_FLOAT) /* isn't plain */
2148 return V_0280A0_SWAP_STD;
2149
2150 if (desc->layout != UTIL_FORMAT_LAYOUT_PLAIN)
2151 return ~0U;
2152
2153 switch (desc->nr_channels) {
2154 case 1:
2155 if (HAS_SWIZZLE(0,X))
2156 return V_0280A0_SWAP_STD; /* X___ */
2157 else if (HAS_SWIZZLE(3,X))
2158 return V_0280A0_SWAP_ALT_REV; /* ___X */
2159 break;
2160 case 2:
2161 if ((HAS_SWIZZLE(0,X) && HAS_SWIZZLE(1,Y)) ||
2162 (HAS_SWIZZLE(0,X) && HAS_SWIZZLE(1,NONE)) ||
2163 (HAS_SWIZZLE(0,NONE) && HAS_SWIZZLE(1,Y)))
2164 return V_0280A0_SWAP_STD; /* XY__ */
2165 else if ((HAS_SWIZZLE(0,Y) && HAS_SWIZZLE(1,X)) ||
2166 (HAS_SWIZZLE(0,Y) && HAS_SWIZZLE(1,NONE)) ||
2167 (HAS_SWIZZLE(0,NONE) && HAS_SWIZZLE(1,X)))
2168 /* YX__ */
2169 return (do_endian_swap ? V_0280A0_SWAP_STD : V_0280A0_SWAP_STD_REV);
2170 else if (HAS_SWIZZLE(0,X) && HAS_SWIZZLE(3,Y))
2171 return V_0280A0_SWAP_ALT; /* X__Y */
2172 else if (HAS_SWIZZLE(0,Y) && HAS_SWIZZLE(3,X))
2173 return V_0280A0_SWAP_ALT_REV; /* Y__X */
2174 break;
2175 case 3:
2176 if (HAS_SWIZZLE(0,X))
2177 return (do_endian_swap ? V_0280A0_SWAP_STD_REV : V_0280A0_SWAP_STD);
2178 else if (HAS_SWIZZLE(0,Z))
2179 return V_0280A0_SWAP_STD_REV; /* ZYX */
2180 break;
2181 case 4:
2182 /* check the middle channels, the 1st and 4th channel can be NONE */
2183 if (HAS_SWIZZLE(1,Y) && HAS_SWIZZLE(2,Z)) {
2184 return V_0280A0_SWAP_STD; /* XYZW */
2185 } else if (HAS_SWIZZLE(1,Z) && HAS_SWIZZLE(2,Y)) {
2186 return V_0280A0_SWAP_STD_REV; /* WZYX */
2187 } else if (HAS_SWIZZLE(1,Y) && HAS_SWIZZLE(2,X)) {
2188 return V_0280A0_SWAP_ALT; /* ZYXW */
2189 } else if (HAS_SWIZZLE(1,Z) && HAS_SWIZZLE(2,W)) {
2190 /* YZWX */
2191 if (desc->is_array)
2192 return V_0280A0_SWAP_ALT_REV;
2193 else
2194 return (do_endian_swap ? V_0280A0_SWAP_ALT : V_0280A0_SWAP_ALT_REV);
2195 }
2196 break;
2197 }
2198 return ~0U;
2199 }
2200
2201 /* PIPELINE_STAT-BASED DCC ENABLEMENT FOR DISPLAYABLE SURFACES */
2202
2203 static void vi_dcc_clean_up_context_slot(struct r600_common_context *rctx,
2204 int slot)
2205 {
2206 int i;
2207
2208 if (rctx->dcc_stats[slot].query_active)
2209 vi_separate_dcc_stop_query(&rctx->b,
2210 rctx->dcc_stats[slot].tex);
2211
2212 for (i = 0; i < ARRAY_SIZE(rctx->dcc_stats[slot].ps_stats); i++)
2213 if (rctx->dcc_stats[slot].ps_stats[i]) {
2214 rctx->b.destroy_query(&rctx->b,
2215 rctx->dcc_stats[slot].ps_stats[i]);
2216 rctx->dcc_stats[slot].ps_stats[i] = NULL;
2217 }
2218
2219 r600_texture_reference(&rctx->dcc_stats[slot].tex, NULL);
2220 }
2221
2222 /**
2223 * Return the per-context slot where DCC statistics queries for the texture live.
2224 */
2225 static unsigned vi_get_context_dcc_stats_index(struct r600_common_context *rctx,
2226 struct r600_texture *tex)
2227 {
2228 int i, empty_slot = -1;
2229
2230 /* Remove zombie textures (textures kept alive by this array only). */
2231 for (i = 0; i < ARRAY_SIZE(rctx->dcc_stats); i++)
2232 if (rctx->dcc_stats[i].tex &&
2233 rctx->dcc_stats[i].tex->resource.b.b.reference.count == 1)
2234 vi_dcc_clean_up_context_slot(rctx, i);
2235
2236 /* Find the texture. */
2237 for (i = 0; i < ARRAY_SIZE(rctx->dcc_stats); i++) {
2238 /* Return if found. */
2239 if (rctx->dcc_stats[i].tex == tex) {
2240 rctx->dcc_stats[i].last_use_timestamp = os_time_get();
2241 return i;
2242 }
2243
2244 /* Record the first seen empty slot. */
2245 if (empty_slot == -1 && !rctx->dcc_stats[i].tex)
2246 empty_slot = i;
2247 }
2248
2249 /* Not found. Remove the oldest member to make space in the array. */
2250 if (empty_slot == -1) {
2251 int oldest_slot = 0;
2252
2253 /* Find the oldest slot. */
2254 for (i = 1; i < ARRAY_SIZE(rctx->dcc_stats); i++)
2255 if (rctx->dcc_stats[oldest_slot].last_use_timestamp >
2256 rctx->dcc_stats[i].last_use_timestamp)
2257 oldest_slot = i;
2258
2259 /* Clean up the oldest slot. */
2260 vi_dcc_clean_up_context_slot(rctx, oldest_slot);
2261 empty_slot = oldest_slot;
2262 }
2263
2264 /* Add the texture to the new slot. */
2265 r600_texture_reference(&rctx->dcc_stats[empty_slot].tex, tex);
2266 rctx->dcc_stats[empty_slot].last_use_timestamp = os_time_get();
2267 return empty_slot;
2268 }
2269
2270 static struct pipe_query *
2271 vi_create_resuming_pipestats_query(struct pipe_context *ctx)
2272 {
2273 struct r600_query_hw *query = (struct r600_query_hw*)
2274 ctx->create_query(ctx, PIPE_QUERY_PIPELINE_STATISTICS, 0);
2275
2276 query->flags |= R600_QUERY_HW_FLAG_BEGIN_RESUMES;
2277 return (struct pipe_query*)query;
2278 }
2279
2280 /**
2281 * Called when binding a color buffer.
2282 */
2283 void vi_separate_dcc_start_query(struct pipe_context *ctx,
2284 struct r600_texture *tex)
2285 {
2286 struct r600_common_context *rctx = (struct r600_common_context*)ctx;
2287 unsigned i = vi_get_context_dcc_stats_index(rctx, tex);
2288
2289 assert(!rctx->dcc_stats[i].query_active);
2290
2291 if (!rctx->dcc_stats[i].ps_stats[0])
2292 rctx->dcc_stats[i].ps_stats[0] = vi_create_resuming_pipestats_query(ctx);
2293
2294 /* begin or resume the query */
2295 ctx->begin_query(ctx, rctx->dcc_stats[i].ps_stats[0]);
2296 rctx->dcc_stats[i].query_active = true;
2297 }
2298
2299 /**
2300 * Called when unbinding a color buffer.
2301 */
2302 void vi_separate_dcc_stop_query(struct pipe_context *ctx,
2303 struct r600_texture *tex)
2304 {
2305 struct r600_common_context *rctx = (struct r600_common_context*)ctx;
2306 unsigned i = vi_get_context_dcc_stats_index(rctx, tex);
2307
2308 assert(rctx->dcc_stats[i].query_active);
2309 assert(rctx->dcc_stats[i].ps_stats[0]);
2310
2311 /* pause or end the query */
2312 ctx->end_query(ctx, rctx->dcc_stats[i].ps_stats[0]);
2313 rctx->dcc_stats[i].query_active = false;
2314 }
2315
2316 static bool vi_should_enable_separate_dcc(struct r600_texture *tex)
2317 {
2318 /* The minimum number of fullscreen draws per frame that is required
2319 * to enable DCC. */
2320 return tex->ps_draw_ratio + tex->num_slow_clears >= 5;
2321 }
2322
2323 /* Called by fast clear. */
2324 static void vi_separate_dcc_try_enable(struct r600_common_context *rctx,
2325 struct r600_texture *tex)
2326 {
2327 /* The intent is to use this with shared displayable back buffers,
2328 * but it's not strictly limited only to them.
2329 */
2330 if (!tex->resource.b.is_shared ||
2331 !(tex->resource.external_usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH) ||
2332 tex->resource.b.b.target != PIPE_TEXTURE_2D ||
2333 tex->resource.b.b.last_level > 0 ||
2334 !tex->surface.dcc_size)
2335 return;
2336
2337 if (tex->dcc_offset)
2338 return; /* already enabled */
2339
2340 /* Enable the DCC stat gathering. */
2341 if (!tex->dcc_gather_statistics) {
2342 tex->dcc_gather_statistics = true;
2343 vi_separate_dcc_start_query(&rctx->b, tex);
2344 }
2345
2346 if (!vi_should_enable_separate_dcc(tex))
2347 return; /* stats show that DCC decompression is too expensive */
2348
2349 assert(tex->surface.num_dcc_levels);
2350 assert(!tex->dcc_separate_buffer);
2351
2352 r600_texture_discard_cmask(rctx->screen, tex);
2353
2354 /* Get a DCC buffer. */
2355 if (tex->last_dcc_separate_buffer) {
2356 assert(tex->dcc_gather_statistics);
2357 assert(!tex->dcc_separate_buffer);
2358 tex->dcc_separate_buffer = tex->last_dcc_separate_buffer;
2359 tex->last_dcc_separate_buffer = NULL;
2360 } else {
2361 tex->dcc_separate_buffer = (struct r600_resource*)
2362 r600_aligned_buffer_create(rctx->b.screen,
2363 R600_RESOURCE_FLAG_UNMAPPABLE,
2364 PIPE_USAGE_DEFAULT,
2365 tex->surface.dcc_size,
2366 tex->surface.dcc_alignment);
2367 if (!tex->dcc_separate_buffer)
2368 return;
2369 }
2370
2371 /* dcc_offset is the absolute GPUVM address. */
2372 tex->dcc_offset = tex->dcc_separate_buffer->gpu_address;
2373
2374 /* no need to flag anything since this is called by fast clear that
2375 * flags framebuffer state
2376 */
2377 }
2378
2379 /**
2380 * Called by pipe_context::flush_resource, the place where DCC decompression
2381 * takes place.
2382 */
2383 void vi_separate_dcc_process_and_reset_stats(struct pipe_context *ctx,
2384 struct r600_texture *tex)
2385 {
2386 struct r600_common_context *rctx = (struct r600_common_context*)ctx;
2387 struct pipe_query *tmp;
2388 unsigned i = vi_get_context_dcc_stats_index(rctx, tex);
2389 bool query_active = rctx->dcc_stats[i].query_active;
2390 bool disable = false;
2391
2392 if (rctx->dcc_stats[i].ps_stats[2]) {
2393 union pipe_query_result result;
2394
2395 /* Read the results. */
2396 ctx->get_query_result(ctx, rctx->dcc_stats[i].ps_stats[2],
2397 true, &result);
2398 r600_query_hw_reset_buffers(rctx,
2399 (struct r600_query_hw*)
2400 rctx->dcc_stats[i].ps_stats[2]);
2401
2402 /* Compute the approximate number of fullscreen draws. */
2403 tex->ps_draw_ratio =
2404 result.pipeline_statistics.ps_invocations /
2405 (tex->resource.b.b.width0 * tex->resource.b.b.height0);
2406 rctx->last_tex_ps_draw_ratio = tex->ps_draw_ratio;
2407
2408 disable = tex->dcc_separate_buffer &&
2409 !vi_should_enable_separate_dcc(tex);
2410 }
2411
2412 tex->num_slow_clears = 0;
2413
2414 /* stop the statistics query for ps_stats[0] */
2415 if (query_active)
2416 vi_separate_dcc_stop_query(ctx, tex);
2417
2418 /* Move the queries in the queue by one. */
2419 tmp = rctx->dcc_stats[i].ps_stats[2];
2420 rctx->dcc_stats[i].ps_stats[2] = rctx->dcc_stats[i].ps_stats[1];
2421 rctx->dcc_stats[i].ps_stats[1] = rctx->dcc_stats[i].ps_stats[0];
2422 rctx->dcc_stats[i].ps_stats[0] = tmp;
2423
2424 /* create and start a new query as ps_stats[0] */
2425 if (query_active)
2426 vi_separate_dcc_start_query(ctx, tex);
2427
2428 if (disable) {
2429 assert(!tex->last_dcc_separate_buffer);
2430 tex->last_dcc_separate_buffer = tex->dcc_separate_buffer;
2431 tex->dcc_separate_buffer = NULL;
2432 tex->dcc_offset = 0;
2433 /* no need to flag anything since this is called after
2434 * decompression that re-sets framebuffer state
2435 */
2436 }
2437 }
2438
2439 /* FAST COLOR CLEAR */
2440
2441 static void evergreen_set_clear_color(struct r600_texture *rtex,
2442 enum pipe_format surface_format,
2443 const union pipe_color_union *color)
2444 {
2445 union util_color uc;
2446
2447 memset(&uc, 0, sizeof(uc));
2448
2449 if (rtex->surface.bpe == 16) {
2450 /* DCC fast clear only:
2451 * CLEAR_WORD0 = R = G = B
2452 * CLEAR_WORD1 = A
2453 */
2454 assert(color->ui[0] == color->ui[1] &&
2455 color->ui[0] == color->ui[2]);
2456 uc.ui[0] = color->ui[0];
2457 uc.ui[1] = color->ui[3];
2458 } else if (util_format_is_pure_uint(surface_format)) {
2459 util_format_write_4ui(surface_format, color->ui, 0, &uc, 0, 0, 0, 1, 1);
2460 } else if (util_format_is_pure_sint(surface_format)) {
2461 util_format_write_4i(surface_format, color->i, 0, &uc, 0, 0, 0, 1, 1);
2462 } else {
2463 util_pack_color(color->f, surface_format, &uc);
2464 }
2465
2466 memcpy(rtex->color_clear_value, &uc, 2 * sizeof(uint32_t));
2467 }
2468
2469 static bool vi_get_fast_clear_parameters(enum pipe_format surface_format,
2470 const union pipe_color_union *color,
2471 uint32_t* reset_value,
2472 bool* clear_words_needed)
2473 {
2474 bool values[4] = {};
2475 int i;
2476 bool main_value = false;
2477 bool extra_value = false;
2478 int extra_channel;
2479
2480 /* This is needed to get the correct DCC clear value for luminance formats.
2481 * 1) Get the linear format (because the next step can't handle L8_SRGB).
2482 * 2) Convert luminance to red. (the real hw format for luminance)
2483 */
2484 surface_format = util_format_linear(surface_format);
2485 surface_format = util_format_luminance_to_red(surface_format);
2486
2487 const struct util_format_description *desc = util_format_description(surface_format);
2488
2489 if (desc->block.bits == 128 &&
2490 (color->ui[0] != color->ui[1] ||
2491 color->ui[0] != color->ui[2]))
2492 return false;
2493
2494 *clear_words_needed = true;
2495 *reset_value = 0x20202020U;
2496
2497 /* If we want to clear without needing a fast clear eliminate step, we
2498 * can set each channel to 0 or 1 (or 0/max for integer formats). We
2499 * have two sets of flags, one for the last or first channel(extra) and
2500 * one for the other channels(main).
2501 */
2502
2503 if (surface_format == PIPE_FORMAT_R11G11B10_FLOAT ||
2504 surface_format == PIPE_FORMAT_B5G6R5_UNORM ||
2505 surface_format == PIPE_FORMAT_B5G6R5_SRGB ||
2506 util_format_is_alpha(surface_format)) {
2507 extra_channel = -1;
2508 } else if (desc->layout == UTIL_FORMAT_LAYOUT_PLAIN) {
2509 if(r600_translate_colorswap(surface_format, false) <= 1)
2510 extra_channel = desc->nr_channels - 1;
2511 else
2512 extra_channel = 0;
2513 } else
2514 return true;
2515
2516 for (i = 0; i < 4; ++i) {
2517 int index = desc->swizzle[i] - PIPE_SWIZZLE_X;
2518
2519 if (desc->swizzle[i] < PIPE_SWIZZLE_X ||
2520 desc->swizzle[i] > PIPE_SWIZZLE_W)
2521 continue;
2522
2523 if (desc->channel[i].pure_integer &&
2524 desc->channel[i].type == UTIL_FORMAT_TYPE_SIGNED) {
2525 /* Use the maximum value for clamping the clear color. */
2526 int max = u_bit_consecutive(0, desc->channel[i].size - 1);
2527
2528 values[i] = color->i[i] != 0;
2529 if (color->i[i] != 0 && MIN2(color->i[i], max) != max)
2530 return true;
2531 } else if (desc->channel[i].pure_integer &&
2532 desc->channel[i].type == UTIL_FORMAT_TYPE_UNSIGNED) {
2533 /* Use the maximum value for clamping the clear color. */
2534 unsigned max = u_bit_consecutive(0, desc->channel[i].size);
2535
2536 values[i] = color->ui[i] != 0U;
2537 if (color->ui[i] != 0U && MIN2(color->ui[i], max) != max)
2538 return true;
2539 } else {
2540 values[i] = color->f[i] != 0.0F;
2541 if (color->f[i] != 0.0F && color->f[i] != 1.0F)
2542 return true;
2543 }
2544
2545 if (index == extra_channel)
2546 extra_value = values[i];
2547 else
2548 main_value = values[i];
2549 }
2550
2551 for (int i = 0; i < 4; ++i)
2552 if (values[i] != main_value &&
2553 desc->swizzle[i] - PIPE_SWIZZLE_X != extra_channel &&
2554 desc->swizzle[i] >= PIPE_SWIZZLE_X &&
2555 desc->swizzle[i] <= PIPE_SWIZZLE_W)
2556 return true;
2557
2558 *clear_words_needed = false;
2559 if (main_value)
2560 *reset_value |= 0x80808080U;
2561
2562 if (extra_value)
2563 *reset_value |= 0x40404040U;
2564 return true;
2565 }
2566
2567 void vi_dcc_clear_level(struct r600_common_context *rctx,
2568 struct r600_texture *rtex,
2569 unsigned level, unsigned clear_value)
2570 {
2571 struct pipe_resource *dcc_buffer;
2572 uint64_t dcc_offset, clear_size;
2573
2574 assert(vi_dcc_enabled(rtex, level));
2575
2576 if (rtex->dcc_separate_buffer) {
2577 dcc_buffer = &rtex->dcc_separate_buffer->b.b;
2578 dcc_offset = 0;
2579 } else {
2580 dcc_buffer = &rtex->resource.b.b;
2581 dcc_offset = rtex->dcc_offset;
2582 }
2583
2584 if (rctx->chip_class >= GFX9) {
2585 /* Mipmap level clears aren't implemented. */
2586 assert(rtex->resource.b.b.last_level == 0);
2587 /* MSAA needs a different clear size. */
2588 assert(rtex->resource.b.b.nr_samples <= 1);
2589 clear_size = rtex->surface.dcc_size;
2590 } else {
2591 dcc_offset += rtex->surface.u.legacy.level[level].dcc_offset;
2592 clear_size = rtex->surface.u.legacy.level[level].dcc_fast_clear_size;
2593 }
2594
2595 rctx->clear_buffer(&rctx->b, dcc_buffer, dcc_offset, clear_size,
2596 clear_value, R600_COHERENCY_CB_META);
2597 }
2598
2599 /* Set the same micro tile mode as the destination of the last MSAA resolve.
2600 * This allows hitting the MSAA resolve fast path, which requires that both
2601 * src and dst micro tile modes match.
2602 */
2603 static void si_set_optimal_micro_tile_mode(struct r600_common_screen *rscreen,
2604 struct r600_texture *rtex)
2605 {
2606 if (rtex->resource.b.is_shared ||
2607 rtex->resource.b.b.nr_samples <= 1 ||
2608 rtex->surface.micro_tile_mode == rtex->last_msaa_resolve_target_micro_mode)
2609 return;
2610
2611 assert(rscreen->chip_class >= GFX9 ||
2612 rtex->surface.u.legacy.level[0].mode == RADEON_SURF_MODE_2D);
2613 assert(rtex->resource.b.b.last_level == 0);
2614
2615 if (rscreen->chip_class >= GFX9) {
2616 /* 4K or larger tiles only. 0 is linear. 1-3 are 256B tiles. */
2617 assert(rtex->surface.u.gfx9.surf.swizzle_mode >= 4);
2618
2619 /* If you do swizzle_mode % 4, you'll get:
2620 * 0 = Depth
2621 * 1 = Standard,
2622 * 2 = Displayable
2623 * 3 = Rotated
2624 *
2625 * Depth-sample order isn't allowed:
2626 */
2627 assert(rtex->surface.u.gfx9.surf.swizzle_mode % 4 != 0);
2628
2629 switch (rtex->last_msaa_resolve_target_micro_mode) {
2630 case RADEON_MICRO_MODE_DISPLAY:
2631 rtex->surface.u.gfx9.surf.swizzle_mode &= ~0x3;
2632 rtex->surface.u.gfx9.surf.swizzle_mode += 2; /* D */
2633 break;
2634 case RADEON_MICRO_MODE_THIN:
2635 rtex->surface.u.gfx9.surf.swizzle_mode &= ~0x3;
2636 rtex->surface.u.gfx9.surf.swizzle_mode += 1; /* S */
2637 break;
2638 case RADEON_MICRO_MODE_ROTATED:
2639 rtex->surface.u.gfx9.surf.swizzle_mode &= ~0x3;
2640 rtex->surface.u.gfx9.surf.swizzle_mode += 3; /* R */
2641 break;
2642 default: /* depth */
2643 assert(!"unexpected micro mode");
2644 return;
2645 }
2646 } else if (rscreen->chip_class >= CIK) {
2647 /* These magic numbers were copied from addrlib. It doesn't use
2648 * any definitions for them either. They are all 2D_TILED_THIN1
2649 * modes with different bpp and micro tile mode.
2650 */
2651 switch (rtex->last_msaa_resolve_target_micro_mode) {
2652 case RADEON_MICRO_MODE_DISPLAY:
2653 rtex->surface.u.legacy.tiling_index[0] = 10;
2654 break;
2655 case RADEON_MICRO_MODE_THIN:
2656 rtex->surface.u.legacy.tiling_index[0] = 14;
2657 break;
2658 case RADEON_MICRO_MODE_ROTATED:
2659 rtex->surface.u.legacy.tiling_index[0] = 28;
2660 break;
2661 default: /* depth, thick */
2662 assert(!"unexpected micro mode");
2663 return;
2664 }
2665 } else { /* SI */
2666 switch (rtex->last_msaa_resolve_target_micro_mode) {
2667 case RADEON_MICRO_MODE_DISPLAY:
2668 switch (rtex->surface.bpe) {
2669 case 1:
2670 rtex->surface.u.legacy.tiling_index[0] = 10;
2671 break;
2672 case 2:
2673 rtex->surface.u.legacy.tiling_index[0] = 11;
2674 break;
2675 default: /* 4, 8 */
2676 rtex->surface.u.legacy.tiling_index[0] = 12;
2677 break;
2678 }
2679 break;
2680 case RADEON_MICRO_MODE_THIN:
2681 switch (rtex->surface.bpe) {
2682 case 1:
2683 rtex->surface.u.legacy.tiling_index[0] = 14;
2684 break;
2685 case 2:
2686 rtex->surface.u.legacy.tiling_index[0] = 15;
2687 break;
2688 case 4:
2689 rtex->surface.u.legacy.tiling_index[0] = 16;
2690 break;
2691 default: /* 8, 16 */
2692 rtex->surface.u.legacy.tiling_index[0] = 17;
2693 break;
2694 }
2695 break;
2696 default: /* depth, thick */
2697 assert(!"unexpected micro mode");
2698 return;
2699 }
2700 }
2701
2702 rtex->surface.micro_tile_mode = rtex->last_msaa_resolve_target_micro_mode;
2703
2704 p_atomic_inc(&rscreen->dirty_tex_counter);
2705 }
2706
2707 void evergreen_do_fast_color_clear(struct r600_common_context *rctx,
2708 struct pipe_framebuffer_state *fb,
2709 struct r600_atom *fb_state,
2710 unsigned *buffers, ubyte *dirty_cbufs,
2711 const union pipe_color_union *color)
2712 {
2713 int i;
2714
2715 /* This function is broken in BE, so just disable this path for now */
2716 #ifdef PIPE_ARCH_BIG_ENDIAN
2717 return;
2718 #endif
2719
2720 if (rctx->render_cond)
2721 return;
2722
2723 for (i = 0; i < fb->nr_cbufs; i++) {
2724 struct r600_texture *tex;
2725 unsigned clear_bit = PIPE_CLEAR_COLOR0 << i;
2726
2727 if (!fb->cbufs[i])
2728 continue;
2729
2730 /* if this colorbuffer is not being cleared */
2731 if (!(*buffers & clear_bit))
2732 continue;
2733
2734 tex = (struct r600_texture *)fb->cbufs[i]->texture;
2735
2736 /* the clear is allowed if all layers are bound */
2737 if (fb->cbufs[i]->u.tex.first_layer != 0 ||
2738 fb->cbufs[i]->u.tex.last_layer != util_max_layer(&tex->resource.b.b, 0)) {
2739 continue;
2740 }
2741
2742 /* cannot clear mipmapped textures */
2743 if (fb->cbufs[i]->texture->last_level != 0) {
2744 continue;
2745 }
2746
2747 /* only supported on tiled surfaces */
2748 if (tex->surface.is_linear) {
2749 continue;
2750 }
2751
2752 /* shared textures can't use fast clear without an explicit flush,
2753 * because there is no way to communicate the clear color among
2754 * all clients
2755 */
2756 if (tex->resource.b.is_shared &&
2757 !(tex->resource.external_usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH))
2758 continue;
2759
2760 /* fast color clear with 1D tiling doesn't work on old kernels and CIK */
2761 if (rctx->chip_class == CIK &&
2762 tex->surface.u.legacy.level[0].mode == RADEON_SURF_MODE_1D &&
2763 rctx->screen->info.drm_major == 2 &&
2764 rctx->screen->info.drm_minor < 38) {
2765 continue;
2766 }
2767
2768 /* Fast clear is the most appropriate place to enable DCC for
2769 * displayable surfaces.
2770 */
2771 if (rctx->chip_class >= VI &&
2772 !(rctx->screen->debug_flags & DBG_NO_DCC_FB)) {
2773 vi_separate_dcc_try_enable(rctx, tex);
2774
2775 /* RB+ isn't supported with a CMASK clear only on Stoney,
2776 * so all clears are considered to be hypothetically slow
2777 * clears, which is weighed when determining whether to
2778 * enable separate DCC.
2779 */
2780 if (tex->dcc_gather_statistics &&
2781 rctx->family == CHIP_STONEY)
2782 tex->num_slow_clears++;
2783 }
2784
2785 /* Try to clear DCC first, otherwise try CMASK. */
2786 if (vi_dcc_enabled(tex, 0)) {
2787 uint32_t reset_value;
2788 bool clear_words_needed;
2789
2790 if (rctx->screen->debug_flags & DBG_NO_DCC_CLEAR)
2791 continue;
2792
2793 if (!vi_get_fast_clear_parameters(fb->cbufs[i]->format,
2794 color, &reset_value,
2795 &clear_words_needed))
2796 continue;
2797
2798 vi_dcc_clear_level(rctx, tex, 0, reset_value);
2799
2800 unsigned level_bit = 1 << fb->cbufs[i]->u.tex.level;
2801 if (clear_words_needed) {
2802 bool need_compressed_update = !tex->dirty_level_mask;
2803
2804 tex->dirty_level_mask |= level_bit;
2805
2806 if (need_compressed_update)
2807 p_atomic_inc(&rctx->screen->compressed_colortex_counter);
2808 }
2809 tex->separate_dcc_dirty = true;
2810 } else {
2811 /* 128-bit formats are unusupported */
2812 if (tex->surface.bpe > 8) {
2813 continue;
2814 }
2815
2816 /* RB+ doesn't work with CMASK fast clear on Stoney. */
2817 if (rctx->family == CHIP_STONEY)
2818 continue;
2819
2820 /* ensure CMASK is enabled */
2821 r600_texture_alloc_cmask_separate(rctx->screen, tex);
2822 if (tex->cmask.size == 0) {
2823 continue;
2824 }
2825
2826 /* Do the fast clear. */
2827 rctx->clear_buffer(&rctx->b, &tex->cmask_buffer->b.b,
2828 tex->cmask.offset, tex->cmask.size, 0,
2829 R600_COHERENCY_CB_META);
2830
2831 bool need_compressed_update = !tex->dirty_level_mask;
2832
2833 tex->dirty_level_mask |= 1 << fb->cbufs[i]->u.tex.level;
2834
2835 if (need_compressed_update)
2836 p_atomic_inc(&rctx->screen->compressed_colortex_counter);
2837 }
2838
2839 /* We can change the micro tile mode before a full clear. */
2840 if (rctx->screen->chip_class >= SI)
2841 si_set_optimal_micro_tile_mode(rctx->screen, tex);
2842
2843 evergreen_set_clear_color(tex, fb->cbufs[i]->format, color);
2844
2845 if (dirty_cbufs)
2846 *dirty_cbufs |= 1 << i;
2847 rctx->set_atom_dirty(rctx, fb_state, true);
2848 *buffers &= ~clear_bit;
2849 }
2850 }
2851
2852 void r600_init_screen_texture_functions(struct r600_common_screen *rscreen)
2853 {
2854 rscreen->b.resource_from_handle = r600_texture_from_handle;
2855 rscreen->b.resource_get_handle = r600_texture_get_handle;
2856 }
2857
2858 void r600_init_context_texture_functions(struct r600_common_context *rctx)
2859 {
2860 rctx->b.create_surface = r600_create_surface;
2861 rctx->b.surface_destroy = r600_surface_destroy;
2862 rctx->b.clear_texture = r600_clear_texture;
2863 }