projects / mesa.git / blob
? search:


508340826e73a50415b9b85e74b2ab1a15c86656
[mesa.git] / src / gallium / drivers / r600 / r600_texture.c
1 /*
2 * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 *
23 * Authors:
24 * Jerome Glisse
25 * Corbin Simpson
26 */
27 #include "r600_formats.h"
28 #include "evergreen_compute.h"
29 #include "r600d.h"
30
31 #include <errno.h>
32 #include "util/u_format_s3tc.h"
33 #include "util/u_memory.h"
34
35
36 /* Same as resource_copy_region, except that both upsampling and downsampling are allowed. */
37 static void r600_copy_region_with_blit(struct pipe_context *pipe,
38 struct pipe_resource *dst,
39 unsigned dst_level,
40 unsigned dstx, unsigned dsty, unsigned dstz,
41 struct pipe_resource *src,
42 unsigned src_level,
43 const struct pipe_box *src_box)
44 {
45 struct pipe_blit_info blit;
46
47 memset(&blit, 0, sizeof(blit));
48 blit.src.resource = src;
49 blit.src.format = src->format;
50 blit.src.level = src_level;
51 blit.src.box = *src_box;
52 blit.dst.resource = dst;
53 blit.dst.format = dst->format;
54 blit.dst.level = dst_level;
55 blit.dst.box.x = dstx;
56 blit.dst.box.y = dsty;
57 blit.dst.box.z = dstz;
58 blit.dst.box.width = src_box->width;
59 blit.dst.box.height = src_box->height;
60 blit.dst.box.depth = src_box->depth;
61 blit.mask = util_format_get_mask(src->format) &
62 util_format_get_mask(dst->format);
63 blit.filter = PIPE_TEX_FILTER_NEAREST;
64
65 if (blit.mask) {
66 pipe->blit(pipe, &blit);
67 }
68 }
69
70 /* Copy from a full GPU texture to a transfer's staging one. */
71 static void r600_copy_to_staging_texture(struct pipe_context *ctx, struct r600_transfer *rtransfer)
72 {
73 struct r600_context *rctx = (struct r600_context*)ctx;
74 struct pipe_transfer *transfer = (struct pipe_transfer*)rtransfer;
75 struct pipe_resource *dst = &rtransfer->staging->b.b;
76 struct pipe_resource *src = transfer->resource;
77
78 if (src->nr_samples > 1) {
79 r600_copy_region_with_blit(ctx, dst, 0, 0, 0, 0,
80 src, transfer->level, &transfer->box);
81 return;
82 }
83
84 if (!rctx->b.dma_copy(ctx, dst, 0, 0, 0, 0,
85 src, transfer->level,
86 &transfer->box)) {
87 ctx->resource_copy_region(ctx, dst, 0, 0, 0, 0,
88 src, transfer->level, &transfer->box);
89 }
90 }
91
92 /* Copy from a transfer's staging texture to a full GPU one. */
93 static void r600_copy_from_staging_texture(struct pipe_context *ctx, struct r600_transfer *rtransfer)
94 {
95 struct r600_context *rctx = (struct r600_context*)ctx;
96 struct pipe_transfer *transfer = (struct pipe_transfer*)rtransfer;
97 struct pipe_resource *dst = transfer->resource;
98 struct pipe_resource *src = &rtransfer->staging->b.b;
99 struct pipe_box sbox;
100
101 u_box_3d(0, 0, 0, transfer->box.width, transfer->box.height, transfer->box.depth, &sbox);
102
103 if (dst->nr_samples > 1) {
104 r600_copy_region_with_blit(ctx, dst, transfer->level,
105 transfer->box.x, transfer->box.y, transfer->box.z,
106 src, 0, &sbox);
107 return;
108 }
109
110 if (!rctx->b.dma_copy(ctx, dst, transfer->level,
111 transfer->box.x, transfer->box.y, transfer->box.z,
112 src, 0, &sbox)) {
113 ctx->resource_copy_region(ctx, dst, transfer->level,
114 transfer->box.x, transfer->box.y, transfer->box.z,
115 src, 0, &sbox);
116 }
117 }
118
119 static unsigned r600_texture_get_offset(struct r600_texture *rtex, unsigned level,
120 const struct pipe_box *box)
121 {
122 enum pipe_format format = rtex->resource.b.b.format;
123
124 return rtex->surface.level[level].offset +
125 box->z * rtex->surface.level[level].slice_size +
126 box->y / util_format_get_blockheight(format) * rtex->surface.level[level].pitch_bytes +
127 box->x / util_format_get_blockwidth(format) * util_format_get_blocksize(format);
128 }
129
130 static int r600_init_surface(struct r600_screen *rscreen,
131 struct radeon_surface *surface,
132 const struct pipe_resource *ptex,
133 unsigned array_mode,
134 bool is_flushed_depth)
135 {
136 const struct util_format_description *desc =
137 util_format_description(ptex->format);
138 bool is_depth, is_stencil;
139
140 is_depth = util_format_has_depth(desc);
141 is_stencil = util_format_has_stencil(desc);
142
143 surface->npix_x = ptex->width0;
144 surface->npix_y = ptex->height0;
145 surface->npix_z = ptex->depth0;
146 surface->blk_w = util_format_get_blockwidth(ptex->format);
147 surface->blk_h = util_format_get_blockheight(ptex->format);
148 surface->blk_d = 1;
149 surface->array_size = 1;
150 surface->last_level = ptex->last_level;
151
152 if (rscreen->b.chip_class >= EVERGREEN && !is_flushed_depth &&
153 ptex->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) {
154 surface->bpe = 4; /* stencil is allocated separately on evergreen */
155 } else {
156 surface->bpe = util_format_get_blocksize(ptex->format);
157 /* align byte per element on dword */
158 if (surface->bpe == 3) {
159 surface->bpe = 4;
160 }
161 }
162
163 surface->nsamples = ptex->nr_samples ? ptex->nr_samples : 1;
164 surface->flags = 0;
165
166 switch (array_mode) {
167 case V_038000_ARRAY_1D_TILED_THIN1:
168 surface->flags |= RADEON_SURF_SET(RADEON_SURF_MODE_1D, MODE);
169 break;
170 case V_038000_ARRAY_2D_TILED_THIN1:
171 surface->flags |= RADEON_SURF_SET(RADEON_SURF_MODE_2D, MODE);
172 break;
173 case V_038000_ARRAY_LINEAR_ALIGNED:
174 surface->flags |= RADEON_SURF_SET(RADEON_SURF_MODE_LINEAR_ALIGNED, MODE);
175 break;
176 case V_038000_ARRAY_LINEAR_GENERAL:
177 default:
178 surface->flags |= RADEON_SURF_SET(RADEON_SURF_MODE_LINEAR, MODE);
179 break;
180 }
181 switch (ptex->target) {
182 case PIPE_TEXTURE_1D:
183 surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_1D, TYPE);
184 break;
185 case PIPE_TEXTURE_RECT:
186 case PIPE_TEXTURE_2D:
187 surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_2D, TYPE);
188 break;
189 case PIPE_TEXTURE_3D:
190 surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_3D, TYPE);
191 break;
192 case PIPE_TEXTURE_1D_ARRAY:
193 surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_1D_ARRAY, TYPE);
194 surface->array_size = ptex->array_size;
195 break;
196 case PIPE_TEXTURE_2D_ARRAY:
197 case PIPE_TEXTURE_CUBE_ARRAY: /* cube array layout like 2d layout for now */
198 surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_2D_ARRAY, TYPE);
199 surface->array_size = ptex->array_size;
200 break;
201 case PIPE_TEXTURE_CUBE:
202 surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_CUBEMAP, TYPE);
203 break;
204 case PIPE_BUFFER:
205 default:
206 return -EINVAL;
207 }
208 if (ptex->bind & PIPE_BIND_SCANOUT) {
209 surface->flags |= RADEON_SURF_SCANOUT;
210 }
211
212 if (!is_flushed_depth && is_depth) {
213 surface->flags |= RADEON_SURF_ZBUFFER;
214
215 if (is_stencil) {
216 surface->flags |= RADEON_SURF_SBUFFER |
217 RADEON_SURF_HAS_SBUFFER_MIPTREE;
218 }
219 }
220 return 0;
221 }
222
223 static int r600_setup_surface(struct pipe_screen *screen,
224 struct r600_texture *rtex,
225 unsigned pitch_in_bytes_override)
226 {
227 struct r600_screen *rscreen = (struct r600_screen*)screen;
228 int r;
229
230 r = rscreen->b.ws->surface_init(rscreen->b.ws, &rtex->surface);
231 if (r) {
232 return r;
233 }
234 rtex->size = rtex->surface.bo_size;
235 if (pitch_in_bytes_override && pitch_in_bytes_override != rtex->surface.level[0].pitch_bytes) {
236 /* old ddx on evergreen over estimate alignment for 1d, only 1 level
237 * for those
238 */
239 rtex->surface.level[0].nblk_x = pitch_in_bytes_override / rtex->surface.bpe;
240 rtex->surface.level[0].pitch_bytes = pitch_in_bytes_override;
241 rtex->surface.level[0].slice_size = pitch_in_bytes_override * rtex->surface.level[0].nblk_y;
242 if (rtex->surface.flags & RADEON_SURF_SBUFFER) {
243 rtex->surface.stencil_offset =
244 rtex->surface.stencil_level[0].offset = rtex->surface.level[0].slice_size;
245 }
246 }
247 return 0;
248 }
249
250 static boolean r600_texture_get_handle(struct pipe_screen* screen,
251 struct pipe_resource *ptex,
252 struct winsys_handle *whandle)
253 {
254 struct r600_texture *rtex = (struct r600_texture*)ptex;
255 struct r600_resource *resource = &rtex->resource;
256 struct radeon_surface *surface = &rtex->surface;
257 struct r600_screen *rscreen = (struct r600_screen*)screen;
258
259 rscreen->b.ws->buffer_set_tiling(resource->buf,
260 NULL,
261 surface->level[0].mode >= RADEON_SURF_MODE_1D ?
262 RADEON_LAYOUT_TILED : RADEON_LAYOUT_LINEAR,
263 surface->level[0].mode >= RADEON_SURF_MODE_2D ?
264 RADEON_LAYOUT_TILED : RADEON_LAYOUT_LINEAR,
265 surface->bankw, surface->bankh,
266 surface->tile_split,
267 surface->stencil_tile_split,
268 surface->mtilea,
269 rtex->surface.level[0].pitch_bytes);
270
271 return rscreen->b.ws->buffer_get_handle(resource->buf,
272 rtex->surface.level[0].pitch_bytes, whandle);
273 }
274
275 static void r600_texture_destroy(struct pipe_screen *screen,
276 struct pipe_resource *ptex)
277 {
278 struct r600_texture *rtex = (struct r600_texture*)ptex;
279 struct r600_resource *resource = &rtex->resource;
280
281 if (rtex->flushed_depth_texture)
282 pipe_resource_reference((struct pipe_resource **)&rtex->flushed_depth_texture, NULL);
283
284 pipe_resource_reference((struct pipe_resource**)&rtex->htile, NULL);
285 if (rtex->cmask_buffer != &rtex->resource) {
286 pipe_resource_reference((struct pipe_resource**)&rtex->cmask_buffer, NULL);
287 }
288 pb_reference(&resource->buf, NULL);
289 FREE(rtex);
290 }
291
292 static const struct u_resource_vtbl r600_texture_vtbl;
293
294 /* The number of samples can be specified independently of the texture. */
295 void r600_texture_get_fmask_info(struct r600_screen *rscreen,
296 struct r600_texture *rtex,
297 unsigned nr_samples,
298 struct r600_fmask_info *out)
299 {
300 /* FMASK is allocated like an ordinary texture. */
301 struct radeon_surface fmask = rtex->surface;
302
303 memset(out, 0, sizeof(*out));
304
305 fmask.bo_alignment = 0;
306 fmask.bo_size = 0;
307 fmask.nsamples = 1;
308 fmask.flags |= RADEON_SURF_FMASK;
309
310 switch (nr_samples) {
311 case 2:
312 case 4:
313 fmask.bpe = 1;
314 fmask.bankh = 4;
315 break;
316 case 8:
317 fmask.bpe = 4;
318 break;
319 default:
320 R600_ERR("Invalid sample count for FMASK allocation.\n");
321 return;
322 }
323
324 /* Overallocate FMASK on R600-R700 to fix colorbuffer corruption.
325 * This can be fixed by writing a separate FMASK allocator specifically
326 * for R600-R700 asics. */
327 if (rscreen->b.chip_class <= R700) {
328 fmask.bpe *= 2;
329 }
330
331 if (rscreen->b.ws->surface_init(rscreen->b.ws, &fmask)) {
332 R600_ERR("Got error in surface_init while allocating FMASK.\n");
333 return;
334 }
335
336 assert(fmask.level[0].mode == RADEON_SURF_MODE_2D);
337
338 out->slice_tile_max = (fmask.level[0].nblk_x * fmask.level[0].nblk_y) / 64;
339 if (out->slice_tile_max)
340 out->slice_tile_max -= 1;
341
342 out->bank_height = fmask.bankh;
343 out->alignment = MAX2(256, fmask.bo_alignment);
344 out->size = fmask.bo_size;
345 }
346
347 static void r600_texture_allocate_fmask(struct r600_screen *rscreen,
348 struct r600_texture *rtex)
349 {
350 struct r600_fmask_info fmask;
351
352 r600_texture_get_fmask_info(rscreen, rtex,
353 rtex->resource.b.b.nr_samples, &fmask);
354
355 rtex->fmask.bank_height = fmask.bank_height;
356 rtex->fmask.slice_tile_max = fmask.slice_tile_max;
357 rtex->fmask.offset = align(rtex->size, fmask.alignment);
358 rtex->fmask.size = fmask.size;
359 rtex->size = rtex->fmask.offset + rtex->fmask.size;
360 #if 0
361 printf("FMASK width=%u, height=%i, bits=%u, size=%u\n",
362 fmask.npix_x, fmask.npix_y, fmask.bpe * fmask.nsamples, rtex->fmask_size);
363 #endif
364 }
365
366 void r600_texture_get_cmask_info(struct r600_screen *rscreen,
367 struct r600_texture *rtex,
368 struct r600_cmask_info *out)
369 {
370 unsigned cmask_tile_width = 8;
371 unsigned cmask_tile_height = 8;
372 unsigned cmask_tile_elements = cmask_tile_width * cmask_tile_height;
373 unsigned element_bits = 4;
374 unsigned cmask_cache_bits = 1024;
375 unsigned num_pipes = rscreen->tiling_info.num_channels;
376 unsigned pipe_interleave_bytes = rscreen->tiling_info.group_bytes;
377
378 unsigned elements_per_macro_tile = (cmask_cache_bits / element_bits) * num_pipes;
379 unsigned pixels_per_macro_tile = elements_per_macro_tile * cmask_tile_elements;
380 unsigned sqrt_pixels_per_macro_tile = sqrt(pixels_per_macro_tile);
381 unsigned macro_tile_width = util_next_power_of_two(sqrt_pixels_per_macro_tile);
382 unsigned macro_tile_height = pixels_per_macro_tile / macro_tile_width;
383
384 unsigned pitch_elements = align(rtex->surface.npix_x, macro_tile_width);
385 unsigned height = align(rtex->surface.npix_y, macro_tile_height);
386
387 unsigned base_align = num_pipes * pipe_interleave_bytes;
388 unsigned slice_bytes =
389 ((pitch_elements * height * element_bits + 7) / 8) / cmask_tile_elements;
390
391 assert(macro_tile_width % 128 == 0);
392 assert(macro_tile_height % 128 == 0);
393
394 out->slice_tile_max = ((pitch_elements * height) / (128*128)) - 1;
395 out->alignment = MAX2(256, base_align);
396 out->size = rtex->surface.array_size * align(slice_bytes, base_align);
397 }
398
399 static void r600_texture_allocate_cmask(struct r600_screen *rscreen,
400 struct r600_texture *rtex)
401 {
402 struct r600_cmask_info cmask;
403
404 r600_texture_get_cmask_info(rscreen, rtex, &cmask);
405
406 rtex->cmask.slice_tile_max = cmask.slice_tile_max;
407 rtex->cmask.offset = align(rtex->size, cmask.alignment);
408 rtex->cmask.size = cmask.size;
409 rtex->size = rtex->cmask.offset + rtex->cmask.size;
410 #if 0
411 printf("CMASK: macro tile width = %u, macro tile height = %u, "
412 "pitch elements = %u, height = %u, slice tile max = %u\n",
413 macro_tile_width, macro_tile_height, pitch_elements, height,
414 rtex->cmask_slice_tile_max);
415 #endif
416 }
417
418 void r600_texture_init_cmask(struct r600_screen *rscreen,
419 struct r600_texture *rtex) {
420 struct r600_cmask_info cmask;
421
422 assert(rtex->cmask.size == 0);
423
424 r600_texture_get_cmask_info(rscreen, rtex, &cmask);
425 rtex->cmask.slice_tile_max = cmask.slice_tile_max;
426 rtex->cmask.offset = 0;
427 rtex->cmask.size = cmask.size;
428 rtex->cmask_buffer = (struct r600_resource *)pipe_buffer_create(&rscreen->b.b,
429 PIPE_BIND_CUSTOM, PIPE_USAGE_STATIC, rtex->cmask.size);
430
431 if (rtex->cmask_buffer == NULL) {
432 rtex->cmask.size = 0;
433 }
434 }
435
436 static struct r600_texture *
437 r600_texture_create_object(struct pipe_screen *screen,
438 const struct pipe_resource *base,
439 unsigned pitch_in_bytes_override,
440 struct pb_buffer *buf,
441 struct radeon_surface *surface)
442 {
443 struct r600_texture *rtex;
444 struct r600_resource *resource;
445 struct r600_screen *rscreen = (struct r600_screen*)screen;
446 int r;
447
448 rtex = CALLOC_STRUCT(r600_texture);
449 if (rtex == NULL)
450 return NULL;
451
452 resource = &rtex->resource;
453 resource->b.b = *base;
454 resource->b.vtbl = &r600_texture_vtbl;
455 pipe_reference_init(&resource->b.b.reference, 1);
456 resource->b.b.screen = screen;
457 rtex->pitch_override = pitch_in_bytes_override;
458
459 /* don't include stencil-only formats which we don't support for rendering */
460 rtex->is_depth = util_format_has_depth(util_format_description(rtex->resource.b.b.format));
461
462 rtex->surface = *surface;
463 r = r600_setup_surface(screen, rtex, pitch_in_bytes_override);
464 if (r) {
465 FREE(rtex);
466 return NULL;
467 }
468
469 rtex->cmask_buffer = NULL;
470 if (base->nr_samples > 1 && !rtex->is_depth && !buf) {
471 r600_texture_allocate_fmask(rscreen, rtex);
472 r600_texture_allocate_cmask(rscreen, rtex);
473 rtex->cmask_buffer = &rtex->resource;
474 }
475
476 if (!rtex->is_depth && base->nr_samples > 1 &&
477 (!rtex->fmask.size || !rtex->cmask.size)) {
478 FREE(rtex);
479 return NULL;
480 }
481
482 /* Tiled depth textures utilize the non-displayable tile order. */
483 rtex->non_disp_tiling = rtex->is_depth && rtex->surface.level[0].mode >= RADEON_SURF_MODE_1D;
484
485 /* only enable hyperz for PIPE_TEXTURE_2D not for PIPE_TEXTURE_2D_ARRAY
486 * Thought it might still be interessting to use hyperz for texture
487 * array without using fast clear features
488 */
489 rtex->htile = NULL;
490 if (!(base->flags & (R600_RESOURCE_FLAG_TRANSFER | R600_RESOURCE_FLAG_FLUSHED_DEPTH)) &&
491 util_format_is_depth_or_stencil(base->format) &&
492 rscreen->b.info.drm_minor >= 26 &&
493 !(rscreen->debug_flags & DBG_NO_HYPERZ) &&
494 base->target == PIPE_TEXTURE_2D &&
495 rtex->surface.level[0].nblk_x >= 32 &&
496 rtex->surface.level[0].nblk_y >= 32) {
497 unsigned sw = rtex->surface.level[0].nblk_x * rtex->surface.blk_w;
498 unsigned sh = rtex->surface.level[0].nblk_y * rtex->surface.blk_h;
499 unsigned htile_size;
500 unsigned npipes = rscreen->b.info.r600_num_tile_pipes;
501
502 /* this alignment and htile size only apply to linear htile buffer */
503 sw = align(sw, 16 << 3);
504 sh = align(sh, npipes << 3);
505 htile_size = (sw >> 3) * (sh >> 3) * 4;
506 /* must be aligned with 2K * npipes */
507 htile_size = align(htile_size, (2 << 10) * npipes);
508
509 rtex->htile = (struct r600_resource*)pipe_buffer_create(&rscreen->b.b, PIPE_BIND_CUSTOM,
510 PIPE_USAGE_STATIC, htile_size);
511 if (rtex->htile == NULL) {
512 /* this is not a fatal error as we can still keep rendering
513 * without htile buffer
514 */
515 R600_ERR("r600: failed to create bo for htile buffers\n");
516 } else {
517 r600_screen_clear_buffer(rscreen, &rtex->htile->b.b, 0, htile_size, 0);
518 }
519 }
520
521 /* Now create the backing buffer. */
522 if (!buf) {
523 unsigned base_align = rtex->surface.bo_alignment;
524 unsigned usage = rtex->surface.level[0].mode >= RADEON_SURF_MODE_1D ?
525 PIPE_USAGE_STATIC : base->usage;
526
527 if (!r600_init_resource(rscreen, resource, rtex->size, base_align, FALSE, usage)) {
528 FREE(rtex);
529 return NULL;
530 }
531 } else {
532 /* This is usually the window framebuffer. We want it in VRAM, always. */
533 resource->buf = buf;
534 resource->cs_buf = rscreen->b.ws->buffer_get_cs_handle(buf);
535 resource->domains = RADEON_DOMAIN_VRAM;
536 }
537
538 if (rtex->cmask.size) {
539 /* Initialize the cmask to 0xCC (= compressed state). */
540 r600_screen_clear_buffer(rscreen, &rtex->cmask_buffer->b.b,
541 rtex->cmask.offset, rtex->cmask.size, 0xCC);
542 }
543
544 if (rscreen->debug_flags & DBG_VM) {
545 fprintf(stderr, "VM start=0x%llX end=0x%llX | Texture %ix%ix%i, %i levels, %i samples, %s\n",
546 r600_resource_va(screen, &rtex->resource.b.b),
547 r600_resource_va(screen, &rtex->resource.b.b) + rtex->resource.buf->size,
548 base->width0, base->height0, util_max_layer(base, 0)+1, base->last_level+1,
549 base->nr_samples ? base->nr_samples : 1, util_format_short_name(base->format));
550 }
551
552 if (rscreen->debug_flags & DBG_TEX_DEPTH && rtex->is_depth && rtex->non_disp_tiling) {
553 printf("Texture: npix_x=%u, npix_y=%u, npix_z=%u, blk_w=%u, "
554 "blk_h=%u, blk_d=%u, array_size=%u, last_level=%u, "
555 "bpe=%u, nsamples=%u, flags=%u\n",
556 rtex->surface.npix_x, rtex->surface.npix_y,
557 rtex->surface.npix_z, rtex->surface.blk_w,
558 rtex->surface.blk_h, rtex->surface.blk_d,
559 rtex->surface.array_size, rtex->surface.last_level,
560 rtex->surface.bpe, rtex->surface.nsamples,
561 rtex->surface.flags);
562 if (rtex->surface.flags & RADEON_SURF_ZBUFFER) {
563 for (int i = 0; i <= rtex->surface.last_level; i++) {
564 printf(" Z %i: offset=%llu, slice_size=%llu, npix_x=%u, "
565 "npix_y=%u, npix_z=%u, nblk_x=%u, nblk_y=%u, "
566 "nblk_z=%u, pitch_bytes=%u, mode=%u\n",
567 i, (unsigned long long)rtex->surface.level[i].offset,
568 (unsigned long long)rtex->surface.level[i].slice_size,
569 u_minify(rtex->resource.b.b.width0, i),
570 u_minify(rtex->resource.b.b.height0, i),
571 u_minify(rtex->resource.b.b.depth0, i),
572 rtex->surface.level[i].nblk_x,
573 rtex->surface.level[i].nblk_y,
574 rtex->surface.level[i].nblk_z,
575 rtex->surface.level[i].pitch_bytes,
576 rtex->surface.level[i].mode);
577 }
578 }
579 if (rtex->surface.flags & RADEON_SURF_SBUFFER) {
580 for (int i = 0; i <= rtex->surface.last_level; i++) {
581 printf(" S %i: offset=%llu, slice_size=%llu, npix_x=%u, "
582 "npix_y=%u, npix_z=%u, nblk_x=%u, nblk_y=%u, "
583 "nblk_z=%u, pitch_bytes=%u, mode=%u\n",
584 i, (unsigned long long)rtex->surface.stencil_level[i].offset,
585 (unsigned long long)rtex->surface.stencil_level[i].slice_size,
586 u_minify(rtex->resource.b.b.width0, i),
587 u_minify(rtex->resource.b.b.height0, i),
588 u_minify(rtex->resource.b.b.depth0, i),
589 rtex->surface.stencil_level[i].nblk_x,
590 rtex->surface.stencil_level[i].nblk_y,
591 rtex->surface.stencil_level[i].nblk_z,
592 rtex->surface.stencil_level[i].pitch_bytes,
593 rtex->surface.stencil_level[i].mode);
594 }
595 }
596 }
597 return rtex;
598 }
599
600 struct pipe_resource *r600_texture_create(struct pipe_screen *screen,
601 const struct pipe_resource *templ)
602 {
603 struct r600_screen *rscreen = (struct r600_screen*)screen;
604 struct radeon_surface surface;
605 const struct util_format_description *desc = util_format_description(templ->format);
606 unsigned array_mode;
607 int r;
608
609 /* Default tiling mode for staging textures. */
610 array_mode = V_038000_ARRAY_LINEAR_ALIGNED;
611
612 /* Tiling doesn't work with the 422 (SUBSAMPLED) formats. That's not an issue,
613 * because 422 formats are used for videos, which prefer linear buffers
614 * for fast uploads anyway. */
615 if (!(templ->flags & R600_RESOURCE_FLAG_TRANSFER) &&
616 (desc->layout != UTIL_FORMAT_LAYOUT_SUBSAMPLED) &&
617 !(templ->bind & PIPE_BIND_LINEAR)) {
618 if (templ->flags & R600_RESOURCE_FLAG_FORCE_TILING) {
619 array_mode = V_038000_ARRAY_2D_TILED_THIN1;
620 } else if (!(templ->bind & PIPE_BIND_SCANOUT) &&
621 templ->usage != PIPE_USAGE_STAGING &&
622 templ->usage != PIPE_USAGE_STREAM &&
623 templ->target != PIPE_TEXTURE_1D &&
624 templ->target != PIPE_TEXTURE_1D_ARRAY &&
625 templ->height0 > 3) {
626 array_mode = V_038000_ARRAY_2D_TILED_THIN1;
627 } else if (util_format_is_compressed(templ->format)) {
628 array_mode = V_038000_ARRAY_1D_TILED_THIN1;
629 }
630 }
631
632 r = r600_init_surface(rscreen, &surface, templ, array_mode,
633 templ->flags & R600_RESOURCE_FLAG_FLUSHED_DEPTH);
634 if (r) {
635 return NULL;
636 }
637 r = rscreen->b.ws->surface_best(rscreen->b.ws, &surface);
638 if (r) {
639 return NULL;
640 }
641 return (struct pipe_resource *)r600_texture_create_object(screen, templ,
642 0, NULL, &surface);
643 }
644
645 struct pipe_resource *r600_texture_from_handle(struct pipe_screen *screen,
646 const struct pipe_resource *templ,
647 struct winsys_handle *whandle)
648 {
649 struct r600_screen *rscreen = (struct r600_screen*)screen;
650 struct pb_buffer *buf = NULL;
651 unsigned stride = 0;
652 unsigned array_mode = 0;
653 enum radeon_bo_layout micro, macro;
654 struct radeon_surface surface;
655 int r;
656
657 /* Support only 2D textures without mipmaps */
658 if ((templ->target != PIPE_TEXTURE_2D && templ->target != PIPE_TEXTURE_RECT) ||
659 templ->depth0 != 1 || templ->last_level != 0)
660 return NULL;
661
662 buf = rscreen->b.ws->buffer_from_handle(rscreen->b.ws, whandle, &stride);
663 if (!buf)
664 return NULL;
665
666 rscreen->b.ws->buffer_get_tiling(buf, &micro, &macro,
667 &surface.bankw, &surface.bankh,
668 &surface.tile_split,
669 &surface.stencil_tile_split,
670 &surface.mtilea);
671
672 if (macro == RADEON_LAYOUT_TILED)
673 array_mode = V_0280A0_ARRAY_2D_TILED_THIN1;
674 else if (micro == RADEON_LAYOUT_TILED)
675 array_mode = V_0280A0_ARRAY_1D_TILED_THIN1;
676 else
677 array_mode = V_038000_ARRAY_LINEAR_ALIGNED;
678
679 r = r600_init_surface(rscreen, &surface, templ, array_mode, false);
680 if (r) {
681 return NULL;
682 }
683 return (struct pipe_resource *)r600_texture_create_object(screen, templ,
684 stride, buf, &surface);
685 }
686
687 bool r600_init_flushed_depth_texture(struct pipe_context *ctx,
688 struct pipe_resource *texture,
689 struct r600_texture **staging)
690 {
691 struct r600_texture *rtex = (struct r600_texture*)texture;
692 struct pipe_resource resource;
693 struct r600_texture **flushed_depth_texture = staging ?
694 staging : &rtex->flushed_depth_texture;
695
696 if (!staging && rtex->flushed_depth_texture)
697 return true; /* it's ready */
698
699 resource.target = texture->target;
700 resource.format = texture->format;
701 resource.width0 = texture->width0;
702 resource.height0 = texture->height0;
703 resource.depth0 = texture->depth0;
704 resource.array_size = texture->array_size;
705 resource.last_level = texture->last_level;
706 resource.nr_samples = texture->nr_samples;
707 resource.usage = staging ? PIPE_USAGE_STAGING : PIPE_USAGE_STATIC;
708 resource.bind = texture->bind & ~PIPE_BIND_DEPTH_STENCIL;
709 resource.flags = texture->flags | R600_RESOURCE_FLAG_FLUSHED_DEPTH;
710
711 if (staging)
712 resource.flags |= R600_RESOURCE_FLAG_TRANSFER;
713
714 *flushed_depth_texture = (struct r600_texture *)ctx->screen->resource_create(ctx->screen, &resource);
715 if (*flushed_depth_texture == NULL) {
716 R600_ERR("failed to create temporary texture to hold flushed depth\n");
717 return false;
718 }
719
720 (*flushed_depth_texture)->is_flushing_texture = TRUE;
721 (*flushed_depth_texture)->non_disp_tiling = false;
722 return true;
723 }
724
725 /**
726 * Initialize the pipe_resource descriptor to be of the same size as the box,
727 * which is supposed to hold a subregion of the texture "orig" at the given
728 * mipmap level.
729 */
730 static void r600_init_temp_resource_from_box(struct pipe_resource *res,
731 struct pipe_resource *orig,
732 const struct pipe_box *box,
733 unsigned level, unsigned flags)
734 {
735 memset(res, 0, sizeof(*res));
736 res->format = orig->format;
737 res->width0 = box->width;
738 res->height0 = box->height;
739 res->depth0 = 1;
740 res->array_size = 1;
741 res->usage = flags & R600_RESOURCE_FLAG_TRANSFER ? PIPE_USAGE_STAGING : PIPE_USAGE_STATIC;
742 res->flags = flags;
743
744 /* We must set the correct texture target and dimensions for a 3D box. */
745 if (box->depth > 1 && util_max_layer(orig, level) > 0)
746 res->target = orig->target;
747 else
748 res->target = PIPE_TEXTURE_2D;
749
750 switch (res->target) {
751 case PIPE_TEXTURE_1D_ARRAY:
752 case PIPE_TEXTURE_2D_ARRAY:
753 case PIPE_TEXTURE_CUBE_ARRAY:
754 res->array_size = box->depth;
755 break;
756 case PIPE_TEXTURE_3D:
757 res->depth0 = box->depth;
758 break;
759 default:;
760 }
761 }
762
763 static void *r600_texture_transfer_map(struct pipe_context *ctx,
764 struct pipe_resource *texture,
765 unsigned level,
766 unsigned usage,
767 const struct pipe_box *box,
768 struct pipe_transfer **ptransfer)
769 {
770 struct r600_context *rctx = (struct r600_context*)ctx;
771 struct r600_texture *rtex = (struct r600_texture*)texture;
772 struct r600_transfer *trans;
773 boolean use_staging_texture = FALSE;
774 struct r600_resource *buf;
775 unsigned offset = 0;
776 char *map;
777
778 if ((texture->bind & PIPE_BIND_GLOBAL) && texture->target == PIPE_BUFFER) {
779 return r600_compute_global_transfer_map(ctx, texture, level, usage, box, ptransfer);
780 }
781
782 /* We cannot map a tiled texture directly because the data is
783 * in a different order, therefore we do detiling using a blit.
784 *
785 * Also, use a temporary in GTT memory for read transfers, as
786 * the CPU is much happier reading out of cached system memory
787 * than uncached VRAM.
788 */
789 if (rtex->surface.level[level].mode >= RADEON_SURF_MODE_1D) {
790 use_staging_texture = TRUE;
791 }
792
793 /* Use a staging texture for uploads if the underlying BO is busy. */
794 if (!(usage & PIPE_TRANSFER_READ) &&
795 (r600_rings_is_buffer_referenced(rctx, rtex->resource.cs_buf, RADEON_USAGE_READWRITE) ||
796 rctx->b.ws->buffer_is_busy(rtex->resource.buf, RADEON_USAGE_READWRITE))) {
797 use_staging_texture = TRUE;
798 }
799
800 if (texture->flags & R600_RESOURCE_FLAG_TRANSFER) {
801 use_staging_texture = FALSE;
802 }
803
804 if (use_staging_texture && (usage & PIPE_TRANSFER_MAP_DIRECTLY)) {
805 return NULL;
806 }
807
808 trans = CALLOC_STRUCT(r600_transfer);
809 if (trans == NULL)
810 return NULL;
811 trans->transfer.resource = texture;
812 trans->transfer.level = level;
813 trans->transfer.usage = usage;
814 trans->transfer.box = *box;
815
816 if (rtex->is_depth) {
817 struct r600_texture *staging_depth;
818
819 if (rtex->resource.b.b.nr_samples > 1) {
820 /* MSAA depth buffers need to be converted to single sample buffers.
821 *
822 * Mapping MSAA depth buffers can occur if ReadPixels is called
823 * with a multisample GLX visual.
824 *
825 * First downsample the depth buffer to a temporary texture,
826 * then decompress the temporary one to staging.
827 *
828 * Only the region being mapped is transfered.
829 */
830 struct pipe_resource resource;
831
832 r600_init_temp_resource_from_box(&resource, texture, box, level, 0);
833
834 if (!r600_init_flushed_depth_texture(ctx, &resource, &staging_depth)) {
835 R600_ERR("failed to create temporary texture to hold untiled copy\n");
836 FREE(trans);
837 return NULL;
838 }
839
840 if (usage & PIPE_TRANSFER_READ) {
841 struct pipe_resource *temp = ctx->screen->resource_create(ctx->screen, &resource);
842
843 r600_copy_region_with_blit(ctx, temp, 0, 0, 0, 0, texture, level, box);
844 r600_blit_decompress_depth(ctx, (struct r600_texture*)temp, staging_depth,
845 0, 0, 0, box->depth, 0, 0);
846 pipe_resource_reference((struct pipe_resource**)&temp, NULL);
847 }
848 }
849 else {
850 /* XXX: only readback the rectangle which is being mapped? */
851 /* XXX: when discard is true, no need to read back from depth texture */
852 if (!r600_init_flushed_depth_texture(ctx, texture, &staging_depth)) {
853 R600_ERR("failed to create temporary texture to hold untiled copy\n");
854 FREE(trans);
855 return NULL;
856 }
857
858 r600_blit_decompress_depth(ctx, rtex, staging_depth,
859 level, level,
860 box->z, box->z + box->depth - 1,
861 0, 0);
862
863 offset = r600_texture_get_offset(staging_depth, level, box);
864 }
865
866 trans->transfer.stride = staging_depth->surface.level[level].pitch_bytes;
867 trans->transfer.layer_stride = staging_depth->surface.level[level].slice_size;
868 trans->staging = (struct r600_resource*)staging_depth;
869 } else if (use_staging_texture) {
870 struct pipe_resource resource;
871 struct r600_texture *staging;
872
873 r600_init_temp_resource_from_box(&resource, texture, box, level,
874 R600_RESOURCE_FLAG_TRANSFER);
875
876 /* Create the temporary texture. */
877 staging = (struct r600_texture*)ctx->screen->resource_create(ctx->screen, &resource);
878 if (staging == NULL) {
879 R600_ERR("failed to create temporary texture to hold untiled copy\n");
880 FREE(trans);
881 return NULL;
882 }
883 trans->staging = &staging->resource;
884 trans->transfer.stride = staging->surface.level[0].pitch_bytes;
885 trans->transfer.layer_stride = staging->surface.level[0].slice_size;
886 if (usage & PIPE_TRANSFER_READ) {
887 r600_copy_to_staging_texture(ctx, trans);
888 }
889 } else {
890 /* the resource is mapped directly */
891 trans->transfer.stride = rtex->surface.level[level].pitch_bytes;
892 trans->transfer.layer_stride = rtex->surface.level[level].slice_size;
893 offset = r600_texture_get_offset(rtex, level, box);
894 }
895
896 if (trans->staging) {
897 buf = trans->staging;
898 } else {
899 buf = &rtex->resource;
900 }
901
902 if (!(map = r600_buffer_mmap_sync_with_rings(rctx, buf, usage))) {
903 pipe_resource_reference((struct pipe_resource**)&trans->staging, NULL);
904 FREE(trans);
905 return NULL;
906 }
907
908 *ptransfer = &trans->transfer;
909 return map + offset;
910 }
911
912 static void r600_texture_transfer_unmap(struct pipe_context *ctx,
913 struct pipe_transfer* transfer)
914 {
915 struct r600_transfer *rtransfer = (struct r600_transfer*)transfer;
916 struct r600_context *rctx = (struct r600_context*)ctx;
917 struct radeon_winsys_cs_handle *buf;
918 struct pipe_resource *texture = transfer->resource;
919 struct r600_texture *rtex = (struct r600_texture*)texture;
920
921 if ((transfer->resource->bind & PIPE_BIND_GLOBAL) && transfer->resource->target == PIPE_BUFFER) {
922 return r600_compute_global_transfer_unmap(ctx, transfer);
923 }
924
925 if (rtransfer->staging) {
926 buf = ((struct r600_resource *)rtransfer->staging)->cs_buf;
927 } else {
928 buf = ((struct r600_resource *)transfer->resource)->cs_buf;
929 }
930 rctx->b.ws->buffer_unmap(buf);
931
932 if ((transfer->usage & PIPE_TRANSFER_WRITE) && rtransfer->staging) {
933 if (rtex->is_depth && rtex->resource.b.b.nr_samples <= 1) {
934 ctx->resource_copy_region(ctx, texture, transfer->level,
935 transfer->box.x, transfer->box.y, transfer->box.z,
936 &rtransfer->staging->b.b, transfer->level,
937 &transfer->box);
938 } else {
939 r600_copy_from_staging_texture(ctx, rtransfer);
940 }
941 }
942
943 if (rtransfer->staging)
944 pipe_resource_reference((struct pipe_resource**)&rtransfer->staging, NULL);
945
946 FREE(transfer);
947 }
948
949 static const struct u_resource_vtbl r600_texture_vtbl =
950 {
951 r600_texture_get_handle, /* get_handle */
952 r600_texture_destroy, /* resource_destroy */
953 r600_texture_transfer_map, /* transfer_map */
954 NULL, /* transfer_flush_region */
955 r600_texture_transfer_unmap, /* transfer_unmap */
956 NULL /* transfer_inline_write */
957 };