2 * Copyright (c) 2012-2015 Etnaviv Project
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sub license,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the
12 * next paragraph) shall be included in all copies or substantial portions
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
24 * Wladimir J. van der Laan <laanwj@gmail.com>
27 #include "etnaviv_transfer.h"
28 #include "etnaviv_clear_blit.h"
29 #include "etnaviv_context.h"
30 #include "etnaviv_debug.h"
31 #include "etnaviv_etc2.h"
32 #include "etnaviv_screen.h"
34 #include "pipe/p_defines.h"
35 #include "pipe/p_format.h"
36 #include "pipe/p_screen.h"
37 #include "pipe/p_state.h"
38 #include "util/format/u_format.h"
39 #include "util/u_inlines.h"
40 #include "util/u_memory.h"
41 #include "util/u_surface.h"
42 #include "util/u_transfer.h"
44 #include "hw/common_3d.xml.h"
46 #include "drm-uapi/drm_fourcc.h"
48 /* Compute offset into a 1D/2D/3D buffer of a certain box.
49 * This box must be aligned to the block width and height of the
50 * underlying format. */
52 etna_compute_offset(enum pipe_format format
, const struct pipe_box
*box
,
53 size_t stride
, size_t layer_stride
)
55 return box
->z
* layer_stride
+
56 box
->y
/ util_format_get_blockheight(format
) * stride
+
57 box
->x
/ util_format_get_blockwidth(format
) *
58 util_format_get_blocksize(format
);
61 static void etna_patch_data(void *buffer
, const struct pipe_transfer
*ptrans
)
63 struct pipe_resource
*prsc
= ptrans
->resource
;
64 struct etna_resource
*rsc
= etna_resource(prsc
);
65 struct etna_resource_level
*level
= &rsc
->levels
[ptrans
->level
];
67 if (likely(!etna_etc2_needs_patching(prsc
)))
73 /* do have the offsets of blocks to patch? */
74 if (!level
->patch_offsets
) {
75 level
->patch_offsets
= CALLOC_STRUCT(util_dynarray
);
77 etna_etc2_calculate_blocks(buffer
, ptrans
->stride
,
78 ptrans
->box
.width
, ptrans
->box
.height
,
79 prsc
->format
, level
->patch_offsets
);
82 etna_etc2_patch(buffer
, level
->patch_offsets
);
84 level
->patched
= true;
87 static void etna_unpatch_data(void *buffer
, const struct pipe_transfer
*ptrans
)
89 struct pipe_resource
*prsc
= ptrans
->resource
;
90 struct etna_resource
*rsc
= etna_resource(prsc
);
91 struct etna_resource_level
*level
= &rsc
->levels
[ptrans
->level
];
96 etna_etc2_patch(buffer
, level
->patch_offsets
);
98 level
->patched
= false;
102 etna_transfer_unmap(struct pipe_context
*pctx
, struct pipe_transfer
*ptrans
)
104 struct etna_context
*ctx
= etna_context(pctx
);
105 struct etna_transfer
*trans
= etna_transfer(ptrans
);
106 struct etna_resource
*rsc
= etna_resource(ptrans
->resource
);
109 * When writing to a resource that is already in use, replace the resource
110 * with a completely new buffer
111 * and free the old one using a fenced free.
112 * The most tricky case to implement will be: tiled or supertiled surface,
113 * partial write, target not aligned to 4/64. */
114 assert(ptrans
->level
<= rsc
->base
.last_level
);
116 if (rsc
->texture
&& !etna_resource_newer(rsc
, etna_resource(rsc
->texture
)))
117 rsc
= etna_resource(rsc
->texture
); /* switch to using the texture resource */
120 * Temporary resources are always pulled into the CPU domain, must push them
121 * back into GPU domain before the RS execs the blit to the base resource.
124 etna_bo_cpu_fini(etna_resource(trans
->rsc
)->bo
);
126 if (ptrans
->usage
& PIPE_TRANSFER_WRITE
) {
128 /* We have a temporary resource due to either tile status or
129 * tiling format. Write back the updated buffer contents.
130 * FIXME: we need to invalidate the tile status. */
131 etna_copy_resource_box(pctx
, ptrans
->resource
, trans
->rsc
, ptrans
->level
, &ptrans
->box
);
132 } else if (trans
->staging
) {
133 /* map buffer object */
134 struct etna_resource_level
*res_level
= &rsc
->levels
[ptrans
->level
];
136 if (rsc
->layout
== ETNA_LAYOUT_TILED
) {
137 for (unsigned z
= 0; z
< ptrans
->box
.depth
; z
++) {
139 trans
->mapped
+ (ptrans
->box
.z
+ z
) * res_level
->layer_stride
,
140 trans
->staging
+ z
* ptrans
->layer_stride
,
141 ptrans
->box
.x
, ptrans
->box
.y
,
142 res_level
->stride
, ptrans
->box
.width
, ptrans
->box
.height
,
143 ptrans
->stride
, util_format_get_blocksize(rsc
->base
.format
));
145 } else if (rsc
->layout
== ETNA_LAYOUT_LINEAR
) {
146 util_copy_box(trans
->mapped
, rsc
->base
.format
, res_level
->stride
,
147 res_level
->layer_stride
, ptrans
->box
.x
,
148 ptrans
->box
.y
, ptrans
->box
.z
, ptrans
->box
.width
,
149 ptrans
->box
.height
, ptrans
->box
.depth
,
150 trans
->staging
, ptrans
->stride
,
151 ptrans
->layer_stride
, 0, 0, 0 /* src x,y,z */);
153 BUG("unsupported tiling %i", rsc
->layout
);
156 FREE(trans
->staging
);
161 if (rsc
->base
.bind
& PIPE_BIND_SAMPLER_VIEW
) {
162 ctx
->dirty
|= ETNA_DIRTY_TEXTURE_CACHES
;
166 /* We need to have the patched data ready for the GPU. */
167 etna_patch_data(trans
->mapped
, ptrans
);
170 * Transfers without a temporary are only pulled into the CPU domain if they
171 * are not mapped unsynchronized. If they are, must push them back into GPU
172 * domain after CPU access is finished.
174 if (!trans
->rsc
&& !(ptrans
->usage
& PIPE_TRANSFER_UNSYNCHRONIZED
))
175 etna_bo_cpu_fini(rsc
->bo
);
177 if ((ptrans
->resource
->target
== PIPE_BUFFER
) &&
178 (ptrans
->usage
& PIPE_TRANSFER_WRITE
)) {
179 util_range_add(&rsc
->base
,
180 &rsc
->valid_buffer_range
,
182 ptrans
->box
.x
+ ptrans
->box
.width
);
185 pipe_resource_reference(&trans
->rsc
, NULL
);
186 pipe_resource_reference(&ptrans
->resource
, NULL
);
187 slab_free(&ctx
->transfer_pool
, trans
);
191 etna_transfer_map(struct pipe_context
*pctx
, struct pipe_resource
*prsc
,
194 const struct pipe_box
*box
,
195 struct pipe_transfer
**out_transfer
)
197 struct etna_context
*ctx
= etna_context(pctx
);
198 struct etna_screen
*screen
= ctx
->screen
;
199 struct etna_resource
*rsc
= etna_resource(prsc
);
200 struct etna_transfer
*trans
;
201 struct pipe_transfer
*ptrans
;
202 enum pipe_format format
= prsc
->format
;
204 trans
= slab_alloc(&ctx
->transfer_pool
);
208 /* slab_alloc() doesn't zero */
209 memset(trans
, 0, sizeof(*trans
));
212 * Upgrade to UNSYNCHRONIZED if target is PIPE_BUFFER and range is uninitialized.
214 if ((usage
& PIPE_TRANSFER_WRITE
) &&
215 (prsc
->target
== PIPE_BUFFER
) &&
216 !util_ranges_intersect(&rsc
->valid_buffer_range
,
218 box
->x
+ box
->width
)) {
219 usage
|= PIPE_TRANSFER_UNSYNCHRONIZED
;
222 /* Upgrade DISCARD_RANGE to WHOLE_RESOURCE if the whole resource is
223 * being mapped. If we add buffer reallocation to avoid CPU/GPU sync this
224 * check needs to be extended to coherent mappings and shared resources.
226 if ((usage
& PIPE_TRANSFER_DISCARD_RANGE
) &&
227 !(usage
& PIPE_TRANSFER_UNSYNCHRONIZED
) &&
228 prsc
->last_level
== 0 &&
229 prsc
->width0
== box
->width
&&
230 prsc
->height0
== box
->height
&&
231 prsc
->depth0
== box
->depth
&&
232 prsc
->array_size
== 1) {
233 usage
|= PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
;
236 ptrans
= &trans
->base
;
237 pipe_resource_reference(&ptrans
->resource
, prsc
);
238 ptrans
->level
= level
;
239 ptrans
->usage
= usage
;
242 assert(level
<= prsc
->last_level
);
244 if (rsc
->texture
&& !etna_resource_newer(rsc
, etna_resource(rsc
->texture
))) {
245 /* We have a texture resource which is the same age or newer than the
246 * render resource. Use the texture resource, which avoids bouncing
247 * pixels between the two resources, and we can de-tile it in s/w. */
248 rsc
= etna_resource(rsc
->texture
);
249 } else if (rsc
->ts_bo
||
250 (rsc
->layout
!= ETNA_LAYOUT_LINEAR
&&
251 etna_resource_hw_tileable(screen
->specs
.use_blt
, prsc
) &&
252 /* HALIGN 4 resources are incompatible with the resolve engine,
253 * so fall back to using software to detile this resource. */
254 rsc
->halign
!= TEXTURE_HALIGN_FOUR
)) {
255 /* If the surface has tile status, we need to resolve it first.
256 * The strategy we implement here is to use the RS to copy the
257 * depth buffer, filling in the "holes" where the tile status
258 * indicates that it's clear. We also do this for tiled
259 * resources, but only if the RS can blit them. */
260 if (usage
& PIPE_TRANSFER_MAP_DIRECTLY
) {
261 slab_free(&ctx
->transfer_pool
, trans
);
262 BUG("unsupported transfer flags %#x with tile status/tiled layout", usage
);
266 if (prsc
->depth0
> 1 && rsc
->ts_bo
) {
267 slab_free(&ctx
->transfer_pool
, trans
);
268 BUG("resource has depth >1 with tile status");
272 struct pipe_resource templ
= *prsc
;
273 templ
.nr_samples
= 0;
274 templ
.bind
= PIPE_BIND_RENDER_TARGET
;
276 trans
->rsc
= etna_resource_alloc(pctx
->screen
, ETNA_LAYOUT_LINEAR
,
277 DRM_FORMAT_MOD_LINEAR
, &templ
);
279 slab_free(&ctx
->transfer_pool
, trans
);
283 if (!screen
->specs
.use_blt
) {
284 /* Need to align the transfer region to satisfy RS restrictions, as we
285 * really want to hit the RS blit path here.
287 unsigned w_align
, h_align
;
289 if (rsc
->layout
& ETNA_LAYOUT_BIT_SUPER
) {
291 h_align
= 64 * ctx
->screen
->specs
.pixel_pipes
;
293 w_align
= ETNA_RS_WIDTH_MASK
+ 1;
294 h_align
= ETNA_RS_HEIGHT_MASK
+ 1;
297 ptrans
->box
.width
+= ptrans
->box
.x
& (w_align
- 1);
298 ptrans
->box
.x
= ptrans
->box
.x
& ~(w_align
- 1);
299 ptrans
->box
.width
= align(ptrans
->box
.width
, (ETNA_RS_WIDTH_MASK
+ 1));
300 ptrans
->box
.height
+= ptrans
->box
.y
& (h_align
- 1);
301 ptrans
->box
.y
= ptrans
->box
.y
& ~(h_align
- 1);
302 ptrans
->box
.height
= align(ptrans
->box
.height
, ETNA_RS_HEIGHT_MASK
+ 1);
305 if (!(usage
& PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
))
306 etna_copy_resource_box(pctx
, trans
->rsc
, prsc
, level
, &ptrans
->box
);
308 /* Switch to using the temporary resource instead */
309 rsc
= etna_resource(trans
->rsc
);
312 struct etna_resource_level
*res_level
= &rsc
->levels
[level
];
314 /* XXX we don't handle PIPE_TRANSFER_FLUSH_EXPLICIT; this flag can be ignored
315 * when mapping in-place,
316 * but when not in place we need to fire off the copy operation in
317 * transfer_flush_region (currently
318 * a no-op) instead of unmap. Need to handle this to support
319 * ARB_map_buffer_range extension at least.
321 /* XXX we don't take care of current operations on the resource; which can
322 be, at some point in the pipeline
323 which is not yet executed:
326 - bound through vertex buffer
327 - bound through index buffer
328 - bound in sampler view
329 - used in clear_render_target / clear_depth_stencil operation
331 - used in resource_copy_region
333 How do other drivers record this information over course of the rendering
335 Is it necessary at all? Only in case we want to provide a fast path and
336 map the resource directly
337 (and for PIPE_TRANSFER_MAP_DIRECTLY) and we don't want to force a sync.
338 We also need to know whether the resource is in use to determine if a sync
339 is needed (or just do it
340 always, but that comes at the expense of performance).
342 A conservative approximation without too much overhead would be to mark
343 all resources that have
344 been bound at some point as busy. A drawback would be that accessing
346 been bound but are no longer in use for a while still carry a performance
347 penalty. On the other hand,
348 the program could be using PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE or
349 PIPE_TRANSFER_UNSYNCHRONIZED to
350 avoid this in the first place...
352 A) We use an in-pipe copy engine, and queue the copy operation after unmap
354 will be performed when all current commands have been executed.
355 Using the RS is possible, not sure if always efficient. This can also
356 do any kind of tiling for us.
357 Only possible when PIPE_TRANSFER_DISCARD_RANGE is set.
358 B) We discard the entire resource (or at least, the mipmap level) and
359 allocate new memory for it.
360 Only possible when mapping the entire resource or
361 PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE is set.
365 * Pull resources into the CPU domain. Only skipped for unsynchronized
366 * transfers without a temporary resource.
368 if (trans
->rsc
|| !(usage
& PIPE_TRANSFER_UNSYNCHRONIZED
)) {
369 uint32_t prep_flags
= 0;
372 * Always flush if we have the temporary resource and have a copy to this
373 * outstanding. Otherwise infer flush requirement from resource access and
374 * current GPU usage (reads must wait for GPU writes, writes must have
375 * exclusive access to the buffer).
377 mtx_lock(&ctx
->lock
);
379 if ((trans
->rsc
&& (etna_resource(trans
->rsc
)->status
& ETNA_PENDING_WRITE
)) ||
381 (((usage
& PIPE_TRANSFER_READ
) && (rsc
->status
& ETNA_PENDING_WRITE
)) ||
382 ((usage
& PIPE_TRANSFER_WRITE
) && rsc
->status
)))) {
383 set_foreach(rsc
->pending_ctx
, entry
) {
384 struct etna_context
*pend_ctx
= (struct etna_context
*)entry
->key
;
385 struct pipe_context
*pend_pctx
= &pend_ctx
->base
;
387 pend_pctx
->flush(pend_pctx
, NULL
, 0);
391 mtx_unlock(&ctx
->lock
);
393 if (usage
& PIPE_TRANSFER_READ
)
394 prep_flags
|= DRM_ETNA_PREP_READ
;
395 if (usage
& PIPE_TRANSFER_WRITE
)
396 prep_flags
|= DRM_ETNA_PREP_WRITE
;
399 * The ETC2 patching operates in-place on the resource, so the resource will
400 * get written even on read-only transfers. This blocks the GPU to sample
401 * from this resource.
403 if ((usage
& PIPE_TRANSFER_READ
) && etna_etc2_needs_patching(prsc
))
404 prep_flags
|= DRM_ETNA_PREP_WRITE
;
406 if (etna_bo_cpu_prep(rsc
->bo
, prep_flags
))
410 /* map buffer object */
411 trans
->mapped
= etna_bo_map(rsc
->bo
);
415 *out_transfer
= ptrans
;
417 if (rsc
->layout
== ETNA_LAYOUT_LINEAR
) {
418 ptrans
->stride
= res_level
->stride
;
419 ptrans
->layer_stride
= res_level
->layer_stride
;
421 trans
->mapped
+= res_level
->offset
+
422 etna_compute_offset(prsc
->format
, box
, res_level
->stride
,
423 res_level
->layer_stride
);
425 /* We need to have the unpatched data ready for the gfx stack. */
426 if (usage
& PIPE_TRANSFER_READ
)
427 etna_unpatch_data(trans
->mapped
, ptrans
);
429 return trans
->mapped
;
431 unsigned divSizeX
= util_format_get_blockwidth(format
);
432 unsigned divSizeY
= util_format_get_blockheight(format
);
434 /* No direct mappings of tiled, since we need to manually
437 if (usage
& PIPE_TRANSFER_MAP_DIRECTLY
)
440 trans
->mapped
+= res_level
->offset
;
441 ptrans
->stride
= align(box
->width
, divSizeX
) * util_format_get_blocksize(format
); /* row stride in bytes */
442 ptrans
->layer_stride
= align(box
->height
, divSizeY
) * ptrans
->stride
;
443 size_t size
= ptrans
->layer_stride
* box
->depth
;
445 trans
->staging
= MALLOC(size
);
449 if (usage
& PIPE_TRANSFER_READ
) {
450 if (rsc
->layout
== ETNA_LAYOUT_TILED
) {
451 for (unsigned z
= 0; z
< ptrans
->box
.depth
; z
++) {
452 etna_texture_untile(trans
->staging
+ z
* ptrans
->layer_stride
,
453 trans
->mapped
+ (ptrans
->box
.z
+ z
) * res_level
->layer_stride
,
454 ptrans
->box
.x
, ptrans
->box
.y
, res_level
->stride
,
455 ptrans
->box
.width
, ptrans
->box
.height
, ptrans
->stride
,
456 util_format_get_blocksize(rsc
->base
.format
));
458 } else if (rsc
->layout
== ETNA_LAYOUT_LINEAR
) {
459 util_copy_box(trans
->staging
, rsc
->base
.format
, ptrans
->stride
,
460 ptrans
->layer_stride
, 0, 0, 0, /* dst x,y,z */
461 ptrans
->box
.width
, ptrans
->box
.height
,
462 ptrans
->box
.depth
, trans
->mapped
, res_level
->stride
,
463 res_level
->layer_stride
, ptrans
->box
.x
,
464 ptrans
->box
.y
, ptrans
->box
.z
);
466 /* TODO supertiling */
467 BUG("unsupported tiling %i for reading", rsc
->layout
);
471 return trans
->staging
;
475 etna_bo_cpu_fini(rsc
->bo
);
477 etna_transfer_unmap(pctx
, ptrans
);
482 etna_transfer_flush_region(struct pipe_context
*pctx
,
483 struct pipe_transfer
*ptrans
,
484 const struct pipe_box
*box
)
486 struct etna_resource
*rsc
= etna_resource(ptrans
->resource
);
488 if (ptrans
->resource
->target
== PIPE_BUFFER
)
489 util_range_add(&rsc
->base
,
490 &rsc
->valid_buffer_range
,
491 ptrans
->box
.x
+ box
->x
,
492 ptrans
->box
.x
+ box
->x
+ box
->width
);
496 etna_transfer_init(struct pipe_context
*pctx
)
498 pctx
->transfer_map
= etna_transfer_map
;
499 pctx
->transfer_flush_region
= etna_transfer_flush_region
;
500 pctx
->transfer_unmap
= etna_transfer_unmap
;
501 pctx
->buffer_subdata
= u_default_buffer_subdata
;
502 pctx
->texture_subdata
= u_default_texture_subdata
;