2 * Copyright (C) 2008 VMware, Inc.
3 * Copyright (C) 2012 Rob Clark <robclark@freedesktop.org>
4 * Copyright (C) 2014-2017 Broadcom
5 * Copyright (C) 2018-2019 Alyssa Rosenzweig
6 * Copyright (C) 2019 Collabora, Ltd.
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the "Software"),
10 * to deal in the Software without restriction, including without limitation
11 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 * and/or sell copies of the Software, and to permit persons to whom the
13 * Software is furnished to do so, subject to the following conditions:
15 * The above copyright notice and this permission notice (including the next
16 * paragraph) shall be included in all copies or substantial portions of the
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27 * Authors (Collabora):
28 * Tomeu Vizoso <tomeu.vizoso@collabora.com>
29 * Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
35 #include "drm-uapi/drm_fourcc.h"
37 #include "frontend/winsys_handle.h"
38 #include "util/format/u_format.h"
39 #include "util/u_memory.h"
40 #include "util/u_surface.h"
41 #include "util/u_transfer.h"
42 #include "util/u_transfer_helper.h"
43 #include "util/u_gen_mipmap.h"
44 #include "util/u_drm.h"
47 #include "pan_context.h"
48 #include "pan_screen.h"
49 #include "pan_resource.h"
51 #include "pan_tiling.h"
53 #include "panfrost-quirks.h"
55 static struct pipe_resource
*
56 panfrost_resource_from_handle(struct pipe_screen
*pscreen
,
57 const struct pipe_resource
*templat
,
58 struct winsys_handle
*whandle
,
61 struct panfrost_device
*dev
= pan_device(pscreen
);
62 struct panfrost_resource
*rsc
;
63 struct pipe_resource
*prsc
;
65 assert(whandle
->type
== WINSYS_HANDLE_TYPE_FD
);
67 rsc
= rzalloc(pscreen
, struct panfrost_resource
);
75 pipe_reference_init(&prsc
->reference
, 1);
76 prsc
->screen
= pscreen
;
78 rsc
->bo
= panfrost_bo_import(dev
, whandle
->handle
);
79 rsc
->internal_format
= templat
->format
;
80 rsc
->modifier
= (whandle
->modifier
== DRM_FORMAT_MOD_INVALID
) ?
81 DRM_FORMAT_MOD_LINEAR
: whandle
->modifier
;
82 rsc
->slices
[0].stride
= whandle
->stride
;
83 rsc
->slices
[0].offset
= whandle
->offset
;
84 rsc
->slices
[0].initialized
= true;
85 panfrost_resource_set_damage_region(NULL
, &rsc
->base
, 0, NULL
);
87 if (dev
->quirks
& IS_BIFROST
&&
88 templat
->bind
& PIPE_BIND_RENDER_TARGET
) {
89 unsigned size
= panfrost_compute_checksum_size(
90 &rsc
->slices
[0], templat
->width0
, templat
->height0
);
91 rsc
->slices
[0].checksum_bo
= panfrost_bo_create(dev
, size
, 0);
92 rsc
->checksummed
= true;
95 if (drm_is_afbc(whandle
->modifier
)) {
96 rsc
->slices
[0].header_size
=
97 panfrost_afbc_header_size(templat
->width0
, templat
->height0
);
102 renderonly_create_gpu_import_for_resource(prsc
, dev
->ro
, NULL
);
103 /* failure is expected in some cases.. */
110 panfrost_resource_get_handle(struct pipe_screen
*pscreen
,
111 struct pipe_context
*ctx
,
112 struct pipe_resource
*pt
,
113 struct winsys_handle
*handle
,
116 struct panfrost_device
*dev
= pan_device(pscreen
);
117 struct panfrost_resource
*rsrc
= (struct panfrost_resource
*) pt
;
118 struct renderonly_scanout
*scanout
= rsrc
->scanout
;
120 handle
->modifier
= rsrc
->modifier
;
122 if (handle
->type
== WINSYS_HANDLE_TYPE_SHARED
) {
124 } else if (handle
->type
== WINSYS_HANDLE_TYPE_KMS
) {
125 if (renderonly_get_handle(scanout
, handle
))
128 handle
->handle
= rsrc
->bo
->gem_handle
;
129 handle
->stride
= rsrc
->slices
[0].stride
;
130 handle
->offset
= rsrc
->slices
[0].offset
;
132 } else if (handle
->type
== WINSYS_HANDLE_TYPE_FD
) {
134 struct drm_prime_handle args
= {
135 .handle
= scanout
->handle
,
136 .flags
= DRM_CLOEXEC
,
139 int ret
= drmIoctl(dev
->ro
->kms_fd
, DRM_IOCTL_PRIME_HANDLE_TO_FD
, &args
);
143 handle
->stride
= scanout
->stride
;
144 handle
->handle
= args
.fd
;
148 int fd
= panfrost_bo_export(rsrc
->bo
);
154 handle
->stride
= rsrc
->slices
[0].stride
;
155 handle
->offset
= rsrc
->slices
[0].offset
;
164 panfrost_flush_resource(struct pipe_context
*pctx
, struct pipe_resource
*prsc
)
169 static struct pipe_surface
*
170 panfrost_create_surface(struct pipe_context
*pipe
,
171 struct pipe_resource
*pt
,
172 const struct pipe_surface
*surf_tmpl
)
174 struct pipe_surface
*ps
= NULL
;
176 ps
= rzalloc(pipe
, struct pipe_surface
);
179 pipe_reference_init(&ps
->reference
, 1);
180 pipe_resource_reference(&ps
->texture
, pt
);
182 ps
->format
= surf_tmpl
->format
;
184 if (pt
->target
!= PIPE_BUFFER
) {
185 assert(surf_tmpl
->u
.tex
.level
<= pt
->last_level
);
186 ps
->width
= u_minify(pt
->width0
, surf_tmpl
->u
.tex
.level
);
187 ps
->height
= u_minify(pt
->height0
, surf_tmpl
->u
.tex
.level
);
188 ps
->nr_samples
= surf_tmpl
->nr_samples
;
189 ps
->u
.tex
.level
= surf_tmpl
->u
.tex
.level
;
190 ps
->u
.tex
.first_layer
= surf_tmpl
->u
.tex
.first_layer
;
191 ps
->u
.tex
.last_layer
= surf_tmpl
->u
.tex
.last_layer
;
193 /* setting width as number of elements should get us correct renderbuffer width */
194 ps
->width
= surf_tmpl
->u
.buf
.last_element
- surf_tmpl
->u
.buf
.first_element
+ 1;
195 ps
->height
= pt
->height0
;
196 ps
->u
.buf
.first_element
= surf_tmpl
->u
.buf
.first_element
;
197 ps
->u
.buf
.last_element
= surf_tmpl
->u
.buf
.last_element
;
198 assert(ps
->u
.buf
.first_element
<= ps
->u
.buf
.last_element
);
199 assert(ps
->u
.buf
.last_element
< ps
->width
);
207 panfrost_surface_destroy(struct pipe_context
*pipe
,
208 struct pipe_surface
*surf
)
210 assert(surf
->texture
);
211 pipe_resource_reference(&surf
->texture
, NULL
);
215 static struct pipe_resource
*
216 panfrost_create_scanout_res(struct pipe_screen
*screen
,
217 const struct pipe_resource
*template,
220 struct panfrost_device
*dev
= pan_device(screen
);
221 struct renderonly_scanout
*scanout
;
222 struct winsys_handle handle
;
223 struct pipe_resource
*res
;
224 struct pipe_resource scanout_templat
= *template;
226 /* Tiled formats need to be tile aligned */
227 if (modifier
== DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED
) {
228 scanout_templat
.width0
= ALIGN_POT(template->width0
, 16);
229 scanout_templat
.height0
= ALIGN_POT(template->height0
, 16);
232 /* AFBC formats need a header. Thankfully we don't care about the
233 * stride so we can just use wonky dimensions as long as the right
234 * number of bytes are allocated at the end of the day... this implies
235 * that stride/pitch is invalid for AFBC buffers */
237 if (drm_is_afbc(modifier
)) {
238 /* Space for the header. We need to keep vaguely similar
239 * dimensions because... reasons... to allocate with renderonly
240 * as a dumb buffer. To do so, after the usual 16x16 alignment,
241 * we add on extra rows for the header. The order of operations
242 * matters here, the extra rows of padding can in fact be
243 * needed and missing them can lead to faults. */
245 unsigned header_size
= panfrost_afbc_header_size(
246 template->width0
, template->height0
);
248 unsigned pitch
= ALIGN_POT(template->width0
, 16) *
249 util_format_get_blocksize(template->format
);
251 unsigned header_rows
=
252 DIV_ROUND_UP(header_size
, pitch
);
254 scanout_templat
.width0
= ALIGN_POT(template->width0
, 16);
255 scanout_templat
.height0
= ALIGN_POT(template->height0
, 16) + header_rows
;
258 scanout
= renderonly_scanout_for_resource(&scanout_templat
,
263 assert(handle
.type
== WINSYS_HANDLE_TYPE_FD
);
264 handle
.modifier
= modifier
;
265 res
= screen
->resource_from_handle(screen
, template, &handle
,
266 PIPE_HANDLE_USAGE_FRAMEBUFFER_WRITE
);
267 close(handle
.handle
);
271 struct panfrost_resource
*pres
= pan_resource(res
);
273 pres
->scanout
= scanout
;
278 /* Setup the mip tree given a particular modifier, possibly with checksumming */
281 panfrost_setup_slices(struct panfrost_resource
*pres
, size_t *bo_size
)
283 struct pipe_resource
*res
= &pres
->base
;
284 unsigned width
= res
->width0
;
285 unsigned height
= res
->height0
;
286 unsigned depth
= res
->depth0
;
287 unsigned bytes_per_pixel
= util_format_get_blocksize(pres
->internal_format
);
289 /* MSAA is implemented as a 3D texture with z corresponding to the
290 * sample #, horrifyingly enough */
292 bool msaa
= res
->nr_samples
> 1;
296 depth
= res
->nr_samples
;
301 /* Tiled operates blockwise; linear is packed. Also, anything
302 * we render to has to be tile-aligned. Maybe not strictly
303 * necessary, but we're not *that* pressed for memory and it
304 * makes code a lot simpler */
306 bool renderable
= res
->bind
&
307 (PIPE_BIND_RENDER_TARGET
| PIPE_BIND_DEPTH_STENCIL
) &&
308 res
->target
!= PIPE_BUFFER
;
309 bool afbc
= drm_is_afbc(pres
->modifier
);
310 bool tiled
= pres
->modifier
== DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED
;
311 bool linear
= pres
->modifier
== DRM_FORMAT_MOD_LINEAR
;
312 bool should_align
= renderable
|| tiled
|| afbc
;
314 /* We don't know how to specify a 2D stride for 3D textures */
316 bool can_align_stride
=
317 res
->target
!= PIPE_TEXTURE_3D
;
319 should_align
&= can_align_stride
;
322 unsigned size_2d
= 0;
324 for (unsigned l
= 0; l
<= res
->last_level
; ++l
) {
325 struct panfrost_slice
*slice
= &pres
->slices
[l
];
327 unsigned effective_width
= width
;
328 unsigned effective_height
= height
;
329 unsigned effective_depth
= depth
;
332 effective_width
= ALIGN_POT(effective_width
, 16);
333 effective_height
= ALIGN_POT(effective_height
, 16);
335 /* We don't need to align depth */
338 /* Align levels to cache-line as a performance improvement for
339 * linear/tiled and as a requirement for AFBC */
341 offset
= ALIGN_POT(offset
, 64);
343 slice
->offset
= offset
;
345 /* Compute the would-be stride */
346 unsigned stride
= bytes_per_pixel
* effective_width
;
348 if (util_format_is_compressed(pres
->internal_format
))
351 /* ..but cache-line align it for performance */
352 if (can_align_stride
&& linear
)
353 stride
= ALIGN_POT(stride
, 64);
355 slice
->stride
= stride
;
357 unsigned slice_one_size
= slice
->stride
* effective_height
;
358 unsigned slice_full_size
= slice_one_size
* effective_depth
;
360 slice
->size0
= slice_one_size
;
362 /* Report 2D size for 3D texturing */
365 size_2d
= slice_one_size
;
367 /* Compute AFBC sizes if necessary */
370 panfrost_afbc_header_size(width
, height
);
372 offset
+= slice
->header_size
;
375 offset
+= slice_full_size
;
377 /* Add a checksum region if necessary */
378 if (pres
->checksummed
) {
379 slice
->checksum_offset
= offset
;
381 unsigned size
= panfrost_compute_checksum_size(
382 slice
, width
, height
);
387 width
= u_minify(width
, 1);
388 height
= u_minify(height
, 1);
390 /* Don't mipmap the sample count */
392 depth
= u_minify(depth
, 1);
395 assert(res
->array_size
);
397 if (res
->target
!= PIPE_TEXTURE_3D
) {
398 /* Arrays and cubemaps have the entire miptree duplicated */
400 pres
->cubemap_stride
= ALIGN_POT(offset
, 64);
401 *bo_size
= ALIGN_POT(pres
->cubemap_stride
* res
->array_size
, 4096);
403 /* 3D strides across the 2D layers */
404 assert(res
->array_size
== 1);
406 pres
->cubemap_stride
= size_2d
;
407 *bo_size
= ALIGN_POT(offset
, 4096);
411 /* Based on the usage, determine if it makes sense to use u-inteleaved tiling.
412 * We only have routines to tile 2D textures of sane bpps. On the hardware
413 * level, not all usages are valid for tiling. Finally, if the app is hinting
414 * that the contents frequently change, tiling will be a loss.
416 * Due to incomplete information on some platforms, we may need to force tiling
419 * On platforms where it is supported, AFBC is even better. */
422 panfrost_can_linear(struct panfrost_device
*dev
, const struct panfrost_resource
*pres
)
424 /* XXX: We should be able to do linear Z/S with the right bits.. */
425 return !((pres
->base
.bind
& PIPE_BIND_DEPTH_STENCIL
) &&
426 (dev
->quirks
& (MIDGARD_SFBD
| IS_BIFROST
)));
430 panfrost_should_afbc(struct panfrost_device
*dev
, const struct panfrost_resource
*pres
)
432 /* AFBC resources may be rendered to, textured from, or shared across
433 * processes, but may not be used as e.g buffers */
434 const unsigned valid_binding
=
435 PIPE_BIND_DEPTH_STENCIL
|
436 PIPE_BIND_RENDER_TARGET
|
437 PIPE_BIND_BLENDABLE
|
438 PIPE_BIND_SAMPLER_VIEW
|
439 PIPE_BIND_DISPLAY_TARGET
|
443 if (pres
->base
.bind
& ~valid_binding
)
446 /* AFBC introduced with Mali T760 */
447 if (dev
->quirks
& MIDGARD_NO_AFBC
)
450 /* AFBC<-->staging is expensive */
451 if (pres
->base
.usage
== PIPE_USAGE_STREAM
)
454 /* Only a small selection of formats are AFBC'able */
455 if (!panfrost_format_supports_afbc(pres
->internal_format
))
458 /* AFBC does not support layered (GLES3 style) multisampling. Use
459 * EXT_multisampled_render_to_texture instead */
460 if (pres
->base
.nr_samples
> 1)
463 /* TODO: Is AFBC of 3D textures possible? */
464 if ((pres
->base
.target
!= PIPE_TEXTURE_2D
) && (pres
->base
.target
!= PIPE_TEXTURE_RECT
))
467 /* For one tile, AFBC is a loss compared to u-interleaved */
468 if (pres
->base
.width0
<= 16 && pres
->base
.height0
<= 16)
471 /* Otherwise, we'd prefer AFBC as it is dramatically more efficient
472 * than linear or usually even u-interleaved */
477 panfrost_should_tile(struct panfrost_device
*dev
, const struct panfrost_resource
*pres
)
479 const unsigned valid_binding
=
480 PIPE_BIND_DEPTH_STENCIL
|
481 PIPE_BIND_RENDER_TARGET
|
482 PIPE_BIND_BLENDABLE
|
483 PIPE_BIND_SAMPLER_VIEW
|
484 PIPE_BIND_DISPLAY_TARGET
|
488 unsigned bpp
= util_format_get_blocksizebits(pres
->internal_format
);
491 bpp
== 8 || bpp
== 16 || bpp
== 24 || bpp
== 32 ||
492 bpp
== 64 || bpp
== 128;
494 bool is_2d
= (pres
->base
.target
== PIPE_TEXTURE_2D
)
495 || (pres
->base
.target
== PIPE_TEXTURE_RECT
);
497 bool can_tile
= is_2d
&& is_sane_bpp
&& ((pres
->base
.bind
& ~valid_binding
) == 0);
499 if (!panfrost_can_linear(dev
, pres
)) {
504 return can_tile
&& (pres
->base
.usage
!= PIPE_USAGE_STREAM
);
508 panfrost_best_modifier(struct panfrost_device
*dev
,
509 const struct panfrost_resource
*pres
)
511 if (panfrost_should_afbc(dev
, pres
)) {
513 AFBC_FORMAT_MOD_BLOCK_SIZE_16x16
|
514 AFBC_FORMAT_MOD_SPARSE
;
516 if (panfrost_afbc_can_ytr(pres
->base
.format
))
517 afbc
|= AFBC_FORMAT_MOD_YTR
;
519 return DRM_FORMAT_MOD_ARM_AFBC(afbc
);
520 } else if (panfrost_should_tile(dev
, pres
))
521 return DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED
;
523 return DRM_FORMAT_MOD_LINEAR
;
527 panfrost_resource_create_bo(struct panfrost_device
*dev
, struct panfrost_resource
*pres
,
530 struct pipe_resource
*res
= &pres
->base
;
532 pres
->modifier
= (modifier
!= DRM_FORMAT_MOD_INVALID
) ? modifier
:
533 panfrost_best_modifier(dev
, pres
);
534 pres
->checksummed
= (res
->bind
& PIPE_BIND_RENDER_TARGET
);
536 /* We can only switch tiled->linear if the resource isn't already
537 * linear, and if we control the modifier, and if the resource can be
539 pres
->modifier_constant
= !((pres
->modifier
!= DRM_FORMAT_MOD_LINEAR
)
540 && (modifier
== DRM_FORMAT_INVALID
)
541 && panfrost_can_linear(dev
, pres
));
545 panfrost_setup_slices(pres
, &bo_size
);
547 /* We create a BO immediately but don't bother mapping, since we don't
548 * care to map e.g. FBOs which the CPU probably won't touch */
549 pres
->bo
= panfrost_bo_create(dev
, bo_size
, PAN_BO_DELAY_MMAP
);
553 panfrost_resource_set_damage_region(struct pipe_screen
*screen
,
554 struct pipe_resource
*res
,
556 const struct pipe_box
*rects
)
558 struct panfrost_resource
*pres
= pan_resource(res
);
559 struct pipe_scissor_state
*damage_extent
= &pres
->damage
.extent
;
562 if (pres
->damage
.inverted_rects
)
563 ralloc_free(pres
->damage
.inverted_rects
);
565 memset(&pres
->damage
, 0, sizeof(pres
->damage
));
567 pres
->damage
.inverted_rects
=
568 pan_subtract_damage(pres
,
569 res
->width0
, res
->height0
,
570 nrects
, rects
, &pres
->damage
.inverted_len
);
572 /* Track the damage extent: the quad including all damage regions. Will
573 * be used restrict the rendering area */
575 damage_extent
->minx
= 0xffff;
576 damage_extent
->miny
= 0xffff;
578 for (i
= 0; i
< nrects
; i
++) {
579 int x
= rects
[i
].x
, w
= rects
[i
].width
, h
= rects
[i
].height
;
580 int y
= res
->height0
- (rects
[i
].y
+ h
);
582 damage_extent
->minx
= MIN2(damage_extent
->minx
, x
);
583 damage_extent
->miny
= MIN2(damage_extent
->miny
, y
);
584 damage_extent
->maxx
= MAX2(damage_extent
->maxx
,
585 MIN2(x
+ w
, res
->width0
));
586 damage_extent
->maxy
= MAX2(damage_extent
->maxy
,
587 MIN2(y
+ h
, res
->height0
));
591 damage_extent
->minx
= 0;
592 damage_extent
->miny
= 0;
593 damage_extent
->maxx
= res
->width0
;
594 damage_extent
->maxy
= res
->height0
;
599 static struct pipe_resource
*
600 panfrost_resource_create_with_modifier(struct pipe_screen
*screen
,
601 const struct pipe_resource
*template,
604 struct panfrost_device
*dev
= pan_device(screen
);
606 /* Make sure we're familiar */
607 switch (template->target
) {
609 case PIPE_TEXTURE_1D
:
610 case PIPE_TEXTURE_2D
:
611 case PIPE_TEXTURE_3D
:
612 case PIPE_TEXTURE_CUBE
:
613 case PIPE_TEXTURE_RECT
:
614 case PIPE_TEXTURE_1D_ARRAY
:
615 case PIPE_TEXTURE_2D_ARRAY
:
618 unreachable("Unknown texture target\n");
621 if (dev
->ro
&& (template->bind
&
622 (PIPE_BIND_DISPLAY_TARGET
| PIPE_BIND_SCANOUT
| PIPE_BIND_SHARED
)))
623 return panfrost_create_scanout_res(screen
, template, modifier
);
625 struct panfrost_resource
*so
= rzalloc(screen
, struct panfrost_resource
);
626 so
->base
= *template;
627 so
->base
.screen
= screen
;
628 so
->internal_format
= template->format
;
630 pipe_reference_init(&so
->base
.reference
, 1);
632 util_range_init(&so
->valid_buffer_range
);
634 panfrost_resource_create_bo(dev
, so
, modifier
);
635 panfrost_resource_set_damage_region(NULL
, &so
->base
, 0, NULL
);
637 if (template->bind
& PIPE_BIND_INDEX_BUFFER
)
638 so
->index_cache
= rzalloc(so
, struct panfrost_minmax_cache
);
640 return (struct pipe_resource
*)so
;
643 /* Default is to create a resource as don't care */
645 static struct pipe_resource
*
646 panfrost_resource_create(struct pipe_screen
*screen
,
647 const struct pipe_resource
*template)
649 return panfrost_resource_create_with_modifier(screen
, template,
650 DRM_FORMAT_MOD_INVALID
);
653 /* If no modifier is specified, we'll choose. Otherwise, the order of
654 * preference is compressed, tiled, linear. */
656 static struct pipe_resource
*
657 panfrost_resource_create_with_modifiers(struct pipe_screen
*screen
,
658 const struct pipe_resource
*template,
659 const uint64_t *modifiers
, int count
)
661 for (unsigned i
= 0; i
< PAN_MODIFIER_COUNT
; ++i
) {
662 if (drm_find_modifier(pan_best_modifiers
[i
], modifiers
, count
)) {
663 return panfrost_resource_create_with_modifier(screen
, template,
664 pan_best_modifiers
[i
]);
668 /* If we didn't find one, app specified invalid */
669 assert(count
== 1 && modifiers
[0] == DRM_FORMAT_MOD_INVALID
);
670 return panfrost_resource_create(screen
, template);
674 panfrost_resource_destroy(struct pipe_screen
*screen
,
675 struct pipe_resource
*pt
)
677 struct panfrost_device
*dev
= pan_device(screen
);
678 struct panfrost_resource
*rsrc
= (struct panfrost_resource
*) pt
;
681 renderonly_scanout_destroy(rsrc
->scanout
, dev
->ro
);
684 panfrost_bo_unreference(rsrc
->bo
);
686 if (rsrc
->slices
[0].checksum_bo
)
687 panfrost_bo_unreference(rsrc
->slices
[0].checksum_bo
);
689 util_range_destroy(&rsrc
->valid_buffer_range
);
693 /* Most of the time we can do CPU-side transfers, but sometimes we need to use
694 * the 3D pipe for this. Let's wrap u_blitter to blit to/from staging textures.
695 * Code adapted from freedreno */
697 static struct panfrost_resource
*
698 pan_alloc_staging(struct panfrost_context
*ctx
, struct panfrost_resource
*rsc
,
699 unsigned level
, const struct pipe_box
*box
)
701 struct pipe_context
*pctx
= &ctx
->base
;
702 struct pipe_resource tmpl
= rsc
->base
;
704 tmpl
.width0
= box
->width
;
705 tmpl
.height0
= box
->height
;
706 /* for array textures, box->depth is the array_size, otherwise
707 * for 3d textures, it is the depth:
709 if (tmpl
.array_size
> 1) {
710 if (tmpl
.target
== PIPE_TEXTURE_CUBE
)
711 tmpl
.target
= PIPE_TEXTURE_2D_ARRAY
;
712 tmpl
.array_size
= box
->depth
;
716 tmpl
.depth0
= box
->depth
;
719 tmpl
.bind
|= PIPE_BIND_LINEAR
;
721 struct pipe_resource
*pstaging
=
722 pctx
->screen
->resource_create(pctx
->screen
, &tmpl
);
726 return pan_resource(pstaging
);
730 pan_blit_from_staging(struct pipe_context
*pctx
, struct panfrost_gtransfer
*trans
)
732 struct pipe_resource
*dst
= trans
->base
.resource
;
733 struct pipe_blit_info blit
= {};
735 blit
.dst
.resource
= dst
;
736 blit
.dst
.format
= dst
->format
;
737 blit
.dst
.level
= trans
->base
.level
;
738 blit
.dst
.box
= trans
->base
.box
;
739 blit
.src
.resource
= trans
->staging
.rsrc
;
740 blit
.src
.format
= trans
->staging
.rsrc
->format
;
742 blit
.src
.box
= trans
->staging
.box
;
743 blit
.mask
= util_format_get_mask(trans
->staging
.rsrc
->format
);
744 blit
.filter
= PIPE_TEX_FILTER_NEAREST
;
746 panfrost_blit(pctx
, &blit
);
750 pan_blit_to_staging(struct pipe_context
*pctx
, struct panfrost_gtransfer
*trans
)
752 struct pipe_resource
*src
= trans
->base
.resource
;
753 struct pipe_blit_info blit
= {};
755 blit
.src
.resource
= src
;
756 blit
.src
.format
= src
->format
;
757 blit
.src
.level
= trans
->base
.level
;
758 blit
.src
.box
= trans
->base
.box
;
759 blit
.dst
.resource
= trans
->staging
.rsrc
;
760 blit
.dst
.format
= trans
->staging
.rsrc
->format
;
762 blit
.dst
.box
= trans
->staging
.box
;
763 blit
.mask
= util_format_get_mask(trans
->staging
.rsrc
->format
);
764 blit
.filter
= PIPE_TEX_FILTER_NEAREST
;
766 panfrost_blit(pctx
, &blit
);
770 panfrost_transfer_map(struct pipe_context
*pctx
,
771 struct pipe_resource
*resource
,
773 unsigned usage
, /* a combination of PIPE_TRANSFER_x */
774 const struct pipe_box
*box
,
775 struct pipe_transfer
**out_transfer
)
777 struct panfrost_context
*ctx
= pan_context(pctx
);
778 struct panfrost_device
*dev
= pan_device(pctx
->screen
);
779 struct panfrost_resource
*rsrc
= pan_resource(resource
);
780 int bytes_per_pixel
= util_format_get_blocksize(rsrc
->internal_format
);
781 struct panfrost_bo
*bo
= rsrc
->bo
;
783 /* Can't map tiled/compressed directly */
784 if ((usage
& PIPE_TRANSFER_MAP_DIRECTLY
) && rsrc
->modifier
!= DRM_FORMAT_MOD_LINEAR
)
787 struct panfrost_gtransfer
*transfer
= rzalloc(pctx
, struct panfrost_gtransfer
);
788 transfer
->base
.level
= level
;
789 transfer
->base
.usage
= usage
;
790 transfer
->base
.box
= *box
;
792 pipe_resource_reference(&transfer
->base
.resource
, resource
);
793 *out_transfer
= &transfer
->base
;
795 /* We don't have s/w routines for AFBC, so use a staging texture */
796 if (drm_is_afbc(rsrc
->modifier
)) {
797 struct panfrost_resource
*staging
= pan_alloc_staging(ctx
, rsrc
, level
, box
);
798 transfer
->base
.stride
= staging
->slices
[0].stride
;
799 transfer
->base
.layer_stride
= transfer
->base
.stride
* box
->height
;
801 transfer
->staging
.rsrc
= &staging
->base
;
803 transfer
->staging
.box
= *box
;
804 transfer
->staging
.box
.x
= 0;
805 transfer
->staging
.box
.y
= 0;
806 transfer
->staging
.box
.z
= 0;
808 assert(transfer
->staging
.rsrc
!= NULL
);
810 /* TODO: Eliminate this flush. It's only there to determine if
811 * we're initialized or not, when the initialization could come
812 * from a pending batch XXX */
813 panfrost_flush_batches_accessing_bo(ctx
, rsrc
->bo
, true);
815 if ((usage
& PIPE_TRANSFER_READ
) && rsrc
->slices
[level
].initialized
) {
816 pan_blit_to_staging(pctx
, transfer
);
817 panfrost_flush_batches_accessing_bo(ctx
, staging
->bo
, true);
818 panfrost_bo_wait(staging
->bo
, INT64_MAX
, false);
821 panfrost_bo_mmap(staging
->bo
);
822 return staging
->bo
->cpu
;
825 /* If we haven't already mmaped, now's the time */
826 panfrost_bo_mmap(bo
);
828 if (dev
->debug
& (PAN_DBG_TRACE
| PAN_DBG_SYNC
))
829 pandecode_inject_mmap(bo
->gpu
, bo
->cpu
, bo
->size
, NULL
);
831 bool create_new_bo
= usage
& PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
;
832 bool copy_resource
= false;
834 if (!create_new_bo
&&
835 !(usage
& PIPE_TRANSFER_UNSYNCHRONIZED
) &&
836 (usage
& PIPE_TRANSFER_WRITE
) &&
837 !(resource
->target
== PIPE_BUFFER
838 && !util_ranges_intersect(&rsrc
->valid_buffer_range
, box
->x
, box
->x
+ box
->width
)) &&
839 panfrost_pending_batches_access_bo(ctx
, bo
)) {
841 /* When a resource to be modified is already being used by a
842 * pending batch, it is often faster to copy the whole BO than
843 * to flush and split the frame in two. This also mostly
844 * mitigates broken depth reload.
847 panfrost_flush_batches_accessing_bo(ctx
, bo
, false);
848 panfrost_bo_wait(bo
, INT64_MAX
, false);
850 create_new_bo
= true;
851 copy_resource
= true;
855 /* If the BO is used by one of the pending batches or if it's
856 * not ready yet (still accessed by one of the already flushed
857 * batches), we try to allocate a new one to avoid waiting.
859 if (panfrost_pending_batches_access_bo(ctx
, bo
) ||
860 !panfrost_bo_wait(bo
, 0, true)) {
861 /* We want the BO to be MMAPed. */
862 uint32_t flags
= bo
->flags
& ~PAN_BO_DELAY_MMAP
;
863 struct panfrost_bo
*newbo
= NULL
;
865 /* When the BO has been imported/exported, we can't
866 * replace it by another one, otherwise the
867 * importer/exporter wouldn't see the change we're
870 if (!(bo
->flags
& PAN_BO_SHARED
))
871 newbo
= panfrost_bo_create(dev
, bo
->size
,
876 memcpy(newbo
->cpu
, rsrc
->bo
->cpu
, bo
->size
);
878 panfrost_bo_unreference(bo
);
882 /* Allocation failed or was impossible, let's
883 * fall back on a flush+wait.
885 panfrost_flush_batches_accessing_bo(ctx
, bo
, true);
886 panfrost_bo_wait(bo
, INT64_MAX
, true);
889 } else if ((usage
& PIPE_TRANSFER_WRITE
)
890 && resource
->target
== PIPE_BUFFER
891 && !util_ranges_intersect(&rsrc
->valid_buffer_range
, box
->x
, box
->x
+ box
->width
)) {
892 /* No flush for writes to uninitialized */
893 } else if (!(usage
& PIPE_TRANSFER_UNSYNCHRONIZED
)) {
894 if (usage
& PIPE_TRANSFER_WRITE
) {
895 panfrost_flush_batches_accessing_bo(ctx
, bo
, true);
896 panfrost_bo_wait(bo
, INT64_MAX
, true);
897 } else if (usage
& PIPE_TRANSFER_READ
) {
898 panfrost_flush_batches_accessing_bo(ctx
, bo
, false);
899 panfrost_bo_wait(bo
, INT64_MAX
, false);
903 if (rsrc
->modifier
== DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED
) {
904 transfer
->base
.stride
= box
->width
* bytes_per_pixel
;
905 transfer
->base
.layer_stride
= transfer
->base
.stride
* box
->height
;
906 transfer
->map
= ralloc_size(transfer
, transfer
->base
.layer_stride
* box
->depth
);
907 assert(box
->depth
== 1);
909 if ((usage
& PIPE_TRANSFER_READ
) && rsrc
->slices
[level
].initialized
) {
910 panfrost_load_tiled_image(
912 bo
->cpu
+ rsrc
->slices
[level
].offset
,
913 box
->x
, box
->y
, box
->width
, box
->height
,
914 transfer
->base
.stride
,
915 rsrc
->slices
[level
].stride
,
916 rsrc
->internal_format
);
919 return transfer
->map
;
921 assert (rsrc
->modifier
== DRM_FORMAT_MOD_LINEAR
);
923 /* Direct, persistent writes create holes in time for
924 * caching... I don't know if this is actually possible but we
925 * should still get it right */
927 unsigned dpw
= PIPE_TRANSFER_MAP_DIRECTLY
| PIPE_TRANSFER_WRITE
| PIPE_TRANSFER_PERSISTENT
;
929 if ((usage
& dpw
) == dpw
&& rsrc
->index_cache
)
932 transfer
->base
.stride
= rsrc
->slices
[level
].stride
;
933 transfer
->base
.layer_stride
= panfrost_get_layer_stride(
934 rsrc
->slices
, rsrc
->base
.target
== PIPE_TEXTURE_3D
,
935 rsrc
->cubemap_stride
, level
);
937 /* By mapping direct-write, we're implicitly already
938 * initialized (maybe), so be conservative */
940 if (usage
& PIPE_TRANSFER_WRITE
) {
941 rsrc
->slices
[level
].initialized
= true;
942 panfrost_minmax_cache_invalidate(rsrc
->index_cache
, &transfer
->base
);
946 + rsrc
->slices
[level
].offset
947 + transfer
->base
.box
.z
* transfer
->base
.layer_stride
948 + transfer
->base
.box
.y
* rsrc
->slices
[level
].stride
949 + transfer
->base
.box
.x
* bytes_per_pixel
;
954 panfrost_transfer_unmap(struct pipe_context
*pctx
,
955 struct pipe_transfer
*transfer
)
957 /* Gallium expects writeback here, so we tile */
959 struct panfrost_gtransfer
*trans
= pan_transfer(transfer
);
960 struct panfrost_resource
*prsrc
= (struct panfrost_resource
*) transfer
->resource
;
962 /* AFBC will use a staging resource. `initialized` will be set when the
963 * fragment job is created; this is deferred to prevent useless surface
964 * reloads that can cascade into DATA_INVALID_FAULTs due to reading
965 * malformed AFBC data if uninitialized */
967 if (trans
->staging
.rsrc
) {
968 if (transfer
->usage
& PIPE_TRANSFER_WRITE
) {
969 pan_blit_from_staging(pctx
, trans
);
970 panfrost_flush_batches_accessing_bo(pan_context(pctx
), pan_resource(trans
->staging
.rsrc
)->bo
, true);
973 pipe_resource_reference(&trans
->staging
.rsrc
, NULL
);
976 /* Tiling will occur in software from a staging cpu buffer */
978 struct panfrost_bo
*bo
= prsrc
->bo
;
980 if (transfer
->usage
& PIPE_TRANSFER_WRITE
) {
981 prsrc
->slices
[transfer
->level
].initialized
= true;
983 if (prsrc
->modifier
== DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED
) {
984 assert(transfer
->box
.depth
== 1);
986 /* Do we overwrite the entire resource? If so,
987 * we don't need an intermediate blit so it's a
988 * good time to switch the modifier. */
990 bool discards_content
= prsrc
->base
.last_level
== 0
991 && transfer
->box
.width
== prsrc
->base
.width0
992 && transfer
->box
.height
== prsrc
->base
.height0
993 && transfer
->box
.x
== 0
994 && transfer
->box
.y
== 0
995 && !prsrc
->modifier_constant
;
997 /* It also serves as a good heuristic for
998 * streaming textures (e.g. in video players),
999 * but we could do better */
1001 if (discards_content
)
1002 ++prsrc
->modifier_updates
;
1004 if (prsrc
->modifier_updates
>= LAYOUT_CONVERT_THRESHOLD
)
1006 prsrc
->modifier
= DRM_FORMAT_MOD_LINEAR
;
1009 bo
->cpu
+ prsrc
->slices
[0].offset
,
1011 prsrc
->slices
[0].stride
,
1013 transfer
->box
.width
,
1014 transfer
->box
.height
,
1019 panfrost_store_tiled_image(
1020 bo
->cpu
+ prsrc
->slices
[transfer
->level
].offset
,
1022 transfer
->box
.x
, transfer
->box
.y
,
1023 transfer
->box
.width
, transfer
->box
.height
,
1024 prsrc
->slices
[transfer
->level
].stride
,
1026 prsrc
->internal_format
);
1033 util_range_add(&prsrc
->base
, &prsrc
->valid_buffer_range
,
1035 transfer
->box
.x
+ transfer
->box
.width
);
1037 panfrost_minmax_cache_invalidate(prsrc
->index_cache
, transfer
);
1039 /* Derefence the resource */
1040 pipe_resource_reference(&transfer
->resource
, NULL
);
1042 /* Transfer itself is RALLOCed at the moment */
1043 ralloc_free(transfer
);
1047 panfrost_transfer_flush_region(struct pipe_context
*pctx
,
1048 struct pipe_transfer
*transfer
,
1049 const struct pipe_box
*box
)
1051 struct panfrost_resource
*rsc
= pan_resource(transfer
->resource
);
1053 if (transfer
->resource
->target
== PIPE_BUFFER
) {
1054 util_range_add(&rsc
->base
, &rsc
->valid_buffer_range
,
1055 transfer
->box
.x
+ box
->x
,
1056 transfer
->box
.x
+ box
->x
+ box
->width
);
1058 unsigned level
= transfer
->level
;
1059 rsc
->slices
[level
].initialized
= true;
1064 panfrost_invalidate_resource(struct pipe_context
*pctx
, struct pipe_resource
*prsc
)
1069 static enum pipe_format
1070 panfrost_resource_get_internal_format(struct pipe_resource
*rsrc
)
1072 struct panfrost_resource
*prsrc
= (struct panfrost_resource
*) rsrc
;
1073 return prsrc
->internal_format
;
1077 panfrost_generate_mipmap(
1078 struct pipe_context
*pctx
,
1079 struct pipe_resource
*prsrc
,
1080 enum pipe_format format
,
1081 unsigned base_level
,
1082 unsigned last_level
,
1083 unsigned first_layer
,
1084 unsigned last_layer
)
1086 struct panfrost_resource
*rsrc
= pan_resource(prsrc
);
1088 /* Generating a mipmap invalidates the written levels, so make that
1089 * explicit so we don't try to wallpaper them back and end up with
1090 * u_blitter recursion */
1093 for (unsigned l
= base_level
+ 1; l
<= last_level
; ++l
)
1094 rsrc
->slices
[l
].initialized
= false;
1096 /* Beyond that, we just delegate the hard stuff. */
1098 bool blit_res
= util_gen_mipmap(
1099 pctx
, prsrc
, format
,
1100 base_level
, last_level
,
1101 first_layer
, last_layer
,
1102 PIPE_TEX_FILTER_LINEAR
);
1107 /* Computes the address to a texture at a particular slice */
1110 panfrost_get_texture_address(
1111 struct panfrost_resource
*rsrc
,
1112 unsigned level
, unsigned face
, unsigned sample
)
1114 bool is_3d
= rsrc
->base
.target
== PIPE_TEXTURE_3D
;
1115 return rsrc
->bo
->gpu
+ panfrost_texture_offset(rsrc
->slices
, is_3d
, rsrc
->cubemap_stride
, level
, face
, sample
);
1119 panfrost_resource_set_stencil(struct pipe_resource
*prsrc
,
1120 struct pipe_resource
*stencil
)
1122 pan_resource(prsrc
)->separate_stencil
= pan_resource(stencil
);
1125 static struct pipe_resource
*
1126 panfrost_resource_get_stencil(struct pipe_resource
*prsrc
)
1128 return &pan_resource(prsrc
)->separate_stencil
->base
;
1131 static const struct u_transfer_vtbl transfer_vtbl
= {
1132 .resource_create
= panfrost_resource_create
,
1133 .resource_destroy
= panfrost_resource_destroy
,
1134 .transfer_map
= panfrost_transfer_map
,
1135 .transfer_unmap
= panfrost_transfer_unmap
,
1136 .transfer_flush_region
= panfrost_transfer_flush_region
,
1137 .get_internal_format
= panfrost_resource_get_internal_format
,
1138 .set_stencil
= panfrost_resource_set_stencil
,
1139 .get_stencil
= panfrost_resource_get_stencil
,
1143 panfrost_resource_screen_init(struct pipe_screen
*pscreen
)
1145 struct panfrost_device
*dev
= pan_device(pscreen
);
1147 bool fake_rgtc
= !panfrost_supports_compressed_format(dev
, MALI_BC4_UNORM
);
1149 pscreen
->resource_create_with_modifiers
=
1150 panfrost_resource_create_with_modifiers
;
1151 pscreen
->resource_create
= u_transfer_helper_resource_create
;
1152 pscreen
->resource_destroy
= u_transfer_helper_resource_destroy
;
1153 pscreen
->resource_from_handle
= panfrost_resource_from_handle
;
1154 pscreen
->resource_get_handle
= panfrost_resource_get_handle
;
1155 pscreen
->transfer_helper
= u_transfer_helper_create(&transfer_vtbl
,
1161 panfrost_resource_context_init(struct pipe_context
*pctx
)
1163 pctx
->transfer_map
= u_transfer_helper_transfer_map
;
1164 pctx
->transfer_unmap
= u_transfer_helper_transfer_unmap
;
1165 pctx
->create_surface
= panfrost_create_surface
;
1166 pctx
->surface_destroy
= panfrost_surface_destroy
;
1167 pctx
->resource_copy_region
= util_resource_copy_region
;
1168 pctx
->blit
= panfrost_blit
;
1169 pctx
->generate_mipmap
= panfrost_generate_mipmap
;
1170 pctx
->flush_resource
= panfrost_flush_resource
;
1171 pctx
->invalidate_resource
= panfrost_invalidate_resource
;
1172 pctx
->transfer_flush_region
= u_transfer_helper_transfer_flush_region
;
1173 pctx
->buffer_subdata
= u_default_buffer_subdata
;
1174 pctx
->texture_subdata
= u_default_texture_subdata
;