2 * Copyright (C) 2008 VMware, Inc.
3 * Copyright (C) 2012 Rob Clark <robclark@freedesktop.org>
4 * Copyright (C) 2014-2017 Broadcom
5 * Copyright (C) 2018-2019 Alyssa Rosenzweig
6 * Copyright (C) 2019 Collabora, Ltd.
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the "Software"),
10 * to deal in the Software without restriction, including without limitation
11 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 * and/or sell copies of the Software, and to permit persons to whom the
13 * Software is furnished to do so, subject to the following conditions:
15 * The above copyright notice and this permission notice (including the next
16 * paragraph) shall be included in all copies or substantial portions of the
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27 * Authors (Collabora):
28 * Tomeu Vizoso <tomeu.vizoso@collabora.com>
29 * Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
35 #include "drm-uapi/drm_fourcc.h"
37 #include "frontend/winsys_handle.h"
38 #include "util/format/u_format.h"
39 #include "util/u_memory.h"
40 #include "util/u_surface.h"
41 #include "util/u_transfer.h"
42 #include "util/u_transfer_helper.h"
43 #include "util/u_gen_mipmap.h"
44 #include "util/u_drm.h"
47 #include "pan_context.h"
48 #include "pan_screen.h"
49 #include "pan_resource.h"
51 #include "pan_tiling.h"
53 #include "panfrost-quirks.h"
55 static struct pipe_resource
*
56 panfrost_resource_from_handle(struct pipe_screen
*pscreen
,
57 const struct pipe_resource
*templat
,
58 struct winsys_handle
*whandle
,
61 struct panfrost_device
*dev
= pan_device(pscreen
);
62 struct panfrost_resource
*rsc
;
63 struct pipe_resource
*prsc
;
65 assert(whandle
->type
== WINSYS_HANDLE_TYPE_FD
);
67 rsc
= rzalloc(pscreen
, struct panfrost_resource
);
75 pipe_reference_init(&prsc
->reference
, 1);
76 prsc
->screen
= pscreen
;
78 rsc
->bo
= panfrost_bo_import(dev
, whandle
->handle
);
79 rsc
->internal_format
= templat
->format
;
80 rsc
->modifier
= (whandle
->modifier
== DRM_FORMAT_MOD_INVALID
) ?
81 DRM_FORMAT_MOD_LINEAR
: whandle
->modifier
;
82 rsc
->slices
[0].stride
= whandle
->stride
;
83 rsc
->slices
[0].offset
= whandle
->offset
;
84 rsc
->slices
[0].initialized
= true;
85 panfrost_resource_set_damage_region(NULL
, &rsc
->base
, 0, NULL
);
87 if (dev
->quirks
& IS_BIFROST
&&
88 templat
->bind
& PIPE_BIND_RENDER_TARGET
) {
89 unsigned size
= panfrost_compute_checksum_size(
90 &rsc
->slices
[0], templat
->width0
, templat
->height0
);
91 rsc
->slices
[0].checksum_bo
= panfrost_bo_create(dev
, size
, 0);
92 rsc
->checksummed
= true;
95 if (drm_is_afbc(whandle
->modifier
)) {
96 rsc
->slices
[0].header_size
=
97 panfrost_afbc_header_size(templat
->width0
, templat
->height0
);
102 renderonly_create_gpu_import_for_resource(prsc
, dev
->ro
, NULL
);
103 /* failure is expected in some cases.. */
110 panfrost_resource_get_handle(struct pipe_screen
*pscreen
,
111 struct pipe_context
*ctx
,
112 struct pipe_resource
*pt
,
113 struct winsys_handle
*handle
,
116 struct panfrost_device
*dev
= pan_device(pscreen
);
117 struct panfrost_resource
*rsrc
= (struct panfrost_resource
*) pt
;
118 struct renderonly_scanout
*scanout
= rsrc
->scanout
;
120 handle
->modifier
= rsrc
->modifier
;
122 if (handle
->type
== WINSYS_HANDLE_TYPE_SHARED
) {
124 } else if (handle
->type
== WINSYS_HANDLE_TYPE_KMS
) {
125 if (renderonly_get_handle(scanout
, handle
))
128 handle
->handle
= rsrc
->bo
->gem_handle
;
129 handle
->stride
= rsrc
->slices
[0].stride
;
130 handle
->offset
= rsrc
->slices
[0].offset
;
132 } else if (handle
->type
== WINSYS_HANDLE_TYPE_FD
) {
134 struct drm_prime_handle args
= {
135 .handle
= scanout
->handle
,
136 .flags
= DRM_CLOEXEC
,
139 int ret
= drmIoctl(dev
->ro
->kms_fd
, DRM_IOCTL_PRIME_HANDLE_TO_FD
, &args
);
143 handle
->stride
= scanout
->stride
;
144 handle
->handle
= args
.fd
;
148 int fd
= panfrost_bo_export(rsrc
->bo
);
154 handle
->stride
= rsrc
->slices
[0].stride
;
155 handle
->offset
= rsrc
->slices
[0].offset
;
164 panfrost_flush_resource(struct pipe_context
*pctx
, struct pipe_resource
*prsc
)
169 static struct pipe_surface
*
170 panfrost_create_surface(struct pipe_context
*pipe
,
171 struct pipe_resource
*pt
,
172 const struct pipe_surface
*surf_tmpl
)
174 struct pipe_surface
*ps
= NULL
;
176 ps
= rzalloc(pipe
, struct pipe_surface
);
179 pipe_reference_init(&ps
->reference
, 1);
180 pipe_resource_reference(&ps
->texture
, pt
);
182 ps
->format
= surf_tmpl
->format
;
184 if (pt
->target
!= PIPE_BUFFER
) {
185 assert(surf_tmpl
->u
.tex
.level
<= pt
->last_level
);
186 ps
->width
= u_minify(pt
->width0
, surf_tmpl
->u
.tex
.level
);
187 ps
->height
= u_minify(pt
->height0
, surf_tmpl
->u
.tex
.level
);
188 ps
->nr_samples
= surf_tmpl
->nr_samples
;
189 ps
->u
.tex
.level
= surf_tmpl
->u
.tex
.level
;
190 ps
->u
.tex
.first_layer
= surf_tmpl
->u
.tex
.first_layer
;
191 ps
->u
.tex
.last_layer
= surf_tmpl
->u
.tex
.last_layer
;
193 /* setting width as number of elements should get us correct renderbuffer width */
194 ps
->width
= surf_tmpl
->u
.buf
.last_element
- surf_tmpl
->u
.buf
.first_element
+ 1;
195 ps
->height
= pt
->height0
;
196 ps
->u
.buf
.first_element
= surf_tmpl
->u
.buf
.first_element
;
197 ps
->u
.buf
.last_element
= surf_tmpl
->u
.buf
.last_element
;
198 assert(ps
->u
.buf
.first_element
<= ps
->u
.buf
.last_element
);
199 assert(ps
->u
.buf
.last_element
< ps
->width
);
207 panfrost_surface_destroy(struct pipe_context
*pipe
,
208 struct pipe_surface
*surf
)
210 assert(surf
->texture
);
211 pipe_resource_reference(&surf
->texture
, NULL
);
215 static struct pipe_resource
*
216 panfrost_create_scanout_res(struct pipe_screen
*screen
,
217 const struct pipe_resource
*template,
220 struct panfrost_device
*dev
= pan_device(screen
);
221 struct pipe_resource scanout_templat
= *template;
222 struct renderonly_scanout
*scanout
;
223 struct winsys_handle handle
;
224 struct pipe_resource
*res
;
226 scanout
= renderonly_scanout_for_resource(&scanout_templat
,
231 assert(handle
.type
== WINSYS_HANDLE_TYPE_FD
);
232 handle
.modifier
= modifier
;
233 res
= screen
->resource_from_handle(screen
, template, &handle
,
234 PIPE_HANDLE_USAGE_FRAMEBUFFER_WRITE
);
235 close(handle
.handle
);
239 struct panfrost_resource
*pres
= pan_resource(res
);
241 pres
->scanout
= scanout
;
246 /* Setup the mip tree given a particular modifier, possibly with checksumming */
249 panfrost_setup_slices(struct panfrost_resource
*pres
, size_t *bo_size
)
251 struct pipe_resource
*res
= &pres
->base
;
252 unsigned width
= res
->width0
;
253 unsigned height
= res
->height0
;
254 unsigned depth
= res
->depth0
;
255 unsigned bytes_per_pixel
= util_format_get_blocksize(pres
->internal_format
);
257 /* MSAA is implemented as a 3D texture with z corresponding to the
258 * sample #, horrifyingly enough */
260 bool msaa
= res
->nr_samples
> 1;
264 depth
= res
->nr_samples
;
269 /* Tiled operates blockwise; linear is packed. Also, anything
270 * we render to has to be tile-aligned. Maybe not strictly
271 * necessary, but we're not *that* pressed for memory and it
272 * makes code a lot simpler */
274 bool renderable
= res
->bind
&
275 (PIPE_BIND_RENDER_TARGET
| PIPE_BIND_DEPTH_STENCIL
) &&
276 res
->target
!= PIPE_BUFFER
;
277 bool afbc
= drm_is_afbc(pres
->modifier
);
278 bool tiled
= pres
->modifier
== DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED
;
279 bool linear
= pres
->modifier
== DRM_FORMAT_MOD_LINEAR
;
280 bool should_align
= renderable
|| tiled
;
282 /* We don't know how to specify a 2D stride for 3D textures */
284 bool can_align_stride
=
285 res
->target
!= PIPE_TEXTURE_3D
;
287 should_align
&= can_align_stride
;
290 unsigned size_2d
= 0;
292 for (unsigned l
= 0; l
<= res
->last_level
; ++l
) {
293 struct panfrost_slice
*slice
= &pres
->slices
[l
];
295 unsigned effective_width
= width
;
296 unsigned effective_height
= height
;
297 unsigned effective_depth
= depth
;
300 effective_width
= ALIGN_POT(effective_width
, 16);
301 effective_height
= ALIGN_POT(effective_height
, 16);
303 /* We don't need to align depth */
306 /* Align levels to cache-line as a performance improvement for
307 * linear/tiled and as a requirement for AFBC */
309 offset
= ALIGN_POT(offset
, 64);
311 slice
->offset
= offset
;
313 /* Compute the would-be stride */
314 unsigned stride
= bytes_per_pixel
* effective_width
;
316 if (util_format_is_compressed(pres
->internal_format
))
319 /* ..but cache-line align it for performance */
320 if (can_align_stride
&& linear
)
321 stride
= ALIGN_POT(stride
, 64);
323 slice
->stride
= stride
;
325 unsigned slice_one_size
= slice
->stride
* effective_height
;
326 unsigned slice_full_size
= slice_one_size
* effective_depth
;
328 slice
->size0
= slice_one_size
;
330 /* Report 2D size for 3D texturing */
333 size_2d
= slice_one_size
;
335 /* Compute AFBC sizes if necessary */
338 panfrost_afbc_header_size(width
, height
);
340 offset
+= slice
->header_size
;
343 offset
+= slice_full_size
;
345 /* Add a checksum region if necessary */
346 if (pres
->checksummed
) {
347 slice
->checksum_offset
= offset
;
349 unsigned size
= panfrost_compute_checksum_size(
350 slice
, width
, height
);
355 width
= u_minify(width
, 1);
356 height
= u_minify(height
, 1);
358 /* Don't mipmap the sample count */
360 depth
= u_minify(depth
, 1);
363 assert(res
->array_size
);
365 if (res
->target
!= PIPE_TEXTURE_3D
) {
366 /* Arrays and cubemaps have the entire miptree duplicated */
368 pres
->cubemap_stride
= ALIGN_POT(offset
, 64);
369 *bo_size
= ALIGN_POT(pres
->cubemap_stride
* res
->array_size
, 4096);
371 /* 3D strides across the 2D layers */
372 assert(res
->array_size
== 1);
374 pres
->cubemap_stride
= size_2d
;
375 *bo_size
= ALIGN_POT(offset
, 4096);
379 /* Based on the usage, determine if it makes sense to use u-inteleaved tiling.
380 * We only have routines to tile 2D textures of sane bpps. On the hardware
381 * level, not all usages are valid for tiling. Finally, if the app is hinting
382 * that the contents frequently change, tiling will be a loss.
384 * Due to incomplete information on some platforms, we may need to force tiling
387 * On platforms where it is supported, AFBC is even better. */
390 panfrost_can_linear(struct panfrost_device
*dev
, const struct panfrost_resource
*pres
)
392 /* XXX: We should be able to do linear Z/S with the right bits.. */
393 return !((pres
->base
.bind
& PIPE_BIND_DEPTH_STENCIL
) &&
394 (dev
->quirks
& (MIDGARD_SFBD
| IS_BIFROST
)));
398 panfrost_should_afbc(struct panfrost_device
*dev
, const struct panfrost_resource
*pres
)
400 /* AFBC resources may be rendered to, textured from, or shared across
401 * processes, but may not be used as e.g buffers */
402 const unsigned valid_binding
=
403 PIPE_BIND_DEPTH_STENCIL
|
404 PIPE_BIND_RENDER_TARGET
|
405 PIPE_BIND_BLENDABLE
|
406 PIPE_BIND_SAMPLER_VIEW
|
407 PIPE_BIND_DISPLAY_TARGET
|
411 if (pres
->base
.bind
& ~valid_binding
)
414 /* AFBC introduced with Mali T760 */
415 if (dev
->quirks
& MIDGARD_NO_AFBC
)
418 /* AFBC<-->staging is expensive */
419 if (pres
->base
.usage
== PIPE_USAGE_STREAM
)
422 /* Only a small selection of formats are AFBC'able */
423 if (!panfrost_format_supports_afbc(pres
->internal_format
))
426 /* AFBC does not support layered (GLES3 style) multisampling. Use
427 * EXT_multisampled_render_to_texture instead */
428 if (pres
->base
.nr_samples
> 1)
431 /* TODO: Is AFBC of 3D textures possible? */
432 if ((pres
->base
.target
!= PIPE_TEXTURE_2D
) && (pres
->base
.target
!= PIPE_TEXTURE_RECT
))
435 /* For one tile, AFBC is a loss compared to u-interleaved */
436 if (pres
->base
.width0
<= 16 && pres
->base
.height0
<= 16)
439 /* Otherwise, we'd prefer AFBC as it is dramatically more efficient
440 * than linear or usually even u-interleaved */
445 panfrost_should_tile(struct panfrost_device
*dev
, const struct panfrost_resource
*pres
)
447 const unsigned valid_binding
=
448 PIPE_BIND_DEPTH_STENCIL
|
449 PIPE_BIND_RENDER_TARGET
|
450 PIPE_BIND_BLENDABLE
|
451 PIPE_BIND_SAMPLER_VIEW
|
452 PIPE_BIND_DISPLAY_TARGET
|
456 unsigned bpp
= util_format_get_blocksizebits(pres
->internal_format
);
459 bpp
== 8 || bpp
== 16 || bpp
== 24 || bpp
== 32 ||
460 bpp
== 64 || bpp
== 128;
462 bool is_2d
= (pres
->base
.target
== PIPE_TEXTURE_2D
)
463 || (pres
->base
.target
== PIPE_TEXTURE_RECT
);
465 bool can_tile
= is_2d
&& is_sane_bpp
&& ((pres
->base
.bind
& ~valid_binding
) == 0);
467 if (!panfrost_can_linear(dev
, pres
)) {
472 return can_tile
&& (pres
->base
.usage
!= PIPE_USAGE_STREAM
);
476 panfrost_best_modifier(struct panfrost_device
*dev
,
477 const struct panfrost_resource
*pres
)
479 if (panfrost_should_afbc(dev
, pres
)) {
481 AFBC_FORMAT_MOD_BLOCK_SIZE_16x16
|
482 AFBC_FORMAT_MOD_SPARSE
;
484 if (panfrost_afbc_can_ytr(pres
->base
.format
))
485 afbc
|= AFBC_FORMAT_MOD_YTR
;
487 return DRM_FORMAT_MOD_ARM_AFBC(afbc
);
488 } else if (panfrost_should_tile(dev
, pres
))
489 return DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED
;
491 return DRM_FORMAT_MOD_LINEAR
;
495 panfrost_resource_create_bo(struct panfrost_device
*dev
, struct panfrost_resource
*pres
,
498 struct pipe_resource
*res
= &pres
->base
;
500 pres
->modifier
= (modifier
!= DRM_FORMAT_MOD_INVALID
) ? modifier
:
501 panfrost_best_modifier(dev
, pres
);
502 pres
->checksummed
= (res
->bind
& PIPE_BIND_RENDER_TARGET
);
504 /* We can only switch tiled->linear if the resource isn't already
505 * linear, and if we control the modifier, and if the resource can be
507 pres
->modifier_constant
= !((pres
->modifier
!= DRM_FORMAT_MOD_LINEAR
)
508 && (modifier
== DRM_FORMAT_INVALID
)
509 && panfrost_can_linear(dev
, pres
));
513 panfrost_setup_slices(pres
, &bo_size
);
515 /* We create a BO immediately but don't bother mapping, since we don't
516 * care to map e.g. FBOs which the CPU probably won't touch */
517 pres
->bo
= panfrost_bo_create(dev
, bo_size
, PAN_BO_DELAY_MMAP
);
521 panfrost_resource_set_damage_region(struct pipe_screen
*screen
,
522 struct pipe_resource
*res
,
524 const struct pipe_box
*rects
)
526 struct panfrost_resource
*pres
= pan_resource(res
);
527 struct pipe_scissor_state
*damage_extent
= &pres
->damage
.extent
;
530 if (pres
->damage
.inverted_rects
)
531 ralloc_free(pres
->damage
.inverted_rects
);
533 memset(&pres
->damage
, 0, sizeof(pres
->damage
));
535 pres
->damage
.inverted_rects
=
536 pan_subtract_damage(pres
,
537 res
->width0
, res
->height0
,
538 nrects
, rects
, &pres
->damage
.inverted_len
);
540 /* Track the damage extent: the quad including all damage regions. Will
541 * be used restrict the rendering area */
543 damage_extent
->minx
= 0xffff;
544 damage_extent
->miny
= 0xffff;
546 for (i
= 0; i
< nrects
; i
++) {
547 int x
= rects
[i
].x
, w
= rects
[i
].width
, h
= rects
[i
].height
;
548 int y
= res
->height0
- (rects
[i
].y
+ h
);
550 damage_extent
->minx
= MIN2(damage_extent
->minx
, x
);
551 damage_extent
->miny
= MIN2(damage_extent
->miny
, y
);
552 damage_extent
->maxx
= MAX2(damage_extent
->maxx
,
553 MIN2(x
+ w
, res
->width0
));
554 damage_extent
->maxy
= MAX2(damage_extent
->maxy
,
555 MIN2(y
+ h
, res
->height0
));
559 damage_extent
->minx
= 0;
560 damage_extent
->miny
= 0;
561 damage_extent
->maxx
= res
->width0
;
562 damage_extent
->maxy
= res
->height0
;
567 static struct pipe_resource
*
568 panfrost_resource_create_with_modifier(struct pipe_screen
*screen
,
569 const struct pipe_resource
*template,
572 struct panfrost_device
*dev
= pan_device(screen
);
574 /* Make sure we're familiar */
575 switch (template->target
) {
577 case PIPE_TEXTURE_1D
:
578 case PIPE_TEXTURE_2D
:
579 case PIPE_TEXTURE_3D
:
580 case PIPE_TEXTURE_CUBE
:
581 case PIPE_TEXTURE_RECT
:
582 case PIPE_TEXTURE_1D_ARRAY
:
583 case PIPE_TEXTURE_2D_ARRAY
:
586 unreachable("Unknown texture target\n");
589 if (dev
->ro
&& (template->bind
&
590 (PIPE_BIND_DISPLAY_TARGET
| PIPE_BIND_SCANOUT
| PIPE_BIND_SHARED
)))
591 return panfrost_create_scanout_res(screen
, template, modifier
);
593 struct panfrost_resource
*so
= rzalloc(screen
, struct panfrost_resource
);
594 so
->base
= *template;
595 so
->base
.screen
= screen
;
596 so
->internal_format
= template->format
;
598 pipe_reference_init(&so
->base
.reference
, 1);
600 util_range_init(&so
->valid_buffer_range
);
602 panfrost_resource_create_bo(dev
, so
, modifier
);
603 panfrost_resource_set_damage_region(NULL
, &so
->base
, 0, NULL
);
605 if (template->bind
& PIPE_BIND_INDEX_BUFFER
)
606 so
->index_cache
= rzalloc(so
, struct panfrost_minmax_cache
);
608 return (struct pipe_resource
*)so
;
611 /* Default is to create a resource as don't care */
613 static struct pipe_resource
*
614 panfrost_resource_create(struct pipe_screen
*screen
,
615 const struct pipe_resource
*template)
617 return panfrost_resource_create_with_modifier(screen
, template,
618 DRM_FORMAT_MOD_INVALID
);
621 /* If no modifier is specified, we'll choose. Otherwise, the order of
622 * preference is compressed, tiled, linear. */
624 static struct pipe_resource
*
625 panfrost_resource_create_with_modifiers(struct pipe_screen
*screen
,
626 const struct pipe_resource
*template,
627 const uint64_t *modifiers
, int count
)
629 for (unsigned i
= 0; i
< PAN_MODIFIER_COUNT
; ++i
) {
630 if (drm_find_modifier(pan_best_modifiers
[i
], modifiers
, count
)) {
631 return panfrost_resource_create_with_modifier(screen
, template,
632 pan_best_modifiers
[i
]);
636 /* If we didn't find one, app specified invalid */
637 assert(count
== 1 && modifiers
[0] == DRM_FORMAT_MOD_INVALID
);
638 return panfrost_resource_create(screen
, template);
642 panfrost_resource_destroy(struct pipe_screen
*screen
,
643 struct pipe_resource
*pt
)
645 struct panfrost_device
*dev
= pan_device(screen
);
646 struct panfrost_resource
*rsrc
= (struct panfrost_resource
*) pt
;
649 renderonly_scanout_destroy(rsrc
->scanout
, dev
->ro
);
652 panfrost_bo_unreference(rsrc
->bo
);
654 if (rsrc
->slices
[0].checksum_bo
)
655 panfrost_bo_unreference(rsrc
->slices
[0].checksum_bo
);
657 util_range_destroy(&rsrc
->valid_buffer_range
);
661 /* Most of the time we can do CPU-side transfers, but sometimes we need to use
662 * the 3D pipe for this. Let's wrap u_blitter to blit to/from staging textures.
663 * Code adapted from freedreno */
665 static struct panfrost_resource
*
666 pan_alloc_staging(struct panfrost_context
*ctx
, struct panfrost_resource
*rsc
,
667 unsigned level
, const struct pipe_box
*box
)
669 struct pipe_context
*pctx
= &ctx
->base
;
670 struct pipe_resource tmpl
= rsc
->base
;
672 tmpl
.width0
= box
->width
;
673 tmpl
.height0
= box
->height
;
674 /* for array textures, box->depth is the array_size, otherwise
675 * for 3d textures, it is the depth:
677 if (tmpl
.array_size
> 1) {
678 if (tmpl
.target
== PIPE_TEXTURE_CUBE
)
679 tmpl
.target
= PIPE_TEXTURE_2D_ARRAY
;
680 tmpl
.array_size
= box
->depth
;
684 tmpl
.depth0
= box
->depth
;
687 tmpl
.bind
|= PIPE_BIND_LINEAR
;
689 struct pipe_resource
*pstaging
=
690 pctx
->screen
->resource_create(pctx
->screen
, &tmpl
);
694 return pan_resource(pstaging
);
698 pan_blit_from_staging(struct pipe_context
*pctx
, struct panfrost_gtransfer
*trans
)
700 struct pipe_resource
*dst
= trans
->base
.resource
;
701 struct pipe_blit_info blit
= {};
703 blit
.dst
.resource
= dst
;
704 blit
.dst
.format
= dst
->format
;
705 blit
.dst
.level
= trans
->base
.level
;
706 blit
.dst
.box
= trans
->base
.box
;
707 blit
.src
.resource
= trans
->staging
.rsrc
;
708 blit
.src
.format
= trans
->staging
.rsrc
->format
;
710 blit
.src
.box
= trans
->staging
.box
;
711 blit
.mask
= util_format_get_mask(trans
->staging
.rsrc
->format
);
712 blit
.filter
= PIPE_TEX_FILTER_NEAREST
;
714 panfrost_blit(pctx
, &blit
);
718 pan_blit_to_staging(struct pipe_context
*pctx
, struct panfrost_gtransfer
*trans
)
720 struct pipe_resource
*src
= trans
->base
.resource
;
721 struct pipe_blit_info blit
= {};
723 blit
.src
.resource
= src
;
724 blit
.src
.format
= src
->format
;
725 blit
.src
.level
= trans
->base
.level
;
726 blit
.src
.box
= trans
->base
.box
;
727 blit
.dst
.resource
= trans
->staging
.rsrc
;
728 blit
.dst
.format
= trans
->staging
.rsrc
->format
;
730 blit
.dst
.box
= trans
->staging
.box
;
731 blit
.mask
= util_format_get_mask(trans
->staging
.rsrc
->format
);
732 blit
.filter
= PIPE_TEX_FILTER_NEAREST
;
734 panfrost_blit(pctx
, &blit
);
738 panfrost_transfer_map(struct pipe_context
*pctx
,
739 struct pipe_resource
*resource
,
741 unsigned usage
, /* a combination of PIPE_TRANSFER_x */
742 const struct pipe_box
*box
,
743 struct pipe_transfer
**out_transfer
)
745 struct panfrost_context
*ctx
= pan_context(pctx
);
746 struct panfrost_device
*dev
= pan_device(pctx
->screen
);
747 struct panfrost_resource
*rsrc
= pan_resource(resource
);
748 int bytes_per_pixel
= util_format_get_blocksize(rsrc
->internal_format
);
749 struct panfrost_bo
*bo
= rsrc
->bo
;
751 /* Can't map tiled/compressed directly */
752 if ((usage
& PIPE_TRANSFER_MAP_DIRECTLY
) && rsrc
->modifier
!= DRM_FORMAT_MOD_LINEAR
)
755 struct panfrost_gtransfer
*transfer
= rzalloc(pctx
, struct panfrost_gtransfer
);
756 transfer
->base
.level
= level
;
757 transfer
->base
.usage
= usage
;
758 transfer
->base
.box
= *box
;
760 pipe_resource_reference(&transfer
->base
.resource
, resource
);
761 *out_transfer
= &transfer
->base
;
763 /* We don't have s/w routines for AFBC, so use a staging texture */
764 if (drm_is_afbc(rsrc
->modifier
)) {
765 struct panfrost_resource
*staging
= pan_alloc_staging(ctx
, rsrc
, level
, box
);
766 transfer
->base
.stride
= staging
->slices
[0].stride
;
767 transfer
->base
.layer_stride
= transfer
->base
.stride
* box
->height
;
769 transfer
->staging
.rsrc
= &staging
->base
;
771 transfer
->staging
.box
= *box
;
772 transfer
->staging
.box
.x
= 0;
773 transfer
->staging
.box
.y
= 0;
774 transfer
->staging
.box
.z
= 0;
776 assert(transfer
->staging
.rsrc
!= NULL
);
778 /* TODO: Eliminate this flush. It's only there to determine if
779 * we're initialized or not, when the initialization could come
780 * from a pending batch XXX */
781 panfrost_flush_batches_accessing_bo(ctx
, rsrc
->bo
, true);
783 if ((usage
& PIPE_TRANSFER_READ
) && rsrc
->slices
[level
].initialized
) {
784 pan_blit_to_staging(pctx
, transfer
);
785 panfrost_flush_batches_accessing_bo(ctx
, staging
->bo
, true);
786 panfrost_bo_wait(staging
->bo
, INT64_MAX
, false);
789 panfrost_bo_mmap(staging
->bo
);
790 return staging
->bo
->cpu
;
793 /* If we haven't already mmaped, now's the time */
794 panfrost_bo_mmap(bo
);
796 if (dev
->debug
& (PAN_DBG_TRACE
| PAN_DBG_SYNC
))
797 pandecode_inject_mmap(bo
->gpu
, bo
->cpu
, bo
->size
, NULL
);
799 bool create_new_bo
= usage
& PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
;
800 bool copy_resource
= false;
802 if (!create_new_bo
&&
803 !(usage
& PIPE_TRANSFER_UNSYNCHRONIZED
) &&
804 (usage
& PIPE_TRANSFER_WRITE
) &&
805 !(resource
->target
== PIPE_BUFFER
806 && !util_ranges_intersect(&rsrc
->valid_buffer_range
, box
->x
, box
->x
+ box
->width
)) &&
807 panfrost_pending_batches_access_bo(ctx
, bo
)) {
809 /* When a resource to be modified is already being used by a
810 * pending batch, it is often faster to copy the whole BO than
811 * to flush and split the frame in two. This also mostly
812 * mitigates broken depth reload.
815 panfrost_flush_batches_accessing_bo(ctx
, bo
, false);
816 panfrost_bo_wait(bo
, INT64_MAX
, false);
818 create_new_bo
= true;
819 copy_resource
= true;
823 /* If the BO is used by one of the pending batches or if it's
824 * not ready yet (still accessed by one of the already flushed
825 * batches), we try to allocate a new one to avoid waiting.
827 if (panfrost_pending_batches_access_bo(ctx
, bo
) ||
828 !panfrost_bo_wait(bo
, 0, true)) {
829 /* We want the BO to be MMAPed. */
830 uint32_t flags
= bo
->flags
& ~PAN_BO_DELAY_MMAP
;
831 struct panfrost_bo
*newbo
= NULL
;
833 /* When the BO has been imported/exported, we can't
834 * replace it by another one, otherwise the
835 * importer/exporter wouldn't see the change we're
838 if (!(bo
->flags
& PAN_BO_SHARED
))
839 newbo
= panfrost_bo_create(dev
, bo
->size
,
844 memcpy(newbo
->cpu
, rsrc
->bo
->cpu
, bo
->size
);
846 panfrost_bo_unreference(bo
);
850 /* Allocation failed or was impossible, let's
851 * fall back on a flush+wait.
853 panfrost_flush_batches_accessing_bo(ctx
, bo
, true);
854 panfrost_bo_wait(bo
, INT64_MAX
, true);
857 } else if ((usage
& PIPE_TRANSFER_WRITE
)
858 && resource
->target
== PIPE_BUFFER
859 && !util_ranges_intersect(&rsrc
->valid_buffer_range
, box
->x
, box
->x
+ box
->width
)) {
860 /* No flush for writes to uninitialized */
861 } else if (!(usage
& PIPE_TRANSFER_UNSYNCHRONIZED
)) {
862 if (usage
& PIPE_TRANSFER_WRITE
) {
863 panfrost_flush_batches_accessing_bo(ctx
, bo
, true);
864 panfrost_bo_wait(bo
, INT64_MAX
, true);
865 } else if (usage
& PIPE_TRANSFER_READ
) {
866 panfrost_flush_batches_accessing_bo(ctx
, bo
, false);
867 panfrost_bo_wait(bo
, INT64_MAX
, false);
871 if (rsrc
->modifier
== DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED
) {
872 transfer
->base
.stride
= box
->width
* bytes_per_pixel
;
873 transfer
->base
.layer_stride
= transfer
->base
.stride
* box
->height
;
874 transfer
->map
= ralloc_size(transfer
, transfer
->base
.layer_stride
* box
->depth
);
875 assert(box
->depth
== 1);
877 if ((usage
& PIPE_TRANSFER_READ
) && rsrc
->slices
[level
].initialized
) {
878 panfrost_load_tiled_image(
880 bo
->cpu
+ rsrc
->slices
[level
].offset
,
881 box
->x
, box
->y
, box
->width
, box
->height
,
882 transfer
->base
.stride
,
883 rsrc
->slices
[level
].stride
,
884 rsrc
->internal_format
);
887 return transfer
->map
;
889 assert (rsrc
->modifier
== DRM_FORMAT_MOD_LINEAR
);
891 /* Direct, persistent writes create holes in time for
892 * caching... I don't know if this is actually possible but we
893 * should still get it right */
895 unsigned dpw
= PIPE_TRANSFER_MAP_DIRECTLY
| PIPE_TRANSFER_WRITE
| PIPE_TRANSFER_PERSISTENT
;
897 if ((usage
& dpw
) == dpw
&& rsrc
->index_cache
)
900 transfer
->base
.stride
= rsrc
->slices
[level
].stride
;
901 transfer
->base
.layer_stride
= panfrost_get_layer_stride(
902 rsrc
->slices
, rsrc
->base
.target
== PIPE_TEXTURE_3D
,
903 rsrc
->cubemap_stride
, level
);
905 /* By mapping direct-write, we're implicitly already
906 * initialized (maybe), so be conservative */
908 if (usage
& PIPE_TRANSFER_WRITE
) {
909 rsrc
->slices
[level
].initialized
= true;
910 panfrost_minmax_cache_invalidate(rsrc
->index_cache
, &transfer
->base
);
914 + rsrc
->slices
[level
].offset
915 + transfer
->base
.box
.z
* transfer
->base
.layer_stride
916 + transfer
->base
.box
.y
* rsrc
->slices
[level
].stride
917 + transfer
->base
.box
.x
* bytes_per_pixel
;
922 panfrost_transfer_unmap(struct pipe_context
*pctx
,
923 struct pipe_transfer
*transfer
)
925 /* Gallium expects writeback here, so we tile */
927 struct panfrost_gtransfer
*trans
= pan_transfer(transfer
);
928 struct panfrost_resource
*prsrc
= (struct panfrost_resource
*) transfer
->resource
;
930 /* AFBC will use a staging resource. `initialized` will be set when the
931 * fragment job is created; this is deferred to prevent useless surface
932 * reloads that can cascade into DATA_INVALID_FAULTs due to reading
933 * malformed AFBC data if uninitialized */
935 if (trans
->staging
.rsrc
) {
936 if (transfer
->usage
& PIPE_TRANSFER_WRITE
) {
937 pan_blit_from_staging(pctx
, trans
);
938 panfrost_flush_batches_accessing_bo(pan_context(pctx
), pan_resource(trans
->staging
.rsrc
)->bo
, true);
941 pipe_resource_reference(&trans
->staging
.rsrc
, NULL
);
944 /* Tiling will occur in software from a staging cpu buffer */
946 struct panfrost_bo
*bo
= prsrc
->bo
;
948 if (transfer
->usage
& PIPE_TRANSFER_WRITE
) {
949 prsrc
->slices
[transfer
->level
].initialized
= true;
951 if (prsrc
->modifier
== DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED
) {
952 assert(transfer
->box
.depth
== 1);
954 /* Do we overwrite the entire resource? If so,
955 * we don't need an intermediate blit so it's a
956 * good time to switch the modifier. */
958 bool discards_content
= prsrc
->base
.last_level
== 0
959 && transfer
->box
.width
== prsrc
->base
.width0
960 && transfer
->box
.height
== prsrc
->base
.height0
961 && transfer
->box
.x
== 0
962 && transfer
->box
.y
== 0
963 && !prsrc
->modifier_constant
;
965 /* It also serves as a good heuristic for
966 * streaming textures (e.g. in video players),
967 * but we could do better */
969 if (discards_content
)
970 ++prsrc
->modifier_updates
;
972 if (prsrc
->modifier_updates
>= LAYOUT_CONVERT_THRESHOLD
)
974 prsrc
->modifier
= DRM_FORMAT_MOD_LINEAR
;
977 bo
->cpu
+ prsrc
->slices
[0].offset
,
979 prsrc
->slices
[0].stride
,
982 transfer
->box
.height
,
987 panfrost_store_tiled_image(
988 bo
->cpu
+ prsrc
->slices
[transfer
->level
].offset
,
990 transfer
->box
.x
, transfer
->box
.y
,
991 transfer
->box
.width
, transfer
->box
.height
,
992 prsrc
->slices
[transfer
->level
].stride
,
994 prsrc
->internal_format
);
1001 util_range_add(&prsrc
->base
, &prsrc
->valid_buffer_range
,
1003 transfer
->box
.x
+ transfer
->box
.width
);
1005 panfrost_minmax_cache_invalidate(prsrc
->index_cache
, transfer
);
1007 /* Derefence the resource */
1008 pipe_resource_reference(&transfer
->resource
, NULL
);
1010 /* Transfer itself is RALLOCed at the moment */
1011 ralloc_free(transfer
);
1015 panfrost_transfer_flush_region(struct pipe_context
*pctx
,
1016 struct pipe_transfer
*transfer
,
1017 const struct pipe_box
*box
)
1019 struct panfrost_resource
*rsc
= pan_resource(transfer
->resource
);
1021 if (transfer
->resource
->target
== PIPE_BUFFER
) {
1022 util_range_add(&rsc
->base
, &rsc
->valid_buffer_range
,
1023 transfer
->box
.x
+ box
->x
,
1024 transfer
->box
.x
+ box
->x
+ box
->width
);
1026 unsigned level
= transfer
->level
;
1027 rsc
->slices
[level
].initialized
= true;
1032 panfrost_invalidate_resource(struct pipe_context
*pctx
, struct pipe_resource
*prsc
)
1037 static enum pipe_format
1038 panfrost_resource_get_internal_format(struct pipe_resource
*rsrc
)
1040 struct panfrost_resource
*prsrc
= (struct panfrost_resource
*) rsrc
;
1041 return prsrc
->internal_format
;
1045 panfrost_generate_mipmap(
1046 struct pipe_context
*pctx
,
1047 struct pipe_resource
*prsrc
,
1048 enum pipe_format format
,
1049 unsigned base_level
,
1050 unsigned last_level
,
1051 unsigned first_layer
,
1052 unsigned last_layer
)
1054 struct panfrost_resource
*rsrc
= pan_resource(prsrc
);
1056 /* Generating a mipmap invalidates the written levels, so make that
1057 * explicit so we don't try to wallpaper them back and end up with
1058 * u_blitter recursion */
1061 for (unsigned l
= base_level
+ 1; l
<= last_level
; ++l
)
1062 rsrc
->slices
[l
].initialized
= false;
1064 /* Beyond that, we just delegate the hard stuff. */
1066 bool blit_res
= util_gen_mipmap(
1067 pctx
, prsrc
, format
,
1068 base_level
, last_level
,
1069 first_layer
, last_layer
,
1070 PIPE_TEX_FILTER_LINEAR
);
1075 /* Computes the address to a texture at a particular slice */
1078 panfrost_get_texture_address(
1079 struct panfrost_resource
*rsrc
,
1080 unsigned level
, unsigned face
, unsigned sample
)
1082 bool is_3d
= rsrc
->base
.target
== PIPE_TEXTURE_3D
;
1083 return rsrc
->bo
->gpu
+ panfrost_texture_offset(rsrc
->slices
, is_3d
, rsrc
->cubemap_stride
, level
, face
, sample
);
1087 panfrost_resource_set_stencil(struct pipe_resource
*prsrc
,
1088 struct pipe_resource
*stencil
)
1090 pan_resource(prsrc
)->separate_stencil
= pan_resource(stencil
);
1093 static struct pipe_resource
*
1094 panfrost_resource_get_stencil(struct pipe_resource
*prsrc
)
1096 return &pan_resource(prsrc
)->separate_stencil
->base
;
1099 static const struct u_transfer_vtbl transfer_vtbl
= {
1100 .resource_create
= panfrost_resource_create
,
1101 .resource_destroy
= panfrost_resource_destroy
,
1102 .transfer_map
= panfrost_transfer_map
,
1103 .transfer_unmap
= panfrost_transfer_unmap
,
1104 .transfer_flush_region
= panfrost_transfer_flush_region
,
1105 .get_internal_format
= panfrost_resource_get_internal_format
,
1106 .set_stencil
= panfrost_resource_set_stencil
,
1107 .get_stencil
= panfrost_resource_get_stencil
,
1111 panfrost_resource_screen_init(struct pipe_screen
*pscreen
)
1113 struct panfrost_device
*dev
= pan_device(pscreen
);
1115 bool fake_rgtc
= !panfrost_supports_compressed_format(dev
, MALI_BC4_UNORM
);
1117 pscreen
->resource_create_with_modifiers
=
1118 panfrost_resource_create_with_modifiers
;
1119 pscreen
->resource_create
= u_transfer_helper_resource_create
;
1120 pscreen
->resource_destroy
= u_transfer_helper_resource_destroy
;
1121 pscreen
->resource_from_handle
= panfrost_resource_from_handle
;
1122 pscreen
->resource_get_handle
= panfrost_resource_get_handle
;
1123 pscreen
->transfer_helper
= u_transfer_helper_create(&transfer_vtbl
,
1129 panfrost_resource_context_init(struct pipe_context
*pctx
)
1131 pctx
->transfer_map
= u_transfer_helper_transfer_map
;
1132 pctx
->transfer_unmap
= u_transfer_helper_transfer_unmap
;
1133 pctx
->create_surface
= panfrost_create_surface
;
1134 pctx
->surface_destroy
= panfrost_surface_destroy
;
1135 pctx
->resource_copy_region
= util_resource_copy_region
;
1136 pctx
->blit
= panfrost_blit
;
1137 pctx
->generate_mipmap
= panfrost_generate_mipmap
;
1138 pctx
->flush_resource
= panfrost_flush_resource
;
1139 pctx
->invalidate_resource
= panfrost_invalidate_resource
;
1140 pctx
->transfer_flush_region
= u_transfer_helper_transfer_flush_region
;
1141 pctx
->buffer_subdata
= u_default_buffer_subdata
;
1142 pctx
->texture_subdata
= u_default_texture_subdata
;