2 * Copyright (c) 2017-2019 Lima Project
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sub license,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the
12 * next paragraph) shall be included in all copies or substantial portions
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
25 #include "util/u_memory.h"
26 #include "util/u_blitter.h"
27 #include "util/format/u_format.h"
28 #include "util/u_inlines.h"
29 #include "util/u_math.h"
30 #include "util/u_debug.h"
31 #include "util/u_transfer.h"
32 #include "util/u_surface.h"
33 #include "util/hash_table.h"
34 #include "util/ralloc.h"
35 #include "util/u_drm.h"
36 #include "renderonly/renderonly.h"
38 #include "state_tracker/drm_driver.h"
40 #include "drm-uapi/drm_fourcc.h"
41 #include "drm-uapi/lima_drm.h"
43 #include "lima_screen.h"
44 #include "lima_context.h"
45 #include "lima_resource.h"
47 #include "lima_util.h"
48 #include "pan_tiling.h"
50 static struct pipe_resource
*
51 lima_resource_create_scanout(struct pipe_screen
*pscreen
,
52 const struct pipe_resource
*templat
,
53 unsigned width
, unsigned height
)
55 struct lima_screen
*screen
= lima_screen(pscreen
);
56 struct renderonly_scanout
*scanout
;
57 struct winsys_handle handle
;
58 struct pipe_resource
*pres
;
60 struct pipe_resource scanout_templat
= *templat
;
61 scanout_templat
.width0
= width
;
62 scanout_templat
.height0
= height
;
63 scanout_templat
.screen
= pscreen
;
65 scanout
= renderonly_scanout_for_resource(&scanout_templat
,
70 assert(handle
.type
== WINSYS_HANDLE_TYPE_FD
);
71 pres
= pscreen
->resource_from_handle(pscreen
, templat
, &handle
,
72 PIPE_HANDLE_USAGE_FRAMEBUFFER_WRITE
);
76 renderonly_scanout_destroy(scanout
, screen
->ro
);
80 struct lima_resource
*res
= lima_resource(pres
);
81 res
->scanout
= scanout
;
87 setup_miptree(struct lima_resource
*res
,
88 unsigned width0
, unsigned height0
,
89 bool should_align_dimensions
)
91 struct pipe_resource
*pres
= &res
->base
;
93 unsigned width
= width0
;
94 unsigned height
= height0
;
95 unsigned depth
= pres
->depth0
;
98 for (level
= 0; level
<= pres
->last_level
; level
++) {
99 uint32_t actual_level_size
;
101 unsigned aligned_width
;
102 unsigned aligned_height
;
104 if (should_align_dimensions
) {
105 aligned_width
= align(width
, 16);
106 aligned_height
= align(height
, 16);
108 aligned_width
= width
;
109 aligned_height
= height
;
112 stride
= util_format_get_stride(pres
->format
, aligned_width
);
113 actual_level_size
= stride
*
114 util_format_get_nblocksy(pres
->format
, aligned_height
) *
115 pres
->array_size
* depth
;
117 res
->levels
[level
].width
= aligned_width
;
118 res
->levels
[level
].stride
= stride
;
119 res
->levels
[level
].offset
= size
;
120 res
->levels
[level
].layer_stride
= util_format_get_stride(pres
->format
, align(width
, 16)) * align(height
, 16);
122 if (util_format_is_compressed(pres
->format
))
123 res
->levels
[level
].layer_stride
/= 4;
125 /* The start address of each level except the last level
126 * must be 64-aligned in order to be able to pass the
127 * addresses to the hardware. */
128 if (level
!= pres
->last_level
)
129 size
+= align(actual_level_size
, 64);
131 size
+= actual_level_size
; /* Save some memory */
133 width
= u_minify(width
, 1);
134 height
= u_minify(height
, 1);
135 depth
= u_minify(depth
, 1);
141 static struct pipe_resource
*
142 lima_resource_create_bo(struct pipe_screen
*pscreen
,
143 const struct pipe_resource
*templat
,
144 unsigned width
, unsigned height
,
145 bool should_align_dimensions
)
147 struct lima_screen
*screen
= lima_screen(pscreen
);
148 struct lima_resource
*res
;
149 struct pipe_resource
*pres
;
151 res
= CALLOC_STRUCT(lima_resource
);
155 res
->base
= *templat
;
156 res
->base
.screen
= pscreen
;
157 pipe_reference_init(&res
->base
.reference
, 1);
161 uint32_t size
= setup_miptree(res
, width
, height
, should_align_dimensions
);
162 size
= align(size
, LIMA_PAGE_SIZE
);
164 res
->bo
= lima_bo_create(screen
, size
, 0);
173 static struct pipe_resource
*
174 _lima_resource_create_with_modifiers(struct pipe_screen
*pscreen
,
175 const struct pipe_resource
*templat
,
176 const uint64_t *modifiers
,
179 struct lima_screen
*screen
= lima_screen(pscreen
);
180 bool should_tile
= lima_debug
& LIMA_DEBUG_NO_TILING
? false : true;
181 unsigned width
, height
;
182 bool should_align_dimensions
;
183 bool has_user_modifiers
= true;
185 if (count
== 1 && modifiers
[0] == DRM_FORMAT_MOD_INVALID
)
186 has_user_modifiers
= false;
188 /* VBOs/PBOs are untiled (and 1 height). */
189 if (templat
->target
== PIPE_BUFFER
)
192 if (templat
->bind
& (PIPE_BIND_LINEAR
| PIPE_BIND_SCANOUT
))
195 /* If there's no user modifiers and buffer is shared we use linear */
196 if (!has_user_modifiers
&& (templat
->bind
& PIPE_BIND_SHARED
))
199 if (has_user_modifiers
&&
200 !drm_find_modifier(DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED
,
204 if (should_tile
|| (templat
->bind
& PIPE_BIND_RENDER_TARGET
) ||
205 (templat
->bind
& PIPE_BIND_DEPTH_STENCIL
)) {
206 should_align_dimensions
= true;
207 width
= align(templat
->width0
, 16);
208 height
= align(templat
->height0
, 16);
211 should_align_dimensions
= false;
212 width
= templat
->width0
;
213 height
= templat
->height0
;
216 struct pipe_resource
*pres
;
217 if (screen
->ro
&& (templat
->bind
& PIPE_BIND_SCANOUT
))
218 pres
= lima_resource_create_scanout(pscreen
, templat
, width
, height
);
220 pres
= lima_resource_create_bo(pscreen
, templat
, width
, height
,
221 should_align_dimensions
);
224 struct lima_resource
*res
= lima_resource(pres
);
225 res
->tiled
= should_tile
;
227 debug_printf("%s: pres=%p width=%u height=%u depth=%u target=%d "
228 "bind=%x usage=%d tile=%d last_level=%d\n", __func__
,
229 pres
, pres
->width0
, pres
->height0
, pres
->depth0
,
230 pres
->target
, pres
->bind
, pres
->usage
, should_tile
, templat
->last_level
);
235 static struct pipe_resource
*
236 lima_resource_create(struct pipe_screen
*pscreen
,
237 const struct pipe_resource
*templat
)
239 const uint64_t mod
= DRM_FORMAT_MOD_INVALID
;
241 return _lima_resource_create_with_modifiers(pscreen
, templat
, &mod
, 1);
244 static struct pipe_resource
*
245 lima_resource_create_with_modifiers(struct pipe_screen
*pscreen
,
246 const struct pipe_resource
*templat
,
247 const uint64_t *modifiers
,
250 struct pipe_resource tmpl
= *templat
;
252 /* gbm_bo_create_with_modifiers & gbm_surface_create_with_modifiers
253 * don't have usage parameter, but buffer created by these functions
254 * may be used for scanout. So we assume buffer created by this
255 * function always enable scanout if linear modifier is permitted.
257 if (drm_find_modifier(DRM_FORMAT_MOD_LINEAR
, modifiers
, count
))
258 tmpl
.bind
|= PIPE_BIND_SCANOUT
;
260 return _lima_resource_create_with_modifiers(pscreen
, &tmpl
, modifiers
, count
);
264 lima_resource_destroy(struct pipe_screen
*pscreen
, struct pipe_resource
*pres
)
266 struct lima_screen
*screen
= lima_screen(pscreen
);
267 struct lima_resource
*res
= lima_resource(pres
);
270 lima_bo_unreference(res
->bo
);
273 renderonly_scanout_destroy(res
->scanout
, screen
->ro
);
275 if (res
->damage
.region
)
276 FREE(res
->damage
.region
);
281 static struct pipe_resource
*
282 lima_resource_from_handle(struct pipe_screen
*pscreen
,
283 const struct pipe_resource
*templat
,
284 struct winsys_handle
*handle
, unsigned usage
)
286 struct lima_resource
*res
;
287 struct lima_screen
*screen
= lima_screen(pscreen
);
289 res
= CALLOC_STRUCT(lima_resource
);
293 struct pipe_resource
*pres
= &res
->base
;
295 pres
->screen
= pscreen
;
296 pipe_reference_init(&pres
->reference
, 1);
297 res
->levels
[0].offset
= 0;
298 res
->levels
[0].stride
= handle
->stride
;
300 res
->bo
= lima_bo_import(screen
, handle
);
306 /* check alignment for the buffer */
307 if (pres
->bind
& PIPE_BIND_RENDER_TARGET
) {
308 unsigned width
, height
, stride
, size
;
310 width
= align(pres
->width0
, 16);
311 height
= align(pres
->height0
, 16);
312 stride
= util_format_get_stride(pres
->format
, width
);
313 size
= util_format_get_2d_size(pres
->format
, stride
, height
);
315 if (res
->levels
[0].stride
!= stride
|| res
->bo
->size
< size
) {
316 debug_error("import buffer not properly aligned\n");
320 res
->levels
[0].width
= width
;
323 res
->levels
[0].width
= pres
->width0
;
325 switch (handle
->modifier
) {
326 case DRM_FORMAT_MOD_LINEAR
:
329 case DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED
:
332 case DRM_FORMAT_MOD_INVALID
:
333 /* Modifier wasn't specified and it's shared buffer. We create these
334 * as linear, so disable tiling.
339 fprintf(stderr
, "Attempted to import unsupported modifier 0x%llx\n",
340 (long long)handle
->modifier
);
347 lima_resource_destroy(pscreen
, pres
);
352 lima_resource_get_handle(struct pipe_screen
*pscreen
,
353 struct pipe_context
*pctx
,
354 struct pipe_resource
*pres
,
355 struct winsys_handle
*handle
, unsigned usage
)
357 struct lima_screen
*screen
= lima_screen(pscreen
);
358 struct lima_resource
*res
= lima_resource(pres
);
361 handle
->modifier
= DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED
;
363 handle
->modifier
= DRM_FORMAT_MOD_LINEAR
;
365 if (handle
->type
== WINSYS_HANDLE_TYPE_KMS
&& screen
->ro
&&
366 renderonly_get_handle(res
->scanout
, handle
))
369 if (!lima_bo_export(res
->bo
, handle
))
372 handle
->stride
= res
->levels
[0].stride
;
377 get_scissor_from_box(struct pipe_scissor_state
*s
,
378 const struct pipe_box
*b
, int h
)
380 int y
= h
- (b
->y
+ b
->height
);
381 /* region in tile unit */
384 s
->maxx
= (b
->x
+ b
->width
+ 0xf) >> 4;
385 s
->maxy
= (y
+ b
->height
+ 0xf) >> 4;
389 get_damage_bound_box(struct pipe_resource
*pres
,
390 const struct pipe_box
*rects
,
392 struct pipe_scissor_state
*bound
)
394 struct pipe_box b
= rects
[0];
396 for (int i
= 1; i
< nrects
; i
++)
397 u_box_union_2d(&b
, &b
, rects
+ i
);
399 int ret
= u_box_clip_2d(&b
, &b
, pres
->width0
, pres
->height0
);
401 memset(bound
, 0, sizeof(*bound
));
403 get_scissor_from_box(bound
, &b
, pres
->height0
);
407 lima_resource_set_damage_region(struct pipe_screen
*pscreen
,
408 struct pipe_resource
*pres
,
410 const struct pipe_box
*rects
)
412 struct lima_resource
*res
= lima_resource(pres
);
413 struct lima_damage_region
*damage
= &res
->damage
;
416 if (damage
->region
) {
417 FREE(damage
->region
);
418 damage
->region
= NULL
;
419 damage
->num_region
= 0;
427 * TODO: currently only check if there is any single damage
428 * region that can cover the full render target; there may
429 * be some accurate way, but a single window size damage
430 * region is most of the case from weston
432 for (i
= 0; i
< nrects
; i
++) {
433 if (rects
[i
].x
<= 0 && rects
[i
].y
<= 0 &&
434 rects
[i
].x
+ rects
[i
].width
>= pres
->width0
&&
435 rects
[i
].y
+ rects
[i
].height
>= pres
->height0
)
439 struct pipe_scissor_state
*bound
= &damage
->bound
;
440 get_damage_bound_box(pres
, rects
, nrects
, bound
);
442 damage
->region
= CALLOC(nrects
, sizeof(*damage
->region
));
446 for (i
= 0; i
< nrects
; i
++)
447 get_scissor_from_box(damage
->region
+ i
, rects
+ i
,
450 /* is region aligned to tiles? */
451 damage
->aligned
= true;
452 for (i
= 0; i
< nrects
; i
++) {
453 if (rects
[i
].x
& 0xf || rects
[i
].y
& 0xf ||
454 rects
[i
].width
& 0xf || rects
[i
].height
& 0xf) {
455 damage
->aligned
= false;
460 damage
->num_region
= nrects
;
464 lima_resource_screen_init(struct lima_screen
*screen
)
466 screen
->base
.resource_create
= lima_resource_create
;
467 screen
->base
.resource_create_with_modifiers
= lima_resource_create_with_modifiers
;
468 screen
->base
.resource_from_handle
= lima_resource_from_handle
;
469 screen
->base
.resource_destroy
= lima_resource_destroy
;
470 screen
->base
.resource_get_handle
= lima_resource_get_handle
;
471 screen
->base
.set_damage_region
= lima_resource_set_damage_region
;
474 static struct pipe_surface
*
475 lima_surface_create(struct pipe_context
*pctx
,
476 struct pipe_resource
*pres
,
477 const struct pipe_surface
*surf_tmpl
)
479 struct lima_surface
*surf
= CALLOC_STRUCT(lima_surface
);
484 assert(surf_tmpl
->u
.tex
.first_layer
== surf_tmpl
->u
.tex
.last_layer
);
486 struct pipe_surface
*psurf
= &surf
->base
;
487 unsigned level
= surf_tmpl
->u
.tex
.level
;
489 pipe_reference_init(&psurf
->reference
, 1);
490 pipe_resource_reference(&psurf
->texture
, pres
);
492 psurf
->context
= pctx
;
493 psurf
->format
= surf_tmpl
->format
;
494 psurf
->width
= u_minify(pres
->width0
, level
);
495 psurf
->height
= u_minify(pres
->height0
, level
);
496 psurf
->u
.tex
.level
= level
;
497 psurf
->u
.tex
.first_layer
= surf_tmpl
->u
.tex
.first_layer
;
498 psurf
->u
.tex
.last_layer
= surf_tmpl
->u
.tex
.last_layer
;
500 surf
->tiled_w
= align(psurf
->width
, 16) >> 4;
501 surf
->tiled_h
= align(psurf
->height
, 16) >> 4;
509 lima_surface_destroy(struct pipe_context
*pctx
, struct pipe_surface
*psurf
)
511 struct lima_surface
*surf
= lima_surface(psurf
);
513 pipe_resource_reference(&psurf
->texture
, NULL
);
518 lima_transfer_map(struct pipe_context
*pctx
,
519 struct pipe_resource
*pres
,
522 const struct pipe_box
*box
,
523 struct pipe_transfer
**pptrans
)
525 struct lima_context
*ctx
= lima_context(pctx
);
526 struct lima_resource
*res
= lima_resource(pres
);
527 struct lima_bo
*bo
= res
->bo
;
528 struct lima_transfer
*trans
;
529 struct pipe_transfer
*ptrans
;
531 /* No direct mappings of tiled, since we need to manually
534 if (res
->tiled
&& (usage
& PIPE_TRANSFER_MAP_DIRECTLY
))
537 /* use once buffers are made sure to not read/write overlapped
538 * range, so no need to sync */
539 if (pres
->usage
!= PIPE_USAGE_STREAM
) {
540 if (usage
& PIPE_TRANSFER_READ_WRITE
) {
541 lima_flush_job_accessing_bo(ctx
, bo
, usage
& PIPE_TRANSFER_WRITE
);
543 unsigned op
= usage
& PIPE_TRANSFER_WRITE
?
544 LIMA_GEM_WAIT_WRITE
: LIMA_GEM_WAIT_READ
;
545 lima_bo_wait(bo
, op
, PIPE_TIMEOUT_INFINITE
);
549 if (!lima_bo_map(bo
))
552 trans
= slab_alloc(&ctx
->transfer_pool
);
556 memset(trans
, 0, sizeof(*trans
));
557 ptrans
= &trans
->base
;
559 pipe_resource_reference(&ptrans
->resource
, pres
);
560 ptrans
->level
= level
;
561 ptrans
->usage
= usage
;
567 ptrans
->stride
= util_format_get_stride(pres
->format
, ptrans
->box
.width
);
568 ptrans
->layer_stride
= ptrans
->stride
* ptrans
->box
.height
;
570 trans
->staging
= malloc(ptrans
->stride
* ptrans
->box
.height
* ptrans
->box
.depth
);
572 if (usage
& PIPE_TRANSFER_READ
) {
574 for (i
= 0; i
< ptrans
->box
.depth
; i
++)
575 panfrost_load_tiled_image(
576 trans
->staging
+ i
* ptrans
->stride
* ptrans
->box
.height
,
577 bo
->map
+ res
->levels
[level
].offset
+ (i
+ box
->z
) * res
->levels
[level
].layer_stride
,
578 ptrans
->box
.x
, ptrans
->box
.y
,
579 ptrans
->box
.width
, ptrans
->box
.height
,
581 res
->levels
[level
].stride
,
585 return trans
->staging
;
587 ptrans
->stride
= res
->levels
[level
].stride
;
588 ptrans
->layer_stride
= res
->levels
[level
].layer_stride
;
590 return bo
->map
+ res
->levels
[level
].offset
+
591 box
->z
* res
->levels
[level
].layer_stride
+
592 box
->y
/ util_format_get_blockheight(pres
->format
) * ptrans
->stride
+
593 box
->x
/ util_format_get_blockwidth(pres
->format
) *
594 util_format_get_blocksize(pres
->format
);
599 lima_transfer_flush_region(struct pipe_context
*pctx
,
600 struct pipe_transfer
*ptrans
,
601 const struct pipe_box
*box
)
607 lima_transfer_unmap(struct pipe_context
*pctx
,
608 struct pipe_transfer
*ptrans
)
610 struct lima_context
*ctx
= lima_context(pctx
);
611 struct lima_transfer
*trans
= lima_transfer(ptrans
);
612 struct lima_resource
*res
= lima_resource(ptrans
->resource
);
613 struct lima_bo
*bo
= res
->bo
;
614 struct pipe_resource
*pres
;
616 if (trans
->staging
) {
618 if (ptrans
->usage
& PIPE_TRANSFER_WRITE
) {
620 for (i
= 0; i
< ptrans
->box
.depth
; i
++)
621 panfrost_store_tiled_image(
622 bo
->map
+ res
->levels
[ptrans
->level
].offset
+ (i
+ ptrans
->box
.z
) * res
->levels
[ptrans
->level
].layer_stride
,
623 trans
->staging
+ i
* ptrans
->stride
* ptrans
->box
.height
,
624 ptrans
->box
.x
, ptrans
->box
.y
,
625 ptrans
->box
.width
, ptrans
->box
.height
,
626 res
->levels
[ptrans
->level
].stride
,
630 free(trans
->staging
);
633 pipe_resource_reference(&ptrans
->resource
, NULL
);
634 slab_free(&ctx
->transfer_pool
, trans
);
638 lima_util_blitter_save_states(struct lima_context
*ctx
)
640 util_blitter_save_blend(ctx
->blitter
, (void *)ctx
->blend
);
641 util_blitter_save_depth_stencil_alpha(ctx
->blitter
, (void *)ctx
->zsa
);
642 util_blitter_save_stencil_ref(ctx
->blitter
, &ctx
->stencil_ref
);
643 util_blitter_save_rasterizer(ctx
->blitter
, (void *)ctx
->rasterizer
);
644 util_blitter_save_fragment_shader(ctx
->blitter
, ctx
->fs
);
645 util_blitter_save_vertex_shader(ctx
->blitter
, ctx
->vs
);
646 util_blitter_save_viewport(ctx
->blitter
,
647 &ctx
->viewport
.transform
);
648 util_blitter_save_scissor(ctx
->blitter
, &ctx
->scissor
);
649 util_blitter_save_vertex_elements(ctx
->blitter
,
650 ctx
->vertex_elements
);
651 util_blitter_save_vertex_buffer_slot(ctx
->blitter
,
652 ctx
->vertex_buffers
.vb
);
654 util_blitter_save_framebuffer(ctx
->blitter
, &ctx
->framebuffer
.base
);
656 util_blitter_save_fragment_sampler_states(ctx
->blitter
,
657 ctx
->tex_stateobj
.num_samplers
,
658 (void**)ctx
->tex_stateobj
.samplers
);
659 util_blitter_save_fragment_sampler_views(ctx
->blitter
,
660 ctx
->tex_stateobj
.num_textures
,
661 ctx
->tex_stateobj
.textures
);
665 lima_blit(struct pipe_context
*pctx
, const struct pipe_blit_info
*blit_info
)
667 struct lima_context
*ctx
= lima_context(pctx
);
668 struct pipe_blit_info info
= *blit_info
;
670 if (util_try_blit_via_copy_region(pctx
, &info
)) {
674 if (info
.mask
& PIPE_MASK_S
) {
675 debug_printf("lima: cannot blit stencil, skipping\n");
676 info
.mask
&= ~PIPE_MASK_S
;
679 if (!util_blitter_is_blit_supported(ctx
->blitter
, &info
)) {
680 debug_printf("lima: blit unsupported %s -> %s\n",
681 util_format_short_name(info
.src
.resource
->format
),
682 util_format_short_name(info
.dst
.resource
->format
));
686 lima_util_blitter_save_states(ctx
);
688 util_blitter_blit(ctx
->blitter
, &info
);
692 lima_flush_resource(struct pipe_context
*pctx
, struct pipe_resource
*resource
)
698 lima_resource_context_init(struct lima_context
*ctx
)
700 ctx
->base
.create_surface
= lima_surface_create
;
701 ctx
->base
.surface_destroy
= lima_surface_destroy
;
703 /* TODO: optimize these functions to read/write data directly
704 * from/to target instead of creating a staging memory for tiled
707 ctx
->base
.buffer_subdata
= u_default_buffer_subdata
;
708 ctx
->base
.texture_subdata
= u_default_texture_subdata
;
709 ctx
->base
.resource_copy_region
= util_resource_copy_region
;
711 ctx
->base
.blit
= lima_blit
;
713 ctx
->base
.transfer_map
= lima_transfer_map
;
714 ctx
->base
.transfer_flush_region
= lima_transfer_flush_region
;
715 ctx
->base
.transfer_unmap
= lima_transfer_unmap
;
717 ctx
->base
.flush_resource
= lima_flush_resource
;