2 * Copyright © 2017 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 #include "pipe/p_defines.h"
26 #include "pipe/p_state.h"
27 #include "pipe/p_context.h"
28 #include "pipe/p_screen.h"
29 #include "util/os_memory.h"
30 #include "util/u_cpu_detect.h"
31 #include "util/u_inlines.h"
32 #include "util/u_format.h"
33 #include "util/u_transfer.h"
34 #include "util/u_upload_mgr.h"
35 #include "util/ralloc.h"
36 #include "iris_batch.h"
37 #include "iris_context.h"
38 #include "iris_resource.h"
39 #include "iris_screen.h"
40 #include "intel/common/gen_debug.h"
42 #include "drm-uapi/drm_fourcc.h"
43 #include "drm-uapi/i915_drm.h"
45 // XXX: u_transfer_helper...for separate stencil...
47 enum modifier_priority
{
48 MODIFIER_PRIORITY_INVALID
= 0,
49 MODIFIER_PRIORITY_LINEAR
,
52 MODIFIER_PRIORITY_Y_CCS
,
55 static const uint64_t priority_to_modifier
[] = {
56 [MODIFIER_PRIORITY_INVALID
] = DRM_FORMAT_MOD_INVALID
,
57 [MODIFIER_PRIORITY_LINEAR
] = DRM_FORMAT_MOD_LINEAR
,
58 [MODIFIER_PRIORITY_X
] = I915_FORMAT_MOD_X_TILED
,
59 [MODIFIER_PRIORITY_Y
] = I915_FORMAT_MOD_Y_TILED
,
60 [MODIFIER_PRIORITY_Y_CCS
] = I915_FORMAT_MOD_Y_TILED_CCS
,
64 modifier_is_supported(const struct gen_device_info
*devinfo
,
67 /* XXX: do something real */
69 case I915_FORMAT_MOD_Y_TILED
:
70 case I915_FORMAT_MOD_X_TILED
:
71 case DRM_FORMAT_MOD_LINEAR
:
73 case I915_FORMAT_MOD_Y_TILED_CCS
:
74 case DRM_FORMAT_MOD_INVALID
:
81 select_best_modifier(struct gen_device_info
*devinfo
,
82 const uint64_t *modifiers
,
85 enum modifier_priority prio
= MODIFIER_PRIORITY_INVALID
;
87 for (int i
= 0; i
< count
; i
++) {
88 if (!modifier_is_supported(devinfo
, modifiers
[i
]))
91 switch (modifiers
[i
]) {
92 case I915_FORMAT_MOD_Y_TILED_CCS
:
93 prio
= MAX2(prio
, MODIFIER_PRIORITY_Y_CCS
);
95 case I915_FORMAT_MOD_Y_TILED
:
96 prio
= MAX2(prio
, MODIFIER_PRIORITY_Y
);
98 case I915_FORMAT_MOD_X_TILED
:
99 prio
= MAX2(prio
, MODIFIER_PRIORITY_X
);
101 case DRM_FORMAT_MOD_LINEAR
:
102 prio
= MAX2(prio
, MODIFIER_PRIORITY_LINEAR
);
104 case DRM_FORMAT_MOD_INVALID
:
110 return priority_to_modifier
[prio
];
113 static enum isl_surf_dim
114 target_to_isl_surf_dim(enum pipe_texture_target target
)
118 case PIPE_TEXTURE_1D
:
119 case PIPE_TEXTURE_1D_ARRAY
:
120 return ISL_SURF_DIM_1D
;
121 case PIPE_TEXTURE_2D
:
122 case PIPE_TEXTURE_CUBE
:
123 case PIPE_TEXTURE_RECT
:
124 case PIPE_TEXTURE_2D_ARRAY
:
125 case PIPE_TEXTURE_CUBE_ARRAY
:
126 return ISL_SURF_DIM_2D
;
127 case PIPE_TEXTURE_3D
:
128 return ISL_SURF_DIM_3D
;
129 case PIPE_MAX_TEXTURE_TYPES
:
132 unreachable("invalid texture type");
135 static isl_surf_usage_flags_t
136 pipe_bind_to_isl_usage(unsigned bindings
)
138 isl_surf_usage_flags_t usage
= 0;
140 if (bindings
& PIPE_BIND_RENDER_TARGET
)
141 usage
|= ISL_SURF_USAGE_RENDER_TARGET_BIT
;
143 if (bindings
& PIPE_BIND_SAMPLER_VIEW
)
144 usage
|= ISL_SURF_USAGE_TEXTURE_BIT
;
146 if (bindings
& (PIPE_BIND_SHADER_IMAGE
| PIPE_BIND_SHADER_BUFFER
))
147 usage
|= ISL_SURF_USAGE_STORAGE_BIT
;
149 if (bindings
& PIPE_BIND_DISPLAY_TARGET
)
150 usage
|= ISL_SURF_USAGE_DISPLAY_BIT
;
156 iris_resource_destroy(struct pipe_screen
*screen
,
157 struct pipe_resource
*resource
)
159 struct iris_resource
*res
= (struct iris_resource
*)resource
;
161 iris_bo_unreference(res
->bo
);
165 static struct iris_resource
*
166 iris_alloc_resource(struct pipe_screen
*pscreen
,
167 const struct pipe_resource
*templ
)
169 struct iris_resource
*res
= calloc(1, sizeof(struct iris_resource
));
174 res
->base
.screen
= pscreen
;
175 pipe_reference_init(&res
->base
.reference
, 1);
180 static struct pipe_resource
*
181 iris_resource_create_with_modifiers(struct pipe_screen
*pscreen
,
182 const struct pipe_resource
*templ
,
183 const uint64_t *modifiers
,
186 struct iris_screen
*screen
= (struct iris_screen
*)pscreen
;
187 struct gen_device_info
*devinfo
= &screen
->devinfo
;
188 struct iris_resource
*res
= iris_alloc_resource(pscreen
, templ
);
192 bool depth
= util_format_is_depth_or_stencil(templ
->format
);
194 uint64_t modifier
= DRM_FORMAT_MOD_INVALID
;
196 if (modifiers_count
== 0 || !modifiers
) {
198 modifier
= I915_FORMAT_MOD_Y_TILED
;
199 } else if (templ
->bind
& PIPE_BIND_DISPLAY_TARGET
) {
200 /* Display is X-tiled for historical reasons. */
201 modifier
= I915_FORMAT_MOD_X_TILED
;
203 modifier
= I915_FORMAT_MOD_Y_TILED
;
205 /* XXX: make sure this doesn't do stupid things for internal textures */
208 if (templ
->target
== PIPE_BUFFER
|| templ
->usage
== PIPE_USAGE_STAGING
)
209 modifier
= DRM_FORMAT_MOD_LINEAR
;
211 if (templ
->bind
& (PIPE_BIND_LINEAR
| PIPE_BIND_CURSOR
))
212 modifier
= DRM_FORMAT_MOD_LINEAR
;
214 if (modifier
== DRM_FORMAT_MOD_INVALID
) {
215 /* User requested specific modifiers */
216 modifier
= select_best_modifier(devinfo
, modifiers
, modifiers_count
);
217 if (modifier
== DRM_FORMAT_MOD_INVALID
)
221 const struct isl_drm_modifier_info
*mod_info
=
222 isl_drm_modifier_get_info(modifier
);
224 isl_surf_usage_flags_t usage
= pipe_bind_to_isl_usage(templ
->bind
);
226 if (templ
->target
== PIPE_TEXTURE_CUBE
)
227 usage
|= ISL_SURF_USAGE_CUBE_BIT
;
229 // XXX: separate stencil...
230 enum pipe_format pfmt
= templ
->format
;
232 if (util_format_is_depth_or_stencil(pfmt
) &&
233 templ
->usage
!= PIPE_USAGE_STAGING
)
234 usage
|= ISL_SURF_USAGE_DEPTH_BIT
;
236 if (util_format_is_depth_and_stencil(pfmt
)) {
238 pfmt
= PIPE_FORMAT_X8Z24_UNORM
;
241 enum isl_format isl_format
= iris_isl_format_for_pipe_format(pfmt
);
242 assert(isl_format
!= ISL_FORMAT_UNSUPPORTED
);
244 UNUSED
const bool isl_surf_created_successfully
=
245 isl_surf_init(&screen
->isl_dev
, &res
->surf
,
246 .dim
= target_to_isl_surf_dim(templ
->target
),
247 .format
= isl_format
,
248 .width
= templ
->width0
,
249 .height
= templ
->height0
,
250 .depth
= templ
->depth0
,
251 .levels
= templ
->last_level
+ 1,
252 .array_len
= templ
->array_size
,
253 .samples
= MAX2(templ
->nr_samples
, 1),
254 .min_alignment_B
= 0,
257 .tiling_flags
= 1 << mod_info
->tiling
);
258 assert(isl_surf_created_successfully
);
260 enum iris_memory_zone memzone
= IRIS_MEMZONE_OTHER
;
261 const char *name
= templ
->target
== PIPE_BUFFER
? "buffer" : "miptree";
262 if (templ
->flags
& IRIS_RESOURCE_FLAG_SHADER_MEMZONE
) {
263 memzone
= IRIS_MEMZONE_SHADER
;
264 name
= "shader kernels";
265 } else if (templ
->flags
& IRIS_RESOURCE_FLAG_SURFACE_MEMZONE
) {
266 memzone
= IRIS_MEMZONE_SURFACE
;
267 name
= "surface state";
268 } else if (templ
->flags
& IRIS_RESOURCE_FLAG_DYNAMIC_MEMZONE
) {
269 memzone
= IRIS_MEMZONE_DYNAMIC
;
270 name
= "dynamic state";
273 res
->bo
= iris_bo_alloc_tiled(screen
->bufmgr
, name
, res
->surf
.size_B
,
275 isl_tiling_to_i915_tiling(res
->surf
.tiling
),
276 res
->surf
.row_pitch_B
, 0);
283 iris_resource_destroy(pscreen
, &res
->base
);
287 static struct pipe_resource
*
288 iris_resource_create(struct pipe_screen
*pscreen
,
289 const struct pipe_resource
*templ
)
291 return iris_resource_create_with_modifiers(pscreen
, templ
, NULL
, 0);
295 tiling_to_modifier(uint32_t tiling
)
297 static const uint64_t map
[] = {
298 [I915_TILING_NONE
] = DRM_FORMAT_MOD_LINEAR
,
299 [I915_TILING_X
] = I915_FORMAT_MOD_X_TILED
,
300 [I915_TILING_Y
] = I915_FORMAT_MOD_Y_TILED
,
303 assert(tiling
< ARRAY_SIZE(map
));
308 static struct pipe_resource
*
309 iris_resource_from_handle(struct pipe_screen
*pscreen
,
310 const struct pipe_resource
*templ
,
311 struct winsys_handle
*whandle
,
314 struct iris_screen
*screen
= (struct iris_screen
*)pscreen
;
315 struct iris_bufmgr
*bufmgr
= screen
->bufmgr
;
316 struct iris_resource
*res
= iris_alloc_resource(pscreen
, templ
);
320 if (whandle
->offset
!= 0) {
321 dbg_printf("Attempt to import unsupported winsys offset %u\n",
326 switch (whandle
->type
) {
327 case WINSYS_HANDLE_TYPE_FD
:
328 res
->bo
= iris_bo_import_dmabuf(bufmgr
, whandle
->handle
);
330 case WINSYS_HANDLE_TYPE_SHARED
:
331 res
->bo
= iris_bo_gem_create_from_name(bufmgr
, "winsys image",
335 unreachable("invalid winsys handle type");
340 uint64_t modifier
= whandle
->modifier
;
341 if (modifier
== DRM_FORMAT_MOD_INVALID
) {
342 modifier
= tiling_to_modifier(res
->bo
->tiling_mode
);
344 const struct isl_drm_modifier_info
*mod_info
=
345 isl_drm_modifier_get_info(modifier
);
349 isl_surf_usage_flags_t isl_usage
= ISL_SURF_USAGE_DISPLAY_BIT
;
351 isl_surf_init(&screen
->isl_dev
, &res
->surf
,
352 .dim
= target_to_isl_surf_dim(templ
->target
),
353 .format
= iris_isl_format_for_pipe_format(templ
->format
),
354 .width
= templ
->width0
,
355 .height
= templ
->height0
,
356 .depth
= templ
->depth0
,
357 .levels
= templ
->last_level
+ 1,
358 .array_len
= templ
->array_size
,
359 .samples
= MAX2(templ
->nr_samples
, 1),
360 .min_alignment_B
= 0,
363 .tiling_flags
= 1 << mod_info
->tiling
);
365 assert(res
->bo
->tiling_mode
== isl_tiling_to_i915_tiling(res
->surf
.tiling
));
370 iris_resource_destroy(pscreen
, &res
->base
);
375 iris_resource_get_handle(struct pipe_screen
*pscreen
,
376 struct pipe_context
*ctx
,
377 struct pipe_resource
*resource
,
378 struct winsys_handle
*whandle
,
381 struct iris_resource
*res
= (struct iris_resource
*)resource
;
383 whandle
->stride
= res
->surf
.row_pitch_B
;
385 switch (whandle
->type
) {
386 case WINSYS_HANDLE_TYPE_SHARED
:
387 return iris_bo_flink(res
->bo
, &whandle
->handle
) == 0;
388 case WINSYS_HANDLE_TYPE_KMS
:
389 return iris_bo_export_gem_handle(res
->bo
) != 0;
390 case WINSYS_HANDLE_TYPE_FD
:
391 return iris_bo_export_dmabuf(res
->bo
, (int *) &whandle
->handle
) == 0;
397 /* Compute extent parameters for use with tiled_memcpy functions.
398 * xs are in units of bytes and ys are in units of strides.
401 tile_extents(struct isl_surf
*surf
,
402 const struct pipe_box
*box
,
404 unsigned int *x1_B
, unsigned int *x2_B
,
405 unsigned int *y1_el
, unsigned int *y2_el
)
407 const struct isl_format_layout
*fmtl
= isl_format_get_layout(surf
->format
);
408 const unsigned cpp
= fmtl
->bpb
/ 8;
410 assert(box
->x
% fmtl
->bw
== 0);
411 assert(box
->y
% fmtl
->bh
== 0);
413 unsigned x0_el
, y0_el
;
414 if (surf
->dim
== ISL_SURF_DIM_3D
) {
415 isl_surf_get_image_offset_el(surf
, level
, 0, box
->z
, &x0_el
, &y0_el
);
417 isl_surf_get_image_offset_el(surf
, level
, box
->z
, 0, &x0_el
, &y0_el
);
420 *x1_B
= (box
->x
/ fmtl
->bw
+ x0_el
) * cpp
;
421 *y1_el
= box
->y
/ fmtl
->bh
+ y0_el
;
422 *x2_B
= (DIV_ROUND_UP(box
->x
+ box
->width
, fmtl
->bw
) + x0_el
) * cpp
;
423 *y2_el
= DIV_ROUND_UP(box
->y
+ box
->height
, fmtl
->bh
) + y0_el
;
427 iris_unmap_tiled_memcpy(struct iris_transfer
*map
)
429 struct pipe_transfer
*xfer
= &map
->base
;
430 struct pipe_box box
= xfer
->box
;
431 struct iris_resource
*res
= (struct iris_resource
*) xfer
->resource
;
432 struct isl_surf
*surf
= &res
->surf
;
434 const bool has_swizzling
= false; // XXX: swizzling?
436 if (xfer
->usage
& PIPE_TRANSFER_WRITE
) {
437 char *dst
= iris_bo_map(map
->dbg
, res
->bo
, xfer
->usage
| MAP_RAW
);
439 for (int s
= 0; s
< box
.depth
; s
++) {
440 unsigned int x1
, x2
, y1
, y2
;
441 tile_extents(surf
, &box
, xfer
->level
, &x1
, &x2
, &y1
, &y2
);
443 void *ptr
= map
->ptr
+ box
.z
* xfer
->layer_stride
;
445 isl_memcpy_linear_to_tiled(x1
, x2
, y1
, y2
, dst
, ptr
,
446 surf
->row_pitch_B
, xfer
->stride
,
447 has_swizzling
, surf
->tiling
, ISL_MEMCPY
);
451 os_free_aligned(map
->buffer
);
452 map
->buffer
= map
->ptr
= NULL
;
456 iris_map_tiled_memcpy(struct iris_transfer
*map
)
458 struct pipe_transfer
*xfer
= &map
->base
;
459 struct iris_resource
*res
= (struct iris_resource
*) xfer
->resource
;
460 struct isl_surf
*surf
= &res
->surf
;
462 xfer
->stride
= ALIGN(surf
->row_pitch_B
, 16);
463 xfer
->layer_stride
= xfer
->stride
* xfer
->box
.height
;
465 unsigned x1
, x2
, y1
, y2
;
466 tile_extents(surf
, &xfer
->box
, xfer
->level
, &x1
, &x2
, &y1
, &y2
);
468 /* The tiling and detiling functions require that the linear buffer has
469 * a 16-byte alignment (that is, its `x0` is 16-byte aligned). Here we
470 * over-allocate the linear buffer to get the proper alignment.
473 os_malloc_aligned(xfer
->layer_stride
* xfer
->box
.depth
, 16);
475 map
->ptr
= (char *)map
->buffer
+ (x1
& 0xf);
477 const bool has_swizzling
= false; // XXX: swizzling?
479 // XXX: PIPE_TRANSFER_READ?
480 if (!(xfer
->usage
& PIPE_TRANSFER_DISCARD_RANGE
)) {
481 char *src
= iris_bo_map(map
->dbg
, res
->bo
, xfer
->usage
| MAP_RAW
);
483 struct pipe_box box
= xfer
->box
;
485 for (int s
= 0; s
< box
.depth
; s
++) {
486 unsigned int x1
, x2
, y1
, y2
;
487 tile_extents(surf
, &box
, xfer
->level
, &x1
, &x2
, &y1
, &y2
);
489 isl_memcpy_tiled_to_linear(x1
, x2
, y1
, y2
, map
->ptr
, src
,
490 xfer
->stride
, surf
->row_pitch_B
,
491 has_swizzling
, surf
->tiling
, ISL_MEMCPY
);
496 map
->unmap
= iris_unmap_tiled_memcpy
;
500 iris_map_direct(struct iris_transfer
*map
)
502 struct pipe_transfer
*xfer
= &map
->base
;
503 struct pipe_box
*box
= &xfer
->box
;
504 struct iris_resource
*res
= (struct iris_resource
*) xfer
->resource
;
505 struct isl_surf
*surf
= &res
->surf
;
506 const struct isl_format_layout
*fmtl
= isl_format_get_layout(surf
->format
);
507 const unsigned cpp
= fmtl
->bpb
/ 8;
509 xfer
->stride
= isl_surf_get_row_pitch_B(surf
);
510 xfer
->layer_stride
= isl_surf_get_array_pitch(surf
);
512 void *ptr
= iris_bo_map(map
->dbg
, res
->bo
, xfer
->usage
);
514 // XXX: level, layer, etc
515 assert(xfer
->level
== 0);
518 map
->ptr
= ptr
+ box
->y
* xfer
->stride
+ box
->x
* cpp
;
522 iris_transfer_map(struct pipe_context
*ctx
,
523 struct pipe_resource
*resource
,
525 enum pipe_transfer_usage usage
,
526 const struct pipe_box
*box
,
527 struct pipe_transfer
**ptransfer
)
529 struct iris_context
*ice
= (struct iris_context
*)ctx
;
530 struct iris_resource
*res
= (struct iris_resource
*)resource
;
531 struct isl_surf
*surf
= &res
->surf
;
533 if (surf
->tiling
!= ISL_TILING_LINEAR
&&
534 (usage
& PIPE_TRANSFER_MAP_DIRECTLY
))
537 if (!(usage
& PIPE_TRANSFER_UNSYNCHRONIZED
) &&
538 iris_batch_references(&ice
->render_batch
, res
->bo
)) {
539 iris_batch_flush(&ice
->render_batch
);
542 if ((usage
& PIPE_TRANSFER_DONTBLOCK
) && iris_bo_busy(res
->bo
))
545 struct iris_transfer
*map
= slab_alloc(&ice
->transfer_pool
);
546 struct pipe_transfer
*xfer
= &map
->base
;
548 // PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
549 // PIPE_TRANSFER_DISCARD_RANGE
554 memset(map
, 0, sizeof(*map
));
555 map
->dbg
= &ice
->dbg
;
557 pipe_resource_reference(&xfer
->resource
, resource
);
563 xfer
->usage
&= (PIPE_TRANSFER_READ
|
564 PIPE_TRANSFER_WRITE
|
565 PIPE_TRANSFER_UNSYNCHRONIZED
|
566 PIPE_TRANSFER_PERSISTENT
|
567 PIPE_TRANSFER_COHERENT
|
568 PIPE_TRANSFER_DISCARD_RANGE
);
570 if (surf
->tiling
!= ISL_TILING_LINEAR
) {
571 iris_map_tiled_memcpy(map
);
573 iris_map_direct(map
);
580 iris_transfer_flush_region(struct pipe_context
*pipe
,
581 struct pipe_transfer
*transfer
,
582 const struct pipe_box
*box
)
587 iris_transfer_unmap(struct pipe_context
*ctx
, struct pipe_transfer
*xfer
)
589 struct iris_context
*ice
= (struct iris_context
*)ctx
;
590 struct iris_transfer
*map
= (void *) xfer
;
595 pipe_resource_reference(&xfer
->resource
, NULL
);
596 slab_free(&ice
->transfer_pool
, map
);
600 iris_flush_resource(struct pipe_context
*ctx
, struct pipe_resource
*resource
)
605 iris_init_screen_resource_functions(struct pipe_screen
*pscreen
)
607 pscreen
->resource_create_with_modifiers
=
608 iris_resource_create_with_modifiers
;
609 pscreen
->resource_create
= iris_resource_create
;
610 pscreen
->resource_from_handle
= iris_resource_from_handle
;
611 pscreen
->resource_get_handle
= iris_resource_get_handle
;
612 pscreen
->resource_destroy
= iris_resource_destroy
;
616 iris_init_resource_functions(struct pipe_context
*ctx
)
618 ctx
->flush_resource
= iris_flush_resource
;
619 ctx
->transfer_map
= iris_transfer_map
;
620 ctx
->transfer_flush_region
= iris_transfer_flush_region
;
621 ctx
->transfer_unmap
= iris_transfer_unmap
;
622 ctx
->buffer_subdata
= u_default_buffer_subdata
;
623 ctx
->texture_subdata
= u_default_texture_subdata
;