2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
5 * based in part on anv driver which is:
6 * Copyright © 2015 Intel Corporation
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the "Software"),
10 * to deal in the Software without restriction, including without limitation
11 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 * and/or sell copies of the Software, and to permit persons to whom the
13 * Software is furnished to do so, subject to the following conditions:
15 * The above copyright notice and this permission notice (including the next
16 * paragraph) shall be included in all copies or substantial portions of the
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
28 #include "radv_private.h"
29 #include "vk_format.h"
31 #include "radv_radeon_winsys.h"
34 #include "util/debug.h"
35 #include "util/u_atomic.h"
37 radv_choose_tiling(struct radv_device
*Device
,
38 const struct radv_image_create_info
*create_info
)
40 const VkImageCreateInfo
*pCreateInfo
= create_info
->vk_info
;
42 if (pCreateInfo
->tiling
== VK_IMAGE_TILING_LINEAR
) {
43 assert(pCreateInfo
->samples
<= 1);
44 return RADEON_SURF_MODE_LINEAR_ALIGNED
;
47 /* Textures with a very small height are recommended to be linear. */
48 if (pCreateInfo
->imageType
== VK_IMAGE_TYPE_1D
||
49 /* Only very thin and long 2D textures should benefit from
51 (pCreateInfo
->extent
.width
> 8 && pCreateInfo
->extent
.height
<= 2))
52 return RADEON_SURF_MODE_LINEAR_ALIGNED
;
54 /* MSAA resources must be 2D tiled. */
55 if (pCreateInfo
->samples
> 1)
56 return RADEON_SURF_MODE_2D
;
58 return RADEON_SURF_MODE_2D
;
61 radv_init_surface(struct radv_device
*device
,
62 struct radeon_surf
*surface
,
63 const struct radv_image_create_info
*create_info
)
65 const VkImageCreateInfo
*pCreateInfo
= create_info
->vk_info
;
66 unsigned array_mode
= radv_choose_tiling(device
, create_info
);
67 const struct vk_format_description
*desc
=
68 vk_format_description(pCreateInfo
->format
);
69 bool is_depth
, is_stencil
, blendable
;
71 is_depth
= vk_format_has_depth(desc
);
72 is_stencil
= vk_format_has_stencil(desc
);
74 surface
->blk_w
= vk_format_get_blockwidth(pCreateInfo
->format
);
75 surface
->blk_h
= vk_format_get_blockheight(pCreateInfo
->format
);
77 surface
->bpe
= vk_format_get_blocksize(vk_format_depth_only(pCreateInfo
->format
));
78 /* align byte per element on dword */
79 if (surface
->bpe
== 3) {
82 surface
->flags
= RADEON_SURF_SET(array_mode
, MODE
);
84 switch (pCreateInfo
->imageType
){
85 case VK_IMAGE_TYPE_1D
:
86 if (pCreateInfo
->arrayLayers
> 1)
87 surface
->flags
|= RADEON_SURF_SET(RADEON_SURF_TYPE_1D_ARRAY
, TYPE
);
89 surface
->flags
|= RADEON_SURF_SET(RADEON_SURF_TYPE_1D
, TYPE
);
91 case VK_IMAGE_TYPE_2D
:
92 if (pCreateInfo
->arrayLayers
> 1)
93 surface
->flags
|= RADEON_SURF_SET(RADEON_SURF_TYPE_2D_ARRAY
, TYPE
);
95 surface
->flags
|= RADEON_SURF_SET(RADEON_SURF_TYPE_2D
, TYPE
);
97 case VK_IMAGE_TYPE_3D
:
98 surface
->flags
|= RADEON_SURF_SET(RADEON_SURF_TYPE_3D
, TYPE
);
101 unreachable("unhandled image type");
105 surface
->flags
|= RADEON_SURF_ZBUFFER
;
109 surface
->flags
|= RADEON_SURF_SBUFFER
;
111 surface
->flags
|= RADEON_SURF_HAS_TILE_MODE_INDEX
;
113 if ((pCreateInfo
->usage
& (VK_IMAGE_USAGE_TRANSFER_SRC_BIT
|
114 VK_IMAGE_USAGE_STORAGE_BIT
)) ||
115 (pCreateInfo
->flags
& VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT
) ||
116 (pCreateInfo
->tiling
== VK_IMAGE_TILING_LINEAR
) ||
117 device
->physical_device
->rad_info
.chip_class
< VI
||
118 create_info
->scanout
|| (device
->debug_flags
& RADV_DEBUG_NO_DCC
) ||
119 !radv_is_colorbuffer_format_supported(pCreateInfo
->format
, &blendable
))
120 surface
->flags
|= RADEON_SURF_DISABLE_DCC
;
121 if (create_info
->scanout
)
122 surface
->flags
|= RADEON_SURF_SCANOUT
;
125 #define ATI_VENDOR_ID 0x1002
126 static uint32_t si_get_bo_metadata_word1(struct radv_device
*device
)
128 return (ATI_VENDOR_ID
<< 16) | device
->physical_device
->rad_info
.pci_id
;
131 static inline unsigned
132 si_tile_mode_index(const struct radv_image
*image
, unsigned level
, bool stencil
)
135 return image
->surface
.u
.legacy
.stencil_tiling_index
[level
];
137 return image
->surface
.u
.legacy
.tiling_index
[level
];
140 static unsigned radv_map_swizzle(unsigned swizzle
)
144 return V_008F0C_SQ_SEL_Y
;
146 return V_008F0C_SQ_SEL_Z
;
148 return V_008F0C_SQ_SEL_W
;
150 return V_008F0C_SQ_SEL_0
;
152 return V_008F0C_SQ_SEL_1
;
153 default: /* VK_SWIZZLE_X */
154 return V_008F0C_SQ_SEL_X
;
159 radv_make_buffer_descriptor(struct radv_device
*device
,
160 struct radv_buffer
*buffer
,
166 const struct vk_format_description
*desc
;
168 uint64_t gpu_address
= device
->ws
->buffer_get_va(buffer
->bo
);
169 uint64_t va
= gpu_address
+ buffer
->offset
;
170 unsigned num_format
, data_format
;
172 desc
= vk_format_description(vk_format
);
173 first_non_void
= vk_format_get_first_non_void_channel(vk_format
);
174 stride
= desc
->block
.bits
/ 8;
176 num_format
= radv_translate_buffer_numformat(desc
, first_non_void
);
177 data_format
= radv_translate_buffer_dataformat(desc
, first_non_void
);
181 state
[1] = S_008F04_BASE_ADDRESS_HI(va
>> 32) |
182 S_008F04_STRIDE(stride
);
184 state
[3] = S_008F0C_DST_SEL_X(radv_map_swizzle(desc
->swizzle
[0])) |
185 S_008F0C_DST_SEL_Y(radv_map_swizzle(desc
->swizzle
[1])) |
186 S_008F0C_DST_SEL_Z(radv_map_swizzle(desc
->swizzle
[2])) |
187 S_008F0C_DST_SEL_W(radv_map_swizzle(desc
->swizzle
[3])) |
188 S_008F0C_NUM_FORMAT(num_format
) |
189 S_008F0C_DATA_FORMAT(data_format
);
193 si_set_mutable_tex_desc_fields(struct radv_device
*device
,
194 struct radv_image
*image
,
195 const struct legacy_surf_level
*base_level_info
,
196 unsigned base_level
, unsigned first_level
,
197 unsigned block_width
, bool is_stencil
,
200 uint64_t gpu_address
= device
->ws
->buffer_get_va(image
->bo
) + image
->offset
;
201 uint64_t va
= gpu_address
;
202 unsigned pitch
= base_level_info
->nblk_x
* block_width
;
203 enum chip_class chip_class
= device
->physical_device
->rad_info
.chip_class
;
204 uint64_t meta_va
= 0;
205 if (chip_class
>= GFX9
) {
207 va
+= image
->surface
.u
.gfx9
.stencil_offset
;
209 va
+= image
->surface
.u
.gfx9
.surf_offset
;
211 va
+= base_level_info
->offset
;
214 if (chip_class
< GFX9
)
215 state
[0] |= image
->surface
.u
.legacy
.tile_swizzle
;
216 state
[1] &= C_008F14_BASE_ADDRESS_HI
;
217 state
[1] |= S_008F14_BASE_ADDRESS_HI(va
>> 40);
218 state
[3] |= S_008F1C_TILING_INDEX(si_tile_mode_index(image
, base_level
,
220 state
[4] |= S_008F20_PITCH_GFX6(pitch
- 1);
222 if (chip_class
>= VI
) {
223 state
[6] &= C_008F28_COMPRESSION_EN
;
225 if (image
->surface
.dcc_size
&& first_level
< image
->surface
.num_dcc_levels
) {
226 uint64_t meta_va
= gpu_address
+ image
->dcc_offset
;
227 if (chip_class
<= VI
)
228 meta_va
+= base_level_info
->dcc_offset
;
229 state
[6] |= S_008F28_COMPRESSION_EN(1);
230 state
[7] = meta_va
>> 8;
231 if (chip_class
< GFX9
)
232 state
[7] |= image
->surface
.u
.legacy
.tile_swizzle
;
236 if (chip_class
>= GFX9
) {
237 state
[3] &= C_008F1C_SW_MODE
;
238 state
[4] &= C_008F20_PITCH_GFX9
;
241 state
[3] |= S_008F1C_SW_MODE(image
->surface
.u
.gfx9
.stencil
.swizzle_mode
);
242 state
[4] |= S_008F20_PITCH_GFX9(image
->surface
.u
.gfx9
.stencil
.epitch
);
244 state
[3] |= S_008F1C_SW_MODE(image
->surface
.u
.gfx9
.surf
.swizzle_mode
);
245 state
[4] |= S_008F20_PITCH_GFX9(image
->surface
.u
.gfx9
.surf
.epitch
);
248 state
[5] &= C_008F24_META_DATA_ADDRESS
&
249 C_008F24_META_PIPE_ALIGNED
&
250 C_008F24_META_RB_ALIGNED
;
252 struct gfx9_surf_meta_flags meta
;
254 if (image
->dcc_offset
)
255 meta
= image
->surface
.u
.gfx9
.dcc
;
257 meta
= image
->surface
.u
.gfx9
.htile
;
259 state
[5] |= S_008F24_META_DATA_ADDRESS(meta_va
>> 40) |
260 S_008F24_META_PIPE_ALIGNED(meta
.pipe_aligned
) |
261 S_008F24_META_RB_ALIGNED(meta
.rb_aligned
);
265 unsigned pitch
= base_level_info
->nblk_x
* block_width
;
266 unsigned index
= si_tile_mode_index(image
, base_level
, is_stencil
);
268 state
[3] &= C_008F1C_TILING_INDEX
;
269 state
[3] |= S_008F1C_TILING_INDEX(index
);
270 state
[4] &= C_008F20_PITCH_GFX6
;
271 state
[4] |= S_008F20_PITCH_GFX6(pitch
- 1);
275 static unsigned radv_tex_dim(VkImageType image_type
, VkImageViewType view_type
,
276 unsigned nr_layers
, unsigned nr_samples
, bool is_storage_image
)
278 if (view_type
== VK_IMAGE_VIEW_TYPE_CUBE
|| view_type
== VK_IMAGE_VIEW_TYPE_CUBE_ARRAY
)
279 return is_storage_image
? V_008F1C_SQ_RSRC_IMG_2D_ARRAY
: V_008F1C_SQ_RSRC_IMG_CUBE
;
280 switch (image_type
) {
281 case VK_IMAGE_TYPE_1D
:
282 return nr_layers
> 1 ? V_008F1C_SQ_RSRC_IMG_1D_ARRAY
: V_008F1C_SQ_RSRC_IMG_1D
;
283 case VK_IMAGE_TYPE_2D
:
285 return nr_layers
> 1 ? V_008F1C_SQ_RSRC_IMG_2D_MSAA_ARRAY
: V_008F1C_SQ_RSRC_IMG_2D_MSAA
;
287 return nr_layers
> 1 ? V_008F1C_SQ_RSRC_IMG_2D_ARRAY
: V_008F1C_SQ_RSRC_IMG_2D
;
288 case VK_IMAGE_TYPE_3D
:
289 if (view_type
== VK_IMAGE_VIEW_TYPE_3D
)
290 return V_008F1C_SQ_RSRC_IMG_3D
;
292 return V_008F1C_SQ_RSRC_IMG_2D_ARRAY
;
294 unreachable("illegale image type");
298 static unsigned gfx9_border_color_swizzle(const unsigned char swizzle
[4])
300 unsigned bc_swizzle
= V_008F20_BC_SWIZZLE_XYZW
;
302 if (swizzle
[3] == VK_SWIZZLE_X
) {
303 /* For the pre-defined border color values (white, opaque
304 * black, transparent black), the only thing that matters is
305 * that the alpha channel winds up in the correct place
306 * (because the RGB channels are all the same) so either of
307 * these enumerations will work.
309 if (swizzle
[2] == VK_SWIZZLE_Y
)
310 bc_swizzle
= V_008F20_BC_SWIZZLE_WZYX
;
312 bc_swizzle
= V_008F20_BC_SWIZZLE_WXYZ
;
313 } else if (swizzle
[0] == VK_SWIZZLE_X
) {
314 if (swizzle
[1] == VK_SWIZZLE_Y
)
315 bc_swizzle
= V_008F20_BC_SWIZZLE_XYZW
;
317 bc_swizzle
= V_008F20_BC_SWIZZLE_XWYZ
;
318 } else if (swizzle
[1] == VK_SWIZZLE_X
) {
319 bc_swizzle
= V_008F20_BC_SWIZZLE_YXWZ
;
320 } else if (swizzle
[2] == VK_SWIZZLE_X
) {
321 bc_swizzle
= V_008F20_BC_SWIZZLE_ZYXW
;
328 * Build the sampler view descriptor for a texture.
331 si_make_texture_descriptor(struct radv_device
*device
,
332 struct radv_image
*image
,
333 bool is_storage_image
,
334 VkImageViewType view_type
,
336 const VkComponentMapping
*mapping
,
337 unsigned first_level
, unsigned last_level
,
338 unsigned first_layer
, unsigned last_layer
,
339 unsigned width
, unsigned height
, unsigned depth
,
341 uint32_t *fmask_state
)
343 const struct vk_format_description
*desc
;
344 enum vk_swizzle swizzle
[4];
346 unsigned num_format
, data_format
, type
;
348 desc
= vk_format_description(vk_format
);
350 if (desc
->colorspace
== VK_FORMAT_COLORSPACE_ZS
) {
351 const unsigned char swizzle_xxxx
[4] = {0, 0, 0, 0};
352 vk_format_compose_swizzles(mapping
, swizzle_xxxx
, swizzle
);
354 vk_format_compose_swizzles(mapping
, desc
->swizzle
, swizzle
);
357 first_non_void
= vk_format_get_first_non_void_channel(vk_format
);
359 num_format
= radv_translate_tex_numformat(vk_format
, desc
, first_non_void
);
360 if (num_format
== ~0) {
364 data_format
= radv_translate_tex_dataformat(vk_format
, desc
, first_non_void
);
365 if (data_format
== ~0) {
369 type
= radv_tex_dim(image
->type
, view_type
, image
->info
.array_size
, image
->info
.samples
,
371 if (type
== V_008F1C_SQ_RSRC_IMG_1D_ARRAY
) {
373 depth
= image
->info
.array_size
;
374 } else if (type
== V_008F1C_SQ_RSRC_IMG_2D_ARRAY
||
375 type
== V_008F1C_SQ_RSRC_IMG_2D_MSAA_ARRAY
) {
376 if (view_type
!= VK_IMAGE_VIEW_TYPE_3D
)
377 depth
= image
->info
.array_size
;
378 } else if (type
== V_008F1C_SQ_RSRC_IMG_CUBE
)
379 depth
= image
->info
.array_size
/ 6;
382 state
[1] = (S_008F14_DATA_FORMAT_GFX6(data_format
) |
383 S_008F14_NUM_FORMAT_GFX6(num_format
));
384 state
[2] = (S_008F18_WIDTH(width
- 1) |
385 S_008F18_HEIGHT(height
- 1) |
386 S_008F18_PERF_MOD(4));
387 state
[3] = (S_008F1C_DST_SEL_X(radv_map_swizzle(swizzle
[0])) |
388 S_008F1C_DST_SEL_Y(radv_map_swizzle(swizzle
[1])) |
389 S_008F1C_DST_SEL_Z(radv_map_swizzle(swizzle
[2])) |
390 S_008F1C_DST_SEL_W(radv_map_swizzle(swizzle
[3])) |
391 S_008F1C_BASE_LEVEL(image
->info
.samples
> 1 ?
393 S_008F1C_LAST_LEVEL(image
->info
.samples
> 1 ?
394 util_logbase2(image
->info
.samples
) :
396 S_008F1C_TYPE(type
));
398 state
[5] = S_008F24_BASE_ARRAY(first_layer
);
402 if (device
->physical_device
->rad_info
.chip_class
>= GFX9
) {
403 unsigned bc_swizzle
= gfx9_border_color_swizzle(desc
->swizzle
);
405 /* Depth is the the last accessible layer on Gfx9.
406 * The hw doesn't need to know the total number of layers.
408 if (type
== V_008F1C_SQ_RSRC_IMG_3D
)
409 state
[4] |= S_008F20_DEPTH(depth
- 1);
411 state
[4] |= S_008F20_DEPTH(last_layer
);
413 state
[4] |= S_008F20_BC_SWIZZLE(bc_swizzle
);
414 state
[5] |= S_008F24_MAX_MIP(image
->info
.samples
> 1 ?
415 util_logbase2(image
->info
.samples
) :
418 state
[3] |= S_008F1C_POW2_PAD(image
->info
.levels
> 1);
419 state
[4] |= S_008F20_DEPTH(depth
- 1);
420 state
[5] |= S_008F24_LAST_ARRAY(last_layer
);
422 if (image
->dcc_offset
) {
423 unsigned swap
= radv_translate_colorswap(vk_format
, FALSE
);
425 state
[6] = S_008F28_ALPHA_IS_ON_MSB(swap
<= 1);
427 /* The last dword is unused by hw. The shader uses it to clear
428 * bits in the first dword of sampler state.
430 if (device
->physical_device
->rad_info
.chip_class
<= CIK
&& image
->info
.samples
<= 1) {
431 if (first_level
== last_level
)
432 state
[7] = C_008F30_MAX_ANISO_RATIO
;
434 state
[7] = 0xffffffff;
438 /* Initialize the sampler view for FMASK. */
439 if (image
->fmask
.size
) {
440 uint32_t fmask_format
, num_format
;
441 uint64_t gpu_address
= device
->ws
->buffer_get_va(image
->bo
);
444 va
= gpu_address
+ image
->offset
+ image
->fmask
.offset
;
446 if (device
->physical_device
->rad_info
.chip_class
>= GFX9
) {
447 fmask_format
= V_008F14_IMG_DATA_FORMAT_FMASK
;
448 switch (image
->info
.samples
) {
450 num_format
= V_008F14_IMG_FMASK_8_2_2
;
453 num_format
= V_008F14_IMG_FMASK_8_4_4
;
456 num_format
= V_008F14_IMG_FMASK_32_8_8
;
459 unreachable("invalid nr_samples");
462 switch (image
->info
.samples
) {
464 fmask_format
= V_008F14_IMG_DATA_FORMAT_FMASK8_S2_F2
;
467 fmask_format
= V_008F14_IMG_DATA_FORMAT_FMASK8_S4_F4
;
470 fmask_format
= V_008F14_IMG_DATA_FORMAT_FMASK32_S8_F8
;
474 fmask_format
= V_008F14_IMG_DATA_FORMAT_INVALID
;
476 num_format
= V_008F14_IMG_NUM_FORMAT_UINT
;
479 fmask_state
[0] = va
>> 8;
480 if (device
->physical_device
->rad_info
.chip_class
< GFX9
)
481 fmask_state
[0] |= image
->surface
.u
.legacy
.tile_swizzle
;
482 fmask_state
[1] = S_008F14_BASE_ADDRESS_HI(va
>> 40) |
483 S_008F14_DATA_FORMAT_GFX6(fmask_format
) |
484 S_008F14_NUM_FORMAT_GFX6(num_format
);
485 fmask_state
[2] = S_008F18_WIDTH(width
- 1) |
486 S_008F18_HEIGHT(height
- 1);
487 fmask_state
[3] = S_008F1C_DST_SEL_X(V_008F1C_SQ_SEL_X
) |
488 S_008F1C_DST_SEL_Y(V_008F1C_SQ_SEL_X
) |
489 S_008F1C_DST_SEL_Z(V_008F1C_SQ_SEL_X
) |
490 S_008F1C_DST_SEL_W(V_008F1C_SQ_SEL_X
) |
491 S_008F1C_TYPE(radv_tex_dim(image
->type
, view_type
, 1, 0, false));
493 fmask_state
[5] = S_008F24_BASE_ARRAY(first_layer
);
497 if (device
->physical_device
->rad_info
.chip_class
>= GFX9
) {
498 fmask_state
[3] |= S_008F1C_SW_MODE(image
->surface
.u
.gfx9
.fmask
.swizzle_mode
);
499 fmask_state
[4] |= S_008F20_DEPTH(last_layer
) |
500 S_008F20_PITCH_GFX9(image
->surface
.u
.gfx9
.fmask
.epitch
);
501 fmask_state
[5] |= S_008F24_META_PIPE_ALIGNED(image
->surface
.u
.gfx9
.cmask
.pipe_aligned
) |
502 S_008F24_META_RB_ALIGNED(image
->surface
.u
.gfx9
.cmask
.rb_aligned
);
504 fmask_state
[3] |= S_008F1C_TILING_INDEX(image
->fmask
.tile_mode_index
);
505 fmask_state
[4] |= S_008F20_DEPTH(depth
- 1) |
506 S_008F20_PITCH_GFX6(image
->fmask
.pitch_in_pixels
- 1);
507 fmask_state
[5] |= S_008F24_LAST_ARRAY(last_layer
);
509 } else if (fmask_state
)
510 memset(fmask_state
, 0, 8 * 4);
514 radv_query_opaque_metadata(struct radv_device
*device
,
515 struct radv_image
*image
,
516 struct radeon_bo_metadata
*md
)
518 static const VkComponentMapping fixedmapping
;
521 /* Metadata image format format version 1:
522 * [0] = 1 (metadata format identifier)
523 * [1] = (VENDOR_ID << 16) | PCI_ID
524 * [2:9] = image descriptor for the whole resource
525 * [2] is always 0, because the base address is cleared
526 * [9] is the DCC offset bits [39:8] from the beginning of
528 * [10:10+LAST_LEVEL] = mipmap level offset bits [39:8] for each level
530 md
->metadata
[0] = 1; /* metadata image format version 1 */
532 /* TILE_MODE_INDEX is ambiguous without a PCI ID. */
533 md
->metadata
[1] = si_get_bo_metadata_word1(device
);
536 si_make_texture_descriptor(device
, image
, false,
537 (VkImageViewType
)image
->type
, image
->vk_format
,
538 &fixedmapping
, 0, image
->info
.levels
- 1, 0,
539 image
->info
.array_size
,
540 image
->info
.width
, image
->info
.height
,
544 si_set_mutable_tex_desc_fields(device
, image
, &image
->surface
.u
.legacy
.level
[0], 0, 0,
545 image
->surface
.blk_w
, false, desc
);
547 /* Clear the base address and set the relative DCC offset. */
549 desc
[1] &= C_008F14_BASE_ADDRESS_HI
;
550 desc
[7] = image
->dcc_offset
>> 8;
552 /* Dwords [2:9] contain the image descriptor. */
553 memcpy(&md
->metadata
[2], desc
, sizeof(desc
));
555 /* Dwords [10:..] contain the mipmap level offsets. */
556 for (i
= 0; i
<= image
->info
.levels
- 1; i
++)
557 md
->metadata
[10+i
] = image
->surface
.u
.legacy
.level
[i
].offset
>> 8;
559 md
->size_metadata
= (11 + image
->info
.levels
- 1) * 4;
563 radv_init_metadata(struct radv_device
*device
,
564 struct radv_image
*image
,
565 struct radeon_bo_metadata
*metadata
)
567 struct radeon_surf
*surface
= &image
->surface
;
569 memset(metadata
, 0, sizeof(*metadata
));
571 if (device
->physical_device
->rad_info
.chip_class
>= GFX9
) {
572 metadata
->u
.gfx9
.swizzle_mode
= surface
->u
.gfx9
.surf
.swizzle_mode
;
574 metadata
->u
.legacy
.microtile
= surface
->u
.legacy
.level
[0].mode
>= RADEON_SURF_MODE_1D
?
575 RADEON_LAYOUT_TILED
: RADEON_LAYOUT_LINEAR
;
576 metadata
->u
.legacy
.macrotile
= surface
->u
.legacy
.level
[0].mode
>= RADEON_SURF_MODE_2D
?
577 RADEON_LAYOUT_TILED
: RADEON_LAYOUT_LINEAR
;
578 metadata
->u
.legacy
.pipe_config
= surface
->u
.legacy
.pipe_config
;
579 metadata
->u
.legacy
.bankw
= surface
->u
.legacy
.bankw
;
580 metadata
->u
.legacy
.bankh
= surface
->u
.legacy
.bankh
;
581 metadata
->u
.legacy
.tile_split
= surface
->u
.legacy
.tile_split
;
582 metadata
->u
.legacy
.mtilea
= surface
->u
.legacy
.mtilea
;
583 metadata
->u
.legacy
.num_banks
= surface
->u
.legacy
.num_banks
;
584 metadata
->u
.legacy
.stride
= surface
->u
.legacy
.level
[0].nblk_x
* surface
->bpe
;
585 metadata
->u
.legacy
.scanout
= (surface
->flags
& RADEON_SURF_SCANOUT
) != 0;
587 radv_query_opaque_metadata(device
, image
, metadata
);
590 /* The number of samples can be specified independently of the texture. */
592 radv_image_get_fmask_info(struct radv_device
*device
,
593 struct radv_image
*image
,
595 struct radv_fmask_info
*out
)
597 /* FMASK is allocated like an ordinary texture. */
598 struct radeon_surf fmask
= {};
599 struct ac_surf_info info
= image
->info
;
600 memset(out
, 0, sizeof(*out
));
602 if (device
->physical_device
->rad_info
.chip_class
>= GFX9
) {
603 out
->alignment
= image
->surface
.u
.gfx9
.fmask_alignment
;
604 out
->size
= image
->surface
.u
.gfx9
.fmask_size
;
608 fmask
.blk_w
= image
->surface
.blk_w
;
609 fmask
.blk_h
= image
->surface
.blk_h
;
611 fmask
.flags
= image
->surface
.flags
| RADEON_SURF_FMASK
;
613 /* Force 2D tiling if it wasn't set. This may occur when creating
614 * FMASK for MSAA resolve on R6xx. On R6xx, the single-sample
615 * destination buffer must have an FMASK too. */
616 fmask
.flags
= RADEON_SURF_CLR(fmask
.flags
, MODE
);
617 fmask
.flags
|= RADEON_SURF_SET(RADEON_SURF_MODE_2D
, MODE
);
619 switch (nr_samples
) {
631 device
->ws
->surface_init(device
->ws
, &info
, &fmask
);
632 assert(fmask
.u
.legacy
.level
[0].mode
== RADEON_SURF_MODE_2D
);
634 out
->slice_tile_max
= (fmask
.u
.legacy
.level
[0].nblk_x
* fmask
.u
.legacy
.level
[0].nblk_y
) / 64;
635 if (out
->slice_tile_max
)
636 out
->slice_tile_max
-= 1;
638 out
->tile_mode_index
= fmask
.u
.legacy
.tiling_index
[0];
639 out
->pitch_in_pixels
= fmask
.u
.legacy
.level
[0].nblk_x
;
640 out
->bank_height
= fmask
.u
.legacy
.bankh
;
641 out
->alignment
= MAX2(256, fmask
.surf_alignment
);
642 out
->size
= fmask
.surf_size
;
646 radv_image_alloc_fmask(struct radv_device
*device
,
647 struct radv_image
*image
)
649 radv_image_get_fmask_info(device
, image
, image
->info
.samples
, &image
->fmask
);
651 image
->fmask
.offset
= align64(image
->size
, image
->fmask
.alignment
);
652 image
->size
= image
->fmask
.offset
+ image
->fmask
.size
;
653 image
->alignment
= MAX2(image
->alignment
, image
->fmask
.alignment
);
657 radv_image_get_cmask_info(struct radv_device
*device
,
658 struct radv_image
*image
,
659 struct radv_cmask_info
*out
)
661 unsigned pipe_interleave_bytes
= device
->physical_device
->rad_info
.pipe_interleave_bytes
;
662 unsigned num_pipes
= device
->physical_device
->rad_info
.num_tile_pipes
;
663 unsigned cl_width
, cl_height
;
665 if (device
->physical_device
->rad_info
.chip_class
>= GFX9
) {
666 out
->alignment
= image
->surface
.u
.gfx9
.cmask_alignment
;
667 out
->size
= image
->surface
.u
.gfx9
.cmask_size
;
684 case 16: /* Hawaii */
693 unsigned base_align
= num_pipes
* pipe_interleave_bytes
;
695 unsigned width
= align(image
->info
.width
, cl_width
*8);
696 unsigned height
= align(image
->info
.height
, cl_height
*8);
697 unsigned slice_elements
= (width
* height
) / (8*8);
699 /* Each element of CMASK is a nibble. */
700 unsigned slice_bytes
= slice_elements
/ 2;
702 out
->slice_tile_max
= (width
* height
) / (128*128);
703 if (out
->slice_tile_max
)
704 out
->slice_tile_max
-= 1;
706 out
->alignment
= MAX2(256, base_align
);
707 out
->size
= (image
->type
== VK_IMAGE_TYPE_3D
? image
->info
.depth
: image
->info
.array_size
) *
708 align(slice_bytes
, base_align
);
712 radv_image_alloc_cmask(struct radv_device
*device
,
713 struct radv_image
*image
)
715 uint32_t clear_value_size
= 0;
716 radv_image_get_cmask_info(device
, image
, &image
->cmask
);
718 image
->cmask
.offset
= align64(image
->size
, image
->cmask
.alignment
);
719 /* + 8 for storing the clear values */
720 if (!image
->clear_value_offset
) {
721 image
->clear_value_offset
= image
->cmask
.offset
+ image
->cmask
.size
;
722 clear_value_size
= 8;
724 image
->size
= image
->cmask
.offset
+ image
->cmask
.size
+ clear_value_size
;
725 image
->alignment
= MAX2(image
->alignment
, image
->cmask
.alignment
);
729 radv_image_alloc_dcc(struct radv_device
*device
,
730 struct radv_image
*image
)
732 image
->dcc_offset
= align64(image
->size
, image
->surface
.dcc_alignment
);
733 /* + 16 for storing the clear values + dcc pred */
734 image
->clear_value_offset
= image
->dcc_offset
+ image
->surface
.dcc_size
;
735 image
->dcc_pred_offset
= image
->clear_value_offset
+ 8;
736 image
->size
= image
->dcc_offset
+ image
->surface
.dcc_size
+ 16;
737 image
->alignment
= MAX2(image
->alignment
, image
->surface
.dcc_alignment
);
741 radv_image_alloc_htile(struct radv_device
*device
,
742 struct radv_image
*image
)
744 if ((device
->debug_flags
& RADV_DEBUG_NO_HIZ
) || image
->info
.levels
> 1) {
745 image
->surface
.htile_size
= 0;
749 image
->htile_offset
= align64(image
->size
, image
->surface
.htile_alignment
);
751 /* + 8 for storing the clear values */
752 image
->clear_value_offset
= image
->htile_offset
+ image
->surface
.htile_size
;
753 image
->size
= image
->clear_value_offset
+ 8;
754 image
->alignment
= align64(image
->alignment
, image
->surface
.htile_alignment
);
758 radv_image_create(VkDevice _device
,
759 const struct radv_image_create_info
*create_info
,
760 const VkAllocationCallbacks
* alloc
,
763 RADV_FROM_HANDLE(radv_device
, device
, _device
);
764 const VkImageCreateInfo
*pCreateInfo
= create_info
->vk_info
;
765 struct radv_image
*image
= NULL
;
766 bool can_cmask_dcc
= false;
767 assert(pCreateInfo
->sType
== VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO
);
769 radv_assert(pCreateInfo
->mipLevels
> 0);
770 radv_assert(pCreateInfo
->arrayLayers
> 0);
771 radv_assert(pCreateInfo
->samples
> 0);
772 radv_assert(pCreateInfo
->extent
.width
> 0);
773 radv_assert(pCreateInfo
->extent
.height
> 0);
774 radv_assert(pCreateInfo
->extent
.depth
> 0);
776 image
= vk_alloc2(&device
->alloc
, alloc
, sizeof(*image
), 8,
777 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT
);
779 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
781 memset(image
, 0, sizeof(*image
));
782 image
->type
= pCreateInfo
->imageType
;
783 image
->info
.width
= pCreateInfo
->extent
.width
;
784 image
->info
.height
= pCreateInfo
->extent
.height
;
785 image
->info
.depth
= pCreateInfo
->extent
.depth
;
786 image
->info
.samples
= pCreateInfo
->samples
;
787 image
->info
.array_size
= pCreateInfo
->arrayLayers
;
788 image
->info
.levels
= pCreateInfo
->mipLevels
;
790 image
->vk_format
= pCreateInfo
->format
;
791 image
->tiling
= pCreateInfo
->tiling
;
792 image
->usage
= pCreateInfo
->usage
;
793 image
->flags
= pCreateInfo
->flags
;
795 image
->exclusive
= pCreateInfo
->sharingMode
== VK_SHARING_MODE_EXCLUSIVE
;
796 if (pCreateInfo
->sharingMode
== VK_SHARING_MODE_CONCURRENT
) {
797 for (uint32_t i
= 0; i
< pCreateInfo
->queueFamilyIndexCount
; ++i
)
798 if (pCreateInfo
->pQueueFamilyIndices
[i
] == VK_QUEUE_FAMILY_EXTERNAL_KHR
)
799 image
->queue_family_mask
|= (1u << RADV_MAX_QUEUE_FAMILIES
) - 1u;
801 image
->queue_family_mask
|= 1u << pCreateInfo
->pQueueFamilyIndices
[i
];
804 image
->shareable
= vk_find_struct_const(pCreateInfo
->pNext
,
805 EXTERNAL_MEMORY_IMAGE_CREATE_INFO_KHR
) != NULL
;
806 if (!vk_format_is_depth(pCreateInfo
->format
) && !create_info
->scanout
&& !image
->shareable
) {
807 image
->info
.surf_index
= p_atomic_inc_return(&device
->image_mrt_offset_counter
) - 1;
810 radv_init_surface(device
, &image
->surface
, create_info
);
812 device
->ws
->surface_init(device
->ws
, &image
->info
, &image
->surface
);
814 image
->size
= image
->surface
.surf_size
;
815 image
->alignment
= image
->surface
.surf_alignment
;
817 if (image
->exclusive
|| image
->queue_family_mask
== 1)
818 can_cmask_dcc
= true;
820 if ((pCreateInfo
->usage
& VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT
) &&
821 image
->surface
.dcc_size
&& can_cmask_dcc
)
822 radv_image_alloc_dcc(device
, image
);
824 image
->surface
.dcc_size
= 0;
826 if ((pCreateInfo
->usage
& VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT
) &&
827 pCreateInfo
->mipLevels
== 1 &&
828 !image
->surface
.dcc_size
&& image
->info
.depth
== 1 && can_cmask_dcc
)
829 radv_image_alloc_cmask(device
, image
);
830 if (image
->info
.samples
> 1 && vk_format_is_color(pCreateInfo
->format
)) {
831 radv_image_alloc_fmask(device
, image
);
832 } else if (vk_format_is_depth(pCreateInfo
->format
)) {
834 radv_image_alloc_htile(device
, image
);
837 if (pCreateInfo
->flags
& VK_IMAGE_CREATE_SPARSE_BINDING_BIT
) {
838 image
->alignment
= MAX2(image
->alignment
, 4096);
839 image
->size
= align64(image
->size
, image
->alignment
);
842 image
->bo
= device
->ws
->buffer_create(device
->ws
, image
->size
, image
->alignment
,
843 0, RADEON_FLAG_VIRTUAL
);
845 vk_free2(&device
->alloc
, alloc
, image
);
846 return vk_error(VK_ERROR_OUT_OF_DEVICE_MEMORY
);
850 *pImage
= radv_image_to_handle(image
);
856 radv_image_view_make_descriptor(struct radv_image_view
*iview
,
857 struct radv_device
*device
,
858 const VkImageViewCreateInfo
* pCreateInfo
,
859 bool is_storage_image
)
861 RADV_FROM_HANDLE(radv_image
, image
, pCreateInfo
->image
);
862 const VkImageSubresourceRange
*range
= &pCreateInfo
->subresourceRange
;
863 bool is_stencil
= iview
->aspect_mask
== VK_IMAGE_ASPECT_STENCIL_BIT
;
865 uint32_t *descriptor
;
866 uint32_t *fmask_descriptor
;
868 if (is_storage_image
) {
869 descriptor
= iview
->storage_descriptor
;
870 fmask_descriptor
= iview
->storage_fmask_descriptor
;
872 descriptor
= iview
->descriptor
;
873 fmask_descriptor
= iview
->fmask_descriptor
;
876 assert(image
->surface
.blk_w
% vk_format_get_blockwidth(image
->vk_format
) == 0);
877 blk_w
= image
->surface
.blk_w
/ vk_format_get_blockwidth(image
->vk_format
) * vk_format_get_blockwidth(iview
->vk_format
);
879 si_make_texture_descriptor(device
, image
, is_storage_image
,
882 &pCreateInfo
->components
,
883 0, radv_get_levelCount(image
, range
) - 1,
884 range
->baseArrayLayer
,
885 range
->baseArrayLayer
+ radv_get_layerCount(image
, range
) - 1,
887 iview
->extent
.height
,
891 si_set_mutable_tex_desc_fields(device
, image
,
892 is_stencil
? &image
->surface
.u
.legacy
.stencil_level
[range
->baseMipLevel
]
893 : &image
->surface
.u
.legacy
.level
[range
->baseMipLevel
],
896 blk_w
, is_stencil
, descriptor
);
900 radv_image_view_init(struct radv_image_view
*iview
,
901 struct radv_device
*device
,
902 const VkImageViewCreateInfo
* pCreateInfo
)
904 RADV_FROM_HANDLE(radv_image
, image
, pCreateInfo
->image
);
905 const VkImageSubresourceRange
*range
= &pCreateInfo
->subresourceRange
;
907 switch (image
->type
) {
908 case VK_IMAGE_TYPE_1D
:
909 case VK_IMAGE_TYPE_2D
:
910 assert(range
->baseArrayLayer
+ radv_get_layerCount(image
, range
) - 1 <= image
->info
.array_size
);
912 case VK_IMAGE_TYPE_3D
:
913 assert(range
->baseArrayLayer
+ radv_get_layerCount(image
, range
) - 1
914 <= radv_minify(image
->info
.depth
, range
->baseMipLevel
));
917 unreachable("bad VkImageType");
919 iview
->image
= image
;
920 iview
->bo
= image
->bo
;
921 iview
->type
= pCreateInfo
->viewType
;
922 iview
->vk_format
= pCreateInfo
->format
;
923 iview
->aspect_mask
= pCreateInfo
->subresourceRange
.aspectMask
;
925 if (iview
->aspect_mask
== VK_IMAGE_ASPECT_STENCIL_BIT
) {
926 iview
->vk_format
= vk_format_stencil_only(iview
->vk_format
);
927 } else if (iview
->aspect_mask
== VK_IMAGE_ASPECT_DEPTH_BIT
) {
928 iview
->vk_format
= vk_format_depth_only(iview
->vk_format
);
931 iview
->extent
= (VkExtent3D
) {
932 .width
= radv_minify(image
->info
.width
, range
->baseMipLevel
),
933 .height
= radv_minify(image
->info
.height
, range
->baseMipLevel
),
934 .depth
= radv_minify(image
->info
.depth
, range
->baseMipLevel
),
937 iview
->extent
.width
= round_up_u32(iview
->extent
.width
* vk_format_get_blockwidth(iview
->vk_format
),
938 vk_format_get_blockwidth(image
->vk_format
));
939 iview
->extent
.height
= round_up_u32(iview
->extent
.height
* vk_format_get_blockheight(iview
->vk_format
),
940 vk_format_get_blockheight(image
->vk_format
));
942 iview
->base_layer
= range
->baseArrayLayer
;
943 iview
->layer_count
= radv_get_layerCount(image
, range
);
944 iview
->base_mip
= range
->baseMipLevel
;
946 radv_image_view_make_descriptor(iview
, device
, pCreateInfo
, false);
948 /* For transfers we may use the image as a storage image. */
949 if (image
->usage
& (VK_IMAGE_USAGE_STORAGE_BIT
| VK_IMAGE_USAGE_TRANSFER_DST_BIT
))
950 radv_image_view_make_descriptor(iview
, device
, pCreateInfo
, true);
953 bool radv_layout_has_htile(const struct radv_image
*image
,
954 VkImageLayout layout
,
957 return image
->surface
.htile_size
&&
958 (layout
== VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL
||
959 layout
== VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL
) &&
960 queue_mask
== (1u << RADV_QUEUE_GENERAL
);
963 bool radv_layout_is_htile_compressed(const struct radv_image
*image
,
964 VkImageLayout layout
,
967 return image
->surface
.htile_size
&&
968 (layout
== VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL
||
969 layout
== VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL
) &&
970 queue_mask
== (1u << RADV_QUEUE_GENERAL
);
973 bool radv_layout_can_fast_clear(const struct radv_image
*image
,
974 VkImageLayout layout
,
977 return layout
== VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL
&&
978 queue_mask
== (1u << RADV_QUEUE_GENERAL
);
982 unsigned radv_image_queue_family_mask(const struct radv_image
*image
, uint32_t family
, uint32_t queue_family
)
984 if (!image
->exclusive
)
985 return image
->queue_family_mask
;
986 if (family
== VK_QUEUE_FAMILY_EXTERNAL_KHR
)
987 return (1u << RADV_MAX_QUEUE_FAMILIES
) - 1u;
988 if (family
== VK_QUEUE_FAMILY_IGNORED
)
989 return 1u << queue_family
;
994 radv_CreateImage(VkDevice device
,
995 const VkImageCreateInfo
*pCreateInfo
,
996 const VkAllocationCallbacks
*pAllocator
,
999 return radv_image_create(device
,
1000 &(struct radv_image_create_info
) {
1001 .vk_info
= pCreateInfo
,
1009 radv_DestroyImage(VkDevice _device
, VkImage _image
,
1010 const VkAllocationCallbacks
*pAllocator
)
1012 RADV_FROM_HANDLE(radv_device
, device
, _device
);
1013 RADV_FROM_HANDLE(radv_image
, image
, _image
);
1018 if (image
->flags
& VK_IMAGE_CREATE_SPARSE_BINDING_BIT
)
1019 device
->ws
->buffer_destroy(image
->bo
);
1021 vk_free2(&device
->alloc
, pAllocator
, image
);
1024 void radv_GetImageSubresourceLayout(
1027 const VkImageSubresource
* pSubresource
,
1028 VkSubresourceLayout
* pLayout
)
1030 RADV_FROM_HANDLE(radv_image
, image
, _image
);
1031 int level
= pSubresource
->mipLevel
;
1032 int layer
= pSubresource
->arrayLayer
;
1033 struct radeon_surf
*surface
= &image
->surface
;
1035 pLayout
->offset
= surface
->u
.legacy
.level
[level
].offset
+ surface
->u
.legacy
.level
[level
].slice_size
* layer
;
1036 pLayout
->rowPitch
= surface
->u
.legacy
.level
[level
].nblk_x
* surface
->bpe
;
1037 pLayout
->arrayPitch
= surface
->u
.legacy
.level
[level
].slice_size
;
1038 pLayout
->depthPitch
= surface
->u
.legacy
.level
[level
].slice_size
;
1039 pLayout
->size
= surface
->u
.legacy
.level
[level
].slice_size
;
1040 if (image
->type
== VK_IMAGE_TYPE_3D
)
1041 pLayout
->size
*= u_minify(image
->info
.depth
, level
);
1046 radv_CreateImageView(VkDevice _device
,
1047 const VkImageViewCreateInfo
*pCreateInfo
,
1048 const VkAllocationCallbacks
*pAllocator
,
1051 RADV_FROM_HANDLE(radv_device
, device
, _device
);
1052 struct radv_image_view
*view
;
1054 view
= vk_alloc2(&device
->alloc
, pAllocator
, sizeof(*view
), 8,
1055 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT
);
1057 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1059 radv_image_view_init(view
, device
, pCreateInfo
);
1061 *pView
= radv_image_view_to_handle(view
);
1067 radv_DestroyImageView(VkDevice _device
, VkImageView _iview
,
1068 const VkAllocationCallbacks
*pAllocator
)
1070 RADV_FROM_HANDLE(radv_device
, device
, _device
);
1071 RADV_FROM_HANDLE(radv_image_view
, iview
, _iview
);
1075 vk_free2(&device
->alloc
, pAllocator
, iview
);
1078 void radv_buffer_view_init(struct radv_buffer_view
*view
,
1079 struct radv_device
*device
,
1080 const VkBufferViewCreateInfo
* pCreateInfo
,
1081 struct radv_cmd_buffer
*cmd_buffer
)
1083 RADV_FROM_HANDLE(radv_buffer
, buffer
, pCreateInfo
->buffer
);
1085 view
->bo
= buffer
->bo
;
1086 view
->range
= pCreateInfo
->range
== VK_WHOLE_SIZE
?
1087 buffer
->size
- pCreateInfo
->offset
: pCreateInfo
->range
;
1088 view
->vk_format
= pCreateInfo
->format
;
1090 radv_make_buffer_descriptor(device
, buffer
, view
->vk_format
,
1091 pCreateInfo
->offset
, view
->range
, view
->state
);
1095 radv_CreateBufferView(VkDevice _device
,
1096 const VkBufferViewCreateInfo
*pCreateInfo
,
1097 const VkAllocationCallbacks
*pAllocator
,
1098 VkBufferView
*pView
)
1100 RADV_FROM_HANDLE(radv_device
, device
, _device
);
1101 struct radv_buffer_view
*view
;
1103 view
= vk_alloc2(&device
->alloc
, pAllocator
, sizeof(*view
), 8,
1104 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT
);
1106 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1108 radv_buffer_view_init(view
, device
, pCreateInfo
, NULL
);
1110 *pView
= radv_buffer_view_to_handle(view
);
1116 radv_DestroyBufferView(VkDevice _device
, VkBufferView bufferView
,
1117 const VkAllocationCallbacks
*pAllocator
)
1119 RADV_FROM_HANDLE(radv_device
, device
, _device
);
1120 RADV_FROM_HANDLE(radv_buffer_view
, view
, bufferView
);
1125 vk_free2(&device
->alloc
, pAllocator
, view
);