2 * Copyright © 2016 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 #include "radv_meta.h"
25 #include "vk_format.h"
28 meta_image_block_size(const struct radv_image
*image
)
30 const struct vk_format_description
*desc
= vk_format_description(image
->vk_format
);
31 return (VkExtent3D
) { desc
->block
.width
, desc
->block
.height
, 1 };
34 /* Returns the user-provided VkBufferImageCopy::imageExtent in units of
35 * elements rather than texels. One element equals one texel or one block
36 * if Image is uncompressed or compressed, respectively.
38 static struct VkExtent3D
39 meta_region_extent_el(const struct radv_image
*image
,
40 const VkImageType imageType
,
41 const struct VkExtent3D
*extent
)
43 const VkExtent3D block
= meta_image_block_size(image
);
44 return radv_sanitize_image_extent(imageType
, (VkExtent3D
) {
45 .width
= DIV_ROUND_UP(extent
->width
, block
.width
),
46 .height
= DIV_ROUND_UP(extent
->height
, block
.height
),
47 .depth
= DIV_ROUND_UP(extent
->depth
, block
.depth
),
51 /* Returns the user-provided VkBufferImageCopy::imageOffset in units of
52 * elements rather than texels. One element equals one texel or one block
53 * if Image is uncompressed or compressed, respectively.
55 static struct VkOffset3D
56 meta_region_offset_el(const struct radv_image
*image
,
57 const struct VkOffset3D
*offset
)
59 const VkExtent3D block
= meta_image_block_size(image
);
60 return radv_sanitize_image_offset(image
->type
, (VkOffset3D
) {
61 .x
= offset
->x
/ block
.width
,
62 .y
= offset
->y
/ block
.height
,
63 .z
= offset
->z
/ block
.depth
,
68 vk_format_for_size(int bs
)
71 case 1: return VK_FORMAT_R8_UINT
;
72 case 2: return VK_FORMAT_R8G8_UINT
;
73 case 4: return VK_FORMAT_R8G8B8A8_UINT
;
74 case 8: return VK_FORMAT_R16G16B16A16_UINT
;
75 case 12: return VK_FORMAT_R32G32B32_UINT
;
76 case 16: return VK_FORMAT_R32G32B32A32_UINT
;
78 unreachable("Invalid format block size");
82 static struct radv_meta_blit2d_surf
83 blit_surf_for_image_level_layer(struct radv_image
*image
,
85 const VkImageSubresourceLayers
*subres
,
86 VkImageAspectFlags aspect_mask
)
88 VkFormat format
= radv_get_aspect_format(image
, aspect_mask
);
90 if (!radv_dcc_enabled(image
, subres
->mipLevel
) &&
91 !(radv_image_is_tc_compat_htile(image
)))
92 format
= vk_format_for_size(vk_format_get_blocksize(format
));
94 format
= vk_format_no_srgb(format
);
96 return (struct radv_meta_blit2d_surf
) {
98 .bs
= vk_format_get_blocksize(format
),
99 .level
= subres
->mipLevel
,
100 .layer
= subres
->baseArrayLayer
,
102 .aspect_mask
= aspect_mask
,
103 .current_layout
= layout
,
108 image_is_renderable(struct radv_device
*device
, struct radv_image
*image
)
110 if (image
->vk_format
== VK_FORMAT_R32G32B32_UINT
||
111 image
->vk_format
== VK_FORMAT_R32G32B32_SINT
||
112 image
->vk_format
== VK_FORMAT_R32G32B32_SFLOAT
)
115 if (device
->physical_device
->rad_info
.chip_class
>= GFX9
&&
116 image
->type
== VK_IMAGE_TYPE_3D
&&
117 vk_format_get_blocksizebits(image
->vk_format
) == 128 &&
118 vk_format_is_compressed(image
->vk_format
))
124 meta_copy_buffer_to_image(struct radv_cmd_buffer
*cmd_buffer
,
125 struct radv_buffer
* buffer
,
126 struct radv_image
* image
,
127 VkImageLayout layout
,
128 uint32_t regionCount
,
129 const VkBufferImageCopy
* pRegions
)
131 bool cs
= cmd_buffer
->queue_family_index
== RADV_QUEUE_COMPUTE
;
132 struct radv_meta_saved_state saved_state
;
133 bool old_predicating
;
135 /* The Vulkan 1.0 spec says "dstImage must have a sample count equal to
136 * VK_SAMPLE_COUNT_1_BIT."
138 assert(image
->info
.samples
== 1);
140 radv_meta_save(&saved_state
, cmd_buffer
,
141 (cs
? RADV_META_SAVE_COMPUTE_PIPELINE
:
142 RADV_META_SAVE_GRAPHICS_PIPELINE
) |
143 RADV_META_SAVE_CONSTANTS
|
144 RADV_META_SAVE_DESCRIPTORS
);
146 /* VK_EXT_conditional_rendering says that copy commands should not be
147 * affected by conditional rendering.
149 old_predicating
= cmd_buffer
->state
.predicating
;
150 cmd_buffer
->state
.predicating
= false;
152 for (unsigned r
= 0; r
< regionCount
; r
++) {
155 * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
156 * extent is the size in texels of the source image to copy in width,
157 * height and depth. 1D images use only x and width. 2D images use x, y,
158 * width and height. 3D images use x, y, z, width, height and depth.
161 * Also, convert the offsets and extent from units of texels to units of
162 * blocks - which is the highest resolution accessible in this command.
164 const VkOffset3D img_offset_el
=
165 meta_region_offset_el(image
, &pRegions
[r
].imageOffset
);
166 const VkExtent3D bufferExtent
= {
167 .width
= pRegions
[r
].bufferRowLength
?
168 pRegions
[r
].bufferRowLength
: pRegions
[r
].imageExtent
.width
,
169 .height
= pRegions
[r
].bufferImageHeight
?
170 pRegions
[r
].bufferImageHeight
: pRegions
[r
].imageExtent
.height
,
172 const VkExtent3D buf_extent_el
=
173 meta_region_extent_el(image
, image
->type
, &bufferExtent
);
175 /* Start creating blit rect */
176 const VkExtent3D img_extent_el
=
177 meta_region_extent_el(image
, image
->type
, &pRegions
[r
].imageExtent
);
178 struct radv_meta_blit2d_rect rect
= {
179 .width
= img_extent_el
.width
,
180 .height
= img_extent_el
.height
,
183 /* Create blit surfaces */
184 struct radv_meta_blit2d_surf img_bsurf
=
185 blit_surf_for_image_level_layer(image
,
187 &pRegions
[r
].imageSubresource
,
188 pRegions
[r
].imageSubresource
.aspectMask
);
190 if (!radv_is_buffer_format_supported(img_bsurf
.format
, NULL
)) {
191 uint32_t queue_mask
= radv_image_queue_family_mask(image
,
192 cmd_buffer
->queue_family_index
,
193 cmd_buffer
->queue_family_index
);
194 bool compressed
= radv_layout_dcc_compressed(cmd_buffer
->device
, image
, layout
, false, queue_mask
);
196 radv_decompress_dcc(cmd_buffer
, image
, &(VkImageSubresourceRange
) {
197 .aspectMask
= pRegions
[r
].imageSubresource
.aspectMask
,
198 .baseMipLevel
= pRegions
[r
].imageSubresource
.mipLevel
,
200 .baseArrayLayer
= pRegions
[r
].imageSubresource
.baseArrayLayer
,
201 .layerCount
= pRegions
[r
].imageSubresource
.layerCount
,
204 img_bsurf
.format
= vk_format_for_size(vk_format_get_blocksize(img_bsurf
.format
));
205 img_bsurf
.current_layout
= VK_IMAGE_LAYOUT_GENERAL
;
208 struct radv_meta_blit2d_buffer buf_bsurf
= {
210 .format
= img_bsurf
.format
,
212 .offset
= pRegions
[r
].bufferOffset
,
213 .pitch
= buf_extent_el
.width
,
216 if (image
->type
== VK_IMAGE_TYPE_3D
)
217 img_bsurf
.layer
= img_offset_el
.z
;
218 /* Loop through each 3D or array slice */
219 unsigned num_slices_3d
= img_extent_el
.depth
;
220 unsigned num_slices_array
= pRegions
[r
].imageSubresource
.layerCount
;
221 unsigned slice_3d
= 0;
222 unsigned slice_array
= 0;
223 while (slice_3d
< num_slices_3d
&& slice_array
< num_slices_array
) {
225 rect
.dst_x
= img_offset_el
.x
;
226 rect
.dst_y
= img_offset_el
.y
;
231 !image_is_renderable(cmd_buffer
->device
, img_bsurf
.image
)) {
232 radv_meta_buffer_to_image_cs(cmd_buffer
, &buf_bsurf
, &img_bsurf
, 1, &rect
);
234 radv_meta_blit2d(cmd_buffer
, NULL
, &buf_bsurf
, &img_bsurf
, 1, &rect
);
237 /* Once we've done the blit, all of the actual information about
238 * the image is embedded in the command buffer so we can just
239 * increment the offset directly in the image effectively
240 * re-binding it to different backing memory.
242 buf_bsurf
.offset
+= buf_extent_el
.width
*
243 buf_extent_el
.height
* buf_bsurf
.bs
;
245 if (image
->type
== VK_IMAGE_TYPE_3D
)
252 /* Restore conditional rendering. */
253 cmd_buffer
->state
.predicating
= old_predicating
;
255 radv_meta_restore(&saved_state
, cmd_buffer
);
258 void radv_CmdCopyBufferToImage(
259 VkCommandBuffer commandBuffer
,
262 VkImageLayout destImageLayout
,
263 uint32_t regionCount
,
264 const VkBufferImageCopy
* pRegions
)
266 RADV_FROM_HANDLE(radv_cmd_buffer
, cmd_buffer
, commandBuffer
);
267 RADV_FROM_HANDLE(radv_image
, dest_image
, destImage
);
268 RADV_FROM_HANDLE(radv_buffer
, src_buffer
, srcBuffer
);
270 meta_copy_buffer_to_image(cmd_buffer
, src_buffer
, dest_image
, destImageLayout
,
271 regionCount
, pRegions
);
275 meta_copy_image_to_buffer(struct radv_cmd_buffer
*cmd_buffer
,
276 struct radv_buffer
* buffer
,
277 struct radv_image
* image
,
278 VkImageLayout layout
,
279 uint32_t regionCount
,
280 const VkBufferImageCopy
* pRegions
)
282 struct radv_meta_saved_state saved_state
;
283 bool old_predicating
;
285 radv_meta_save(&saved_state
, cmd_buffer
,
286 RADV_META_SAVE_COMPUTE_PIPELINE
|
287 RADV_META_SAVE_CONSTANTS
|
288 RADV_META_SAVE_DESCRIPTORS
);
290 /* VK_EXT_conditional_rendering says that copy commands should not be
291 * affected by conditional rendering.
293 old_predicating
= cmd_buffer
->state
.predicating
;
294 cmd_buffer
->state
.predicating
= false;
296 for (unsigned r
= 0; r
< regionCount
; r
++) {
299 * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
300 * extent is the size in texels of the source image to copy in width,
301 * height and depth. 1D images use only x and width. 2D images use x, y,
302 * width and height. 3D images use x, y, z, width, height and depth.
305 * Also, convert the offsets and extent from units of texels to units of
306 * blocks - which is the highest resolution accessible in this command.
308 const VkOffset3D img_offset_el
=
309 meta_region_offset_el(image
, &pRegions
[r
].imageOffset
);
310 const VkExtent3D bufferExtent
= {
311 .width
= pRegions
[r
].bufferRowLength
?
312 pRegions
[r
].bufferRowLength
: pRegions
[r
].imageExtent
.width
,
313 .height
= pRegions
[r
].bufferImageHeight
?
314 pRegions
[r
].bufferImageHeight
: pRegions
[r
].imageExtent
.height
,
316 const VkExtent3D buf_extent_el
=
317 meta_region_extent_el(image
, image
->type
, &bufferExtent
);
319 /* Start creating blit rect */
320 const VkExtent3D img_extent_el
=
321 meta_region_extent_el(image
, image
->type
, &pRegions
[r
].imageExtent
);
322 struct radv_meta_blit2d_rect rect
= {
323 .width
= img_extent_el
.width
,
324 .height
= img_extent_el
.height
,
327 /* Create blit surfaces */
328 struct radv_meta_blit2d_surf img_info
=
329 blit_surf_for_image_level_layer(image
,
331 &pRegions
[r
].imageSubresource
,
332 pRegions
[r
].imageSubresource
.aspectMask
);
334 if (!radv_is_buffer_format_supported(img_info
.format
, NULL
)) {
335 uint32_t queue_mask
= radv_image_queue_family_mask(image
,
336 cmd_buffer
->queue_family_index
,
337 cmd_buffer
->queue_family_index
);
338 bool compressed
= radv_layout_dcc_compressed(cmd_buffer
->device
, image
, layout
, false, queue_mask
);
340 radv_decompress_dcc(cmd_buffer
, image
, &(VkImageSubresourceRange
) {
341 .aspectMask
= pRegions
[r
].imageSubresource
.aspectMask
,
342 .baseMipLevel
= pRegions
[r
].imageSubresource
.mipLevel
,
344 .baseArrayLayer
= pRegions
[r
].imageSubresource
.baseArrayLayer
,
345 .layerCount
= pRegions
[r
].imageSubresource
.layerCount
,
348 img_info
.format
= vk_format_for_size(vk_format_get_blocksize(img_info
.format
));
349 img_info
.current_layout
= VK_IMAGE_LAYOUT_GENERAL
;
352 struct radv_meta_blit2d_buffer buf_info
= {
354 .format
= img_info
.format
,
356 .offset
= pRegions
[r
].bufferOffset
,
357 .pitch
= buf_extent_el
.width
,
360 if (image
->type
== VK_IMAGE_TYPE_3D
)
361 img_info
.layer
= img_offset_el
.z
;
362 /* Loop through each 3D or array slice */
363 unsigned num_slices_3d
= img_extent_el
.depth
;
364 unsigned num_slices_array
= pRegions
[r
].imageSubresource
.layerCount
;
365 unsigned slice_3d
= 0;
366 unsigned slice_array
= 0;
367 while (slice_3d
< num_slices_3d
&& slice_array
< num_slices_array
) {
369 rect
.src_x
= img_offset_el
.x
;
370 rect
.src_y
= img_offset_el
.y
;
374 radv_meta_image_to_buffer(cmd_buffer
, &img_info
, &buf_info
, 1, &rect
);
376 buf_info
.offset
+= buf_extent_el
.width
*
377 buf_extent_el
.height
* buf_info
.bs
;
379 if (image
->type
== VK_IMAGE_TYPE_3D
)
386 /* Restore conditional rendering. */
387 cmd_buffer
->state
.predicating
= old_predicating
;
389 radv_meta_restore(&saved_state
, cmd_buffer
);
392 void radv_CmdCopyImageToBuffer(
393 VkCommandBuffer commandBuffer
,
395 VkImageLayout srcImageLayout
,
397 uint32_t regionCount
,
398 const VkBufferImageCopy
* pRegions
)
400 RADV_FROM_HANDLE(radv_cmd_buffer
, cmd_buffer
, commandBuffer
);
401 RADV_FROM_HANDLE(radv_image
, src_image
, srcImage
);
402 RADV_FROM_HANDLE(radv_buffer
, dst_buffer
, destBuffer
);
404 meta_copy_image_to_buffer(cmd_buffer
, dst_buffer
, src_image
,
406 regionCount
, pRegions
);
410 meta_copy_image(struct radv_cmd_buffer
*cmd_buffer
,
411 struct radv_image
*src_image
,
412 VkImageLayout src_image_layout
,
413 struct radv_image
*dest_image
,
414 VkImageLayout dest_image_layout
,
415 uint32_t regionCount
,
416 const VkImageCopy
*pRegions
)
418 bool cs
= cmd_buffer
->queue_family_index
== RADV_QUEUE_COMPUTE
;
419 struct radv_meta_saved_state saved_state
;
420 bool old_predicating
;
422 /* From the Vulkan 1.0 spec:
424 * vkCmdCopyImage can be used to copy image data between multisample
425 * images, but both images must have the same number of samples.
427 assert(src_image
->info
.samples
== dest_image
->info
.samples
);
429 radv_meta_save(&saved_state
, cmd_buffer
,
430 (cs
? RADV_META_SAVE_COMPUTE_PIPELINE
:
431 RADV_META_SAVE_GRAPHICS_PIPELINE
) |
432 RADV_META_SAVE_CONSTANTS
|
433 RADV_META_SAVE_DESCRIPTORS
);
435 /* VK_EXT_conditional_rendering says that copy commands should not be
436 * affected by conditional rendering.
438 old_predicating
= cmd_buffer
->state
.predicating
;
439 cmd_buffer
->state
.predicating
= false;
441 for (unsigned r
= 0; r
< regionCount
; r
++) {
442 VkImageAspectFlags src_aspects
[3] = {VK_IMAGE_ASPECT_PLANE_0_BIT
, VK_IMAGE_ASPECT_PLANE_1_BIT
, VK_IMAGE_ASPECT_PLANE_2_BIT
};
443 VkImageAspectFlags dst_aspects
[3] = {VK_IMAGE_ASPECT_PLANE_0_BIT
, VK_IMAGE_ASPECT_PLANE_1_BIT
, VK_IMAGE_ASPECT_PLANE_2_BIT
};
444 unsigned aspect_count
= pRegions
[r
].srcSubresource
.aspectMask
== VK_IMAGE_ASPECT_COLOR_BIT
? src_image
->plane_count
: 1;
445 if (pRegions
[r
].srcSubresource
.aspectMask
!= VK_IMAGE_ASPECT_COLOR_BIT
)
446 src_aspects
[0] = pRegions
[r
].srcSubresource
.aspectMask
;
447 if (pRegions
[r
].dstSubresource
.aspectMask
!= VK_IMAGE_ASPECT_COLOR_BIT
)
448 dst_aspects
[0] = pRegions
[r
].dstSubresource
.aspectMask
;
450 for (unsigned a
= 0; a
< aspect_count
; ++a
) {
451 /* Create blit surfaces */
452 struct radv_meta_blit2d_surf b_src
=
453 blit_surf_for_image_level_layer(src_image
,
455 &pRegions
[r
].srcSubresource
,
458 struct radv_meta_blit2d_surf b_dst
=
459 blit_surf_for_image_level_layer(dest_image
,
461 &pRegions
[r
].dstSubresource
,
464 uint32_t dst_queue_mask
= radv_image_queue_family_mask(dest_image
,
465 cmd_buffer
->queue_family_index
,
466 cmd_buffer
->queue_family_index
);
467 bool dst_compressed
= radv_layout_dcc_compressed(cmd_buffer
->device
, dest_image
, dest_image_layout
, false, dst_queue_mask
);
468 uint32_t src_queue_mask
= radv_image_queue_family_mask(src_image
,
469 cmd_buffer
->queue_family_index
,
470 cmd_buffer
->queue_family_index
);
471 bool src_compressed
= radv_layout_dcc_compressed(cmd_buffer
->device
, src_image
, src_image_layout
, false, src_queue_mask
);
473 if (!src_compressed
|| radv_dcc_formats_compatible(b_src
.format
, b_dst
.format
)) {
474 b_src
.format
= b_dst
.format
;
475 } else if (!dst_compressed
) {
476 b_dst
.format
= b_src
.format
;
478 radv_decompress_dcc(cmd_buffer
, dest_image
, &(VkImageSubresourceRange
) {
479 .aspectMask
= dst_aspects
[a
],
480 .baseMipLevel
= pRegions
[r
].dstSubresource
.mipLevel
,
482 .baseArrayLayer
= pRegions
[r
].dstSubresource
.baseArrayLayer
,
483 .layerCount
= pRegions
[r
].dstSubresource
.layerCount
,
485 b_dst
.format
= b_src
.format
;
486 b_dst
.current_layout
= VK_IMAGE_LAYOUT_GENERAL
;
491 * From the Vulkan 1.0.6 spec: 18.4 Copying Data Between Buffers and Images
492 * imageExtent is the size in texels of the image to copy in width, height
493 * and depth. 1D images use only x and width. 2D images use x, y, width
494 * and height. 3D images use x, y, z, width, height and depth.
496 * Also, convert the offsets and extent from units of texels to units of
497 * blocks - which is the highest resolution accessible in this command.
499 const VkOffset3D dst_offset_el
=
500 meta_region_offset_el(dest_image
, &pRegions
[r
].dstOffset
);
501 const VkOffset3D src_offset_el
=
502 meta_region_offset_el(src_image
, &pRegions
[r
].srcOffset
);
505 * From Vulkan 1.0.68, "Copying Data Between Images":
506 * "When copying between compressed and uncompressed formats
507 * the extent members represent the texel dimensions of the
508 * source image and not the destination."
509 * However, we must use the destination image type to avoid
510 * clamping depth when copying multiple layers of a 2D image to
513 const VkExtent3D img_extent_el
=
514 meta_region_extent_el(src_image
, dest_image
->type
, &pRegions
[r
].extent
);
516 /* Start creating blit rect */
517 struct radv_meta_blit2d_rect rect
= {
518 .width
= img_extent_el
.width
,
519 .height
= img_extent_el
.height
,
522 if (src_image
->type
== VK_IMAGE_TYPE_3D
)
523 b_src
.layer
= src_offset_el
.z
;
525 if (dest_image
->type
== VK_IMAGE_TYPE_3D
)
526 b_dst
.layer
= dst_offset_el
.z
;
528 /* Loop through each 3D or array slice */
529 unsigned num_slices_3d
= img_extent_el
.depth
;
530 unsigned num_slices_array
= pRegions
[r
].dstSubresource
.layerCount
;
531 unsigned slice_3d
= 0;
532 unsigned slice_array
= 0;
533 while (slice_3d
< num_slices_3d
&& slice_array
< num_slices_array
) {
535 /* Finish creating blit rect */
536 rect
.dst_x
= dst_offset_el
.x
;
537 rect
.dst_y
= dst_offset_el
.y
;
538 rect
.src_x
= src_offset_el
.x
;
539 rect
.src_y
= src_offset_el
.y
;
543 !image_is_renderable(cmd_buffer
->device
, b_dst
.image
)) {
544 radv_meta_image_to_image_cs(cmd_buffer
, &b_src
, &b_dst
, 1, &rect
);
546 radv_meta_blit2d(cmd_buffer
, &b_src
, NULL
, &b_dst
, 1, &rect
);
551 if (dest_image
->type
== VK_IMAGE_TYPE_3D
)
559 /* Restore conditional rendering. */
560 cmd_buffer
->state
.predicating
= old_predicating
;
562 radv_meta_restore(&saved_state
, cmd_buffer
);
565 void radv_CmdCopyImage(
566 VkCommandBuffer commandBuffer
,
568 VkImageLayout srcImageLayout
,
570 VkImageLayout destImageLayout
,
571 uint32_t regionCount
,
572 const VkImageCopy
* pRegions
)
574 RADV_FROM_HANDLE(radv_cmd_buffer
, cmd_buffer
, commandBuffer
);
575 RADV_FROM_HANDLE(radv_image
, src_image
, srcImage
);
576 RADV_FROM_HANDLE(radv_image
, dest_image
, destImage
);
578 meta_copy_image(cmd_buffer
,
579 src_image
, srcImageLayout
,
580 dest_image
, destImageLayout
,
581 regionCount
, pRegions
);