2 * Copyright © 2016 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 #include "radv_meta.h"
25 #include "vk_format.h"
28 meta_image_block_size(const struct radv_image
*image
)
30 const struct vk_format_description
*desc
= vk_format_description(image
->vk_format
);
31 return (VkExtent3D
) { desc
->block
.width
, desc
->block
.height
, 1 };
34 /* Returns the user-provided VkBufferImageCopy::imageExtent in units of
35 * elements rather than texels. One element equals one texel or one block
36 * if Image is uncompressed or compressed, respectively.
38 static struct VkExtent3D
39 meta_region_extent_el(const struct radv_image
*image
,
40 const struct VkExtent3D
*extent
)
42 const VkExtent3D block
= meta_image_block_size(image
);
43 return radv_sanitize_image_extent(image
->type
, (VkExtent3D
) {
44 .width
= DIV_ROUND_UP(extent
->width
, block
.width
),
45 .height
= DIV_ROUND_UP(extent
->height
, block
.height
),
46 .depth
= DIV_ROUND_UP(extent
->depth
, block
.depth
),
50 /* Returns the user-provided VkBufferImageCopy::imageOffset in units of
51 * elements rather than texels. One element equals one texel or one block
52 * if Image is uncompressed or compressed, respectively.
54 static struct VkOffset3D
55 meta_region_offset_el(const struct radv_image
*image
,
56 const struct VkOffset3D
*offset
)
58 const VkExtent3D block
= meta_image_block_size(image
);
59 return radv_sanitize_image_offset(image
->type
, (VkOffset3D
) {
60 .x
= offset
->x
/ block
.width
,
61 .y
= offset
->y
/ block
.height
,
62 .z
= offset
->z
/ block
.depth
,
67 vk_format_for_size(int bs
)
70 case 1: return VK_FORMAT_R8_UINT
;
71 case 2: return VK_FORMAT_R8G8_UINT
;
72 case 4: return VK_FORMAT_R8G8B8A8_UINT
;
73 case 8: return VK_FORMAT_R16G16B16A16_UINT
;
74 case 16: return VK_FORMAT_R32G32B32A32_UINT
;
76 unreachable("Invalid format block size");
80 static struct radv_meta_blit2d_surf
81 blit_surf_for_image_level_layer(struct radv_image
*image
,
83 const VkImageSubresourceLayers
*subres
)
85 VkFormat format
= image
->vk_format
;
86 if (subres
->aspectMask
& VK_IMAGE_ASPECT_DEPTH_BIT
)
87 format
= vk_format_depth_only(format
);
88 else if (subres
->aspectMask
& VK_IMAGE_ASPECT_STENCIL_BIT
)
89 format
= vk_format_stencil_only(format
);
91 if (!image
->surface
.dcc_size
&&
92 !(image
->surface
.htile_size
&& image
->tc_compatible_htile
))
93 format
= vk_format_for_size(vk_format_get_blocksize(format
));
95 return (struct radv_meta_blit2d_surf
) {
97 .bs
= vk_format_get_blocksize(format
),
98 .level
= subres
->mipLevel
,
99 .layer
= subres
->baseArrayLayer
,
101 .aspect_mask
= subres
->aspectMask
,
102 .current_layout
= layout
,
107 meta_copy_buffer_to_image(struct radv_cmd_buffer
*cmd_buffer
,
108 struct radv_buffer
* buffer
,
109 struct radv_image
* image
,
110 VkImageLayout layout
,
111 uint32_t regionCount
,
112 const VkBufferImageCopy
* pRegions
)
114 bool cs
= cmd_buffer
->queue_family_index
== RADV_QUEUE_COMPUTE
;
115 struct radv_meta_saved_state saved_state
;
117 /* The Vulkan 1.0 spec says "dstImage must have a sample count equal to
118 * VK_SAMPLE_COUNT_1_BIT."
120 assert(image
->info
.samples
== 1);
122 radv_meta_save(&saved_state
, cmd_buffer
,
123 (cs
? RADV_META_SAVE_COMPUTE_PIPELINE
:
124 RADV_META_SAVE_GRAPHICS_PIPELINE
) |
125 RADV_META_SAVE_CONSTANTS
|
126 RADV_META_SAVE_DESCRIPTORS
);
128 for (unsigned r
= 0; r
< regionCount
; r
++) {
131 * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
132 * extent is the size in texels of the source image to copy in width,
133 * height and depth. 1D images use only x and width. 2D images use x, y,
134 * width and height. 3D images use x, y, z, width, height and depth.
137 * Also, convert the offsets and extent from units of texels to units of
138 * blocks - which is the highest resolution accessible in this command.
140 const VkOffset3D img_offset_el
=
141 meta_region_offset_el(image
, &pRegions
[r
].imageOffset
);
142 const VkExtent3D bufferExtent
= {
143 .width
= pRegions
[r
].bufferRowLength
?
144 pRegions
[r
].bufferRowLength
: pRegions
[r
].imageExtent
.width
,
145 .height
= pRegions
[r
].bufferImageHeight
?
146 pRegions
[r
].bufferImageHeight
: pRegions
[r
].imageExtent
.height
,
148 const VkExtent3D buf_extent_el
=
149 meta_region_extent_el(image
, &bufferExtent
);
151 /* Start creating blit rect */
152 const VkExtent3D img_extent_el
=
153 meta_region_extent_el(image
, &pRegions
[r
].imageExtent
);
154 struct radv_meta_blit2d_rect rect
= {
155 .width
= img_extent_el
.width
,
156 .height
= img_extent_el
.height
,
159 /* Create blit surfaces */
160 struct radv_meta_blit2d_surf img_bsurf
=
161 blit_surf_for_image_level_layer(image
,
163 &pRegions
[r
].imageSubresource
);
165 struct radv_meta_blit2d_buffer buf_bsurf
= {
167 .format
= img_bsurf
.format
,
169 .offset
= pRegions
[r
].bufferOffset
,
170 .pitch
= buf_extent_el
.width
,
173 if (image
->type
== VK_IMAGE_TYPE_3D
)
174 img_bsurf
.layer
= img_offset_el
.z
;
175 /* Loop through each 3D or array slice */
176 unsigned num_slices_3d
= img_extent_el
.depth
;
177 unsigned num_slices_array
= pRegions
[r
].imageSubresource
.layerCount
;
178 unsigned slice_3d
= 0;
179 unsigned slice_array
= 0;
180 while (slice_3d
< num_slices_3d
&& slice_array
< num_slices_array
) {
182 rect
.dst_x
= img_offset_el
.x
;
183 rect
.dst_y
= img_offset_el
.y
;
188 radv_meta_buffer_to_image_cs(cmd_buffer
, &buf_bsurf
, &img_bsurf
, 1, &rect
);
190 radv_meta_blit2d(cmd_buffer
, NULL
, &buf_bsurf
, &img_bsurf
, 1, &rect
);
192 /* Once we've done the blit, all of the actual information about
193 * the image is embedded in the command buffer so we can just
194 * increment the offset directly in the image effectively
195 * re-binding it to different backing memory.
197 buf_bsurf
.offset
+= buf_extent_el
.width
*
198 buf_extent_el
.height
* buf_bsurf
.bs
;
200 if (image
->type
== VK_IMAGE_TYPE_3D
)
207 radv_meta_restore(&saved_state
, cmd_buffer
);
210 void radv_CmdCopyBufferToImage(
211 VkCommandBuffer commandBuffer
,
214 VkImageLayout destImageLayout
,
215 uint32_t regionCount
,
216 const VkBufferImageCopy
* pRegions
)
218 RADV_FROM_HANDLE(radv_cmd_buffer
, cmd_buffer
, commandBuffer
);
219 RADV_FROM_HANDLE(radv_image
, dest_image
, destImage
);
220 RADV_FROM_HANDLE(radv_buffer
, src_buffer
, srcBuffer
);
222 meta_copy_buffer_to_image(cmd_buffer
, src_buffer
, dest_image
, destImageLayout
,
223 regionCount
, pRegions
);
227 meta_copy_image_to_buffer(struct radv_cmd_buffer
*cmd_buffer
,
228 struct radv_buffer
* buffer
,
229 struct radv_image
* image
,
230 VkImageLayout layout
,
231 uint32_t regionCount
,
232 const VkBufferImageCopy
* pRegions
)
234 struct radv_meta_saved_state saved_state
;
236 radv_meta_save(&saved_state
, cmd_buffer
,
237 RADV_META_SAVE_COMPUTE_PIPELINE
|
238 RADV_META_SAVE_CONSTANTS
|
239 RADV_META_SAVE_DESCRIPTORS
);
241 for (unsigned r
= 0; r
< regionCount
; r
++) {
244 * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
245 * extent is the size in texels of the source image to copy in width,
246 * height and depth. 1D images use only x and width. 2D images use x, y,
247 * width and height. 3D images use x, y, z, width, height and depth.
250 * Also, convert the offsets and extent from units of texels to units of
251 * blocks - which is the highest resolution accessible in this command.
253 const VkOffset3D img_offset_el
=
254 meta_region_offset_el(image
, &pRegions
[r
].imageOffset
);
255 const VkExtent3D bufferExtent
= {
256 .width
= pRegions
[r
].bufferRowLength
?
257 pRegions
[r
].bufferRowLength
: pRegions
[r
].imageExtent
.width
,
258 .height
= pRegions
[r
].bufferImageHeight
?
259 pRegions
[r
].bufferImageHeight
: pRegions
[r
].imageExtent
.height
,
261 const VkExtent3D buf_extent_el
=
262 meta_region_extent_el(image
, &bufferExtent
);
264 /* Start creating blit rect */
265 const VkExtent3D img_extent_el
=
266 meta_region_extent_el(image
, &pRegions
[r
].imageExtent
);
267 struct radv_meta_blit2d_rect rect
= {
268 .width
= img_extent_el
.width
,
269 .height
= img_extent_el
.height
,
272 /* Create blit surfaces */
273 struct radv_meta_blit2d_surf img_info
=
274 blit_surf_for_image_level_layer(image
,
276 &pRegions
[r
].imageSubresource
);
278 struct radv_meta_blit2d_buffer buf_info
= {
280 .format
= img_info
.format
,
282 .offset
= pRegions
[r
].bufferOffset
,
283 .pitch
= buf_extent_el
.width
,
286 if (image
->type
== VK_IMAGE_TYPE_3D
)
287 img_info
.layer
= img_offset_el
.z
;
288 /* Loop through each 3D or array slice */
289 unsigned num_slices_3d
= img_extent_el
.depth
;
290 unsigned num_slices_array
= pRegions
[r
].imageSubresource
.layerCount
;
291 unsigned slice_3d
= 0;
292 unsigned slice_array
= 0;
293 while (slice_3d
< num_slices_3d
&& slice_array
< num_slices_array
) {
295 rect
.src_x
= img_offset_el
.x
;
296 rect
.src_y
= img_offset_el
.y
;
300 radv_meta_image_to_buffer(cmd_buffer
, &img_info
, &buf_info
, 1, &rect
);
302 buf_info
.offset
+= buf_extent_el
.width
*
303 buf_extent_el
.height
* buf_info
.bs
;
305 if (image
->type
== VK_IMAGE_TYPE_3D
)
312 radv_meta_restore(&saved_state
, cmd_buffer
);
315 void radv_CmdCopyImageToBuffer(
316 VkCommandBuffer commandBuffer
,
318 VkImageLayout srcImageLayout
,
320 uint32_t regionCount
,
321 const VkBufferImageCopy
* pRegions
)
323 RADV_FROM_HANDLE(radv_cmd_buffer
, cmd_buffer
, commandBuffer
);
324 RADV_FROM_HANDLE(radv_image
, src_image
, srcImage
);
325 RADV_FROM_HANDLE(radv_buffer
, dst_buffer
, destBuffer
);
327 meta_copy_image_to_buffer(cmd_buffer
, dst_buffer
, src_image
,
329 regionCount
, pRegions
);
333 meta_copy_image(struct radv_cmd_buffer
*cmd_buffer
,
334 struct radv_image
*src_image
,
335 VkImageLayout src_image_layout
,
336 struct radv_image
*dest_image
,
337 VkImageLayout dest_image_layout
,
338 uint32_t regionCount
,
339 const VkImageCopy
*pRegions
)
341 bool cs
= cmd_buffer
->queue_family_index
== RADV_QUEUE_COMPUTE
;
342 struct radv_meta_saved_state saved_state
;
344 /* From the Vulkan 1.0 spec:
346 * vkCmdCopyImage can be used to copy image data between multisample
347 * images, but both images must have the same number of samples.
349 assert(src_image
->info
.samples
== dest_image
->info
.samples
);
351 radv_meta_save(&saved_state
, cmd_buffer
,
352 (cs
? RADV_META_SAVE_COMPUTE_PIPELINE
:
353 RADV_META_SAVE_GRAPHICS_PIPELINE
) |
354 RADV_META_SAVE_CONSTANTS
|
355 RADV_META_SAVE_DESCRIPTORS
);
357 for (unsigned r
= 0; r
< regionCount
; r
++) {
358 assert(pRegions
[r
].srcSubresource
.aspectMask
==
359 pRegions
[r
].dstSubresource
.aspectMask
);
361 /* Create blit surfaces */
362 struct radv_meta_blit2d_surf b_src
=
363 blit_surf_for_image_level_layer(src_image
,
365 &pRegions
[r
].srcSubresource
);
367 struct radv_meta_blit2d_surf b_dst
=
368 blit_surf_for_image_level_layer(dest_image
,
370 &pRegions
[r
].dstSubresource
);
372 uint32_t dst_queue_mask
= radv_image_queue_family_mask(dest_image
,
373 cmd_buffer
->queue_family_index
,
374 cmd_buffer
->queue_family_index
);
375 bool dst_compressed
= radv_layout_dcc_compressed(dest_image
, dest_image_layout
, dst_queue_mask
);
376 uint32_t src_queue_mask
= radv_image_queue_family_mask(src_image
,
377 cmd_buffer
->queue_family_index
,
378 cmd_buffer
->queue_family_index
);
379 bool src_compressed
= radv_layout_dcc_compressed(src_image
, src_image_layout
, src_queue_mask
);
381 if (!src_compressed
|| radv_dcc_formats_compatible(b_src
.format
, b_dst
.format
)) {
382 b_src
.format
= b_dst
.format
;
383 } else if (!dst_compressed
) {
384 b_dst
.format
= b_src
.format
;
386 radv_decompress_dcc(cmd_buffer
, dest_image
, &(VkImageSubresourceRange
) {
387 .aspectMask
= pRegions
[r
].dstSubresource
.aspectMask
,
388 .baseMipLevel
= pRegions
[r
].dstSubresource
.mipLevel
,
390 .baseArrayLayer
= pRegions
[r
].dstSubresource
.baseArrayLayer
,
391 .layerCount
= pRegions
[r
].dstSubresource
.layerCount
,
393 b_dst
.format
= b_src
.format
;
394 b_dst
.current_layout
= VK_IMAGE_LAYOUT_GENERAL
;
399 * From the Vulkan 1.0.6 spec: 18.4 Copying Data Between Buffers and Images
400 * imageExtent is the size in texels of the image to copy in width, height
401 * and depth. 1D images use only x and width. 2D images use x, y, width
402 * and height. 3D images use x, y, z, width, height and depth.
404 * Also, convert the offsets and extent from units of texels to units of
405 * blocks - which is the highest resolution accessible in this command.
407 const VkOffset3D dst_offset_el
=
408 meta_region_offset_el(dest_image
, &pRegions
[r
].dstOffset
);
409 const VkOffset3D src_offset_el
=
410 meta_region_offset_el(src_image
, &pRegions
[r
].srcOffset
);
411 const VkExtent3D img_extent_el
=
412 meta_region_extent_el(dest_image
, &pRegions
[r
].extent
);
414 /* Start creating blit rect */
415 struct radv_meta_blit2d_rect rect
= {
416 .width
= img_extent_el
.width
,
417 .height
= img_extent_el
.height
,
420 if (dest_image
->type
== VK_IMAGE_TYPE_3D
)
421 b_dst
.layer
= dst_offset_el
.z
;
423 /* Loop through each 3D or array slice */
424 unsigned num_slices_3d
= img_extent_el
.depth
;
425 unsigned num_slices_array
= pRegions
[r
].dstSubresource
.layerCount
;
426 unsigned slice_3d
= 0;
427 unsigned slice_array
= 0;
428 while (slice_3d
< num_slices_3d
&& slice_array
< num_slices_array
) {
430 /* Finish creating blit rect */
431 rect
.dst_x
= dst_offset_el
.x
;
432 rect
.dst_y
= dst_offset_el
.y
;
433 rect
.src_x
= src_offset_el
.x
;
434 rect
.src_y
= src_offset_el
.y
;
438 radv_meta_image_to_image_cs(cmd_buffer
, &b_src
, &b_dst
, 1, &rect
);
440 radv_meta_blit2d(cmd_buffer
, &b_src
, NULL
, &b_dst
, 1, &rect
);
444 if (dest_image
->type
== VK_IMAGE_TYPE_3D
)
451 radv_meta_restore(&saved_state
, cmd_buffer
);
454 void radv_CmdCopyImage(
455 VkCommandBuffer commandBuffer
,
457 VkImageLayout srcImageLayout
,
459 VkImageLayout destImageLayout
,
460 uint32_t regionCount
,
461 const VkImageCopy
* pRegions
)
463 RADV_FROM_HANDLE(radv_cmd_buffer
, cmd_buffer
, commandBuffer
);
464 RADV_FROM_HANDLE(radv_image
, src_image
, srcImage
);
465 RADV_FROM_HANDLE(radv_image
, dest_image
, destImage
);
467 meta_copy_image(cmd_buffer
,
468 src_image
, srcImageLayout
,
469 dest_image
, destImageLayout
,
470 regionCount
, pRegions
);
473 void radv_blit_to_prime_linear(struct radv_cmd_buffer
*cmd_buffer
,
474 struct radv_image
*image
,
475 struct radv_image
*linear_image
)
477 struct VkImageCopy image_copy
= { 0 };
479 image_copy
.srcSubresource
.aspectMask
= VK_IMAGE_ASPECT_COLOR_BIT
;
480 image_copy
.srcSubresource
.layerCount
= 1;
482 image_copy
.dstSubresource
.aspectMask
= VK_IMAGE_ASPECT_COLOR_BIT
;
483 image_copy
.dstSubresource
.layerCount
= 1;
485 image_copy
.extent
.width
= image
->info
.width
;
486 image_copy
.extent
.height
= image
->info
.height
;
487 image_copy
.extent
.depth
= 1;
489 meta_copy_image(cmd_buffer
, image
, VK_IMAGE_LAYOUT_GENERAL
, linear_image
,
490 VK_IMAGE_LAYOUT_GENERAL
,