2 * Copyright © 2016 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
27 meta_image_block_size(const struct anv_image
*image
)
29 if (image
->aspects
== VK_IMAGE_ASPECT_COLOR_BIT
) {
30 const struct isl_format_layout
*isl_layout
=
31 isl_format_get_layout(image
->color_surface
.isl
.format
);
32 return (VkExtent3D
) { isl_layout
->bw
, isl_layout
->bh
, isl_layout
->bd
};
34 return (VkExtent3D
) { 1, 1, 1 };
38 /* Returns the user-provided VkBufferImageCopy::imageExtent in units of
39 * elements rather than texels. One element equals one texel or one block
40 * if Image is uncompressed or compressed, respectively.
42 static struct VkExtent3D
43 meta_region_extent_el(const struct anv_image
*image
,
44 const struct VkExtent3D
*extent
)
46 const VkExtent3D block
= meta_image_block_size(image
);
47 return anv_sanitize_image_extent(image
->type
, (VkExtent3D
) {
48 .width
= DIV_ROUND_UP(extent
->width
, block
.width
),
49 .height
= DIV_ROUND_UP(extent
->height
, block
.height
),
50 .depth
= DIV_ROUND_UP(extent
->depth
, block
.depth
),
54 /* Returns the user-provided VkBufferImageCopy::imageOffset in units of
55 * elements rather than texels. One element equals one texel or one block
56 * if Image is uncompressed or compressed, respectively.
58 static struct VkOffset3D
59 meta_region_offset_el(const struct anv_image
*image
,
60 const struct VkOffset3D
*offset
)
62 const VkExtent3D block
= meta_image_block_size(image
);
63 return anv_sanitize_image_offset(image
->type
, (VkOffset3D
) {
64 .x
= offset
->x
/ block
.width
,
65 .y
= offset
->y
/ block
.height
,
66 .z
= offset
->z
/ block
.depth
,
70 static struct anv_meta_blit2d_surf
71 blit_surf_for_image(const struct anv_image
* image
,
72 const struct isl_surf
*img_isl_surf
)
74 return (struct anv_meta_blit2d_surf
) {
76 .tiling
= img_isl_surf
->tiling
,
77 .base_offset
= image
->offset
,
78 .bs
= isl_format_get_layout(img_isl_surf
->format
)->bs
,
79 .pitch
= isl_surf_get_row_pitch(img_isl_surf
),
84 do_buffer_copy(struct anv_cmd_buffer
*cmd_buffer
,
85 struct anv_bo
*src
, uint64_t src_offset
,
86 struct anv_bo
*dest
, uint64_t dest_offset
,
87 int width
, int height
, int bs
)
89 struct anv_meta_blit2d_surf b_src
= {
91 .tiling
= ISL_TILING_LINEAR
,
92 .base_offset
= src_offset
,
96 struct anv_meta_blit2d_surf b_dst
= {
98 .tiling
= ISL_TILING_LINEAR
,
99 .base_offset
= dest_offset
,
103 struct anv_meta_blit2d_rect rect
= {
107 anv_meta_blit2d(cmd_buffer
, &b_src
, &b_dst
, 1, &rect
);
111 meta_copy_buffer_to_image(struct anv_cmd_buffer
*cmd_buffer
,
112 struct anv_buffer
* buffer
,
113 struct anv_image
* image
,
114 uint32_t regionCount
,
115 const VkBufferImageCopy
* pRegions
,
118 struct anv_meta_saved_state saved_state
;
120 /* The Vulkan 1.0 spec says "dstImage must have a sample count equal to
121 * VK_SAMPLE_COUNT_1_BIT."
123 assert(image
->samples
== 1);
125 anv_meta_begin_blit2d(cmd_buffer
, &saved_state
);
127 for (unsigned r
= 0; r
< regionCount
; r
++) {
130 * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
131 * extent is the size in texels of the source image to copy in width,
132 * height and depth. 1D images use only x and width. 2D images use x, y,
133 * width and height. 3D images use x, y, z, width, height and depth.
136 * Also, convert the offsets and extent from units of texels to units of
137 * blocks - which is the highest resolution accessible in this command.
139 const VkOffset3D img_offset_el
=
140 meta_region_offset_el(image
, &pRegions
[r
].imageOffset
);
141 const VkExtent3D bufferExtent
= {
142 .width
= pRegions
[r
].bufferRowLength
?
143 pRegions
[r
].bufferRowLength
: pRegions
[r
].imageExtent
.width
,
144 .height
= pRegions
[r
].bufferImageHeight
?
145 pRegions
[r
].bufferImageHeight
: pRegions
[r
].imageExtent
.height
,
147 const VkExtent3D buf_extent_el
=
148 meta_region_extent_el(image
, &bufferExtent
);
150 /* Start creating blit rect */
151 const VkExtent3D img_extent_el
=
152 meta_region_extent_el(image
, &pRegions
[r
].imageExtent
);
153 struct anv_meta_blit2d_rect rect
= {
154 .width
= img_extent_el
.width
,
155 .height
= img_extent_el
.height
,
158 /* Create blit surfaces */
159 VkImageAspectFlags aspect
= pRegions
[r
].imageSubresource
.aspectMask
;
160 const struct isl_surf
*img_isl_surf
=
161 &anv_image_get_surface_for_aspect_mask(image
, aspect
)->isl
;
162 struct anv_meta_blit2d_surf img_bsurf
=
163 blit_surf_for_image(image
, img_isl_surf
);
164 enum isl_format buf_format
= anv_get_isl_format(&cmd_buffer
->device
->info
,
165 image
->vk_format
, aspect
,
166 VK_IMAGE_TILING_LINEAR
);
167 struct anv_meta_blit2d_surf buf_bsurf
= {
169 .tiling
= ISL_TILING_LINEAR
,
170 .base_offset
= buffer
->offset
+ pRegions
[r
].bufferOffset
,
171 .bs
= isl_format_get_layout(buf_format
)->bs
,
172 .pitch
= buf_extent_el
.width
* buf_bsurf
.bs
,
175 /* Set direction-dependent variables */
176 struct anv_meta_blit2d_surf
*dst_bsurf
= forward
? &img_bsurf
: &buf_bsurf
;
177 struct anv_meta_blit2d_surf
*src_bsurf
= forward
? &buf_bsurf
: &img_bsurf
;
178 uint32_t *x_offset
= forward
? &rect
.dst_x
: &rect
.src_x
;
179 uint32_t *y_offset
= forward
? &rect
.dst_y
: &rect
.src_y
;
181 /* Loop through each 3D or array slice */
182 unsigned num_slices_3d
= img_extent_el
.depth
;
183 unsigned num_slices_array
= pRegions
[r
].imageSubresource
.layerCount
;
184 unsigned slice_3d
= 0;
185 unsigned slice_array
= 0;
186 while (slice_3d
< num_slices_3d
&& slice_array
< num_slices_array
) {
188 /* Finish creating blit rect */
189 isl_surf_get_image_offset_el(img_isl_surf
,
190 pRegions
[r
].imageSubresource
.mipLevel
,
191 pRegions
[r
].imageSubresource
.baseArrayLayer
193 img_offset_el
.z
+ slice_3d
,
196 *x_offset
+= img_offset_el
.x
;
197 *y_offset
+= img_offset_el
.y
;
200 anv_meta_blit2d(cmd_buffer
, src_bsurf
, dst_bsurf
, 1, &rect
);
202 /* Once we've done the blit, all of the actual information about
203 * the image is embedded in the command buffer so we can just
204 * increment the offset directly in the image effectively
205 * re-binding it to different backing memory.
207 buf_bsurf
.base_offset
+= buf_extent_el
.width
*
208 buf_extent_el
.height
* buf_bsurf
.bs
;
210 if (image
->type
== VK_IMAGE_TYPE_3D
)
216 anv_meta_end_blit2d(cmd_buffer
, &saved_state
);
219 void anv_CmdCopyBufferToImage(
220 VkCommandBuffer commandBuffer
,
223 VkImageLayout destImageLayout
,
224 uint32_t regionCount
,
225 const VkBufferImageCopy
* pRegions
)
227 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
, commandBuffer
);
228 ANV_FROM_HANDLE(anv_image
, dest_image
, destImage
);
229 ANV_FROM_HANDLE(anv_buffer
, src_buffer
, srcBuffer
);
231 meta_copy_buffer_to_image(cmd_buffer
, src_buffer
, dest_image
,
232 regionCount
, pRegions
, true);
235 void anv_CmdCopyImageToBuffer(
236 VkCommandBuffer commandBuffer
,
238 VkImageLayout srcImageLayout
,
240 uint32_t regionCount
,
241 const VkBufferImageCopy
* pRegions
)
243 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
, commandBuffer
);
244 ANV_FROM_HANDLE(anv_image
, src_image
, srcImage
);
245 ANV_FROM_HANDLE(anv_buffer
, dst_buffer
, destBuffer
);
247 meta_copy_buffer_to_image(cmd_buffer
, dst_buffer
, src_image
,
248 regionCount
, pRegions
, false);
251 void anv_CmdCopyImage(
252 VkCommandBuffer commandBuffer
,
254 VkImageLayout srcImageLayout
,
256 VkImageLayout destImageLayout
,
257 uint32_t regionCount
,
258 const VkImageCopy
* pRegions
)
260 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
, commandBuffer
);
261 ANV_FROM_HANDLE(anv_image
, src_image
, srcImage
);
262 ANV_FROM_HANDLE(anv_image
, dest_image
, destImage
);
263 struct anv_meta_saved_state saved_state
;
265 /* From the Vulkan 1.0 spec:
267 * vkCmdCopyImage can be used to copy image data between multisample
268 * images, but both images must have the same number of samples.
270 assert(src_image
->samples
== dest_image
->samples
);
272 anv_meta_begin_blit2d(cmd_buffer
, &saved_state
);
274 for (unsigned r
= 0; r
< regionCount
; r
++) {
275 assert(pRegions
[r
].srcSubresource
.aspectMask
==
276 pRegions
[r
].dstSubresource
.aspectMask
);
278 VkImageAspectFlags aspect
= pRegions
[r
].srcSubresource
.aspectMask
;
280 /* Create blit surfaces */
281 struct isl_surf
*src_isl_surf
=
282 &anv_image_get_surface_for_aspect_mask(src_image
, aspect
)->isl
;
283 struct isl_surf
*dst_isl_surf
=
284 &anv_image_get_surface_for_aspect_mask(dest_image
, aspect
)->isl
;
285 struct anv_meta_blit2d_surf b_src
=
286 blit_surf_for_image(src_image
, src_isl_surf
);
287 struct anv_meta_blit2d_surf b_dst
=
288 blit_surf_for_image(dest_image
, dst_isl_surf
);
291 * From the Vulkan 1.0.6 spec: 18.4 Copying Data Between Buffers and Images
292 * imageExtent is the size in texels of the image to copy in width, height
293 * and depth. 1D images use only x and width. 2D images use x, y, width
294 * and height. 3D images use x, y, z, width, height and depth.
296 * Also, convert the offsets and extent from units of texels to units of
297 * blocks - which is the highest resolution accessible in this command.
299 const VkOffset3D dst_offset_el
=
300 meta_region_offset_el(dest_image
, &pRegions
[r
].dstOffset
);
301 const VkOffset3D src_offset_el
=
302 meta_region_offset_el(src_image
, &pRegions
[r
].srcOffset
);
303 const VkExtent3D img_extent_el
=
304 meta_region_extent_el(src_image
, &pRegions
[r
].extent
);
306 /* Start creating blit rect */
307 struct anv_meta_blit2d_rect rect
= {
308 .width
= img_extent_el
.width
,
309 .height
= img_extent_el
.height
,
312 /* Loop through each 3D or array slice */
313 unsigned num_slices_3d
= img_extent_el
.depth
;
314 unsigned num_slices_array
= pRegions
[r
].dstSubresource
.layerCount
;
315 unsigned slice_3d
= 0;
316 unsigned slice_array
= 0;
317 while (slice_3d
< num_slices_3d
&& slice_array
< num_slices_array
) {
319 /* Finish creating blit rect */
320 isl_surf_get_image_offset_el(dst_isl_surf
,
321 pRegions
[r
].dstSubresource
.mipLevel
,
322 pRegions
[r
].dstSubresource
.baseArrayLayer
324 dst_offset_el
.z
+ slice_3d
,
327 isl_surf_get_image_offset_el(src_isl_surf
,
328 pRegions
[r
].srcSubresource
.mipLevel
,
329 pRegions
[r
].srcSubresource
.baseArrayLayer
331 src_offset_el
.z
+ slice_3d
,
334 rect
.dst_x
+= dst_offset_el
.x
;
335 rect
.dst_y
+= dst_offset_el
.y
;
336 rect
.src_x
+= src_offset_el
.x
;
337 rect
.src_y
+= src_offset_el
.y
;
340 anv_meta_blit2d(cmd_buffer
, &b_src
, &b_dst
, 1, &rect
);
342 if (dest_image
->type
== VK_IMAGE_TYPE_3D
)
349 anv_meta_end_blit2d(cmd_buffer
, &saved_state
);
352 void anv_CmdCopyBuffer(
353 VkCommandBuffer commandBuffer
,
356 uint32_t regionCount
,
357 const VkBufferCopy
* pRegions
)
359 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
, commandBuffer
);
360 ANV_FROM_HANDLE(anv_buffer
, src_buffer
, srcBuffer
);
361 ANV_FROM_HANDLE(anv_buffer
, dest_buffer
, destBuffer
);
363 struct anv_meta_saved_state saved_state
;
365 anv_meta_begin_blit2d(cmd_buffer
, &saved_state
);
367 for (unsigned r
= 0; r
< regionCount
; r
++) {
368 uint64_t src_offset
= src_buffer
->offset
+ pRegions
[r
].srcOffset
;
369 uint64_t dest_offset
= dest_buffer
->offset
+ pRegions
[r
].dstOffset
;
370 uint64_t copy_size
= pRegions
[r
].size
;
372 /* First, we compute the biggest format that can be used with the
373 * given offsets and size.
377 int fs
= ffs(src_offset
) - 1;
379 bs
= MIN2(bs
, 1 << fs
);
380 assert(src_offset
% bs
== 0);
382 fs
= ffs(dest_offset
) - 1;
384 bs
= MIN2(bs
, 1 << fs
);
385 assert(dest_offset
% bs
== 0);
387 fs
= ffs(pRegions
[r
].size
) - 1;
389 bs
= MIN2(bs
, 1 << fs
);
390 assert(pRegions
[r
].size
% bs
== 0);
392 /* This is maximum possible width/height our HW can handle */
393 uint64_t max_surface_dim
= 1 << 14;
395 /* First, we make a bunch of max-sized copies */
396 uint64_t max_copy_size
= max_surface_dim
* max_surface_dim
* bs
;
397 while (copy_size
>= max_copy_size
) {
398 do_buffer_copy(cmd_buffer
, src_buffer
->bo
, src_offset
,
399 dest_buffer
->bo
, dest_offset
,
400 max_surface_dim
, max_surface_dim
, bs
);
401 copy_size
-= max_copy_size
;
402 src_offset
+= max_copy_size
;
403 dest_offset
+= max_copy_size
;
406 uint64_t height
= copy_size
/ (max_surface_dim
* bs
);
407 assert(height
< max_surface_dim
);
409 uint64_t rect_copy_size
= height
* max_surface_dim
* bs
;
410 do_buffer_copy(cmd_buffer
, src_buffer
->bo
, src_offset
,
411 dest_buffer
->bo
, dest_offset
,
412 max_surface_dim
, height
, bs
);
413 copy_size
-= rect_copy_size
;
414 src_offset
+= rect_copy_size
;
415 dest_offset
+= rect_copy_size
;
418 if (copy_size
!= 0) {
419 do_buffer_copy(cmd_buffer
, src_buffer
->bo
, src_offset
,
420 dest_buffer
->bo
, dest_offset
,
421 copy_size
/ bs
, 1, bs
);
425 anv_meta_end_blit2d(cmd_buffer
, &saved_state
);
428 void anv_CmdUpdateBuffer(
429 VkCommandBuffer commandBuffer
,
431 VkDeviceSize dstOffset
,
432 VkDeviceSize dataSize
,
433 const uint32_t* pData
)
435 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
, commandBuffer
);
436 ANV_FROM_HANDLE(anv_buffer
, dst_buffer
, dstBuffer
);
437 struct anv_meta_saved_state saved_state
;
439 anv_meta_begin_blit2d(cmd_buffer
, &saved_state
);
441 /* We can't quite grab a full block because the state stream needs a
442 * little data at the top to build its linked list.
444 const uint32_t max_update_size
=
445 cmd_buffer
->device
->dynamic_state_block_pool
.block_size
- 64;
447 assert(max_update_size
< (1 << 14) * 4);
450 const uint32_t copy_size
= MIN2(dataSize
, max_update_size
);
452 struct anv_state tmp_data
=
453 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer
, copy_size
, 64);
455 memcpy(tmp_data
.map
, pData
, copy_size
);
458 if ((copy_size
& 15) == 0 && (dstOffset
& 15) == 0) {
460 } else if ((copy_size
& 7) == 0 && (dstOffset
& 7) == 0) {
463 assert((copy_size
& 3) == 0 && (dstOffset
& 3) == 0);
467 do_buffer_copy(cmd_buffer
,
468 &cmd_buffer
->device
->dynamic_state_block_pool
.bo
,
470 dst_buffer
->bo
, dst_buffer
->offset
+ dstOffset
,
471 copy_size
/ bs
, 1, bs
);
473 dataSize
-= copy_size
;
474 dstOffset
+= copy_size
;
475 pData
= (void *)pData
+ copy_size
;
478 anv_meta_end_blit2d(cmd_buffer
, &saved_state
);