2 * Copyright © 2016 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
26 /* Returns the user-provided VkBufferImageCopy::imageExtent in units of
27 * elements rather than texels. One element equals one texel or one block
28 * if Image is uncompressed or compressed, respectively.
30 static struct VkExtent3D
31 meta_region_extent_el(const VkFormat format
,
32 const struct VkExtent3D
*extent
)
34 const struct isl_format_layout
*isl_layout
=
35 anv_format_for_vk_format(format
)->isl_layout
;
37 .width
= DIV_ROUND_UP(extent
->width
, isl_layout
->bw
),
38 .height
= DIV_ROUND_UP(extent
->height
, isl_layout
->bh
),
39 .depth
= DIV_ROUND_UP(extent
->depth
, isl_layout
->bd
),
43 /* Returns the user-provided VkBufferImageCopy::imageOffset in units of
44 * elements rather than texels. One element equals one texel or one block
45 * if Image is uncompressed or compressed, respectively.
47 static struct VkOffset3D
48 meta_region_offset_el(const struct anv_image
*image
,
49 const struct VkOffset3D
*offset
)
51 const struct isl_format_layout
*isl_layout
= image
->format
->isl_layout
;
53 .x
= offset
->x
/ isl_layout
->bw
,
54 .y
= offset
->y
/ isl_layout
->bh
,
55 .z
= offset
->z
/ isl_layout
->bd
,
59 static struct anv_meta_blit2d_surf
60 blit_surf_for_image(const struct anv_image
* image
,
61 const struct isl_surf
*img_isl_surf
)
63 return (struct anv_meta_blit2d_surf
) {
65 .tiling
= img_isl_surf
->tiling
,
66 .base_offset
= image
->offset
,
67 .bs
= isl_format_get_layout(img_isl_surf
->format
)->bs
,
68 .pitch
= isl_surf_get_row_pitch(img_isl_surf
),
73 do_buffer_copy(struct anv_cmd_buffer
*cmd_buffer
,
74 struct anv_bo
*src
, uint64_t src_offset
,
75 struct anv_bo
*dest
, uint64_t dest_offset
,
76 int width
, int height
, int bs
)
78 struct anv_meta_blit2d_surf b_src
= {
80 .tiling
= ISL_TILING_LINEAR
,
81 .base_offset
= src_offset
,
85 struct anv_meta_blit2d_surf b_dst
= {
87 .tiling
= ISL_TILING_LINEAR
,
88 .base_offset
= dest_offset
,
92 struct anv_meta_blit2d_rect rect
= {
96 anv_meta_blit2d(cmd_buffer
, &b_src
, &b_dst
, 1, &rect
);
100 meta_copy_buffer_to_image(struct anv_cmd_buffer
*cmd_buffer
,
101 struct anv_buffer
* buffer
,
102 struct anv_image
* image
,
103 uint32_t regionCount
,
104 const VkBufferImageCopy
* pRegions
,
107 struct anv_meta_saved_state saved_state
;
109 /* The Vulkan 1.0 spec says "dstImage must have a sample count equal to
110 * VK_SAMPLE_COUNT_1_BIT."
112 assert(image
->samples
== 1);
114 anv_meta_begin_blit2d(cmd_buffer
, &saved_state
);
116 for (unsigned r
= 0; r
< regionCount
; r
++) {
118 /* Start creating blit rect */
119 const VkOffset3D img_offset_el
=
120 meta_region_offset_el(image
, &pRegions
[r
].imageOffset
);
121 const VkExtent3D bufferExtent
= {
122 .width
= pRegions
[r
].bufferRowLength
,
123 .height
= pRegions
[r
].bufferImageHeight
,
125 const VkExtent3D buf_extent_el
=
126 meta_region_extent_el(image
->vk_format
, &bufferExtent
);
127 const VkExtent3D img_extent_el
=
128 meta_region_extent_el(image
->vk_format
, &pRegions
[r
].imageExtent
);
129 struct anv_meta_blit2d_rect rect
= {
130 .width
= MAX2(buf_extent_el
.width
, img_extent_el
.width
),
131 .height
= MAX2(buf_extent_el
.height
, img_extent_el
.height
),
134 /* Create blit surfaces */
135 VkImageAspectFlags aspect
= pRegions
[r
].imageSubresource
.aspectMask
;
136 const struct isl_surf
*img_isl_surf
=
137 &anv_image_get_surface_for_aspect_mask(image
, aspect
)->isl
;
138 struct anv_meta_blit2d_surf img_bsurf
=
139 blit_surf_for_image(image
, img_isl_surf
);
140 struct anv_meta_blit2d_surf buf_bsurf
= {
142 .tiling
= ISL_TILING_LINEAR
,
143 .base_offset
= buffer
->offset
+ pRegions
[r
].bufferOffset
,
144 .bs
= forward
? image
->format
->isl_layout
->bs
: img_bsurf
.bs
,
145 .pitch
= rect
.width
* buf_bsurf
.bs
,
148 /* Set direction-dependent variables */
149 struct anv_meta_blit2d_surf
*dst_bsurf
= forward
? &img_bsurf
: &buf_bsurf
;
150 struct anv_meta_blit2d_surf
*src_bsurf
= forward
? &buf_bsurf
: &img_bsurf
;
151 uint32_t *x_offset
= forward
? &rect
.dst_x
: &rect
.src_x
;
152 uint32_t *y_offset
= forward
? &rect
.dst_y
: &rect
.src_y
;
154 /* Loop through each 3D or array slice */
155 unsigned num_slices_3d
= pRegions
[r
].imageExtent
.depth
;
156 unsigned num_slices_array
= pRegions
[r
].imageSubresource
.layerCount
;
157 unsigned slice_3d
= 0;
158 unsigned slice_array
= 0;
159 while (slice_3d
< num_slices_3d
&& slice_array
< num_slices_array
) {
161 /* Finish creating blit rect */
162 isl_surf_get_image_offset_el(img_isl_surf
,
163 pRegions
[r
].imageSubresource
.mipLevel
,
164 pRegions
[r
].imageSubresource
.baseArrayLayer
166 pRegions
[r
].imageOffset
.z
+ slice_3d
,
169 *x_offset
+= img_offset_el
.x
;
170 *y_offset
+= img_offset_el
.y
;
173 anv_meta_blit2d(cmd_buffer
, src_bsurf
, dst_bsurf
, 1, &rect
);
175 /* Once we've done the blit, all of the actual information about
176 * the image is embedded in the command buffer so we can just
177 * increment the offset directly in the image effectively
178 * re-binding it to different backing memory.
180 buf_bsurf
.base_offset
+= rect
.width
* rect
.height
* buf_bsurf
.bs
;
182 if (image
->type
== VK_IMAGE_TYPE_3D
)
188 anv_meta_end_blit2d(cmd_buffer
, &saved_state
);
191 void anv_CmdCopyBufferToImage(
192 VkCommandBuffer commandBuffer
,
195 VkImageLayout destImageLayout
,
196 uint32_t regionCount
,
197 const VkBufferImageCopy
* pRegions
)
199 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
, commandBuffer
);
200 ANV_FROM_HANDLE(anv_image
, dest_image
, destImage
);
201 ANV_FROM_HANDLE(anv_buffer
, src_buffer
, srcBuffer
);
203 meta_copy_buffer_to_image(cmd_buffer
, src_buffer
, dest_image
,
204 regionCount
, pRegions
, true);
207 void anv_CmdCopyImageToBuffer(
208 VkCommandBuffer commandBuffer
,
210 VkImageLayout srcImageLayout
,
212 uint32_t regionCount
,
213 const VkBufferImageCopy
* pRegions
)
215 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
, commandBuffer
);
216 ANV_FROM_HANDLE(anv_image
, src_image
, srcImage
);
217 ANV_FROM_HANDLE(anv_buffer
, dst_buffer
, destBuffer
);
219 meta_copy_buffer_to_image(cmd_buffer
, dst_buffer
, src_image
,
220 regionCount
, pRegions
, false);
223 void anv_CmdCopyImage(
224 VkCommandBuffer commandBuffer
,
226 VkImageLayout srcImageLayout
,
228 VkImageLayout destImageLayout
,
229 uint32_t regionCount
,
230 const VkImageCopy
* pRegions
)
232 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
, commandBuffer
);
233 ANV_FROM_HANDLE(anv_image
, src_image
, srcImage
);
234 ANV_FROM_HANDLE(anv_image
, dest_image
, destImage
);
235 struct anv_meta_saved_state saved_state
;
237 /* From the Vulkan 1.0 spec:
239 * vkCmdCopyImage can be used to copy image data between multisample
240 * images, but both images must have the same number of samples.
242 assert(src_image
->samples
== dest_image
->samples
);
244 anv_meta_begin_blit2d(cmd_buffer
, &saved_state
);
246 for (unsigned r
= 0; r
< regionCount
; r
++) {
247 assert(pRegions
[r
].srcSubresource
.aspectMask
==
248 pRegions
[r
].dstSubresource
.aspectMask
);
250 VkImageAspectFlags aspect
= pRegions
[r
].srcSubresource
.aspectMask
;
252 /* Create blit surfaces */
253 struct isl_surf
*src_isl_surf
=
254 &anv_image_get_surface_for_aspect_mask(src_image
, aspect
)->isl
;
255 struct isl_surf
*dst_isl_surf
=
256 &anv_image_get_surface_for_aspect_mask(dest_image
, aspect
)->isl
;
257 struct anv_meta_blit2d_surf b_src
=
258 blit_surf_for_image(src_image
, src_isl_surf
);
259 struct anv_meta_blit2d_surf b_dst
=
260 blit_surf_for_image(dest_image
, dst_isl_surf
);
262 /* Start creating blit rect */
263 const VkOffset3D dst_offset_el
=
264 meta_region_offset_el(dest_image
, &pRegions
[r
].dstOffset
);
265 const VkOffset3D src_offset_el
=
266 meta_region_offset_el(src_image
, &pRegions
[r
].srcOffset
);
267 const VkExtent3D img_extent_el
=
268 meta_region_extent_el(src_image
->vk_format
, &pRegions
[r
].extent
);
269 struct anv_meta_blit2d_rect rect
= {
270 .width
= img_extent_el
.width
,
271 .height
= img_extent_el
.height
,
274 /* Loop through each 3D or array slice */
275 unsigned num_slices_3d
= pRegions
[r
].extent
.depth
;
276 unsigned num_slices_array
= pRegions
[r
].dstSubresource
.layerCount
;
277 unsigned slice_3d
= 0;
278 unsigned slice_array
= 0;
279 while (slice_3d
< num_slices_3d
&& slice_array
< num_slices_array
) {
281 /* Finish creating blit rect */
282 isl_surf_get_image_offset_el(dst_isl_surf
,
283 pRegions
[r
].dstSubresource
.mipLevel
,
284 pRegions
[r
].dstSubresource
.baseArrayLayer
286 pRegions
[r
].dstOffset
.z
+ slice_3d
,
289 isl_surf_get_image_offset_el(src_isl_surf
,
290 pRegions
[r
].srcSubresource
.mipLevel
,
291 pRegions
[r
].srcSubresource
.baseArrayLayer
293 pRegions
[r
].srcOffset
.z
+ slice_3d
,
296 rect
.dst_x
+= dst_offset_el
.x
;
297 rect
.dst_y
+= dst_offset_el
.y
;
298 rect
.src_x
+= src_offset_el
.x
;
299 rect
.src_y
+= src_offset_el
.y
;
302 anv_meta_blit2d(cmd_buffer
, &b_src
, &b_dst
, 1, &rect
);
304 if (dest_image
->type
== VK_IMAGE_TYPE_3D
)
311 anv_meta_end_blit2d(cmd_buffer
, &saved_state
);
314 void anv_CmdCopyBuffer(
315 VkCommandBuffer commandBuffer
,
318 uint32_t regionCount
,
319 const VkBufferCopy
* pRegions
)
321 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
, commandBuffer
);
322 ANV_FROM_HANDLE(anv_buffer
, src_buffer
, srcBuffer
);
323 ANV_FROM_HANDLE(anv_buffer
, dest_buffer
, destBuffer
);
325 struct anv_meta_saved_state saved_state
;
327 anv_meta_begin_blit2d(cmd_buffer
, &saved_state
);
329 for (unsigned r
= 0; r
< regionCount
; r
++) {
330 uint64_t src_offset
= src_buffer
->offset
+ pRegions
[r
].srcOffset
;
331 uint64_t dest_offset
= dest_buffer
->offset
+ pRegions
[r
].dstOffset
;
332 uint64_t copy_size
= pRegions
[r
].size
;
334 /* First, we compute the biggest format that can be used with the
335 * given offsets and size.
339 int fs
= ffs(src_offset
) - 1;
341 bs
= MIN2(bs
, 1 << fs
);
342 assert(src_offset
% bs
== 0);
344 fs
= ffs(dest_offset
) - 1;
346 bs
= MIN2(bs
, 1 << fs
);
347 assert(dest_offset
% bs
== 0);
349 fs
= ffs(pRegions
[r
].size
) - 1;
351 bs
= MIN2(bs
, 1 << fs
);
352 assert(pRegions
[r
].size
% bs
== 0);
354 /* This is maximum possible width/height our HW can handle */
355 uint64_t max_surface_dim
= 1 << 14;
357 /* First, we make a bunch of max-sized copies */
358 uint64_t max_copy_size
= max_surface_dim
* max_surface_dim
* bs
;
359 while (copy_size
>= max_copy_size
) {
360 do_buffer_copy(cmd_buffer
, src_buffer
->bo
, src_offset
,
361 dest_buffer
->bo
, dest_offset
,
362 max_surface_dim
, max_surface_dim
, bs
);
363 copy_size
-= max_copy_size
;
364 src_offset
+= max_copy_size
;
365 dest_offset
+= max_copy_size
;
368 uint64_t height
= copy_size
/ (max_surface_dim
* bs
);
369 assert(height
< max_surface_dim
);
371 uint64_t rect_copy_size
= height
* max_surface_dim
* bs
;
372 do_buffer_copy(cmd_buffer
, src_buffer
->bo
, src_offset
,
373 dest_buffer
->bo
, dest_offset
,
374 max_surface_dim
, height
, bs
);
375 copy_size
-= rect_copy_size
;
376 src_offset
+= rect_copy_size
;
377 dest_offset
+= rect_copy_size
;
380 if (copy_size
!= 0) {
381 do_buffer_copy(cmd_buffer
, src_buffer
->bo
, src_offset
,
382 dest_buffer
->bo
, dest_offset
,
383 copy_size
/ bs
, 1, bs
);
387 anv_meta_end_blit2d(cmd_buffer
, &saved_state
);
390 void anv_CmdUpdateBuffer(
391 VkCommandBuffer commandBuffer
,
393 VkDeviceSize dstOffset
,
394 VkDeviceSize dataSize
,
395 const uint32_t* pData
)
397 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
, commandBuffer
);
398 ANV_FROM_HANDLE(anv_buffer
, dst_buffer
, dstBuffer
);
399 struct anv_meta_saved_state saved_state
;
401 anv_meta_begin_blit2d(cmd_buffer
, &saved_state
);
403 /* We can't quite grab a full block because the state stream needs a
404 * little data at the top to build its linked list.
406 const uint32_t max_update_size
=
407 cmd_buffer
->device
->dynamic_state_block_pool
.block_size
- 64;
409 assert(max_update_size
< (1 << 14) * 4);
412 const uint32_t copy_size
= MIN2(dataSize
, max_update_size
);
414 struct anv_state tmp_data
=
415 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer
, copy_size
, 64);
417 memcpy(tmp_data
.map
, pData
, copy_size
);
420 if ((copy_size
& 15) == 0 && (dstOffset
& 15) == 0) {
422 } else if ((copy_size
& 7) == 0 && (dstOffset
& 7) == 0) {
425 assert((copy_size
& 3) == 0 && (dstOffset
& 3) == 0);
429 do_buffer_copy(cmd_buffer
,
430 &cmd_buffer
->device
->dynamic_state_block_pool
.bo
,
432 dst_buffer
->bo
, dst_buffer
->offset
+ dstOffset
,
433 copy_size
/ bs
, 1, bs
);
435 dataSize
-= copy_size
;
436 dstOffset
+= copy_size
;
437 pData
= (void *)pData
+ copy_size
;
440 anv_meta_end_blit2d(cmd_buffer
, &saved_state
);