2 * Copyright © 2016 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 #include "anv_private.h"
27 lookup_blorp_shader(struct blorp_context
*blorp
,
28 const void *key
, uint32_t key_size
,
29 uint32_t *kernel_out
, void *prog_data_out
)
31 struct anv_device
*device
= blorp
->driver_ctx
;
33 /* The blorp cache must be a real cache */
34 assert(device
->blorp_shader_cache
.cache
);
36 struct anv_shader_bin
*bin
=
37 anv_pipeline_cache_search(&device
->blorp_shader_cache
, key
, key_size
);
41 /* The cache already has a reference and it's not going anywhere so there
42 * is no need to hold a second reference.
44 anv_shader_bin_unref(device
, bin
);
46 *kernel_out
= bin
->kernel
.offset
;
47 *(const struct brw_stage_prog_data
**)prog_data_out
= bin
->prog_data
;
53 upload_blorp_shader(struct blorp_context
*blorp
,
54 const void *key
, uint32_t key_size
,
55 const void *kernel
, uint32_t kernel_size
,
56 const struct brw_stage_prog_data
*prog_data
,
57 uint32_t prog_data_size
,
58 uint32_t *kernel_out
, void *prog_data_out
)
60 struct anv_device
*device
= blorp
->driver_ctx
;
62 /* The blorp cache must be a real cache */
63 assert(device
->blorp_shader_cache
.cache
);
65 struct anv_pipeline_bind_map bind_map
= {
70 struct anv_shader_bin
*bin
=
71 anv_pipeline_cache_upload_kernel(&device
->blorp_shader_cache
,
72 key
, key_size
, kernel
, kernel_size
,
73 prog_data
, prog_data_size
, &bind_map
);
78 /* The cache already has a reference and it's not going anywhere so there
79 * is no need to hold a second reference.
81 anv_shader_bin_unref(device
, bin
);
83 *kernel_out
= bin
->kernel
.offset
;
84 *(const struct brw_stage_prog_data
**)prog_data_out
= bin
->prog_data
;
90 anv_device_init_blorp(struct anv_device
*device
)
92 anv_pipeline_cache_init(&device
->blorp_shader_cache
, device
, true);
93 blorp_init(&device
->blorp
, device
, &device
->isl_dev
);
94 device
->blorp
.compiler
= device
->instance
->physicalDevice
.compiler
;
95 device
->blorp
.mocs
.tex
= device
->default_mocs
;
96 device
->blorp
.mocs
.rb
= device
->default_mocs
;
97 device
->blorp
.mocs
.vb
= device
->default_mocs
;
98 device
->blorp
.lookup_shader
= lookup_blorp_shader
;
99 device
->blorp
.upload_shader
= upload_blorp_shader
;
100 switch (device
->info
.gen
) {
102 if (device
->info
.is_haswell
) {
103 device
->blorp
.exec
= gen75_blorp_exec
;
105 device
->blorp
.exec
= gen7_blorp_exec
;
109 device
->blorp
.exec
= gen8_blorp_exec
;
112 device
->blorp
.exec
= gen9_blorp_exec
;
115 device
->blorp
.exec
= gen10_blorp_exec
;
118 unreachable("Unknown hardware generation");
123 anv_device_finish_blorp(struct anv_device
*device
)
125 blorp_finish(&device
->blorp
);
126 anv_pipeline_cache_finish(&device
->blorp_shader_cache
);
130 get_blorp_surf_for_anv_buffer(struct anv_device
*device
,
131 struct anv_buffer
*buffer
, uint64_t offset
,
132 uint32_t width
, uint32_t height
,
133 uint32_t row_pitch
, enum isl_format format
,
134 struct blorp_surf
*blorp_surf
,
135 struct isl_surf
*isl_surf
)
137 const struct isl_format_layout
*fmtl
=
138 isl_format_get_layout(format
);
141 /* ASTC is the only format which doesn't support linear layouts.
142 * Create an equivalently sized surface with ISL to get around this.
144 if (fmtl
->txc
== ISL_TXC_ASTC
) {
145 /* Use an equivalently sized format */
146 format
= ISL_FORMAT_R32G32B32A32_UINT
;
147 assert(fmtl
->bpb
== isl_format_get_layout(format
)->bpb
);
149 /* Shrink the dimensions for the new format */
150 width
= DIV_ROUND_UP(width
, fmtl
->bw
);
151 height
= DIV_ROUND_UP(height
, fmtl
->bh
);
154 *blorp_surf
= (struct blorp_surf
) {
157 .buffer
= buffer
->bo
,
158 .offset
= buffer
->offset
+ offset
,
162 ok
= isl_surf_init(&device
->isl_dev
, isl_surf
,
163 .dim
= ISL_SURF_DIM_2D
,
171 .row_pitch
= row_pitch
,
172 .usage
= ISL_SURF_USAGE_TEXTURE_BIT
|
173 ISL_SURF_USAGE_RENDER_TARGET_BIT
,
174 .tiling_flags
= ISL_TILING_LINEAR_BIT
);
179 get_blorp_surf_for_anv_image(const struct anv_image
*image
,
180 VkImageAspectFlags aspect
,
181 enum isl_aux_usage aux_usage
,
182 struct blorp_surf
*blorp_surf
)
184 if (aspect
== VK_IMAGE_ASPECT_STENCIL_BIT
||
185 aux_usage
== ISL_AUX_USAGE_HIZ
)
186 aux_usage
= ISL_AUX_USAGE_NONE
;
188 const struct anv_surface
*surface
=
189 anv_image_get_surface_for_aspect_mask(image
, aspect
);
191 *blorp_surf
= (struct blorp_surf
) {
192 .surf
= &surface
->isl
,
195 .offset
= image
->offset
+ surface
->offset
,
199 if (aux_usage
!= ISL_AUX_USAGE_NONE
) {
200 blorp_surf
->aux_surf
= &image
->aux_surface
.isl
,
201 blorp_surf
->aux_addr
= (struct blorp_address
) {
203 .offset
= image
->offset
+ image
->aux_surface
.offset
,
205 blorp_surf
->aux_usage
= aux_usage
;
209 void anv_CmdCopyImage(
210 VkCommandBuffer commandBuffer
,
212 VkImageLayout srcImageLayout
,
214 VkImageLayout dstImageLayout
,
215 uint32_t regionCount
,
216 const VkImageCopy
* pRegions
)
218 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
, commandBuffer
);
219 ANV_FROM_HANDLE(anv_image
, src_image
, srcImage
);
220 ANV_FROM_HANDLE(anv_image
, dst_image
, dstImage
);
222 struct blorp_batch batch
;
223 blorp_batch_init(&cmd_buffer
->device
->blorp
, &batch
, cmd_buffer
, 0);
225 for (unsigned r
= 0; r
< regionCount
; r
++) {
226 VkOffset3D srcOffset
=
227 anv_sanitize_image_offset(src_image
->type
, pRegions
[r
].srcOffset
);
228 VkOffset3D dstOffset
=
229 anv_sanitize_image_offset(dst_image
->type
, pRegions
[r
].dstOffset
);
231 anv_sanitize_image_extent(src_image
->type
, pRegions
[r
].extent
);
233 unsigned dst_base_layer
, layer_count
;
234 if (dst_image
->type
== VK_IMAGE_TYPE_3D
) {
235 dst_base_layer
= pRegions
[r
].dstOffset
.z
;
236 layer_count
= pRegions
[r
].extent
.depth
;
238 dst_base_layer
= pRegions
[r
].dstSubresource
.baseArrayLayer
;
240 anv_get_layerCount(dst_image
, &pRegions
[r
].dstSubresource
);
243 unsigned src_base_layer
;
244 if (src_image
->type
== VK_IMAGE_TYPE_3D
) {
245 src_base_layer
= pRegions
[r
].srcOffset
.z
;
247 src_base_layer
= pRegions
[r
].srcSubresource
.baseArrayLayer
;
248 assert(layer_count
==
249 anv_get_layerCount(src_image
, &pRegions
[r
].srcSubresource
));
252 assert(pRegions
[r
].srcSubresource
.aspectMask
==
253 pRegions
[r
].dstSubresource
.aspectMask
);
256 for_each_bit(a
, pRegions
[r
].dstSubresource
.aspectMask
) {
257 VkImageAspectFlagBits aspect
= (1 << a
);
259 struct blorp_surf src_surf
, dst_surf
;
260 get_blorp_surf_for_anv_image(src_image
, aspect
, src_image
->aux_usage
,
262 get_blorp_surf_for_anv_image(dst_image
, aspect
, dst_image
->aux_usage
,
265 for (unsigned i
= 0; i
< layer_count
; i
++) {
266 blorp_copy(&batch
, &src_surf
, pRegions
[r
].srcSubresource
.mipLevel
,
268 &dst_surf
, pRegions
[r
].dstSubresource
.mipLevel
,
270 srcOffset
.x
, srcOffset
.y
,
271 dstOffset
.x
, dstOffset
.y
,
272 extent
.width
, extent
.height
);
277 blorp_batch_finish(&batch
);
281 copy_buffer_to_image(struct anv_cmd_buffer
*cmd_buffer
,
282 struct anv_buffer
*anv_buffer
,
283 struct anv_image
*anv_image
,
284 uint32_t regionCount
,
285 const VkBufferImageCopy
* pRegions
,
286 bool buffer_to_image
)
288 struct blorp_batch batch
;
289 blorp_batch_init(&cmd_buffer
->device
->blorp
, &batch
, cmd_buffer
, 0);
292 struct blorp_surf surf
;
295 } image
, buffer
, *src
, *dst
;
298 buffer
.offset
= (VkOffset3D
) { 0, 0, 0 };
300 if (buffer_to_image
) {
308 for (unsigned r
= 0; r
< regionCount
; r
++) {
309 const VkImageAspectFlags aspect
= pRegions
[r
].imageSubresource
.aspectMask
;
311 get_blorp_surf_for_anv_image(anv_image
, aspect
, anv_image
->aux_usage
,
314 anv_sanitize_image_offset(anv_image
->type
, pRegions
[r
].imageOffset
);
315 image
.level
= pRegions
[r
].imageSubresource
.mipLevel
;
318 anv_sanitize_image_extent(anv_image
->type
, pRegions
[r
].imageExtent
);
319 if (anv_image
->type
!= VK_IMAGE_TYPE_3D
) {
320 image
.offset
.z
= pRegions
[r
].imageSubresource
.baseArrayLayer
;
322 anv_get_layerCount(anv_image
, &pRegions
[r
].imageSubresource
);
325 const enum isl_format buffer_format
=
326 anv_get_isl_format(&cmd_buffer
->device
->info
, anv_image
->vk_format
,
327 aspect
, VK_IMAGE_TILING_LINEAR
);
329 const VkExtent3D bufferImageExtent
= {
330 .width
= pRegions
[r
].bufferRowLength
?
331 pRegions
[r
].bufferRowLength
: extent
.width
,
332 .height
= pRegions
[r
].bufferImageHeight
?
333 pRegions
[r
].bufferImageHeight
: extent
.height
,
336 const struct isl_format_layout
*buffer_fmtl
=
337 isl_format_get_layout(buffer_format
);
339 const uint32_t buffer_row_pitch
=
340 DIV_ROUND_UP(bufferImageExtent
.width
, buffer_fmtl
->bw
) *
341 (buffer_fmtl
->bpb
/ 8);
343 const uint32_t buffer_layer_stride
=
344 DIV_ROUND_UP(bufferImageExtent
.height
, buffer_fmtl
->bh
) *
347 struct isl_surf buffer_isl_surf
;
348 get_blorp_surf_for_anv_buffer(cmd_buffer
->device
,
349 anv_buffer
, pRegions
[r
].bufferOffset
,
350 extent
.width
, extent
.height
,
351 buffer_row_pitch
, buffer_format
,
352 &buffer
.surf
, &buffer_isl_surf
);
354 for (unsigned z
= 0; z
< extent
.depth
; z
++) {
355 blorp_copy(&batch
, &src
->surf
, src
->level
, src
->offset
.z
,
356 &dst
->surf
, dst
->level
, dst
->offset
.z
,
357 src
->offset
.x
, src
->offset
.y
, dst
->offset
.x
, dst
->offset
.y
,
358 extent
.width
, extent
.height
);
361 buffer
.surf
.addr
.offset
+= buffer_layer_stride
;
365 blorp_batch_finish(&batch
);
368 void anv_CmdCopyBufferToImage(
369 VkCommandBuffer commandBuffer
,
372 VkImageLayout dstImageLayout
,
373 uint32_t regionCount
,
374 const VkBufferImageCopy
* pRegions
)
376 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
, commandBuffer
);
377 ANV_FROM_HANDLE(anv_buffer
, src_buffer
, srcBuffer
);
378 ANV_FROM_HANDLE(anv_image
, dst_image
, dstImage
);
380 copy_buffer_to_image(cmd_buffer
, src_buffer
, dst_image
,
381 regionCount
, pRegions
, true);
384 void anv_CmdCopyImageToBuffer(
385 VkCommandBuffer commandBuffer
,
387 VkImageLayout srcImageLayout
,
389 uint32_t regionCount
,
390 const VkBufferImageCopy
* pRegions
)
392 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
, commandBuffer
);
393 ANV_FROM_HANDLE(anv_image
, src_image
, srcImage
);
394 ANV_FROM_HANDLE(anv_buffer
, dst_buffer
, dstBuffer
);
396 copy_buffer_to_image(cmd_buffer
, dst_buffer
, src_image
,
397 regionCount
, pRegions
, false);
401 flip_coords(unsigned *src0
, unsigned *src1
, unsigned *dst0
, unsigned *dst1
)
405 unsigned tmp
= *src0
;
412 unsigned tmp
= *dst0
;
421 void anv_CmdBlitImage(
422 VkCommandBuffer commandBuffer
,
424 VkImageLayout srcImageLayout
,
426 VkImageLayout dstImageLayout
,
427 uint32_t regionCount
,
428 const VkImageBlit
* pRegions
,
432 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
, commandBuffer
);
433 ANV_FROM_HANDLE(anv_image
, src_image
, srcImage
);
434 ANV_FROM_HANDLE(anv_image
, dst_image
, dstImage
);
436 struct blorp_surf src
, dst
;
440 case VK_FILTER_NEAREST
:
441 gl_filter
= 0x2600; /* GL_NEAREST */
443 case VK_FILTER_LINEAR
:
444 gl_filter
= 0x2601; /* GL_LINEAR */
447 unreachable("Invalid filter");
450 struct blorp_batch batch
;
451 blorp_batch_init(&cmd_buffer
->device
->blorp
, &batch
, cmd_buffer
, 0);
453 for (unsigned r
= 0; r
< regionCount
; r
++) {
454 const VkImageSubresourceLayers
*src_res
= &pRegions
[r
].srcSubresource
;
455 const VkImageSubresourceLayers
*dst_res
= &pRegions
[r
].dstSubresource
;
457 get_blorp_surf_for_anv_image(src_image
, src_res
->aspectMask
,
458 src_image
->aux_usage
, &src
);
459 get_blorp_surf_for_anv_image(dst_image
, dst_res
->aspectMask
,
460 dst_image
->aux_usage
, &dst
);
462 struct anv_format src_format
=
463 anv_get_format(&cmd_buffer
->device
->info
, src_image
->vk_format
,
464 src_res
->aspectMask
, src_image
->tiling
);
465 struct anv_format dst_format
=
466 anv_get_format(&cmd_buffer
->device
->info
, dst_image
->vk_format
,
467 dst_res
->aspectMask
, dst_image
->tiling
);
469 unsigned dst_start
, dst_end
;
470 if (dst_image
->type
== VK_IMAGE_TYPE_3D
) {
471 assert(dst_res
->baseArrayLayer
== 0);
472 dst_start
= pRegions
[r
].dstOffsets
[0].z
;
473 dst_end
= pRegions
[r
].dstOffsets
[1].z
;
475 dst_start
= dst_res
->baseArrayLayer
;
476 dst_end
= dst_start
+ anv_get_layerCount(dst_image
, dst_res
);
479 unsigned src_start
, src_end
;
480 if (src_image
->type
== VK_IMAGE_TYPE_3D
) {
481 assert(src_res
->baseArrayLayer
== 0);
482 src_start
= pRegions
[r
].srcOffsets
[0].z
;
483 src_end
= pRegions
[r
].srcOffsets
[1].z
;
485 src_start
= src_res
->baseArrayLayer
;
486 src_end
= src_start
+ anv_get_layerCount(src_image
, src_res
);
489 bool flip_z
= flip_coords(&src_start
, &src_end
, &dst_start
, &dst_end
);
490 float src_z_step
= (float)(src_end
+ 1 - src_start
) /
491 (float)(dst_end
+ 1 - dst_start
);
498 unsigned src_x0
= pRegions
[r
].srcOffsets
[0].x
;
499 unsigned src_x1
= pRegions
[r
].srcOffsets
[1].x
;
500 unsigned dst_x0
= pRegions
[r
].dstOffsets
[0].x
;
501 unsigned dst_x1
= pRegions
[r
].dstOffsets
[1].x
;
502 bool flip_x
= flip_coords(&src_x0
, &src_x1
, &dst_x0
, &dst_x1
);
504 unsigned src_y0
= pRegions
[r
].srcOffsets
[0].y
;
505 unsigned src_y1
= pRegions
[r
].srcOffsets
[1].y
;
506 unsigned dst_y0
= pRegions
[r
].dstOffsets
[0].y
;
507 unsigned dst_y1
= pRegions
[r
].dstOffsets
[1].y
;
508 bool flip_y
= flip_coords(&src_y0
, &src_y1
, &dst_y0
, &dst_y1
);
510 const unsigned num_layers
= dst_end
- dst_start
;
511 for (unsigned i
= 0; i
< num_layers
; i
++) {
512 unsigned dst_z
= dst_start
+ i
;
513 unsigned src_z
= src_start
+ i
* src_z_step
;
515 blorp_blit(&batch
, &src
, src_res
->mipLevel
, src_z
,
516 src_format
.isl_format
, src_format
.swizzle
,
517 &dst
, dst_res
->mipLevel
, dst_z
,
518 dst_format
.isl_format
,
519 anv_swizzle_for_render(dst_format
.swizzle
),
520 src_x0
, src_y0
, src_x1
, src_y1
,
521 dst_x0
, dst_y0
, dst_x1
, dst_y1
,
522 gl_filter
, flip_x
, flip_y
);
527 blorp_batch_finish(&batch
);
530 static enum isl_format
531 isl_format_for_size(unsigned size_B
)
534 case 4: return ISL_FORMAT_R32_UINT
;
535 case 8: return ISL_FORMAT_R32G32_UINT
;
536 case 16: return ISL_FORMAT_R32G32B32A32_UINT
;
538 unreachable("Not a power-of-two format size");
543 * Returns the greatest common divisor of a and b that is a power of two.
546 gcd_pow2_u64(uint64_t a
, uint64_t b
)
548 assert(a
> 0 || b
> 0);
550 unsigned a_log2
= ffsll(a
) - 1;
551 unsigned b_log2
= ffsll(b
) - 1;
553 /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which
554 * case, the MIN2() will take the other one. If both are 0 then we will
555 * hit the assert above.
557 return 1 << MIN2(a_log2
, b_log2
);
560 /* This is maximum possible width/height our HW can handle */
561 #define MAX_SURFACE_DIM (1ull << 14)
563 void anv_CmdCopyBuffer(
564 VkCommandBuffer commandBuffer
,
567 uint32_t regionCount
,
568 const VkBufferCopy
* pRegions
)
570 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
, commandBuffer
);
571 ANV_FROM_HANDLE(anv_buffer
, src_buffer
, srcBuffer
);
572 ANV_FROM_HANDLE(anv_buffer
, dst_buffer
, dstBuffer
);
574 struct blorp_batch batch
;
575 blorp_batch_init(&cmd_buffer
->device
->blorp
, &batch
, cmd_buffer
, 0);
577 for (unsigned r
= 0; r
< regionCount
; r
++) {
578 struct blorp_address src
= {
579 .buffer
= src_buffer
->bo
,
580 .offset
= src_buffer
->offset
+ pRegions
[r
].srcOffset
,
582 struct blorp_address dst
= {
583 .buffer
= dst_buffer
->bo
,
584 .offset
= dst_buffer
->offset
+ pRegions
[r
].dstOffset
,
587 blorp_buffer_copy(&batch
, src
, dst
, pRegions
[r
].size
);
590 blorp_batch_finish(&batch
);
593 void anv_CmdUpdateBuffer(
594 VkCommandBuffer commandBuffer
,
596 VkDeviceSize dstOffset
,
597 VkDeviceSize dataSize
,
600 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
, commandBuffer
);
601 ANV_FROM_HANDLE(anv_buffer
, dst_buffer
, dstBuffer
);
603 struct blorp_batch batch
;
604 blorp_batch_init(&cmd_buffer
->device
->blorp
, &batch
, cmd_buffer
, 0);
606 /* We can't quite grab a full block because the state stream needs a
607 * little data at the top to build its linked list.
609 const uint32_t max_update_size
=
610 cmd_buffer
->device
->dynamic_state_pool
.block_size
- 64;
612 assert(max_update_size
< MAX_SURFACE_DIM
* 4);
614 /* We're about to read data that was written from the CPU. Flush the
615 * texture cache so we don't get anything stale.
617 cmd_buffer
->state
.pending_pipe_bits
|= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT
;
620 const uint32_t copy_size
= MIN2(dataSize
, max_update_size
);
622 struct anv_state tmp_data
=
623 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer
, copy_size
, 64);
625 memcpy(tmp_data
.map
, pData
, copy_size
);
627 anv_state_flush(cmd_buffer
->device
, tmp_data
);
629 struct blorp_address src
= {
630 .buffer
= &cmd_buffer
->device
->dynamic_state_pool
.block_pool
.bo
,
631 .offset
= tmp_data
.offset
,
633 struct blorp_address dst
= {
634 .buffer
= dst_buffer
->bo
,
635 .offset
= dst_buffer
->offset
+ dstOffset
,
638 blorp_buffer_copy(&batch
, src
, dst
, copy_size
);
640 dataSize
-= copy_size
;
641 dstOffset
+= copy_size
;
642 pData
= (void *)pData
+ copy_size
;
645 blorp_batch_finish(&batch
);
648 void anv_CmdFillBuffer(
649 VkCommandBuffer commandBuffer
,
651 VkDeviceSize dstOffset
,
652 VkDeviceSize fillSize
,
655 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
, commandBuffer
);
656 ANV_FROM_HANDLE(anv_buffer
, dst_buffer
, dstBuffer
);
657 struct blorp_surf surf
;
658 struct isl_surf isl_surf
;
660 struct blorp_batch batch
;
661 blorp_batch_init(&cmd_buffer
->device
->blorp
, &batch
, cmd_buffer
, 0);
663 fillSize
= anv_buffer_get_range(dst_buffer
, dstOffset
, fillSize
);
665 /* From the Vulkan spec:
667 * "size is the number of bytes to fill, and must be either a multiple
668 * of 4, or VK_WHOLE_SIZE to fill the range from offset to the end of
669 * the buffer. If VK_WHOLE_SIZE is used and the remaining size of the
670 * buffer is not a multiple of 4, then the nearest smaller multiple is
675 /* First, we compute the biggest format that can be used with the
676 * given offsets and size.
679 bs
= gcd_pow2_u64(bs
, dstOffset
);
680 bs
= gcd_pow2_u64(bs
, fillSize
);
681 enum isl_format isl_format
= isl_format_for_size(bs
);
683 union isl_color_value color
= {
684 .u32
= { data
, data
, data
, data
},
687 const uint64_t max_fill_size
= MAX_SURFACE_DIM
* MAX_SURFACE_DIM
* bs
;
688 while (fillSize
>= max_fill_size
) {
689 get_blorp_surf_for_anv_buffer(cmd_buffer
->device
,
690 dst_buffer
, dstOffset
,
691 MAX_SURFACE_DIM
, MAX_SURFACE_DIM
,
692 MAX_SURFACE_DIM
* bs
, isl_format
,
695 blorp_clear(&batch
, &surf
, isl_format
, ISL_SWIZZLE_IDENTITY
,
696 0, 0, 1, 0, 0, MAX_SURFACE_DIM
, MAX_SURFACE_DIM
,
698 fillSize
-= max_fill_size
;
699 dstOffset
+= max_fill_size
;
702 uint64_t height
= fillSize
/ (MAX_SURFACE_DIM
* bs
);
703 assert(height
< MAX_SURFACE_DIM
);
705 const uint64_t rect_fill_size
= height
* MAX_SURFACE_DIM
* bs
;
706 get_blorp_surf_for_anv_buffer(cmd_buffer
->device
,
707 dst_buffer
, dstOffset
,
708 MAX_SURFACE_DIM
, height
,
709 MAX_SURFACE_DIM
* bs
, isl_format
,
712 blorp_clear(&batch
, &surf
, isl_format
, ISL_SWIZZLE_IDENTITY
,
713 0, 0, 1, 0, 0, MAX_SURFACE_DIM
, height
,
715 fillSize
-= rect_fill_size
;
716 dstOffset
+= rect_fill_size
;
720 const uint32_t width
= fillSize
/ bs
;
721 get_blorp_surf_for_anv_buffer(cmd_buffer
->device
,
722 dst_buffer
, dstOffset
,
724 width
* bs
, isl_format
,
727 blorp_clear(&batch
, &surf
, isl_format
, ISL_SWIZZLE_IDENTITY
,
728 0, 0, 1, 0, 0, width
, 1,
732 blorp_batch_finish(&batch
);
735 void anv_CmdClearColorImage(
736 VkCommandBuffer commandBuffer
,
738 VkImageLayout imageLayout
,
739 const VkClearColorValue
* pColor
,
741 const VkImageSubresourceRange
* pRanges
)
743 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
, commandBuffer
);
744 ANV_FROM_HANDLE(anv_image
, image
, _image
);
746 static const bool color_write_disable
[4] = { false, false, false, false };
748 struct blorp_batch batch
;
749 blorp_batch_init(&cmd_buffer
->device
->blorp
, &batch
, cmd_buffer
, 0);
751 struct blorp_surf surf
;
752 get_blorp_surf_for_anv_image(image
, VK_IMAGE_ASPECT_COLOR_BIT
,
753 image
->aux_usage
, &surf
);
755 for (unsigned r
= 0; r
< rangeCount
; r
++) {
756 if (pRanges
[r
].aspectMask
== 0)
759 assert(pRanges
[r
].aspectMask
== VK_IMAGE_ASPECT_COLOR_BIT
);
761 struct anv_format src_format
=
762 anv_get_format(&cmd_buffer
->device
->info
, image
->vk_format
,
763 VK_IMAGE_ASPECT_COLOR_BIT
, image
->tiling
);
765 unsigned base_layer
= pRanges
[r
].baseArrayLayer
;
766 unsigned layer_count
= anv_get_layerCount(image
, &pRanges
[r
]);
768 for (unsigned i
= 0; i
< anv_get_levelCount(image
, &pRanges
[r
]); i
++) {
769 const unsigned level
= pRanges
[r
].baseMipLevel
+ i
;
770 const unsigned level_width
= anv_minify(image
->extent
.width
, level
);
771 const unsigned level_height
= anv_minify(image
->extent
.height
, level
);
773 if (image
->type
== VK_IMAGE_TYPE_3D
) {
775 layer_count
= anv_minify(image
->extent
.depth
, level
);
778 blorp_clear(&batch
, &surf
,
779 src_format
.isl_format
, src_format
.swizzle
,
780 level
, base_layer
, layer_count
,
781 0, 0, level_width
, level_height
,
782 vk_to_isl_color(*pColor
), color_write_disable
);
786 blorp_batch_finish(&batch
);
789 void anv_CmdClearDepthStencilImage(
790 VkCommandBuffer commandBuffer
,
792 VkImageLayout imageLayout
,
793 const VkClearDepthStencilValue
* pDepthStencil
,
795 const VkImageSubresourceRange
* pRanges
)
797 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
, commandBuffer
);
798 ANV_FROM_HANDLE(anv_image
, image
, image_h
);
800 struct blorp_batch batch
;
801 blorp_batch_init(&cmd_buffer
->device
->blorp
, &batch
, cmd_buffer
, 0);
803 struct blorp_surf depth
, stencil
;
804 if (image
->aspects
& VK_IMAGE_ASPECT_DEPTH_BIT
) {
805 get_blorp_surf_for_anv_image(image
, VK_IMAGE_ASPECT_DEPTH_BIT
,
806 ISL_AUX_USAGE_NONE
, &depth
);
808 memset(&depth
, 0, sizeof(depth
));
811 if (image
->aspects
& VK_IMAGE_ASPECT_STENCIL_BIT
) {
812 get_blorp_surf_for_anv_image(image
, VK_IMAGE_ASPECT_STENCIL_BIT
,
813 ISL_AUX_USAGE_NONE
, &stencil
);
815 memset(&stencil
, 0, sizeof(stencil
));
818 for (unsigned r
= 0; r
< rangeCount
; r
++) {
819 if (pRanges
[r
].aspectMask
== 0)
822 bool clear_depth
= pRanges
[r
].aspectMask
& VK_IMAGE_ASPECT_DEPTH_BIT
;
823 bool clear_stencil
= pRanges
[r
].aspectMask
& VK_IMAGE_ASPECT_STENCIL_BIT
;
825 unsigned base_layer
= pRanges
[r
].baseArrayLayer
;
826 unsigned layer_count
= anv_get_layerCount(image
, &pRanges
[r
]);
828 for (unsigned i
= 0; i
< anv_get_levelCount(image
, &pRanges
[r
]); i
++) {
829 const unsigned level
= pRanges
[r
].baseMipLevel
+ i
;
830 const unsigned level_width
= anv_minify(image
->extent
.width
, level
);
831 const unsigned level_height
= anv_minify(image
->extent
.height
, level
);
833 if (image
->type
== VK_IMAGE_TYPE_3D
)
834 layer_count
= anv_minify(image
->extent
.depth
, level
);
836 blorp_clear_depth_stencil(&batch
, &depth
, &stencil
,
837 level
, base_layer
, layer_count
,
838 0, 0, level_width
, level_height
,
839 clear_depth
, pDepthStencil
->depth
,
840 clear_stencil
? 0xff : 0,
841 pDepthStencil
->stencil
);
845 blorp_batch_finish(&batch
);
849 anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer
*cmd_buffer
,
850 uint32_t num_entries
,
851 uint32_t *state_offset
,
852 struct anv_state
*bt_state
)
854 *bt_state
= anv_cmd_buffer_alloc_binding_table(cmd_buffer
, num_entries
,
856 if (bt_state
->map
== NULL
) {
857 /* We ran out of space. Grab a new binding table block. */
858 VkResult result
= anv_cmd_buffer_new_binding_table_block(cmd_buffer
);
859 if (result
!= VK_SUCCESS
)
862 /* Re-emit state base addresses so we get the new surface state base
863 * address before we start emitting binding tables etc.
865 anv_cmd_buffer_emit_state_base_address(cmd_buffer
);
867 *bt_state
= anv_cmd_buffer_alloc_binding_table(cmd_buffer
, num_entries
,
869 assert(bt_state
->map
!= NULL
);
876 binding_table_for_surface_state(struct anv_cmd_buffer
*cmd_buffer
,
877 struct anv_state surface_state
,
880 uint32_t state_offset
;
881 struct anv_state bt_state
;
884 anv_cmd_buffer_alloc_blorp_binding_table(cmd_buffer
, 1, &state_offset
,
886 if (result
!= VK_SUCCESS
)
889 uint32_t *bt_map
= bt_state
.map
;
890 bt_map
[0] = surface_state
.offset
+ state_offset
;
892 *bt_offset
= bt_state
.offset
;
897 clear_color_attachment(struct anv_cmd_buffer
*cmd_buffer
,
898 struct blorp_batch
*batch
,
899 const VkClearAttachment
*attachment
,
900 uint32_t rectCount
, const VkClearRect
*pRects
)
902 const struct anv_subpass
*subpass
= cmd_buffer
->state
.subpass
;
903 const uint32_t color_att
= attachment
->colorAttachment
;
904 const uint32_t att_idx
= subpass
->color_attachments
[color_att
].attachment
;
906 if (att_idx
== VK_ATTACHMENT_UNUSED
)
909 struct anv_render_pass_attachment
*pass_att
=
910 &cmd_buffer
->state
.pass
->attachments
[att_idx
];
911 struct anv_attachment_state
*att_state
=
912 &cmd_buffer
->state
.attachments
[att_idx
];
914 uint32_t binding_table
;
916 binding_table_for_surface_state(cmd_buffer
, att_state
->color
.state
,
918 if (result
!= VK_SUCCESS
)
921 union isl_color_value clear_color
=
922 vk_to_isl_color(attachment
->clearValue
.color
);
924 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
925 if (subpass
->view_mask
) {
927 for_each_bit(view_idx
, subpass
->view_mask
) {
928 for (uint32_t r
= 0; r
< rectCount
; ++r
) {
929 const VkOffset2D offset
= pRects
[r
].rect
.offset
;
930 const VkExtent2D extent
= pRects
[r
].rect
.extent
;
931 blorp_clear_attachments(batch
, binding_table
,
932 ISL_FORMAT_UNSUPPORTED
, pass_att
->samples
,
935 offset
.x
+ extent
.width
,
936 offset
.y
+ extent
.height
,
937 true, clear_color
, false, 0.0f
, 0, 0);
943 for (uint32_t r
= 0; r
< rectCount
; ++r
) {
944 const VkOffset2D offset
= pRects
[r
].rect
.offset
;
945 const VkExtent2D extent
= pRects
[r
].rect
.extent
;
946 blorp_clear_attachments(batch
, binding_table
,
947 ISL_FORMAT_UNSUPPORTED
, pass_att
->samples
,
948 pRects
[r
].baseArrayLayer
,
949 pRects
[r
].layerCount
,
951 offset
.x
+ extent
.width
, offset
.y
+ extent
.height
,
952 true, clear_color
, false, 0.0f
, 0, 0);
957 clear_depth_stencil_attachment(struct anv_cmd_buffer
*cmd_buffer
,
958 struct blorp_batch
*batch
,
959 const VkClearAttachment
*attachment
,
960 uint32_t rectCount
, const VkClearRect
*pRects
)
962 static const union isl_color_value color_value
= { .u32
= { 0, } };
963 const struct anv_subpass
*subpass
= cmd_buffer
->state
.subpass
;
964 const uint32_t att_idx
= subpass
->depth_stencil_attachment
.attachment
;
966 if (att_idx
== VK_ATTACHMENT_UNUSED
)
969 struct anv_render_pass_attachment
*pass_att
=
970 &cmd_buffer
->state
.pass
->attachments
[att_idx
];
972 bool clear_depth
= attachment
->aspectMask
& VK_IMAGE_ASPECT_DEPTH_BIT
;
973 bool clear_stencil
= attachment
->aspectMask
& VK_IMAGE_ASPECT_STENCIL_BIT
;
975 enum isl_format depth_format
= ISL_FORMAT_UNSUPPORTED
;
977 depth_format
= anv_get_isl_format(&cmd_buffer
->device
->info
,
979 VK_IMAGE_ASPECT_DEPTH_BIT
,
980 VK_IMAGE_TILING_OPTIMAL
);
983 uint32_t binding_table
;
985 binding_table_for_surface_state(cmd_buffer
,
986 cmd_buffer
->state
.null_surface_state
,
988 if (result
!= VK_SUCCESS
)
991 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
992 if (subpass
->view_mask
) {
994 for_each_bit(view_idx
, subpass
->view_mask
) {
995 for (uint32_t r
= 0; r
< rectCount
; ++r
) {
996 const VkOffset2D offset
= pRects
[r
].rect
.offset
;
997 const VkExtent2D extent
= pRects
[r
].rect
.extent
;
998 VkClearDepthStencilValue value
= attachment
->clearValue
.depthStencil
;
999 blorp_clear_attachments(batch
, binding_table
,
1000 depth_format
, pass_att
->samples
,
1003 offset
.x
+ extent
.width
,
1004 offset
.y
+ extent
.height
,
1006 clear_depth
, value
.depth
,
1007 clear_stencil
? 0xff : 0, value
.stencil
);
1013 for (uint32_t r
= 0; r
< rectCount
; ++r
) {
1014 const VkOffset2D offset
= pRects
[r
].rect
.offset
;
1015 const VkExtent2D extent
= pRects
[r
].rect
.extent
;
1016 VkClearDepthStencilValue value
= attachment
->clearValue
.depthStencil
;
1017 blorp_clear_attachments(batch
, binding_table
,
1018 depth_format
, pass_att
->samples
,
1019 pRects
[r
].baseArrayLayer
,
1020 pRects
[r
].layerCount
,
1022 offset
.x
+ extent
.width
, offset
.y
+ extent
.height
,
1024 clear_depth
, value
.depth
,
1025 clear_stencil
? 0xff : 0, value
.stencil
);
1029 void anv_CmdClearAttachments(
1030 VkCommandBuffer commandBuffer
,
1031 uint32_t attachmentCount
,
1032 const VkClearAttachment
* pAttachments
,
1034 const VkClearRect
* pRects
)
1036 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
, commandBuffer
);
1038 /* Because this gets called within a render pass, we tell blorp not to
1039 * trash our depth and stencil buffers.
1041 struct blorp_batch batch
;
1042 blorp_batch_init(&cmd_buffer
->device
->blorp
, &batch
, cmd_buffer
,
1043 BLORP_BATCH_NO_EMIT_DEPTH_STENCIL
);
1045 for (uint32_t a
= 0; a
< attachmentCount
; ++a
) {
1046 if (pAttachments
[a
].aspectMask
== VK_IMAGE_ASPECT_COLOR_BIT
) {
1047 clear_color_attachment(cmd_buffer
, &batch
,
1051 clear_depth_stencil_attachment(cmd_buffer
, &batch
,
1057 blorp_batch_finish(&batch
);
1060 enum subpass_stage
{
1063 SUBPASS_STAGE_RESOLVE
,
1067 subpass_needs_clear(const struct anv_cmd_buffer
*cmd_buffer
)
1069 const struct anv_cmd_state
*cmd_state
= &cmd_buffer
->state
;
1070 uint32_t ds
= cmd_state
->subpass
->depth_stencil_attachment
.attachment
;
1072 for (uint32_t i
= 0; i
< cmd_state
->subpass
->color_count
; ++i
) {
1073 uint32_t a
= cmd_state
->subpass
->color_attachments
[i
].attachment
;
1074 if (a
== VK_ATTACHMENT_UNUSED
)
1077 assert(a
< cmd_state
->pass
->attachment_count
);
1078 if (cmd_state
->attachments
[a
].pending_clear_aspects
) {
1083 if (ds
!= VK_ATTACHMENT_UNUSED
) {
1084 assert(ds
< cmd_state
->pass
->attachment_count
);
1085 if (cmd_state
->attachments
[ds
].pending_clear_aspects
)
1093 anv_cmd_buffer_clear_subpass(struct anv_cmd_buffer
*cmd_buffer
)
1095 const struct anv_cmd_state
*cmd_state
= &cmd_buffer
->state
;
1096 const VkRect2D render_area
= cmd_buffer
->state
.render_area
;
1099 if (!subpass_needs_clear(cmd_buffer
))
1102 /* Because this gets called within a render pass, we tell blorp not to
1103 * trash our depth and stencil buffers.
1105 struct blorp_batch batch
;
1106 blorp_batch_init(&cmd_buffer
->device
->blorp
, &batch
, cmd_buffer
,
1107 BLORP_BATCH_NO_EMIT_DEPTH_STENCIL
);
1109 VkClearRect clear_rect
= {
1110 .rect
= cmd_buffer
->state
.render_area
,
1111 .baseArrayLayer
= 0,
1112 .layerCount
= cmd_buffer
->state
.framebuffer
->layers
,
1115 struct anv_framebuffer
*fb
= cmd_buffer
->state
.framebuffer
;
1116 for (uint32_t i
= 0; i
< cmd_state
->subpass
->color_count
; ++i
) {
1117 const uint32_t a
= cmd_state
->subpass
->color_attachments
[i
].attachment
;
1118 if (a
== VK_ATTACHMENT_UNUSED
)
1121 assert(a
< cmd_state
->pass
->attachment_count
);
1122 struct anv_attachment_state
*att_state
= &cmd_state
->attachments
[a
];
1124 if (!att_state
->pending_clear_aspects
)
1127 assert(att_state
->pending_clear_aspects
== VK_IMAGE_ASPECT_COLOR_BIT
);
1129 struct anv_image_view
*iview
= fb
->attachments
[a
];
1130 const struct anv_image
*image
= iview
->image
;
1131 struct blorp_surf surf
;
1132 get_blorp_surf_for_anv_image(image
, VK_IMAGE_ASPECT_COLOR_BIT
,
1133 att_state
->aux_usage
, &surf
);
1135 if (att_state
->fast_clear
) {
1136 surf
.clear_color
= vk_to_isl_color(att_state
->clear_value
.color
);
1138 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1140 * "After Render target fast clear, pipe-control with color cache
1141 * write-flush must be issued before sending any DRAW commands on
1142 * that render target."
1144 * This comment is a bit cryptic and doesn't really tell you what's
1145 * going or what's really needed. It appears that fast clear ops are
1146 * not properly synchronized with other drawing. This means that we
1147 * cannot have a fast clear operation in the pipe at the same time as
1148 * other regular drawing operations. We need to use a PIPE_CONTROL
1149 * to ensure that the contents of the previous draw hit the render
1150 * target before we resolve and then use a second PIPE_CONTROL after
1151 * the resolve to ensure that it is completed before any additional
1154 cmd_buffer
->state
.pending_pipe_bits
|=
1155 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT
| ANV_PIPE_CS_STALL_BIT
;
1157 blorp_fast_clear(&batch
, &surf
, iview
->isl
.format
,
1158 iview
->isl
.base_level
,
1159 iview
->isl
.base_array_layer
, fb
->layers
,
1160 render_area
.offset
.x
, render_area
.offset
.y
,
1161 render_area
.offset
.x
+ render_area
.extent
.width
,
1162 render_area
.offset
.y
+ render_area
.extent
.height
);
1164 cmd_buffer
->state
.pending_pipe_bits
|=
1165 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT
| ANV_PIPE_CS_STALL_BIT
;
1167 blorp_clear(&batch
, &surf
, iview
->isl
.format
,
1168 anv_swizzle_for_render(iview
->isl
.swizzle
),
1169 iview
->isl
.base_level
,
1170 iview
->isl
.base_array_layer
, fb
->layers
,
1171 render_area
.offset
.x
, render_area
.offset
.y
,
1172 render_area
.offset
.x
+ render_area
.extent
.width
,
1173 render_area
.offset
.y
+ render_area
.extent
.height
,
1174 vk_to_isl_color(att_state
->clear_value
.color
), NULL
);
1177 att_state
->pending_clear_aspects
= 0;
1180 const uint32_t ds
= cmd_state
->subpass
->depth_stencil_attachment
.attachment
;
1181 assert(ds
== VK_ATTACHMENT_UNUSED
|| ds
< cmd_state
->pass
->attachment_count
);
1183 if (ds
!= VK_ATTACHMENT_UNUSED
&&
1184 cmd_state
->attachments
[ds
].pending_clear_aspects
) {
1186 VkClearAttachment clear_att
= {
1187 .aspectMask
= cmd_state
->attachments
[ds
].pending_clear_aspects
,
1188 .clearValue
= cmd_state
->attachments
[ds
].clear_value
,
1192 const uint8_t gen
= cmd_buffer
->device
->info
.gen
;
1193 bool clear_with_hiz
= gen
>= 8 && cmd_state
->attachments
[ds
].aux_usage
==
1195 const struct anv_image_view
*iview
= fb
->attachments
[ds
];
1197 if (clear_with_hiz
) {
1198 const bool clear_depth
= clear_att
.aspectMask
&
1199 VK_IMAGE_ASPECT_DEPTH_BIT
;
1200 const bool clear_stencil
= clear_att
.aspectMask
&
1201 VK_IMAGE_ASPECT_STENCIL_BIT
;
1203 /* Check against restrictions for depth buffer clearing. A great GPU
1204 * performance benefit isn't expected when using the HZ sequence for
1205 * stencil-only clears. Therefore, we don't emit a HZ op sequence for
1206 * a stencil clear in addition to using the BLORP-fallback for depth.
1209 if (!blorp_can_hiz_clear_depth(gen
, iview
->isl
.format
,
1210 iview
->image
->samples
,
1211 render_area
.offset
.x
,
1212 render_area
.offset
.y
,
1213 render_area
.offset
.x
+
1214 render_area
.extent
.width
,
1215 render_area
.offset
.y
+
1216 render_area
.extent
.height
)) {
1217 clear_with_hiz
= false;
1218 } else if (clear_att
.clearValue
.depthStencil
.depth
!=
1220 /* Don't enable fast depth clears for any color not equal to
1223 clear_with_hiz
= false;
1224 } else if (gen
== 8 &&
1225 anv_can_sample_with_hiz(&cmd_buffer
->device
->info
,
1227 iview
->image
->samples
)) {
1228 /* Only gen9+ supports returning ANV_HZ_FC_VAL when sampling a
1229 * fast-cleared portion of a HiZ buffer. Testing has revealed
1230 * that Gen8 only supports returning 0.0f. Gens prior to gen8 do
1231 * not support this feature at all.
1233 clear_with_hiz
= false;
1237 if (clear_with_hiz
) {
1238 blorp_gen8_hiz_clear_attachments(&batch
, iview
->image
->samples
,
1239 render_area
.offset
.x
,
1240 render_area
.offset
.y
,
1241 render_area
.offset
.x
+
1242 render_area
.extent
.width
,
1243 render_area
.offset
.y
+
1244 render_area
.extent
.height
,
1245 clear_depth
, clear_stencil
,
1246 clear_att
.clearValue
.
1247 depthStencil
.stencil
);
1249 /* From the SKL PRM, Depth Buffer Clear:
1251 * Depth Buffer Clear Workaround
1252 * Depth buffer clear pass using any of the methods (WM_STATE,
1253 * 3DSTATE_WM or 3DSTATE_WM_HZ_OP) must be followed by a
1254 * PIPE_CONTROL command with DEPTH_STALL bit and Depth FLUSH bits
1255 * “set” before starting to render. DepthStall and DepthFlush are
1256 * not needed between consecutive depth clear passes nor is it
1257 * required if the depth-clear pass was done with “full_surf_clear”
1258 * bit set in the 3DSTATE_WM_HZ_OP.
1261 cmd_buffer
->state
.pending_pipe_bits
|=
1262 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT
| ANV_PIPE_DEPTH_STALL_BIT
;
1267 if (!clear_with_hiz
) {
1268 clear_depth_stencil_attachment(cmd_buffer
, &batch
,
1269 &clear_att
, 1, &clear_rect
);
1272 cmd_state
->attachments
[ds
].pending_clear_aspects
= 0;
1275 blorp_batch_finish(&batch
);
1279 resolve_image(struct blorp_batch
*batch
,
1280 const struct anv_image
*src_image
,
1281 enum isl_aux_usage src_aux_usage
,
1282 uint32_t src_level
, uint32_t src_layer
,
1283 const struct anv_image
*dst_image
,
1284 enum isl_aux_usage dst_aux_usage
,
1285 uint32_t dst_level
, uint32_t dst_layer
,
1286 VkImageAspectFlags aspect_mask
,
1287 uint32_t src_x
, uint32_t src_y
, uint32_t dst_x
, uint32_t dst_y
,
1288 uint32_t width
, uint32_t height
)
1290 assert(src_image
->type
== VK_IMAGE_TYPE_2D
);
1291 assert(src_image
->samples
> 1);
1292 assert(dst_image
->type
== VK_IMAGE_TYPE_2D
);
1293 assert(dst_image
->samples
== 1);
1296 for_each_bit(a
, aspect_mask
) {
1297 VkImageAspectFlagBits aspect
= 1 << a
;
1299 struct blorp_surf src_surf
, dst_surf
;
1300 get_blorp_surf_for_anv_image(src_image
, aspect
,
1301 src_aux_usage
, &src_surf
);
1302 get_blorp_surf_for_anv_image(dst_image
, aspect
,
1303 dst_aux_usage
, &dst_surf
);
1306 &src_surf
, src_level
, src_layer
,
1307 ISL_FORMAT_UNSUPPORTED
, ISL_SWIZZLE_IDENTITY
,
1308 &dst_surf
, dst_level
, dst_layer
,
1309 ISL_FORMAT_UNSUPPORTED
, ISL_SWIZZLE_IDENTITY
,
1310 src_x
, src_y
, src_x
+ width
, src_y
+ height
,
1311 dst_x
, dst_y
, dst_x
+ width
, dst_y
+ height
,
1312 0x2600 /* GL_NEAREST */, false, false);
1316 void anv_CmdResolveImage(
1317 VkCommandBuffer commandBuffer
,
1319 VkImageLayout srcImageLayout
,
1321 VkImageLayout dstImageLayout
,
1322 uint32_t regionCount
,
1323 const VkImageResolve
* pRegions
)
1325 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
, commandBuffer
);
1326 ANV_FROM_HANDLE(anv_image
, src_image
, srcImage
);
1327 ANV_FROM_HANDLE(anv_image
, dst_image
, dstImage
);
1329 struct blorp_batch batch
;
1330 blorp_batch_init(&cmd_buffer
->device
->blorp
, &batch
, cmd_buffer
, 0);
1332 for (uint32_t r
= 0; r
< regionCount
; r
++) {
1333 assert(pRegions
[r
].srcSubresource
.aspectMask
==
1334 pRegions
[r
].dstSubresource
.aspectMask
);
1335 assert(anv_get_layerCount(src_image
, &pRegions
[r
].srcSubresource
) ==
1336 anv_get_layerCount(dst_image
, &pRegions
[r
].dstSubresource
));
1338 const uint32_t layer_count
=
1339 anv_get_layerCount(dst_image
, &pRegions
[r
].dstSubresource
);
1341 for (uint32_t layer
= 0; layer
< layer_count
; layer
++) {
1342 resolve_image(&batch
,
1343 src_image
, src_image
->aux_usage
,
1344 pRegions
[r
].srcSubresource
.mipLevel
,
1345 pRegions
[r
].srcSubresource
.baseArrayLayer
+ layer
,
1346 dst_image
, dst_image
->aux_usage
,
1347 pRegions
[r
].dstSubresource
.mipLevel
,
1348 pRegions
[r
].dstSubresource
.baseArrayLayer
+ layer
,
1349 pRegions
[r
].dstSubresource
.aspectMask
,
1350 pRegions
[r
].srcOffset
.x
, pRegions
[r
].srcOffset
.y
,
1351 pRegions
[r
].dstOffset
.x
, pRegions
[r
].dstOffset
.y
,
1352 pRegions
[r
].extent
.width
, pRegions
[r
].extent
.height
);
1356 blorp_batch_finish(&batch
);
1360 anv_image_fast_clear(struct anv_cmd_buffer
*cmd_buffer
,
1361 const struct anv_image
*image
,
1362 const uint32_t base_level
, const uint32_t level_count
,
1363 const uint32_t base_layer
, uint32_t layer_count
)
1365 assert(image
->type
== VK_IMAGE_TYPE_3D
|| image
->extent
.depth
== 1);
1367 if (image
->type
== VK_IMAGE_TYPE_3D
) {
1368 assert(base_layer
== 0);
1369 assert(layer_count
== anv_minify(image
->extent
.depth
, base_level
));
1372 struct blorp_batch batch
;
1373 blorp_batch_init(&cmd_buffer
->device
->blorp
, &batch
, cmd_buffer
, 0);
1375 struct blorp_surf surf
;
1376 get_blorp_surf_for_anv_image(image
, VK_IMAGE_ASPECT_COLOR_BIT
,
1377 image
->aux_usage
== ISL_AUX_USAGE_NONE
?
1378 ISL_AUX_USAGE_CCS_D
: image
->aux_usage
,
1381 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1383 * "After Render target fast clear, pipe-control with color cache
1384 * write-flush must be issued before sending any DRAW commands on
1385 * that render target."
1387 * This comment is a bit cryptic and doesn't really tell you what's going
1388 * or what's really needed. It appears that fast clear ops are not
1389 * properly synchronized with other drawing. This means that we cannot
1390 * have a fast clear operation in the pipe at the same time as other
1391 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
1392 * that the contents of the previous draw hit the render target before we
1393 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
1394 * that it is completed before any additional drawing occurs.
1396 cmd_buffer
->state
.pending_pipe_bits
|=
1397 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT
| ANV_PIPE_CS_STALL_BIT
;
1399 for (uint32_t l
= 0; l
< level_count
; l
++) {
1400 const uint32_t level
= base_level
+ l
;
1402 const VkExtent3D extent
= {
1403 .width
= anv_minify(image
->extent
.width
, level
),
1404 .height
= anv_minify(image
->extent
.height
, level
),
1405 .depth
= anv_minify(image
->extent
.depth
, level
),
1408 if (image
->type
== VK_IMAGE_TYPE_3D
)
1409 layer_count
= extent
.depth
;
1411 assert(level
< anv_image_aux_levels(image
));
1412 assert(base_layer
+ layer_count
<= anv_image_aux_layers(image
, level
));
1413 blorp_fast_clear(&batch
, &surf
, surf
.surf
->format
,
1414 level
, base_layer
, layer_count
,
1415 0, 0, extent
.width
, extent
.height
);
1418 cmd_buffer
->state
.pending_pipe_bits
|=
1419 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT
| ANV_PIPE_CS_STALL_BIT
;
1423 anv_cmd_buffer_resolve_subpass(struct anv_cmd_buffer
*cmd_buffer
)
1425 struct anv_framebuffer
*fb
= cmd_buffer
->state
.framebuffer
;
1426 struct anv_subpass
*subpass
= cmd_buffer
->state
.subpass
;
1428 if (subpass
->has_resolve
) {
1429 struct blorp_batch batch
;
1430 blorp_batch_init(&cmd_buffer
->device
->blorp
, &batch
, cmd_buffer
, 0);
1432 /* We are about to do some MSAA resolves. We need to flush so that the
1433 * result of writes to the MSAA color attachments show up in the sampler
1434 * when we blit to the single-sampled resolve target.
1436 cmd_buffer
->state
.pending_pipe_bits
|=
1437 ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT
|
1438 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT
;
1440 for (uint32_t i
= 0; i
< subpass
->color_count
; ++i
) {
1441 uint32_t src_att
= subpass
->color_attachments
[i
].attachment
;
1442 uint32_t dst_att
= subpass
->resolve_attachments
[i
].attachment
;
1444 if (dst_att
== VK_ATTACHMENT_UNUSED
)
1447 assert(src_att
< cmd_buffer
->state
.pass
->attachment_count
);
1448 assert(dst_att
< cmd_buffer
->state
.pass
->attachment_count
);
1450 if (cmd_buffer
->state
.attachments
[dst_att
].pending_clear_aspects
) {
1451 /* From the Vulkan 1.0 spec:
1453 * If the first use of an attachment in a render pass is as a
1454 * resolve attachment, then the loadOp is effectively ignored
1455 * as the resolve is guaranteed to overwrite all pixels in the
1458 cmd_buffer
->state
.attachments
[dst_att
].pending_clear_aspects
= 0;
1461 struct anv_image_view
*src_iview
= fb
->attachments
[src_att
];
1462 struct anv_image_view
*dst_iview
= fb
->attachments
[dst_att
];
1464 enum isl_aux_usage src_aux_usage
=
1465 cmd_buffer
->state
.attachments
[src_att
].aux_usage
;
1466 enum isl_aux_usage dst_aux_usage
=
1467 cmd_buffer
->state
.attachments
[dst_att
].aux_usage
;
1469 const VkRect2D render_area
= cmd_buffer
->state
.render_area
;
1471 assert(src_iview
->aspect_mask
== dst_iview
->aspect_mask
);
1473 resolve_image(&batch
, src_iview
->image
, src_aux_usage
,
1474 src_iview
->isl
.base_level
,
1475 src_iview
->isl
.base_array_layer
,
1476 dst_iview
->image
, dst_aux_usage
,
1477 dst_iview
->isl
.base_level
,
1478 dst_iview
->isl
.base_array_layer
,
1479 src_iview
->aspect_mask
,
1480 render_area
.offset
.x
, render_area
.offset
.y
,
1481 render_area
.offset
.x
, render_area
.offset
.y
,
1482 render_area
.extent
.width
, render_area
.extent
.height
);
1485 blorp_batch_finish(&batch
);
1490 anv_image_copy_to_shadow(struct anv_cmd_buffer
*cmd_buffer
,
1491 const struct anv_image
*image
,
1492 VkImageAspectFlagBits aspect
,
1493 uint32_t base_level
, uint32_t level_count
,
1494 uint32_t base_layer
, uint32_t layer_count
)
1496 struct blorp_batch batch
;
1497 blorp_batch_init(&cmd_buffer
->device
->blorp
, &batch
, cmd_buffer
, 0);
1499 struct blorp_surf surf
;
1500 get_blorp_surf_for_anv_image(image
, VK_IMAGE_ASPECT_COLOR_BIT
,
1501 ISL_AUX_USAGE_NONE
, &surf
);
1503 struct blorp_surf shadow_surf
= {
1504 .surf
= &image
->shadow_surface
.isl
,
1506 .buffer
= image
->bo
,
1507 .offset
= image
->offset
+ image
->shadow_surface
.offset
,
1511 for (uint32_t l
= 0; l
< level_count
; l
++) {
1512 const uint32_t level
= base_level
+ l
;
1514 const VkExtent3D extent
= {
1515 .width
= anv_minify(image
->extent
.width
, level
),
1516 .height
= anv_minify(image
->extent
.height
, level
),
1517 .depth
= anv_minify(image
->extent
.depth
, level
),
1520 if (image
->type
== VK_IMAGE_TYPE_3D
)
1521 layer_count
= extent
.depth
;
1523 for (uint32_t a
= 0; a
< layer_count
; a
++) {
1524 const uint32_t layer
= base_layer
+ a
;
1526 blorp_copy(&batch
, &surf
, level
, layer
,
1527 &shadow_surf
, level
, layer
,
1528 0, 0, 0, 0, extent
.width
, extent
.height
);
1532 blorp_batch_finish(&batch
);
1536 anv_gen8_hiz_op_resolve(struct anv_cmd_buffer
*cmd_buffer
,
1537 const struct anv_image
*image
,
1538 enum blorp_hiz_op op
)
1542 /* Don't resolve depth buffers without an auxiliary HiZ buffer and
1543 * don't perform such a resolve on gens that don't support it.
1545 if (cmd_buffer
->device
->info
.gen
< 8 ||
1546 image
->aux_usage
!= ISL_AUX_USAGE_HIZ
)
1549 assert(op
== BLORP_HIZ_OP_HIZ_RESOLVE
||
1550 op
== BLORP_HIZ_OP_DEPTH_RESOLVE
);
1552 struct blorp_batch batch
;
1553 blorp_batch_init(&cmd_buffer
->device
->blorp
, &batch
, cmd_buffer
, 0);
1555 struct blorp_surf surf
;
1556 get_blorp_surf_for_anv_image(image
, VK_IMAGE_ASPECT_DEPTH_BIT
,
1557 ISL_AUX_USAGE_NONE
, &surf
);
1559 /* Manually add the aux HiZ surf */
1560 surf
.aux_surf
= &image
->aux_surface
.isl
,
1561 surf
.aux_addr
= (struct blorp_address
) {
1562 .buffer
= image
->bo
,
1563 .offset
= image
->offset
+ image
->aux_surface
.offset
,
1565 surf
.aux_usage
= ISL_AUX_USAGE_HIZ
;
1567 surf
.clear_color
.f32
[0] = ANV_HZ_FC_VAL
;
1569 blorp_hiz_op(&batch
, &surf
, 0, 0, 1, op
);
1570 blorp_batch_finish(&batch
);
1574 anv_ccs_resolve(struct anv_cmd_buffer
* const cmd_buffer
,
1575 const struct anv_state surface_state
,
1576 const struct anv_image
* const image
,
1577 const uint8_t level
, const uint32_t layer_count
,
1578 const enum blorp_fast_clear_op op
)
1580 assert(cmd_buffer
&& image
);
1582 /* The resolved subresource range must have a CCS buffer. */
1583 assert(level
< anv_image_aux_levels(image
));
1584 assert(layer_count
<= anv_image_aux_layers(image
, level
));
1585 assert(image
->aspects
== VK_IMAGE_ASPECT_COLOR_BIT
&& image
->samples
== 1);
1587 /* Create a binding table for this surface state. */
1588 uint32_t binding_table
;
1590 binding_table_for_surface_state(cmd_buffer
, surface_state
,
1592 if (result
!= VK_SUCCESS
)
1595 struct blorp_batch batch
;
1596 blorp_batch_init(&cmd_buffer
->device
->blorp
, &batch
, cmd_buffer
,
1597 BLORP_BATCH_PREDICATE_ENABLE
);
1599 struct blorp_surf surf
;
1600 get_blorp_surf_for_anv_image(image
, VK_IMAGE_ASPECT_COLOR_BIT
,
1601 image
->aux_usage
== ISL_AUX_USAGE_CCS_E
?
1602 ISL_AUX_USAGE_CCS_E
: ISL_AUX_USAGE_CCS_D
,
1605 blorp_ccs_resolve_attachment(&batch
, binding_table
, &surf
, level
,
1606 layer_count
, image
->color_surface
.isl
.format
,
1609 blorp_batch_finish(&batch
);