projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
anv: refresh cached current batch bo after emitting some commands
[mesa.git] / src / intel / vulkan / anv_blorp.c
1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "anv_private.h"
25
26 static bool
27 lookup_blorp_shader(struct blorp_batch *batch,
28 const void *key, uint32_t key_size,
29 uint32_t *kernel_out, void *prog_data_out)
30 {
31 struct blorp_context *blorp = batch->blorp;
32 struct anv_device *device = blorp->driver_ctx;
33
34 /* The default cache must be a real cache */
35 assert(device->default_pipeline_cache.cache);
36
37 struct anv_shader_bin *bin =
38 anv_pipeline_cache_search(&device->default_pipeline_cache, key, key_size);
39 if (!bin)
40 return false;
41
42 /* The cache already has a reference and it's not going anywhere so there
43 * is no need to hold a second reference.
44 */
45 anv_shader_bin_unref(device, bin);
46
47 *kernel_out = bin->kernel.offset;
48 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
49
50 return true;
51 }
52
53 static bool
54 upload_blorp_shader(struct blorp_batch *batch, uint32_t stage,
55 const void *key, uint32_t key_size,
56 const void *kernel, uint32_t kernel_size,
57 const struct brw_stage_prog_data *prog_data,
58 uint32_t prog_data_size,
59 uint32_t *kernel_out, void *prog_data_out)
60 {
61 struct blorp_context *blorp = batch->blorp;
62 struct anv_device *device = blorp->driver_ctx;
63
64 /* The blorp cache must be a real cache */
65 assert(device->default_pipeline_cache.cache);
66
67 struct anv_pipeline_bind_map bind_map = {
68 .surface_count = 0,
69 .sampler_count = 0,
70 };
71
72 struct anv_shader_bin *bin =
73 anv_pipeline_cache_upload_kernel(&device->default_pipeline_cache, stage,
74 key, key_size, kernel, kernel_size,
75 NULL, 0,
76 prog_data, prog_data_size,
77 NULL, 0, NULL, &bind_map);
78
79 if (!bin)
80 return false;
81
82 /* The cache already has a reference and it's not going anywhere so there
83 * is no need to hold a second reference.
84 */
85 anv_shader_bin_unref(device, bin);
86
87 *kernel_out = bin->kernel.offset;
88 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
89
90 return true;
91 }
92
93 void
94 anv_device_init_blorp(struct anv_device *device)
95 {
96 blorp_init(&device->blorp, device, &device->isl_dev);
97 device->blorp.compiler = device->physical->compiler;
98 device->blorp.lookup_shader = lookup_blorp_shader;
99 device->blorp.upload_shader = upload_blorp_shader;
100 switch (device->info.gen) {
101 case 7:
102 if (device->info.is_haswell) {
103 device->blorp.exec = gen75_blorp_exec;
104 } else {
105 device->blorp.exec = gen7_blorp_exec;
106 }
107 break;
108 case 8:
109 device->blorp.exec = gen8_blorp_exec;
110 break;
111 case 9:
112 device->blorp.exec = gen9_blorp_exec;
113 break;
114 case 10:
115 device->blorp.exec = gen10_blorp_exec;
116 break;
117 case 11:
118 device->blorp.exec = gen11_blorp_exec;
119 break;
120 case 12:
121 device->blorp.exec = gen12_blorp_exec;
122 break;
123 default:
124 unreachable("Unknown hardware generation");
125 }
126 }
127
128 void
129 anv_device_finish_blorp(struct anv_device *device)
130 {
131 blorp_finish(&device->blorp);
132 }
133
134 static void
135 get_blorp_surf_for_anv_buffer(struct anv_device *device,
136 struct anv_buffer *buffer, uint64_t offset,
137 uint32_t width, uint32_t height,
138 uint32_t row_pitch, enum isl_format format,
139 struct blorp_surf *blorp_surf,
140 struct isl_surf *isl_surf)
141 {
142 const struct isl_format_layout *fmtl =
143 isl_format_get_layout(format);
144 bool ok UNUSED;
145
146 /* ASTC is the only format which doesn't support linear layouts.
147 * Create an equivalently sized surface with ISL to get around this.
148 */
149 if (fmtl->txc == ISL_TXC_ASTC) {
150 /* Use an equivalently sized format */
151 format = ISL_FORMAT_R32G32B32A32_UINT;
152 assert(fmtl->bpb == isl_format_get_layout(format)->bpb);
153
154 /* Shrink the dimensions for the new format */
155 width = DIV_ROUND_UP(width, fmtl->bw);
156 height = DIV_ROUND_UP(height, fmtl->bh);
157 }
158
159 *blorp_surf = (struct blorp_surf) {
160 .surf = isl_surf,
161 .addr = {
162 .buffer = buffer->address.bo,
163 .offset = buffer->address.offset + offset,
164 .mocs = anv_mocs_for_bo(device, buffer->address.bo),
165 },
166 };
167
168 ok = isl_surf_init(&device->isl_dev, isl_surf,
169 .dim = ISL_SURF_DIM_2D,
170 .format = format,
171 .width = width,
172 .height = height,
173 .depth = 1,
174 .levels = 1,
175 .array_len = 1,
176 .samples = 1,
177 .row_pitch_B = row_pitch,
178 .usage = ISL_SURF_USAGE_TEXTURE_BIT |
179 ISL_SURF_USAGE_RENDER_TARGET_BIT,
180 .tiling_flags = ISL_TILING_LINEAR_BIT);
181 assert(ok);
182 }
183
184 /* Pick something high enough that it won't be used in core and low enough it
185 * will never map to an extension.
186 */
187 #define ANV_IMAGE_LAYOUT_EXPLICIT_AUX (VkImageLayout)10000000
188
189 static struct blorp_address
190 anv_to_blorp_address(struct anv_address addr)
191 {
192 return (struct blorp_address) {
193 .buffer = addr.bo,
194 .offset = addr.offset,
195 };
196 }
197
198 static void
199 get_blorp_surf_for_anv_image(const struct anv_device *device,
200 const struct anv_image *image,
201 VkImageAspectFlags aspect,
202 VkImageUsageFlags usage,
203 VkImageLayout layout,
204 enum isl_aux_usage aux_usage,
205 struct blorp_surf *blorp_surf)
206 {
207 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
208
209 if (layout != ANV_IMAGE_LAYOUT_EXPLICIT_AUX) {
210 assert(usage != 0);
211 aux_usage = anv_layout_to_aux_usage(&device->info, image,
212 aspect, usage, layout);
213 }
214
215 const struct anv_surface *surface = &image->planes[plane].surface;
216 *blorp_surf = (struct blorp_surf) {
217 .surf = &surface->isl,
218 .addr = {
219 .buffer = image->planes[plane].address.bo,
220 .offset = image->planes[plane].address.offset + surface->offset,
221 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
222 },
223 };
224
225 if (aux_usage != ISL_AUX_USAGE_NONE) {
226 const struct anv_surface *aux_surface = &image->planes[plane].aux_surface;
227 blorp_surf->aux_surf = &aux_surface->isl,
228 blorp_surf->aux_addr = (struct blorp_address) {
229 .buffer = image->planes[plane].address.bo,
230 .offset = image->planes[plane].address.offset + aux_surface->offset,
231 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
232 };
233 blorp_surf->aux_usage = aux_usage;
234
235 /* If we're doing a partial resolve, then we need the indirect clear
236 * color. If we are doing a fast clear and want to store/update the
237 * clear color, we also pass the address to blorp, otherwise it will only
238 * stomp the CCS to a particular value and won't care about format or
239 * clear value
240 */
241 if (aspect & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
242 const struct anv_address clear_color_addr =
243 anv_image_get_clear_color_addr(device, image, aspect);
244 blorp_surf->clear_color_addr = anv_to_blorp_address(clear_color_addr);
245 } else if (aspect & VK_IMAGE_ASPECT_DEPTH_BIT) {
246 if (device->info.gen >= 10) {
247 /* Vulkan always clears to 1.0. On gen < 10, we set that directly
248 * in the state packet. For gen >= 10, must provide the clear
249 * value in a buffer. We have a single global buffer that stores
250 * the 1.0 value.
251 */
252 const struct anv_address clear_color_addr = (struct anv_address) {
253 .bo = device->hiz_clear_bo,
254 };
255 blorp_surf->clear_color_addr =
256 anv_to_blorp_address(clear_color_addr);
257 } else {
258 blorp_surf->clear_color = (union isl_color_value) {
259 .f32 = { ANV_HZ_FC_VAL },
260 };
261 }
262 }
263 }
264 }
265
266 static bool
267 get_blorp_surf_for_anv_shadow_image(const struct anv_device *device,
268 const struct anv_image *image,
269 VkImageAspectFlags aspect,
270 struct blorp_surf *blorp_surf)
271 {
272
273 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
274 if (image->planes[plane].shadow_surface.isl.size_B == 0)
275 return false;
276
277 *blorp_surf = (struct blorp_surf) {
278 .surf = &image->planes[plane].shadow_surface.isl,
279 .addr = {
280 .buffer = image->planes[plane].address.bo,
281 .offset = image->planes[plane].address.offset +
282 image->planes[plane].shadow_surface.offset,
283 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
284 },
285 };
286
287 return true;
288 }
289
290 void anv_CmdCopyImage(
291 VkCommandBuffer commandBuffer,
292 VkImage srcImage,
293 VkImageLayout srcImageLayout,
294 VkImage dstImage,
295 VkImageLayout dstImageLayout,
296 uint32_t regionCount,
297 const VkImageCopy* pRegions)
298 {
299 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
300 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
301 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
302
303 struct blorp_batch batch;
304 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
305
306 for (unsigned r = 0; r < regionCount; r++) {
307 VkOffset3D srcOffset =
308 anv_sanitize_image_offset(src_image->type, pRegions[r].srcOffset);
309 VkOffset3D dstOffset =
310 anv_sanitize_image_offset(dst_image->type, pRegions[r].dstOffset);
311 VkExtent3D extent =
312 anv_sanitize_image_extent(src_image->type, pRegions[r].extent);
313
314 const uint32_t dst_level = pRegions[r].dstSubresource.mipLevel;
315 unsigned dst_base_layer, layer_count;
316 if (dst_image->type == VK_IMAGE_TYPE_3D) {
317 dst_base_layer = pRegions[r].dstOffset.z;
318 layer_count = pRegions[r].extent.depth;
319 } else {
320 dst_base_layer = pRegions[r].dstSubresource.baseArrayLayer;
321 layer_count =
322 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource);
323 }
324
325 const uint32_t src_level = pRegions[r].srcSubresource.mipLevel;
326 unsigned src_base_layer;
327 if (src_image->type == VK_IMAGE_TYPE_3D) {
328 src_base_layer = pRegions[r].srcOffset.z;
329 } else {
330 src_base_layer = pRegions[r].srcSubresource.baseArrayLayer;
331 assert(layer_count ==
332 anv_get_layerCount(src_image, &pRegions[r].srcSubresource));
333 }
334
335 VkImageAspectFlags src_mask = pRegions[r].srcSubresource.aspectMask,
336 dst_mask = pRegions[r].dstSubresource.aspectMask;
337
338 assert(anv_image_aspects_compatible(src_mask, dst_mask));
339
340 if (util_bitcount(src_mask) > 1) {
341 uint32_t aspect_bit;
342 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_mask) {
343 struct blorp_surf src_surf, dst_surf;
344 get_blorp_surf_for_anv_image(cmd_buffer->device,
345 src_image, 1UL << aspect_bit,
346 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
347 srcImageLayout, ISL_AUX_USAGE_NONE,
348 &src_surf);
349 get_blorp_surf_for_anv_image(cmd_buffer->device,
350 dst_image, 1UL << aspect_bit,
351 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
352 dstImageLayout, ISL_AUX_USAGE_NONE,
353 &dst_surf);
354 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
355 1UL << aspect_bit,
356 dst_surf.aux_usage, dst_level,
357 dst_base_layer, layer_count);
358
359 for (unsigned i = 0; i < layer_count; i++) {
360 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
361 &dst_surf, dst_level, dst_base_layer + i,
362 srcOffset.x, srcOffset.y,
363 dstOffset.x, dstOffset.y,
364 extent.width, extent.height);
365 }
366
367 struct blorp_surf dst_shadow_surf;
368 if (get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
369 dst_image,
370 1UL << aspect_bit,
371 &dst_shadow_surf)) {
372 for (unsigned i = 0; i < layer_count; i++) {
373 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
374 &dst_shadow_surf, dst_level, dst_base_layer + i,
375 srcOffset.x, srcOffset.y,
376 dstOffset.x, dstOffset.y,
377 extent.width, extent.height);
378 }
379 }
380 }
381 } else {
382 struct blorp_surf src_surf, dst_surf;
383 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, src_mask,
384 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
385 srcImageLayout, ISL_AUX_USAGE_NONE,
386 &src_surf);
387 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, dst_mask,
388 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
389 dstImageLayout, ISL_AUX_USAGE_NONE,
390 &dst_surf);
391 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image, dst_mask,
392 dst_surf.aux_usage, dst_level,
393 dst_base_layer, layer_count);
394
395 for (unsigned i = 0; i < layer_count; i++) {
396 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
397 &dst_surf, dst_level, dst_base_layer + i,
398 srcOffset.x, srcOffset.y,
399 dstOffset.x, dstOffset.y,
400 extent.width, extent.height);
401 }
402
403 struct blorp_surf dst_shadow_surf;
404 if (get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
405 dst_image, dst_mask,
406 &dst_shadow_surf)) {
407 for (unsigned i = 0; i < layer_count; i++) {
408 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
409 &dst_shadow_surf, dst_level, dst_base_layer + i,
410 srcOffset.x, srcOffset.y,
411 dstOffset.x, dstOffset.y,
412 extent.width, extent.height);
413 }
414 }
415 }
416 }
417
418 blorp_batch_finish(&batch);
419 }
420
421 static enum isl_format
422 isl_format_for_size(unsigned size_B)
423 {
424 /* Prefer 32-bit per component formats for CmdFillBuffer */
425 switch (size_B) {
426 case 1: return ISL_FORMAT_R8_UINT;
427 case 2: return ISL_FORMAT_R16_UINT;
428 case 3: return ISL_FORMAT_R8G8B8_UINT;
429 case 4: return ISL_FORMAT_R32_UINT;
430 case 6: return ISL_FORMAT_R16G16B16_UINT;
431 case 8: return ISL_FORMAT_R32G32_UINT;
432 case 12: return ISL_FORMAT_R32G32B32_UINT;
433 case 16: return ISL_FORMAT_R32G32B32A32_UINT;
434 default:
435 unreachable("Unknown format size");
436 }
437 }
438
439 static void
440 copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer,
441 struct anv_buffer *anv_buffer,
442 struct anv_image *anv_image,
443 VkImageLayout image_layout,
444 uint32_t regionCount,
445 const VkBufferImageCopy* pRegions,
446 bool buffer_to_image)
447 {
448 struct blorp_batch batch;
449 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
450
451 struct {
452 struct blorp_surf surf;
453 uint32_t level;
454 VkOffset3D offset;
455 } image, buffer, *src, *dst;
456
457 buffer.level = 0;
458 buffer.offset = (VkOffset3D) { 0, 0, 0 };
459
460 if (buffer_to_image) {
461 src = &buffer;
462 dst = &image;
463 } else {
464 src = &image;
465 dst = &buffer;
466 }
467
468 for (unsigned r = 0; r < regionCount; r++) {
469 const VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask;
470
471 get_blorp_surf_for_anv_image(cmd_buffer->device, anv_image, aspect,
472 buffer_to_image ?
473 VK_IMAGE_USAGE_TRANSFER_DST_BIT :
474 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
475 image_layout, ISL_AUX_USAGE_NONE,
476 &image.surf);
477 image.offset =
478 anv_sanitize_image_offset(anv_image->type, pRegions[r].imageOffset);
479 image.level = pRegions[r].imageSubresource.mipLevel;
480
481 VkExtent3D extent =
482 anv_sanitize_image_extent(anv_image->type, pRegions[r].imageExtent);
483 if (anv_image->type != VK_IMAGE_TYPE_3D) {
484 image.offset.z = pRegions[r].imageSubresource.baseArrayLayer;
485 extent.depth =
486 anv_get_layerCount(anv_image, &pRegions[r].imageSubresource);
487 }
488
489 const enum isl_format linear_format =
490 anv_get_isl_format(&cmd_buffer->device->info, anv_image->vk_format,
491 aspect, VK_IMAGE_TILING_LINEAR);
492 const struct isl_format_layout *linear_fmtl =
493 isl_format_get_layout(linear_format);
494
495 const uint32_t buffer_row_length =
496 pRegions[r].bufferRowLength ?
497 pRegions[r].bufferRowLength : extent.width;
498
499 const uint32_t buffer_image_height =
500 pRegions[r].bufferImageHeight ?
501 pRegions[r].bufferImageHeight : extent.height;
502
503 const uint32_t buffer_row_pitch =
504 DIV_ROUND_UP(buffer_row_length, linear_fmtl->bw) *
505 (linear_fmtl->bpb / 8);
506
507 const uint32_t buffer_layer_stride =
508 DIV_ROUND_UP(buffer_image_height, linear_fmtl->bh) *
509 buffer_row_pitch;
510
511 /* Some formats have additional restrictions which may cause ISL to
512 * fail to create a surface for us. Some examples include:
513 *
514 * 1. ASTC formats are not allowed to be LINEAR and must be tiled
515 * 2. YCbCr formats have to have 2-pixel aligned strides
516 *
517 * To avoid these issues, we always bind the buffer as if it's a
518 * "normal" format like RGBA32_UINT. Since we're using blorp_copy,
519 * the format doesn't matter as long as it has the right bpb.
520 */
521 const VkExtent2D buffer_extent = {
522 .width = DIV_ROUND_UP(extent.width, linear_fmtl->bw),
523 .height = DIV_ROUND_UP(extent.height, linear_fmtl->bh),
524 };
525 const enum isl_format buffer_format =
526 isl_format_for_size(linear_fmtl->bpb / 8);
527
528 struct isl_surf buffer_isl_surf;
529 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
530 anv_buffer, pRegions[r].bufferOffset,
531 buffer_extent.width, buffer_extent.height,
532 buffer_row_pitch, buffer_format,
533 &buffer.surf, &buffer_isl_surf);
534
535 bool dst_has_shadow = false;
536 struct blorp_surf dst_shadow_surf;
537 if (&image == dst) {
538 /* In this case, the source is the buffer and, since blorp takes its
539 * copy dimensions in terms of the source format, we have to use the
540 * scaled down version for compressed textures because the source
541 * format is an RGB format.
542 */
543 extent.width = buffer_extent.width;
544 extent.height = buffer_extent.height;
545
546 anv_cmd_buffer_mark_image_written(cmd_buffer, anv_image,
547 aspect, dst->surf.aux_usage,
548 dst->level,
549 dst->offset.z, extent.depth);
550
551 dst_has_shadow =
552 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
553 anv_image, aspect,
554 &dst_shadow_surf);
555 }
556
557 for (unsigned z = 0; z < extent.depth; z++) {
558 blorp_copy(&batch, &src->surf, src->level, src->offset.z,
559 &dst->surf, dst->level, dst->offset.z,
560 src->offset.x, src->offset.y, dst->offset.x, dst->offset.y,
561 extent.width, extent.height);
562
563 if (dst_has_shadow) {
564 blorp_copy(&batch, &src->surf, src->level, src->offset.z,
565 &dst_shadow_surf, dst->level, dst->offset.z,
566 src->offset.x, src->offset.y,
567 dst->offset.x, dst->offset.y,
568 extent.width, extent.height);
569 }
570
571 image.offset.z++;
572 buffer.surf.addr.offset += buffer_layer_stride;
573 }
574 }
575
576 blorp_batch_finish(&batch);
577 }
578
579 void anv_CmdCopyBufferToImage(
580 VkCommandBuffer commandBuffer,
581 VkBuffer srcBuffer,
582 VkImage dstImage,
583 VkImageLayout dstImageLayout,
584 uint32_t regionCount,
585 const VkBufferImageCopy* pRegions)
586 {
587 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
588 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
589 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
590
591 copy_buffer_to_image(cmd_buffer, src_buffer, dst_image, dstImageLayout,
592 regionCount, pRegions, true);
593 }
594
595 void anv_CmdCopyImageToBuffer(
596 VkCommandBuffer commandBuffer,
597 VkImage srcImage,
598 VkImageLayout srcImageLayout,
599 VkBuffer dstBuffer,
600 uint32_t regionCount,
601 const VkBufferImageCopy* pRegions)
602 {
603 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
604 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
605 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
606
607 copy_buffer_to_image(cmd_buffer, dst_buffer, src_image, srcImageLayout,
608 regionCount, pRegions, false);
609
610 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
611 }
612
613 static bool
614 flip_coords(unsigned *src0, unsigned *src1, unsigned *dst0, unsigned *dst1)
615 {
616 bool flip = false;
617 if (*src0 > *src1) {
618 unsigned tmp = *src0;
619 *src0 = *src1;
620 *src1 = tmp;
621 flip = !flip;
622 }
623
624 if (*dst0 > *dst1) {
625 unsigned tmp = *dst0;
626 *dst0 = *dst1;
627 *dst1 = tmp;
628 flip = !flip;
629 }
630
631 return flip;
632 }
633
634 void anv_CmdBlitImage(
635 VkCommandBuffer commandBuffer,
636 VkImage srcImage,
637 VkImageLayout srcImageLayout,
638 VkImage dstImage,
639 VkImageLayout dstImageLayout,
640 uint32_t regionCount,
641 const VkImageBlit* pRegions,
642 VkFilter filter)
643
644 {
645 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
646 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
647 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
648
649 struct blorp_surf src, dst;
650
651 enum blorp_filter blorp_filter;
652 switch (filter) {
653 case VK_FILTER_NEAREST:
654 blorp_filter = BLORP_FILTER_NEAREST;
655 break;
656 case VK_FILTER_LINEAR:
657 blorp_filter = BLORP_FILTER_BILINEAR;
658 break;
659 default:
660 unreachable("Invalid filter");
661 }
662
663 struct blorp_batch batch;
664 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
665
666 for (unsigned r = 0; r < regionCount; r++) {
667 const VkImageSubresourceLayers *src_res = &pRegions[r].srcSubresource;
668 const VkImageSubresourceLayers *dst_res = &pRegions[r].dstSubresource;
669
670 assert(anv_image_aspects_compatible(src_res->aspectMask,
671 dst_res->aspectMask));
672
673 uint32_t aspect_bit;
674 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_res->aspectMask) {
675 get_blorp_surf_for_anv_image(cmd_buffer->device,
676 src_image, 1U << aspect_bit,
677 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
678 srcImageLayout, ISL_AUX_USAGE_NONE, &src);
679 get_blorp_surf_for_anv_image(cmd_buffer->device,
680 dst_image, 1U << aspect_bit,
681 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
682 dstImageLayout, ISL_AUX_USAGE_NONE, &dst);
683
684 struct anv_format_plane src_format =
685 anv_get_format_plane(&cmd_buffer->device->info, src_image->vk_format,
686 1U << aspect_bit, src_image->tiling);
687 struct anv_format_plane dst_format =
688 anv_get_format_plane(&cmd_buffer->device->info, dst_image->vk_format,
689 1U << aspect_bit, dst_image->tiling);
690
691 unsigned dst_start, dst_end;
692 if (dst_image->type == VK_IMAGE_TYPE_3D) {
693 assert(dst_res->baseArrayLayer == 0);
694 dst_start = pRegions[r].dstOffsets[0].z;
695 dst_end = pRegions[r].dstOffsets[1].z;
696 } else {
697 dst_start = dst_res->baseArrayLayer;
698 dst_end = dst_start + anv_get_layerCount(dst_image, dst_res);
699 }
700
701 unsigned src_start, src_end;
702 if (src_image->type == VK_IMAGE_TYPE_3D) {
703 assert(src_res->baseArrayLayer == 0);
704 src_start = pRegions[r].srcOffsets[0].z;
705 src_end = pRegions[r].srcOffsets[1].z;
706 } else {
707 src_start = src_res->baseArrayLayer;
708 src_end = src_start + anv_get_layerCount(src_image, src_res);
709 }
710
711 bool flip_z = flip_coords(&src_start, &src_end, &dst_start, &dst_end);
712 float src_z_step = (float)(src_end + 1 - src_start) /
713 (float)(dst_end + 1 - dst_start);
714
715 if (flip_z) {
716 src_start = src_end;
717 src_z_step *= -1;
718 }
719
720 unsigned src_x0 = pRegions[r].srcOffsets[0].x;
721 unsigned src_x1 = pRegions[r].srcOffsets[1].x;
722 unsigned dst_x0 = pRegions[r].dstOffsets[0].x;
723 unsigned dst_x1 = pRegions[r].dstOffsets[1].x;
724 bool flip_x = flip_coords(&src_x0, &src_x1, &dst_x0, &dst_x1);
725
726 unsigned src_y0 = pRegions[r].srcOffsets[0].y;
727 unsigned src_y1 = pRegions[r].srcOffsets[1].y;
728 unsigned dst_y0 = pRegions[r].dstOffsets[0].y;
729 unsigned dst_y1 = pRegions[r].dstOffsets[1].y;
730 bool flip_y = flip_coords(&src_y0, &src_y1, &dst_y0, &dst_y1);
731
732 const unsigned num_layers = dst_end - dst_start;
733 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
734 1U << aspect_bit,
735 dst.aux_usage,
736 dst_res->mipLevel,
737 dst_start, num_layers);
738
739 for (unsigned i = 0; i < num_layers; i++) {
740 unsigned dst_z = dst_start + i;
741 unsigned src_z = src_start + i * src_z_step;
742
743 blorp_blit(&batch, &src, src_res->mipLevel, src_z,
744 src_format.isl_format, src_format.swizzle,
745 &dst, dst_res->mipLevel, dst_z,
746 dst_format.isl_format, dst_format.swizzle,
747 src_x0, src_y0, src_x1, src_y1,
748 dst_x0, dst_y0, dst_x1, dst_y1,
749 blorp_filter, flip_x, flip_y);
750 }
751 }
752 }
753
754 blorp_batch_finish(&batch);
755 }
756
757 /**
758 * Returns the greatest common divisor of a and b that is a power of two.
759 */
760 static uint64_t
761 gcd_pow2_u64(uint64_t a, uint64_t b)
762 {
763 assert(a > 0 || b > 0);
764
765 unsigned a_log2 = ffsll(a) - 1;
766 unsigned b_log2 = ffsll(b) - 1;
767
768 /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which
769 * case, the MIN2() will take the other one. If both are 0 then we will
770 * hit the assert above.
771 */
772 return 1 << MIN2(a_log2, b_log2);
773 }
774
775 /* This is maximum possible width/height our HW can handle */
776 #define MAX_SURFACE_DIM (1ull << 14)
777
778 void anv_CmdCopyBuffer(
779 VkCommandBuffer commandBuffer,
780 VkBuffer srcBuffer,
781 VkBuffer dstBuffer,
782 uint32_t regionCount,
783 const VkBufferCopy* pRegions)
784 {
785 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
786 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
787 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
788
789 struct blorp_batch batch;
790 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
791
792 for (unsigned r = 0; r < regionCount; r++) {
793 struct blorp_address src = {
794 .buffer = src_buffer->address.bo,
795 .offset = src_buffer->address.offset + pRegions[r].srcOffset,
796 .mocs = anv_mocs_for_bo(cmd_buffer->device, src_buffer->address.bo),
797 };
798 struct blorp_address dst = {
799 .buffer = dst_buffer->address.bo,
800 .offset = dst_buffer->address.offset + pRegions[r].dstOffset,
801 .mocs = anv_mocs_for_bo(cmd_buffer->device, dst_buffer->address.bo),
802 };
803
804 blorp_buffer_copy(&batch, src, dst, pRegions[r].size);
805 }
806
807 blorp_batch_finish(&batch);
808
809 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
810 }
811
812 void anv_CmdUpdateBuffer(
813 VkCommandBuffer commandBuffer,
814 VkBuffer dstBuffer,
815 VkDeviceSize dstOffset,
816 VkDeviceSize dataSize,
817 const void* pData)
818 {
819 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
820 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
821
822 struct blorp_batch batch;
823 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
824
825 /* We can't quite grab a full block because the state stream needs a
826 * little data at the top to build its linked list.
827 */
828 const uint32_t max_update_size =
829 cmd_buffer->device->dynamic_state_pool.block_size - 64;
830
831 assert(max_update_size < MAX_SURFACE_DIM * 4);
832
833 /* We're about to read data that was written from the CPU. Flush the
834 * texture cache so we don't get anything stale.
835 */
836 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
837
838 while (dataSize) {
839 const uint32_t copy_size = MIN2(dataSize, max_update_size);
840
841 struct anv_state tmp_data =
842 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, copy_size, 64);
843
844 memcpy(tmp_data.map, pData, copy_size);
845
846 struct blorp_address src = {
847 .buffer = cmd_buffer->device->dynamic_state_pool.block_pool.bo,
848 .offset = tmp_data.offset,
849 .mocs = cmd_buffer->device->isl_dev.mocs.internal,
850 };
851 struct blorp_address dst = {
852 .buffer = dst_buffer->address.bo,
853 .offset = dst_buffer->address.offset + dstOffset,
854 .mocs = anv_mocs_for_bo(cmd_buffer->device, dst_buffer->address.bo),
855 };
856
857 blorp_buffer_copy(&batch, src, dst, copy_size);
858
859 dataSize -= copy_size;
860 dstOffset += copy_size;
861 pData = (void *)pData + copy_size;
862 }
863
864 blorp_batch_finish(&batch);
865
866 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
867 }
868
869 void anv_CmdFillBuffer(
870 VkCommandBuffer commandBuffer,
871 VkBuffer dstBuffer,
872 VkDeviceSize dstOffset,
873 VkDeviceSize fillSize,
874 uint32_t data)
875 {
876 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
877 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
878 struct blorp_surf surf;
879 struct isl_surf isl_surf;
880
881 struct blorp_batch batch;
882 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
883
884 fillSize = anv_buffer_get_range(dst_buffer, dstOffset, fillSize);
885
886 /* From the Vulkan spec:
887 *
888 * "size is the number of bytes to fill, and must be either a multiple
889 * of 4, or VK_WHOLE_SIZE to fill the range from offset to the end of
890 * the buffer. If VK_WHOLE_SIZE is used and the remaining size of the
891 * buffer is not a multiple of 4, then the nearest smaller multiple is
892 * used."
893 */
894 fillSize &= ~3ull;
895
896 /* First, we compute the biggest format that can be used with the
897 * given offsets and size.
898 */
899 int bs = 16;
900 bs = gcd_pow2_u64(bs, dstOffset);
901 bs = gcd_pow2_u64(bs, fillSize);
902 enum isl_format isl_format = isl_format_for_size(bs);
903
904 union isl_color_value color = {
905 .u32 = { data, data, data, data },
906 };
907
908 const uint64_t max_fill_size = MAX_SURFACE_DIM * MAX_SURFACE_DIM * bs;
909 while (fillSize >= max_fill_size) {
910 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
911 dst_buffer, dstOffset,
912 MAX_SURFACE_DIM, MAX_SURFACE_DIM,
913 MAX_SURFACE_DIM * bs, isl_format,
914 &surf, &isl_surf);
915
916 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
917 0, 0, 1, 0, 0, MAX_SURFACE_DIM, MAX_SURFACE_DIM,
918 color, NULL);
919 fillSize -= max_fill_size;
920 dstOffset += max_fill_size;
921 }
922
923 uint64_t height = fillSize / (MAX_SURFACE_DIM * bs);
924 assert(height < MAX_SURFACE_DIM);
925 if (height != 0) {
926 const uint64_t rect_fill_size = height * MAX_SURFACE_DIM * bs;
927 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
928 dst_buffer, dstOffset,
929 MAX_SURFACE_DIM, height,
930 MAX_SURFACE_DIM * bs, isl_format,
931 &surf, &isl_surf);
932
933 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
934 0, 0, 1, 0, 0, MAX_SURFACE_DIM, height,
935 color, NULL);
936 fillSize -= rect_fill_size;
937 dstOffset += rect_fill_size;
938 }
939
940 if (fillSize != 0) {
941 const uint32_t width = fillSize / bs;
942 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
943 dst_buffer, dstOffset,
944 width, 1,
945 width * bs, isl_format,
946 &surf, &isl_surf);
947
948 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
949 0, 0, 1, 0, 0, width, 1,
950 color, NULL);
951 }
952
953 blorp_batch_finish(&batch);
954
955 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
956 }
957
958 void anv_CmdClearColorImage(
959 VkCommandBuffer commandBuffer,
960 VkImage _image,
961 VkImageLayout imageLayout,
962 const VkClearColorValue* pColor,
963 uint32_t rangeCount,
964 const VkImageSubresourceRange* pRanges)
965 {
966 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
967 ANV_FROM_HANDLE(anv_image, image, _image);
968
969 static const bool color_write_disable[4] = { false, false, false, false };
970
971 struct blorp_batch batch;
972 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
973
974
975 for (unsigned r = 0; r < rangeCount; r++) {
976 if (pRanges[r].aspectMask == 0)
977 continue;
978
979 assert(pRanges[r].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
980
981 struct blorp_surf surf;
982 get_blorp_surf_for_anv_image(cmd_buffer->device,
983 image, pRanges[r].aspectMask,
984 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
985 imageLayout, ISL_AUX_USAGE_NONE, &surf);
986
987 struct anv_format_plane src_format =
988 anv_get_format_plane(&cmd_buffer->device->info, image->vk_format,
989 VK_IMAGE_ASPECT_COLOR_BIT, image->tiling);
990
991 unsigned base_layer = pRanges[r].baseArrayLayer;
992 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
993
994 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
995 const unsigned level = pRanges[r].baseMipLevel + i;
996 const unsigned level_width = anv_minify(image->extent.width, level);
997 const unsigned level_height = anv_minify(image->extent.height, level);
998
999 if (image->type == VK_IMAGE_TYPE_3D) {
1000 base_layer = 0;
1001 layer_count = anv_minify(image->extent.depth, level);
1002 }
1003
1004 anv_cmd_buffer_mark_image_written(cmd_buffer, image,
1005 pRanges[r].aspectMask,
1006 surf.aux_usage, level,
1007 base_layer, layer_count);
1008
1009 blorp_clear(&batch, &surf,
1010 src_format.isl_format, src_format.swizzle,
1011 level, base_layer, layer_count,
1012 0, 0, level_width, level_height,
1013 vk_to_isl_color(*pColor), color_write_disable);
1014 }
1015 }
1016
1017 blorp_batch_finish(&batch);
1018 }
1019
1020 void anv_CmdClearDepthStencilImage(
1021 VkCommandBuffer commandBuffer,
1022 VkImage image_h,
1023 VkImageLayout imageLayout,
1024 const VkClearDepthStencilValue* pDepthStencil,
1025 uint32_t rangeCount,
1026 const VkImageSubresourceRange* pRanges)
1027 {
1028 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1029 ANV_FROM_HANDLE(anv_image, image, image_h);
1030
1031 struct blorp_batch batch;
1032 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1033
1034 struct blorp_surf depth, stencil, stencil_shadow;
1035 if (image->aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1036 get_blorp_surf_for_anv_image(cmd_buffer->device,
1037 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1038 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
1039 imageLayout, ISL_AUX_USAGE_NONE, &depth);
1040 } else {
1041 memset(&depth, 0, sizeof(depth));
1042 }
1043
1044 bool has_stencil_shadow = false;
1045 if (image->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1046 get_blorp_surf_for_anv_image(cmd_buffer->device,
1047 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1048 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
1049 imageLayout, ISL_AUX_USAGE_NONE, &stencil);
1050
1051 has_stencil_shadow =
1052 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device, image,
1053 VK_IMAGE_ASPECT_STENCIL_BIT,
1054 &stencil_shadow);
1055 } else {
1056 memset(&stencil, 0, sizeof(stencil));
1057 }
1058
1059 for (unsigned r = 0; r < rangeCount; r++) {
1060 if (pRanges[r].aspectMask == 0)
1061 continue;
1062
1063 bool clear_depth = pRanges[r].aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
1064 bool clear_stencil = pRanges[r].aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
1065
1066 unsigned base_layer = pRanges[r].baseArrayLayer;
1067 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
1068
1069 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
1070 const unsigned level = pRanges[r].baseMipLevel + i;
1071 const unsigned level_width = anv_minify(image->extent.width, level);
1072 const unsigned level_height = anv_minify(image->extent.height, level);
1073
1074 if (image->type == VK_IMAGE_TYPE_3D)
1075 layer_count = anv_minify(image->extent.depth, level);
1076
1077 blorp_clear_depth_stencil(&batch, &depth, &stencil,
1078 level, base_layer, layer_count,
1079 0, 0, level_width, level_height,
1080 clear_depth, pDepthStencil->depth,
1081 clear_stencil ? 0xff : 0,
1082 pDepthStencil->stencil);
1083
1084 if (clear_stencil && has_stencil_shadow) {
1085 union isl_color_value stencil_color = {
1086 .u32 = { pDepthStencil->stencil, },
1087 };
1088 blorp_clear(&batch, &stencil_shadow,
1089 ISL_FORMAT_R8_UINT, ISL_SWIZZLE_IDENTITY,
1090 level, base_layer, layer_count,
1091 0, 0, level_width, level_height,
1092 stencil_color, NULL);
1093 }
1094 }
1095 }
1096
1097 blorp_batch_finish(&batch);
1098 }
1099
1100 VkResult
1101 anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer *cmd_buffer,
1102 uint32_t num_entries,
1103 uint32_t *state_offset,
1104 struct anv_state *bt_state)
1105 {
1106 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
1107 state_offset);
1108 if (bt_state->map == NULL) {
1109 /* We ran out of space. Grab a new binding table block. */
1110 VkResult result = anv_cmd_buffer_new_binding_table_block(cmd_buffer);
1111 if (result != VK_SUCCESS)
1112 return result;
1113
1114 /* Re-emit state base addresses so we get the new surface state base
1115 * address before we start emitting binding tables etc.
1116 */
1117 anv_cmd_buffer_emit_state_base_address(cmd_buffer);
1118
1119 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
1120 state_offset);
1121 assert(bt_state->map != NULL);
1122 }
1123
1124 return VK_SUCCESS;
1125 }
1126
1127 static VkResult
1128 binding_table_for_surface_state(struct anv_cmd_buffer *cmd_buffer,
1129 struct anv_state surface_state,
1130 uint32_t *bt_offset)
1131 {
1132 uint32_t state_offset;
1133 struct anv_state bt_state;
1134
1135 VkResult result =
1136 anv_cmd_buffer_alloc_blorp_binding_table(cmd_buffer, 1, &state_offset,
1137 &bt_state);
1138 if (result != VK_SUCCESS)
1139 return result;
1140
1141 uint32_t *bt_map = bt_state.map;
1142 bt_map[0] = surface_state.offset + state_offset;
1143
1144 *bt_offset = bt_state.offset;
1145 return VK_SUCCESS;
1146 }
1147
1148 static void
1149 clear_color_attachment(struct anv_cmd_buffer *cmd_buffer,
1150 struct blorp_batch *batch,
1151 const VkClearAttachment *attachment,
1152 uint32_t rectCount, const VkClearRect *pRects)
1153 {
1154 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
1155 const uint32_t color_att = attachment->colorAttachment;
1156 assert(color_att < subpass->color_count);
1157 const uint32_t att_idx = subpass->color_attachments[color_att].attachment;
1158
1159 if (att_idx == VK_ATTACHMENT_UNUSED)
1160 return;
1161
1162 struct anv_render_pass_attachment *pass_att =
1163 &cmd_buffer->state.pass->attachments[att_idx];
1164 struct anv_attachment_state *att_state =
1165 &cmd_buffer->state.attachments[att_idx];
1166
1167 uint32_t binding_table;
1168 VkResult result =
1169 binding_table_for_surface_state(cmd_buffer, att_state->color.state,
1170 &binding_table);
1171 if (result != VK_SUCCESS)
1172 return;
1173
1174 union isl_color_value clear_color =
1175 vk_to_isl_color(attachment->clearValue.color);
1176
1177 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1178 if (subpass->view_mask) {
1179 uint32_t view_idx;
1180 for_each_bit(view_idx, subpass->view_mask) {
1181 for (uint32_t r = 0; r < rectCount; ++r) {
1182 const VkOffset2D offset = pRects[r].rect.offset;
1183 const VkExtent2D extent = pRects[r].rect.extent;
1184 blorp_clear_attachments(batch, binding_table,
1185 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1186 view_idx, 1,
1187 offset.x, offset.y,
1188 offset.x + extent.width,
1189 offset.y + extent.height,
1190 true, clear_color, false, 0.0f, 0, 0);
1191 }
1192 }
1193 return;
1194 }
1195
1196 for (uint32_t r = 0; r < rectCount; ++r) {
1197 const VkOffset2D offset = pRects[r].rect.offset;
1198 const VkExtent2D extent = pRects[r].rect.extent;
1199 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1200 blorp_clear_attachments(batch, binding_table,
1201 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1202 pRects[r].baseArrayLayer,
1203 pRects[r].layerCount,
1204 offset.x, offset.y,
1205 offset.x + extent.width, offset.y + extent.height,
1206 true, clear_color, false, 0.0f, 0, 0);
1207 }
1208 }
1209
1210 static void
1211 clear_depth_stencil_attachment(struct anv_cmd_buffer *cmd_buffer,
1212 struct blorp_batch *batch,
1213 const VkClearAttachment *attachment,
1214 uint32_t rectCount, const VkClearRect *pRects)
1215 {
1216 static const union isl_color_value color_value = { .u32 = { 0, } };
1217 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
1218 if (!subpass->depth_stencil_attachment)
1219 return;
1220
1221 const uint32_t att_idx = subpass->depth_stencil_attachment->attachment;
1222 assert(att_idx != VK_ATTACHMENT_UNUSED);
1223 struct anv_render_pass_attachment *pass_att =
1224 &cmd_buffer->state.pass->attachments[att_idx];
1225
1226 bool clear_depth = attachment->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
1227 bool clear_stencil = attachment->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
1228
1229 enum isl_format depth_format = ISL_FORMAT_UNSUPPORTED;
1230 if (clear_depth) {
1231 depth_format = anv_get_isl_format(&cmd_buffer->device->info,
1232 pass_att->format,
1233 VK_IMAGE_ASPECT_DEPTH_BIT,
1234 VK_IMAGE_TILING_OPTIMAL);
1235 }
1236
1237 uint32_t binding_table;
1238 VkResult result =
1239 binding_table_for_surface_state(cmd_buffer,
1240 cmd_buffer->state.null_surface_state,
1241 &binding_table);
1242 if (result != VK_SUCCESS)
1243 return;
1244
1245 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1246 if (subpass->view_mask) {
1247 uint32_t view_idx;
1248 for_each_bit(view_idx, subpass->view_mask) {
1249 for (uint32_t r = 0; r < rectCount; ++r) {
1250 const VkOffset2D offset = pRects[r].rect.offset;
1251 const VkExtent2D extent = pRects[r].rect.extent;
1252 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1253 blorp_clear_attachments(batch, binding_table,
1254 depth_format, pass_att->samples,
1255 view_idx, 1,
1256 offset.x, offset.y,
1257 offset.x + extent.width,
1258 offset.y + extent.height,
1259 false, color_value,
1260 clear_depth, value.depth,
1261 clear_stencil ? 0xff : 0, value.stencil);
1262 }
1263 }
1264 return;
1265 }
1266
1267 for (uint32_t r = 0; r < rectCount; ++r) {
1268 const VkOffset2D offset = pRects[r].rect.offset;
1269 const VkExtent2D extent = pRects[r].rect.extent;
1270 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1271 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1272 blorp_clear_attachments(batch, binding_table,
1273 depth_format, pass_att->samples,
1274 pRects[r].baseArrayLayer,
1275 pRects[r].layerCount,
1276 offset.x, offset.y,
1277 offset.x + extent.width, offset.y + extent.height,
1278 false, color_value,
1279 clear_depth, value.depth,
1280 clear_stencil ? 0xff : 0, value.stencil);
1281 }
1282 }
1283
1284 void anv_CmdClearAttachments(
1285 VkCommandBuffer commandBuffer,
1286 uint32_t attachmentCount,
1287 const VkClearAttachment* pAttachments,
1288 uint32_t rectCount,
1289 const VkClearRect* pRects)
1290 {
1291 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1292
1293 /* Because this gets called within a render pass, we tell blorp not to
1294 * trash our depth and stencil buffers.
1295 */
1296 struct blorp_batch batch;
1297 enum blorp_batch_flags flags = BLORP_BATCH_NO_EMIT_DEPTH_STENCIL;
1298 if (cmd_buffer->state.conditional_render_enabled) {
1299 anv_cmd_emit_conditional_render_predicate(cmd_buffer);
1300 flags |= BLORP_BATCH_PREDICATE_ENABLE;
1301 }
1302 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, flags);
1303
1304 for (uint32_t a = 0; a < attachmentCount; ++a) {
1305 if (pAttachments[a].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
1306 assert(pAttachments[a].aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
1307 clear_color_attachment(cmd_buffer, &batch,
1308 &pAttachments[a],
1309 rectCount, pRects);
1310 } else {
1311 clear_depth_stencil_attachment(cmd_buffer, &batch,
1312 &pAttachments[a],
1313 rectCount, pRects);
1314 }
1315 }
1316
1317 blorp_batch_finish(&batch);
1318 }
1319
1320 enum subpass_stage {
1321 SUBPASS_STAGE_LOAD,
1322 SUBPASS_STAGE_DRAW,
1323 SUBPASS_STAGE_RESOLVE,
1324 };
1325
1326 void
1327 anv_image_msaa_resolve(struct anv_cmd_buffer *cmd_buffer,
1328 const struct anv_image *src_image,
1329 enum isl_aux_usage src_aux_usage,
1330 uint32_t src_level, uint32_t src_base_layer,
1331 const struct anv_image *dst_image,
1332 enum isl_aux_usage dst_aux_usage,
1333 uint32_t dst_level, uint32_t dst_base_layer,
1334 VkImageAspectFlagBits aspect,
1335 uint32_t src_x, uint32_t src_y,
1336 uint32_t dst_x, uint32_t dst_y,
1337 uint32_t width, uint32_t height,
1338 uint32_t layer_count,
1339 enum blorp_filter filter)
1340 {
1341 struct blorp_batch batch;
1342 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1343
1344 assert(src_image->type == VK_IMAGE_TYPE_2D);
1345 assert(src_image->samples > 1);
1346 assert(dst_image->type == VK_IMAGE_TYPE_2D);
1347 assert(dst_image->samples == 1);
1348 assert(src_image->n_planes == dst_image->n_planes);
1349 assert(!src_image->format->can_ycbcr);
1350 assert(!dst_image->format->can_ycbcr);
1351
1352 struct blorp_surf src_surf, dst_surf;
1353 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, aspect,
1354 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1355 src_aux_usage, &src_surf);
1356 if (src_aux_usage == ISL_AUX_USAGE_MCS) {
1357 src_surf.clear_color_addr = anv_to_blorp_address(
1358 anv_image_get_clear_color_addr(cmd_buffer->device, src_image,
1359 VK_IMAGE_ASPECT_COLOR_BIT));
1360 }
1361 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, aspect,
1362 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1363 dst_aux_usage, &dst_surf);
1364 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
1365 aspect, dst_aux_usage,
1366 dst_level, dst_base_layer, layer_count);
1367
1368 if (filter == BLORP_FILTER_NONE) {
1369 /* If no explicit filter is provided, then it's implied by the type of
1370 * the source image.
1371 */
1372 if ((src_surf.surf->usage & ISL_SURF_USAGE_DEPTH_BIT) ||
1373 (src_surf.surf->usage & ISL_SURF_USAGE_STENCIL_BIT) ||
1374 isl_format_has_int_channel(src_surf.surf->format)) {
1375 filter = BLORP_FILTER_SAMPLE_0;
1376 } else {
1377 filter = BLORP_FILTER_AVERAGE;
1378 }
1379 }
1380
1381 for (uint32_t l = 0; l < layer_count; l++) {
1382 blorp_blit(&batch,
1383 &src_surf, src_level, src_base_layer + l,
1384 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1385 &dst_surf, dst_level, dst_base_layer + l,
1386 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1387 src_x, src_y, src_x + width, src_y + height,
1388 dst_x, dst_y, dst_x + width, dst_y + height,
1389 filter, false, false);
1390 }
1391
1392 blorp_batch_finish(&batch);
1393 }
1394
1395 void anv_CmdResolveImage(
1396 VkCommandBuffer commandBuffer,
1397 VkImage srcImage,
1398 VkImageLayout srcImageLayout,
1399 VkImage dstImage,
1400 VkImageLayout dstImageLayout,
1401 uint32_t regionCount,
1402 const VkImageResolve* pRegions)
1403 {
1404 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1405 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
1406 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
1407
1408 assert(!src_image->format->can_ycbcr);
1409
1410 for (uint32_t r = 0; r < regionCount; r++) {
1411 assert(pRegions[r].srcSubresource.aspectMask ==
1412 pRegions[r].dstSubresource.aspectMask);
1413 assert(anv_get_layerCount(src_image, &pRegions[r].srcSubresource) ==
1414 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource));
1415
1416 const uint32_t layer_count =
1417 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource);
1418
1419 uint32_t aspect_bit;
1420 anv_foreach_image_aspect_bit(aspect_bit, src_image,
1421 pRegions[r].srcSubresource.aspectMask) {
1422 enum isl_aux_usage src_aux_usage =
1423 anv_layout_to_aux_usage(&cmd_buffer->device->info, src_image,
1424 (1 << aspect_bit),
1425 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
1426 srcImageLayout);
1427 enum isl_aux_usage dst_aux_usage =
1428 anv_layout_to_aux_usage(&cmd_buffer->device->info, dst_image,
1429 (1 << aspect_bit),
1430 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
1431 dstImageLayout);
1432
1433 anv_image_msaa_resolve(cmd_buffer,
1434 src_image, src_aux_usage,
1435 pRegions[r].srcSubresource.mipLevel,
1436 pRegions[r].srcSubresource.baseArrayLayer,
1437 dst_image, dst_aux_usage,
1438 pRegions[r].dstSubresource.mipLevel,
1439 pRegions[r].dstSubresource.baseArrayLayer,
1440 (1 << aspect_bit),
1441 pRegions[r].srcOffset.x,
1442 pRegions[r].srcOffset.y,
1443 pRegions[r].dstOffset.x,
1444 pRegions[r].dstOffset.y,
1445 pRegions[r].extent.width,
1446 pRegions[r].extent.height,
1447 layer_count, BLORP_FILTER_NONE);
1448 }
1449 }
1450 }
1451
1452 void
1453 anv_image_copy_to_shadow(struct anv_cmd_buffer *cmd_buffer,
1454 const struct anv_image *image,
1455 VkImageAspectFlagBits aspect,
1456 uint32_t base_level, uint32_t level_count,
1457 uint32_t base_layer, uint32_t layer_count)
1458 {
1459 struct blorp_batch batch;
1460 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1461
1462 /* We don't know who touched the main surface last so flush a bunch of
1463 * caches to ensure we get good data.
1464 */
1465 cmd_buffer->state.pending_pipe_bits |=
1466 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT |
1467 ANV_PIPE_DATA_CACHE_FLUSH_BIT |
1468 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT |
1469 ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
1470
1471 struct blorp_surf surf;
1472 get_blorp_surf_for_anv_image(cmd_buffer->device,
1473 image, aspect,
1474 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
1475 VK_IMAGE_LAYOUT_GENERAL,
1476 ISL_AUX_USAGE_NONE, &surf);
1477 assert(surf.aux_usage == ISL_AUX_USAGE_NONE);
1478
1479 struct blorp_surf shadow_surf;
1480 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
1481 image, aspect, &shadow_surf);
1482
1483 for (uint32_t l = 0; l < level_count; l++) {
1484 const uint32_t level = base_level + l;
1485
1486 const VkExtent3D extent = {
1487 .width = anv_minify(image->extent.width, level),
1488 .height = anv_minify(image->extent.height, level),
1489 .depth = anv_minify(image->extent.depth, level),
1490 };
1491
1492 if (image->type == VK_IMAGE_TYPE_3D)
1493 layer_count = extent.depth;
1494
1495 for (uint32_t a = 0; a < layer_count; a++) {
1496 const uint32_t layer = base_layer + a;
1497
1498 blorp_copy(&batch, &surf, level, layer,
1499 &shadow_surf, level, layer,
1500 0, 0, 0, 0, extent.width, extent.height);
1501 }
1502 }
1503
1504 /* We just wrote to the buffer with the render cache. Flush it. */
1505 cmd_buffer->state.pending_pipe_bits |=
1506 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
1507
1508 blorp_batch_finish(&batch);
1509 }
1510
1511 void
1512 anv_image_clear_color(struct anv_cmd_buffer *cmd_buffer,
1513 const struct anv_image *image,
1514 VkImageAspectFlagBits aspect,
1515 enum isl_aux_usage aux_usage,
1516 enum isl_format format, struct isl_swizzle swizzle,
1517 uint32_t level, uint32_t base_layer, uint32_t layer_count,
1518 VkRect2D area, union isl_color_value clear_color)
1519 {
1520 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1521
1522 /* We don't support planar images with multisampling yet */
1523 assert(image->n_planes == 1);
1524
1525 struct blorp_batch batch;
1526 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1527
1528 struct blorp_surf surf;
1529 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1530 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1531 aux_usage, &surf);
1532 anv_cmd_buffer_mark_image_written(cmd_buffer, image, aspect, aux_usage,
1533 level, base_layer, layer_count);
1534
1535 blorp_clear(&batch, &surf, format, anv_swizzle_for_render(swizzle),
1536 level, base_layer, layer_count,
1537 area.offset.x, area.offset.y,
1538 area.offset.x + area.extent.width,
1539 area.offset.y + area.extent.height,
1540 clear_color, NULL);
1541
1542 blorp_batch_finish(&batch);
1543 }
1544
1545 void
1546 anv_image_clear_depth_stencil(struct anv_cmd_buffer *cmd_buffer,
1547 const struct anv_image *image,
1548 VkImageAspectFlags aspects,
1549 enum isl_aux_usage depth_aux_usage,
1550 uint32_t level,
1551 uint32_t base_layer, uint32_t layer_count,
1552 VkRect2D area,
1553 float depth_value, uint8_t stencil_value)
1554 {
1555 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
1556 VK_IMAGE_ASPECT_STENCIL_BIT));
1557
1558 struct blorp_batch batch;
1559 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1560
1561 struct blorp_surf depth = {};
1562 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1563 get_blorp_surf_for_anv_image(cmd_buffer->device,
1564 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1565 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1566 depth_aux_usage, &depth);
1567 depth.clear_color.f32[0] = ANV_HZ_FC_VAL;
1568 }
1569
1570 struct blorp_surf stencil = {};
1571 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1572 get_blorp_surf_for_anv_image(cmd_buffer->device,
1573 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1574 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1575 ISL_AUX_USAGE_NONE, &stencil);
1576 }
1577
1578 /* Blorp may choose to clear stencil using RGBA32_UINT for better
1579 * performance. If it does this, we need to flush it out of the depth
1580 * cache before rendering to it.
1581 */
1582 cmd_buffer->state.pending_pipe_bits |=
1583 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_END_OF_PIPE_SYNC_BIT;
1584
1585 blorp_clear_depth_stencil(&batch, &depth, &stencil,
1586 level, base_layer, layer_count,
1587 area.offset.x, area.offset.y,
1588 area.offset.x + area.extent.width,
1589 area.offset.y + area.extent.height,
1590 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
1591 depth_value,
1592 (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) ? 0xff : 0,
1593 stencil_value);
1594
1595 /* Blorp may choose to clear stencil using RGBA32_UINT for better
1596 * performance. If it does this, we need to flush it out of the render
1597 * cache before someone starts trying to do stencil on it.
1598 */
1599 cmd_buffer->state.pending_pipe_bits |=
1600 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_END_OF_PIPE_SYNC_BIT;
1601
1602 struct blorp_surf stencil_shadow;
1603 if ((aspects & VK_IMAGE_ASPECT_STENCIL_BIT) &&
1604 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device, image,
1605 VK_IMAGE_ASPECT_STENCIL_BIT,
1606 &stencil_shadow)) {
1607 union isl_color_value stencil_color = {
1608 .u32 = { stencil_value },
1609 };
1610 blorp_clear(&batch, &stencil_shadow,
1611 ISL_FORMAT_R8_UINT, ISL_SWIZZLE_IDENTITY,
1612 level, base_layer, layer_count,
1613 area.offset.x, area.offset.y,
1614 area.offset.x + area.extent.width,
1615 area.offset.y + area.extent.height,
1616 stencil_color, NULL);
1617 }
1618
1619 blorp_batch_finish(&batch);
1620 }
1621
1622 void
1623 anv_image_hiz_op(struct anv_cmd_buffer *cmd_buffer,
1624 const struct anv_image *image,
1625 VkImageAspectFlagBits aspect, uint32_t level,
1626 uint32_t base_layer, uint32_t layer_count,
1627 enum isl_aux_op hiz_op)
1628 {
1629 assert(aspect == VK_IMAGE_ASPECT_DEPTH_BIT);
1630 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, level));
1631 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
1632 assert(plane == 0);
1633
1634 struct blorp_batch batch;
1635 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1636
1637 struct blorp_surf surf;
1638 get_blorp_surf_for_anv_image(cmd_buffer->device,
1639 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1640 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1641 image->planes[plane].aux_usage, &surf);
1642 surf.clear_color.f32[0] = ANV_HZ_FC_VAL;
1643
1644 blorp_hiz_op(&batch, &surf, level, base_layer, layer_count, hiz_op);
1645
1646 blorp_batch_finish(&batch);
1647 }
1648
1649 void
1650 anv_image_hiz_clear(struct anv_cmd_buffer *cmd_buffer,
1651 const struct anv_image *image,
1652 VkImageAspectFlags aspects,
1653 uint32_t level,
1654 uint32_t base_layer, uint32_t layer_count,
1655 VkRect2D area, uint8_t stencil_value)
1656 {
1657 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
1658 VK_IMAGE_ASPECT_STENCIL_BIT));
1659
1660 struct blorp_batch batch;
1661 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1662
1663 struct blorp_surf depth = {};
1664 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1665 uint32_t plane = anv_image_aspect_to_plane(image->aspects,
1666 VK_IMAGE_ASPECT_DEPTH_BIT);
1667 assert(base_layer + layer_count <=
1668 anv_image_aux_layers(image, VK_IMAGE_ASPECT_DEPTH_BIT, level));
1669 get_blorp_surf_for_anv_image(cmd_buffer->device,
1670 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1671 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1672 image->planes[plane].aux_usage, &depth);
1673 depth.clear_color.f32[0] = ANV_HZ_FC_VAL;
1674 }
1675
1676 struct blorp_surf stencil = {};
1677 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1678 uint32_t plane = anv_image_aspect_to_plane(image->aspects,
1679 VK_IMAGE_ASPECT_STENCIL_BIT);
1680 get_blorp_surf_for_anv_image(cmd_buffer->device,
1681 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1682 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1683 image->planes[plane].aux_usage, &stencil);
1684 }
1685
1686 /* From the Sky Lake PRM Volume 7, "Depth Buffer Clear":
1687 *
1688 * "The following is required when performing a depth buffer clear with
1689 * using the WM_STATE or 3DSTATE_WM:
1690 *
1691 * * If other rendering operations have preceded this clear, a
1692 * PIPE_CONTROL with depth cache flush enabled, Depth Stall bit
1693 * enabled must be issued before the rectangle primitive used for
1694 * the depth buffer clear operation.
1695 * * [...]"
1696 *
1697 * Even though the PRM only says that this is required if using 3DSTATE_WM
1698 * and a 3DPRIMITIVE, the GPU appears to also need this to avoid occasional
1699 * hangs when doing a clear with WM_HZ_OP.
1700 */
1701 cmd_buffer->state.pending_pipe_bits |=
1702 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT;
1703
1704 blorp_hiz_clear_depth_stencil(&batch, &depth, &stencil,
1705 level, base_layer, layer_count,
1706 area.offset.x, area.offset.y,
1707 area.offset.x + area.extent.width,
1708 area.offset.y + area.extent.height,
1709 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
1710 ANV_HZ_FC_VAL,
1711 aspects & VK_IMAGE_ASPECT_STENCIL_BIT,
1712 stencil_value);
1713
1714 blorp_batch_finish(&batch);
1715
1716 /* From the SKL PRM, Depth Buffer Clear:
1717 *
1718 * "Depth Buffer Clear Workaround
1719 *
1720 * Depth buffer clear pass using any of the methods (WM_STATE,
1721 * 3DSTATE_WM or 3DSTATE_WM_HZ_OP) must be followed by a PIPE_CONTROL
1722 * command with DEPTH_STALL bit and Depth FLUSH bits “set” before
1723 * starting to render. DepthStall and DepthFlush are not needed between
1724 * consecutive depth clear passes nor is it required if the depth-clear
1725 * pass was done with “full_surf_clear” bit set in the
1726 * 3DSTATE_WM_HZ_OP."
1727 *
1728 * Even though the PRM provides a bunch of conditions under which this is
1729 * supposedly unnecessary, we choose to perform the flush unconditionally
1730 * just to be safe.
1731 */
1732 cmd_buffer->state.pending_pipe_bits |=
1733 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT;
1734 }
1735
1736 void
1737 anv_image_mcs_op(struct anv_cmd_buffer *cmd_buffer,
1738 const struct anv_image *image,
1739 enum isl_format format, struct isl_swizzle swizzle,
1740 VkImageAspectFlagBits aspect,
1741 uint32_t base_layer, uint32_t layer_count,
1742 enum isl_aux_op mcs_op, union isl_color_value *clear_value,
1743 bool predicate)
1744 {
1745 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1746 assert(image->samples > 1);
1747 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, 0));
1748
1749 /* Multisampling with multi-planar formats is not supported */
1750 assert(image->n_planes == 1);
1751
1752 struct blorp_batch batch;
1753 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1754 BLORP_BATCH_PREDICATE_ENABLE * predicate +
1755 BLORP_BATCH_NO_UPDATE_CLEAR_COLOR * !clear_value);
1756
1757 struct blorp_surf surf;
1758 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1759 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1760 ISL_AUX_USAGE_MCS, &surf);
1761
1762 /* Blorp will store the clear color for us if we provide the clear color
1763 * address and we are doing a fast clear. So we save the clear value into
1764 * the blorp surface.
1765 */
1766 if (clear_value)
1767 surf.clear_color = *clear_value;
1768
1769 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1770 *
1771 * "After Render target fast clear, pipe-control with color cache
1772 * write-flush must be issued before sending any DRAW commands on
1773 * that render target."
1774 *
1775 * This comment is a bit cryptic and doesn't really tell you what's going
1776 * or what's really needed. It appears that fast clear ops are not
1777 * properly synchronized with other drawing. This means that we cannot
1778 * have a fast clear operation in the pipe at the same time as other
1779 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
1780 * that the contents of the previous draw hit the render target before we
1781 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
1782 * that it is completed before any additional drawing occurs.
1783 */
1784 cmd_buffer->state.pending_pipe_bits |=
1785 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_END_OF_PIPE_SYNC_BIT;
1786
1787 switch (mcs_op) {
1788 case ISL_AUX_OP_FAST_CLEAR:
1789 blorp_fast_clear(&batch, &surf, format, swizzle,
1790 0, base_layer, layer_count,
1791 0, 0, image->extent.width, image->extent.height);
1792 break;
1793 case ISL_AUX_OP_PARTIAL_RESOLVE:
1794 blorp_mcs_partial_resolve(&batch, &surf, format,
1795 base_layer, layer_count);
1796 break;
1797 case ISL_AUX_OP_FULL_RESOLVE:
1798 case ISL_AUX_OP_AMBIGUATE:
1799 default:
1800 unreachable("Unsupported MCS operation");
1801 }
1802
1803 cmd_buffer->state.pending_pipe_bits |=
1804 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_END_OF_PIPE_SYNC_BIT;
1805
1806 blorp_batch_finish(&batch);
1807 }
1808
1809 void
1810 anv_image_ccs_op(struct anv_cmd_buffer *cmd_buffer,
1811 const struct anv_image *image,
1812 enum isl_format format, struct isl_swizzle swizzle,
1813 VkImageAspectFlagBits aspect, uint32_t level,
1814 uint32_t base_layer, uint32_t layer_count,
1815 enum isl_aux_op ccs_op, union isl_color_value *clear_value,
1816 bool predicate)
1817 {
1818 assert(image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
1819 assert(image->samples == 1);
1820 assert(level < anv_image_aux_levels(image, aspect));
1821 /* Multi-LOD YcBcR is not allowed */
1822 assert(image->n_planes == 1 || level == 0);
1823 assert(base_layer + layer_count <=
1824 anv_image_aux_layers(image, aspect, level));
1825
1826 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
1827 uint32_t width_div = image->format->planes[plane].denominator_scales[0];
1828 uint32_t height_div = image->format->planes[plane].denominator_scales[1];
1829 uint32_t level_width = anv_minify(image->extent.width, level) / width_div;
1830 uint32_t level_height = anv_minify(image->extent.height, level) / height_div;
1831
1832 struct blorp_batch batch;
1833 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1834 BLORP_BATCH_PREDICATE_ENABLE * predicate +
1835 BLORP_BATCH_NO_UPDATE_CLEAR_COLOR * !clear_value);
1836
1837 struct blorp_surf surf;
1838 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1839 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1840 image->planes[plane].aux_usage,
1841 &surf);
1842
1843 /* Blorp will store the clear color for us if we provide the clear color
1844 * address and we are doing a fast clear. So we save the clear value into
1845 * the blorp surface.
1846 */
1847 if (clear_value)
1848 surf.clear_color = *clear_value;
1849
1850 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1851 *
1852 * "After Render target fast clear, pipe-control with color cache
1853 * write-flush must be issued before sending any DRAW commands on
1854 * that render target."
1855 *
1856 * This comment is a bit cryptic and doesn't really tell you what's going
1857 * or what's really needed. It appears that fast clear ops are not
1858 * properly synchronized with other drawing. This means that we cannot
1859 * have a fast clear operation in the pipe at the same time as other
1860 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
1861 * that the contents of the previous draw hit the render target before we
1862 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
1863 * that it is completed before any additional drawing occurs.
1864 */
1865 cmd_buffer->state.pending_pipe_bits |=
1866 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_END_OF_PIPE_SYNC_BIT;
1867
1868 switch (ccs_op) {
1869 case ISL_AUX_OP_FAST_CLEAR:
1870 blorp_fast_clear(&batch, &surf, format, swizzle,
1871 level, base_layer, layer_count,
1872 0, 0, level_width, level_height);
1873 break;
1874 case ISL_AUX_OP_FULL_RESOLVE:
1875 case ISL_AUX_OP_PARTIAL_RESOLVE:
1876 blorp_ccs_resolve(&batch, &surf, level, base_layer, layer_count,
1877 format, ccs_op);
1878 break;
1879 case ISL_AUX_OP_AMBIGUATE:
1880 for (uint32_t a = 0; a < layer_count; a++) {
1881 const uint32_t layer = base_layer + a;
1882 blorp_ccs_ambiguate(&batch, &surf, level, layer);
1883 }
1884 break;
1885 default:
1886 unreachable("Unsupported CCS operation");
1887 }
1888
1889 cmd_buffer->state.pending_pipe_bits |=
1890 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_END_OF_PIPE_SYNC_BIT;
1891
1892 blorp_batch_finish(&batch);
1893 }