projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
anv: Add an anv_physical_device field to anv_device
[mesa.git] / src / intel / vulkan / anv_blorp.c
1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "anv_private.h"
25
26 static bool
27 lookup_blorp_shader(struct blorp_batch *batch,
28 const void *key, uint32_t key_size,
29 uint32_t *kernel_out, void *prog_data_out)
30 {
31 struct blorp_context *blorp = batch->blorp;
32 struct anv_device *device = blorp->driver_ctx;
33
34 /* The default cache must be a real cache */
35 assert(device->default_pipeline_cache.cache);
36
37 struct anv_shader_bin *bin =
38 anv_pipeline_cache_search(&device->default_pipeline_cache, key, key_size);
39 if (!bin)
40 return false;
41
42 /* The cache already has a reference and it's not going anywhere so there
43 * is no need to hold a second reference.
44 */
45 anv_shader_bin_unref(device, bin);
46
47 *kernel_out = bin->kernel.offset;
48 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
49
50 return true;
51 }
52
53 static bool
54 upload_blorp_shader(struct blorp_batch *batch,
55 const void *key, uint32_t key_size,
56 const void *kernel, uint32_t kernel_size,
57 const struct brw_stage_prog_data *prog_data,
58 uint32_t prog_data_size,
59 uint32_t *kernel_out, void *prog_data_out)
60 {
61 struct blorp_context *blorp = batch->blorp;
62 struct anv_device *device = blorp->driver_ctx;
63
64 /* The blorp cache must be a real cache */
65 assert(device->default_pipeline_cache.cache);
66
67 struct anv_pipeline_bind_map bind_map = {
68 .surface_count = 0,
69 .sampler_count = 0,
70 };
71
72 struct anv_shader_bin *bin =
73 anv_pipeline_cache_upload_kernel(&device->default_pipeline_cache,
74 key, key_size, kernel, kernel_size,
75 NULL, 0,
76 prog_data, prog_data_size,
77 NULL, 0, NULL, &bind_map);
78
79 if (!bin)
80 return false;
81
82 /* The cache already has a reference and it's not going anywhere so there
83 * is no need to hold a second reference.
84 */
85 anv_shader_bin_unref(device, bin);
86
87 *kernel_out = bin->kernel.offset;
88 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
89
90 return true;
91 }
92
93 void
94 anv_device_init_blorp(struct anv_device *device)
95 {
96 blorp_init(&device->blorp, device, &device->isl_dev);
97 device->blorp.compiler = device->physical->compiler;
98 device->blorp.lookup_shader = lookup_blorp_shader;
99 device->blorp.upload_shader = upload_blorp_shader;
100 switch (device->info.gen) {
101 case 7:
102 if (device->info.is_haswell) {
103 device->blorp.exec = gen75_blorp_exec;
104 } else {
105 device->blorp.exec = gen7_blorp_exec;
106 }
107 break;
108 case 8:
109 device->blorp.exec = gen8_blorp_exec;
110 break;
111 case 9:
112 device->blorp.exec = gen9_blorp_exec;
113 break;
114 case 10:
115 device->blorp.exec = gen10_blorp_exec;
116 break;
117 case 11:
118 device->blorp.exec = gen11_blorp_exec;
119 break;
120 case 12:
121 device->blorp.exec = gen12_blorp_exec;
122 break;
123 default:
124 unreachable("Unknown hardware generation");
125 }
126 }
127
128 void
129 anv_device_finish_blorp(struct anv_device *device)
130 {
131 blorp_finish(&device->blorp);
132 }
133
134 static void
135 get_blorp_surf_for_anv_buffer(struct anv_device *device,
136 struct anv_buffer *buffer, uint64_t offset,
137 uint32_t width, uint32_t height,
138 uint32_t row_pitch, enum isl_format format,
139 struct blorp_surf *blorp_surf,
140 struct isl_surf *isl_surf)
141 {
142 const struct isl_format_layout *fmtl =
143 isl_format_get_layout(format);
144 bool ok UNUSED;
145
146 /* ASTC is the only format which doesn't support linear layouts.
147 * Create an equivalently sized surface with ISL to get around this.
148 */
149 if (fmtl->txc == ISL_TXC_ASTC) {
150 /* Use an equivalently sized format */
151 format = ISL_FORMAT_R32G32B32A32_UINT;
152 assert(fmtl->bpb == isl_format_get_layout(format)->bpb);
153
154 /* Shrink the dimensions for the new format */
155 width = DIV_ROUND_UP(width, fmtl->bw);
156 height = DIV_ROUND_UP(height, fmtl->bh);
157 }
158
159 *blorp_surf = (struct blorp_surf) {
160 .surf = isl_surf,
161 .addr = {
162 .buffer = buffer->address.bo,
163 .offset = buffer->address.offset + offset,
164 .mocs = anv_mocs_for_bo(device, buffer->address.bo),
165 },
166 };
167
168 ok = isl_surf_init(&device->isl_dev, isl_surf,
169 .dim = ISL_SURF_DIM_2D,
170 .format = format,
171 .width = width,
172 .height = height,
173 .depth = 1,
174 .levels = 1,
175 .array_len = 1,
176 .samples = 1,
177 .row_pitch_B = row_pitch,
178 .usage = ISL_SURF_USAGE_TEXTURE_BIT |
179 ISL_SURF_USAGE_RENDER_TARGET_BIT,
180 .tiling_flags = ISL_TILING_LINEAR_BIT);
181 assert(ok);
182 }
183
184 /* Pick something high enough that it won't be used in core and low enough it
185 * will never map to an extension.
186 */
187 #define ANV_IMAGE_LAYOUT_EXPLICIT_AUX (VkImageLayout)10000000
188
189 static struct blorp_address
190 anv_to_blorp_address(struct anv_address addr)
191 {
192 return (struct blorp_address) {
193 .buffer = addr.bo,
194 .offset = addr.offset,
195 };
196 }
197
198 static void
199 get_blorp_surf_for_anv_image(const struct anv_device *device,
200 const struct anv_image *image,
201 VkImageAspectFlags aspect,
202 VkImageLayout layout,
203 enum isl_aux_usage aux_usage,
204 struct blorp_surf *blorp_surf)
205 {
206 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
207
208 if (layout != ANV_IMAGE_LAYOUT_EXPLICIT_AUX)
209 aux_usage = anv_layout_to_aux_usage(&device->info, image, aspect, layout);
210
211 const struct anv_surface *surface = &image->planes[plane].surface;
212 *blorp_surf = (struct blorp_surf) {
213 .surf = &surface->isl,
214 .addr = {
215 .buffer = image->planes[plane].address.bo,
216 .offset = image->planes[plane].address.offset + surface->offset,
217 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
218 },
219 };
220
221 if (aux_usage != ISL_AUX_USAGE_NONE) {
222 const struct anv_surface *aux_surface = &image->planes[plane].aux_surface;
223 blorp_surf->aux_surf = &aux_surface->isl,
224 blorp_surf->aux_addr = (struct blorp_address) {
225 .buffer = image->planes[plane].address.bo,
226 .offset = image->planes[plane].address.offset + aux_surface->offset,
227 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
228 };
229 blorp_surf->aux_usage = aux_usage;
230
231 /* If we're doing a partial resolve, then we need the indirect clear
232 * color. If we are doing a fast clear and want to store/update the
233 * clear color, we also pass the address to blorp, otherwise it will only
234 * stomp the CCS to a particular value and won't care about format or
235 * clear value
236 */
237 if (aspect & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
238 const struct anv_address clear_color_addr =
239 anv_image_get_clear_color_addr(device, image, aspect);
240 blorp_surf->clear_color_addr = anv_to_blorp_address(clear_color_addr);
241 } else if (aspect & VK_IMAGE_ASPECT_DEPTH_BIT) {
242 if (device->info.gen >= 10) {
243 /* Vulkan always clears to 1.0. On gen < 10, we set that directly
244 * in the state packet. For gen >= 10, must provide the clear
245 * value in a buffer. We have a single global buffer that stores
246 * the 1.0 value.
247 */
248 const struct anv_address clear_color_addr = (struct anv_address) {
249 .bo = device->hiz_clear_bo,
250 };
251 blorp_surf->clear_color_addr =
252 anv_to_blorp_address(clear_color_addr);
253 } else {
254 blorp_surf->clear_color = (union isl_color_value) {
255 .f32 = { ANV_HZ_FC_VAL },
256 };
257 }
258 }
259 }
260 }
261
262 static bool
263 get_blorp_surf_for_anv_shadow_image(const struct anv_device *device,
264 const struct anv_image *image,
265 VkImageAspectFlags aspect,
266 struct blorp_surf *blorp_surf)
267 {
268
269 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
270 if (image->planes[plane].shadow_surface.isl.size_B == 0)
271 return false;
272
273 *blorp_surf = (struct blorp_surf) {
274 .surf = &image->planes[plane].shadow_surface.isl,
275 .addr = {
276 .buffer = image->planes[plane].address.bo,
277 .offset = image->planes[plane].address.offset +
278 image->planes[plane].shadow_surface.offset,
279 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
280 },
281 };
282
283 return true;
284 }
285
286 void anv_CmdCopyImage(
287 VkCommandBuffer commandBuffer,
288 VkImage srcImage,
289 VkImageLayout srcImageLayout,
290 VkImage dstImage,
291 VkImageLayout dstImageLayout,
292 uint32_t regionCount,
293 const VkImageCopy* pRegions)
294 {
295 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
296 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
297 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
298
299 struct blorp_batch batch;
300 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
301
302 for (unsigned r = 0; r < regionCount; r++) {
303 VkOffset3D srcOffset =
304 anv_sanitize_image_offset(src_image->type, pRegions[r].srcOffset);
305 VkOffset3D dstOffset =
306 anv_sanitize_image_offset(dst_image->type, pRegions[r].dstOffset);
307 VkExtent3D extent =
308 anv_sanitize_image_extent(src_image->type, pRegions[r].extent);
309
310 const uint32_t dst_level = pRegions[r].dstSubresource.mipLevel;
311 unsigned dst_base_layer, layer_count;
312 if (dst_image->type == VK_IMAGE_TYPE_3D) {
313 dst_base_layer = pRegions[r].dstOffset.z;
314 layer_count = pRegions[r].extent.depth;
315 } else {
316 dst_base_layer = pRegions[r].dstSubresource.baseArrayLayer;
317 layer_count =
318 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource);
319 }
320
321 const uint32_t src_level = pRegions[r].srcSubresource.mipLevel;
322 unsigned src_base_layer;
323 if (src_image->type == VK_IMAGE_TYPE_3D) {
324 src_base_layer = pRegions[r].srcOffset.z;
325 } else {
326 src_base_layer = pRegions[r].srcSubresource.baseArrayLayer;
327 assert(layer_count ==
328 anv_get_layerCount(src_image, &pRegions[r].srcSubresource));
329 }
330
331 VkImageAspectFlags src_mask = pRegions[r].srcSubresource.aspectMask,
332 dst_mask = pRegions[r].dstSubresource.aspectMask;
333
334 assert(anv_image_aspects_compatible(src_mask, dst_mask));
335
336 if (util_bitcount(src_mask) > 1) {
337 uint32_t aspect_bit;
338 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_mask) {
339 struct blorp_surf src_surf, dst_surf;
340 get_blorp_surf_for_anv_image(cmd_buffer->device,
341 src_image, 1UL << aspect_bit,
342 srcImageLayout, ISL_AUX_USAGE_NONE,
343 &src_surf);
344 get_blorp_surf_for_anv_image(cmd_buffer->device,
345 dst_image, 1UL << aspect_bit,
346 dstImageLayout, ISL_AUX_USAGE_NONE,
347 &dst_surf);
348 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
349 1UL << aspect_bit,
350 dst_surf.aux_usage, dst_level,
351 dst_base_layer, layer_count);
352
353 for (unsigned i = 0; i < layer_count; i++) {
354 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
355 &dst_surf, dst_level, dst_base_layer + i,
356 srcOffset.x, srcOffset.y,
357 dstOffset.x, dstOffset.y,
358 extent.width, extent.height);
359 }
360
361 struct blorp_surf dst_shadow_surf;
362 if (get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
363 dst_image,
364 1UL << aspect_bit,
365 &dst_shadow_surf)) {
366 for (unsigned i = 0; i < layer_count; i++) {
367 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
368 &dst_shadow_surf, dst_level, dst_base_layer + i,
369 srcOffset.x, srcOffset.y,
370 dstOffset.x, dstOffset.y,
371 extent.width, extent.height);
372 }
373 }
374 }
375 } else {
376 struct blorp_surf src_surf, dst_surf;
377 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, src_mask,
378 srcImageLayout, ISL_AUX_USAGE_NONE,
379 &src_surf);
380 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, dst_mask,
381 dstImageLayout, ISL_AUX_USAGE_NONE,
382 &dst_surf);
383 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image, dst_mask,
384 dst_surf.aux_usage, dst_level,
385 dst_base_layer, layer_count);
386
387 for (unsigned i = 0; i < layer_count; i++) {
388 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
389 &dst_surf, dst_level, dst_base_layer + i,
390 srcOffset.x, srcOffset.y,
391 dstOffset.x, dstOffset.y,
392 extent.width, extent.height);
393 }
394
395 struct blorp_surf dst_shadow_surf;
396 if (get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
397 dst_image, dst_mask,
398 &dst_shadow_surf)) {
399 for (unsigned i = 0; i < layer_count; i++) {
400 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
401 &dst_shadow_surf, dst_level, dst_base_layer + i,
402 srcOffset.x, srcOffset.y,
403 dstOffset.x, dstOffset.y,
404 extent.width, extent.height);
405 }
406 }
407 }
408 }
409
410 blorp_batch_finish(&batch);
411 }
412
413 static enum isl_format
414 isl_format_for_size(unsigned size_B)
415 {
416 /* Prefer 32-bit per component formats for CmdFillBuffer */
417 switch (size_B) {
418 case 1: return ISL_FORMAT_R8_UINT;
419 case 2: return ISL_FORMAT_R16_UINT;
420 case 3: return ISL_FORMAT_R8G8B8_UINT;
421 case 4: return ISL_FORMAT_R32_UINT;
422 case 6: return ISL_FORMAT_R16G16B16_UINT;
423 case 8: return ISL_FORMAT_R32G32_UINT;
424 case 12: return ISL_FORMAT_R32G32B32_UINT;
425 case 16: return ISL_FORMAT_R32G32B32A32_UINT;
426 default:
427 unreachable("Unknown format size");
428 }
429 }
430
431 static void
432 copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer,
433 struct anv_buffer *anv_buffer,
434 struct anv_image *anv_image,
435 VkImageLayout image_layout,
436 uint32_t regionCount,
437 const VkBufferImageCopy* pRegions,
438 bool buffer_to_image)
439 {
440 struct blorp_batch batch;
441 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
442
443 struct {
444 struct blorp_surf surf;
445 uint32_t level;
446 VkOffset3D offset;
447 } image, buffer, *src, *dst;
448
449 buffer.level = 0;
450 buffer.offset = (VkOffset3D) { 0, 0, 0 };
451
452 if (buffer_to_image) {
453 src = &buffer;
454 dst = &image;
455 } else {
456 src = &image;
457 dst = &buffer;
458 }
459
460 for (unsigned r = 0; r < regionCount; r++) {
461 const VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask;
462
463 get_blorp_surf_for_anv_image(cmd_buffer->device, anv_image, aspect,
464 image_layout, ISL_AUX_USAGE_NONE,
465 &image.surf);
466 image.offset =
467 anv_sanitize_image_offset(anv_image->type, pRegions[r].imageOffset);
468 image.level = pRegions[r].imageSubresource.mipLevel;
469
470 VkExtent3D extent =
471 anv_sanitize_image_extent(anv_image->type, pRegions[r].imageExtent);
472 if (anv_image->type != VK_IMAGE_TYPE_3D) {
473 image.offset.z = pRegions[r].imageSubresource.baseArrayLayer;
474 extent.depth =
475 anv_get_layerCount(anv_image, &pRegions[r].imageSubresource);
476 }
477
478 const enum isl_format linear_format =
479 anv_get_isl_format(&cmd_buffer->device->info, anv_image->vk_format,
480 aspect, VK_IMAGE_TILING_LINEAR);
481 const struct isl_format_layout *linear_fmtl =
482 isl_format_get_layout(linear_format);
483
484 const uint32_t buffer_row_length =
485 pRegions[r].bufferRowLength ?
486 pRegions[r].bufferRowLength : extent.width;
487
488 const uint32_t buffer_image_height =
489 pRegions[r].bufferImageHeight ?
490 pRegions[r].bufferImageHeight : extent.height;
491
492 const uint32_t buffer_row_pitch =
493 DIV_ROUND_UP(buffer_row_length, linear_fmtl->bw) *
494 (linear_fmtl->bpb / 8);
495
496 const uint32_t buffer_layer_stride =
497 DIV_ROUND_UP(buffer_image_height, linear_fmtl->bh) *
498 buffer_row_pitch;
499
500 /* Some formats have additional restrictions which may cause ISL to
501 * fail to create a surface for us. Some examples include:
502 *
503 * 1. ASTC formats are not allowed to be LINEAR and must be tiled
504 * 2. YCbCr formats have to have 2-pixel aligned strides
505 *
506 * To avoid these issues, we always bind the buffer as if it's a
507 * "normal" format like RGBA32_UINT. Since we're using blorp_copy,
508 * the format doesn't matter as long as it has the right bpb.
509 */
510 const VkExtent2D buffer_extent = {
511 .width = DIV_ROUND_UP(extent.width, linear_fmtl->bw),
512 .height = DIV_ROUND_UP(extent.height, linear_fmtl->bh),
513 };
514 const enum isl_format buffer_format =
515 isl_format_for_size(linear_fmtl->bpb / 8);
516
517 struct isl_surf buffer_isl_surf;
518 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
519 anv_buffer, pRegions[r].bufferOffset,
520 buffer_extent.width, buffer_extent.height,
521 buffer_row_pitch, buffer_format,
522 &buffer.surf, &buffer_isl_surf);
523
524 bool dst_has_shadow = false;
525 struct blorp_surf dst_shadow_surf;
526 if (&image == dst) {
527 anv_cmd_buffer_mark_image_written(cmd_buffer, anv_image,
528 aspect, dst->surf.aux_usage,
529 dst->level,
530 dst->offset.z, extent.depth);
531
532 dst_has_shadow =
533 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
534 anv_image, aspect,
535 &dst_shadow_surf);
536 }
537
538 for (unsigned z = 0; z < extent.depth; z++) {
539 blorp_copy(&batch, &src->surf, src->level, src->offset.z,
540 &dst->surf, dst->level, dst->offset.z,
541 src->offset.x, src->offset.y, dst->offset.x, dst->offset.y,
542 extent.width, extent.height);
543
544 if (dst_has_shadow) {
545 blorp_copy(&batch, &src->surf, src->level, src->offset.z,
546 &dst_shadow_surf, dst->level, dst->offset.z,
547 src->offset.x, src->offset.y,
548 dst->offset.x, dst->offset.y,
549 extent.width, extent.height);
550 }
551
552 image.offset.z++;
553 buffer.surf.addr.offset += buffer_layer_stride;
554 }
555 }
556
557 blorp_batch_finish(&batch);
558 }
559
560 void anv_CmdCopyBufferToImage(
561 VkCommandBuffer commandBuffer,
562 VkBuffer srcBuffer,
563 VkImage dstImage,
564 VkImageLayout dstImageLayout,
565 uint32_t regionCount,
566 const VkBufferImageCopy* pRegions)
567 {
568 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
569 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
570 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
571
572 copy_buffer_to_image(cmd_buffer, src_buffer, dst_image, dstImageLayout,
573 regionCount, pRegions, true);
574 }
575
576 void anv_CmdCopyImageToBuffer(
577 VkCommandBuffer commandBuffer,
578 VkImage srcImage,
579 VkImageLayout srcImageLayout,
580 VkBuffer dstBuffer,
581 uint32_t regionCount,
582 const VkBufferImageCopy* pRegions)
583 {
584 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
585 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
586 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
587
588 copy_buffer_to_image(cmd_buffer, dst_buffer, src_image, srcImageLayout,
589 regionCount, pRegions, false);
590
591 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
592 }
593
594 static bool
595 flip_coords(unsigned *src0, unsigned *src1, unsigned *dst0, unsigned *dst1)
596 {
597 bool flip = false;
598 if (*src0 > *src1) {
599 unsigned tmp = *src0;
600 *src0 = *src1;
601 *src1 = tmp;
602 flip = !flip;
603 }
604
605 if (*dst0 > *dst1) {
606 unsigned tmp = *dst0;
607 *dst0 = *dst1;
608 *dst1 = tmp;
609 flip = !flip;
610 }
611
612 return flip;
613 }
614
615 void anv_CmdBlitImage(
616 VkCommandBuffer commandBuffer,
617 VkImage srcImage,
618 VkImageLayout srcImageLayout,
619 VkImage dstImage,
620 VkImageLayout dstImageLayout,
621 uint32_t regionCount,
622 const VkImageBlit* pRegions,
623 VkFilter filter)
624
625 {
626 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
627 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
628 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
629
630 struct blorp_surf src, dst;
631
632 enum blorp_filter blorp_filter;
633 switch (filter) {
634 case VK_FILTER_NEAREST:
635 blorp_filter = BLORP_FILTER_NEAREST;
636 break;
637 case VK_FILTER_LINEAR:
638 blorp_filter = BLORP_FILTER_BILINEAR;
639 break;
640 default:
641 unreachable("Invalid filter");
642 }
643
644 struct blorp_batch batch;
645 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
646
647 for (unsigned r = 0; r < regionCount; r++) {
648 const VkImageSubresourceLayers *src_res = &pRegions[r].srcSubresource;
649 const VkImageSubresourceLayers *dst_res = &pRegions[r].dstSubresource;
650
651 assert(anv_image_aspects_compatible(src_res->aspectMask,
652 dst_res->aspectMask));
653
654 uint32_t aspect_bit;
655 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_res->aspectMask) {
656 get_blorp_surf_for_anv_image(cmd_buffer->device,
657 src_image, 1U << aspect_bit,
658 srcImageLayout, ISL_AUX_USAGE_NONE, &src);
659 get_blorp_surf_for_anv_image(cmd_buffer->device,
660 dst_image, 1U << aspect_bit,
661 dstImageLayout, ISL_AUX_USAGE_NONE, &dst);
662
663 struct anv_format_plane src_format =
664 anv_get_format_plane(&cmd_buffer->device->info, src_image->vk_format,
665 1U << aspect_bit, src_image->tiling);
666 struct anv_format_plane dst_format =
667 anv_get_format_plane(&cmd_buffer->device->info, dst_image->vk_format,
668 1U << aspect_bit, dst_image->tiling);
669
670 unsigned dst_start, dst_end;
671 if (dst_image->type == VK_IMAGE_TYPE_3D) {
672 assert(dst_res->baseArrayLayer == 0);
673 dst_start = pRegions[r].dstOffsets[0].z;
674 dst_end = pRegions[r].dstOffsets[1].z;
675 } else {
676 dst_start = dst_res->baseArrayLayer;
677 dst_end = dst_start + anv_get_layerCount(dst_image, dst_res);
678 }
679
680 unsigned src_start, src_end;
681 if (src_image->type == VK_IMAGE_TYPE_3D) {
682 assert(src_res->baseArrayLayer == 0);
683 src_start = pRegions[r].srcOffsets[0].z;
684 src_end = pRegions[r].srcOffsets[1].z;
685 } else {
686 src_start = src_res->baseArrayLayer;
687 src_end = src_start + anv_get_layerCount(src_image, src_res);
688 }
689
690 bool flip_z = flip_coords(&src_start, &src_end, &dst_start, &dst_end);
691 float src_z_step = (float)(src_end + 1 - src_start) /
692 (float)(dst_end + 1 - dst_start);
693
694 if (flip_z) {
695 src_start = src_end;
696 src_z_step *= -1;
697 }
698
699 unsigned src_x0 = pRegions[r].srcOffsets[0].x;
700 unsigned src_x1 = pRegions[r].srcOffsets[1].x;
701 unsigned dst_x0 = pRegions[r].dstOffsets[0].x;
702 unsigned dst_x1 = pRegions[r].dstOffsets[1].x;
703 bool flip_x = flip_coords(&src_x0, &src_x1, &dst_x0, &dst_x1);
704
705 unsigned src_y0 = pRegions[r].srcOffsets[0].y;
706 unsigned src_y1 = pRegions[r].srcOffsets[1].y;
707 unsigned dst_y0 = pRegions[r].dstOffsets[0].y;
708 unsigned dst_y1 = pRegions[r].dstOffsets[1].y;
709 bool flip_y = flip_coords(&src_y0, &src_y1, &dst_y0, &dst_y1);
710
711 const unsigned num_layers = dst_end - dst_start;
712 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
713 1U << aspect_bit,
714 dst.aux_usage,
715 dst_res->mipLevel,
716 dst_start, num_layers);
717
718 for (unsigned i = 0; i < num_layers; i++) {
719 unsigned dst_z = dst_start + i;
720 unsigned src_z = src_start + i * src_z_step;
721
722 blorp_blit(&batch, &src, src_res->mipLevel, src_z,
723 src_format.isl_format, src_format.swizzle,
724 &dst, dst_res->mipLevel, dst_z,
725 dst_format.isl_format, dst_format.swizzle,
726 src_x0, src_y0, src_x1, src_y1,
727 dst_x0, dst_y0, dst_x1, dst_y1,
728 blorp_filter, flip_x, flip_y);
729 }
730 }
731 }
732
733 blorp_batch_finish(&batch);
734 }
735
736 /**
737 * Returns the greatest common divisor of a and b that is a power of two.
738 */
739 static uint64_t
740 gcd_pow2_u64(uint64_t a, uint64_t b)
741 {
742 assert(a > 0 || b > 0);
743
744 unsigned a_log2 = ffsll(a) - 1;
745 unsigned b_log2 = ffsll(b) - 1;
746
747 /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which
748 * case, the MIN2() will take the other one. If both are 0 then we will
749 * hit the assert above.
750 */
751 return 1 << MIN2(a_log2, b_log2);
752 }
753
754 /* This is maximum possible width/height our HW can handle */
755 #define MAX_SURFACE_DIM (1ull << 14)
756
757 void anv_CmdCopyBuffer(
758 VkCommandBuffer commandBuffer,
759 VkBuffer srcBuffer,
760 VkBuffer dstBuffer,
761 uint32_t regionCount,
762 const VkBufferCopy* pRegions)
763 {
764 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
765 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
766 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
767
768 struct blorp_batch batch;
769 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
770
771 for (unsigned r = 0; r < regionCount; r++) {
772 struct blorp_address src = {
773 .buffer = src_buffer->address.bo,
774 .offset = src_buffer->address.offset + pRegions[r].srcOffset,
775 .mocs = anv_mocs_for_bo(cmd_buffer->device, src_buffer->address.bo),
776 };
777 struct blorp_address dst = {
778 .buffer = dst_buffer->address.bo,
779 .offset = dst_buffer->address.offset + pRegions[r].dstOffset,
780 .mocs = anv_mocs_for_bo(cmd_buffer->device, dst_buffer->address.bo),
781 };
782
783 blorp_buffer_copy(&batch, src, dst, pRegions[r].size);
784 }
785
786 blorp_batch_finish(&batch);
787
788 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
789 }
790
791 void anv_CmdUpdateBuffer(
792 VkCommandBuffer commandBuffer,
793 VkBuffer dstBuffer,
794 VkDeviceSize dstOffset,
795 VkDeviceSize dataSize,
796 const void* pData)
797 {
798 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
799 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
800
801 struct blorp_batch batch;
802 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
803
804 /* We can't quite grab a full block because the state stream needs a
805 * little data at the top to build its linked list.
806 */
807 const uint32_t max_update_size =
808 cmd_buffer->device->dynamic_state_pool.block_size - 64;
809
810 assert(max_update_size < MAX_SURFACE_DIM * 4);
811
812 /* We're about to read data that was written from the CPU. Flush the
813 * texture cache so we don't get anything stale.
814 */
815 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
816
817 while (dataSize) {
818 const uint32_t copy_size = MIN2(dataSize, max_update_size);
819
820 struct anv_state tmp_data =
821 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, copy_size, 64);
822
823 memcpy(tmp_data.map, pData, copy_size);
824
825 struct blorp_address src = {
826 .buffer = cmd_buffer->device->dynamic_state_pool.block_pool.bo,
827 .offset = tmp_data.offset,
828 .mocs = cmd_buffer->device->isl_dev.mocs.internal,
829 };
830 struct blorp_address dst = {
831 .buffer = dst_buffer->address.bo,
832 .offset = dst_buffer->address.offset + dstOffset,
833 .mocs = anv_mocs_for_bo(cmd_buffer->device, dst_buffer->address.bo),
834 };
835
836 blorp_buffer_copy(&batch, src, dst, copy_size);
837
838 dataSize -= copy_size;
839 dstOffset += copy_size;
840 pData = (void *)pData + copy_size;
841 }
842
843 blorp_batch_finish(&batch);
844
845 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
846 }
847
848 void anv_CmdFillBuffer(
849 VkCommandBuffer commandBuffer,
850 VkBuffer dstBuffer,
851 VkDeviceSize dstOffset,
852 VkDeviceSize fillSize,
853 uint32_t data)
854 {
855 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
856 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
857 struct blorp_surf surf;
858 struct isl_surf isl_surf;
859
860 struct blorp_batch batch;
861 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
862
863 fillSize = anv_buffer_get_range(dst_buffer, dstOffset, fillSize);
864
865 /* From the Vulkan spec:
866 *
867 * "size is the number of bytes to fill, and must be either a multiple
868 * of 4, or VK_WHOLE_SIZE to fill the range from offset to the end of
869 * the buffer. If VK_WHOLE_SIZE is used and the remaining size of the
870 * buffer is not a multiple of 4, then the nearest smaller multiple is
871 * used."
872 */
873 fillSize &= ~3ull;
874
875 /* First, we compute the biggest format that can be used with the
876 * given offsets and size.
877 */
878 int bs = 16;
879 bs = gcd_pow2_u64(bs, dstOffset);
880 bs = gcd_pow2_u64(bs, fillSize);
881 enum isl_format isl_format = isl_format_for_size(bs);
882
883 union isl_color_value color = {
884 .u32 = { data, data, data, data },
885 };
886
887 const uint64_t max_fill_size = MAX_SURFACE_DIM * MAX_SURFACE_DIM * bs;
888 while (fillSize >= max_fill_size) {
889 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
890 dst_buffer, dstOffset,
891 MAX_SURFACE_DIM, MAX_SURFACE_DIM,
892 MAX_SURFACE_DIM * bs, isl_format,
893 &surf, &isl_surf);
894
895 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
896 0, 0, 1, 0, 0, MAX_SURFACE_DIM, MAX_SURFACE_DIM,
897 color, NULL);
898 fillSize -= max_fill_size;
899 dstOffset += max_fill_size;
900 }
901
902 uint64_t height = fillSize / (MAX_SURFACE_DIM * bs);
903 assert(height < MAX_SURFACE_DIM);
904 if (height != 0) {
905 const uint64_t rect_fill_size = height * MAX_SURFACE_DIM * bs;
906 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
907 dst_buffer, dstOffset,
908 MAX_SURFACE_DIM, height,
909 MAX_SURFACE_DIM * bs, isl_format,
910 &surf, &isl_surf);
911
912 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
913 0, 0, 1, 0, 0, MAX_SURFACE_DIM, height,
914 color, NULL);
915 fillSize -= rect_fill_size;
916 dstOffset += rect_fill_size;
917 }
918
919 if (fillSize != 0) {
920 const uint32_t width = fillSize / bs;
921 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
922 dst_buffer, dstOffset,
923 width, 1,
924 width * bs, isl_format,
925 &surf, &isl_surf);
926
927 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
928 0, 0, 1, 0, 0, width, 1,
929 color, NULL);
930 }
931
932 blorp_batch_finish(&batch);
933
934 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
935 }
936
937 void anv_CmdClearColorImage(
938 VkCommandBuffer commandBuffer,
939 VkImage _image,
940 VkImageLayout imageLayout,
941 const VkClearColorValue* pColor,
942 uint32_t rangeCount,
943 const VkImageSubresourceRange* pRanges)
944 {
945 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
946 ANV_FROM_HANDLE(anv_image, image, _image);
947
948 static const bool color_write_disable[4] = { false, false, false, false };
949
950 struct blorp_batch batch;
951 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
952
953
954 for (unsigned r = 0; r < rangeCount; r++) {
955 if (pRanges[r].aspectMask == 0)
956 continue;
957
958 assert(pRanges[r].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
959
960 struct blorp_surf surf;
961 get_blorp_surf_for_anv_image(cmd_buffer->device,
962 image, pRanges[r].aspectMask,
963 imageLayout, ISL_AUX_USAGE_NONE, &surf);
964
965 struct anv_format_plane src_format =
966 anv_get_format_plane(&cmd_buffer->device->info, image->vk_format,
967 VK_IMAGE_ASPECT_COLOR_BIT, image->tiling);
968
969 unsigned base_layer = pRanges[r].baseArrayLayer;
970 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
971
972 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
973 const unsigned level = pRanges[r].baseMipLevel + i;
974 const unsigned level_width = anv_minify(image->extent.width, level);
975 const unsigned level_height = anv_minify(image->extent.height, level);
976
977 if (image->type == VK_IMAGE_TYPE_3D) {
978 base_layer = 0;
979 layer_count = anv_minify(image->extent.depth, level);
980 }
981
982 anv_cmd_buffer_mark_image_written(cmd_buffer, image,
983 pRanges[r].aspectMask,
984 surf.aux_usage, level,
985 base_layer, layer_count);
986
987 blorp_clear(&batch, &surf,
988 src_format.isl_format, src_format.swizzle,
989 level, base_layer, layer_count,
990 0, 0, level_width, level_height,
991 vk_to_isl_color(*pColor), color_write_disable);
992 }
993 }
994
995 blorp_batch_finish(&batch);
996 }
997
998 void anv_CmdClearDepthStencilImage(
999 VkCommandBuffer commandBuffer,
1000 VkImage image_h,
1001 VkImageLayout imageLayout,
1002 const VkClearDepthStencilValue* pDepthStencil,
1003 uint32_t rangeCount,
1004 const VkImageSubresourceRange* pRanges)
1005 {
1006 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1007 ANV_FROM_HANDLE(anv_image, image, image_h);
1008
1009 struct blorp_batch batch;
1010 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1011
1012 struct blorp_surf depth, stencil, stencil_shadow;
1013 if (image->aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1014 get_blorp_surf_for_anv_image(cmd_buffer->device,
1015 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1016 imageLayout, ISL_AUX_USAGE_NONE, &depth);
1017 } else {
1018 memset(&depth, 0, sizeof(depth));
1019 }
1020
1021 bool has_stencil_shadow = false;
1022 if (image->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1023 get_blorp_surf_for_anv_image(cmd_buffer->device,
1024 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1025 imageLayout, ISL_AUX_USAGE_NONE, &stencil);
1026
1027 has_stencil_shadow =
1028 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device, image,
1029 VK_IMAGE_ASPECT_STENCIL_BIT,
1030 &stencil_shadow);
1031 } else {
1032 memset(&stencil, 0, sizeof(stencil));
1033 }
1034
1035 for (unsigned r = 0; r < rangeCount; r++) {
1036 if (pRanges[r].aspectMask == 0)
1037 continue;
1038
1039 bool clear_depth = pRanges[r].aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
1040 bool clear_stencil = pRanges[r].aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
1041
1042 unsigned base_layer = pRanges[r].baseArrayLayer;
1043 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
1044
1045 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
1046 const unsigned level = pRanges[r].baseMipLevel + i;
1047 const unsigned level_width = anv_minify(image->extent.width, level);
1048 const unsigned level_height = anv_minify(image->extent.height, level);
1049
1050 if (image->type == VK_IMAGE_TYPE_3D)
1051 layer_count = anv_minify(image->extent.depth, level);
1052
1053 blorp_clear_depth_stencil(&batch, &depth, &stencil,
1054 level, base_layer, layer_count,
1055 0, 0, level_width, level_height,
1056 clear_depth, pDepthStencil->depth,
1057 clear_stencil ? 0xff : 0,
1058 pDepthStencil->stencil);
1059
1060 if (clear_stencil && has_stencil_shadow) {
1061 union isl_color_value stencil_color = {
1062 .u32 = { pDepthStencil->stencil, },
1063 };
1064 blorp_clear(&batch, &stencil_shadow,
1065 ISL_FORMAT_R8_UINT, ISL_SWIZZLE_IDENTITY,
1066 level, base_layer, layer_count,
1067 0, 0, level_width, level_height,
1068 stencil_color, NULL);
1069 }
1070 }
1071 }
1072
1073 blorp_batch_finish(&batch);
1074 }
1075
1076 VkResult
1077 anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer *cmd_buffer,
1078 uint32_t num_entries,
1079 uint32_t *state_offset,
1080 struct anv_state *bt_state)
1081 {
1082 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
1083 state_offset);
1084 if (bt_state->map == NULL) {
1085 /* We ran out of space. Grab a new binding table block. */
1086 VkResult result = anv_cmd_buffer_new_binding_table_block(cmd_buffer);
1087 if (result != VK_SUCCESS)
1088 return result;
1089
1090 /* Re-emit state base addresses so we get the new surface state base
1091 * address before we start emitting binding tables etc.
1092 */
1093 anv_cmd_buffer_emit_state_base_address(cmd_buffer);
1094
1095 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
1096 state_offset);
1097 assert(bt_state->map != NULL);
1098 }
1099
1100 return VK_SUCCESS;
1101 }
1102
1103 static VkResult
1104 binding_table_for_surface_state(struct anv_cmd_buffer *cmd_buffer,
1105 struct anv_state surface_state,
1106 uint32_t *bt_offset)
1107 {
1108 uint32_t state_offset;
1109 struct anv_state bt_state;
1110
1111 VkResult result =
1112 anv_cmd_buffer_alloc_blorp_binding_table(cmd_buffer, 1, &state_offset,
1113 &bt_state);
1114 if (result != VK_SUCCESS)
1115 return result;
1116
1117 uint32_t *bt_map = bt_state.map;
1118 bt_map[0] = surface_state.offset + state_offset;
1119
1120 *bt_offset = bt_state.offset;
1121 return VK_SUCCESS;
1122 }
1123
1124 static void
1125 clear_color_attachment(struct anv_cmd_buffer *cmd_buffer,
1126 struct blorp_batch *batch,
1127 const VkClearAttachment *attachment,
1128 uint32_t rectCount, const VkClearRect *pRects)
1129 {
1130 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
1131 const uint32_t color_att = attachment->colorAttachment;
1132 assert(color_att < subpass->color_count);
1133 const uint32_t att_idx = subpass->color_attachments[color_att].attachment;
1134
1135 if (att_idx == VK_ATTACHMENT_UNUSED)
1136 return;
1137
1138 struct anv_render_pass_attachment *pass_att =
1139 &cmd_buffer->state.pass->attachments[att_idx];
1140 struct anv_attachment_state *att_state =
1141 &cmd_buffer->state.attachments[att_idx];
1142
1143 uint32_t binding_table;
1144 VkResult result =
1145 binding_table_for_surface_state(cmd_buffer, att_state->color.state,
1146 &binding_table);
1147 if (result != VK_SUCCESS)
1148 return;
1149
1150 union isl_color_value clear_color =
1151 vk_to_isl_color(attachment->clearValue.color);
1152
1153 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1154 if (subpass->view_mask) {
1155 uint32_t view_idx;
1156 for_each_bit(view_idx, subpass->view_mask) {
1157 for (uint32_t r = 0; r < rectCount; ++r) {
1158 const VkOffset2D offset = pRects[r].rect.offset;
1159 const VkExtent2D extent = pRects[r].rect.extent;
1160 blorp_clear_attachments(batch, binding_table,
1161 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1162 view_idx, 1,
1163 offset.x, offset.y,
1164 offset.x + extent.width,
1165 offset.y + extent.height,
1166 true, clear_color, false, 0.0f, 0, 0);
1167 }
1168 }
1169 return;
1170 }
1171
1172 for (uint32_t r = 0; r < rectCount; ++r) {
1173 const VkOffset2D offset = pRects[r].rect.offset;
1174 const VkExtent2D extent = pRects[r].rect.extent;
1175 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1176 blorp_clear_attachments(batch, binding_table,
1177 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1178 pRects[r].baseArrayLayer,
1179 pRects[r].layerCount,
1180 offset.x, offset.y,
1181 offset.x + extent.width, offset.y + extent.height,
1182 true, clear_color, false, 0.0f, 0, 0);
1183 }
1184 }
1185
1186 static void
1187 clear_depth_stencil_attachment(struct anv_cmd_buffer *cmd_buffer,
1188 struct blorp_batch *batch,
1189 const VkClearAttachment *attachment,
1190 uint32_t rectCount, const VkClearRect *pRects)
1191 {
1192 static const union isl_color_value color_value = { .u32 = { 0, } };
1193 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
1194 if (!subpass->depth_stencil_attachment)
1195 return;
1196
1197 const uint32_t att_idx = subpass->depth_stencil_attachment->attachment;
1198 assert(att_idx != VK_ATTACHMENT_UNUSED);
1199 struct anv_render_pass_attachment *pass_att =
1200 &cmd_buffer->state.pass->attachments[att_idx];
1201
1202 bool clear_depth = attachment->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
1203 bool clear_stencil = attachment->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
1204
1205 enum isl_format depth_format = ISL_FORMAT_UNSUPPORTED;
1206 if (clear_depth) {
1207 depth_format = anv_get_isl_format(&cmd_buffer->device->info,
1208 pass_att->format,
1209 VK_IMAGE_ASPECT_DEPTH_BIT,
1210 VK_IMAGE_TILING_OPTIMAL);
1211 }
1212
1213 uint32_t binding_table;
1214 VkResult result =
1215 binding_table_for_surface_state(cmd_buffer,
1216 cmd_buffer->state.null_surface_state,
1217 &binding_table);
1218 if (result != VK_SUCCESS)
1219 return;
1220
1221 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1222 if (subpass->view_mask) {
1223 uint32_t view_idx;
1224 for_each_bit(view_idx, subpass->view_mask) {
1225 for (uint32_t r = 0; r < rectCount; ++r) {
1226 const VkOffset2D offset = pRects[r].rect.offset;
1227 const VkExtent2D extent = pRects[r].rect.extent;
1228 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1229 blorp_clear_attachments(batch, binding_table,
1230 depth_format, pass_att->samples,
1231 view_idx, 1,
1232 offset.x, offset.y,
1233 offset.x + extent.width,
1234 offset.y + extent.height,
1235 false, color_value,
1236 clear_depth, value.depth,
1237 clear_stencil ? 0xff : 0, value.stencil);
1238 }
1239 }
1240 return;
1241 }
1242
1243 for (uint32_t r = 0; r < rectCount; ++r) {
1244 const VkOffset2D offset = pRects[r].rect.offset;
1245 const VkExtent2D extent = pRects[r].rect.extent;
1246 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1247 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1248 blorp_clear_attachments(batch, binding_table,
1249 depth_format, pass_att->samples,
1250 pRects[r].baseArrayLayer,
1251 pRects[r].layerCount,
1252 offset.x, offset.y,
1253 offset.x + extent.width, offset.y + extent.height,
1254 false, color_value,
1255 clear_depth, value.depth,
1256 clear_stencil ? 0xff : 0, value.stencil);
1257 }
1258 }
1259
1260 void anv_CmdClearAttachments(
1261 VkCommandBuffer commandBuffer,
1262 uint32_t attachmentCount,
1263 const VkClearAttachment* pAttachments,
1264 uint32_t rectCount,
1265 const VkClearRect* pRects)
1266 {
1267 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1268
1269 /* Because this gets called within a render pass, we tell blorp not to
1270 * trash our depth and stencil buffers.
1271 */
1272 struct blorp_batch batch;
1273 enum blorp_batch_flags flags = BLORP_BATCH_NO_EMIT_DEPTH_STENCIL;
1274 if (cmd_buffer->state.conditional_render_enabled) {
1275 anv_cmd_emit_conditional_render_predicate(cmd_buffer);
1276 flags |= BLORP_BATCH_PREDICATE_ENABLE;
1277 }
1278 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, flags);
1279
1280 for (uint32_t a = 0; a < attachmentCount; ++a) {
1281 if (pAttachments[a].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
1282 assert(pAttachments[a].aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
1283 clear_color_attachment(cmd_buffer, &batch,
1284 &pAttachments[a],
1285 rectCount, pRects);
1286 } else {
1287 clear_depth_stencil_attachment(cmd_buffer, &batch,
1288 &pAttachments[a],
1289 rectCount, pRects);
1290 }
1291 }
1292
1293 blorp_batch_finish(&batch);
1294 }
1295
1296 enum subpass_stage {
1297 SUBPASS_STAGE_LOAD,
1298 SUBPASS_STAGE_DRAW,
1299 SUBPASS_STAGE_RESOLVE,
1300 };
1301
1302 void
1303 anv_image_msaa_resolve(struct anv_cmd_buffer *cmd_buffer,
1304 const struct anv_image *src_image,
1305 enum isl_aux_usage src_aux_usage,
1306 uint32_t src_level, uint32_t src_base_layer,
1307 const struct anv_image *dst_image,
1308 enum isl_aux_usage dst_aux_usage,
1309 uint32_t dst_level, uint32_t dst_base_layer,
1310 VkImageAspectFlagBits aspect,
1311 uint32_t src_x, uint32_t src_y,
1312 uint32_t dst_x, uint32_t dst_y,
1313 uint32_t width, uint32_t height,
1314 uint32_t layer_count,
1315 enum blorp_filter filter)
1316 {
1317 struct blorp_batch batch;
1318 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1319
1320 assert(src_image->type == VK_IMAGE_TYPE_2D);
1321 assert(src_image->samples > 1);
1322 assert(dst_image->type == VK_IMAGE_TYPE_2D);
1323 assert(dst_image->samples == 1);
1324 assert(src_image->n_planes == dst_image->n_planes);
1325 assert(!src_image->format->can_ycbcr);
1326 assert(!dst_image->format->can_ycbcr);
1327
1328 struct blorp_surf src_surf, dst_surf;
1329 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, aspect,
1330 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1331 src_aux_usage, &src_surf);
1332 if (src_aux_usage == ISL_AUX_USAGE_MCS) {
1333 src_surf.clear_color_addr = anv_to_blorp_address(
1334 anv_image_get_clear_color_addr(cmd_buffer->device, src_image,
1335 VK_IMAGE_ASPECT_COLOR_BIT));
1336 }
1337 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, aspect,
1338 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1339 dst_aux_usage, &dst_surf);
1340 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
1341 aspect, dst_aux_usage,
1342 dst_level, dst_base_layer, layer_count);
1343
1344 if (filter == BLORP_FILTER_NONE) {
1345 /* If no explicit filter is provided, then it's implied by the type of
1346 * the source image.
1347 */
1348 if ((src_surf.surf->usage & ISL_SURF_USAGE_DEPTH_BIT) ||
1349 (src_surf.surf->usage & ISL_SURF_USAGE_STENCIL_BIT) ||
1350 isl_format_has_int_channel(src_surf.surf->format)) {
1351 filter = BLORP_FILTER_SAMPLE_0;
1352 } else {
1353 filter = BLORP_FILTER_AVERAGE;
1354 }
1355 }
1356
1357 for (uint32_t l = 0; l < layer_count; l++) {
1358 blorp_blit(&batch,
1359 &src_surf, src_level, src_base_layer + l,
1360 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1361 &dst_surf, dst_level, dst_base_layer + l,
1362 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1363 src_x, src_y, src_x + width, src_y + height,
1364 dst_x, dst_y, dst_x + width, dst_y + height,
1365 filter, false, false);
1366 }
1367
1368 blorp_batch_finish(&batch);
1369 }
1370
1371 void anv_CmdResolveImage(
1372 VkCommandBuffer commandBuffer,
1373 VkImage srcImage,
1374 VkImageLayout srcImageLayout,
1375 VkImage dstImage,
1376 VkImageLayout dstImageLayout,
1377 uint32_t regionCount,
1378 const VkImageResolve* pRegions)
1379 {
1380 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1381 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
1382 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
1383
1384 assert(!src_image->format->can_ycbcr);
1385
1386 for (uint32_t r = 0; r < regionCount; r++) {
1387 assert(pRegions[r].srcSubresource.aspectMask ==
1388 pRegions[r].dstSubresource.aspectMask);
1389 assert(anv_get_layerCount(src_image, &pRegions[r].srcSubresource) ==
1390 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource));
1391
1392 const uint32_t layer_count =
1393 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource);
1394
1395 uint32_t aspect_bit;
1396 anv_foreach_image_aspect_bit(aspect_bit, src_image,
1397 pRegions[r].srcSubresource.aspectMask) {
1398 enum isl_aux_usage src_aux_usage =
1399 anv_layout_to_aux_usage(&cmd_buffer->device->info, src_image,
1400 (1 << aspect_bit), srcImageLayout);
1401 enum isl_aux_usage dst_aux_usage =
1402 anv_layout_to_aux_usage(&cmd_buffer->device->info, dst_image,
1403 (1 << aspect_bit), dstImageLayout);
1404
1405 anv_image_msaa_resolve(cmd_buffer,
1406 src_image, src_aux_usage,
1407 pRegions[r].srcSubresource.mipLevel,
1408 pRegions[r].srcSubresource.baseArrayLayer,
1409 dst_image, dst_aux_usage,
1410 pRegions[r].dstSubresource.mipLevel,
1411 pRegions[r].dstSubresource.baseArrayLayer,
1412 (1 << aspect_bit),
1413 pRegions[r].srcOffset.x,
1414 pRegions[r].srcOffset.y,
1415 pRegions[r].dstOffset.x,
1416 pRegions[r].dstOffset.y,
1417 pRegions[r].extent.width,
1418 pRegions[r].extent.height,
1419 layer_count, BLORP_FILTER_NONE);
1420 }
1421 }
1422 }
1423
1424 static enum isl_aux_usage
1425 fast_clear_aux_usage(const struct anv_image *image,
1426 VkImageAspectFlagBits aspect)
1427 {
1428 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
1429 if (image->planes[plane].aux_usage == ISL_AUX_USAGE_NONE)
1430 return ISL_AUX_USAGE_CCS_D;
1431 else
1432 return image->planes[plane].aux_usage;
1433 }
1434
1435 void
1436 anv_image_copy_to_shadow(struct anv_cmd_buffer *cmd_buffer,
1437 const struct anv_image *image,
1438 VkImageAspectFlagBits aspect,
1439 uint32_t base_level, uint32_t level_count,
1440 uint32_t base_layer, uint32_t layer_count)
1441 {
1442 struct blorp_batch batch;
1443 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1444
1445 /* We don't know who touched the main surface last so flush a bunch of
1446 * caches to ensure we get good data.
1447 */
1448 cmd_buffer->state.pending_pipe_bits |=
1449 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT |
1450 ANV_PIPE_DATA_CACHE_FLUSH_BIT |
1451 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT |
1452 ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
1453
1454 struct blorp_surf surf;
1455 get_blorp_surf_for_anv_image(cmd_buffer->device,
1456 image, aspect,
1457 VK_IMAGE_LAYOUT_GENERAL,
1458 ISL_AUX_USAGE_NONE, &surf);
1459 assert(surf.aux_usage == ISL_AUX_USAGE_NONE);
1460
1461 struct blorp_surf shadow_surf;
1462 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
1463 image, aspect, &shadow_surf);
1464
1465 for (uint32_t l = 0; l < level_count; l++) {
1466 const uint32_t level = base_level + l;
1467
1468 const VkExtent3D extent = {
1469 .width = anv_minify(image->extent.width, level),
1470 .height = anv_minify(image->extent.height, level),
1471 .depth = anv_minify(image->extent.depth, level),
1472 };
1473
1474 if (image->type == VK_IMAGE_TYPE_3D)
1475 layer_count = extent.depth;
1476
1477 for (uint32_t a = 0; a < layer_count; a++) {
1478 const uint32_t layer = base_layer + a;
1479
1480 blorp_copy(&batch, &surf, level, layer,
1481 &shadow_surf, level, layer,
1482 0, 0, 0, 0, extent.width, extent.height);
1483 }
1484 }
1485
1486 /* We just wrote to the buffer with the render cache. Flush it. */
1487 cmd_buffer->state.pending_pipe_bits |=
1488 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
1489
1490 blorp_batch_finish(&batch);
1491 }
1492
1493 void
1494 anv_image_clear_color(struct anv_cmd_buffer *cmd_buffer,
1495 const struct anv_image *image,
1496 VkImageAspectFlagBits aspect,
1497 enum isl_aux_usage aux_usage,
1498 enum isl_format format, struct isl_swizzle swizzle,
1499 uint32_t level, uint32_t base_layer, uint32_t layer_count,
1500 VkRect2D area, union isl_color_value clear_color)
1501 {
1502 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1503
1504 /* We don't support planar images with multisampling yet */
1505 assert(image->n_planes == 1);
1506
1507 struct blorp_batch batch;
1508 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1509
1510 struct blorp_surf surf;
1511 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1512 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1513 aux_usage, &surf);
1514 anv_cmd_buffer_mark_image_written(cmd_buffer, image, aspect, aux_usage,
1515 level, base_layer, layer_count);
1516
1517 blorp_clear(&batch, &surf, format, anv_swizzle_for_render(swizzle),
1518 level, base_layer, layer_count,
1519 area.offset.x, area.offset.y,
1520 area.offset.x + area.extent.width,
1521 area.offset.y + area.extent.height,
1522 clear_color, NULL);
1523
1524 blorp_batch_finish(&batch);
1525 }
1526
1527 void
1528 anv_image_clear_depth_stencil(struct anv_cmd_buffer *cmd_buffer,
1529 const struct anv_image *image,
1530 VkImageAspectFlags aspects,
1531 enum isl_aux_usage depth_aux_usage,
1532 uint32_t level,
1533 uint32_t base_layer, uint32_t layer_count,
1534 VkRect2D area,
1535 float depth_value, uint8_t stencil_value)
1536 {
1537 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
1538 VK_IMAGE_ASPECT_STENCIL_BIT));
1539
1540 struct blorp_batch batch;
1541 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1542
1543 struct blorp_surf depth = {};
1544 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1545 get_blorp_surf_for_anv_image(cmd_buffer->device,
1546 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1547 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1548 depth_aux_usage, &depth);
1549 depth.clear_color.f32[0] = ANV_HZ_FC_VAL;
1550 }
1551
1552 struct blorp_surf stencil = {};
1553 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1554 get_blorp_surf_for_anv_image(cmd_buffer->device,
1555 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1556 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1557 ISL_AUX_USAGE_NONE, &stencil);
1558 }
1559
1560 /* Blorp may choose to clear stencil using RGBA32_UINT for better
1561 * performance. If it does this, we need to flush it out of the depth
1562 * cache before rendering to it.
1563 */
1564 cmd_buffer->state.pending_pipe_bits |=
1565 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1566
1567 blorp_clear_depth_stencil(&batch, &depth, &stencil,
1568 level, base_layer, layer_count,
1569 area.offset.x, area.offset.y,
1570 area.offset.x + area.extent.width,
1571 area.offset.y + area.extent.height,
1572 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
1573 depth_value,
1574 (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) ? 0xff : 0,
1575 stencil_value);
1576
1577 /* Blorp may choose to clear stencil using RGBA32_UINT for better
1578 * performance. If it does this, we need to flush it out of the render
1579 * cache before someone starts trying to do stencil on it.
1580 */
1581 cmd_buffer->state.pending_pipe_bits |=
1582 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1583
1584 struct blorp_surf stencil_shadow;
1585 if ((aspects & VK_IMAGE_ASPECT_STENCIL_BIT) &&
1586 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device, image,
1587 VK_IMAGE_ASPECT_STENCIL_BIT,
1588 &stencil_shadow)) {
1589 union isl_color_value stencil_color = {
1590 .u32 = { stencil_value },
1591 };
1592 blorp_clear(&batch, &stencil_shadow,
1593 ISL_FORMAT_R8_UINT, ISL_SWIZZLE_IDENTITY,
1594 level, base_layer, layer_count,
1595 area.offset.x, area.offset.y,
1596 area.offset.x + area.extent.width,
1597 area.offset.y + area.extent.height,
1598 stencil_color, NULL);
1599 }
1600
1601 blorp_batch_finish(&batch);
1602 }
1603
1604 void
1605 anv_image_hiz_op(struct anv_cmd_buffer *cmd_buffer,
1606 const struct anv_image *image,
1607 VkImageAspectFlagBits aspect, uint32_t level,
1608 uint32_t base_layer, uint32_t layer_count,
1609 enum isl_aux_op hiz_op)
1610 {
1611 assert(aspect == VK_IMAGE_ASPECT_DEPTH_BIT);
1612 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, level));
1613 assert(anv_image_aspect_to_plane(image->aspects,
1614 VK_IMAGE_ASPECT_DEPTH_BIT) == 0);
1615
1616 struct blorp_batch batch;
1617 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1618
1619 struct blorp_surf surf;
1620 get_blorp_surf_for_anv_image(cmd_buffer->device,
1621 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1622 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1623 ISL_AUX_USAGE_HIZ, &surf);
1624 surf.clear_color.f32[0] = ANV_HZ_FC_VAL;
1625
1626 blorp_hiz_op(&batch, &surf, level, base_layer, layer_count, hiz_op);
1627
1628 blorp_batch_finish(&batch);
1629 }
1630
1631 void
1632 anv_image_hiz_clear(struct anv_cmd_buffer *cmd_buffer,
1633 const struct anv_image *image,
1634 VkImageAspectFlags aspects,
1635 uint32_t level,
1636 uint32_t base_layer, uint32_t layer_count,
1637 VkRect2D area, uint8_t stencil_value)
1638 {
1639 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
1640 VK_IMAGE_ASPECT_STENCIL_BIT));
1641
1642 struct blorp_batch batch;
1643 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1644
1645 struct blorp_surf depth = {};
1646 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1647 assert(base_layer + layer_count <=
1648 anv_image_aux_layers(image, VK_IMAGE_ASPECT_DEPTH_BIT, level));
1649 get_blorp_surf_for_anv_image(cmd_buffer->device,
1650 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1651 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1652 ISL_AUX_USAGE_HIZ, &depth);
1653 depth.clear_color.f32[0] = ANV_HZ_FC_VAL;
1654 }
1655
1656 struct blorp_surf stencil = {};
1657 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1658 get_blorp_surf_for_anv_image(cmd_buffer->device,
1659 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1660 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1661 ISL_AUX_USAGE_NONE, &stencil);
1662 }
1663
1664 /* From the Sky Lake PRM Volume 7, "Depth Buffer Clear":
1665 *
1666 * "The following is required when performing a depth buffer clear with
1667 * using the WM_STATE or 3DSTATE_WM:
1668 *
1669 * * If other rendering operations have preceded this clear, a
1670 * PIPE_CONTROL with depth cache flush enabled, Depth Stall bit
1671 * enabled must be issued before the rectangle primitive used for
1672 * the depth buffer clear operation.
1673 * * [...]"
1674 *
1675 * Even though the PRM only says that this is required if using 3DSTATE_WM
1676 * and a 3DPRIMITIVE, the GPU appears to also need this to avoid occasional
1677 * hangs when doing a clear with WM_HZ_OP.
1678 */
1679 cmd_buffer->state.pending_pipe_bits |=
1680 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT;
1681
1682 blorp_hiz_clear_depth_stencil(&batch, &depth, &stencil,
1683 level, base_layer, layer_count,
1684 area.offset.x, area.offset.y,
1685 area.offset.x + area.extent.width,
1686 area.offset.y + area.extent.height,
1687 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
1688 ANV_HZ_FC_VAL,
1689 aspects & VK_IMAGE_ASPECT_STENCIL_BIT,
1690 stencil_value);
1691
1692 blorp_batch_finish(&batch);
1693
1694 /* From the SKL PRM, Depth Buffer Clear:
1695 *
1696 * "Depth Buffer Clear Workaround
1697 *
1698 * Depth buffer clear pass using any of the methods (WM_STATE,
1699 * 3DSTATE_WM or 3DSTATE_WM_HZ_OP) must be followed by a PIPE_CONTROL
1700 * command with DEPTH_STALL bit and Depth FLUSH bits “set” before
1701 * starting to render. DepthStall and DepthFlush are not needed between
1702 * consecutive depth clear passes nor is it required if the depth-clear
1703 * pass was done with “full_surf_clear” bit set in the
1704 * 3DSTATE_WM_HZ_OP."
1705 *
1706 * Even though the PRM provides a bunch of conditions under which this is
1707 * supposedly unnecessary, we choose to perform the flush unconditionally
1708 * just to be safe.
1709 */
1710 cmd_buffer->state.pending_pipe_bits |=
1711 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT;
1712 }
1713
1714 void
1715 anv_image_mcs_op(struct anv_cmd_buffer *cmd_buffer,
1716 const struct anv_image *image,
1717 enum isl_format format,
1718 VkImageAspectFlagBits aspect,
1719 uint32_t base_layer, uint32_t layer_count,
1720 enum isl_aux_op mcs_op, union isl_color_value *clear_value,
1721 bool predicate)
1722 {
1723 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1724 assert(image->samples > 1);
1725 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, 0));
1726
1727 /* Multisampling with multi-planar formats is not supported */
1728 assert(image->n_planes == 1);
1729
1730 struct blorp_batch batch;
1731 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1732 BLORP_BATCH_PREDICATE_ENABLE * predicate +
1733 BLORP_BATCH_NO_UPDATE_CLEAR_COLOR * !clear_value);
1734
1735 struct blorp_surf surf;
1736 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1737 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1738 ISL_AUX_USAGE_MCS, &surf);
1739
1740 /* Blorp will store the clear color for us if we provide the clear color
1741 * address and we are doing a fast clear. So we save the clear value into
1742 * the blorp surface.
1743 */
1744 if (clear_value)
1745 surf.clear_color = *clear_value;
1746
1747 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1748 *
1749 * "After Render target fast clear, pipe-control with color cache
1750 * write-flush must be issued before sending any DRAW commands on
1751 * that render target."
1752 *
1753 * This comment is a bit cryptic and doesn't really tell you what's going
1754 * or what's really needed. It appears that fast clear ops are not
1755 * properly synchronized with other drawing. This means that we cannot
1756 * have a fast clear operation in the pipe at the same time as other
1757 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
1758 * that the contents of the previous draw hit the render target before we
1759 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
1760 * that it is completed before any additional drawing occurs.
1761 */
1762 cmd_buffer->state.pending_pipe_bits |=
1763 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1764
1765 switch (mcs_op) {
1766 case ISL_AUX_OP_FAST_CLEAR:
1767 blorp_fast_clear(&batch, &surf, format,
1768 0, base_layer, layer_count,
1769 0, 0, image->extent.width, image->extent.height);
1770 break;
1771 case ISL_AUX_OP_PARTIAL_RESOLVE:
1772 blorp_mcs_partial_resolve(&batch, &surf, format,
1773 base_layer, layer_count);
1774 break;
1775 case ISL_AUX_OP_FULL_RESOLVE:
1776 case ISL_AUX_OP_AMBIGUATE:
1777 default:
1778 unreachable("Unsupported MCS operation");
1779 }
1780
1781 cmd_buffer->state.pending_pipe_bits |=
1782 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1783
1784 blorp_batch_finish(&batch);
1785 }
1786
1787 void
1788 anv_image_ccs_op(struct anv_cmd_buffer *cmd_buffer,
1789 const struct anv_image *image,
1790 enum isl_format format,
1791 VkImageAspectFlagBits aspect, uint32_t level,
1792 uint32_t base_layer, uint32_t layer_count,
1793 enum isl_aux_op ccs_op, union isl_color_value *clear_value,
1794 bool predicate)
1795 {
1796 assert(image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
1797 assert(image->samples == 1);
1798 assert(level < anv_image_aux_levels(image, aspect));
1799 /* Multi-LOD YcBcR is not allowed */
1800 assert(image->n_planes == 1 || level == 0);
1801 assert(base_layer + layer_count <=
1802 anv_image_aux_layers(image, aspect, level));
1803
1804 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
1805 uint32_t width_div = image->format->planes[plane].denominator_scales[0];
1806 uint32_t height_div = image->format->planes[plane].denominator_scales[1];
1807 uint32_t level_width = anv_minify(image->extent.width, level) / width_div;
1808 uint32_t level_height = anv_minify(image->extent.height, level) / height_div;
1809
1810 struct blorp_batch batch;
1811 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1812 BLORP_BATCH_PREDICATE_ENABLE * predicate +
1813 BLORP_BATCH_NO_UPDATE_CLEAR_COLOR * !clear_value);
1814
1815 struct blorp_surf surf;
1816 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1817 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1818 fast_clear_aux_usage(image, aspect),
1819 &surf);
1820
1821 /* Blorp will store the clear color for us if we provide the clear color
1822 * address and we are doing a fast clear. So we save the clear value into
1823 * the blorp surface.
1824 */
1825 if (clear_value)
1826 surf.clear_color = *clear_value;
1827
1828 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1829 *
1830 * "After Render target fast clear, pipe-control with color cache
1831 * write-flush must be issued before sending any DRAW commands on
1832 * that render target."
1833 *
1834 * This comment is a bit cryptic and doesn't really tell you what's going
1835 * or what's really needed. It appears that fast clear ops are not
1836 * properly synchronized with other drawing. This means that we cannot
1837 * have a fast clear operation in the pipe at the same time as other
1838 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
1839 * that the contents of the previous draw hit the render target before we
1840 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
1841 * that it is completed before any additional drawing occurs.
1842 */
1843 cmd_buffer->state.pending_pipe_bits |=
1844 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1845
1846 switch (ccs_op) {
1847 case ISL_AUX_OP_FAST_CLEAR:
1848 blorp_fast_clear(&batch, &surf, format,
1849 level, base_layer, layer_count,
1850 0, 0, level_width, level_height);
1851 break;
1852 case ISL_AUX_OP_FULL_RESOLVE:
1853 case ISL_AUX_OP_PARTIAL_RESOLVE:
1854 blorp_ccs_resolve(&batch, &surf, level, base_layer, layer_count,
1855 format, ccs_op);
1856 break;
1857 case ISL_AUX_OP_AMBIGUATE:
1858 for (uint32_t a = 0; a < layer_count; a++) {
1859 const uint32_t layer = base_layer + a;
1860 blorp_ccs_ambiguate(&batch, &surf, level, layer);
1861 }
1862 break;
1863 default:
1864 unreachable("Unsupported CCS operation");
1865 }
1866
1867 cmd_buffer->state.pending_pipe_bits |=
1868 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1869
1870 blorp_batch_finish(&batch);
1871 }