projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
anv: fix crash in vkCmdClearAttachments with unused attachment
[mesa.git] / src / intel / vulkan / anv_blorp.c
1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "anv_private.h"
25
26 static bool
27 lookup_blorp_shader(struct blorp_batch *batch,
28 const void *key, uint32_t key_size,
29 uint32_t *kernel_out, void *prog_data_out)
30 {
31 struct blorp_context *blorp = batch->blorp;
32 struct anv_device *device = blorp->driver_ctx;
33
34 /* The default cache must be a real cache */
35 assert(device->default_pipeline_cache.cache);
36
37 struct anv_shader_bin *bin =
38 anv_pipeline_cache_search(&device->default_pipeline_cache, key, key_size);
39 if (!bin)
40 return false;
41
42 /* The cache already has a reference and it's not going anywhere so there
43 * is no need to hold a second reference.
44 */
45 anv_shader_bin_unref(device, bin);
46
47 *kernel_out = bin->kernel.offset;
48 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
49
50 return true;
51 }
52
53 static bool
54 upload_blorp_shader(struct blorp_batch *batch,
55 const void *key, uint32_t key_size,
56 const void *kernel, uint32_t kernel_size,
57 const struct brw_stage_prog_data *prog_data,
58 uint32_t prog_data_size,
59 uint32_t *kernel_out, void *prog_data_out)
60 {
61 struct blorp_context *blorp = batch->blorp;
62 struct anv_device *device = blorp->driver_ctx;
63
64 /* The blorp cache must be a real cache */
65 assert(device->default_pipeline_cache.cache);
66
67 struct anv_pipeline_bind_map bind_map = {
68 .surface_count = 0,
69 .sampler_count = 0,
70 };
71
72 struct anv_shader_bin *bin =
73 anv_pipeline_cache_upload_kernel(&device->default_pipeline_cache,
74 key, key_size, kernel, kernel_size,
75 NULL, 0,
76 prog_data, prog_data_size,
77 NULL, &bind_map);
78
79 if (!bin)
80 return false;
81
82 /* The cache already has a reference and it's not going anywhere so there
83 * is no need to hold a second reference.
84 */
85 anv_shader_bin_unref(device, bin);
86
87 *kernel_out = bin->kernel.offset;
88 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
89
90 return true;
91 }
92
93 void
94 anv_device_init_blorp(struct anv_device *device)
95 {
96 blorp_init(&device->blorp, device, &device->isl_dev);
97 device->blorp.compiler = device->instance->physicalDevice.compiler;
98 device->blorp.lookup_shader = lookup_blorp_shader;
99 device->blorp.upload_shader = upload_blorp_shader;
100 switch (device->info.gen) {
101 case 7:
102 if (device->info.is_haswell) {
103 device->blorp.exec = gen75_blorp_exec;
104 } else {
105 device->blorp.exec = gen7_blorp_exec;
106 }
107 break;
108 case 8:
109 device->blorp.exec = gen8_blorp_exec;
110 break;
111 case 9:
112 device->blorp.exec = gen9_blorp_exec;
113 break;
114 case 10:
115 device->blorp.exec = gen10_blorp_exec;
116 break;
117 case 11:
118 device->blorp.exec = gen11_blorp_exec;
119 break;
120 default:
121 unreachable("Unknown hardware generation");
122 }
123 }
124
125 void
126 anv_device_finish_blorp(struct anv_device *device)
127 {
128 blorp_finish(&device->blorp);
129 }
130
131 static void
132 get_blorp_surf_for_anv_buffer(struct anv_device *device,
133 struct anv_buffer *buffer, uint64_t offset,
134 uint32_t width, uint32_t height,
135 uint32_t row_pitch, enum isl_format format,
136 struct blorp_surf *blorp_surf,
137 struct isl_surf *isl_surf)
138 {
139 const struct isl_format_layout *fmtl =
140 isl_format_get_layout(format);
141 bool ok UNUSED;
142
143 /* ASTC is the only format which doesn't support linear layouts.
144 * Create an equivalently sized surface with ISL to get around this.
145 */
146 if (fmtl->txc == ISL_TXC_ASTC) {
147 /* Use an equivalently sized format */
148 format = ISL_FORMAT_R32G32B32A32_UINT;
149 assert(fmtl->bpb == isl_format_get_layout(format)->bpb);
150
151 /* Shrink the dimensions for the new format */
152 width = DIV_ROUND_UP(width, fmtl->bw);
153 height = DIV_ROUND_UP(height, fmtl->bh);
154 }
155
156 *blorp_surf = (struct blorp_surf) {
157 .surf = isl_surf,
158 .addr = {
159 .buffer = buffer->address.bo,
160 .offset = buffer->address.offset + offset,
161 .mocs = anv_mocs_for_bo(device, buffer->address.bo),
162 },
163 };
164
165 ok = isl_surf_init(&device->isl_dev, isl_surf,
166 .dim = ISL_SURF_DIM_2D,
167 .format = format,
168 .width = width,
169 .height = height,
170 .depth = 1,
171 .levels = 1,
172 .array_len = 1,
173 .samples = 1,
174 .row_pitch_B = row_pitch,
175 .usage = ISL_SURF_USAGE_TEXTURE_BIT |
176 ISL_SURF_USAGE_RENDER_TARGET_BIT,
177 .tiling_flags = ISL_TILING_LINEAR_BIT);
178 assert(ok);
179 }
180
181 /* Pick something high enough that it won't be used in core and low enough it
182 * will never map to an extension.
183 */
184 #define ANV_IMAGE_LAYOUT_EXPLICIT_AUX (VkImageLayout)10000000
185
186 static struct blorp_address
187 anv_to_blorp_address(struct anv_address addr)
188 {
189 return (struct blorp_address) {
190 .buffer = addr.bo,
191 .offset = addr.offset,
192 };
193 }
194
195 static void
196 get_blorp_surf_for_anv_image(const struct anv_device *device,
197 const struct anv_image *image,
198 VkImageAspectFlags aspect,
199 VkImageLayout layout,
200 enum isl_aux_usage aux_usage,
201 struct blorp_surf *blorp_surf)
202 {
203 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
204
205 if (layout != ANV_IMAGE_LAYOUT_EXPLICIT_AUX)
206 aux_usage = anv_layout_to_aux_usage(&device->info, image, aspect, layout);
207
208 const struct anv_surface *surface = &image->planes[plane].surface;
209 *blorp_surf = (struct blorp_surf) {
210 .surf = &surface->isl,
211 .addr = {
212 .buffer = image->planes[plane].address.bo,
213 .offset = image->planes[plane].address.offset + surface->offset,
214 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
215 },
216 };
217
218 if (aux_usage != ISL_AUX_USAGE_NONE) {
219 const struct anv_surface *aux_surface = &image->planes[plane].aux_surface;
220 blorp_surf->aux_surf = &aux_surface->isl,
221 blorp_surf->aux_addr = (struct blorp_address) {
222 .buffer = image->planes[plane].address.bo,
223 .offset = image->planes[plane].address.offset + aux_surface->offset,
224 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
225 };
226 blorp_surf->aux_usage = aux_usage;
227
228 /* If we're doing a partial resolve, then we need the indirect clear
229 * color. If we are doing a fast clear and want to store/update the
230 * clear color, we also pass the address to blorp, otherwise it will only
231 * stomp the CCS to a particular value and won't care about format or
232 * clear value
233 */
234 if (aspect & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
235 const struct anv_address clear_color_addr =
236 anv_image_get_clear_color_addr(device, image, aspect);
237 blorp_surf->clear_color_addr = anv_to_blorp_address(clear_color_addr);
238 } else if (aspect & VK_IMAGE_ASPECT_DEPTH_BIT
239 && device->info.gen >= 10) {
240 /* Vulkan always clears to 1.0. On gen < 10, we set that directly in
241 * the state packet. For gen >= 10, must provide the clear value in a
242 * buffer. We have a single global buffer that stores the 1.0 value.
243 */
244 const struct anv_address clear_color_addr = (struct anv_address) {
245 .bo = (struct anv_bo *)&device->hiz_clear_bo
246 };
247 blorp_surf->clear_color_addr = anv_to_blorp_address(clear_color_addr);
248 }
249 }
250 }
251
252 static bool
253 get_blorp_surf_for_anv_shadow_image(const struct anv_device *device,
254 const struct anv_image *image,
255 VkImageAspectFlags aspect,
256 struct blorp_surf *blorp_surf)
257 {
258
259 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
260 if (image->planes[plane].shadow_surface.isl.size_B == 0)
261 return false;
262
263 *blorp_surf = (struct blorp_surf) {
264 .surf = &image->planes[plane].shadow_surface.isl,
265 .addr = {
266 .buffer = image->planes[plane].address.bo,
267 .offset = image->planes[plane].address.offset +
268 image->planes[plane].shadow_surface.offset,
269 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
270 },
271 };
272
273 return true;
274 }
275
276 void anv_CmdCopyImage(
277 VkCommandBuffer commandBuffer,
278 VkImage srcImage,
279 VkImageLayout srcImageLayout,
280 VkImage dstImage,
281 VkImageLayout dstImageLayout,
282 uint32_t regionCount,
283 const VkImageCopy* pRegions)
284 {
285 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
286 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
287 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
288
289 struct blorp_batch batch;
290 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
291
292 for (unsigned r = 0; r < regionCount; r++) {
293 VkOffset3D srcOffset =
294 anv_sanitize_image_offset(src_image->type, pRegions[r].srcOffset);
295 VkOffset3D dstOffset =
296 anv_sanitize_image_offset(dst_image->type, pRegions[r].dstOffset);
297 VkExtent3D extent =
298 anv_sanitize_image_extent(src_image->type, pRegions[r].extent);
299
300 const uint32_t dst_level = pRegions[r].dstSubresource.mipLevel;
301 unsigned dst_base_layer, layer_count;
302 if (dst_image->type == VK_IMAGE_TYPE_3D) {
303 dst_base_layer = pRegions[r].dstOffset.z;
304 layer_count = pRegions[r].extent.depth;
305 } else {
306 dst_base_layer = pRegions[r].dstSubresource.baseArrayLayer;
307 layer_count =
308 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource);
309 }
310
311 const uint32_t src_level = pRegions[r].srcSubresource.mipLevel;
312 unsigned src_base_layer;
313 if (src_image->type == VK_IMAGE_TYPE_3D) {
314 src_base_layer = pRegions[r].srcOffset.z;
315 } else {
316 src_base_layer = pRegions[r].srcSubresource.baseArrayLayer;
317 assert(layer_count ==
318 anv_get_layerCount(src_image, &pRegions[r].srcSubresource));
319 }
320
321 VkImageAspectFlags src_mask = pRegions[r].srcSubresource.aspectMask,
322 dst_mask = pRegions[r].dstSubresource.aspectMask;
323
324 assert(anv_image_aspects_compatible(src_mask, dst_mask));
325
326 if (util_bitcount(src_mask) > 1) {
327 uint32_t aspect_bit;
328 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_mask) {
329 struct blorp_surf src_surf, dst_surf;
330 get_blorp_surf_for_anv_image(cmd_buffer->device,
331 src_image, 1UL << aspect_bit,
332 srcImageLayout, ISL_AUX_USAGE_NONE,
333 &src_surf);
334 get_blorp_surf_for_anv_image(cmd_buffer->device,
335 dst_image, 1UL << aspect_bit,
336 dstImageLayout, ISL_AUX_USAGE_NONE,
337 &dst_surf);
338 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
339 1UL << aspect_bit,
340 dst_surf.aux_usage, dst_level,
341 dst_base_layer, layer_count);
342
343 for (unsigned i = 0; i < layer_count; i++) {
344 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
345 &dst_surf, dst_level, dst_base_layer + i,
346 srcOffset.x, srcOffset.y,
347 dstOffset.x, dstOffset.y,
348 extent.width, extent.height);
349 }
350
351 struct blorp_surf dst_shadow_surf;
352 if (get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
353 dst_image,
354 1UL << aspect_bit,
355 &dst_shadow_surf)) {
356 for (unsigned i = 0; i < layer_count; i++) {
357 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
358 &dst_shadow_surf, dst_level, dst_base_layer + i,
359 srcOffset.x, srcOffset.y,
360 dstOffset.x, dstOffset.y,
361 extent.width, extent.height);
362 }
363 }
364 }
365 } else {
366 struct blorp_surf src_surf, dst_surf;
367 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, src_mask,
368 srcImageLayout, ISL_AUX_USAGE_NONE,
369 &src_surf);
370 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, dst_mask,
371 dstImageLayout, ISL_AUX_USAGE_NONE,
372 &dst_surf);
373 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image, dst_mask,
374 dst_surf.aux_usage, dst_level,
375 dst_base_layer, layer_count);
376
377 for (unsigned i = 0; i < layer_count; i++) {
378 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
379 &dst_surf, dst_level, dst_base_layer + i,
380 srcOffset.x, srcOffset.y,
381 dstOffset.x, dstOffset.y,
382 extent.width, extent.height);
383 }
384
385 struct blorp_surf dst_shadow_surf;
386 if (get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
387 dst_image, dst_mask,
388 &dst_shadow_surf)) {
389 for (unsigned i = 0; i < layer_count; i++) {
390 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
391 &dst_shadow_surf, dst_level, dst_base_layer + i,
392 srcOffset.x, srcOffset.y,
393 dstOffset.x, dstOffset.y,
394 extent.width, extent.height);
395 }
396 }
397 }
398 }
399
400 blorp_batch_finish(&batch);
401 }
402
403 static void
404 copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer,
405 struct anv_buffer *anv_buffer,
406 struct anv_image *anv_image,
407 VkImageLayout image_layout,
408 uint32_t regionCount,
409 const VkBufferImageCopy* pRegions,
410 bool buffer_to_image)
411 {
412 struct blorp_batch batch;
413 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
414
415 struct {
416 struct blorp_surf surf;
417 uint32_t level;
418 VkOffset3D offset;
419 } image, buffer, *src, *dst;
420
421 buffer.level = 0;
422 buffer.offset = (VkOffset3D) { 0, 0, 0 };
423
424 if (buffer_to_image) {
425 src = &buffer;
426 dst = &image;
427 } else {
428 src = &image;
429 dst = &buffer;
430 }
431
432 for (unsigned r = 0; r < regionCount; r++) {
433 const VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask;
434
435 get_blorp_surf_for_anv_image(cmd_buffer->device, anv_image, aspect,
436 image_layout, ISL_AUX_USAGE_NONE,
437 &image.surf);
438 image.offset =
439 anv_sanitize_image_offset(anv_image->type, pRegions[r].imageOffset);
440 image.level = pRegions[r].imageSubresource.mipLevel;
441
442 VkExtent3D extent =
443 anv_sanitize_image_extent(anv_image->type, pRegions[r].imageExtent);
444 if (anv_image->type != VK_IMAGE_TYPE_3D) {
445 image.offset.z = pRegions[r].imageSubresource.baseArrayLayer;
446 extent.depth =
447 anv_get_layerCount(anv_image, &pRegions[r].imageSubresource);
448 }
449
450 const enum isl_format buffer_format =
451 anv_get_isl_format(&cmd_buffer->device->info, anv_image->vk_format,
452 aspect, VK_IMAGE_TILING_LINEAR);
453
454 const VkExtent3D bufferImageExtent = {
455 .width = pRegions[r].bufferRowLength ?
456 pRegions[r].bufferRowLength : extent.width,
457 .height = pRegions[r].bufferImageHeight ?
458 pRegions[r].bufferImageHeight : extent.height,
459 };
460
461 const struct isl_format_layout *buffer_fmtl =
462 isl_format_get_layout(buffer_format);
463
464 const uint32_t buffer_row_pitch =
465 DIV_ROUND_UP(bufferImageExtent.width, buffer_fmtl->bw) *
466 (buffer_fmtl->bpb / 8);
467
468 const uint32_t buffer_layer_stride =
469 DIV_ROUND_UP(bufferImageExtent.height, buffer_fmtl->bh) *
470 buffer_row_pitch;
471
472 struct isl_surf buffer_isl_surf;
473 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
474 anv_buffer, pRegions[r].bufferOffset,
475 extent.width, extent.height,
476 buffer_row_pitch, buffer_format,
477 &buffer.surf, &buffer_isl_surf);
478
479 bool dst_has_shadow = false;
480 struct blorp_surf dst_shadow_surf;
481 if (&image == dst) {
482 anv_cmd_buffer_mark_image_written(cmd_buffer, anv_image,
483 aspect, dst->surf.aux_usage,
484 dst->level,
485 dst->offset.z, extent.depth);
486
487 dst_has_shadow =
488 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
489 anv_image, aspect,
490 &dst_shadow_surf);
491 }
492
493 for (unsigned z = 0; z < extent.depth; z++) {
494 blorp_copy(&batch, &src->surf, src->level, src->offset.z,
495 &dst->surf, dst->level, dst->offset.z,
496 src->offset.x, src->offset.y, dst->offset.x, dst->offset.y,
497 extent.width, extent.height);
498
499 if (dst_has_shadow) {
500 blorp_copy(&batch, &src->surf, src->level, src->offset.z,
501 &dst_shadow_surf, dst->level, dst->offset.z,
502 src->offset.x, src->offset.y,
503 dst->offset.x, dst->offset.y,
504 extent.width, extent.height);
505 }
506
507 image.offset.z++;
508 buffer.surf.addr.offset += buffer_layer_stride;
509 }
510 }
511
512 blorp_batch_finish(&batch);
513 }
514
515 void anv_CmdCopyBufferToImage(
516 VkCommandBuffer commandBuffer,
517 VkBuffer srcBuffer,
518 VkImage dstImage,
519 VkImageLayout dstImageLayout,
520 uint32_t regionCount,
521 const VkBufferImageCopy* pRegions)
522 {
523 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
524 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
525 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
526
527 copy_buffer_to_image(cmd_buffer, src_buffer, dst_image, dstImageLayout,
528 regionCount, pRegions, true);
529 }
530
531 void anv_CmdCopyImageToBuffer(
532 VkCommandBuffer commandBuffer,
533 VkImage srcImage,
534 VkImageLayout srcImageLayout,
535 VkBuffer dstBuffer,
536 uint32_t regionCount,
537 const VkBufferImageCopy* pRegions)
538 {
539 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
540 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
541 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
542
543 copy_buffer_to_image(cmd_buffer, dst_buffer, src_image, srcImageLayout,
544 regionCount, pRegions, false);
545
546 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
547 }
548
549 static bool
550 flip_coords(unsigned *src0, unsigned *src1, unsigned *dst0, unsigned *dst1)
551 {
552 bool flip = false;
553 if (*src0 > *src1) {
554 unsigned tmp = *src0;
555 *src0 = *src1;
556 *src1 = tmp;
557 flip = !flip;
558 }
559
560 if (*dst0 > *dst1) {
561 unsigned tmp = *dst0;
562 *dst0 = *dst1;
563 *dst1 = tmp;
564 flip = !flip;
565 }
566
567 return flip;
568 }
569
570 void anv_CmdBlitImage(
571 VkCommandBuffer commandBuffer,
572 VkImage srcImage,
573 VkImageLayout srcImageLayout,
574 VkImage dstImage,
575 VkImageLayout dstImageLayout,
576 uint32_t regionCount,
577 const VkImageBlit* pRegions,
578 VkFilter filter)
579
580 {
581 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
582 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
583 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
584
585 struct blorp_surf src, dst;
586
587 enum blorp_filter blorp_filter;
588 switch (filter) {
589 case VK_FILTER_NEAREST:
590 blorp_filter = BLORP_FILTER_NEAREST;
591 break;
592 case VK_FILTER_LINEAR:
593 blorp_filter = BLORP_FILTER_BILINEAR;
594 break;
595 default:
596 unreachable("Invalid filter");
597 }
598
599 struct blorp_batch batch;
600 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
601
602 for (unsigned r = 0; r < regionCount; r++) {
603 const VkImageSubresourceLayers *src_res = &pRegions[r].srcSubresource;
604 const VkImageSubresourceLayers *dst_res = &pRegions[r].dstSubresource;
605
606 assert(anv_image_aspects_compatible(src_res->aspectMask,
607 dst_res->aspectMask));
608
609 uint32_t aspect_bit;
610 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_res->aspectMask) {
611 get_blorp_surf_for_anv_image(cmd_buffer->device,
612 src_image, 1U << aspect_bit,
613 srcImageLayout, ISL_AUX_USAGE_NONE, &src);
614 get_blorp_surf_for_anv_image(cmd_buffer->device,
615 dst_image, 1U << aspect_bit,
616 dstImageLayout, ISL_AUX_USAGE_NONE, &dst);
617
618 struct anv_format_plane src_format =
619 anv_get_format_plane(&cmd_buffer->device->info, src_image->vk_format,
620 1U << aspect_bit, src_image->tiling);
621 struct anv_format_plane dst_format =
622 anv_get_format_plane(&cmd_buffer->device->info, dst_image->vk_format,
623 1U << aspect_bit, dst_image->tiling);
624
625 unsigned dst_start, dst_end;
626 if (dst_image->type == VK_IMAGE_TYPE_3D) {
627 assert(dst_res->baseArrayLayer == 0);
628 dst_start = pRegions[r].dstOffsets[0].z;
629 dst_end = pRegions[r].dstOffsets[1].z;
630 } else {
631 dst_start = dst_res->baseArrayLayer;
632 dst_end = dst_start + anv_get_layerCount(dst_image, dst_res);
633 }
634
635 unsigned src_start, src_end;
636 if (src_image->type == VK_IMAGE_TYPE_3D) {
637 assert(src_res->baseArrayLayer == 0);
638 src_start = pRegions[r].srcOffsets[0].z;
639 src_end = pRegions[r].srcOffsets[1].z;
640 } else {
641 src_start = src_res->baseArrayLayer;
642 src_end = src_start + anv_get_layerCount(src_image, src_res);
643 }
644
645 bool flip_z = flip_coords(&src_start, &src_end, &dst_start, &dst_end);
646 float src_z_step = (float)(src_end + 1 - src_start) /
647 (float)(dst_end + 1 - dst_start);
648
649 if (flip_z) {
650 src_start = src_end;
651 src_z_step *= -1;
652 }
653
654 unsigned src_x0 = pRegions[r].srcOffsets[0].x;
655 unsigned src_x1 = pRegions[r].srcOffsets[1].x;
656 unsigned dst_x0 = pRegions[r].dstOffsets[0].x;
657 unsigned dst_x1 = pRegions[r].dstOffsets[1].x;
658 bool flip_x = flip_coords(&src_x0, &src_x1, &dst_x0, &dst_x1);
659
660 unsigned src_y0 = pRegions[r].srcOffsets[0].y;
661 unsigned src_y1 = pRegions[r].srcOffsets[1].y;
662 unsigned dst_y0 = pRegions[r].dstOffsets[0].y;
663 unsigned dst_y1 = pRegions[r].dstOffsets[1].y;
664 bool flip_y = flip_coords(&src_y0, &src_y1, &dst_y0, &dst_y1);
665
666 const unsigned num_layers = dst_end - dst_start;
667 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
668 1U << aspect_bit,
669 dst.aux_usage,
670 dst_res->mipLevel,
671 dst_start, num_layers);
672
673 for (unsigned i = 0; i < num_layers; i++) {
674 unsigned dst_z = dst_start + i;
675 unsigned src_z = src_start + i * src_z_step;
676
677 blorp_blit(&batch, &src, src_res->mipLevel, src_z,
678 src_format.isl_format, src_format.swizzle,
679 &dst, dst_res->mipLevel, dst_z,
680 dst_format.isl_format, dst_format.swizzle,
681 src_x0, src_y0, src_x1, src_y1,
682 dst_x0, dst_y0, dst_x1, dst_y1,
683 blorp_filter, flip_x, flip_y);
684 }
685 }
686 }
687
688 blorp_batch_finish(&batch);
689 }
690
691 static enum isl_format
692 isl_format_for_size(unsigned size_B)
693 {
694 switch (size_B) {
695 case 4: return ISL_FORMAT_R32_UINT;
696 case 8: return ISL_FORMAT_R32G32_UINT;
697 case 16: return ISL_FORMAT_R32G32B32A32_UINT;
698 default:
699 unreachable("Not a power-of-two format size");
700 }
701 }
702
703 /**
704 * Returns the greatest common divisor of a and b that is a power of two.
705 */
706 static uint64_t
707 gcd_pow2_u64(uint64_t a, uint64_t b)
708 {
709 assert(a > 0 || b > 0);
710
711 unsigned a_log2 = ffsll(a) - 1;
712 unsigned b_log2 = ffsll(b) - 1;
713
714 /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which
715 * case, the MIN2() will take the other one. If both are 0 then we will
716 * hit the assert above.
717 */
718 return 1 << MIN2(a_log2, b_log2);
719 }
720
721 /* This is maximum possible width/height our HW can handle */
722 #define MAX_SURFACE_DIM (1ull << 14)
723
724 void anv_CmdCopyBuffer(
725 VkCommandBuffer commandBuffer,
726 VkBuffer srcBuffer,
727 VkBuffer dstBuffer,
728 uint32_t regionCount,
729 const VkBufferCopy* pRegions)
730 {
731 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
732 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
733 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
734
735 struct blorp_batch batch;
736 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
737
738 for (unsigned r = 0; r < regionCount; r++) {
739 struct blorp_address src = {
740 .buffer = src_buffer->address.bo,
741 .offset = src_buffer->address.offset + pRegions[r].srcOffset,
742 .mocs = anv_mocs_for_bo(cmd_buffer->device, src_buffer->address.bo),
743 };
744 struct blorp_address dst = {
745 .buffer = dst_buffer->address.bo,
746 .offset = dst_buffer->address.offset + pRegions[r].dstOffset,
747 .mocs = anv_mocs_for_bo(cmd_buffer->device, dst_buffer->address.bo),
748 };
749
750 blorp_buffer_copy(&batch, src, dst, pRegions[r].size);
751 }
752
753 blorp_batch_finish(&batch);
754
755 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
756 }
757
758 void anv_CmdUpdateBuffer(
759 VkCommandBuffer commandBuffer,
760 VkBuffer dstBuffer,
761 VkDeviceSize dstOffset,
762 VkDeviceSize dataSize,
763 const void* pData)
764 {
765 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
766 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
767
768 struct blorp_batch batch;
769 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
770
771 /* We can't quite grab a full block because the state stream needs a
772 * little data at the top to build its linked list.
773 */
774 const uint32_t max_update_size =
775 cmd_buffer->device->dynamic_state_pool.block_size - 64;
776
777 assert(max_update_size < MAX_SURFACE_DIM * 4);
778
779 /* We're about to read data that was written from the CPU. Flush the
780 * texture cache so we don't get anything stale.
781 */
782 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
783
784 while (dataSize) {
785 const uint32_t copy_size = MIN2(dataSize, max_update_size);
786
787 struct anv_state tmp_data =
788 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, copy_size, 64);
789
790 memcpy(tmp_data.map, pData, copy_size);
791
792 struct blorp_address src = {
793 .buffer = cmd_buffer->device->dynamic_state_pool.block_pool.bo,
794 .offset = tmp_data.offset,
795 .mocs = cmd_buffer->device->default_mocs,
796 };
797 struct blorp_address dst = {
798 .buffer = dst_buffer->address.bo,
799 .offset = dst_buffer->address.offset + dstOffset,
800 .mocs = anv_mocs_for_bo(cmd_buffer->device, dst_buffer->address.bo),
801 };
802
803 blorp_buffer_copy(&batch, src, dst, copy_size);
804
805 dataSize -= copy_size;
806 dstOffset += copy_size;
807 pData = (void *)pData + copy_size;
808 }
809
810 blorp_batch_finish(&batch);
811
812 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
813 }
814
815 void anv_CmdFillBuffer(
816 VkCommandBuffer commandBuffer,
817 VkBuffer dstBuffer,
818 VkDeviceSize dstOffset,
819 VkDeviceSize fillSize,
820 uint32_t data)
821 {
822 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
823 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
824 struct blorp_surf surf;
825 struct isl_surf isl_surf;
826
827 struct blorp_batch batch;
828 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
829
830 fillSize = anv_buffer_get_range(dst_buffer, dstOffset, fillSize);
831
832 /* From the Vulkan spec:
833 *
834 * "size is the number of bytes to fill, and must be either a multiple
835 * of 4, or VK_WHOLE_SIZE to fill the range from offset to the end of
836 * the buffer. If VK_WHOLE_SIZE is used and the remaining size of the
837 * buffer is not a multiple of 4, then the nearest smaller multiple is
838 * used."
839 */
840 fillSize &= ~3ull;
841
842 /* First, we compute the biggest format that can be used with the
843 * given offsets and size.
844 */
845 int bs = 16;
846 bs = gcd_pow2_u64(bs, dstOffset);
847 bs = gcd_pow2_u64(bs, fillSize);
848 enum isl_format isl_format = isl_format_for_size(bs);
849
850 union isl_color_value color = {
851 .u32 = { data, data, data, data },
852 };
853
854 const uint64_t max_fill_size = MAX_SURFACE_DIM * MAX_SURFACE_DIM * bs;
855 while (fillSize >= max_fill_size) {
856 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
857 dst_buffer, dstOffset,
858 MAX_SURFACE_DIM, MAX_SURFACE_DIM,
859 MAX_SURFACE_DIM * bs, isl_format,
860 &surf, &isl_surf);
861
862 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
863 0, 0, 1, 0, 0, MAX_SURFACE_DIM, MAX_SURFACE_DIM,
864 color, NULL);
865 fillSize -= max_fill_size;
866 dstOffset += max_fill_size;
867 }
868
869 uint64_t height = fillSize / (MAX_SURFACE_DIM * bs);
870 assert(height < MAX_SURFACE_DIM);
871 if (height != 0) {
872 const uint64_t rect_fill_size = height * MAX_SURFACE_DIM * bs;
873 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
874 dst_buffer, dstOffset,
875 MAX_SURFACE_DIM, height,
876 MAX_SURFACE_DIM * bs, isl_format,
877 &surf, &isl_surf);
878
879 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
880 0, 0, 1, 0, 0, MAX_SURFACE_DIM, height,
881 color, NULL);
882 fillSize -= rect_fill_size;
883 dstOffset += rect_fill_size;
884 }
885
886 if (fillSize != 0) {
887 const uint32_t width = fillSize / bs;
888 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
889 dst_buffer, dstOffset,
890 width, 1,
891 width * bs, isl_format,
892 &surf, &isl_surf);
893
894 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
895 0, 0, 1, 0, 0, width, 1,
896 color, NULL);
897 }
898
899 blorp_batch_finish(&batch);
900
901 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
902 }
903
904 void anv_CmdClearColorImage(
905 VkCommandBuffer commandBuffer,
906 VkImage _image,
907 VkImageLayout imageLayout,
908 const VkClearColorValue* pColor,
909 uint32_t rangeCount,
910 const VkImageSubresourceRange* pRanges)
911 {
912 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
913 ANV_FROM_HANDLE(anv_image, image, _image);
914
915 static const bool color_write_disable[4] = { false, false, false, false };
916
917 struct blorp_batch batch;
918 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
919
920
921 for (unsigned r = 0; r < rangeCount; r++) {
922 if (pRanges[r].aspectMask == 0)
923 continue;
924
925 assert(pRanges[r].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
926
927 struct blorp_surf surf;
928 get_blorp_surf_for_anv_image(cmd_buffer->device,
929 image, pRanges[r].aspectMask,
930 imageLayout, ISL_AUX_USAGE_NONE, &surf);
931
932 struct anv_format_plane src_format =
933 anv_get_format_plane(&cmd_buffer->device->info, image->vk_format,
934 VK_IMAGE_ASPECT_COLOR_BIT, image->tiling);
935
936 unsigned base_layer = pRanges[r].baseArrayLayer;
937 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
938
939 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
940 const unsigned level = pRanges[r].baseMipLevel + i;
941 const unsigned level_width = anv_minify(image->extent.width, level);
942 const unsigned level_height = anv_minify(image->extent.height, level);
943
944 if (image->type == VK_IMAGE_TYPE_3D) {
945 base_layer = 0;
946 layer_count = anv_minify(image->extent.depth, level);
947 }
948
949 anv_cmd_buffer_mark_image_written(cmd_buffer, image,
950 pRanges[r].aspectMask,
951 surf.aux_usage, level,
952 base_layer, layer_count);
953
954 blorp_clear(&batch, &surf,
955 src_format.isl_format, src_format.swizzle,
956 level, base_layer, layer_count,
957 0, 0, level_width, level_height,
958 vk_to_isl_color(*pColor), color_write_disable);
959 }
960 }
961
962 blorp_batch_finish(&batch);
963 }
964
965 void anv_CmdClearDepthStencilImage(
966 VkCommandBuffer commandBuffer,
967 VkImage image_h,
968 VkImageLayout imageLayout,
969 const VkClearDepthStencilValue* pDepthStencil,
970 uint32_t rangeCount,
971 const VkImageSubresourceRange* pRanges)
972 {
973 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
974 ANV_FROM_HANDLE(anv_image, image, image_h);
975
976 struct blorp_batch batch;
977 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
978
979 struct blorp_surf depth, stencil, stencil_shadow;
980 if (image->aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
981 get_blorp_surf_for_anv_image(cmd_buffer->device,
982 image, VK_IMAGE_ASPECT_DEPTH_BIT,
983 imageLayout, ISL_AUX_USAGE_NONE, &depth);
984 } else {
985 memset(&depth, 0, sizeof(depth));
986 }
987
988 bool has_stencil_shadow = false;
989 if (image->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
990 get_blorp_surf_for_anv_image(cmd_buffer->device,
991 image, VK_IMAGE_ASPECT_STENCIL_BIT,
992 imageLayout, ISL_AUX_USAGE_NONE, &stencil);
993
994 has_stencil_shadow =
995 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device, image,
996 VK_IMAGE_ASPECT_STENCIL_BIT,
997 &stencil_shadow);
998 } else {
999 memset(&stencil, 0, sizeof(stencil));
1000 }
1001
1002 for (unsigned r = 0; r < rangeCount; r++) {
1003 if (pRanges[r].aspectMask == 0)
1004 continue;
1005
1006 bool clear_depth = pRanges[r].aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
1007 bool clear_stencil = pRanges[r].aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
1008
1009 unsigned base_layer = pRanges[r].baseArrayLayer;
1010 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
1011
1012 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
1013 const unsigned level = pRanges[r].baseMipLevel + i;
1014 const unsigned level_width = anv_minify(image->extent.width, level);
1015 const unsigned level_height = anv_minify(image->extent.height, level);
1016
1017 if (image->type == VK_IMAGE_TYPE_3D)
1018 layer_count = anv_minify(image->extent.depth, level);
1019
1020 blorp_clear_depth_stencil(&batch, &depth, &stencil,
1021 level, base_layer, layer_count,
1022 0, 0, level_width, level_height,
1023 clear_depth, pDepthStencil->depth,
1024 clear_stencil ? 0xff : 0,
1025 pDepthStencil->stencil);
1026
1027 if (clear_stencil && has_stencil_shadow) {
1028 union isl_color_value stencil_color = {
1029 .u32 = { pDepthStencil->stencil, },
1030 };
1031 blorp_clear(&batch, &stencil_shadow,
1032 ISL_FORMAT_R8_UINT, ISL_SWIZZLE_IDENTITY,
1033 level, base_layer, layer_count,
1034 0, 0, level_width, level_height,
1035 stencil_color, NULL);
1036 }
1037 }
1038 }
1039
1040 blorp_batch_finish(&batch);
1041 }
1042
1043 VkResult
1044 anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer *cmd_buffer,
1045 uint32_t num_entries,
1046 uint32_t *state_offset,
1047 struct anv_state *bt_state)
1048 {
1049 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
1050 state_offset);
1051 if (bt_state->map == NULL) {
1052 /* We ran out of space. Grab a new binding table block. */
1053 VkResult result = anv_cmd_buffer_new_binding_table_block(cmd_buffer);
1054 if (result != VK_SUCCESS)
1055 return result;
1056
1057 /* Re-emit state base addresses so we get the new surface state base
1058 * address before we start emitting binding tables etc.
1059 */
1060 anv_cmd_buffer_emit_state_base_address(cmd_buffer);
1061
1062 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
1063 state_offset);
1064 assert(bt_state->map != NULL);
1065 }
1066
1067 return VK_SUCCESS;
1068 }
1069
1070 static VkResult
1071 binding_table_for_surface_state(struct anv_cmd_buffer *cmd_buffer,
1072 struct anv_state surface_state,
1073 uint32_t *bt_offset)
1074 {
1075 uint32_t state_offset;
1076 struct anv_state bt_state;
1077
1078 VkResult result =
1079 anv_cmd_buffer_alloc_blorp_binding_table(cmd_buffer, 1, &state_offset,
1080 &bt_state);
1081 if (result != VK_SUCCESS)
1082 return result;
1083
1084 uint32_t *bt_map = bt_state.map;
1085 bt_map[0] = surface_state.offset + state_offset;
1086
1087 *bt_offset = bt_state.offset;
1088 return VK_SUCCESS;
1089 }
1090
1091 static void
1092 clear_color_attachment(struct anv_cmd_buffer *cmd_buffer,
1093 struct blorp_batch *batch,
1094 const VkClearAttachment *attachment,
1095 uint32_t rectCount, const VkClearRect *pRects)
1096 {
1097 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
1098 const uint32_t color_att = attachment->colorAttachment;
1099 assert(color_att < subpass->color_count);
1100 const uint32_t att_idx = subpass->color_attachments[color_att].attachment;
1101
1102 if (att_idx == VK_ATTACHMENT_UNUSED)
1103 return;
1104
1105 struct anv_render_pass_attachment *pass_att =
1106 &cmd_buffer->state.pass->attachments[att_idx];
1107 struct anv_attachment_state *att_state =
1108 &cmd_buffer->state.attachments[att_idx];
1109
1110 uint32_t binding_table;
1111 VkResult result =
1112 binding_table_for_surface_state(cmd_buffer, att_state->color.state,
1113 &binding_table);
1114 if (result != VK_SUCCESS)
1115 return;
1116
1117 union isl_color_value clear_color =
1118 vk_to_isl_color(attachment->clearValue.color);
1119
1120 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1121 if (subpass->view_mask) {
1122 uint32_t view_idx;
1123 for_each_bit(view_idx, subpass->view_mask) {
1124 for (uint32_t r = 0; r < rectCount; ++r) {
1125 const VkOffset2D offset = pRects[r].rect.offset;
1126 const VkExtent2D extent = pRects[r].rect.extent;
1127 blorp_clear_attachments(batch, binding_table,
1128 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1129 view_idx, 1,
1130 offset.x, offset.y,
1131 offset.x + extent.width,
1132 offset.y + extent.height,
1133 true, clear_color, false, 0.0f, 0, 0);
1134 }
1135 }
1136 return;
1137 }
1138
1139 for (uint32_t r = 0; r < rectCount; ++r) {
1140 const VkOffset2D offset = pRects[r].rect.offset;
1141 const VkExtent2D extent = pRects[r].rect.extent;
1142 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1143 blorp_clear_attachments(batch, binding_table,
1144 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1145 pRects[r].baseArrayLayer,
1146 pRects[r].layerCount,
1147 offset.x, offset.y,
1148 offset.x + extent.width, offset.y + extent.height,
1149 true, clear_color, false, 0.0f, 0, 0);
1150 }
1151 }
1152
1153 static void
1154 clear_depth_stencil_attachment(struct anv_cmd_buffer *cmd_buffer,
1155 struct blorp_batch *batch,
1156 const VkClearAttachment *attachment,
1157 uint32_t rectCount, const VkClearRect *pRects)
1158 {
1159 static const union isl_color_value color_value = { .u32 = { 0, } };
1160 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
1161 if (!subpass->depth_stencil_attachment)
1162 return;
1163
1164 const uint32_t att_idx = subpass->depth_stencil_attachment->attachment;
1165 assert(att_idx != VK_ATTACHMENT_UNUSED);
1166 struct anv_render_pass_attachment *pass_att =
1167 &cmd_buffer->state.pass->attachments[att_idx];
1168
1169 bool clear_depth = attachment->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
1170 bool clear_stencil = attachment->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
1171
1172 enum isl_format depth_format = ISL_FORMAT_UNSUPPORTED;
1173 if (clear_depth) {
1174 depth_format = anv_get_isl_format(&cmd_buffer->device->info,
1175 pass_att->format,
1176 VK_IMAGE_ASPECT_DEPTH_BIT,
1177 VK_IMAGE_TILING_OPTIMAL);
1178 }
1179
1180 uint32_t binding_table;
1181 VkResult result =
1182 binding_table_for_surface_state(cmd_buffer,
1183 cmd_buffer->state.null_surface_state,
1184 &binding_table);
1185 if (result != VK_SUCCESS)
1186 return;
1187
1188 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1189 if (subpass->view_mask) {
1190 uint32_t view_idx;
1191 for_each_bit(view_idx, subpass->view_mask) {
1192 for (uint32_t r = 0; r < rectCount; ++r) {
1193 const VkOffset2D offset = pRects[r].rect.offset;
1194 const VkExtent2D extent = pRects[r].rect.extent;
1195 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1196 blorp_clear_attachments(batch, binding_table,
1197 depth_format, pass_att->samples,
1198 view_idx, 1,
1199 offset.x, offset.y,
1200 offset.x + extent.width,
1201 offset.y + extent.height,
1202 false, color_value,
1203 clear_depth, value.depth,
1204 clear_stencil ? 0xff : 0, value.stencil);
1205 }
1206 }
1207 return;
1208 }
1209
1210 for (uint32_t r = 0; r < rectCount; ++r) {
1211 const VkOffset2D offset = pRects[r].rect.offset;
1212 const VkExtent2D extent = pRects[r].rect.extent;
1213 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1214 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1215 blorp_clear_attachments(batch, binding_table,
1216 depth_format, pass_att->samples,
1217 pRects[r].baseArrayLayer,
1218 pRects[r].layerCount,
1219 offset.x, offset.y,
1220 offset.x + extent.width, offset.y + extent.height,
1221 false, color_value,
1222 clear_depth, value.depth,
1223 clear_stencil ? 0xff : 0, value.stencil);
1224 }
1225 }
1226
1227 void anv_CmdClearAttachments(
1228 VkCommandBuffer commandBuffer,
1229 uint32_t attachmentCount,
1230 const VkClearAttachment* pAttachments,
1231 uint32_t rectCount,
1232 const VkClearRect* pRects)
1233 {
1234 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1235
1236 /* Because this gets called within a render pass, we tell blorp not to
1237 * trash our depth and stencil buffers.
1238 */
1239 struct blorp_batch batch;
1240 enum blorp_batch_flags flags = BLORP_BATCH_NO_EMIT_DEPTH_STENCIL;
1241 if (cmd_buffer->state.conditional_render_enabled) {
1242 anv_cmd_emit_conditional_render_predicate(cmd_buffer);
1243 flags |= BLORP_BATCH_PREDICATE_ENABLE;
1244 }
1245 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, flags);
1246
1247 for (uint32_t a = 0; a < attachmentCount; ++a) {
1248 if (pAttachments[a].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
1249 assert(pAttachments[a].aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
1250 clear_color_attachment(cmd_buffer, &batch,
1251 &pAttachments[a],
1252 rectCount, pRects);
1253 } else {
1254 clear_depth_stencil_attachment(cmd_buffer, &batch,
1255 &pAttachments[a],
1256 rectCount, pRects);
1257 }
1258 }
1259
1260 blorp_batch_finish(&batch);
1261 }
1262
1263 enum subpass_stage {
1264 SUBPASS_STAGE_LOAD,
1265 SUBPASS_STAGE_DRAW,
1266 SUBPASS_STAGE_RESOLVE,
1267 };
1268
1269 void
1270 anv_image_msaa_resolve(struct anv_cmd_buffer *cmd_buffer,
1271 const struct anv_image *src_image,
1272 enum isl_aux_usage src_aux_usage,
1273 uint32_t src_level, uint32_t src_base_layer,
1274 const struct anv_image *dst_image,
1275 enum isl_aux_usage dst_aux_usage,
1276 uint32_t dst_level, uint32_t dst_base_layer,
1277 VkImageAspectFlagBits aspect,
1278 uint32_t src_x, uint32_t src_y,
1279 uint32_t dst_x, uint32_t dst_y,
1280 uint32_t width, uint32_t height,
1281 uint32_t layer_count,
1282 enum blorp_filter filter)
1283 {
1284 struct blorp_batch batch;
1285 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1286
1287 assert(src_image->type == VK_IMAGE_TYPE_2D);
1288 assert(src_image->samples > 1);
1289 assert(dst_image->type == VK_IMAGE_TYPE_2D);
1290 assert(dst_image->samples == 1);
1291 assert(src_image->n_planes == dst_image->n_planes);
1292 assert(!src_image->format->can_ycbcr);
1293 assert(!dst_image->format->can_ycbcr);
1294
1295 struct blorp_surf src_surf, dst_surf;
1296 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, aspect,
1297 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1298 src_aux_usage, &src_surf);
1299 if (src_aux_usage == ISL_AUX_USAGE_MCS) {
1300 src_surf.clear_color_addr = anv_to_blorp_address(
1301 anv_image_get_clear_color_addr(cmd_buffer->device, src_image,
1302 VK_IMAGE_ASPECT_COLOR_BIT));
1303 }
1304 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, aspect,
1305 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1306 dst_aux_usage, &dst_surf);
1307 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
1308 aspect, dst_aux_usage,
1309 dst_level, dst_base_layer, layer_count);
1310
1311 if (filter == BLORP_FILTER_NONE) {
1312 /* If no explicit filter is provided, then it's implied by the type of
1313 * the source image.
1314 */
1315 if ((src_surf.surf->usage & ISL_SURF_USAGE_DEPTH_BIT) ||
1316 (src_surf.surf->usage & ISL_SURF_USAGE_STENCIL_BIT) ||
1317 isl_format_has_int_channel(src_surf.surf->format)) {
1318 filter = BLORP_FILTER_SAMPLE_0;
1319 } else {
1320 filter = BLORP_FILTER_AVERAGE;
1321 }
1322 }
1323
1324 for (uint32_t l = 0; l < layer_count; l++) {
1325 blorp_blit(&batch,
1326 &src_surf, src_level, src_base_layer + l,
1327 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1328 &dst_surf, dst_level, dst_base_layer + l,
1329 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1330 src_x, src_y, src_x + width, src_y + height,
1331 dst_x, dst_y, dst_x + width, dst_y + height,
1332 filter, false, false);
1333 }
1334
1335 blorp_batch_finish(&batch);
1336 }
1337
1338 void anv_CmdResolveImage(
1339 VkCommandBuffer commandBuffer,
1340 VkImage srcImage,
1341 VkImageLayout srcImageLayout,
1342 VkImage dstImage,
1343 VkImageLayout dstImageLayout,
1344 uint32_t regionCount,
1345 const VkImageResolve* pRegions)
1346 {
1347 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1348 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
1349 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
1350
1351 assert(!src_image->format->can_ycbcr);
1352
1353 for (uint32_t r = 0; r < regionCount; r++) {
1354 assert(pRegions[r].srcSubresource.aspectMask ==
1355 pRegions[r].dstSubresource.aspectMask);
1356 assert(anv_get_layerCount(src_image, &pRegions[r].srcSubresource) ==
1357 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource));
1358
1359 const uint32_t layer_count =
1360 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource);
1361
1362 uint32_t aspect_bit;
1363 anv_foreach_image_aspect_bit(aspect_bit, src_image,
1364 pRegions[r].srcSubresource.aspectMask) {
1365 enum isl_aux_usage src_aux_usage =
1366 anv_layout_to_aux_usage(&cmd_buffer->device->info, src_image,
1367 (1 << aspect_bit), srcImageLayout);
1368 enum isl_aux_usage dst_aux_usage =
1369 anv_layout_to_aux_usage(&cmd_buffer->device->info, dst_image,
1370 (1 << aspect_bit), dstImageLayout);
1371
1372 anv_image_msaa_resolve(cmd_buffer,
1373 src_image, src_aux_usage,
1374 pRegions[r].srcSubresource.mipLevel,
1375 pRegions[r].srcSubresource.baseArrayLayer,
1376 dst_image, dst_aux_usage,
1377 pRegions[r].dstSubresource.mipLevel,
1378 pRegions[r].dstSubresource.baseArrayLayer,
1379 (1 << aspect_bit),
1380 pRegions[r].srcOffset.x,
1381 pRegions[r].srcOffset.y,
1382 pRegions[r].dstOffset.x,
1383 pRegions[r].dstOffset.y,
1384 pRegions[r].extent.width,
1385 pRegions[r].extent.height,
1386 layer_count, BLORP_FILTER_NONE);
1387 }
1388 }
1389 }
1390
1391 static enum isl_aux_usage
1392 fast_clear_aux_usage(const struct anv_image *image,
1393 VkImageAspectFlagBits aspect)
1394 {
1395 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
1396 if (image->planes[plane].aux_usage == ISL_AUX_USAGE_NONE)
1397 return ISL_AUX_USAGE_CCS_D;
1398 else
1399 return image->planes[plane].aux_usage;
1400 }
1401
1402 void
1403 anv_image_copy_to_shadow(struct anv_cmd_buffer *cmd_buffer,
1404 const struct anv_image *image,
1405 VkImageAspectFlagBits aspect,
1406 uint32_t base_level, uint32_t level_count,
1407 uint32_t base_layer, uint32_t layer_count)
1408 {
1409 struct blorp_batch batch;
1410 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1411
1412 /* We don't know who touched the main surface last so flush a bunch of
1413 * caches to ensure we get good data.
1414 */
1415 cmd_buffer->state.pending_pipe_bits |=
1416 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT |
1417 ANV_PIPE_DATA_CACHE_FLUSH_BIT |
1418 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT |
1419 ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
1420
1421 struct blorp_surf surf;
1422 get_blorp_surf_for_anv_image(cmd_buffer->device,
1423 image, aspect,
1424 VK_IMAGE_LAYOUT_GENERAL,
1425 ISL_AUX_USAGE_NONE, &surf);
1426 assert(surf.aux_usage == ISL_AUX_USAGE_NONE);
1427
1428 struct blorp_surf shadow_surf;
1429 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
1430 image, aspect, &shadow_surf);
1431
1432 for (uint32_t l = 0; l < level_count; l++) {
1433 const uint32_t level = base_level + l;
1434
1435 const VkExtent3D extent = {
1436 .width = anv_minify(image->extent.width, level),
1437 .height = anv_minify(image->extent.height, level),
1438 .depth = anv_minify(image->extent.depth, level),
1439 };
1440
1441 if (image->type == VK_IMAGE_TYPE_3D)
1442 layer_count = extent.depth;
1443
1444 for (uint32_t a = 0; a < layer_count; a++) {
1445 const uint32_t layer = base_layer + a;
1446
1447 blorp_copy(&batch, &surf, level, layer,
1448 &shadow_surf, level, layer,
1449 0, 0, 0, 0, extent.width, extent.height);
1450 }
1451 }
1452
1453 /* We just wrote to the buffer with the render cache. Flush it. */
1454 cmd_buffer->state.pending_pipe_bits |=
1455 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
1456
1457 blorp_batch_finish(&batch);
1458 }
1459
1460 void
1461 anv_image_clear_color(struct anv_cmd_buffer *cmd_buffer,
1462 const struct anv_image *image,
1463 VkImageAspectFlagBits aspect,
1464 enum isl_aux_usage aux_usage,
1465 enum isl_format format, struct isl_swizzle swizzle,
1466 uint32_t level, uint32_t base_layer, uint32_t layer_count,
1467 VkRect2D area, union isl_color_value clear_color)
1468 {
1469 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1470
1471 /* We don't support planar images with multisampling yet */
1472 assert(image->n_planes == 1);
1473
1474 struct blorp_batch batch;
1475 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1476
1477 struct blorp_surf surf;
1478 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1479 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1480 aux_usage, &surf);
1481 anv_cmd_buffer_mark_image_written(cmd_buffer, image, aspect, aux_usage,
1482 level, base_layer, layer_count);
1483
1484 blorp_clear(&batch, &surf, format, anv_swizzle_for_render(swizzle),
1485 level, base_layer, layer_count,
1486 area.offset.x, area.offset.y,
1487 area.offset.x + area.extent.width,
1488 area.offset.y + area.extent.height,
1489 clear_color, NULL);
1490
1491 blorp_batch_finish(&batch);
1492 }
1493
1494 void
1495 anv_image_clear_depth_stencil(struct anv_cmd_buffer *cmd_buffer,
1496 const struct anv_image *image,
1497 VkImageAspectFlags aspects,
1498 enum isl_aux_usage depth_aux_usage,
1499 uint32_t level,
1500 uint32_t base_layer, uint32_t layer_count,
1501 VkRect2D area,
1502 float depth_value, uint8_t stencil_value)
1503 {
1504 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
1505 VK_IMAGE_ASPECT_STENCIL_BIT));
1506
1507 struct blorp_batch batch;
1508 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1509
1510 struct blorp_surf depth = {};
1511 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1512 get_blorp_surf_for_anv_image(cmd_buffer->device,
1513 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1514 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1515 depth_aux_usage, &depth);
1516 depth.clear_color.f32[0] = ANV_HZ_FC_VAL;
1517 }
1518
1519 struct blorp_surf stencil = {};
1520 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1521 get_blorp_surf_for_anv_image(cmd_buffer->device,
1522 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1523 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1524 ISL_AUX_USAGE_NONE, &stencil);
1525 }
1526
1527 blorp_clear_depth_stencil(&batch, &depth, &stencil,
1528 level, base_layer, layer_count,
1529 area.offset.x, area.offset.y,
1530 area.offset.x + area.extent.width,
1531 area.offset.y + area.extent.height,
1532 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
1533 depth_value,
1534 (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) ? 0xff : 0,
1535 stencil_value);
1536
1537 struct blorp_surf stencil_shadow;
1538 if ((aspects & VK_IMAGE_ASPECT_STENCIL_BIT) &&
1539 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device, image,
1540 VK_IMAGE_ASPECT_STENCIL_BIT,
1541 &stencil_shadow)) {
1542 union isl_color_value stencil_color = {
1543 .u32 = { stencil_value },
1544 };
1545 blorp_clear(&batch, &stencil_shadow,
1546 ISL_FORMAT_R8_UINT, ISL_SWIZZLE_IDENTITY,
1547 level, base_layer, layer_count,
1548 area.offset.x, area.offset.y,
1549 area.offset.x + area.extent.width,
1550 area.offset.y + area.extent.height,
1551 stencil_color, NULL);
1552 }
1553
1554 blorp_batch_finish(&batch);
1555 }
1556
1557 void
1558 anv_image_hiz_op(struct anv_cmd_buffer *cmd_buffer,
1559 const struct anv_image *image,
1560 VkImageAspectFlagBits aspect, uint32_t level,
1561 uint32_t base_layer, uint32_t layer_count,
1562 enum isl_aux_op hiz_op)
1563 {
1564 assert(aspect == VK_IMAGE_ASPECT_DEPTH_BIT);
1565 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, level));
1566 assert(anv_image_aspect_to_plane(image->aspects,
1567 VK_IMAGE_ASPECT_DEPTH_BIT) == 0);
1568
1569 struct blorp_batch batch;
1570 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1571
1572 struct blorp_surf surf;
1573 get_blorp_surf_for_anv_image(cmd_buffer->device,
1574 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1575 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1576 ISL_AUX_USAGE_HIZ, &surf);
1577 surf.clear_color.f32[0] = ANV_HZ_FC_VAL;
1578
1579 blorp_hiz_op(&batch, &surf, level, base_layer, layer_count, hiz_op);
1580
1581 blorp_batch_finish(&batch);
1582 }
1583
1584 void
1585 anv_image_hiz_clear(struct anv_cmd_buffer *cmd_buffer,
1586 const struct anv_image *image,
1587 VkImageAspectFlags aspects,
1588 uint32_t level,
1589 uint32_t base_layer, uint32_t layer_count,
1590 VkRect2D area, uint8_t stencil_value)
1591 {
1592 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
1593 VK_IMAGE_ASPECT_STENCIL_BIT));
1594
1595 struct blorp_batch batch;
1596 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1597
1598 struct blorp_surf depth = {};
1599 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1600 assert(base_layer + layer_count <=
1601 anv_image_aux_layers(image, VK_IMAGE_ASPECT_DEPTH_BIT, level));
1602 get_blorp_surf_for_anv_image(cmd_buffer->device,
1603 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1604 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1605 ISL_AUX_USAGE_HIZ, &depth);
1606 depth.clear_color.f32[0] = ANV_HZ_FC_VAL;
1607 }
1608
1609 struct blorp_surf stencil = {};
1610 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1611 get_blorp_surf_for_anv_image(cmd_buffer->device,
1612 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1613 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1614 ISL_AUX_USAGE_NONE, &stencil);
1615 }
1616
1617 /* From the Sky Lake PRM Volume 7, "Depth Buffer Clear":
1618 *
1619 * "The following is required when performing a depth buffer clear with
1620 * using the WM_STATE or 3DSTATE_WM:
1621 *
1622 * * If other rendering operations have preceded this clear, a
1623 * PIPE_CONTROL with depth cache flush enabled, Depth Stall bit
1624 * enabled must be issued before the rectangle primitive used for
1625 * the depth buffer clear operation.
1626 * * [...]"
1627 *
1628 * Even though the PRM only says that this is required if using 3DSTATE_WM
1629 * and a 3DPRIMITIVE, the GPU appears to also need this to avoid occasional
1630 * hangs when doing a clear with WM_HZ_OP.
1631 */
1632 cmd_buffer->state.pending_pipe_bits |=
1633 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT;
1634
1635 blorp_hiz_clear_depth_stencil(&batch, &depth, &stencil,
1636 level, base_layer, layer_count,
1637 area.offset.x, area.offset.y,
1638 area.offset.x + area.extent.width,
1639 area.offset.y + area.extent.height,
1640 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
1641 ANV_HZ_FC_VAL,
1642 aspects & VK_IMAGE_ASPECT_STENCIL_BIT,
1643 stencil_value);
1644
1645 blorp_batch_finish(&batch);
1646
1647 /* From the SKL PRM, Depth Buffer Clear:
1648 *
1649 * "Depth Buffer Clear Workaround
1650 *
1651 * Depth buffer clear pass using any of the methods (WM_STATE,
1652 * 3DSTATE_WM or 3DSTATE_WM_HZ_OP) must be followed by a PIPE_CONTROL
1653 * command with DEPTH_STALL bit and Depth FLUSH bits “set” before
1654 * starting to render. DepthStall and DepthFlush are not needed between
1655 * consecutive depth clear passes nor is it required if the depth-clear
1656 * pass was done with “full_surf_clear” bit set in the
1657 * 3DSTATE_WM_HZ_OP."
1658 *
1659 * Even though the PRM provides a bunch of conditions under which this is
1660 * supposedly unnecessary, we choose to perform the flush unconditionally
1661 * just to be safe.
1662 */
1663 cmd_buffer->state.pending_pipe_bits |=
1664 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT;
1665 }
1666
1667 void
1668 anv_image_mcs_op(struct anv_cmd_buffer *cmd_buffer,
1669 const struct anv_image *image,
1670 enum isl_format format,
1671 VkImageAspectFlagBits aspect,
1672 uint32_t base_layer, uint32_t layer_count,
1673 enum isl_aux_op mcs_op, union isl_color_value *clear_value,
1674 bool predicate)
1675 {
1676 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1677 assert(image->samples > 1);
1678 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, 0));
1679
1680 /* Multisampling with multi-planar formats is not supported */
1681 assert(image->n_planes == 1);
1682
1683 struct blorp_batch batch;
1684 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1685 predicate ? BLORP_BATCH_PREDICATE_ENABLE : 0);
1686
1687 struct blorp_surf surf;
1688 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1689 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1690 ISL_AUX_USAGE_MCS, &surf);
1691
1692 /* Blorp will store the clear color for us if we provide the clear color
1693 * address and we are doing a fast clear. So we save the clear value into
1694 * the blorp surface. However, in some situations we want to do a fast clear
1695 * without changing the clear value stored in the state buffer. For those
1696 * cases, we set the clear color address pointer to NULL, so blorp will not
1697 * try to store a garbage color.
1698 */
1699 if (mcs_op == ISL_AUX_OP_FAST_CLEAR) {
1700 if (clear_value)
1701 surf.clear_color = *clear_value;
1702 else
1703 surf.clear_color_addr.buffer = NULL;
1704 }
1705
1706 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1707 *
1708 * "After Render target fast clear, pipe-control with color cache
1709 * write-flush must be issued before sending any DRAW commands on
1710 * that render target."
1711 *
1712 * This comment is a bit cryptic and doesn't really tell you what's going
1713 * or what's really needed. It appears that fast clear ops are not
1714 * properly synchronized with other drawing. This means that we cannot
1715 * have a fast clear operation in the pipe at the same time as other
1716 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
1717 * that the contents of the previous draw hit the render target before we
1718 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
1719 * that it is completed before any additional drawing occurs.
1720 */
1721 cmd_buffer->state.pending_pipe_bits |=
1722 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1723
1724 switch (mcs_op) {
1725 case ISL_AUX_OP_FAST_CLEAR:
1726 blorp_fast_clear(&batch, &surf, format,
1727 0, base_layer, layer_count,
1728 0, 0, image->extent.width, image->extent.height);
1729 break;
1730 case ISL_AUX_OP_PARTIAL_RESOLVE:
1731 blorp_mcs_partial_resolve(&batch, &surf, format,
1732 base_layer, layer_count);
1733 break;
1734 case ISL_AUX_OP_FULL_RESOLVE:
1735 case ISL_AUX_OP_AMBIGUATE:
1736 default:
1737 unreachable("Unsupported MCS operation");
1738 }
1739
1740 cmd_buffer->state.pending_pipe_bits |=
1741 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1742
1743 blorp_batch_finish(&batch);
1744 }
1745
1746 void
1747 anv_image_ccs_op(struct anv_cmd_buffer *cmd_buffer,
1748 const struct anv_image *image,
1749 enum isl_format format,
1750 VkImageAspectFlagBits aspect, uint32_t level,
1751 uint32_t base_layer, uint32_t layer_count,
1752 enum isl_aux_op ccs_op, union isl_color_value *clear_value,
1753 bool predicate)
1754 {
1755 assert(image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
1756 assert(image->samples == 1);
1757 assert(level < anv_image_aux_levels(image, aspect));
1758 /* Multi-LOD YcBcR is not allowed */
1759 assert(image->n_planes == 1 || level == 0);
1760 assert(base_layer + layer_count <=
1761 anv_image_aux_layers(image, aspect, level));
1762
1763 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
1764 uint32_t width_div = image->format->planes[plane].denominator_scales[0];
1765 uint32_t height_div = image->format->planes[plane].denominator_scales[1];
1766 uint32_t level_width = anv_minify(image->extent.width, level) / width_div;
1767 uint32_t level_height = anv_minify(image->extent.height, level) / height_div;
1768
1769 struct blorp_batch batch;
1770 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1771 predicate ? BLORP_BATCH_PREDICATE_ENABLE : 0);
1772
1773 struct blorp_surf surf;
1774 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1775 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1776 fast_clear_aux_usage(image, aspect),
1777 &surf);
1778
1779 /* Blorp will store the clear color for us if we provide the clear color
1780 * address and we are doing a fast clear. So we save the clear value into
1781 * the blorp surface. However, in some situations we want to do a fast clear
1782 * without changing the clear value stored in the state buffer. For those
1783 * cases, we set the clear color address pointer to NULL, so blorp will not
1784 * try to store a garbage color.
1785 */
1786 if (ccs_op == ISL_AUX_OP_FAST_CLEAR) {
1787 if (clear_value)
1788 surf.clear_color = *clear_value;
1789 else
1790 surf.clear_color_addr.buffer = NULL;
1791 }
1792
1793 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1794 *
1795 * "After Render target fast clear, pipe-control with color cache
1796 * write-flush must be issued before sending any DRAW commands on
1797 * that render target."
1798 *
1799 * This comment is a bit cryptic and doesn't really tell you what's going
1800 * or what's really needed. It appears that fast clear ops are not
1801 * properly synchronized with other drawing. This means that we cannot
1802 * have a fast clear operation in the pipe at the same time as other
1803 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
1804 * that the contents of the previous draw hit the render target before we
1805 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
1806 * that it is completed before any additional drawing occurs.
1807 */
1808 cmd_buffer->state.pending_pipe_bits |=
1809 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1810
1811 switch (ccs_op) {
1812 case ISL_AUX_OP_FAST_CLEAR:
1813 blorp_fast_clear(&batch, &surf, format,
1814 level, base_layer, layer_count,
1815 0, 0, level_width, level_height);
1816 break;
1817 case ISL_AUX_OP_FULL_RESOLVE:
1818 case ISL_AUX_OP_PARTIAL_RESOLVE:
1819 blorp_ccs_resolve(&batch, &surf, level, base_layer, layer_count,
1820 format, ccs_op);
1821 break;
1822 case ISL_AUX_OP_AMBIGUATE:
1823 for (uint32_t a = 0; a < layer_count; a++) {
1824 const uint32_t layer = base_layer + a;
1825 blorp_ccs_ambiguate(&batch, &surf, level, layer);
1826 }
1827 break;
1828 default:
1829 unreachable("Unsupported CCS operation");
1830 }
1831
1832 cmd_buffer->state.pending_pipe_bits |=
1833 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1834
1835 blorp_batch_finish(&batch);
1836 }