projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
anv: narrow flushing of the render target to buffer writes
[mesa.git] / src / intel / vulkan / anv_blorp.c
1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "anv_private.h"
25
26 static bool
27 lookup_blorp_shader(struct blorp_batch *batch,
28 const void *key, uint32_t key_size,
29 uint32_t *kernel_out, void *prog_data_out)
30 {
31 struct blorp_context *blorp = batch->blorp;
32 struct anv_device *device = blorp->driver_ctx;
33
34 /* The default cache must be a real cache */
35 assert(device->default_pipeline_cache.cache);
36
37 struct anv_shader_bin *bin =
38 anv_pipeline_cache_search(&device->default_pipeline_cache, key, key_size);
39 if (!bin)
40 return false;
41
42 /* The cache already has a reference and it's not going anywhere so there
43 * is no need to hold a second reference.
44 */
45 anv_shader_bin_unref(device, bin);
46
47 *kernel_out = bin->kernel.offset;
48 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
49
50 return true;
51 }
52
53 static bool
54 upload_blorp_shader(struct blorp_batch *batch,
55 const void *key, uint32_t key_size,
56 const void *kernel, uint32_t kernel_size,
57 const struct brw_stage_prog_data *prog_data,
58 uint32_t prog_data_size,
59 uint32_t *kernel_out, void *prog_data_out)
60 {
61 struct blorp_context *blorp = batch->blorp;
62 struct anv_device *device = blorp->driver_ctx;
63
64 /* The blorp cache must be a real cache */
65 assert(device->default_pipeline_cache.cache);
66
67 struct anv_pipeline_bind_map bind_map = {
68 .surface_count = 0,
69 .sampler_count = 0,
70 };
71
72 struct anv_shader_bin *bin =
73 anv_pipeline_cache_upload_kernel(&device->default_pipeline_cache,
74 key, key_size, kernel, kernel_size,
75 NULL, 0,
76 prog_data, prog_data_size, &bind_map);
77
78 if (!bin)
79 return false;
80
81 /* The cache already has a reference and it's not going anywhere so there
82 * is no need to hold a second reference.
83 */
84 anv_shader_bin_unref(device, bin);
85
86 *kernel_out = bin->kernel.offset;
87 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
88
89 return true;
90 }
91
92 void
93 anv_device_init_blorp(struct anv_device *device)
94 {
95 blorp_init(&device->blorp, device, &device->isl_dev);
96 device->blorp.compiler = device->instance->physicalDevice.compiler;
97 device->blorp.lookup_shader = lookup_blorp_shader;
98 device->blorp.upload_shader = upload_blorp_shader;
99 switch (device->info.gen) {
100 case 7:
101 if (device->info.is_haswell) {
102 device->blorp.exec = gen75_blorp_exec;
103 } else {
104 device->blorp.exec = gen7_blorp_exec;
105 }
106 break;
107 case 8:
108 device->blorp.exec = gen8_blorp_exec;
109 break;
110 case 9:
111 device->blorp.exec = gen9_blorp_exec;
112 break;
113 case 10:
114 device->blorp.exec = gen10_blorp_exec;
115 break;
116 case 11:
117 device->blorp.exec = gen11_blorp_exec;
118 break;
119 default:
120 unreachable("Unknown hardware generation");
121 }
122 }
123
124 void
125 anv_device_finish_blorp(struct anv_device *device)
126 {
127 blorp_finish(&device->blorp);
128 }
129
130 static void
131 get_blorp_surf_for_anv_buffer(struct anv_device *device,
132 struct anv_buffer *buffer, uint64_t offset,
133 uint32_t width, uint32_t height,
134 uint32_t row_pitch, enum isl_format format,
135 struct blorp_surf *blorp_surf,
136 struct isl_surf *isl_surf)
137 {
138 const struct isl_format_layout *fmtl =
139 isl_format_get_layout(format);
140 bool ok UNUSED;
141
142 /* ASTC is the only format which doesn't support linear layouts.
143 * Create an equivalently sized surface with ISL to get around this.
144 */
145 if (fmtl->txc == ISL_TXC_ASTC) {
146 /* Use an equivalently sized format */
147 format = ISL_FORMAT_R32G32B32A32_UINT;
148 assert(fmtl->bpb == isl_format_get_layout(format)->bpb);
149
150 /* Shrink the dimensions for the new format */
151 width = DIV_ROUND_UP(width, fmtl->bw);
152 height = DIV_ROUND_UP(height, fmtl->bh);
153 }
154
155 *blorp_surf = (struct blorp_surf) {
156 .surf = isl_surf,
157 .addr = {
158 .buffer = buffer->address.bo,
159 .offset = buffer->address.offset + offset,
160 .mocs = anv_mocs_for_bo(device, buffer->address.bo),
161 },
162 };
163
164 ok = isl_surf_init(&device->isl_dev, isl_surf,
165 .dim = ISL_SURF_DIM_2D,
166 .format = format,
167 .width = width,
168 .height = height,
169 .depth = 1,
170 .levels = 1,
171 .array_len = 1,
172 .samples = 1,
173 .row_pitch_B = row_pitch,
174 .usage = ISL_SURF_USAGE_TEXTURE_BIT |
175 ISL_SURF_USAGE_RENDER_TARGET_BIT,
176 .tiling_flags = ISL_TILING_LINEAR_BIT);
177 assert(ok);
178 }
179
180 /* Pick something high enough that it won't be used in core and low enough it
181 * will never map to an extension.
182 */
183 #define ANV_IMAGE_LAYOUT_EXPLICIT_AUX (VkImageLayout)10000000
184
185 static struct blorp_address
186 anv_to_blorp_address(struct anv_address addr)
187 {
188 return (struct blorp_address) {
189 .buffer = addr.bo,
190 .offset = addr.offset,
191 };
192 }
193
194 static void
195 get_blorp_surf_for_anv_image(const struct anv_device *device,
196 const struct anv_image *image,
197 VkImageAspectFlags aspect,
198 VkImageLayout layout,
199 enum isl_aux_usage aux_usage,
200 struct blorp_surf *blorp_surf)
201 {
202 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
203
204 if (layout != ANV_IMAGE_LAYOUT_EXPLICIT_AUX)
205 aux_usage = anv_layout_to_aux_usage(&device->info, image, aspect, layout);
206
207 const struct anv_surface *surface = &image->planes[plane].surface;
208 *blorp_surf = (struct blorp_surf) {
209 .surf = &surface->isl,
210 .addr = {
211 .buffer = image->planes[plane].address.bo,
212 .offset = image->planes[plane].address.offset + surface->offset,
213 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
214 },
215 };
216
217 if (aux_usage != ISL_AUX_USAGE_NONE) {
218 const struct anv_surface *aux_surface = &image->planes[plane].aux_surface;
219 blorp_surf->aux_surf = &aux_surface->isl,
220 blorp_surf->aux_addr = (struct blorp_address) {
221 .buffer = image->planes[plane].address.bo,
222 .offset = image->planes[plane].address.offset + aux_surface->offset,
223 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
224 };
225 blorp_surf->aux_usage = aux_usage;
226
227 /* If we're doing a partial resolve, then we need the indirect clear
228 * color. If we are doing a fast clear and want to store/update the
229 * clear color, we also pass the address to blorp, otherwise it will only
230 * stomp the CCS to a particular value and won't care about format or
231 * clear value
232 */
233 if (aspect & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
234 const struct anv_address clear_color_addr =
235 anv_image_get_clear_color_addr(device, image, aspect);
236 blorp_surf->clear_color_addr = anv_to_blorp_address(clear_color_addr);
237 } else if (aspect & VK_IMAGE_ASPECT_DEPTH_BIT
238 && device->info.gen >= 10) {
239 /* Vulkan always clears to 1.0. On gen < 10, we set that directly in
240 * the state packet. For gen >= 10, must provide the clear value in a
241 * buffer. We have a single global buffer that stores the 1.0 value.
242 */
243 const struct anv_address clear_color_addr = (struct anv_address) {
244 .bo = (struct anv_bo *)&device->hiz_clear_bo
245 };
246 blorp_surf->clear_color_addr = anv_to_blorp_address(clear_color_addr);
247 }
248 }
249 }
250
251 void anv_CmdCopyImage(
252 VkCommandBuffer commandBuffer,
253 VkImage srcImage,
254 VkImageLayout srcImageLayout,
255 VkImage dstImage,
256 VkImageLayout dstImageLayout,
257 uint32_t regionCount,
258 const VkImageCopy* pRegions)
259 {
260 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
261 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
262 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
263
264 struct blorp_batch batch;
265 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
266
267 for (unsigned r = 0; r < regionCount; r++) {
268 VkOffset3D srcOffset =
269 anv_sanitize_image_offset(src_image->type, pRegions[r].srcOffset);
270 VkOffset3D dstOffset =
271 anv_sanitize_image_offset(dst_image->type, pRegions[r].dstOffset);
272 VkExtent3D extent =
273 anv_sanitize_image_extent(src_image->type, pRegions[r].extent);
274
275 const uint32_t dst_level = pRegions[r].dstSubresource.mipLevel;
276 unsigned dst_base_layer, layer_count;
277 if (dst_image->type == VK_IMAGE_TYPE_3D) {
278 dst_base_layer = pRegions[r].dstOffset.z;
279 layer_count = pRegions[r].extent.depth;
280 } else {
281 dst_base_layer = pRegions[r].dstSubresource.baseArrayLayer;
282 layer_count =
283 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource);
284 }
285
286 const uint32_t src_level = pRegions[r].srcSubresource.mipLevel;
287 unsigned src_base_layer;
288 if (src_image->type == VK_IMAGE_TYPE_3D) {
289 src_base_layer = pRegions[r].srcOffset.z;
290 } else {
291 src_base_layer = pRegions[r].srcSubresource.baseArrayLayer;
292 assert(layer_count ==
293 anv_get_layerCount(src_image, &pRegions[r].srcSubresource));
294 }
295
296 VkImageAspectFlags src_mask = pRegions[r].srcSubresource.aspectMask,
297 dst_mask = pRegions[r].dstSubresource.aspectMask;
298
299 assert(anv_image_aspects_compatible(src_mask, dst_mask));
300
301 if (util_bitcount(src_mask) > 1) {
302 uint32_t aspect_bit;
303 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_mask) {
304 struct blorp_surf src_surf, dst_surf;
305 get_blorp_surf_for_anv_image(cmd_buffer->device,
306 src_image, 1UL << aspect_bit,
307 srcImageLayout, ISL_AUX_USAGE_NONE,
308 &src_surf);
309 get_blorp_surf_for_anv_image(cmd_buffer->device,
310 dst_image, 1UL << aspect_bit,
311 dstImageLayout, ISL_AUX_USAGE_NONE,
312 &dst_surf);
313 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
314 1UL << aspect_bit,
315 dst_surf.aux_usage, dst_level,
316 dst_base_layer, layer_count);
317
318 for (unsigned i = 0; i < layer_count; i++) {
319 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
320 &dst_surf, dst_level, dst_base_layer + i,
321 srcOffset.x, srcOffset.y,
322 dstOffset.x, dstOffset.y,
323 extent.width, extent.height);
324 }
325 }
326 } else {
327 struct blorp_surf src_surf, dst_surf;
328 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, src_mask,
329 srcImageLayout, ISL_AUX_USAGE_NONE,
330 &src_surf);
331 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, dst_mask,
332 dstImageLayout, ISL_AUX_USAGE_NONE,
333 &dst_surf);
334 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image, dst_mask,
335 dst_surf.aux_usage, dst_level,
336 dst_base_layer, layer_count);
337
338 for (unsigned i = 0; i < layer_count; i++) {
339 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
340 &dst_surf, dst_level, dst_base_layer + i,
341 srcOffset.x, srcOffset.y,
342 dstOffset.x, dstOffset.y,
343 extent.width, extent.height);
344 }
345 }
346 }
347
348 blorp_batch_finish(&batch);
349 }
350
351 static void
352 copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer,
353 struct anv_buffer *anv_buffer,
354 struct anv_image *anv_image,
355 VkImageLayout image_layout,
356 uint32_t regionCount,
357 const VkBufferImageCopy* pRegions,
358 bool buffer_to_image)
359 {
360 struct blorp_batch batch;
361 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
362
363 struct {
364 struct blorp_surf surf;
365 uint32_t level;
366 VkOffset3D offset;
367 } image, buffer, *src, *dst;
368
369 buffer.level = 0;
370 buffer.offset = (VkOffset3D) { 0, 0, 0 };
371
372 if (buffer_to_image) {
373 src = &buffer;
374 dst = &image;
375 } else {
376 src = &image;
377 dst = &buffer;
378 }
379
380 for (unsigned r = 0; r < regionCount; r++) {
381 const VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask;
382
383 get_blorp_surf_for_anv_image(cmd_buffer->device, anv_image, aspect,
384 image_layout, ISL_AUX_USAGE_NONE,
385 &image.surf);
386 image.offset =
387 anv_sanitize_image_offset(anv_image->type, pRegions[r].imageOffset);
388 image.level = pRegions[r].imageSubresource.mipLevel;
389
390 VkExtent3D extent =
391 anv_sanitize_image_extent(anv_image->type, pRegions[r].imageExtent);
392 if (anv_image->type != VK_IMAGE_TYPE_3D) {
393 image.offset.z = pRegions[r].imageSubresource.baseArrayLayer;
394 extent.depth =
395 anv_get_layerCount(anv_image, &pRegions[r].imageSubresource);
396 }
397
398 const enum isl_format buffer_format =
399 anv_get_isl_format(&cmd_buffer->device->info, anv_image->vk_format,
400 aspect, VK_IMAGE_TILING_LINEAR);
401
402 const VkExtent3D bufferImageExtent = {
403 .width = pRegions[r].bufferRowLength ?
404 pRegions[r].bufferRowLength : extent.width,
405 .height = pRegions[r].bufferImageHeight ?
406 pRegions[r].bufferImageHeight : extent.height,
407 };
408
409 const struct isl_format_layout *buffer_fmtl =
410 isl_format_get_layout(buffer_format);
411
412 const uint32_t buffer_row_pitch =
413 DIV_ROUND_UP(bufferImageExtent.width, buffer_fmtl->bw) *
414 (buffer_fmtl->bpb / 8);
415
416 const uint32_t buffer_layer_stride =
417 DIV_ROUND_UP(bufferImageExtent.height, buffer_fmtl->bh) *
418 buffer_row_pitch;
419
420 struct isl_surf buffer_isl_surf;
421 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
422 anv_buffer, pRegions[r].bufferOffset,
423 extent.width, extent.height,
424 buffer_row_pitch, buffer_format,
425 &buffer.surf, &buffer_isl_surf);
426
427 if (&image == dst) {
428 anv_cmd_buffer_mark_image_written(cmd_buffer, anv_image,
429 aspect, dst->surf.aux_usage,
430 dst->level,
431 dst->offset.z, extent.depth);
432 }
433
434 for (unsigned z = 0; z < extent.depth; z++) {
435 blorp_copy(&batch, &src->surf, src->level, src->offset.z,
436 &dst->surf, dst->level, dst->offset.z,
437 src->offset.x, src->offset.y, dst->offset.x, dst->offset.y,
438 extent.width, extent.height);
439
440 image.offset.z++;
441 buffer.surf.addr.offset += buffer_layer_stride;
442 }
443 }
444
445 blorp_batch_finish(&batch);
446 }
447
448 void anv_CmdCopyBufferToImage(
449 VkCommandBuffer commandBuffer,
450 VkBuffer srcBuffer,
451 VkImage dstImage,
452 VkImageLayout dstImageLayout,
453 uint32_t regionCount,
454 const VkBufferImageCopy* pRegions)
455 {
456 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
457 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
458 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
459
460 copy_buffer_to_image(cmd_buffer, src_buffer, dst_image, dstImageLayout,
461 regionCount, pRegions, true);
462 }
463
464 void anv_CmdCopyImageToBuffer(
465 VkCommandBuffer commandBuffer,
466 VkImage srcImage,
467 VkImageLayout srcImageLayout,
468 VkBuffer dstBuffer,
469 uint32_t regionCount,
470 const VkBufferImageCopy* pRegions)
471 {
472 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
473 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
474 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
475
476 copy_buffer_to_image(cmd_buffer, dst_buffer, src_image, srcImageLayout,
477 regionCount, pRegions, false);
478
479 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
480 }
481
482 static bool
483 flip_coords(unsigned *src0, unsigned *src1, unsigned *dst0, unsigned *dst1)
484 {
485 bool flip = false;
486 if (*src0 > *src1) {
487 unsigned tmp = *src0;
488 *src0 = *src1;
489 *src1 = tmp;
490 flip = !flip;
491 }
492
493 if (*dst0 > *dst1) {
494 unsigned tmp = *dst0;
495 *dst0 = *dst1;
496 *dst1 = tmp;
497 flip = !flip;
498 }
499
500 return flip;
501 }
502
503 void anv_CmdBlitImage(
504 VkCommandBuffer commandBuffer,
505 VkImage srcImage,
506 VkImageLayout srcImageLayout,
507 VkImage dstImage,
508 VkImageLayout dstImageLayout,
509 uint32_t regionCount,
510 const VkImageBlit* pRegions,
511 VkFilter filter)
512
513 {
514 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
515 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
516 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
517
518 struct blorp_surf src, dst;
519
520 enum blorp_filter blorp_filter;
521 switch (filter) {
522 case VK_FILTER_NEAREST:
523 blorp_filter = BLORP_FILTER_NEAREST;
524 break;
525 case VK_FILTER_LINEAR:
526 blorp_filter = BLORP_FILTER_BILINEAR;
527 break;
528 default:
529 unreachable("Invalid filter");
530 }
531
532 struct blorp_batch batch;
533 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
534
535 for (unsigned r = 0; r < regionCount; r++) {
536 const VkImageSubresourceLayers *src_res = &pRegions[r].srcSubresource;
537 const VkImageSubresourceLayers *dst_res = &pRegions[r].dstSubresource;
538
539 assert(anv_image_aspects_compatible(src_res->aspectMask,
540 dst_res->aspectMask));
541
542 uint32_t aspect_bit;
543 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_res->aspectMask) {
544 get_blorp_surf_for_anv_image(cmd_buffer->device,
545 src_image, 1U << aspect_bit,
546 srcImageLayout, ISL_AUX_USAGE_NONE, &src);
547 get_blorp_surf_for_anv_image(cmd_buffer->device,
548 dst_image, 1U << aspect_bit,
549 dstImageLayout, ISL_AUX_USAGE_NONE, &dst);
550
551 struct anv_format_plane src_format =
552 anv_get_format_plane(&cmd_buffer->device->info, src_image->vk_format,
553 1U << aspect_bit, src_image->tiling);
554 struct anv_format_plane dst_format =
555 anv_get_format_plane(&cmd_buffer->device->info, dst_image->vk_format,
556 1U << aspect_bit, dst_image->tiling);
557
558 unsigned dst_start, dst_end;
559 if (dst_image->type == VK_IMAGE_TYPE_3D) {
560 assert(dst_res->baseArrayLayer == 0);
561 dst_start = pRegions[r].dstOffsets[0].z;
562 dst_end = pRegions[r].dstOffsets[1].z;
563 } else {
564 dst_start = dst_res->baseArrayLayer;
565 dst_end = dst_start + anv_get_layerCount(dst_image, dst_res);
566 }
567
568 unsigned src_start, src_end;
569 if (src_image->type == VK_IMAGE_TYPE_3D) {
570 assert(src_res->baseArrayLayer == 0);
571 src_start = pRegions[r].srcOffsets[0].z;
572 src_end = pRegions[r].srcOffsets[1].z;
573 } else {
574 src_start = src_res->baseArrayLayer;
575 src_end = src_start + anv_get_layerCount(src_image, src_res);
576 }
577
578 bool flip_z = flip_coords(&src_start, &src_end, &dst_start, &dst_end);
579 float src_z_step = (float)(src_end + 1 - src_start) /
580 (float)(dst_end + 1 - dst_start);
581
582 if (flip_z) {
583 src_start = src_end;
584 src_z_step *= -1;
585 }
586
587 unsigned src_x0 = pRegions[r].srcOffsets[0].x;
588 unsigned src_x1 = pRegions[r].srcOffsets[1].x;
589 unsigned dst_x0 = pRegions[r].dstOffsets[0].x;
590 unsigned dst_x1 = pRegions[r].dstOffsets[1].x;
591 bool flip_x = flip_coords(&src_x0, &src_x1, &dst_x0, &dst_x1);
592
593 unsigned src_y0 = pRegions[r].srcOffsets[0].y;
594 unsigned src_y1 = pRegions[r].srcOffsets[1].y;
595 unsigned dst_y0 = pRegions[r].dstOffsets[0].y;
596 unsigned dst_y1 = pRegions[r].dstOffsets[1].y;
597 bool flip_y = flip_coords(&src_y0, &src_y1, &dst_y0, &dst_y1);
598
599 const unsigned num_layers = dst_end - dst_start;
600 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
601 1U << aspect_bit,
602 dst.aux_usage,
603 dst_res->mipLevel,
604 dst_start, num_layers);
605
606 for (unsigned i = 0; i < num_layers; i++) {
607 unsigned dst_z = dst_start + i;
608 unsigned src_z = src_start + i * src_z_step;
609
610 blorp_blit(&batch, &src, src_res->mipLevel, src_z,
611 src_format.isl_format, src_format.swizzle,
612 &dst, dst_res->mipLevel, dst_z,
613 dst_format.isl_format, dst_format.swizzle,
614 src_x0, src_y0, src_x1, src_y1,
615 dst_x0, dst_y0, dst_x1, dst_y1,
616 blorp_filter, flip_x, flip_y);
617 }
618 }
619 }
620
621 blorp_batch_finish(&batch);
622 }
623
624 static enum isl_format
625 isl_format_for_size(unsigned size_B)
626 {
627 switch (size_B) {
628 case 4: return ISL_FORMAT_R32_UINT;
629 case 8: return ISL_FORMAT_R32G32_UINT;
630 case 16: return ISL_FORMAT_R32G32B32A32_UINT;
631 default:
632 unreachable("Not a power-of-two format size");
633 }
634 }
635
636 /**
637 * Returns the greatest common divisor of a and b that is a power of two.
638 */
639 static uint64_t
640 gcd_pow2_u64(uint64_t a, uint64_t b)
641 {
642 assert(a > 0 || b > 0);
643
644 unsigned a_log2 = ffsll(a) - 1;
645 unsigned b_log2 = ffsll(b) - 1;
646
647 /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which
648 * case, the MIN2() will take the other one. If both are 0 then we will
649 * hit the assert above.
650 */
651 return 1 << MIN2(a_log2, b_log2);
652 }
653
654 /* This is maximum possible width/height our HW can handle */
655 #define MAX_SURFACE_DIM (1ull << 14)
656
657 void anv_CmdCopyBuffer(
658 VkCommandBuffer commandBuffer,
659 VkBuffer srcBuffer,
660 VkBuffer dstBuffer,
661 uint32_t regionCount,
662 const VkBufferCopy* pRegions)
663 {
664 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
665 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
666 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
667
668 struct blorp_batch batch;
669 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
670
671 for (unsigned r = 0; r < regionCount; r++) {
672 struct blorp_address src = {
673 .buffer = src_buffer->address.bo,
674 .offset = src_buffer->address.offset + pRegions[r].srcOffset,
675 .mocs = anv_mocs_for_bo(cmd_buffer->device, src_buffer->address.bo),
676 };
677 struct blorp_address dst = {
678 .buffer = dst_buffer->address.bo,
679 .offset = dst_buffer->address.offset + pRegions[r].dstOffset,
680 .mocs = anv_mocs_for_bo(cmd_buffer->device, dst_buffer->address.bo),
681 };
682
683 blorp_buffer_copy(&batch, src, dst, pRegions[r].size);
684 }
685
686 blorp_batch_finish(&batch);
687
688 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
689 }
690
691 void anv_CmdUpdateBuffer(
692 VkCommandBuffer commandBuffer,
693 VkBuffer dstBuffer,
694 VkDeviceSize dstOffset,
695 VkDeviceSize dataSize,
696 const void* pData)
697 {
698 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
699 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
700
701 struct blorp_batch batch;
702 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
703
704 /* We can't quite grab a full block because the state stream needs a
705 * little data at the top to build its linked list.
706 */
707 const uint32_t max_update_size =
708 cmd_buffer->device->dynamic_state_pool.block_size - 64;
709
710 assert(max_update_size < MAX_SURFACE_DIM * 4);
711
712 /* We're about to read data that was written from the CPU. Flush the
713 * texture cache so we don't get anything stale.
714 */
715 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
716
717 while (dataSize) {
718 const uint32_t copy_size = MIN2(dataSize, max_update_size);
719
720 struct anv_state tmp_data =
721 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, copy_size, 64);
722
723 memcpy(tmp_data.map, pData, copy_size);
724
725 struct blorp_address src = {
726 .buffer = cmd_buffer->device->dynamic_state_pool.block_pool.bo,
727 .offset = tmp_data.offset,
728 .mocs = cmd_buffer->device->default_mocs,
729 };
730 struct blorp_address dst = {
731 .buffer = dst_buffer->address.bo,
732 .offset = dst_buffer->address.offset + dstOffset,
733 .mocs = anv_mocs_for_bo(cmd_buffer->device, dst_buffer->address.bo),
734 };
735
736 blorp_buffer_copy(&batch, src, dst, copy_size);
737
738 dataSize -= copy_size;
739 dstOffset += copy_size;
740 pData = (void *)pData + copy_size;
741 }
742
743 blorp_batch_finish(&batch);
744
745 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
746 }
747
748 void anv_CmdFillBuffer(
749 VkCommandBuffer commandBuffer,
750 VkBuffer dstBuffer,
751 VkDeviceSize dstOffset,
752 VkDeviceSize fillSize,
753 uint32_t data)
754 {
755 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
756 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
757 struct blorp_surf surf;
758 struct isl_surf isl_surf;
759
760 struct blorp_batch batch;
761 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
762
763 fillSize = anv_buffer_get_range(dst_buffer, dstOffset, fillSize);
764
765 /* From the Vulkan spec:
766 *
767 * "size is the number of bytes to fill, and must be either a multiple
768 * of 4, or VK_WHOLE_SIZE to fill the range from offset to the end of
769 * the buffer. If VK_WHOLE_SIZE is used and the remaining size of the
770 * buffer is not a multiple of 4, then the nearest smaller multiple is
771 * used."
772 */
773 fillSize &= ~3ull;
774
775 /* First, we compute the biggest format that can be used with the
776 * given offsets and size.
777 */
778 int bs = 16;
779 bs = gcd_pow2_u64(bs, dstOffset);
780 bs = gcd_pow2_u64(bs, fillSize);
781 enum isl_format isl_format = isl_format_for_size(bs);
782
783 union isl_color_value color = {
784 .u32 = { data, data, data, data },
785 };
786
787 const uint64_t max_fill_size = MAX_SURFACE_DIM * MAX_SURFACE_DIM * bs;
788 while (fillSize >= max_fill_size) {
789 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
790 dst_buffer, dstOffset,
791 MAX_SURFACE_DIM, MAX_SURFACE_DIM,
792 MAX_SURFACE_DIM * bs, isl_format,
793 &surf, &isl_surf);
794
795 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
796 0, 0, 1, 0, 0, MAX_SURFACE_DIM, MAX_SURFACE_DIM,
797 color, NULL);
798 fillSize -= max_fill_size;
799 dstOffset += max_fill_size;
800 }
801
802 uint64_t height = fillSize / (MAX_SURFACE_DIM * bs);
803 assert(height < MAX_SURFACE_DIM);
804 if (height != 0) {
805 const uint64_t rect_fill_size = height * MAX_SURFACE_DIM * bs;
806 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
807 dst_buffer, dstOffset,
808 MAX_SURFACE_DIM, height,
809 MAX_SURFACE_DIM * bs, isl_format,
810 &surf, &isl_surf);
811
812 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
813 0, 0, 1, 0, 0, MAX_SURFACE_DIM, height,
814 color, NULL);
815 fillSize -= rect_fill_size;
816 dstOffset += rect_fill_size;
817 }
818
819 if (fillSize != 0) {
820 const uint32_t width = fillSize / bs;
821 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
822 dst_buffer, dstOffset,
823 width, 1,
824 width * bs, isl_format,
825 &surf, &isl_surf);
826
827 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
828 0, 0, 1, 0, 0, width, 1,
829 color, NULL);
830 }
831
832 blorp_batch_finish(&batch);
833
834 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
835 }
836
837 void anv_CmdClearColorImage(
838 VkCommandBuffer commandBuffer,
839 VkImage _image,
840 VkImageLayout imageLayout,
841 const VkClearColorValue* pColor,
842 uint32_t rangeCount,
843 const VkImageSubresourceRange* pRanges)
844 {
845 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
846 ANV_FROM_HANDLE(anv_image, image, _image);
847
848 static const bool color_write_disable[4] = { false, false, false, false };
849
850 struct blorp_batch batch;
851 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
852
853
854 for (unsigned r = 0; r < rangeCount; r++) {
855 if (pRanges[r].aspectMask == 0)
856 continue;
857
858 assert(pRanges[r].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
859
860 struct blorp_surf surf;
861 get_blorp_surf_for_anv_image(cmd_buffer->device,
862 image, pRanges[r].aspectMask,
863 imageLayout, ISL_AUX_USAGE_NONE, &surf);
864
865 struct anv_format_plane src_format =
866 anv_get_format_plane(&cmd_buffer->device->info, image->vk_format,
867 VK_IMAGE_ASPECT_COLOR_BIT, image->tiling);
868
869 unsigned base_layer = pRanges[r].baseArrayLayer;
870 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
871
872 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
873 const unsigned level = pRanges[r].baseMipLevel + i;
874 const unsigned level_width = anv_minify(image->extent.width, level);
875 const unsigned level_height = anv_minify(image->extent.height, level);
876
877 if (image->type == VK_IMAGE_TYPE_3D) {
878 base_layer = 0;
879 layer_count = anv_minify(image->extent.depth, level);
880 }
881
882 anv_cmd_buffer_mark_image_written(cmd_buffer, image,
883 pRanges[r].aspectMask,
884 surf.aux_usage, level,
885 base_layer, layer_count);
886
887 blorp_clear(&batch, &surf,
888 src_format.isl_format, src_format.swizzle,
889 level, base_layer, layer_count,
890 0, 0, level_width, level_height,
891 vk_to_isl_color(*pColor), color_write_disable);
892 }
893 }
894
895 blorp_batch_finish(&batch);
896 }
897
898 void anv_CmdClearDepthStencilImage(
899 VkCommandBuffer commandBuffer,
900 VkImage image_h,
901 VkImageLayout imageLayout,
902 const VkClearDepthStencilValue* pDepthStencil,
903 uint32_t rangeCount,
904 const VkImageSubresourceRange* pRanges)
905 {
906 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
907 ANV_FROM_HANDLE(anv_image, image, image_h);
908
909 struct blorp_batch batch;
910 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
911
912 struct blorp_surf depth, stencil;
913 if (image->aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
914 get_blorp_surf_for_anv_image(cmd_buffer->device,
915 image, VK_IMAGE_ASPECT_DEPTH_BIT,
916 imageLayout, ISL_AUX_USAGE_NONE, &depth);
917 } else {
918 memset(&depth, 0, sizeof(depth));
919 }
920
921 if (image->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
922 get_blorp_surf_for_anv_image(cmd_buffer->device,
923 image, VK_IMAGE_ASPECT_STENCIL_BIT,
924 imageLayout, ISL_AUX_USAGE_NONE, &stencil);
925 } else {
926 memset(&stencil, 0, sizeof(stencil));
927 }
928
929 for (unsigned r = 0; r < rangeCount; r++) {
930 if (pRanges[r].aspectMask == 0)
931 continue;
932
933 bool clear_depth = pRanges[r].aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
934 bool clear_stencil = pRanges[r].aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
935
936 unsigned base_layer = pRanges[r].baseArrayLayer;
937 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
938
939 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
940 const unsigned level = pRanges[r].baseMipLevel + i;
941 const unsigned level_width = anv_minify(image->extent.width, level);
942 const unsigned level_height = anv_minify(image->extent.height, level);
943
944 if (image->type == VK_IMAGE_TYPE_3D)
945 layer_count = anv_minify(image->extent.depth, level);
946
947 blorp_clear_depth_stencil(&batch, &depth, &stencil,
948 level, base_layer, layer_count,
949 0, 0, level_width, level_height,
950 clear_depth, pDepthStencil->depth,
951 clear_stencil ? 0xff : 0,
952 pDepthStencil->stencil);
953 }
954 }
955
956 blorp_batch_finish(&batch);
957 }
958
959 VkResult
960 anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer *cmd_buffer,
961 uint32_t num_entries,
962 uint32_t *state_offset,
963 struct anv_state *bt_state)
964 {
965 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
966 state_offset);
967 if (bt_state->map == NULL) {
968 /* We ran out of space. Grab a new binding table block. */
969 VkResult result = anv_cmd_buffer_new_binding_table_block(cmd_buffer);
970 if (result != VK_SUCCESS)
971 return result;
972
973 /* Re-emit state base addresses so we get the new surface state base
974 * address before we start emitting binding tables etc.
975 */
976 anv_cmd_buffer_emit_state_base_address(cmd_buffer);
977
978 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
979 state_offset);
980 assert(bt_state->map != NULL);
981 }
982
983 return VK_SUCCESS;
984 }
985
986 static VkResult
987 binding_table_for_surface_state(struct anv_cmd_buffer *cmd_buffer,
988 struct anv_state surface_state,
989 uint32_t *bt_offset)
990 {
991 uint32_t state_offset;
992 struct anv_state bt_state;
993
994 VkResult result =
995 anv_cmd_buffer_alloc_blorp_binding_table(cmd_buffer, 1, &state_offset,
996 &bt_state);
997 if (result != VK_SUCCESS)
998 return result;
999
1000 uint32_t *bt_map = bt_state.map;
1001 bt_map[0] = surface_state.offset + state_offset;
1002
1003 *bt_offset = bt_state.offset;
1004 return VK_SUCCESS;
1005 }
1006
1007 static void
1008 clear_color_attachment(struct anv_cmd_buffer *cmd_buffer,
1009 struct blorp_batch *batch,
1010 const VkClearAttachment *attachment,
1011 uint32_t rectCount, const VkClearRect *pRects)
1012 {
1013 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
1014 const uint32_t color_att = attachment->colorAttachment;
1015 const uint32_t att_idx = subpass->color_attachments[color_att].attachment;
1016
1017 if (att_idx == VK_ATTACHMENT_UNUSED)
1018 return;
1019
1020 struct anv_render_pass_attachment *pass_att =
1021 &cmd_buffer->state.pass->attachments[att_idx];
1022 struct anv_attachment_state *att_state =
1023 &cmd_buffer->state.attachments[att_idx];
1024
1025 uint32_t binding_table;
1026 VkResult result =
1027 binding_table_for_surface_state(cmd_buffer, att_state->color.state,
1028 &binding_table);
1029 if (result != VK_SUCCESS)
1030 return;
1031
1032 union isl_color_value clear_color =
1033 vk_to_isl_color(attachment->clearValue.color);
1034
1035 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1036 if (subpass->view_mask) {
1037 uint32_t view_idx;
1038 for_each_bit(view_idx, subpass->view_mask) {
1039 for (uint32_t r = 0; r < rectCount; ++r) {
1040 const VkOffset2D offset = pRects[r].rect.offset;
1041 const VkExtent2D extent = pRects[r].rect.extent;
1042 blorp_clear_attachments(batch, binding_table,
1043 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1044 view_idx, 1,
1045 offset.x, offset.y,
1046 offset.x + extent.width,
1047 offset.y + extent.height,
1048 true, clear_color, false, 0.0f, 0, 0);
1049 }
1050 }
1051 return;
1052 }
1053
1054 for (uint32_t r = 0; r < rectCount; ++r) {
1055 const VkOffset2D offset = pRects[r].rect.offset;
1056 const VkExtent2D extent = pRects[r].rect.extent;
1057 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1058 blorp_clear_attachments(batch, binding_table,
1059 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1060 pRects[r].baseArrayLayer,
1061 pRects[r].layerCount,
1062 offset.x, offset.y,
1063 offset.x + extent.width, offset.y + extent.height,
1064 true, clear_color, false, 0.0f, 0, 0);
1065 }
1066 }
1067
1068 static void
1069 clear_depth_stencil_attachment(struct anv_cmd_buffer *cmd_buffer,
1070 struct blorp_batch *batch,
1071 const VkClearAttachment *attachment,
1072 uint32_t rectCount, const VkClearRect *pRects)
1073 {
1074 static const union isl_color_value color_value = { .u32 = { 0, } };
1075 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
1076 const uint32_t att_idx = subpass->depth_stencil_attachment->attachment;
1077
1078 if (att_idx == VK_ATTACHMENT_UNUSED)
1079 return;
1080
1081 struct anv_render_pass_attachment *pass_att =
1082 &cmd_buffer->state.pass->attachments[att_idx];
1083
1084 bool clear_depth = attachment->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
1085 bool clear_stencil = attachment->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
1086
1087 enum isl_format depth_format = ISL_FORMAT_UNSUPPORTED;
1088 if (clear_depth) {
1089 depth_format = anv_get_isl_format(&cmd_buffer->device->info,
1090 pass_att->format,
1091 VK_IMAGE_ASPECT_DEPTH_BIT,
1092 VK_IMAGE_TILING_OPTIMAL);
1093 }
1094
1095 uint32_t binding_table;
1096 VkResult result =
1097 binding_table_for_surface_state(cmd_buffer,
1098 cmd_buffer->state.null_surface_state,
1099 &binding_table);
1100 if (result != VK_SUCCESS)
1101 return;
1102
1103 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1104 if (subpass->view_mask) {
1105 uint32_t view_idx;
1106 for_each_bit(view_idx, subpass->view_mask) {
1107 for (uint32_t r = 0; r < rectCount; ++r) {
1108 const VkOffset2D offset = pRects[r].rect.offset;
1109 const VkExtent2D extent = pRects[r].rect.extent;
1110 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1111 blorp_clear_attachments(batch, binding_table,
1112 depth_format, pass_att->samples,
1113 view_idx, 1,
1114 offset.x, offset.y,
1115 offset.x + extent.width,
1116 offset.y + extent.height,
1117 false, color_value,
1118 clear_depth, value.depth,
1119 clear_stencil ? 0xff : 0, value.stencil);
1120 }
1121 }
1122 return;
1123 }
1124
1125 for (uint32_t r = 0; r < rectCount; ++r) {
1126 const VkOffset2D offset = pRects[r].rect.offset;
1127 const VkExtent2D extent = pRects[r].rect.extent;
1128 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1129 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1130 blorp_clear_attachments(batch, binding_table,
1131 depth_format, pass_att->samples,
1132 pRects[r].baseArrayLayer,
1133 pRects[r].layerCount,
1134 offset.x, offset.y,
1135 offset.x + extent.width, offset.y + extent.height,
1136 false, color_value,
1137 clear_depth, value.depth,
1138 clear_stencil ? 0xff : 0, value.stencil);
1139 }
1140 }
1141
1142 void anv_CmdClearAttachments(
1143 VkCommandBuffer commandBuffer,
1144 uint32_t attachmentCount,
1145 const VkClearAttachment* pAttachments,
1146 uint32_t rectCount,
1147 const VkClearRect* pRects)
1148 {
1149 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1150
1151 /* Because this gets called within a render pass, we tell blorp not to
1152 * trash our depth and stencil buffers.
1153 */
1154 struct blorp_batch batch;
1155 enum blorp_batch_flags flags = BLORP_BATCH_NO_EMIT_DEPTH_STENCIL;
1156 if (cmd_buffer->state.conditional_render_enabled) {
1157 anv_cmd_emit_conditional_render_predicate(cmd_buffer);
1158 flags |= BLORP_BATCH_PREDICATE_ENABLE;
1159 }
1160 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, flags);
1161
1162 for (uint32_t a = 0; a < attachmentCount; ++a) {
1163 if (pAttachments[a].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
1164 assert(pAttachments[a].aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
1165 clear_color_attachment(cmd_buffer, &batch,
1166 &pAttachments[a],
1167 rectCount, pRects);
1168 } else {
1169 clear_depth_stencil_attachment(cmd_buffer, &batch,
1170 &pAttachments[a],
1171 rectCount, pRects);
1172 }
1173 }
1174
1175 blorp_batch_finish(&batch);
1176 }
1177
1178 enum subpass_stage {
1179 SUBPASS_STAGE_LOAD,
1180 SUBPASS_STAGE_DRAW,
1181 SUBPASS_STAGE_RESOLVE,
1182 };
1183
1184 void
1185 anv_image_msaa_resolve(struct anv_cmd_buffer *cmd_buffer,
1186 const struct anv_image *src_image,
1187 enum isl_aux_usage src_aux_usage,
1188 uint32_t src_level, uint32_t src_base_layer,
1189 const struct anv_image *dst_image,
1190 enum isl_aux_usage dst_aux_usage,
1191 uint32_t dst_level, uint32_t dst_base_layer,
1192 VkImageAspectFlagBits aspect,
1193 uint32_t src_x, uint32_t src_y,
1194 uint32_t dst_x, uint32_t dst_y,
1195 uint32_t width, uint32_t height,
1196 uint32_t layer_count,
1197 enum blorp_filter filter)
1198 {
1199 struct blorp_batch batch;
1200 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1201
1202 assert(src_image->type == VK_IMAGE_TYPE_2D);
1203 assert(src_image->samples > 1);
1204 assert(dst_image->type == VK_IMAGE_TYPE_2D);
1205 assert(dst_image->samples == 1);
1206 assert(src_image->n_planes == dst_image->n_planes);
1207 assert(!src_image->format->can_ycbcr);
1208 assert(!dst_image->format->can_ycbcr);
1209
1210 struct blorp_surf src_surf, dst_surf;
1211 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, aspect,
1212 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1213 src_aux_usage, &src_surf);
1214 if (src_aux_usage == ISL_AUX_USAGE_MCS) {
1215 src_surf.clear_color_addr = anv_to_blorp_address(
1216 anv_image_get_clear_color_addr(cmd_buffer->device, src_image,
1217 VK_IMAGE_ASPECT_COLOR_BIT));
1218 }
1219 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, aspect,
1220 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1221 dst_aux_usage, &dst_surf);
1222 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
1223 aspect, dst_aux_usage,
1224 dst_level, dst_base_layer, layer_count);
1225
1226 if (filter == BLORP_FILTER_NONE) {
1227 /* If no explicit filter is provided, then it's implied by the type of
1228 * the source image.
1229 */
1230 if ((src_surf.surf->usage & ISL_SURF_USAGE_DEPTH_BIT) ||
1231 (src_surf.surf->usage & ISL_SURF_USAGE_STENCIL_BIT) ||
1232 isl_format_has_int_channel(src_surf.surf->format)) {
1233 filter = BLORP_FILTER_SAMPLE_0;
1234 } else {
1235 filter = BLORP_FILTER_AVERAGE;
1236 }
1237 }
1238
1239 for (uint32_t l = 0; l < layer_count; l++) {
1240 blorp_blit(&batch,
1241 &src_surf, src_level, src_base_layer + l,
1242 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1243 &dst_surf, dst_level, dst_base_layer + l,
1244 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1245 src_x, src_y, src_x + width, src_y + height,
1246 dst_x, dst_y, dst_x + width, dst_y + height,
1247 filter, false, false);
1248 }
1249
1250 blorp_batch_finish(&batch);
1251 }
1252
1253 void anv_CmdResolveImage(
1254 VkCommandBuffer commandBuffer,
1255 VkImage srcImage,
1256 VkImageLayout srcImageLayout,
1257 VkImage dstImage,
1258 VkImageLayout dstImageLayout,
1259 uint32_t regionCount,
1260 const VkImageResolve* pRegions)
1261 {
1262 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1263 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
1264 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
1265
1266 assert(!src_image->format->can_ycbcr);
1267
1268 for (uint32_t r = 0; r < regionCount; r++) {
1269 assert(pRegions[r].srcSubresource.aspectMask ==
1270 pRegions[r].dstSubresource.aspectMask);
1271 assert(anv_get_layerCount(src_image, &pRegions[r].srcSubresource) ==
1272 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource));
1273
1274 const uint32_t layer_count =
1275 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource);
1276
1277 VkImageAspectFlags src_mask = pRegions[r].srcSubresource.aspectMask;
1278 VkImageAspectFlags dst_mask = pRegions[r].dstSubresource.aspectMask;
1279
1280 assert(anv_image_aspects_compatible(src_mask, dst_mask));
1281
1282 uint32_t aspect_bit;
1283 anv_foreach_image_aspect_bit(aspect_bit, src_image,
1284 pRegions[r].srcSubresource.aspectMask) {
1285 enum isl_aux_usage src_aux_usage =
1286 anv_layout_to_aux_usage(&cmd_buffer->device->info, src_image,
1287 (1 << aspect_bit), srcImageLayout);
1288 enum isl_aux_usage dst_aux_usage =
1289 anv_layout_to_aux_usage(&cmd_buffer->device->info, dst_image,
1290 (1 << aspect_bit), dstImageLayout);
1291
1292 anv_image_msaa_resolve(cmd_buffer,
1293 src_image, src_aux_usage,
1294 pRegions[r].srcSubresource.mipLevel,
1295 pRegions[r].srcSubresource.baseArrayLayer,
1296 dst_image, dst_aux_usage,
1297 pRegions[r].dstSubresource.mipLevel,
1298 pRegions[r].dstSubresource.baseArrayLayer,
1299 (1 << aspect_bit),
1300 pRegions[r].srcOffset.x,
1301 pRegions[r].srcOffset.y,
1302 pRegions[r].dstOffset.x,
1303 pRegions[r].dstOffset.y,
1304 pRegions[r].extent.width,
1305 pRegions[r].extent.height,
1306 layer_count, BLORP_FILTER_NONE);
1307 }
1308 }
1309 }
1310
1311 static enum isl_aux_usage
1312 fast_clear_aux_usage(const struct anv_image *image,
1313 VkImageAspectFlagBits aspect)
1314 {
1315 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
1316 if (image->planes[plane].aux_usage == ISL_AUX_USAGE_NONE)
1317 return ISL_AUX_USAGE_CCS_D;
1318 else
1319 return image->planes[plane].aux_usage;
1320 }
1321
1322 void
1323 anv_image_copy_to_shadow(struct anv_cmd_buffer *cmd_buffer,
1324 const struct anv_image *image,
1325 uint32_t base_level, uint32_t level_count,
1326 uint32_t base_layer, uint32_t layer_count)
1327 {
1328 struct blorp_batch batch;
1329 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1330
1331 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT && image->n_planes == 1);
1332
1333 struct blorp_surf surf;
1334 get_blorp_surf_for_anv_image(cmd_buffer->device,
1335 image, VK_IMAGE_ASPECT_COLOR_BIT,
1336 VK_IMAGE_LAYOUT_GENERAL,
1337 ISL_AUX_USAGE_NONE, &surf);
1338 assert(surf.aux_usage == ISL_AUX_USAGE_NONE);
1339
1340 struct blorp_surf shadow_surf = {
1341 .surf = &image->planes[0].shadow_surface.isl,
1342 .addr = {
1343 .buffer = image->planes[0].address.bo,
1344 .offset = image->planes[0].address.offset +
1345 image->planes[0].shadow_surface.offset,
1346 .mocs = anv_mocs_for_bo(cmd_buffer->device,
1347 image->planes[0].address.bo),
1348 },
1349 };
1350
1351 for (uint32_t l = 0; l < level_count; l++) {
1352 const uint32_t level = base_level + l;
1353
1354 const VkExtent3D extent = {
1355 .width = anv_minify(image->extent.width, level),
1356 .height = anv_minify(image->extent.height, level),
1357 .depth = anv_minify(image->extent.depth, level),
1358 };
1359
1360 if (image->type == VK_IMAGE_TYPE_3D)
1361 layer_count = extent.depth;
1362
1363 for (uint32_t a = 0; a < layer_count; a++) {
1364 const uint32_t layer = base_layer + a;
1365
1366 blorp_copy(&batch, &surf, level, layer,
1367 &shadow_surf, level, layer,
1368 0, 0, 0, 0, extent.width, extent.height);
1369 }
1370 }
1371
1372 blorp_batch_finish(&batch);
1373 }
1374
1375 void
1376 anv_image_clear_color(struct anv_cmd_buffer *cmd_buffer,
1377 const struct anv_image *image,
1378 VkImageAspectFlagBits aspect,
1379 enum isl_aux_usage aux_usage,
1380 enum isl_format format, struct isl_swizzle swizzle,
1381 uint32_t level, uint32_t base_layer, uint32_t layer_count,
1382 VkRect2D area, union isl_color_value clear_color)
1383 {
1384 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1385
1386 /* We don't support planar images with multisampling yet */
1387 assert(image->n_planes == 1);
1388
1389 struct blorp_batch batch;
1390 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1391
1392 struct blorp_surf surf;
1393 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1394 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1395 aux_usage, &surf);
1396 anv_cmd_buffer_mark_image_written(cmd_buffer, image, aspect, aux_usage,
1397 level, base_layer, layer_count);
1398
1399 blorp_clear(&batch, &surf, format, anv_swizzle_for_render(swizzle),
1400 level, base_layer, layer_count,
1401 area.offset.x, area.offset.y,
1402 area.offset.x + area.extent.width,
1403 area.offset.y + area.extent.height,
1404 clear_color, NULL);
1405
1406 blorp_batch_finish(&batch);
1407 }
1408
1409 void
1410 anv_image_clear_depth_stencil(struct anv_cmd_buffer *cmd_buffer,
1411 const struct anv_image *image,
1412 VkImageAspectFlags aspects,
1413 enum isl_aux_usage depth_aux_usage,
1414 uint32_t level,
1415 uint32_t base_layer, uint32_t layer_count,
1416 VkRect2D area,
1417 float depth_value, uint8_t stencil_value)
1418 {
1419 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
1420 VK_IMAGE_ASPECT_STENCIL_BIT));
1421
1422 struct blorp_batch batch;
1423 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1424
1425 struct blorp_surf depth = {};
1426 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1427 get_blorp_surf_for_anv_image(cmd_buffer->device,
1428 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1429 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1430 depth_aux_usage, &depth);
1431 depth.clear_color.f32[0] = ANV_HZ_FC_VAL;
1432 }
1433
1434 struct blorp_surf stencil = {};
1435 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1436 get_blorp_surf_for_anv_image(cmd_buffer->device,
1437 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1438 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1439 ISL_AUX_USAGE_NONE, &stencil);
1440 }
1441
1442 blorp_clear_depth_stencil(&batch, &depth, &stencil,
1443 level, base_layer, layer_count,
1444 area.offset.x, area.offset.y,
1445 area.offset.x + area.extent.width,
1446 area.offset.y + area.extent.height,
1447 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
1448 depth_value,
1449 (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) ? 0xff : 0,
1450 stencil_value);
1451
1452 blorp_batch_finish(&batch);
1453 }
1454
1455 void
1456 anv_image_hiz_op(struct anv_cmd_buffer *cmd_buffer,
1457 const struct anv_image *image,
1458 VkImageAspectFlagBits aspect, uint32_t level,
1459 uint32_t base_layer, uint32_t layer_count,
1460 enum isl_aux_op hiz_op)
1461 {
1462 assert(aspect == VK_IMAGE_ASPECT_DEPTH_BIT);
1463 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, level));
1464 assert(anv_image_aspect_to_plane(image->aspects,
1465 VK_IMAGE_ASPECT_DEPTH_BIT) == 0);
1466
1467 struct blorp_batch batch;
1468 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1469
1470 struct blorp_surf surf;
1471 get_blorp_surf_for_anv_image(cmd_buffer->device,
1472 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1473 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1474 ISL_AUX_USAGE_HIZ, &surf);
1475 surf.clear_color.f32[0] = ANV_HZ_FC_VAL;
1476
1477 blorp_hiz_op(&batch, &surf, level, base_layer, layer_count, hiz_op);
1478
1479 blorp_batch_finish(&batch);
1480 }
1481
1482 void
1483 anv_image_hiz_clear(struct anv_cmd_buffer *cmd_buffer,
1484 const struct anv_image *image,
1485 VkImageAspectFlags aspects,
1486 uint32_t level,
1487 uint32_t base_layer, uint32_t layer_count,
1488 VkRect2D area, uint8_t stencil_value)
1489 {
1490 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
1491 VK_IMAGE_ASPECT_STENCIL_BIT));
1492
1493 struct blorp_batch batch;
1494 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1495
1496 struct blorp_surf depth = {};
1497 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1498 assert(base_layer + layer_count <=
1499 anv_image_aux_layers(image, VK_IMAGE_ASPECT_DEPTH_BIT, level));
1500 get_blorp_surf_for_anv_image(cmd_buffer->device,
1501 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1502 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1503 ISL_AUX_USAGE_HIZ, &depth);
1504 depth.clear_color.f32[0] = ANV_HZ_FC_VAL;
1505 }
1506
1507 struct blorp_surf stencil = {};
1508 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1509 get_blorp_surf_for_anv_image(cmd_buffer->device,
1510 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1511 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1512 ISL_AUX_USAGE_NONE, &stencil);
1513 }
1514
1515 /* From the Sky Lake PRM Volume 7, "Depth Buffer Clear":
1516 *
1517 * "The following is required when performing a depth buffer clear with
1518 * using the WM_STATE or 3DSTATE_WM:
1519 *
1520 * * If other rendering operations have preceded this clear, a
1521 * PIPE_CONTROL with depth cache flush enabled, Depth Stall bit
1522 * enabled must be issued before the rectangle primitive used for
1523 * the depth buffer clear operation.
1524 * * [...]"
1525 *
1526 * Even though the PRM only says that this is required if using 3DSTATE_WM
1527 * and a 3DPRIMITIVE, the GPU appears to also need this to avoid occasional
1528 * hangs when doing a clear with WM_HZ_OP.
1529 */
1530 cmd_buffer->state.pending_pipe_bits |=
1531 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT;
1532
1533 blorp_hiz_clear_depth_stencil(&batch, &depth, &stencil,
1534 level, base_layer, layer_count,
1535 area.offset.x, area.offset.y,
1536 area.offset.x + area.extent.width,
1537 area.offset.y + area.extent.height,
1538 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
1539 ANV_HZ_FC_VAL,
1540 aspects & VK_IMAGE_ASPECT_STENCIL_BIT,
1541 stencil_value);
1542
1543 blorp_batch_finish(&batch);
1544
1545 /* From the SKL PRM, Depth Buffer Clear:
1546 *
1547 * "Depth Buffer Clear Workaround
1548 *
1549 * Depth buffer clear pass using any of the methods (WM_STATE,
1550 * 3DSTATE_WM or 3DSTATE_WM_HZ_OP) must be followed by a PIPE_CONTROL
1551 * command with DEPTH_STALL bit and Depth FLUSH bits “set” before
1552 * starting to render. DepthStall and DepthFlush are not needed between
1553 * consecutive depth clear passes nor is it required if the depth-clear
1554 * pass was done with “full_surf_clear” bit set in the
1555 * 3DSTATE_WM_HZ_OP."
1556 *
1557 * Even though the PRM provides a bunch of conditions under which this is
1558 * supposedly unnecessary, we choose to perform the flush unconditionally
1559 * just to be safe.
1560 */
1561 cmd_buffer->state.pending_pipe_bits |=
1562 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT;
1563 }
1564
1565 void
1566 anv_image_mcs_op(struct anv_cmd_buffer *cmd_buffer,
1567 const struct anv_image *image,
1568 enum isl_format format,
1569 VkImageAspectFlagBits aspect,
1570 uint32_t base_layer, uint32_t layer_count,
1571 enum isl_aux_op mcs_op, union isl_color_value *clear_value,
1572 bool predicate)
1573 {
1574 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1575 assert(image->samples > 1);
1576 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, 0));
1577
1578 /* Multisampling with multi-planar formats is not supported */
1579 assert(image->n_planes == 1);
1580
1581 struct blorp_batch batch;
1582 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1583 predicate ? BLORP_BATCH_PREDICATE_ENABLE : 0);
1584
1585 struct blorp_surf surf;
1586 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1587 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1588 ISL_AUX_USAGE_MCS, &surf);
1589
1590 /* Blorp will store the clear color for us if we provide the clear color
1591 * address and we are doing a fast clear. So we save the clear value into
1592 * the blorp surface. However, in some situations we want to do a fast clear
1593 * without changing the clear value stored in the state buffer. For those
1594 * cases, we set the clear color address pointer to NULL, so blorp will not
1595 * try to store a garbage color.
1596 */
1597 if (mcs_op == ISL_AUX_OP_FAST_CLEAR) {
1598 if (clear_value)
1599 surf.clear_color = *clear_value;
1600 else
1601 surf.clear_color_addr.buffer = NULL;
1602 }
1603
1604 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1605 *
1606 * "After Render target fast clear, pipe-control with color cache
1607 * write-flush must be issued before sending any DRAW commands on
1608 * that render target."
1609 *
1610 * This comment is a bit cryptic and doesn't really tell you what's going
1611 * or what's really needed. It appears that fast clear ops are not
1612 * properly synchronized with other drawing. This means that we cannot
1613 * have a fast clear operation in the pipe at the same time as other
1614 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
1615 * that the contents of the previous draw hit the render target before we
1616 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
1617 * that it is completed before any additional drawing occurs.
1618 */
1619 cmd_buffer->state.pending_pipe_bits |=
1620 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1621
1622 switch (mcs_op) {
1623 case ISL_AUX_OP_FAST_CLEAR:
1624 blorp_fast_clear(&batch, &surf, format,
1625 0, base_layer, layer_count,
1626 0, 0, image->extent.width, image->extent.height);
1627 break;
1628 case ISL_AUX_OP_PARTIAL_RESOLVE:
1629 blorp_mcs_partial_resolve(&batch, &surf, format,
1630 base_layer, layer_count);
1631 break;
1632 case ISL_AUX_OP_FULL_RESOLVE:
1633 case ISL_AUX_OP_AMBIGUATE:
1634 default:
1635 unreachable("Unsupported MCS operation");
1636 }
1637
1638 cmd_buffer->state.pending_pipe_bits |=
1639 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1640
1641 blorp_batch_finish(&batch);
1642 }
1643
1644 void
1645 anv_image_ccs_op(struct anv_cmd_buffer *cmd_buffer,
1646 const struct anv_image *image,
1647 enum isl_format format,
1648 VkImageAspectFlagBits aspect, uint32_t level,
1649 uint32_t base_layer, uint32_t layer_count,
1650 enum isl_aux_op ccs_op, union isl_color_value *clear_value,
1651 bool predicate)
1652 {
1653 assert(image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
1654 assert(image->samples == 1);
1655 assert(level < anv_image_aux_levels(image, aspect));
1656 /* Multi-LOD YcBcR is not allowed */
1657 assert(image->n_planes == 1 || level == 0);
1658 assert(base_layer + layer_count <=
1659 anv_image_aux_layers(image, aspect, level));
1660
1661 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
1662 uint32_t width_div = image->format->planes[plane].denominator_scales[0];
1663 uint32_t height_div = image->format->planes[plane].denominator_scales[1];
1664 uint32_t level_width = anv_minify(image->extent.width, level) / width_div;
1665 uint32_t level_height = anv_minify(image->extent.height, level) / height_div;
1666
1667 struct blorp_batch batch;
1668 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1669 predicate ? BLORP_BATCH_PREDICATE_ENABLE : 0);
1670
1671 struct blorp_surf surf;
1672 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1673 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1674 fast_clear_aux_usage(image, aspect),
1675 &surf);
1676
1677 /* Blorp will store the clear color for us if we provide the clear color
1678 * address and we are doing a fast clear. So we save the clear value into
1679 * the blorp surface. However, in some situations we want to do a fast clear
1680 * without changing the clear value stored in the state buffer. For those
1681 * cases, we set the clear color address pointer to NULL, so blorp will not
1682 * try to store a garbage color.
1683 */
1684 if (ccs_op == ISL_AUX_OP_FAST_CLEAR) {
1685 if (clear_value)
1686 surf.clear_color = *clear_value;
1687 else
1688 surf.clear_color_addr.buffer = NULL;
1689 }
1690
1691 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1692 *
1693 * "After Render target fast clear, pipe-control with color cache
1694 * write-flush must be issued before sending any DRAW commands on
1695 * that render target."
1696 *
1697 * This comment is a bit cryptic and doesn't really tell you what's going
1698 * or what's really needed. It appears that fast clear ops are not
1699 * properly synchronized with other drawing. This means that we cannot
1700 * have a fast clear operation in the pipe at the same time as other
1701 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
1702 * that the contents of the previous draw hit the render target before we
1703 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
1704 * that it is completed before any additional drawing occurs.
1705 */
1706 cmd_buffer->state.pending_pipe_bits |=
1707 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1708
1709 switch (ccs_op) {
1710 case ISL_AUX_OP_FAST_CLEAR:
1711 blorp_fast_clear(&batch, &surf, format,
1712 level, base_layer, layer_count,
1713 0, 0, level_width, level_height);
1714 break;
1715 case ISL_AUX_OP_FULL_RESOLVE:
1716 case ISL_AUX_OP_PARTIAL_RESOLVE:
1717 blorp_ccs_resolve(&batch, &surf, level, base_layer, layer_count,
1718 format, ccs_op);
1719 break;
1720 case ISL_AUX_OP_AMBIGUATE:
1721 for (uint32_t a = 0; a < layer_count; a++) {
1722 const uint32_t layer = base_layer + a;
1723 blorp_ccs_ambiguate(&batch, &surf, level, layer);
1724 }
1725 break;
1726 default:
1727 unreachable("Unsupported CCS operation");
1728 }
1729
1730 cmd_buffer->state.pending_pipe_bits |=
1731 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1732
1733 blorp_batch_finish(&batch);
1734 }