projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
anv/blorp: Use BLORP_BATCH_NO_UPDATE_CLEAR_COLOR
[mesa.git] / src / intel / vulkan / anv_blorp.c
1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "anv_private.h"
25
26 static bool
27 lookup_blorp_shader(struct blorp_batch *batch,
28 const void *key, uint32_t key_size,
29 uint32_t *kernel_out, void *prog_data_out)
30 {
31 struct blorp_context *blorp = batch->blorp;
32 struct anv_device *device = blorp->driver_ctx;
33
34 /* The default cache must be a real cache */
35 assert(device->default_pipeline_cache.cache);
36
37 struct anv_shader_bin *bin =
38 anv_pipeline_cache_search(&device->default_pipeline_cache, key, key_size);
39 if (!bin)
40 return false;
41
42 /* The cache already has a reference and it's not going anywhere so there
43 * is no need to hold a second reference.
44 */
45 anv_shader_bin_unref(device, bin);
46
47 *kernel_out = bin->kernel.offset;
48 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
49
50 return true;
51 }
52
53 static bool
54 upload_blorp_shader(struct blorp_batch *batch,
55 const void *key, uint32_t key_size,
56 const void *kernel, uint32_t kernel_size,
57 const struct brw_stage_prog_data *prog_data,
58 uint32_t prog_data_size,
59 uint32_t *kernel_out, void *prog_data_out)
60 {
61 struct blorp_context *blorp = batch->blorp;
62 struct anv_device *device = blorp->driver_ctx;
63
64 /* The blorp cache must be a real cache */
65 assert(device->default_pipeline_cache.cache);
66
67 struct anv_pipeline_bind_map bind_map = {
68 .surface_count = 0,
69 .sampler_count = 0,
70 };
71
72 struct anv_shader_bin *bin =
73 anv_pipeline_cache_upload_kernel(&device->default_pipeline_cache,
74 key, key_size, kernel, kernel_size,
75 NULL, 0,
76 prog_data, prog_data_size,
77 NULL, 0, NULL, &bind_map);
78
79 if (!bin)
80 return false;
81
82 /* The cache already has a reference and it's not going anywhere so there
83 * is no need to hold a second reference.
84 */
85 anv_shader_bin_unref(device, bin);
86
87 *kernel_out = bin->kernel.offset;
88 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
89
90 return true;
91 }
92
93 void
94 anv_device_init_blorp(struct anv_device *device)
95 {
96 blorp_init(&device->blorp, device, &device->isl_dev);
97 device->blorp.compiler = device->instance->physicalDevice.compiler;
98 device->blorp.lookup_shader = lookup_blorp_shader;
99 device->blorp.upload_shader = upload_blorp_shader;
100 switch (device->info.gen) {
101 case 7:
102 if (device->info.is_haswell) {
103 device->blorp.exec = gen75_blorp_exec;
104 } else {
105 device->blorp.exec = gen7_blorp_exec;
106 }
107 break;
108 case 8:
109 device->blorp.exec = gen8_blorp_exec;
110 break;
111 case 9:
112 device->blorp.exec = gen9_blorp_exec;
113 break;
114 case 10:
115 device->blorp.exec = gen10_blorp_exec;
116 break;
117 case 11:
118 device->blorp.exec = gen11_blorp_exec;
119 break;
120 case 12:
121 device->blorp.exec = gen12_blorp_exec;
122 break;
123 default:
124 unreachable("Unknown hardware generation");
125 }
126 }
127
128 void
129 anv_device_finish_blorp(struct anv_device *device)
130 {
131 blorp_finish(&device->blorp);
132 }
133
134 static void
135 get_blorp_surf_for_anv_buffer(struct anv_device *device,
136 struct anv_buffer *buffer, uint64_t offset,
137 uint32_t width, uint32_t height,
138 uint32_t row_pitch, enum isl_format format,
139 struct blorp_surf *blorp_surf,
140 struct isl_surf *isl_surf)
141 {
142 const struct isl_format_layout *fmtl =
143 isl_format_get_layout(format);
144 bool ok UNUSED;
145
146 /* ASTC is the only format which doesn't support linear layouts.
147 * Create an equivalently sized surface with ISL to get around this.
148 */
149 if (fmtl->txc == ISL_TXC_ASTC) {
150 /* Use an equivalently sized format */
151 format = ISL_FORMAT_R32G32B32A32_UINT;
152 assert(fmtl->bpb == isl_format_get_layout(format)->bpb);
153
154 /* Shrink the dimensions for the new format */
155 width = DIV_ROUND_UP(width, fmtl->bw);
156 height = DIV_ROUND_UP(height, fmtl->bh);
157 }
158
159 *blorp_surf = (struct blorp_surf) {
160 .surf = isl_surf,
161 .addr = {
162 .buffer = buffer->address.bo,
163 .offset = buffer->address.offset + offset,
164 .mocs = anv_mocs_for_bo(device, buffer->address.bo),
165 },
166 };
167
168 ok = isl_surf_init(&device->isl_dev, isl_surf,
169 .dim = ISL_SURF_DIM_2D,
170 .format = format,
171 .width = width,
172 .height = height,
173 .depth = 1,
174 .levels = 1,
175 .array_len = 1,
176 .samples = 1,
177 .row_pitch_B = row_pitch,
178 .usage = ISL_SURF_USAGE_TEXTURE_BIT |
179 ISL_SURF_USAGE_RENDER_TARGET_BIT,
180 .tiling_flags = ISL_TILING_LINEAR_BIT);
181 assert(ok);
182 }
183
184 /* Pick something high enough that it won't be used in core and low enough it
185 * will never map to an extension.
186 */
187 #define ANV_IMAGE_LAYOUT_EXPLICIT_AUX (VkImageLayout)10000000
188
189 static struct blorp_address
190 anv_to_blorp_address(struct anv_address addr)
191 {
192 return (struct blorp_address) {
193 .buffer = addr.bo,
194 .offset = addr.offset,
195 };
196 }
197
198 static void
199 get_blorp_surf_for_anv_image(const struct anv_device *device,
200 const struct anv_image *image,
201 VkImageAspectFlags aspect,
202 VkImageLayout layout,
203 enum isl_aux_usage aux_usage,
204 struct blorp_surf *blorp_surf)
205 {
206 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
207
208 if (layout != ANV_IMAGE_LAYOUT_EXPLICIT_AUX)
209 aux_usage = anv_layout_to_aux_usage(&device->info, image, aspect, layout);
210
211 const struct anv_surface *surface = &image->planes[plane].surface;
212 *blorp_surf = (struct blorp_surf) {
213 .surf = &surface->isl,
214 .addr = {
215 .buffer = image->planes[plane].address.bo,
216 .offset = image->planes[plane].address.offset + surface->offset,
217 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
218 },
219 };
220
221 if (aux_usage != ISL_AUX_USAGE_NONE) {
222 const struct anv_surface *aux_surface = &image->planes[plane].aux_surface;
223 blorp_surf->aux_surf = &aux_surface->isl,
224 blorp_surf->aux_addr = (struct blorp_address) {
225 .buffer = image->planes[plane].address.bo,
226 .offset = image->planes[plane].address.offset + aux_surface->offset,
227 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
228 };
229 blorp_surf->aux_usage = aux_usage;
230
231 /* If we're doing a partial resolve, then we need the indirect clear
232 * color. If we are doing a fast clear and want to store/update the
233 * clear color, we also pass the address to blorp, otherwise it will only
234 * stomp the CCS to a particular value and won't care about format or
235 * clear value
236 */
237 if (aspect & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
238 const struct anv_address clear_color_addr =
239 anv_image_get_clear_color_addr(device, image, aspect);
240 blorp_surf->clear_color_addr = anv_to_blorp_address(clear_color_addr);
241 } else if (aspect & VK_IMAGE_ASPECT_DEPTH_BIT
242 && device->info.gen >= 10) {
243 /* Vulkan always clears to 1.0. On gen < 10, we set that directly in
244 * the state packet. For gen >= 10, must provide the clear value in a
245 * buffer. We have a single global buffer that stores the 1.0 value.
246 */
247 const struct anv_address clear_color_addr = (struct anv_address) {
248 .bo = (struct anv_bo *)&device->hiz_clear_bo
249 };
250 blorp_surf->clear_color_addr = anv_to_blorp_address(clear_color_addr);
251 }
252 }
253 }
254
255 static bool
256 get_blorp_surf_for_anv_shadow_image(const struct anv_device *device,
257 const struct anv_image *image,
258 VkImageAspectFlags aspect,
259 struct blorp_surf *blorp_surf)
260 {
261
262 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
263 if (image->planes[plane].shadow_surface.isl.size_B == 0)
264 return false;
265
266 *blorp_surf = (struct blorp_surf) {
267 .surf = &image->planes[plane].shadow_surface.isl,
268 .addr = {
269 .buffer = image->planes[plane].address.bo,
270 .offset = image->planes[plane].address.offset +
271 image->planes[plane].shadow_surface.offset,
272 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
273 },
274 };
275
276 return true;
277 }
278
279 void anv_CmdCopyImage(
280 VkCommandBuffer commandBuffer,
281 VkImage srcImage,
282 VkImageLayout srcImageLayout,
283 VkImage dstImage,
284 VkImageLayout dstImageLayout,
285 uint32_t regionCount,
286 const VkImageCopy* pRegions)
287 {
288 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
289 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
290 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
291
292 struct blorp_batch batch;
293 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
294
295 for (unsigned r = 0; r < regionCount; r++) {
296 VkOffset3D srcOffset =
297 anv_sanitize_image_offset(src_image->type, pRegions[r].srcOffset);
298 VkOffset3D dstOffset =
299 anv_sanitize_image_offset(dst_image->type, pRegions[r].dstOffset);
300 VkExtent3D extent =
301 anv_sanitize_image_extent(src_image->type, pRegions[r].extent);
302
303 const uint32_t dst_level = pRegions[r].dstSubresource.mipLevel;
304 unsigned dst_base_layer, layer_count;
305 if (dst_image->type == VK_IMAGE_TYPE_3D) {
306 dst_base_layer = pRegions[r].dstOffset.z;
307 layer_count = pRegions[r].extent.depth;
308 } else {
309 dst_base_layer = pRegions[r].dstSubresource.baseArrayLayer;
310 layer_count =
311 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource);
312 }
313
314 const uint32_t src_level = pRegions[r].srcSubresource.mipLevel;
315 unsigned src_base_layer;
316 if (src_image->type == VK_IMAGE_TYPE_3D) {
317 src_base_layer = pRegions[r].srcOffset.z;
318 } else {
319 src_base_layer = pRegions[r].srcSubresource.baseArrayLayer;
320 assert(layer_count ==
321 anv_get_layerCount(src_image, &pRegions[r].srcSubresource));
322 }
323
324 VkImageAspectFlags src_mask = pRegions[r].srcSubresource.aspectMask,
325 dst_mask = pRegions[r].dstSubresource.aspectMask;
326
327 assert(anv_image_aspects_compatible(src_mask, dst_mask));
328
329 if (util_bitcount(src_mask) > 1) {
330 uint32_t aspect_bit;
331 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_mask) {
332 struct blorp_surf src_surf, dst_surf;
333 get_blorp_surf_for_anv_image(cmd_buffer->device,
334 src_image, 1UL << aspect_bit,
335 srcImageLayout, ISL_AUX_USAGE_NONE,
336 &src_surf);
337 get_blorp_surf_for_anv_image(cmd_buffer->device,
338 dst_image, 1UL << aspect_bit,
339 dstImageLayout, ISL_AUX_USAGE_NONE,
340 &dst_surf);
341 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
342 1UL << aspect_bit,
343 dst_surf.aux_usage, dst_level,
344 dst_base_layer, layer_count);
345
346 for (unsigned i = 0; i < layer_count; i++) {
347 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
348 &dst_surf, dst_level, dst_base_layer + i,
349 srcOffset.x, srcOffset.y,
350 dstOffset.x, dstOffset.y,
351 extent.width, extent.height);
352 }
353
354 struct blorp_surf dst_shadow_surf;
355 if (get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
356 dst_image,
357 1UL << aspect_bit,
358 &dst_shadow_surf)) {
359 for (unsigned i = 0; i < layer_count; i++) {
360 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
361 &dst_shadow_surf, dst_level, dst_base_layer + i,
362 srcOffset.x, srcOffset.y,
363 dstOffset.x, dstOffset.y,
364 extent.width, extent.height);
365 }
366 }
367 }
368 } else {
369 struct blorp_surf src_surf, dst_surf;
370 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, src_mask,
371 srcImageLayout, ISL_AUX_USAGE_NONE,
372 &src_surf);
373 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, dst_mask,
374 dstImageLayout, ISL_AUX_USAGE_NONE,
375 &dst_surf);
376 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image, dst_mask,
377 dst_surf.aux_usage, dst_level,
378 dst_base_layer, layer_count);
379
380 for (unsigned i = 0; i < layer_count; i++) {
381 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
382 &dst_surf, dst_level, dst_base_layer + i,
383 srcOffset.x, srcOffset.y,
384 dstOffset.x, dstOffset.y,
385 extent.width, extent.height);
386 }
387
388 struct blorp_surf dst_shadow_surf;
389 if (get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
390 dst_image, dst_mask,
391 &dst_shadow_surf)) {
392 for (unsigned i = 0; i < layer_count; i++) {
393 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
394 &dst_shadow_surf, dst_level, dst_base_layer + i,
395 srcOffset.x, srcOffset.y,
396 dstOffset.x, dstOffset.y,
397 extent.width, extent.height);
398 }
399 }
400 }
401 }
402
403 blorp_batch_finish(&batch);
404 }
405
406 static void
407 copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer,
408 struct anv_buffer *anv_buffer,
409 struct anv_image *anv_image,
410 VkImageLayout image_layout,
411 uint32_t regionCount,
412 const VkBufferImageCopy* pRegions,
413 bool buffer_to_image)
414 {
415 struct blorp_batch batch;
416 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
417
418 struct {
419 struct blorp_surf surf;
420 uint32_t level;
421 VkOffset3D offset;
422 } image, buffer, *src, *dst;
423
424 buffer.level = 0;
425 buffer.offset = (VkOffset3D) { 0, 0, 0 };
426
427 if (buffer_to_image) {
428 src = &buffer;
429 dst = &image;
430 } else {
431 src = &image;
432 dst = &buffer;
433 }
434
435 for (unsigned r = 0; r < regionCount; r++) {
436 const VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask;
437
438 get_blorp_surf_for_anv_image(cmd_buffer->device, anv_image, aspect,
439 image_layout, ISL_AUX_USAGE_NONE,
440 &image.surf);
441 image.offset =
442 anv_sanitize_image_offset(anv_image->type, pRegions[r].imageOffset);
443 image.level = pRegions[r].imageSubresource.mipLevel;
444
445 VkExtent3D extent =
446 anv_sanitize_image_extent(anv_image->type, pRegions[r].imageExtent);
447 if (anv_image->type != VK_IMAGE_TYPE_3D) {
448 image.offset.z = pRegions[r].imageSubresource.baseArrayLayer;
449 extent.depth =
450 anv_get_layerCount(anv_image, &pRegions[r].imageSubresource);
451 }
452
453 const enum isl_format buffer_format =
454 anv_get_isl_format(&cmd_buffer->device->info, anv_image->vk_format,
455 aspect, VK_IMAGE_TILING_LINEAR);
456
457 const VkExtent3D bufferImageExtent = {
458 .width = pRegions[r].bufferRowLength ?
459 pRegions[r].bufferRowLength : extent.width,
460 .height = pRegions[r].bufferImageHeight ?
461 pRegions[r].bufferImageHeight : extent.height,
462 };
463
464 const struct isl_format_layout *buffer_fmtl =
465 isl_format_get_layout(buffer_format);
466
467 const uint32_t buffer_row_pitch =
468 DIV_ROUND_UP(bufferImageExtent.width, buffer_fmtl->bw) *
469 (buffer_fmtl->bpb / 8);
470
471 const uint32_t buffer_layer_stride =
472 DIV_ROUND_UP(bufferImageExtent.height, buffer_fmtl->bh) *
473 buffer_row_pitch;
474
475 struct isl_surf buffer_isl_surf;
476 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
477 anv_buffer, pRegions[r].bufferOffset,
478 extent.width, extent.height,
479 buffer_row_pitch, buffer_format,
480 &buffer.surf, &buffer_isl_surf);
481
482 bool dst_has_shadow = false;
483 struct blorp_surf dst_shadow_surf;
484 if (&image == dst) {
485 anv_cmd_buffer_mark_image_written(cmd_buffer, anv_image,
486 aspect, dst->surf.aux_usage,
487 dst->level,
488 dst->offset.z, extent.depth);
489
490 dst_has_shadow =
491 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
492 anv_image, aspect,
493 &dst_shadow_surf);
494 }
495
496 for (unsigned z = 0; z < extent.depth; z++) {
497 blorp_copy(&batch, &src->surf, src->level, src->offset.z,
498 &dst->surf, dst->level, dst->offset.z,
499 src->offset.x, src->offset.y, dst->offset.x, dst->offset.y,
500 extent.width, extent.height);
501
502 if (dst_has_shadow) {
503 blorp_copy(&batch, &src->surf, src->level, src->offset.z,
504 &dst_shadow_surf, dst->level, dst->offset.z,
505 src->offset.x, src->offset.y,
506 dst->offset.x, dst->offset.y,
507 extent.width, extent.height);
508 }
509
510 image.offset.z++;
511 buffer.surf.addr.offset += buffer_layer_stride;
512 }
513 }
514
515 blorp_batch_finish(&batch);
516 }
517
518 void anv_CmdCopyBufferToImage(
519 VkCommandBuffer commandBuffer,
520 VkBuffer srcBuffer,
521 VkImage dstImage,
522 VkImageLayout dstImageLayout,
523 uint32_t regionCount,
524 const VkBufferImageCopy* pRegions)
525 {
526 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
527 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
528 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
529
530 copy_buffer_to_image(cmd_buffer, src_buffer, dst_image, dstImageLayout,
531 regionCount, pRegions, true);
532 }
533
534 void anv_CmdCopyImageToBuffer(
535 VkCommandBuffer commandBuffer,
536 VkImage srcImage,
537 VkImageLayout srcImageLayout,
538 VkBuffer dstBuffer,
539 uint32_t regionCount,
540 const VkBufferImageCopy* pRegions)
541 {
542 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
543 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
544 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
545
546 copy_buffer_to_image(cmd_buffer, dst_buffer, src_image, srcImageLayout,
547 regionCount, pRegions, false);
548
549 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
550 }
551
552 static bool
553 flip_coords(unsigned *src0, unsigned *src1, unsigned *dst0, unsigned *dst1)
554 {
555 bool flip = false;
556 if (*src0 > *src1) {
557 unsigned tmp = *src0;
558 *src0 = *src1;
559 *src1 = tmp;
560 flip = !flip;
561 }
562
563 if (*dst0 > *dst1) {
564 unsigned tmp = *dst0;
565 *dst0 = *dst1;
566 *dst1 = tmp;
567 flip = !flip;
568 }
569
570 return flip;
571 }
572
573 void anv_CmdBlitImage(
574 VkCommandBuffer commandBuffer,
575 VkImage srcImage,
576 VkImageLayout srcImageLayout,
577 VkImage dstImage,
578 VkImageLayout dstImageLayout,
579 uint32_t regionCount,
580 const VkImageBlit* pRegions,
581 VkFilter filter)
582
583 {
584 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
585 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
586 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
587
588 struct blorp_surf src, dst;
589
590 enum blorp_filter blorp_filter;
591 switch (filter) {
592 case VK_FILTER_NEAREST:
593 blorp_filter = BLORP_FILTER_NEAREST;
594 break;
595 case VK_FILTER_LINEAR:
596 blorp_filter = BLORP_FILTER_BILINEAR;
597 break;
598 default:
599 unreachable("Invalid filter");
600 }
601
602 struct blorp_batch batch;
603 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
604
605 for (unsigned r = 0; r < regionCount; r++) {
606 const VkImageSubresourceLayers *src_res = &pRegions[r].srcSubresource;
607 const VkImageSubresourceLayers *dst_res = &pRegions[r].dstSubresource;
608
609 assert(anv_image_aspects_compatible(src_res->aspectMask,
610 dst_res->aspectMask));
611
612 uint32_t aspect_bit;
613 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_res->aspectMask) {
614 get_blorp_surf_for_anv_image(cmd_buffer->device,
615 src_image, 1U << aspect_bit,
616 srcImageLayout, ISL_AUX_USAGE_NONE, &src);
617 get_blorp_surf_for_anv_image(cmd_buffer->device,
618 dst_image, 1U << aspect_bit,
619 dstImageLayout, ISL_AUX_USAGE_NONE, &dst);
620
621 struct anv_format_plane src_format =
622 anv_get_format_plane(&cmd_buffer->device->info, src_image->vk_format,
623 1U << aspect_bit, src_image->tiling);
624 struct anv_format_plane dst_format =
625 anv_get_format_plane(&cmd_buffer->device->info, dst_image->vk_format,
626 1U << aspect_bit, dst_image->tiling);
627
628 unsigned dst_start, dst_end;
629 if (dst_image->type == VK_IMAGE_TYPE_3D) {
630 assert(dst_res->baseArrayLayer == 0);
631 dst_start = pRegions[r].dstOffsets[0].z;
632 dst_end = pRegions[r].dstOffsets[1].z;
633 } else {
634 dst_start = dst_res->baseArrayLayer;
635 dst_end = dst_start + anv_get_layerCount(dst_image, dst_res);
636 }
637
638 unsigned src_start, src_end;
639 if (src_image->type == VK_IMAGE_TYPE_3D) {
640 assert(src_res->baseArrayLayer == 0);
641 src_start = pRegions[r].srcOffsets[0].z;
642 src_end = pRegions[r].srcOffsets[1].z;
643 } else {
644 src_start = src_res->baseArrayLayer;
645 src_end = src_start + anv_get_layerCount(src_image, src_res);
646 }
647
648 bool flip_z = flip_coords(&src_start, &src_end, &dst_start, &dst_end);
649 float src_z_step = (float)(src_end + 1 - src_start) /
650 (float)(dst_end + 1 - dst_start);
651
652 if (flip_z) {
653 src_start = src_end;
654 src_z_step *= -1;
655 }
656
657 unsigned src_x0 = pRegions[r].srcOffsets[0].x;
658 unsigned src_x1 = pRegions[r].srcOffsets[1].x;
659 unsigned dst_x0 = pRegions[r].dstOffsets[0].x;
660 unsigned dst_x1 = pRegions[r].dstOffsets[1].x;
661 bool flip_x = flip_coords(&src_x0, &src_x1, &dst_x0, &dst_x1);
662
663 unsigned src_y0 = pRegions[r].srcOffsets[0].y;
664 unsigned src_y1 = pRegions[r].srcOffsets[1].y;
665 unsigned dst_y0 = pRegions[r].dstOffsets[0].y;
666 unsigned dst_y1 = pRegions[r].dstOffsets[1].y;
667 bool flip_y = flip_coords(&src_y0, &src_y1, &dst_y0, &dst_y1);
668
669 const unsigned num_layers = dst_end - dst_start;
670 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
671 1U << aspect_bit,
672 dst.aux_usage,
673 dst_res->mipLevel,
674 dst_start, num_layers);
675
676 for (unsigned i = 0; i < num_layers; i++) {
677 unsigned dst_z = dst_start + i;
678 unsigned src_z = src_start + i * src_z_step;
679
680 blorp_blit(&batch, &src, src_res->mipLevel, src_z,
681 src_format.isl_format, src_format.swizzle,
682 &dst, dst_res->mipLevel, dst_z,
683 dst_format.isl_format, dst_format.swizzle,
684 src_x0, src_y0, src_x1, src_y1,
685 dst_x0, dst_y0, dst_x1, dst_y1,
686 blorp_filter, flip_x, flip_y);
687 }
688 }
689 }
690
691 blorp_batch_finish(&batch);
692 }
693
694 static enum isl_format
695 isl_format_for_size(unsigned size_B)
696 {
697 switch (size_B) {
698 case 4: return ISL_FORMAT_R32_UINT;
699 case 8: return ISL_FORMAT_R32G32_UINT;
700 case 16: return ISL_FORMAT_R32G32B32A32_UINT;
701 default:
702 unreachable("Not a power-of-two format size");
703 }
704 }
705
706 /**
707 * Returns the greatest common divisor of a and b that is a power of two.
708 */
709 static uint64_t
710 gcd_pow2_u64(uint64_t a, uint64_t b)
711 {
712 assert(a > 0 || b > 0);
713
714 unsigned a_log2 = ffsll(a) - 1;
715 unsigned b_log2 = ffsll(b) - 1;
716
717 /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which
718 * case, the MIN2() will take the other one. If both are 0 then we will
719 * hit the assert above.
720 */
721 return 1 << MIN2(a_log2, b_log2);
722 }
723
724 /* This is maximum possible width/height our HW can handle */
725 #define MAX_SURFACE_DIM (1ull << 14)
726
727 void anv_CmdCopyBuffer(
728 VkCommandBuffer commandBuffer,
729 VkBuffer srcBuffer,
730 VkBuffer dstBuffer,
731 uint32_t regionCount,
732 const VkBufferCopy* pRegions)
733 {
734 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
735 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
736 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
737
738 struct blorp_batch batch;
739 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
740
741 for (unsigned r = 0; r < regionCount; r++) {
742 struct blorp_address src = {
743 .buffer = src_buffer->address.bo,
744 .offset = src_buffer->address.offset + pRegions[r].srcOffset,
745 .mocs = anv_mocs_for_bo(cmd_buffer->device, src_buffer->address.bo),
746 };
747 struct blorp_address dst = {
748 .buffer = dst_buffer->address.bo,
749 .offset = dst_buffer->address.offset + pRegions[r].dstOffset,
750 .mocs = anv_mocs_for_bo(cmd_buffer->device, dst_buffer->address.bo),
751 };
752
753 blorp_buffer_copy(&batch, src, dst, pRegions[r].size);
754 }
755
756 blorp_batch_finish(&batch);
757
758 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
759 }
760
761 void anv_CmdUpdateBuffer(
762 VkCommandBuffer commandBuffer,
763 VkBuffer dstBuffer,
764 VkDeviceSize dstOffset,
765 VkDeviceSize dataSize,
766 const void* pData)
767 {
768 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
769 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
770
771 struct blorp_batch batch;
772 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
773
774 /* We can't quite grab a full block because the state stream needs a
775 * little data at the top to build its linked list.
776 */
777 const uint32_t max_update_size =
778 cmd_buffer->device->dynamic_state_pool.block_size - 64;
779
780 assert(max_update_size < MAX_SURFACE_DIM * 4);
781
782 /* We're about to read data that was written from the CPU. Flush the
783 * texture cache so we don't get anything stale.
784 */
785 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
786
787 while (dataSize) {
788 const uint32_t copy_size = MIN2(dataSize, max_update_size);
789
790 struct anv_state tmp_data =
791 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, copy_size, 64);
792
793 memcpy(tmp_data.map, pData, copy_size);
794
795 struct blorp_address src = {
796 .buffer = cmd_buffer->device->dynamic_state_pool.block_pool.bo,
797 .offset = tmp_data.offset,
798 .mocs = cmd_buffer->device->default_mocs,
799 };
800 struct blorp_address dst = {
801 .buffer = dst_buffer->address.bo,
802 .offset = dst_buffer->address.offset + dstOffset,
803 .mocs = anv_mocs_for_bo(cmd_buffer->device, dst_buffer->address.bo),
804 };
805
806 blorp_buffer_copy(&batch, src, dst, copy_size);
807
808 dataSize -= copy_size;
809 dstOffset += copy_size;
810 pData = (void *)pData + copy_size;
811 }
812
813 blorp_batch_finish(&batch);
814
815 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
816 }
817
818 void anv_CmdFillBuffer(
819 VkCommandBuffer commandBuffer,
820 VkBuffer dstBuffer,
821 VkDeviceSize dstOffset,
822 VkDeviceSize fillSize,
823 uint32_t data)
824 {
825 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
826 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
827 struct blorp_surf surf;
828 struct isl_surf isl_surf;
829
830 struct blorp_batch batch;
831 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
832
833 fillSize = anv_buffer_get_range(dst_buffer, dstOffset, fillSize);
834
835 /* From the Vulkan spec:
836 *
837 * "size is the number of bytes to fill, and must be either a multiple
838 * of 4, or VK_WHOLE_SIZE to fill the range from offset to the end of
839 * the buffer. If VK_WHOLE_SIZE is used and the remaining size of the
840 * buffer is not a multiple of 4, then the nearest smaller multiple is
841 * used."
842 */
843 fillSize &= ~3ull;
844
845 /* First, we compute the biggest format that can be used with the
846 * given offsets and size.
847 */
848 int bs = 16;
849 bs = gcd_pow2_u64(bs, dstOffset);
850 bs = gcd_pow2_u64(bs, fillSize);
851 enum isl_format isl_format = isl_format_for_size(bs);
852
853 union isl_color_value color = {
854 .u32 = { data, data, data, data },
855 };
856
857 const uint64_t max_fill_size = MAX_SURFACE_DIM * MAX_SURFACE_DIM * bs;
858 while (fillSize >= max_fill_size) {
859 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
860 dst_buffer, dstOffset,
861 MAX_SURFACE_DIM, MAX_SURFACE_DIM,
862 MAX_SURFACE_DIM * bs, isl_format,
863 &surf, &isl_surf);
864
865 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
866 0, 0, 1, 0, 0, MAX_SURFACE_DIM, MAX_SURFACE_DIM,
867 color, NULL);
868 fillSize -= max_fill_size;
869 dstOffset += max_fill_size;
870 }
871
872 uint64_t height = fillSize / (MAX_SURFACE_DIM * bs);
873 assert(height < MAX_SURFACE_DIM);
874 if (height != 0) {
875 const uint64_t rect_fill_size = height * MAX_SURFACE_DIM * bs;
876 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
877 dst_buffer, dstOffset,
878 MAX_SURFACE_DIM, height,
879 MAX_SURFACE_DIM * bs, isl_format,
880 &surf, &isl_surf);
881
882 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
883 0, 0, 1, 0, 0, MAX_SURFACE_DIM, height,
884 color, NULL);
885 fillSize -= rect_fill_size;
886 dstOffset += rect_fill_size;
887 }
888
889 if (fillSize != 0) {
890 const uint32_t width = fillSize / bs;
891 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
892 dst_buffer, dstOffset,
893 width, 1,
894 width * bs, isl_format,
895 &surf, &isl_surf);
896
897 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
898 0, 0, 1, 0, 0, width, 1,
899 color, NULL);
900 }
901
902 blorp_batch_finish(&batch);
903
904 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
905 }
906
907 void anv_CmdClearColorImage(
908 VkCommandBuffer commandBuffer,
909 VkImage _image,
910 VkImageLayout imageLayout,
911 const VkClearColorValue* pColor,
912 uint32_t rangeCount,
913 const VkImageSubresourceRange* pRanges)
914 {
915 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
916 ANV_FROM_HANDLE(anv_image, image, _image);
917
918 static const bool color_write_disable[4] = { false, false, false, false };
919
920 struct blorp_batch batch;
921 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
922
923
924 for (unsigned r = 0; r < rangeCount; r++) {
925 if (pRanges[r].aspectMask == 0)
926 continue;
927
928 assert(pRanges[r].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
929
930 struct blorp_surf surf;
931 get_blorp_surf_for_anv_image(cmd_buffer->device,
932 image, pRanges[r].aspectMask,
933 imageLayout, ISL_AUX_USAGE_NONE, &surf);
934
935 struct anv_format_plane src_format =
936 anv_get_format_plane(&cmd_buffer->device->info, image->vk_format,
937 VK_IMAGE_ASPECT_COLOR_BIT, image->tiling);
938
939 unsigned base_layer = pRanges[r].baseArrayLayer;
940 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
941
942 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
943 const unsigned level = pRanges[r].baseMipLevel + i;
944 const unsigned level_width = anv_minify(image->extent.width, level);
945 const unsigned level_height = anv_minify(image->extent.height, level);
946
947 if (image->type == VK_IMAGE_TYPE_3D) {
948 base_layer = 0;
949 layer_count = anv_minify(image->extent.depth, level);
950 }
951
952 anv_cmd_buffer_mark_image_written(cmd_buffer, image,
953 pRanges[r].aspectMask,
954 surf.aux_usage, level,
955 base_layer, layer_count);
956
957 blorp_clear(&batch, &surf,
958 src_format.isl_format, src_format.swizzle,
959 level, base_layer, layer_count,
960 0, 0, level_width, level_height,
961 vk_to_isl_color(*pColor), color_write_disable);
962 }
963 }
964
965 blorp_batch_finish(&batch);
966 }
967
968 void anv_CmdClearDepthStencilImage(
969 VkCommandBuffer commandBuffer,
970 VkImage image_h,
971 VkImageLayout imageLayout,
972 const VkClearDepthStencilValue* pDepthStencil,
973 uint32_t rangeCount,
974 const VkImageSubresourceRange* pRanges)
975 {
976 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
977 ANV_FROM_HANDLE(anv_image, image, image_h);
978
979 struct blorp_batch batch;
980 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
981
982 struct blorp_surf depth, stencil, stencil_shadow;
983 if (image->aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
984 get_blorp_surf_for_anv_image(cmd_buffer->device,
985 image, VK_IMAGE_ASPECT_DEPTH_BIT,
986 imageLayout, ISL_AUX_USAGE_NONE, &depth);
987 } else {
988 memset(&depth, 0, sizeof(depth));
989 }
990
991 bool has_stencil_shadow = false;
992 if (image->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
993 get_blorp_surf_for_anv_image(cmd_buffer->device,
994 image, VK_IMAGE_ASPECT_STENCIL_BIT,
995 imageLayout, ISL_AUX_USAGE_NONE, &stencil);
996
997 has_stencil_shadow =
998 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device, image,
999 VK_IMAGE_ASPECT_STENCIL_BIT,
1000 &stencil_shadow);
1001 } else {
1002 memset(&stencil, 0, sizeof(stencil));
1003 }
1004
1005 for (unsigned r = 0; r < rangeCount; r++) {
1006 if (pRanges[r].aspectMask == 0)
1007 continue;
1008
1009 bool clear_depth = pRanges[r].aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
1010 bool clear_stencil = pRanges[r].aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
1011
1012 unsigned base_layer = pRanges[r].baseArrayLayer;
1013 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
1014
1015 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
1016 const unsigned level = pRanges[r].baseMipLevel + i;
1017 const unsigned level_width = anv_minify(image->extent.width, level);
1018 const unsigned level_height = anv_minify(image->extent.height, level);
1019
1020 if (image->type == VK_IMAGE_TYPE_3D)
1021 layer_count = anv_minify(image->extent.depth, level);
1022
1023 blorp_clear_depth_stencil(&batch, &depth, &stencil,
1024 level, base_layer, layer_count,
1025 0, 0, level_width, level_height,
1026 clear_depth, pDepthStencil->depth,
1027 clear_stencil ? 0xff : 0,
1028 pDepthStencil->stencil);
1029
1030 if (clear_stencil && has_stencil_shadow) {
1031 union isl_color_value stencil_color = {
1032 .u32 = { pDepthStencil->stencil, },
1033 };
1034 blorp_clear(&batch, &stencil_shadow,
1035 ISL_FORMAT_R8_UINT, ISL_SWIZZLE_IDENTITY,
1036 level, base_layer, layer_count,
1037 0, 0, level_width, level_height,
1038 stencil_color, NULL);
1039 }
1040 }
1041 }
1042
1043 blorp_batch_finish(&batch);
1044 }
1045
1046 VkResult
1047 anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer *cmd_buffer,
1048 uint32_t num_entries,
1049 uint32_t *state_offset,
1050 struct anv_state *bt_state)
1051 {
1052 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
1053 state_offset);
1054 if (bt_state->map == NULL) {
1055 /* We ran out of space. Grab a new binding table block. */
1056 VkResult result = anv_cmd_buffer_new_binding_table_block(cmd_buffer);
1057 if (result != VK_SUCCESS)
1058 return result;
1059
1060 /* Re-emit state base addresses so we get the new surface state base
1061 * address before we start emitting binding tables etc.
1062 */
1063 anv_cmd_buffer_emit_state_base_address(cmd_buffer);
1064
1065 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
1066 state_offset);
1067 assert(bt_state->map != NULL);
1068 }
1069
1070 return VK_SUCCESS;
1071 }
1072
1073 static VkResult
1074 binding_table_for_surface_state(struct anv_cmd_buffer *cmd_buffer,
1075 struct anv_state surface_state,
1076 uint32_t *bt_offset)
1077 {
1078 uint32_t state_offset;
1079 struct anv_state bt_state;
1080
1081 VkResult result =
1082 anv_cmd_buffer_alloc_blorp_binding_table(cmd_buffer, 1, &state_offset,
1083 &bt_state);
1084 if (result != VK_SUCCESS)
1085 return result;
1086
1087 uint32_t *bt_map = bt_state.map;
1088 bt_map[0] = surface_state.offset + state_offset;
1089
1090 *bt_offset = bt_state.offset;
1091 return VK_SUCCESS;
1092 }
1093
1094 static void
1095 clear_color_attachment(struct anv_cmd_buffer *cmd_buffer,
1096 struct blorp_batch *batch,
1097 const VkClearAttachment *attachment,
1098 uint32_t rectCount, const VkClearRect *pRects)
1099 {
1100 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
1101 const uint32_t color_att = attachment->colorAttachment;
1102 assert(color_att < subpass->color_count);
1103 const uint32_t att_idx = subpass->color_attachments[color_att].attachment;
1104
1105 if (att_idx == VK_ATTACHMENT_UNUSED)
1106 return;
1107
1108 struct anv_render_pass_attachment *pass_att =
1109 &cmd_buffer->state.pass->attachments[att_idx];
1110 struct anv_attachment_state *att_state =
1111 &cmd_buffer->state.attachments[att_idx];
1112
1113 uint32_t binding_table;
1114 VkResult result =
1115 binding_table_for_surface_state(cmd_buffer, att_state->color.state,
1116 &binding_table);
1117 if (result != VK_SUCCESS)
1118 return;
1119
1120 union isl_color_value clear_color =
1121 vk_to_isl_color(attachment->clearValue.color);
1122
1123 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1124 if (subpass->view_mask) {
1125 uint32_t view_idx;
1126 for_each_bit(view_idx, subpass->view_mask) {
1127 for (uint32_t r = 0; r < rectCount; ++r) {
1128 const VkOffset2D offset = pRects[r].rect.offset;
1129 const VkExtent2D extent = pRects[r].rect.extent;
1130 blorp_clear_attachments(batch, binding_table,
1131 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1132 view_idx, 1,
1133 offset.x, offset.y,
1134 offset.x + extent.width,
1135 offset.y + extent.height,
1136 true, clear_color, false, 0.0f, 0, 0);
1137 }
1138 }
1139 return;
1140 }
1141
1142 for (uint32_t r = 0; r < rectCount; ++r) {
1143 const VkOffset2D offset = pRects[r].rect.offset;
1144 const VkExtent2D extent = pRects[r].rect.extent;
1145 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1146 blorp_clear_attachments(batch, binding_table,
1147 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1148 pRects[r].baseArrayLayer,
1149 pRects[r].layerCount,
1150 offset.x, offset.y,
1151 offset.x + extent.width, offset.y + extent.height,
1152 true, clear_color, false, 0.0f, 0, 0);
1153 }
1154 }
1155
1156 static void
1157 clear_depth_stencil_attachment(struct anv_cmd_buffer *cmd_buffer,
1158 struct blorp_batch *batch,
1159 const VkClearAttachment *attachment,
1160 uint32_t rectCount, const VkClearRect *pRects)
1161 {
1162 static const union isl_color_value color_value = { .u32 = { 0, } };
1163 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
1164 if (!subpass->depth_stencil_attachment)
1165 return;
1166
1167 const uint32_t att_idx = subpass->depth_stencil_attachment->attachment;
1168 assert(att_idx != VK_ATTACHMENT_UNUSED);
1169 struct anv_render_pass_attachment *pass_att =
1170 &cmd_buffer->state.pass->attachments[att_idx];
1171
1172 bool clear_depth = attachment->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
1173 bool clear_stencil = attachment->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
1174
1175 enum isl_format depth_format = ISL_FORMAT_UNSUPPORTED;
1176 if (clear_depth) {
1177 depth_format = anv_get_isl_format(&cmd_buffer->device->info,
1178 pass_att->format,
1179 VK_IMAGE_ASPECT_DEPTH_BIT,
1180 VK_IMAGE_TILING_OPTIMAL);
1181 }
1182
1183 uint32_t binding_table;
1184 VkResult result =
1185 binding_table_for_surface_state(cmd_buffer,
1186 cmd_buffer->state.null_surface_state,
1187 &binding_table);
1188 if (result != VK_SUCCESS)
1189 return;
1190
1191 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1192 if (subpass->view_mask) {
1193 uint32_t view_idx;
1194 for_each_bit(view_idx, subpass->view_mask) {
1195 for (uint32_t r = 0; r < rectCount; ++r) {
1196 const VkOffset2D offset = pRects[r].rect.offset;
1197 const VkExtent2D extent = pRects[r].rect.extent;
1198 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1199 blorp_clear_attachments(batch, binding_table,
1200 depth_format, pass_att->samples,
1201 view_idx, 1,
1202 offset.x, offset.y,
1203 offset.x + extent.width,
1204 offset.y + extent.height,
1205 false, color_value,
1206 clear_depth, value.depth,
1207 clear_stencil ? 0xff : 0, value.stencil);
1208 }
1209 }
1210 return;
1211 }
1212
1213 for (uint32_t r = 0; r < rectCount; ++r) {
1214 const VkOffset2D offset = pRects[r].rect.offset;
1215 const VkExtent2D extent = pRects[r].rect.extent;
1216 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1217 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1218 blorp_clear_attachments(batch, binding_table,
1219 depth_format, pass_att->samples,
1220 pRects[r].baseArrayLayer,
1221 pRects[r].layerCount,
1222 offset.x, offset.y,
1223 offset.x + extent.width, offset.y + extent.height,
1224 false, color_value,
1225 clear_depth, value.depth,
1226 clear_stencil ? 0xff : 0, value.stencil);
1227 }
1228 }
1229
1230 void anv_CmdClearAttachments(
1231 VkCommandBuffer commandBuffer,
1232 uint32_t attachmentCount,
1233 const VkClearAttachment* pAttachments,
1234 uint32_t rectCount,
1235 const VkClearRect* pRects)
1236 {
1237 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1238
1239 /* Because this gets called within a render pass, we tell blorp not to
1240 * trash our depth and stencil buffers.
1241 */
1242 struct blorp_batch batch;
1243 enum blorp_batch_flags flags = BLORP_BATCH_NO_EMIT_DEPTH_STENCIL;
1244 if (cmd_buffer->state.conditional_render_enabled) {
1245 anv_cmd_emit_conditional_render_predicate(cmd_buffer);
1246 flags |= BLORP_BATCH_PREDICATE_ENABLE;
1247 }
1248 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, flags);
1249
1250 for (uint32_t a = 0; a < attachmentCount; ++a) {
1251 if (pAttachments[a].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
1252 assert(pAttachments[a].aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
1253 clear_color_attachment(cmd_buffer, &batch,
1254 &pAttachments[a],
1255 rectCount, pRects);
1256 } else {
1257 clear_depth_stencil_attachment(cmd_buffer, &batch,
1258 &pAttachments[a],
1259 rectCount, pRects);
1260 }
1261 }
1262
1263 blorp_batch_finish(&batch);
1264 }
1265
1266 enum subpass_stage {
1267 SUBPASS_STAGE_LOAD,
1268 SUBPASS_STAGE_DRAW,
1269 SUBPASS_STAGE_RESOLVE,
1270 };
1271
1272 void
1273 anv_image_msaa_resolve(struct anv_cmd_buffer *cmd_buffer,
1274 const struct anv_image *src_image,
1275 enum isl_aux_usage src_aux_usage,
1276 uint32_t src_level, uint32_t src_base_layer,
1277 const struct anv_image *dst_image,
1278 enum isl_aux_usage dst_aux_usage,
1279 uint32_t dst_level, uint32_t dst_base_layer,
1280 VkImageAspectFlagBits aspect,
1281 uint32_t src_x, uint32_t src_y,
1282 uint32_t dst_x, uint32_t dst_y,
1283 uint32_t width, uint32_t height,
1284 uint32_t layer_count,
1285 enum blorp_filter filter)
1286 {
1287 struct blorp_batch batch;
1288 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1289
1290 assert(src_image->type == VK_IMAGE_TYPE_2D);
1291 assert(src_image->samples > 1);
1292 assert(dst_image->type == VK_IMAGE_TYPE_2D);
1293 assert(dst_image->samples == 1);
1294 assert(src_image->n_planes == dst_image->n_planes);
1295 assert(!src_image->format->can_ycbcr);
1296 assert(!dst_image->format->can_ycbcr);
1297
1298 struct blorp_surf src_surf, dst_surf;
1299 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, aspect,
1300 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1301 src_aux_usage, &src_surf);
1302 if (src_aux_usage == ISL_AUX_USAGE_MCS) {
1303 src_surf.clear_color_addr = anv_to_blorp_address(
1304 anv_image_get_clear_color_addr(cmd_buffer->device, src_image,
1305 VK_IMAGE_ASPECT_COLOR_BIT));
1306 }
1307 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, aspect,
1308 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1309 dst_aux_usage, &dst_surf);
1310 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
1311 aspect, dst_aux_usage,
1312 dst_level, dst_base_layer, layer_count);
1313
1314 if (filter == BLORP_FILTER_NONE) {
1315 /* If no explicit filter is provided, then it's implied by the type of
1316 * the source image.
1317 */
1318 if ((src_surf.surf->usage & ISL_SURF_USAGE_DEPTH_BIT) ||
1319 (src_surf.surf->usage & ISL_SURF_USAGE_STENCIL_BIT) ||
1320 isl_format_has_int_channel(src_surf.surf->format)) {
1321 filter = BLORP_FILTER_SAMPLE_0;
1322 } else {
1323 filter = BLORP_FILTER_AVERAGE;
1324 }
1325 }
1326
1327 for (uint32_t l = 0; l < layer_count; l++) {
1328 blorp_blit(&batch,
1329 &src_surf, src_level, src_base_layer + l,
1330 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1331 &dst_surf, dst_level, dst_base_layer + l,
1332 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1333 src_x, src_y, src_x + width, src_y + height,
1334 dst_x, dst_y, dst_x + width, dst_y + height,
1335 filter, false, false);
1336 }
1337
1338 blorp_batch_finish(&batch);
1339 }
1340
1341 void anv_CmdResolveImage(
1342 VkCommandBuffer commandBuffer,
1343 VkImage srcImage,
1344 VkImageLayout srcImageLayout,
1345 VkImage dstImage,
1346 VkImageLayout dstImageLayout,
1347 uint32_t regionCount,
1348 const VkImageResolve* pRegions)
1349 {
1350 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1351 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
1352 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
1353
1354 assert(!src_image->format->can_ycbcr);
1355
1356 for (uint32_t r = 0; r < regionCount; r++) {
1357 assert(pRegions[r].srcSubresource.aspectMask ==
1358 pRegions[r].dstSubresource.aspectMask);
1359 assert(anv_get_layerCount(src_image, &pRegions[r].srcSubresource) ==
1360 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource));
1361
1362 const uint32_t layer_count =
1363 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource);
1364
1365 uint32_t aspect_bit;
1366 anv_foreach_image_aspect_bit(aspect_bit, src_image,
1367 pRegions[r].srcSubresource.aspectMask) {
1368 enum isl_aux_usage src_aux_usage =
1369 anv_layout_to_aux_usage(&cmd_buffer->device->info, src_image,
1370 (1 << aspect_bit), srcImageLayout);
1371 enum isl_aux_usage dst_aux_usage =
1372 anv_layout_to_aux_usage(&cmd_buffer->device->info, dst_image,
1373 (1 << aspect_bit), dstImageLayout);
1374
1375 anv_image_msaa_resolve(cmd_buffer,
1376 src_image, src_aux_usage,
1377 pRegions[r].srcSubresource.mipLevel,
1378 pRegions[r].srcSubresource.baseArrayLayer,
1379 dst_image, dst_aux_usage,
1380 pRegions[r].dstSubresource.mipLevel,
1381 pRegions[r].dstSubresource.baseArrayLayer,
1382 (1 << aspect_bit),
1383 pRegions[r].srcOffset.x,
1384 pRegions[r].srcOffset.y,
1385 pRegions[r].dstOffset.x,
1386 pRegions[r].dstOffset.y,
1387 pRegions[r].extent.width,
1388 pRegions[r].extent.height,
1389 layer_count, BLORP_FILTER_NONE);
1390 }
1391 }
1392 }
1393
1394 static enum isl_aux_usage
1395 fast_clear_aux_usage(const struct anv_image *image,
1396 VkImageAspectFlagBits aspect)
1397 {
1398 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
1399 if (image->planes[plane].aux_usage == ISL_AUX_USAGE_NONE)
1400 return ISL_AUX_USAGE_CCS_D;
1401 else
1402 return image->planes[plane].aux_usage;
1403 }
1404
1405 void
1406 anv_image_copy_to_shadow(struct anv_cmd_buffer *cmd_buffer,
1407 const struct anv_image *image,
1408 VkImageAspectFlagBits aspect,
1409 uint32_t base_level, uint32_t level_count,
1410 uint32_t base_layer, uint32_t layer_count)
1411 {
1412 struct blorp_batch batch;
1413 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1414
1415 /* We don't know who touched the main surface last so flush a bunch of
1416 * caches to ensure we get good data.
1417 */
1418 cmd_buffer->state.pending_pipe_bits |=
1419 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT |
1420 ANV_PIPE_DATA_CACHE_FLUSH_BIT |
1421 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT |
1422 ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
1423
1424 struct blorp_surf surf;
1425 get_blorp_surf_for_anv_image(cmd_buffer->device,
1426 image, aspect,
1427 VK_IMAGE_LAYOUT_GENERAL,
1428 ISL_AUX_USAGE_NONE, &surf);
1429 assert(surf.aux_usage == ISL_AUX_USAGE_NONE);
1430
1431 struct blorp_surf shadow_surf;
1432 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
1433 image, aspect, &shadow_surf);
1434
1435 for (uint32_t l = 0; l < level_count; l++) {
1436 const uint32_t level = base_level + l;
1437
1438 const VkExtent3D extent = {
1439 .width = anv_minify(image->extent.width, level),
1440 .height = anv_minify(image->extent.height, level),
1441 .depth = anv_minify(image->extent.depth, level),
1442 };
1443
1444 if (image->type == VK_IMAGE_TYPE_3D)
1445 layer_count = extent.depth;
1446
1447 for (uint32_t a = 0; a < layer_count; a++) {
1448 const uint32_t layer = base_layer + a;
1449
1450 blorp_copy(&batch, &surf, level, layer,
1451 &shadow_surf, level, layer,
1452 0, 0, 0, 0, extent.width, extent.height);
1453 }
1454 }
1455
1456 /* We just wrote to the buffer with the render cache. Flush it. */
1457 cmd_buffer->state.pending_pipe_bits |=
1458 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
1459
1460 blorp_batch_finish(&batch);
1461 }
1462
1463 void
1464 anv_image_clear_color(struct anv_cmd_buffer *cmd_buffer,
1465 const struct anv_image *image,
1466 VkImageAspectFlagBits aspect,
1467 enum isl_aux_usage aux_usage,
1468 enum isl_format format, struct isl_swizzle swizzle,
1469 uint32_t level, uint32_t base_layer, uint32_t layer_count,
1470 VkRect2D area, union isl_color_value clear_color)
1471 {
1472 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1473
1474 /* We don't support planar images with multisampling yet */
1475 assert(image->n_planes == 1);
1476
1477 struct blorp_batch batch;
1478 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1479
1480 struct blorp_surf surf;
1481 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1482 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1483 aux_usage, &surf);
1484 anv_cmd_buffer_mark_image_written(cmd_buffer, image, aspect, aux_usage,
1485 level, base_layer, layer_count);
1486
1487 blorp_clear(&batch, &surf, format, anv_swizzle_for_render(swizzle),
1488 level, base_layer, layer_count,
1489 area.offset.x, area.offset.y,
1490 area.offset.x + area.extent.width,
1491 area.offset.y + area.extent.height,
1492 clear_color, NULL);
1493
1494 blorp_batch_finish(&batch);
1495 }
1496
1497 void
1498 anv_image_clear_depth_stencil(struct anv_cmd_buffer *cmd_buffer,
1499 const struct anv_image *image,
1500 VkImageAspectFlags aspects,
1501 enum isl_aux_usage depth_aux_usage,
1502 uint32_t level,
1503 uint32_t base_layer, uint32_t layer_count,
1504 VkRect2D area,
1505 float depth_value, uint8_t stencil_value)
1506 {
1507 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
1508 VK_IMAGE_ASPECT_STENCIL_BIT));
1509
1510 struct blorp_batch batch;
1511 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1512
1513 struct blorp_surf depth = {};
1514 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1515 get_blorp_surf_for_anv_image(cmd_buffer->device,
1516 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1517 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1518 depth_aux_usage, &depth);
1519 depth.clear_color.f32[0] = ANV_HZ_FC_VAL;
1520 }
1521
1522 struct blorp_surf stencil = {};
1523 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1524 get_blorp_surf_for_anv_image(cmd_buffer->device,
1525 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1526 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1527 ISL_AUX_USAGE_NONE, &stencil);
1528 }
1529
1530 /* Blorp may choose to clear stencil using RGBA32_UINT for better
1531 * performance. If it does this, we need to flush it out of the depth
1532 * cache before rendering to it.
1533 */
1534 cmd_buffer->state.pending_pipe_bits |=
1535 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1536
1537 blorp_clear_depth_stencil(&batch, &depth, &stencil,
1538 level, base_layer, layer_count,
1539 area.offset.x, area.offset.y,
1540 area.offset.x + area.extent.width,
1541 area.offset.y + area.extent.height,
1542 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
1543 depth_value,
1544 (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) ? 0xff : 0,
1545 stencil_value);
1546
1547 /* Blorp may choose to clear stencil using RGBA32_UINT for better
1548 * performance. If it does this, we need to flush it out of the render
1549 * cache before someone starts trying to do stencil on it.
1550 */
1551 cmd_buffer->state.pending_pipe_bits |=
1552 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1553
1554 struct blorp_surf stencil_shadow;
1555 if ((aspects & VK_IMAGE_ASPECT_STENCIL_BIT) &&
1556 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device, image,
1557 VK_IMAGE_ASPECT_STENCIL_BIT,
1558 &stencil_shadow)) {
1559 union isl_color_value stencil_color = {
1560 .u32 = { stencil_value },
1561 };
1562 blorp_clear(&batch, &stencil_shadow,
1563 ISL_FORMAT_R8_UINT, ISL_SWIZZLE_IDENTITY,
1564 level, base_layer, layer_count,
1565 area.offset.x, area.offset.y,
1566 area.offset.x + area.extent.width,
1567 area.offset.y + area.extent.height,
1568 stencil_color, NULL);
1569 }
1570
1571 blorp_batch_finish(&batch);
1572 }
1573
1574 void
1575 anv_image_hiz_op(struct anv_cmd_buffer *cmd_buffer,
1576 const struct anv_image *image,
1577 VkImageAspectFlagBits aspect, uint32_t level,
1578 uint32_t base_layer, uint32_t layer_count,
1579 enum isl_aux_op hiz_op)
1580 {
1581 assert(aspect == VK_IMAGE_ASPECT_DEPTH_BIT);
1582 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, level));
1583 assert(anv_image_aspect_to_plane(image->aspects,
1584 VK_IMAGE_ASPECT_DEPTH_BIT) == 0);
1585
1586 struct blorp_batch batch;
1587 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1588
1589 struct blorp_surf surf;
1590 get_blorp_surf_for_anv_image(cmd_buffer->device,
1591 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1592 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1593 ISL_AUX_USAGE_HIZ, &surf);
1594 surf.clear_color.f32[0] = ANV_HZ_FC_VAL;
1595
1596 blorp_hiz_op(&batch, &surf, level, base_layer, layer_count, hiz_op);
1597
1598 blorp_batch_finish(&batch);
1599 }
1600
1601 void
1602 anv_image_hiz_clear(struct anv_cmd_buffer *cmd_buffer,
1603 const struct anv_image *image,
1604 VkImageAspectFlags aspects,
1605 uint32_t level,
1606 uint32_t base_layer, uint32_t layer_count,
1607 VkRect2D area, uint8_t stencil_value)
1608 {
1609 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
1610 VK_IMAGE_ASPECT_STENCIL_BIT));
1611
1612 struct blorp_batch batch;
1613 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1614
1615 struct blorp_surf depth = {};
1616 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1617 assert(base_layer + layer_count <=
1618 anv_image_aux_layers(image, VK_IMAGE_ASPECT_DEPTH_BIT, level));
1619 get_blorp_surf_for_anv_image(cmd_buffer->device,
1620 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1621 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1622 ISL_AUX_USAGE_HIZ, &depth);
1623 depth.clear_color.f32[0] = ANV_HZ_FC_VAL;
1624 }
1625
1626 struct blorp_surf stencil = {};
1627 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1628 get_blorp_surf_for_anv_image(cmd_buffer->device,
1629 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1630 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1631 ISL_AUX_USAGE_NONE, &stencil);
1632 }
1633
1634 /* From the Sky Lake PRM Volume 7, "Depth Buffer Clear":
1635 *
1636 * "The following is required when performing a depth buffer clear with
1637 * using the WM_STATE or 3DSTATE_WM:
1638 *
1639 * * If other rendering operations have preceded this clear, a
1640 * PIPE_CONTROL with depth cache flush enabled, Depth Stall bit
1641 * enabled must be issued before the rectangle primitive used for
1642 * the depth buffer clear operation.
1643 * * [...]"
1644 *
1645 * Even though the PRM only says that this is required if using 3DSTATE_WM
1646 * and a 3DPRIMITIVE, the GPU appears to also need this to avoid occasional
1647 * hangs when doing a clear with WM_HZ_OP.
1648 */
1649 cmd_buffer->state.pending_pipe_bits |=
1650 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT;
1651
1652 blorp_hiz_clear_depth_stencil(&batch, &depth, &stencil,
1653 level, base_layer, layer_count,
1654 area.offset.x, area.offset.y,
1655 area.offset.x + area.extent.width,
1656 area.offset.y + area.extent.height,
1657 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
1658 ANV_HZ_FC_VAL,
1659 aspects & VK_IMAGE_ASPECT_STENCIL_BIT,
1660 stencil_value);
1661
1662 blorp_batch_finish(&batch);
1663
1664 /* From the SKL PRM, Depth Buffer Clear:
1665 *
1666 * "Depth Buffer Clear Workaround
1667 *
1668 * Depth buffer clear pass using any of the methods (WM_STATE,
1669 * 3DSTATE_WM or 3DSTATE_WM_HZ_OP) must be followed by a PIPE_CONTROL
1670 * command with DEPTH_STALL bit and Depth FLUSH bits “set” before
1671 * starting to render. DepthStall and DepthFlush are not needed between
1672 * consecutive depth clear passes nor is it required if the depth-clear
1673 * pass was done with “full_surf_clear” bit set in the
1674 * 3DSTATE_WM_HZ_OP."
1675 *
1676 * Even though the PRM provides a bunch of conditions under which this is
1677 * supposedly unnecessary, we choose to perform the flush unconditionally
1678 * just to be safe.
1679 */
1680 cmd_buffer->state.pending_pipe_bits |=
1681 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT;
1682 }
1683
1684 void
1685 anv_image_mcs_op(struct anv_cmd_buffer *cmd_buffer,
1686 const struct anv_image *image,
1687 enum isl_format format,
1688 VkImageAspectFlagBits aspect,
1689 uint32_t base_layer, uint32_t layer_count,
1690 enum isl_aux_op mcs_op, union isl_color_value *clear_value,
1691 bool predicate)
1692 {
1693 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1694 assert(image->samples > 1);
1695 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, 0));
1696
1697 /* Multisampling with multi-planar formats is not supported */
1698 assert(image->n_planes == 1);
1699
1700 struct blorp_batch batch;
1701 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1702 BLORP_BATCH_PREDICATE_ENABLE * predicate +
1703 BLORP_BATCH_NO_UPDATE_CLEAR_COLOR * !clear_value);
1704
1705 struct blorp_surf surf;
1706 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1707 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1708 ISL_AUX_USAGE_MCS, &surf);
1709
1710 /* Blorp will store the clear color for us if we provide the clear color
1711 * address and we are doing a fast clear. So we save the clear value into
1712 * the blorp surface.
1713 */
1714 if (clear_value)
1715 surf.clear_color = *clear_value;
1716
1717 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1718 *
1719 * "After Render target fast clear, pipe-control with color cache
1720 * write-flush must be issued before sending any DRAW commands on
1721 * that render target."
1722 *
1723 * This comment is a bit cryptic and doesn't really tell you what's going
1724 * or what's really needed. It appears that fast clear ops are not
1725 * properly synchronized with other drawing. This means that we cannot
1726 * have a fast clear operation in the pipe at the same time as other
1727 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
1728 * that the contents of the previous draw hit the render target before we
1729 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
1730 * that it is completed before any additional drawing occurs.
1731 */
1732 cmd_buffer->state.pending_pipe_bits |=
1733 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1734
1735 switch (mcs_op) {
1736 case ISL_AUX_OP_FAST_CLEAR:
1737 blorp_fast_clear(&batch, &surf, format,
1738 0, base_layer, layer_count,
1739 0, 0, image->extent.width, image->extent.height);
1740 break;
1741 case ISL_AUX_OP_PARTIAL_RESOLVE:
1742 blorp_mcs_partial_resolve(&batch, &surf, format,
1743 base_layer, layer_count);
1744 break;
1745 case ISL_AUX_OP_FULL_RESOLVE:
1746 case ISL_AUX_OP_AMBIGUATE:
1747 default:
1748 unreachable("Unsupported MCS operation");
1749 }
1750
1751 cmd_buffer->state.pending_pipe_bits |=
1752 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1753
1754 blorp_batch_finish(&batch);
1755 }
1756
1757 void
1758 anv_image_ccs_op(struct anv_cmd_buffer *cmd_buffer,
1759 const struct anv_image *image,
1760 enum isl_format format,
1761 VkImageAspectFlagBits aspect, uint32_t level,
1762 uint32_t base_layer, uint32_t layer_count,
1763 enum isl_aux_op ccs_op, union isl_color_value *clear_value,
1764 bool predicate)
1765 {
1766 assert(image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
1767 assert(image->samples == 1);
1768 assert(level < anv_image_aux_levels(image, aspect));
1769 /* Multi-LOD YcBcR is not allowed */
1770 assert(image->n_planes == 1 || level == 0);
1771 assert(base_layer + layer_count <=
1772 anv_image_aux_layers(image, aspect, level));
1773
1774 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
1775 uint32_t width_div = image->format->planes[plane].denominator_scales[0];
1776 uint32_t height_div = image->format->planes[plane].denominator_scales[1];
1777 uint32_t level_width = anv_minify(image->extent.width, level) / width_div;
1778 uint32_t level_height = anv_minify(image->extent.height, level) / height_div;
1779
1780 struct blorp_batch batch;
1781 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1782 BLORP_BATCH_PREDICATE_ENABLE * predicate +
1783 BLORP_BATCH_NO_UPDATE_CLEAR_COLOR * !clear_value);
1784
1785 struct blorp_surf surf;
1786 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1787 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1788 fast_clear_aux_usage(image, aspect),
1789 &surf);
1790
1791 /* Blorp will store the clear color for us if we provide the clear color
1792 * address and we are doing a fast clear. So we save the clear value into
1793 * the blorp surface.
1794 */
1795 if (clear_value)
1796 surf.clear_color = *clear_value;
1797
1798 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1799 *
1800 * "After Render target fast clear, pipe-control with color cache
1801 * write-flush must be issued before sending any DRAW commands on
1802 * that render target."
1803 *
1804 * This comment is a bit cryptic and doesn't really tell you what's going
1805 * or what's really needed. It appears that fast clear ops are not
1806 * properly synchronized with other drawing. This means that we cannot
1807 * have a fast clear operation in the pipe at the same time as other
1808 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
1809 * that the contents of the previous draw hit the render target before we
1810 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
1811 * that it is completed before any additional drawing occurs.
1812 */
1813 cmd_buffer->state.pending_pipe_bits |=
1814 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1815
1816 switch (ccs_op) {
1817 case ISL_AUX_OP_FAST_CLEAR:
1818 blorp_fast_clear(&batch, &surf, format,
1819 level, base_layer, layer_count,
1820 0, 0, level_width, level_height);
1821 break;
1822 case ISL_AUX_OP_FULL_RESOLVE:
1823 case ISL_AUX_OP_PARTIAL_RESOLVE:
1824 blorp_ccs_resolve(&batch, &surf, level, base_layer, layer_count,
1825 format, ccs_op);
1826 break;
1827 case ISL_AUX_OP_AMBIGUATE:
1828 for (uint32_t a = 0; a < layer_count; a++) {
1829 const uint32_t layer = base_layer + a;
1830 blorp_ccs_ambiguate(&batch, &surf, level, layer);
1831 }
1832 break;
1833 default:
1834 unreachable("Unsupported CCS operation");
1835 }
1836
1837 cmd_buffer->state.pending_pipe_bits |=
1838 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1839
1840 blorp_batch_finish(&batch);
1841 }