projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
anv/blorp: Rename buffer image stride parameters
[mesa.git] / src / intel / vulkan / anv_blorp.c
1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "anv_private.h"
25
26 static bool
27 lookup_blorp_shader(struct blorp_batch *batch,
28 const void *key, uint32_t key_size,
29 uint32_t *kernel_out, void *prog_data_out)
30 {
31 struct blorp_context *blorp = batch->blorp;
32 struct anv_device *device = blorp->driver_ctx;
33
34 /* The default cache must be a real cache */
35 assert(device->default_pipeline_cache.cache);
36
37 struct anv_shader_bin *bin =
38 anv_pipeline_cache_search(&device->default_pipeline_cache, key, key_size);
39 if (!bin)
40 return false;
41
42 /* The cache already has a reference and it's not going anywhere so there
43 * is no need to hold a second reference.
44 */
45 anv_shader_bin_unref(device, bin);
46
47 *kernel_out = bin->kernel.offset;
48 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
49
50 return true;
51 }
52
53 static bool
54 upload_blorp_shader(struct blorp_batch *batch,
55 const void *key, uint32_t key_size,
56 const void *kernel, uint32_t kernel_size,
57 const struct brw_stage_prog_data *prog_data,
58 uint32_t prog_data_size,
59 uint32_t *kernel_out, void *prog_data_out)
60 {
61 struct blorp_context *blorp = batch->blorp;
62 struct anv_device *device = blorp->driver_ctx;
63
64 /* The blorp cache must be a real cache */
65 assert(device->default_pipeline_cache.cache);
66
67 struct anv_pipeline_bind_map bind_map = {
68 .surface_count = 0,
69 .sampler_count = 0,
70 };
71
72 struct anv_shader_bin *bin =
73 anv_pipeline_cache_upload_kernel(&device->default_pipeline_cache,
74 key, key_size, kernel, kernel_size,
75 NULL, 0,
76 prog_data, prog_data_size,
77 NULL, 0, NULL, &bind_map);
78
79 if (!bin)
80 return false;
81
82 /* The cache already has a reference and it's not going anywhere so there
83 * is no need to hold a second reference.
84 */
85 anv_shader_bin_unref(device, bin);
86
87 *kernel_out = bin->kernel.offset;
88 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
89
90 return true;
91 }
92
93 void
94 anv_device_init_blorp(struct anv_device *device)
95 {
96 blorp_init(&device->blorp, device, &device->isl_dev);
97 device->blorp.compiler = device->instance->physicalDevice.compiler;
98 device->blorp.lookup_shader = lookup_blorp_shader;
99 device->blorp.upload_shader = upload_blorp_shader;
100 switch (device->info.gen) {
101 case 7:
102 if (device->info.is_haswell) {
103 device->blorp.exec = gen75_blorp_exec;
104 } else {
105 device->blorp.exec = gen7_blorp_exec;
106 }
107 break;
108 case 8:
109 device->blorp.exec = gen8_blorp_exec;
110 break;
111 case 9:
112 device->blorp.exec = gen9_blorp_exec;
113 break;
114 case 10:
115 device->blorp.exec = gen10_blorp_exec;
116 break;
117 case 11:
118 device->blorp.exec = gen11_blorp_exec;
119 break;
120 case 12:
121 device->blorp.exec = gen12_blorp_exec;
122 break;
123 default:
124 unreachable("Unknown hardware generation");
125 }
126 }
127
128 void
129 anv_device_finish_blorp(struct anv_device *device)
130 {
131 blorp_finish(&device->blorp);
132 }
133
134 static void
135 get_blorp_surf_for_anv_buffer(struct anv_device *device,
136 struct anv_buffer *buffer, uint64_t offset,
137 uint32_t width, uint32_t height,
138 uint32_t row_pitch, enum isl_format format,
139 struct blorp_surf *blorp_surf,
140 struct isl_surf *isl_surf)
141 {
142 const struct isl_format_layout *fmtl =
143 isl_format_get_layout(format);
144 bool ok UNUSED;
145
146 /* ASTC is the only format which doesn't support linear layouts.
147 * Create an equivalently sized surface with ISL to get around this.
148 */
149 if (fmtl->txc == ISL_TXC_ASTC) {
150 /* Use an equivalently sized format */
151 format = ISL_FORMAT_R32G32B32A32_UINT;
152 assert(fmtl->bpb == isl_format_get_layout(format)->bpb);
153
154 /* Shrink the dimensions for the new format */
155 width = DIV_ROUND_UP(width, fmtl->bw);
156 height = DIV_ROUND_UP(height, fmtl->bh);
157 }
158
159 *blorp_surf = (struct blorp_surf) {
160 .surf = isl_surf,
161 .addr = {
162 .buffer = buffer->address.bo,
163 .offset = buffer->address.offset + offset,
164 .mocs = anv_mocs_for_bo(device, buffer->address.bo),
165 },
166 };
167
168 ok = isl_surf_init(&device->isl_dev, isl_surf,
169 .dim = ISL_SURF_DIM_2D,
170 .format = format,
171 .width = width,
172 .height = height,
173 .depth = 1,
174 .levels = 1,
175 .array_len = 1,
176 .samples = 1,
177 .row_pitch_B = row_pitch,
178 .usage = ISL_SURF_USAGE_TEXTURE_BIT |
179 ISL_SURF_USAGE_RENDER_TARGET_BIT,
180 .tiling_flags = ISL_TILING_LINEAR_BIT);
181 assert(ok);
182 }
183
184 /* Pick something high enough that it won't be used in core and low enough it
185 * will never map to an extension.
186 */
187 #define ANV_IMAGE_LAYOUT_EXPLICIT_AUX (VkImageLayout)10000000
188
189 static struct blorp_address
190 anv_to_blorp_address(struct anv_address addr)
191 {
192 return (struct blorp_address) {
193 .buffer = addr.bo,
194 .offset = addr.offset,
195 };
196 }
197
198 static void
199 get_blorp_surf_for_anv_image(const struct anv_device *device,
200 const struct anv_image *image,
201 VkImageAspectFlags aspect,
202 VkImageLayout layout,
203 enum isl_aux_usage aux_usage,
204 struct blorp_surf *blorp_surf)
205 {
206 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
207
208 if (layout != ANV_IMAGE_LAYOUT_EXPLICIT_AUX)
209 aux_usage = anv_layout_to_aux_usage(&device->info, image, aspect, layout);
210
211 const struct anv_surface *surface = &image->planes[plane].surface;
212 *blorp_surf = (struct blorp_surf) {
213 .surf = &surface->isl,
214 .addr = {
215 .buffer = image->planes[plane].address.bo,
216 .offset = image->planes[plane].address.offset + surface->offset,
217 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
218 },
219 };
220
221 if (aux_usage != ISL_AUX_USAGE_NONE) {
222 const struct anv_surface *aux_surface = &image->planes[plane].aux_surface;
223 blorp_surf->aux_surf = &aux_surface->isl,
224 blorp_surf->aux_addr = (struct blorp_address) {
225 .buffer = image->planes[plane].address.bo,
226 .offset = image->planes[plane].address.offset + aux_surface->offset,
227 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
228 };
229 blorp_surf->aux_usage = aux_usage;
230
231 /* If we're doing a partial resolve, then we need the indirect clear
232 * color. If we are doing a fast clear and want to store/update the
233 * clear color, we also pass the address to blorp, otherwise it will only
234 * stomp the CCS to a particular value and won't care about format or
235 * clear value
236 */
237 if (aspect & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
238 const struct anv_address clear_color_addr =
239 anv_image_get_clear_color_addr(device, image, aspect);
240 blorp_surf->clear_color_addr = anv_to_blorp_address(clear_color_addr);
241 } else if (aspect & VK_IMAGE_ASPECT_DEPTH_BIT) {
242 if (device->info.gen >= 10) {
243 /* Vulkan always clears to 1.0. On gen < 10, we set that directly
244 * in the state packet. For gen >= 10, must provide the clear
245 * value in a buffer. We have a single global buffer that stores
246 * the 1.0 value.
247 */
248 const struct anv_address clear_color_addr = (struct anv_address) {
249 .bo = device->hiz_clear_bo,
250 };
251 blorp_surf->clear_color_addr =
252 anv_to_blorp_address(clear_color_addr);
253 } else {
254 blorp_surf->clear_color = (union isl_color_value) {
255 .f32 = { ANV_HZ_FC_VAL },
256 };
257 }
258 }
259 }
260 }
261
262 static bool
263 get_blorp_surf_for_anv_shadow_image(const struct anv_device *device,
264 const struct anv_image *image,
265 VkImageAspectFlags aspect,
266 struct blorp_surf *blorp_surf)
267 {
268
269 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
270 if (image->planes[plane].shadow_surface.isl.size_B == 0)
271 return false;
272
273 *blorp_surf = (struct blorp_surf) {
274 .surf = &image->planes[plane].shadow_surface.isl,
275 .addr = {
276 .buffer = image->planes[plane].address.bo,
277 .offset = image->planes[plane].address.offset +
278 image->planes[plane].shadow_surface.offset,
279 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
280 },
281 };
282
283 return true;
284 }
285
286 void anv_CmdCopyImage(
287 VkCommandBuffer commandBuffer,
288 VkImage srcImage,
289 VkImageLayout srcImageLayout,
290 VkImage dstImage,
291 VkImageLayout dstImageLayout,
292 uint32_t regionCount,
293 const VkImageCopy* pRegions)
294 {
295 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
296 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
297 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
298
299 struct blorp_batch batch;
300 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
301
302 for (unsigned r = 0; r < regionCount; r++) {
303 VkOffset3D srcOffset =
304 anv_sanitize_image_offset(src_image->type, pRegions[r].srcOffset);
305 VkOffset3D dstOffset =
306 anv_sanitize_image_offset(dst_image->type, pRegions[r].dstOffset);
307 VkExtent3D extent =
308 anv_sanitize_image_extent(src_image->type, pRegions[r].extent);
309
310 const uint32_t dst_level = pRegions[r].dstSubresource.mipLevel;
311 unsigned dst_base_layer, layer_count;
312 if (dst_image->type == VK_IMAGE_TYPE_3D) {
313 dst_base_layer = pRegions[r].dstOffset.z;
314 layer_count = pRegions[r].extent.depth;
315 } else {
316 dst_base_layer = pRegions[r].dstSubresource.baseArrayLayer;
317 layer_count =
318 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource);
319 }
320
321 const uint32_t src_level = pRegions[r].srcSubresource.mipLevel;
322 unsigned src_base_layer;
323 if (src_image->type == VK_IMAGE_TYPE_3D) {
324 src_base_layer = pRegions[r].srcOffset.z;
325 } else {
326 src_base_layer = pRegions[r].srcSubresource.baseArrayLayer;
327 assert(layer_count ==
328 anv_get_layerCount(src_image, &pRegions[r].srcSubresource));
329 }
330
331 VkImageAspectFlags src_mask = pRegions[r].srcSubresource.aspectMask,
332 dst_mask = pRegions[r].dstSubresource.aspectMask;
333
334 assert(anv_image_aspects_compatible(src_mask, dst_mask));
335
336 if (util_bitcount(src_mask) > 1) {
337 uint32_t aspect_bit;
338 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_mask) {
339 struct blorp_surf src_surf, dst_surf;
340 get_blorp_surf_for_anv_image(cmd_buffer->device,
341 src_image, 1UL << aspect_bit,
342 srcImageLayout, ISL_AUX_USAGE_NONE,
343 &src_surf);
344 get_blorp_surf_for_anv_image(cmd_buffer->device,
345 dst_image, 1UL << aspect_bit,
346 dstImageLayout, ISL_AUX_USAGE_NONE,
347 &dst_surf);
348 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
349 1UL << aspect_bit,
350 dst_surf.aux_usage, dst_level,
351 dst_base_layer, layer_count);
352
353 for (unsigned i = 0; i < layer_count; i++) {
354 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
355 &dst_surf, dst_level, dst_base_layer + i,
356 srcOffset.x, srcOffset.y,
357 dstOffset.x, dstOffset.y,
358 extent.width, extent.height);
359 }
360
361 struct blorp_surf dst_shadow_surf;
362 if (get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
363 dst_image,
364 1UL << aspect_bit,
365 &dst_shadow_surf)) {
366 for (unsigned i = 0; i < layer_count; i++) {
367 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
368 &dst_shadow_surf, dst_level, dst_base_layer + i,
369 srcOffset.x, srcOffset.y,
370 dstOffset.x, dstOffset.y,
371 extent.width, extent.height);
372 }
373 }
374 }
375 } else {
376 struct blorp_surf src_surf, dst_surf;
377 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, src_mask,
378 srcImageLayout, ISL_AUX_USAGE_NONE,
379 &src_surf);
380 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, dst_mask,
381 dstImageLayout, ISL_AUX_USAGE_NONE,
382 &dst_surf);
383 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image, dst_mask,
384 dst_surf.aux_usage, dst_level,
385 dst_base_layer, layer_count);
386
387 for (unsigned i = 0; i < layer_count; i++) {
388 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
389 &dst_surf, dst_level, dst_base_layer + i,
390 srcOffset.x, srcOffset.y,
391 dstOffset.x, dstOffset.y,
392 extent.width, extent.height);
393 }
394
395 struct blorp_surf dst_shadow_surf;
396 if (get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
397 dst_image, dst_mask,
398 &dst_shadow_surf)) {
399 for (unsigned i = 0; i < layer_count; i++) {
400 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
401 &dst_shadow_surf, dst_level, dst_base_layer + i,
402 srcOffset.x, srcOffset.y,
403 dstOffset.x, dstOffset.y,
404 extent.width, extent.height);
405 }
406 }
407 }
408 }
409
410 blorp_batch_finish(&batch);
411 }
412
413 static void
414 copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer,
415 struct anv_buffer *anv_buffer,
416 struct anv_image *anv_image,
417 VkImageLayout image_layout,
418 uint32_t regionCount,
419 const VkBufferImageCopy* pRegions,
420 bool buffer_to_image)
421 {
422 struct blorp_batch batch;
423 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
424
425 struct {
426 struct blorp_surf surf;
427 uint32_t level;
428 VkOffset3D offset;
429 } image, buffer, *src, *dst;
430
431 buffer.level = 0;
432 buffer.offset = (VkOffset3D) { 0, 0, 0 };
433
434 if (buffer_to_image) {
435 src = &buffer;
436 dst = &image;
437 } else {
438 src = &image;
439 dst = &buffer;
440 }
441
442 for (unsigned r = 0; r < regionCount; r++) {
443 const VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask;
444
445 get_blorp_surf_for_anv_image(cmd_buffer->device, anv_image, aspect,
446 image_layout, ISL_AUX_USAGE_NONE,
447 &image.surf);
448 image.offset =
449 anv_sanitize_image_offset(anv_image->type, pRegions[r].imageOffset);
450 image.level = pRegions[r].imageSubresource.mipLevel;
451
452 VkExtent3D extent =
453 anv_sanitize_image_extent(anv_image->type, pRegions[r].imageExtent);
454 if (anv_image->type != VK_IMAGE_TYPE_3D) {
455 image.offset.z = pRegions[r].imageSubresource.baseArrayLayer;
456 extent.depth =
457 anv_get_layerCount(anv_image, &pRegions[r].imageSubresource);
458 }
459
460 const enum isl_format buffer_format =
461 anv_get_isl_format(&cmd_buffer->device->info, anv_image->vk_format,
462 aspect, VK_IMAGE_TILING_LINEAR);
463
464 const struct isl_format_layout *buffer_fmtl =
465 isl_format_get_layout(buffer_format);
466
467 const uint32_t buffer_row_length =
468 pRegions[r].bufferRowLength ?
469 pRegions[r].bufferRowLength : extent.width;
470
471 const uint32_t buffer_image_height =
472 pRegions[r].bufferImageHeight ?
473 pRegions[r].bufferImageHeight : extent.height;
474
475 const uint32_t buffer_row_pitch =
476 DIV_ROUND_UP(buffer_row_length, buffer_fmtl->bw) *
477 (buffer_fmtl->bpb / 8);
478
479 const uint32_t buffer_layer_stride =
480 DIV_ROUND_UP(buffer_image_height, buffer_fmtl->bh) *
481 buffer_row_pitch;
482
483 struct isl_surf buffer_isl_surf;
484 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
485 anv_buffer, pRegions[r].bufferOffset,
486 extent.width, extent.height,
487 buffer_row_pitch, buffer_format,
488 &buffer.surf, &buffer_isl_surf);
489
490 bool dst_has_shadow = false;
491 struct blorp_surf dst_shadow_surf;
492 if (&image == dst) {
493 anv_cmd_buffer_mark_image_written(cmd_buffer, anv_image,
494 aspect, dst->surf.aux_usage,
495 dst->level,
496 dst->offset.z, extent.depth);
497
498 dst_has_shadow =
499 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
500 anv_image, aspect,
501 &dst_shadow_surf);
502 }
503
504 for (unsigned z = 0; z < extent.depth; z++) {
505 blorp_copy(&batch, &src->surf, src->level, src->offset.z,
506 &dst->surf, dst->level, dst->offset.z,
507 src->offset.x, src->offset.y, dst->offset.x, dst->offset.y,
508 extent.width, extent.height);
509
510 if (dst_has_shadow) {
511 blorp_copy(&batch, &src->surf, src->level, src->offset.z,
512 &dst_shadow_surf, dst->level, dst->offset.z,
513 src->offset.x, src->offset.y,
514 dst->offset.x, dst->offset.y,
515 extent.width, extent.height);
516 }
517
518 image.offset.z++;
519 buffer.surf.addr.offset += buffer_layer_stride;
520 }
521 }
522
523 blorp_batch_finish(&batch);
524 }
525
526 void anv_CmdCopyBufferToImage(
527 VkCommandBuffer commandBuffer,
528 VkBuffer srcBuffer,
529 VkImage dstImage,
530 VkImageLayout dstImageLayout,
531 uint32_t regionCount,
532 const VkBufferImageCopy* pRegions)
533 {
534 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
535 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
536 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
537
538 copy_buffer_to_image(cmd_buffer, src_buffer, dst_image, dstImageLayout,
539 regionCount, pRegions, true);
540 }
541
542 void anv_CmdCopyImageToBuffer(
543 VkCommandBuffer commandBuffer,
544 VkImage srcImage,
545 VkImageLayout srcImageLayout,
546 VkBuffer dstBuffer,
547 uint32_t regionCount,
548 const VkBufferImageCopy* pRegions)
549 {
550 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
551 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
552 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
553
554 copy_buffer_to_image(cmd_buffer, dst_buffer, src_image, srcImageLayout,
555 regionCount, pRegions, false);
556
557 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
558 }
559
560 static bool
561 flip_coords(unsigned *src0, unsigned *src1, unsigned *dst0, unsigned *dst1)
562 {
563 bool flip = false;
564 if (*src0 > *src1) {
565 unsigned tmp = *src0;
566 *src0 = *src1;
567 *src1 = tmp;
568 flip = !flip;
569 }
570
571 if (*dst0 > *dst1) {
572 unsigned tmp = *dst0;
573 *dst0 = *dst1;
574 *dst1 = tmp;
575 flip = !flip;
576 }
577
578 return flip;
579 }
580
581 void anv_CmdBlitImage(
582 VkCommandBuffer commandBuffer,
583 VkImage srcImage,
584 VkImageLayout srcImageLayout,
585 VkImage dstImage,
586 VkImageLayout dstImageLayout,
587 uint32_t regionCount,
588 const VkImageBlit* pRegions,
589 VkFilter filter)
590
591 {
592 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
593 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
594 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
595
596 struct blorp_surf src, dst;
597
598 enum blorp_filter blorp_filter;
599 switch (filter) {
600 case VK_FILTER_NEAREST:
601 blorp_filter = BLORP_FILTER_NEAREST;
602 break;
603 case VK_FILTER_LINEAR:
604 blorp_filter = BLORP_FILTER_BILINEAR;
605 break;
606 default:
607 unreachable("Invalid filter");
608 }
609
610 struct blorp_batch batch;
611 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
612
613 for (unsigned r = 0; r < regionCount; r++) {
614 const VkImageSubresourceLayers *src_res = &pRegions[r].srcSubresource;
615 const VkImageSubresourceLayers *dst_res = &pRegions[r].dstSubresource;
616
617 assert(anv_image_aspects_compatible(src_res->aspectMask,
618 dst_res->aspectMask));
619
620 uint32_t aspect_bit;
621 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_res->aspectMask) {
622 get_blorp_surf_for_anv_image(cmd_buffer->device,
623 src_image, 1U << aspect_bit,
624 srcImageLayout, ISL_AUX_USAGE_NONE, &src);
625 get_blorp_surf_for_anv_image(cmd_buffer->device,
626 dst_image, 1U << aspect_bit,
627 dstImageLayout, ISL_AUX_USAGE_NONE, &dst);
628
629 struct anv_format_plane src_format =
630 anv_get_format_plane(&cmd_buffer->device->info, src_image->vk_format,
631 1U << aspect_bit, src_image->tiling);
632 struct anv_format_plane dst_format =
633 anv_get_format_plane(&cmd_buffer->device->info, dst_image->vk_format,
634 1U << aspect_bit, dst_image->tiling);
635
636 unsigned dst_start, dst_end;
637 if (dst_image->type == VK_IMAGE_TYPE_3D) {
638 assert(dst_res->baseArrayLayer == 0);
639 dst_start = pRegions[r].dstOffsets[0].z;
640 dst_end = pRegions[r].dstOffsets[1].z;
641 } else {
642 dst_start = dst_res->baseArrayLayer;
643 dst_end = dst_start + anv_get_layerCount(dst_image, dst_res);
644 }
645
646 unsigned src_start, src_end;
647 if (src_image->type == VK_IMAGE_TYPE_3D) {
648 assert(src_res->baseArrayLayer == 0);
649 src_start = pRegions[r].srcOffsets[0].z;
650 src_end = pRegions[r].srcOffsets[1].z;
651 } else {
652 src_start = src_res->baseArrayLayer;
653 src_end = src_start + anv_get_layerCount(src_image, src_res);
654 }
655
656 bool flip_z = flip_coords(&src_start, &src_end, &dst_start, &dst_end);
657 float src_z_step = (float)(src_end + 1 - src_start) /
658 (float)(dst_end + 1 - dst_start);
659
660 if (flip_z) {
661 src_start = src_end;
662 src_z_step *= -1;
663 }
664
665 unsigned src_x0 = pRegions[r].srcOffsets[0].x;
666 unsigned src_x1 = pRegions[r].srcOffsets[1].x;
667 unsigned dst_x0 = pRegions[r].dstOffsets[0].x;
668 unsigned dst_x1 = pRegions[r].dstOffsets[1].x;
669 bool flip_x = flip_coords(&src_x0, &src_x1, &dst_x0, &dst_x1);
670
671 unsigned src_y0 = pRegions[r].srcOffsets[0].y;
672 unsigned src_y1 = pRegions[r].srcOffsets[1].y;
673 unsigned dst_y0 = pRegions[r].dstOffsets[0].y;
674 unsigned dst_y1 = pRegions[r].dstOffsets[1].y;
675 bool flip_y = flip_coords(&src_y0, &src_y1, &dst_y0, &dst_y1);
676
677 const unsigned num_layers = dst_end - dst_start;
678 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
679 1U << aspect_bit,
680 dst.aux_usage,
681 dst_res->mipLevel,
682 dst_start, num_layers);
683
684 for (unsigned i = 0; i < num_layers; i++) {
685 unsigned dst_z = dst_start + i;
686 unsigned src_z = src_start + i * src_z_step;
687
688 blorp_blit(&batch, &src, src_res->mipLevel, src_z,
689 src_format.isl_format, src_format.swizzle,
690 &dst, dst_res->mipLevel, dst_z,
691 dst_format.isl_format, dst_format.swizzle,
692 src_x0, src_y0, src_x1, src_y1,
693 dst_x0, dst_y0, dst_x1, dst_y1,
694 blorp_filter, flip_x, flip_y);
695 }
696 }
697 }
698
699 blorp_batch_finish(&batch);
700 }
701
702 static enum isl_format
703 isl_format_for_size(unsigned size_B)
704 {
705 switch (size_B) {
706 case 4: return ISL_FORMAT_R32_UINT;
707 case 8: return ISL_FORMAT_R32G32_UINT;
708 case 16: return ISL_FORMAT_R32G32B32A32_UINT;
709 default:
710 unreachable("Not a power-of-two format size");
711 }
712 }
713
714 /**
715 * Returns the greatest common divisor of a and b that is a power of two.
716 */
717 static uint64_t
718 gcd_pow2_u64(uint64_t a, uint64_t b)
719 {
720 assert(a > 0 || b > 0);
721
722 unsigned a_log2 = ffsll(a) - 1;
723 unsigned b_log2 = ffsll(b) - 1;
724
725 /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which
726 * case, the MIN2() will take the other one. If both are 0 then we will
727 * hit the assert above.
728 */
729 return 1 << MIN2(a_log2, b_log2);
730 }
731
732 /* This is maximum possible width/height our HW can handle */
733 #define MAX_SURFACE_DIM (1ull << 14)
734
735 void anv_CmdCopyBuffer(
736 VkCommandBuffer commandBuffer,
737 VkBuffer srcBuffer,
738 VkBuffer dstBuffer,
739 uint32_t regionCount,
740 const VkBufferCopy* pRegions)
741 {
742 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
743 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
744 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
745
746 struct blorp_batch batch;
747 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
748
749 for (unsigned r = 0; r < regionCount; r++) {
750 struct blorp_address src = {
751 .buffer = src_buffer->address.bo,
752 .offset = src_buffer->address.offset + pRegions[r].srcOffset,
753 .mocs = anv_mocs_for_bo(cmd_buffer->device, src_buffer->address.bo),
754 };
755 struct blorp_address dst = {
756 .buffer = dst_buffer->address.bo,
757 .offset = dst_buffer->address.offset + pRegions[r].dstOffset,
758 .mocs = anv_mocs_for_bo(cmd_buffer->device, dst_buffer->address.bo),
759 };
760
761 blorp_buffer_copy(&batch, src, dst, pRegions[r].size);
762 }
763
764 blorp_batch_finish(&batch);
765
766 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
767 }
768
769 void anv_CmdUpdateBuffer(
770 VkCommandBuffer commandBuffer,
771 VkBuffer dstBuffer,
772 VkDeviceSize dstOffset,
773 VkDeviceSize dataSize,
774 const void* pData)
775 {
776 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
777 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
778
779 struct blorp_batch batch;
780 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
781
782 /* We can't quite grab a full block because the state stream needs a
783 * little data at the top to build its linked list.
784 */
785 const uint32_t max_update_size =
786 cmd_buffer->device->dynamic_state_pool.block_size - 64;
787
788 assert(max_update_size < MAX_SURFACE_DIM * 4);
789
790 /* We're about to read data that was written from the CPU. Flush the
791 * texture cache so we don't get anything stale.
792 */
793 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
794
795 while (dataSize) {
796 const uint32_t copy_size = MIN2(dataSize, max_update_size);
797
798 struct anv_state tmp_data =
799 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, copy_size, 64);
800
801 memcpy(tmp_data.map, pData, copy_size);
802
803 struct blorp_address src = {
804 .buffer = cmd_buffer->device->dynamic_state_pool.block_pool.bo,
805 .offset = tmp_data.offset,
806 .mocs = cmd_buffer->device->isl_dev.mocs.internal,
807 };
808 struct blorp_address dst = {
809 .buffer = dst_buffer->address.bo,
810 .offset = dst_buffer->address.offset + dstOffset,
811 .mocs = anv_mocs_for_bo(cmd_buffer->device, dst_buffer->address.bo),
812 };
813
814 blorp_buffer_copy(&batch, src, dst, copy_size);
815
816 dataSize -= copy_size;
817 dstOffset += copy_size;
818 pData = (void *)pData + copy_size;
819 }
820
821 blorp_batch_finish(&batch);
822
823 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
824 }
825
826 void anv_CmdFillBuffer(
827 VkCommandBuffer commandBuffer,
828 VkBuffer dstBuffer,
829 VkDeviceSize dstOffset,
830 VkDeviceSize fillSize,
831 uint32_t data)
832 {
833 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
834 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
835 struct blorp_surf surf;
836 struct isl_surf isl_surf;
837
838 struct blorp_batch batch;
839 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
840
841 fillSize = anv_buffer_get_range(dst_buffer, dstOffset, fillSize);
842
843 /* From the Vulkan spec:
844 *
845 * "size is the number of bytes to fill, and must be either a multiple
846 * of 4, or VK_WHOLE_SIZE to fill the range from offset to the end of
847 * the buffer. If VK_WHOLE_SIZE is used and the remaining size of the
848 * buffer is not a multiple of 4, then the nearest smaller multiple is
849 * used."
850 */
851 fillSize &= ~3ull;
852
853 /* First, we compute the biggest format that can be used with the
854 * given offsets and size.
855 */
856 int bs = 16;
857 bs = gcd_pow2_u64(bs, dstOffset);
858 bs = gcd_pow2_u64(bs, fillSize);
859 enum isl_format isl_format = isl_format_for_size(bs);
860
861 union isl_color_value color = {
862 .u32 = { data, data, data, data },
863 };
864
865 const uint64_t max_fill_size = MAX_SURFACE_DIM * MAX_SURFACE_DIM * bs;
866 while (fillSize >= max_fill_size) {
867 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
868 dst_buffer, dstOffset,
869 MAX_SURFACE_DIM, MAX_SURFACE_DIM,
870 MAX_SURFACE_DIM * bs, isl_format,
871 &surf, &isl_surf);
872
873 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
874 0, 0, 1, 0, 0, MAX_SURFACE_DIM, MAX_SURFACE_DIM,
875 color, NULL);
876 fillSize -= max_fill_size;
877 dstOffset += max_fill_size;
878 }
879
880 uint64_t height = fillSize / (MAX_SURFACE_DIM * bs);
881 assert(height < MAX_SURFACE_DIM);
882 if (height != 0) {
883 const uint64_t rect_fill_size = height * MAX_SURFACE_DIM * bs;
884 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
885 dst_buffer, dstOffset,
886 MAX_SURFACE_DIM, height,
887 MAX_SURFACE_DIM * bs, isl_format,
888 &surf, &isl_surf);
889
890 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
891 0, 0, 1, 0, 0, MAX_SURFACE_DIM, height,
892 color, NULL);
893 fillSize -= rect_fill_size;
894 dstOffset += rect_fill_size;
895 }
896
897 if (fillSize != 0) {
898 const uint32_t width = fillSize / bs;
899 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
900 dst_buffer, dstOffset,
901 width, 1,
902 width * bs, isl_format,
903 &surf, &isl_surf);
904
905 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
906 0, 0, 1, 0, 0, width, 1,
907 color, NULL);
908 }
909
910 blorp_batch_finish(&batch);
911
912 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
913 }
914
915 void anv_CmdClearColorImage(
916 VkCommandBuffer commandBuffer,
917 VkImage _image,
918 VkImageLayout imageLayout,
919 const VkClearColorValue* pColor,
920 uint32_t rangeCount,
921 const VkImageSubresourceRange* pRanges)
922 {
923 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
924 ANV_FROM_HANDLE(anv_image, image, _image);
925
926 static const bool color_write_disable[4] = { false, false, false, false };
927
928 struct blorp_batch batch;
929 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
930
931
932 for (unsigned r = 0; r < rangeCount; r++) {
933 if (pRanges[r].aspectMask == 0)
934 continue;
935
936 assert(pRanges[r].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
937
938 struct blorp_surf surf;
939 get_blorp_surf_for_anv_image(cmd_buffer->device,
940 image, pRanges[r].aspectMask,
941 imageLayout, ISL_AUX_USAGE_NONE, &surf);
942
943 struct anv_format_plane src_format =
944 anv_get_format_plane(&cmd_buffer->device->info, image->vk_format,
945 VK_IMAGE_ASPECT_COLOR_BIT, image->tiling);
946
947 unsigned base_layer = pRanges[r].baseArrayLayer;
948 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
949
950 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
951 const unsigned level = pRanges[r].baseMipLevel + i;
952 const unsigned level_width = anv_minify(image->extent.width, level);
953 const unsigned level_height = anv_minify(image->extent.height, level);
954
955 if (image->type == VK_IMAGE_TYPE_3D) {
956 base_layer = 0;
957 layer_count = anv_minify(image->extent.depth, level);
958 }
959
960 anv_cmd_buffer_mark_image_written(cmd_buffer, image,
961 pRanges[r].aspectMask,
962 surf.aux_usage, level,
963 base_layer, layer_count);
964
965 blorp_clear(&batch, &surf,
966 src_format.isl_format, src_format.swizzle,
967 level, base_layer, layer_count,
968 0, 0, level_width, level_height,
969 vk_to_isl_color(*pColor), color_write_disable);
970 }
971 }
972
973 blorp_batch_finish(&batch);
974 }
975
976 void anv_CmdClearDepthStencilImage(
977 VkCommandBuffer commandBuffer,
978 VkImage image_h,
979 VkImageLayout imageLayout,
980 const VkClearDepthStencilValue* pDepthStencil,
981 uint32_t rangeCount,
982 const VkImageSubresourceRange* pRanges)
983 {
984 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
985 ANV_FROM_HANDLE(anv_image, image, image_h);
986
987 struct blorp_batch batch;
988 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
989
990 struct blorp_surf depth, stencil, stencil_shadow;
991 if (image->aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
992 get_blorp_surf_for_anv_image(cmd_buffer->device,
993 image, VK_IMAGE_ASPECT_DEPTH_BIT,
994 imageLayout, ISL_AUX_USAGE_NONE, &depth);
995 } else {
996 memset(&depth, 0, sizeof(depth));
997 }
998
999 bool has_stencil_shadow = false;
1000 if (image->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1001 get_blorp_surf_for_anv_image(cmd_buffer->device,
1002 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1003 imageLayout, ISL_AUX_USAGE_NONE, &stencil);
1004
1005 has_stencil_shadow =
1006 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device, image,
1007 VK_IMAGE_ASPECT_STENCIL_BIT,
1008 &stencil_shadow);
1009 } else {
1010 memset(&stencil, 0, sizeof(stencil));
1011 }
1012
1013 for (unsigned r = 0; r < rangeCount; r++) {
1014 if (pRanges[r].aspectMask == 0)
1015 continue;
1016
1017 bool clear_depth = pRanges[r].aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
1018 bool clear_stencil = pRanges[r].aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
1019
1020 unsigned base_layer = pRanges[r].baseArrayLayer;
1021 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
1022
1023 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
1024 const unsigned level = pRanges[r].baseMipLevel + i;
1025 const unsigned level_width = anv_minify(image->extent.width, level);
1026 const unsigned level_height = anv_minify(image->extent.height, level);
1027
1028 if (image->type == VK_IMAGE_TYPE_3D)
1029 layer_count = anv_minify(image->extent.depth, level);
1030
1031 blorp_clear_depth_stencil(&batch, &depth, &stencil,
1032 level, base_layer, layer_count,
1033 0, 0, level_width, level_height,
1034 clear_depth, pDepthStencil->depth,
1035 clear_stencil ? 0xff : 0,
1036 pDepthStencil->stencil);
1037
1038 if (clear_stencil && has_stencil_shadow) {
1039 union isl_color_value stencil_color = {
1040 .u32 = { pDepthStencil->stencil, },
1041 };
1042 blorp_clear(&batch, &stencil_shadow,
1043 ISL_FORMAT_R8_UINT, ISL_SWIZZLE_IDENTITY,
1044 level, base_layer, layer_count,
1045 0, 0, level_width, level_height,
1046 stencil_color, NULL);
1047 }
1048 }
1049 }
1050
1051 blorp_batch_finish(&batch);
1052 }
1053
1054 VkResult
1055 anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer *cmd_buffer,
1056 uint32_t num_entries,
1057 uint32_t *state_offset,
1058 struct anv_state *bt_state)
1059 {
1060 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
1061 state_offset);
1062 if (bt_state->map == NULL) {
1063 /* We ran out of space. Grab a new binding table block. */
1064 VkResult result = anv_cmd_buffer_new_binding_table_block(cmd_buffer);
1065 if (result != VK_SUCCESS)
1066 return result;
1067
1068 /* Re-emit state base addresses so we get the new surface state base
1069 * address before we start emitting binding tables etc.
1070 */
1071 anv_cmd_buffer_emit_state_base_address(cmd_buffer);
1072
1073 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
1074 state_offset);
1075 assert(bt_state->map != NULL);
1076 }
1077
1078 return VK_SUCCESS;
1079 }
1080
1081 static VkResult
1082 binding_table_for_surface_state(struct anv_cmd_buffer *cmd_buffer,
1083 struct anv_state surface_state,
1084 uint32_t *bt_offset)
1085 {
1086 uint32_t state_offset;
1087 struct anv_state bt_state;
1088
1089 VkResult result =
1090 anv_cmd_buffer_alloc_blorp_binding_table(cmd_buffer, 1, &state_offset,
1091 &bt_state);
1092 if (result != VK_SUCCESS)
1093 return result;
1094
1095 uint32_t *bt_map = bt_state.map;
1096 bt_map[0] = surface_state.offset + state_offset;
1097
1098 *bt_offset = bt_state.offset;
1099 return VK_SUCCESS;
1100 }
1101
1102 static void
1103 clear_color_attachment(struct anv_cmd_buffer *cmd_buffer,
1104 struct blorp_batch *batch,
1105 const VkClearAttachment *attachment,
1106 uint32_t rectCount, const VkClearRect *pRects)
1107 {
1108 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
1109 const uint32_t color_att = attachment->colorAttachment;
1110 assert(color_att < subpass->color_count);
1111 const uint32_t att_idx = subpass->color_attachments[color_att].attachment;
1112
1113 if (att_idx == VK_ATTACHMENT_UNUSED)
1114 return;
1115
1116 struct anv_render_pass_attachment *pass_att =
1117 &cmd_buffer->state.pass->attachments[att_idx];
1118 struct anv_attachment_state *att_state =
1119 &cmd_buffer->state.attachments[att_idx];
1120
1121 uint32_t binding_table;
1122 VkResult result =
1123 binding_table_for_surface_state(cmd_buffer, att_state->color.state,
1124 &binding_table);
1125 if (result != VK_SUCCESS)
1126 return;
1127
1128 union isl_color_value clear_color =
1129 vk_to_isl_color(attachment->clearValue.color);
1130
1131 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1132 if (subpass->view_mask) {
1133 uint32_t view_idx;
1134 for_each_bit(view_idx, subpass->view_mask) {
1135 for (uint32_t r = 0; r < rectCount; ++r) {
1136 const VkOffset2D offset = pRects[r].rect.offset;
1137 const VkExtent2D extent = pRects[r].rect.extent;
1138 blorp_clear_attachments(batch, binding_table,
1139 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1140 view_idx, 1,
1141 offset.x, offset.y,
1142 offset.x + extent.width,
1143 offset.y + extent.height,
1144 true, clear_color, false, 0.0f, 0, 0);
1145 }
1146 }
1147 return;
1148 }
1149
1150 for (uint32_t r = 0; r < rectCount; ++r) {
1151 const VkOffset2D offset = pRects[r].rect.offset;
1152 const VkExtent2D extent = pRects[r].rect.extent;
1153 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1154 blorp_clear_attachments(batch, binding_table,
1155 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1156 pRects[r].baseArrayLayer,
1157 pRects[r].layerCount,
1158 offset.x, offset.y,
1159 offset.x + extent.width, offset.y + extent.height,
1160 true, clear_color, false, 0.0f, 0, 0);
1161 }
1162 }
1163
1164 static void
1165 clear_depth_stencil_attachment(struct anv_cmd_buffer *cmd_buffer,
1166 struct blorp_batch *batch,
1167 const VkClearAttachment *attachment,
1168 uint32_t rectCount, const VkClearRect *pRects)
1169 {
1170 static const union isl_color_value color_value = { .u32 = { 0, } };
1171 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
1172 if (!subpass->depth_stencil_attachment)
1173 return;
1174
1175 const uint32_t att_idx = subpass->depth_stencil_attachment->attachment;
1176 assert(att_idx != VK_ATTACHMENT_UNUSED);
1177 struct anv_render_pass_attachment *pass_att =
1178 &cmd_buffer->state.pass->attachments[att_idx];
1179
1180 bool clear_depth = attachment->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
1181 bool clear_stencil = attachment->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
1182
1183 enum isl_format depth_format = ISL_FORMAT_UNSUPPORTED;
1184 if (clear_depth) {
1185 depth_format = anv_get_isl_format(&cmd_buffer->device->info,
1186 pass_att->format,
1187 VK_IMAGE_ASPECT_DEPTH_BIT,
1188 VK_IMAGE_TILING_OPTIMAL);
1189 }
1190
1191 uint32_t binding_table;
1192 VkResult result =
1193 binding_table_for_surface_state(cmd_buffer,
1194 cmd_buffer->state.null_surface_state,
1195 &binding_table);
1196 if (result != VK_SUCCESS)
1197 return;
1198
1199 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1200 if (subpass->view_mask) {
1201 uint32_t view_idx;
1202 for_each_bit(view_idx, subpass->view_mask) {
1203 for (uint32_t r = 0; r < rectCount; ++r) {
1204 const VkOffset2D offset = pRects[r].rect.offset;
1205 const VkExtent2D extent = pRects[r].rect.extent;
1206 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1207 blorp_clear_attachments(batch, binding_table,
1208 depth_format, pass_att->samples,
1209 view_idx, 1,
1210 offset.x, offset.y,
1211 offset.x + extent.width,
1212 offset.y + extent.height,
1213 false, color_value,
1214 clear_depth, value.depth,
1215 clear_stencil ? 0xff : 0, value.stencil);
1216 }
1217 }
1218 return;
1219 }
1220
1221 for (uint32_t r = 0; r < rectCount; ++r) {
1222 const VkOffset2D offset = pRects[r].rect.offset;
1223 const VkExtent2D extent = pRects[r].rect.extent;
1224 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1225 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1226 blorp_clear_attachments(batch, binding_table,
1227 depth_format, pass_att->samples,
1228 pRects[r].baseArrayLayer,
1229 pRects[r].layerCount,
1230 offset.x, offset.y,
1231 offset.x + extent.width, offset.y + extent.height,
1232 false, color_value,
1233 clear_depth, value.depth,
1234 clear_stencil ? 0xff : 0, value.stencil);
1235 }
1236 }
1237
1238 void anv_CmdClearAttachments(
1239 VkCommandBuffer commandBuffer,
1240 uint32_t attachmentCount,
1241 const VkClearAttachment* pAttachments,
1242 uint32_t rectCount,
1243 const VkClearRect* pRects)
1244 {
1245 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1246
1247 /* Because this gets called within a render pass, we tell blorp not to
1248 * trash our depth and stencil buffers.
1249 */
1250 struct blorp_batch batch;
1251 enum blorp_batch_flags flags = BLORP_BATCH_NO_EMIT_DEPTH_STENCIL;
1252 if (cmd_buffer->state.conditional_render_enabled) {
1253 anv_cmd_emit_conditional_render_predicate(cmd_buffer);
1254 flags |= BLORP_BATCH_PREDICATE_ENABLE;
1255 }
1256 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, flags);
1257
1258 for (uint32_t a = 0; a < attachmentCount; ++a) {
1259 if (pAttachments[a].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
1260 assert(pAttachments[a].aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
1261 clear_color_attachment(cmd_buffer, &batch,
1262 &pAttachments[a],
1263 rectCount, pRects);
1264 } else {
1265 clear_depth_stencil_attachment(cmd_buffer, &batch,
1266 &pAttachments[a],
1267 rectCount, pRects);
1268 }
1269 }
1270
1271 blorp_batch_finish(&batch);
1272 }
1273
1274 enum subpass_stage {
1275 SUBPASS_STAGE_LOAD,
1276 SUBPASS_STAGE_DRAW,
1277 SUBPASS_STAGE_RESOLVE,
1278 };
1279
1280 void
1281 anv_image_msaa_resolve(struct anv_cmd_buffer *cmd_buffer,
1282 const struct anv_image *src_image,
1283 enum isl_aux_usage src_aux_usage,
1284 uint32_t src_level, uint32_t src_base_layer,
1285 const struct anv_image *dst_image,
1286 enum isl_aux_usage dst_aux_usage,
1287 uint32_t dst_level, uint32_t dst_base_layer,
1288 VkImageAspectFlagBits aspect,
1289 uint32_t src_x, uint32_t src_y,
1290 uint32_t dst_x, uint32_t dst_y,
1291 uint32_t width, uint32_t height,
1292 uint32_t layer_count,
1293 enum blorp_filter filter)
1294 {
1295 struct blorp_batch batch;
1296 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1297
1298 assert(src_image->type == VK_IMAGE_TYPE_2D);
1299 assert(src_image->samples > 1);
1300 assert(dst_image->type == VK_IMAGE_TYPE_2D);
1301 assert(dst_image->samples == 1);
1302 assert(src_image->n_planes == dst_image->n_planes);
1303 assert(!src_image->format->can_ycbcr);
1304 assert(!dst_image->format->can_ycbcr);
1305
1306 struct blorp_surf src_surf, dst_surf;
1307 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, aspect,
1308 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1309 src_aux_usage, &src_surf);
1310 if (src_aux_usage == ISL_AUX_USAGE_MCS) {
1311 src_surf.clear_color_addr = anv_to_blorp_address(
1312 anv_image_get_clear_color_addr(cmd_buffer->device, src_image,
1313 VK_IMAGE_ASPECT_COLOR_BIT));
1314 }
1315 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, aspect,
1316 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1317 dst_aux_usage, &dst_surf);
1318 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
1319 aspect, dst_aux_usage,
1320 dst_level, dst_base_layer, layer_count);
1321
1322 if (filter == BLORP_FILTER_NONE) {
1323 /* If no explicit filter is provided, then it's implied by the type of
1324 * the source image.
1325 */
1326 if ((src_surf.surf->usage & ISL_SURF_USAGE_DEPTH_BIT) ||
1327 (src_surf.surf->usage & ISL_SURF_USAGE_STENCIL_BIT) ||
1328 isl_format_has_int_channel(src_surf.surf->format)) {
1329 filter = BLORP_FILTER_SAMPLE_0;
1330 } else {
1331 filter = BLORP_FILTER_AVERAGE;
1332 }
1333 }
1334
1335 for (uint32_t l = 0; l < layer_count; l++) {
1336 blorp_blit(&batch,
1337 &src_surf, src_level, src_base_layer + l,
1338 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1339 &dst_surf, dst_level, dst_base_layer + l,
1340 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1341 src_x, src_y, src_x + width, src_y + height,
1342 dst_x, dst_y, dst_x + width, dst_y + height,
1343 filter, false, false);
1344 }
1345
1346 blorp_batch_finish(&batch);
1347 }
1348
1349 void anv_CmdResolveImage(
1350 VkCommandBuffer commandBuffer,
1351 VkImage srcImage,
1352 VkImageLayout srcImageLayout,
1353 VkImage dstImage,
1354 VkImageLayout dstImageLayout,
1355 uint32_t regionCount,
1356 const VkImageResolve* pRegions)
1357 {
1358 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1359 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
1360 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
1361
1362 assert(!src_image->format->can_ycbcr);
1363
1364 for (uint32_t r = 0; r < regionCount; r++) {
1365 assert(pRegions[r].srcSubresource.aspectMask ==
1366 pRegions[r].dstSubresource.aspectMask);
1367 assert(anv_get_layerCount(src_image, &pRegions[r].srcSubresource) ==
1368 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource));
1369
1370 const uint32_t layer_count =
1371 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource);
1372
1373 uint32_t aspect_bit;
1374 anv_foreach_image_aspect_bit(aspect_bit, src_image,
1375 pRegions[r].srcSubresource.aspectMask) {
1376 enum isl_aux_usage src_aux_usage =
1377 anv_layout_to_aux_usage(&cmd_buffer->device->info, src_image,
1378 (1 << aspect_bit), srcImageLayout);
1379 enum isl_aux_usage dst_aux_usage =
1380 anv_layout_to_aux_usage(&cmd_buffer->device->info, dst_image,
1381 (1 << aspect_bit), dstImageLayout);
1382
1383 anv_image_msaa_resolve(cmd_buffer,
1384 src_image, src_aux_usage,
1385 pRegions[r].srcSubresource.mipLevel,
1386 pRegions[r].srcSubresource.baseArrayLayer,
1387 dst_image, dst_aux_usage,
1388 pRegions[r].dstSubresource.mipLevel,
1389 pRegions[r].dstSubresource.baseArrayLayer,
1390 (1 << aspect_bit),
1391 pRegions[r].srcOffset.x,
1392 pRegions[r].srcOffset.y,
1393 pRegions[r].dstOffset.x,
1394 pRegions[r].dstOffset.y,
1395 pRegions[r].extent.width,
1396 pRegions[r].extent.height,
1397 layer_count, BLORP_FILTER_NONE);
1398 }
1399 }
1400 }
1401
1402 static enum isl_aux_usage
1403 fast_clear_aux_usage(const struct anv_image *image,
1404 VkImageAspectFlagBits aspect)
1405 {
1406 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
1407 if (image->planes[plane].aux_usage == ISL_AUX_USAGE_NONE)
1408 return ISL_AUX_USAGE_CCS_D;
1409 else
1410 return image->planes[plane].aux_usage;
1411 }
1412
1413 void
1414 anv_image_copy_to_shadow(struct anv_cmd_buffer *cmd_buffer,
1415 const struct anv_image *image,
1416 VkImageAspectFlagBits aspect,
1417 uint32_t base_level, uint32_t level_count,
1418 uint32_t base_layer, uint32_t layer_count)
1419 {
1420 struct blorp_batch batch;
1421 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1422
1423 /* We don't know who touched the main surface last so flush a bunch of
1424 * caches to ensure we get good data.
1425 */
1426 cmd_buffer->state.pending_pipe_bits |=
1427 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT |
1428 ANV_PIPE_DATA_CACHE_FLUSH_BIT |
1429 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT |
1430 ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
1431
1432 struct blorp_surf surf;
1433 get_blorp_surf_for_anv_image(cmd_buffer->device,
1434 image, aspect,
1435 VK_IMAGE_LAYOUT_GENERAL,
1436 ISL_AUX_USAGE_NONE, &surf);
1437 assert(surf.aux_usage == ISL_AUX_USAGE_NONE);
1438
1439 struct blorp_surf shadow_surf;
1440 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
1441 image, aspect, &shadow_surf);
1442
1443 for (uint32_t l = 0; l < level_count; l++) {
1444 const uint32_t level = base_level + l;
1445
1446 const VkExtent3D extent = {
1447 .width = anv_minify(image->extent.width, level),
1448 .height = anv_minify(image->extent.height, level),
1449 .depth = anv_minify(image->extent.depth, level),
1450 };
1451
1452 if (image->type == VK_IMAGE_TYPE_3D)
1453 layer_count = extent.depth;
1454
1455 for (uint32_t a = 0; a < layer_count; a++) {
1456 const uint32_t layer = base_layer + a;
1457
1458 blorp_copy(&batch, &surf, level, layer,
1459 &shadow_surf, level, layer,
1460 0, 0, 0, 0, extent.width, extent.height);
1461 }
1462 }
1463
1464 /* We just wrote to the buffer with the render cache. Flush it. */
1465 cmd_buffer->state.pending_pipe_bits |=
1466 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
1467
1468 blorp_batch_finish(&batch);
1469 }
1470
1471 void
1472 anv_image_clear_color(struct anv_cmd_buffer *cmd_buffer,
1473 const struct anv_image *image,
1474 VkImageAspectFlagBits aspect,
1475 enum isl_aux_usage aux_usage,
1476 enum isl_format format, struct isl_swizzle swizzle,
1477 uint32_t level, uint32_t base_layer, uint32_t layer_count,
1478 VkRect2D area, union isl_color_value clear_color)
1479 {
1480 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1481
1482 /* We don't support planar images with multisampling yet */
1483 assert(image->n_planes == 1);
1484
1485 struct blorp_batch batch;
1486 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1487
1488 struct blorp_surf surf;
1489 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1490 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1491 aux_usage, &surf);
1492 anv_cmd_buffer_mark_image_written(cmd_buffer, image, aspect, aux_usage,
1493 level, base_layer, layer_count);
1494
1495 blorp_clear(&batch, &surf, format, anv_swizzle_for_render(swizzle),
1496 level, base_layer, layer_count,
1497 area.offset.x, area.offset.y,
1498 area.offset.x + area.extent.width,
1499 area.offset.y + area.extent.height,
1500 clear_color, NULL);
1501
1502 blorp_batch_finish(&batch);
1503 }
1504
1505 void
1506 anv_image_clear_depth_stencil(struct anv_cmd_buffer *cmd_buffer,
1507 const struct anv_image *image,
1508 VkImageAspectFlags aspects,
1509 enum isl_aux_usage depth_aux_usage,
1510 uint32_t level,
1511 uint32_t base_layer, uint32_t layer_count,
1512 VkRect2D area,
1513 float depth_value, uint8_t stencil_value)
1514 {
1515 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
1516 VK_IMAGE_ASPECT_STENCIL_BIT));
1517
1518 struct blorp_batch batch;
1519 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1520
1521 struct blorp_surf depth = {};
1522 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1523 get_blorp_surf_for_anv_image(cmd_buffer->device,
1524 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1525 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1526 depth_aux_usage, &depth);
1527 depth.clear_color.f32[0] = ANV_HZ_FC_VAL;
1528 }
1529
1530 struct blorp_surf stencil = {};
1531 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1532 get_blorp_surf_for_anv_image(cmd_buffer->device,
1533 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1534 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1535 ISL_AUX_USAGE_NONE, &stencil);
1536 }
1537
1538 /* Blorp may choose to clear stencil using RGBA32_UINT for better
1539 * performance. If it does this, we need to flush it out of the depth
1540 * cache before rendering to it.
1541 */
1542 cmd_buffer->state.pending_pipe_bits |=
1543 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1544
1545 blorp_clear_depth_stencil(&batch, &depth, &stencil,
1546 level, base_layer, layer_count,
1547 area.offset.x, area.offset.y,
1548 area.offset.x + area.extent.width,
1549 area.offset.y + area.extent.height,
1550 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
1551 depth_value,
1552 (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) ? 0xff : 0,
1553 stencil_value);
1554
1555 /* Blorp may choose to clear stencil using RGBA32_UINT for better
1556 * performance. If it does this, we need to flush it out of the render
1557 * cache before someone starts trying to do stencil on it.
1558 */
1559 cmd_buffer->state.pending_pipe_bits |=
1560 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1561
1562 struct blorp_surf stencil_shadow;
1563 if ((aspects & VK_IMAGE_ASPECT_STENCIL_BIT) &&
1564 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device, image,
1565 VK_IMAGE_ASPECT_STENCIL_BIT,
1566 &stencil_shadow)) {
1567 union isl_color_value stencil_color = {
1568 .u32 = { stencil_value },
1569 };
1570 blorp_clear(&batch, &stencil_shadow,
1571 ISL_FORMAT_R8_UINT, ISL_SWIZZLE_IDENTITY,
1572 level, base_layer, layer_count,
1573 area.offset.x, area.offset.y,
1574 area.offset.x + area.extent.width,
1575 area.offset.y + area.extent.height,
1576 stencil_color, NULL);
1577 }
1578
1579 blorp_batch_finish(&batch);
1580 }
1581
1582 void
1583 anv_image_hiz_op(struct anv_cmd_buffer *cmd_buffer,
1584 const struct anv_image *image,
1585 VkImageAspectFlagBits aspect, uint32_t level,
1586 uint32_t base_layer, uint32_t layer_count,
1587 enum isl_aux_op hiz_op)
1588 {
1589 assert(aspect == VK_IMAGE_ASPECT_DEPTH_BIT);
1590 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, level));
1591 assert(anv_image_aspect_to_plane(image->aspects,
1592 VK_IMAGE_ASPECT_DEPTH_BIT) == 0);
1593
1594 struct blorp_batch batch;
1595 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1596
1597 struct blorp_surf surf;
1598 get_blorp_surf_for_anv_image(cmd_buffer->device,
1599 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1600 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1601 ISL_AUX_USAGE_HIZ, &surf);
1602 surf.clear_color.f32[0] = ANV_HZ_FC_VAL;
1603
1604 blorp_hiz_op(&batch, &surf, level, base_layer, layer_count, hiz_op);
1605
1606 blorp_batch_finish(&batch);
1607 }
1608
1609 void
1610 anv_image_hiz_clear(struct anv_cmd_buffer *cmd_buffer,
1611 const struct anv_image *image,
1612 VkImageAspectFlags aspects,
1613 uint32_t level,
1614 uint32_t base_layer, uint32_t layer_count,
1615 VkRect2D area, uint8_t stencil_value)
1616 {
1617 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
1618 VK_IMAGE_ASPECT_STENCIL_BIT));
1619
1620 struct blorp_batch batch;
1621 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1622
1623 struct blorp_surf depth = {};
1624 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1625 assert(base_layer + layer_count <=
1626 anv_image_aux_layers(image, VK_IMAGE_ASPECT_DEPTH_BIT, level));
1627 get_blorp_surf_for_anv_image(cmd_buffer->device,
1628 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1629 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1630 ISL_AUX_USAGE_HIZ, &depth);
1631 depth.clear_color.f32[0] = ANV_HZ_FC_VAL;
1632 }
1633
1634 struct blorp_surf stencil = {};
1635 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1636 get_blorp_surf_for_anv_image(cmd_buffer->device,
1637 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1638 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1639 ISL_AUX_USAGE_NONE, &stencil);
1640 }
1641
1642 /* From the Sky Lake PRM Volume 7, "Depth Buffer Clear":
1643 *
1644 * "The following is required when performing a depth buffer clear with
1645 * using the WM_STATE or 3DSTATE_WM:
1646 *
1647 * * If other rendering operations have preceded this clear, a
1648 * PIPE_CONTROL with depth cache flush enabled, Depth Stall bit
1649 * enabled must be issued before the rectangle primitive used for
1650 * the depth buffer clear operation.
1651 * * [...]"
1652 *
1653 * Even though the PRM only says that this is required if using 3DSTATE_WM
1654 * and a 3DPRIMITIVE, the GPU appears to also need this to avoid occasional
1655 * hangs when doing a clear with WM_HZ_OP.
1656 */
1657 cmd_buffer->state.pending_pipe_bits |=
1658 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT;
1659
1660 blorp_hiz_clear_depth_stencil(&batch, &depth, &stencil,
1661 level, base_layer, layer_count,
1662 area.offset.x, area.offset.y,
1663 area.offset.x + area.extent.width,
1664 area.offset.y + area.extent.height,
1665 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
1666 ANV_HZ_FC_VAL,
1667 aspects & VK_IMAGE_ASPECT_STENCIL_BIT,
1668 stencil_value);
1669
1670 blorp_batch_finish(&batch);
1671
1672 /* From the SKL PRM, Depth Buffer Clear:
1673 *
1674 * "Depth Buffer Clear Workaround
1675 *
1676 * Depth buffer clear pass using any of the methods (WM_STATE,
1677 * 3DSTATE_WM or 3DSTATE_WM_HZ_OP) must be followed by a PIPE_CONTROL
1678 * command with DEPTH_STALL bit and Depth FLUSH bits “set” before
1679 * starting to render. DepthStall and DepthFlush are not needed between
1680 * consecutive depth clear passes nor is it required if the depth-clear
1681 * pass was done with “full_surf_clear” bit set in the
1682 * 3DSTATE_WM_HZ_OP."
1683 *
1684 * Even though the PRM provides a bunch of conditions under which this is
1685 * supposedly unnecessary, we choose to perform the flush unconditionally
1686 * just to be safe.
1687 */
1688 cmd_buffer->state.pending_pipe_bits |=
1689 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT;
1690 }
1691
1692 void
1693 anv_image_mcs_op(struct anv_cmd_buffer *cmd_buffer,
1694 const struct anv_image *image,
1695 enum isl_format format,
1696 VkImageAspectFlagBits aspect,
1697 uint32_t base_layer, uint32_t layer_count,
1698 enum isl_aux_op mcs_op, union isl_color_value *clear_value,
1699 bool predicate)
1700 {
1701 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1702 assert(image->samples > 1);
1703 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, 0));
1704
1705 /* Multisampling with multi-planar formats is not supported */
1706 assert(image->n_planes == 1);
1707
1708 struct blorp_batch batch;
1709 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1710 BLORP_BATCH_PREDICATE_ENABLE * predicate +
1711 BLORP_BATCH_NO_UPDATE_CLEAR_COLOR * !clear_value);
1712
1713 struct blorp_surf surf;
1714 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1715 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1716 ISL_AUX_USAGE_MCS, &surf);
1717
1718 /* Blorp will store the clear color for us if we provide the clear color
1719 * address and we are doing a fast clear. So we save the clear value into
1720 * the blorp surface.
1721 */
1722 if (clear_value)
1723 surf.clear_color = *clear_value;
1724
1725 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1726 *
1727 * "After Render target fast clear, pipe-control with color cache
1728 * write-flush must be issued before sending any DRAW commands on
1729 * that render target."
1730 *
1731 * This comment is a bit cryptic and doesn't really tell you what's going
1732 * or what's really needed. It appears that fast clear ops are not
1733 * properly synchronized with other drawing. This means that we cannot
1734 * have a fast clear operation in the pipe at the same time as other
1735 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
1736 * that the contents of the previous draw hit the render target before we
1737 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
1738 * that it is completed before any additional drawing occurs.
1739 */
1740 cmd_buffer->state.pending_pipe_bits |=
1741 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1742
1743 switch (mcs_op) {
1744 case ISL_AUX_OP_FAST_CLEAR:
1745 blorp_fast_clear(&batch, &surf, format,
1746 0, base_layer, layer_count,
1747 0, 0, image->extent.width, image->extent.height);
1748 break;
1749 case ISL_AUX_OP_PARTIAL_RESOLVE:
1750 blorp_mcs_partial_resolve(&batch, &surf, format,
1751 base_layer, layer_count);
1752 break;
1753 case ISL_AUX_OP_FULL_RESOLVE:
1754 case ISL_AUX_OP_AMBIGUATE:
1755 default:
1756 unreachable("Unsupported MCS operation");
1757 }
1758
1759 cmd_buffer->state.pending_pipe_bits |=
1760 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1761
1762 blorp_batch_finish(&batch);
1763 }
1764
1765 void
1766 anv_image_ccs_op(struct anv_cmd_buffer *cmd_buffer,
1767 const struct anv_image *image,
1768 enum isl_format format,
1769 VkImageAspectFlagBits aspect, uint32_t level,
1770 uint32_t base_layer, uint32_t layer_count,
1771 enum isl_aux_op ccs_op, union isl_color_value *clear_value,
1772 bool predicate)
1773 {
1774 assert(image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
1775 assert(image->samples == 1);
1776 assert(level < anv_image_aux_levels(image, aspect));
1777 /* Multi-LOD YcBcR is not allowed */
1778 assert(image->n_planes == 1 || level == 0);
1779 assert(base_layer + layer_count <=
1780 anv_image_aux_layers(image, aspect, level));
1781
1782 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
1783 uint32_t width_div = image->format->planes[plane].denominator_scales[0];
1784 uint32_t height_div = image->format->planes[plane].denominator_scales[1];
1785 uint32_t level_width = anv_minify(image->extent.width, level) / width_div;
1786 uint32_t level_height = anv_minify(image->extent.height, level) / height_div;
1787
1788 struct blorp_batch batch;
1789 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1790 BLORP_BATCH_PREDICATE_ENABLE * predicate +
1791 BLORP_BATCH_NO_UPDATE_CLEAR_COLOR * !clear_value);
1792
1793 struct blorp_surf surf;
1794 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1795 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1796 fast_clear_aux_usage(image, aspect),
1797 &surf);
1798
1799 /* Blorp will store the clear color for us if we provide the clear color
1800 * address and we are doing a fast clear. So we save the clear value into
1801 * the blorp surface.
1802 */
1803 if (clear_value)
1804 surf.clear_color = *clear_value;
1805
1806 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1807 *
1808 * "After Render target fast clear, pipe-control with color cache
1809 * write-flush must be issued before sending any DRAW commands on
1810 * that render target."
1811 *
1812 * This comment is a bit cryptic and doesn't really tell you what's going
1813 * or what's really needed. It appears that fast clear ops are not
1814 * properly synchronized with other drawing. This means that we cannot
1815 * have a fast clear operation in the pipe at the same time as other
1816 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
1817 * that the contents of the previous draw hit the render target before we
1818 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
1819 * that it is completed before any additional drawing occurs.
1820 */
1821 cmd_buffer->state.pending_pipe_bits |=
1822 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1823
1824 switch (ccs_op) {
1825 case ISL_AUX_OP_FAST_CLEAR:
1826 blorp_fast_clear(&batch, &surf, format,
1827 level, base_layer, layer_count,
1828 0, 0, level_width, level_height);
1829 break;
1830 case ISL_AUX_OP_FULL_RESOLVE:
1831 case ISL_AUX_OP_PARTIAL_RESOLVE:
1832 blorp_ccs_resolve(&batch, &surf, level, base_layer, layer_count,
1833 format, ccs_op);
1834 break;
1835 case ISL_AUX_OP_AMBIGUATE:
1836 for (uint32_t a = 0; a < layer_count; a++) {
1837 const uint32_t layer = base_layer + a;
1838 blorp_ccs_ambiguate(&batch, &surf, level, layer);
1839 }
1840 break;
1841 default:
1842 unreachable("Unsupported CCS operation");
1843 }
1844
1845 cmd_buffer->state.pending_pipe_bits |=
1846 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1847
1848 blorp_batch_finish(&batch);
1849 }