projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
anv: Rename has_resolve to has_color_resolve
[mesa.git] / src / intel / vulkan / anv_blorp.c
1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "anv_private.h"
25
26 static bool
27 lookup_blorp_shader(struct blorp_batch *batch,
28 const void *key, uint32_t key_size,
29 uint32_t *kernel_out, void *prog_data_out)
30 {
31 struct blorp_context *blorp = batch->blorp;
32 struct anv_device *device = blorp->driver_ctx;
33
34 /* The default cache must be a real cache */
35 assert(device->default_pipeline_cache.cache);
36
37 struct anv_shader_bin *bin =
38 anv_pipeline_cache_search(&device->default_pipeline_cache, key, key_size);
39 if (!bin)
40 return false;
41
42 /* The cache already has a reference and it's not going anywhere so there
43 * is no need to hold a second reference.
44 */
45 anv_shader_bin_unref(device, bin);
46
47 *kernel_out = bin->kernel.offset;
48 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
49
50 return true;
51 }
52
53 static bool
54 upload_blorp_shader(struct blorp_batch *batch,
55 const void *key, uint32_t key_size,
56 const void *kernel, uint32_t kernel_size,
57 const struct brw_stage_prog_data *prog_data,
58 uint32_t prog_data_size,
59 uint32_t *kernel_out, void *prog_data_out)
60 {
61 struct blorp_context *blorp = batch->blorp;
62 struct anv_device *device = blorp->driver_ctx;
63
64 /* The blorp cache must be a real cache */
65 assert(device->default_pipeline_cache.cache);
66
67 struct anv_pipeline_bind_map bind_map = {
68 .surface_count = 0,
69 .sampler_count = 0,
70 };
71
72 struct anv_shader_bin *bin =
73 anv_pipeline_cache_upload_kernel(&device->default_pipeline_cache,
74 key, key_size, kernel, kernel_size,
75 NULL, 0,
76 prog_data, prog_data_size, &bind_map);
77
78 if (!bin)
79 return false;
80
81 /* The cache already has a reference and it's not going anywhere so there
82 * is no need to hold a second reference.
83 */
84 anv_shader_bin_unref(device, bin);
85
86 *kernel_out = bin->kernel.offset;
87 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
88
89 return true;
90 }
91
92 void
93 anv_device_init_blorp(struct anv_device *device)
94 {
95 blorp_init(&device->blorp, device, &device->isl_dev);
96 device->blorp.compiler = device->instance->physicalDevice.compiler;
97 device->blorp.lookup_shader = lookup_blorp_shader;
98 device->blorp.upload_shader = upload_blorp_shader;
99 switch (device->info.gen) {
100 case 7:
101 if (device->info.is_haswell) {
102 device->blorp.exec = gen75_blorp_exec;
103 } else {
104 device->blorp.exec = gen7_blorp_exec;
105 }
106 break;
107 case 8:
108 device->blorp.exec = gen8_blorp_exec;
109 break;
110 case 9:
111 device->blorp.exec = gen9_blorp_exec;
112 break;
113 case 10:
114 device->blorp.exec = gen10_blorp_exec;
115 break;
116 case 11:
117 device->blorp.exec = gen11_blorp_exec;
118 break;
119 default:
120 unreachable("Unknown hardware generation");
121 }
122 }
123
124 void
125 anv_device_finish_blorp(struct anv_device *device)
126 {
127 blorp_finish(&device->blorp);
128 }
129
130 static void
131 get_blorp_surf_for_anv_buffer(struct anv_device *device,
132 struct anv_buffer *buffer, uint64_t offset,
133 uint32_t width, uint32_t height,
134 uint32_t row_pitch, enum isl_format format,
135 struct blorp_surf *blorp_surf,
136 struct isl_surf *isl_surf)
137 {
138 const struct isl_format_layout *fmtl =
139 isl_format_get_layout(format);
140 bool ok UNUSED;
141
142 /* ASTC is the only format which doesn't support linear layouts.
143 * Create an equivalently sized surface with ISL to get around this.
144 */
145 if (fmtl->txc == ISL_TXC_ASTC) {
146 /* Use an equivalently sized format */
147 format = ISL_FORMAT_R32G32B32A32_UINT;
148 assert(fmtl->bpb == isl_format_get_layout(format)->bpb);
149
150 /* Shrink the dimensions for the new format */
151 width = DIV_ROUND_UP(width, fmtl->bw);
152 height = DIV_ROUND_UP(height, fmtl->bh);
153 }
154
155 *blorp_surf = (struct blorp_surf) {
156 .surf = isl_surf,
157 .addr = {
158 .buffer = buffer->address.bo,
159 .offset = buffer->address.offset + offset,
160 .mocs = anv_mocs_for_bo(device, buffer->address.bo),
161 },
162 };
163
164 ok = isl_surf_init(&device->isl_dev, isl_surf,
165 .dim = ISL_SURF_DIM_2D,
166 .format = format,
167 .width = width,
168 .height = height,
169 .depth = 1,
170 .levels = 1,
171 .array_len = 1,
172 .samples = 1,
173 .row_pitch_B = row_pitch,
174 .usage = ISL_SURF_USAGE_TEXTURE_BIT |
175 ISL_SURF_USAGE_RENDER_TARGET_BIT,
176 .tiling_flags = ISL_TILING_LINEAR_BIT);
177 assert(ok);
178 }
179
180 /* Pick something high enough that it won't be used in core and low enough it
181 * will never map to an extension.
182 */
183 #define ANV_IMAGE_LAYOUT_EXPLICIT_AUX (VkImageLayout)10000000
184
185 static struct blorp_address
186 anv_to_blorp_address(struct anv_address addr)
187 {
188 return (struct blorp_address) {
189 .buffer = addr.bo,
190 .offset = addr.offset,
191 };
192 }
193
194 static void
195 get_blorp_surf_for_anv_image(const struct anv_device *device,
196 const struct anv_image *image,
197 VkImageAspectFlags aspect,
198 VkImageLayout layout,
199 enum isl_aux_usage aux_usage,
200 struct blorp_surf *blorp_surf)
201 {
202 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
203
204 if (layout != ANV_IMAGE_LAYOUT_EXPLICIT_AUX)
205 aux_usage = anv_layout_to_aux_usage(&device->info, image, aspect, layout);
206
207 const struct anv_surface *surface = &image->planes[plane].surface;
208 *blorp_surf = (struct blorp_surf) {
209 .surf = &surface->isl,
210 .addr = {
211 .buffer = image->planes[plane].address.bo,
212 .offset = image->planes[plane].address.offset + surface->offset,
213 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
214 },
215 };
216
217 if (aux_usage != ISL_AUX_USAGE_NONE) {
218 const struct anv_surface *aux_surface = &image->planes[plane].aux_surface;
219 blorp_surf->aux_surf = &aux_surface->isl,
220 blorp_surf->aux_addr = (struct blorp_address) {
221 .buffer = image->planes[plane].address.bo,
222 .offset = image->planes[plane].address.offset + aux_surface->offset,
223 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
224 };
225 blorp_surf->aux_usage = aux_usage;
226
227 /* If we're doing a partial resolve, then we need the indirect clear
228 * color. If we are doing a fast clear and want to store/update the
229 * clear color, we also pass the address to blorp, otherwise it will only
230 * stomp the CCS to a particular value and won't care about format or
231 * clear value
232 */
233 if (aspect & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
234 const struct anv_address clear_color_addr =
235 anv_image_get_clear_color_addr(device, image, aspect);
236 blorp_surf->clear_color_addr = anv_to_blorp_address(clear_color_addr);
237 } else if (aspect & VK_IMAGE_ASPECT_DEPTH_BIT
238 && device->info.gen >= 10) {
239 /* Vulkan always clears to 1.0. On gen < 10, we set that directly in
240 * the state packet. For gen >= 10, must provide the clear value in a
241 * buffer. We have a single global buffer that stores the 1.0 value.
242 */
243 const struct anv_address clear_color_addr = (struct anv_address) {
244 .bo = (struct anv_bo *)&device->hiz_clear_bo
245 };
246 blorp_surf->clear_color_addr = anv_to_blorp_address(clear_color_addr);
247 }
248 }
249 }
250
251 void anv_CmdCopyImage(
252 VkCommandBuffer commandBuffer,
253 VkImage srcImage,
254 VkImageLayout srcImageLayout,
255 VkImage dstImage,
256 VkImageLayout dstImageLayout,
257 uint32_t regionCount,
258 const VkImageCopy* pRegions)
259 {
260 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
261 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
262 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
263
264 struct blorp_batch batch;
265 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
266
267 for (unsigned r = 0; r < regionCount; r++) {
268 VkOffset3D srcOffset =
269 anv_sanitize_image_offset(src_image->type, pRegions[r].srcOffset);
270 VkOffset3D dstOffset =
271 anv_sanitize_image_offset(dst_image->type, pRegions[r].dstOffset);
272 VkExtent3D extent =
273 anv_sanitize_image_extent(src_image->type, pRegions[r].extent);
274
275 const uint32_t dst_level = pRegions[r].dstSubresource.mipLevel;
276 unsigned dst_base_layer, layer_count;
277 if (dst_image->type == VK_IMAGE_TYPE_3D) {
278 dst_base_layer = pRegions[r].dstOffset.z;
279 layer_count = pRegions[r].extent.depth;
280 } else {
281 dst_base_layer = pRegions[r].dstSubresource.baseArrayLayer;
282 layer_count =
283 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource);
284 }
285
286 const uint32_t src_level = pRegions[r].srcSubresource.mipLevel;
287 unsigned src_base_layer;
288 if (src_image->type == VK_IMAGE_TYPE_3D) {
289 src_base_layer = pRegions[r].srcOffset.z;
290 } else {
291 src_base_layer = pRegions[r].srcSubresource.baseArrayLayer;
292 assert(layer_count ==
293 anv_get_layerCount(src_image, &pRegions[r].srcSubresource));
294 }
295
296 VkImageAspectFlags src_mask = pRegions[r].srcSubresource.aspectMask,
297 dst_mask = pRegions[r].dstSubresource.aspectMask;
298
299 assert(anv_image_aspects_compatible(src_mask, dst_mask));
300
301 if (util_bitcount(src_mask) > 1) {
302 uint32_t aspect_bit;
303 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_mask) {
304 struct blorp_surf src_surf, dst_surf;
305 get_blorp_surf_for_anv_image(cmd_buffer->device,
306 src_image, 1UL << aspect_bit,
307 srcImageLayout, ISL_AUX_USAGE_NONE,
308 &src_surf);
309 get_blorp_surf_for_anv_image(cmd_buffer->device,
310 dst_image, 1UL << aspect_bit,
311 dstImageLayout, ISL_AUX_USAGE_NONE,
312 &dst_surf);
313 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
314 1UL << aspect_bit,
315 dst_surf.aux_usage, dst_level,
316 dst_base_layer, layer_count);
317
318 for (unsigned i = 0; i < layer_count; i++) {
319 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
320 &dst_surf, dst_level, dst_base_layer + i,
321 srcOffset.x, srcOffset.y,
322 dstOffset.x, dstOffset.y,
323 extent.width, extent.height);
324 }
325 }
326 } else {
327 struct blorp_surf src_surf, dst_surf;
328 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, src_mask,
329 srcImageLayout, ISL_AUX_USAGE_NONE,
330 &src_surf);
331 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, dst_mask,
332 dstImageLayout, ISL_AUX_USAGE_NONE,
333 &dst_surf);
334 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image, dst_mask,
335 dst_surf.aux_usage, dst_level,
336 dst_base_layer, layer_count);
337
338 for (unsigned i = 0; i < layer_count; i++) {
339 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
340 &dst_surf, dst_level, dst_base_layer + i,
341 srcOffset.x, srcOffset.y,
342 dstOffset.x, dstOffset.y,
343 extent.width, extent.height);
344 }
345 }
346 }
347
348 blorp_batch_finish(&batch);
349 }
350
351 static void
352 copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer,
353 struct anv_buffer *anv_buffer,
354 struct anv_image *anv_image,
355 VkImageLayout image_layout,
356 uint32_t regionCount,
357 const VkBufferImageCopy* pRegions,
358 bool buffer_to_image)
359 {
360 struct blorp_batch batch;
361 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
362
363 struct {
364 struct blorp_surf surf;
365 uint32_t level;
366 VkOffset3D offset;
367 } image, buffer, *src, *dst;
368
369 buffer.level = 0;
370 buffer.offset = (VkOffset3D) { 0, 0, 0 };
371
372 if (buffer_to_image) {
373 src = &buffer;
374 dst = &image;
375 } else {
376 src = &image;
377 dst = &buffer;
378 }
379
380 for (unsigned r = 0; r < regionCount; r++) {
381 const VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask;
382
383 get_blorp_surf_for_anv_image(cmd_buffer->device, anv_image, aspect,
384 image_layout, ISL_AUX_USAGE_NONE,
385 &image.surf);
386 image.offset =
387 anv_sanitize_image_offset(anv_image->type, pRegions[r].imageOffset);
388 image.level = pRegions[r].imageSubresource.mipLevel;
389
390 VkExtent3D extent =
391 anv_sanitize_image_extent(anv_image->type, pRegions[r].imageExtent);
392 if (anv_image->type != VK_IMAGE_TYPE_3D) {
393 image.offset.z = pRegions[r].imageSubresource.baseArrayLayer;
394 extent.depth =
395 anv_get_layerCount(anv_image, &pRegions[r].imageSubresource);
396 }
397
398 const enum isl_format buffer_format =
399 anv_get_isl_format(&cmd_buffer->device->info, anv_image->vk_format,
400 aspect, VK_IMAGE_TILING_LINEAR);
401
402 const VkExtent3D bufferImageExtent = {
403 .width = pRegions[r].bufferRowLength ?
404 pRegions[r].bufferRowLength : extent.width,
405 .height = pRegions[r].bufferImageHeight ?
406 pRegions[r].bufferImageHeight : extent.height,
407 };
408
409 const struct isl_format_layout *buffer_fmtl =
410 isl_format_get_layout(buffer_format);
411
412 const uint32_t buffer_row_pitch =
413 DIV_ROUND_UP(bufferImageExtent.width, buffer_fmtl->bw) *
414 (buffer_fmtl->bpb / 8);
415
416 const uint32_t buffer_layer_stride =
417 DIV_ROUND_UP(bufferImageExtent.height, buffer_fmtl->bh) *
418 buffer_row_pitch;
419
420 struct isl_surf buffer_isl_surf;
421 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
422 anv_buffer, pRegions[r].bufferOffset,
423 extent.width, extent.height,
424 buffer_row_pitch, buffer_format,
425 &buffer.surf, &buffer_isl_surf);
426
427 if (&image == dst) {
428 anv_cmd_buffer_mark_image_written(cmd_buffer, anv_image,
429 aspect, dst->surf.aux_usage,
430 dst->level,
431 dst->offset.z, extent.depth);
432 }
433
434 for (unsigned z = 0; z < extent.depth; z++) {
435 blorp_copy(&batch, &src->surf, src->level, src->offset.z,
436 &dst->surf, dst->level, dst->offset.z,
437 src->offset.x, src->offset.y, dst->offset.x, dst->offset.y,
438 extent.width, extent.height);
439
440 image.offset.z++;
441 buffer.surf.addr.offset += buffer_layer_stride;
442 }
443 }
444
445 blorp_batch_finish(&batch);
446 }
447
448 void anv_CmdCopyBufferToImage(
449 VkCommandBuffer commandBuffer,
450 VkBuffer srcBuffer,
451 VkImage dstImage,
452 VkImageLayout dstImageLayout,
453 uint32_t regionCount,
454 const VkBufferImageCopy* pRegions)
455 {
456 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
457 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
458 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
459
460 copy_buffer_to_image(cmd_buffer, src_buffer, dst_image, dstImageLayout,
461 regionCount, pRegions, true);
462 }
463
464 void anv_CmdCopyImageToBuffer(
465 VkCommandBuffer commandBuffer,
466 VkImage srcImage,
467 VkImageLayout srcImageLayout,
468 VkBuffer dstBuffer,
469 uint32_t regionCount,
470 const VkBufferImageCopy* pRegions)
471 {
472 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
473 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
474 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
475
476 copy_buffer_to_image(cmd_buffer, dst_buffer, src_image, srcImageLayout,
477 regionCount, pRegions, false);
478 }
479
480 static bool
481 flip_coords(unsigned *src0, unsigned *src1, unsigned *dst0, unsigned *dst1)
482 {
483 bool flip = false;
484 if (*src0 > *src1) {
485 unsigned tmp = *src0;
486 *src0 = *src1;
487 *src1 = tmp;
488 flip = !flip;
489 }
490
491 if (*dst0 > *dst1) {
492 unsigned tmp = *dst0;
493 *dst0 = *dst1;
494 *dst1 = tmp;
495 flip = !flip;
496 }
497
498 return flip;
499 }
500
501 void anv_CmdBlitImage(
502 VkCommandBuffer commandBuffer,
503 VkImage srcImage,
504 VkImageLayout srcImageLayout,
505 VkImage dstImage,
506 VkImageLayout dstImageLayout,
507 uint32_t regionCount,
508 const VkImageBlit* pRegions,
509 VkFilter filter)
510
511 {
512 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
513 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
514 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
515
516 struct blorp_surf src, dst;
517
518 enum blorp_filter blorp_filter;
519 switch (filter) {
520 case VK_FILTER_NEAREST:
521 blorp_filter = BLORP_FILTER_NEAREST;
522 break;
523 case VK_FILTER_LINEAR:
524 blorp_filter = BLORP_FILTER_BILINEAR;
525 break;
526 default:
527 unreachable("Invalid filter");
528 }
529
530 struct blorp_batch batch;
531 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
532
533 for (unsigned r = 0; r < regionCount; r++) {
534 const VkImageSubresourceLayers *src_res = &pRegions[r].srcSubresource;
535 const VkImageSubresourceLayers *dst_res = &pRegions[r].dstSubresource;
536
537 assert(anv_image_aspects_compatible(src_res->aspectMask,
538 dst_res->aspectMask));
539
540 uint32_t aspect_bit;
541 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_res->aspectMask) {
542 get_blorp_surf_for_anv_image(cmd_buffer->device,
543 src_image, 1U << aspect_bit,
544 srcImageLayout, ISL_AUX_USAGE_NONE, &src);
545 get_blorp_surf_for_anv_image(cmd_buffer->device,
546 dst_image, 1U << aspect_bit,
547 dstImageLayout, ISL_AUX_USAGE_NONE, &dst);
548
549 struct anv_format_plane src_format =
550 anv_get_format_plane(&cmd_buffer->device->info, src_image->vk_format,
551 1U << aspect_bit, src_image->tiling);
552 struct anv_format_plane dst_format =
553 anv_get_format_plane(&cmd_buffer->device->info, dst_image->vk_format,
554 1U << aspect_bit, dst_image->tiling);
555
556 unsigned dst_start, dst_end;
557 if (dst_image->type == VK_IMAGE_TYPE_3D) {
558 assert(dst_res->baseArrayLayer == 0);
559 dst_start = pRegions[r].dstOffsets[0].z;
560 dst_end = pRegions[r].dstOffsets[1].z;
561 } else {
562 dst_start = dst_res->baseArrayLayer;
563 dst_end = dst_start + anv_get_layerCount(dst_image, dst_res);
564 }
565
566 unsigned src_start, src_end;
567 if (src_image->type == VK_IMAGE_TYPE_3D) {
568 assert(src_res->baseArrayLayer == 0);
569 src_start = pRegions[r].srcOffsets[0].z;
570 src_end = pRegions[r].srcOffsets[1].z;
571 } else {
572 src_start = src_res->baseArrayLayer;
573 src_end = src_start + anv_get_layerCount(src_image, src_res);
574 }
575
576 bool flip_z = flip_coords(&src_start, &src_end, &dst_start, &dst_end);
577 float src_z_step = (float)(src_end + 1 - src_start) /
578 (float)(dst_end + 1 - dst_start);
579
580 if (flip_z) {
581 src_start = src_end;
582 src_z_step *= -1;
583 }
584
585 unsigned src_x0 = pRegions[r].srcOffsets[0].x;
586 unsigned src_x1 = pRegions[r].srcOffsets[1].x;
587 unsigned dst_x0 = pRegions[r].dstOffsets[0].x;
588 unsigned dst_x1 = pRegions[r].dstOffsets[1].x;
589 bool flip_x = flip_coords(&src_x0, &src_x1, &dst_x0, &dst_x1);
590
591 unsigned src_y0 = pRegions[r].srcOffsets[0].y;
592 unsigned src_y1 = pRegions[r].srcOffsets[1].y;
593 unsigned dst_y0 = pRegions[r].dstOffsets[0].y;
594 unsigned dst_y1 = pRegions[r].dstOffsets[1].y;
595 bool flip_y = flip_coords(&src_y0, &src_y1, &dst_y0, &dst_y1);
596
597 const unsigned num_layers = dst_end - dst_start;
598 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
599 1U << aspect_bit,
600 dst.aux_usage,
601 dst_res->mipLevel,
602 dst_start, num_layers);
603
604 for (unsigned i = 0; i < num_layers; i++) {
605 unsigned dst_z = dst_start + i;
606 unsigned src_z = src_start + i * src_z_step;
607
608 blorp_blit(&batch, &src, src_res->mipLevel, src_z,
609 src_format.isl_format, src_format.swizzle,
610 &dst, dst_res->mipLevel, dst_z,
611 dst_format.isl_format, dst_format.swizzle,
612 src_x0, src_y0, src_x1, src_y1,
613 dst_x0, dst_y0, dst_x1, dst_y1,
614 blorp_filter, flip_x, flip_y);
615 }
616 }
617 }
618
619 blorp_batch_finish(&batch);
620 }
621
622 static enum isl_format
623 isl_format_for_size(unsigned size_B)
624 {
625 switch (size_B) {
626 case 4: return ISL_FORMAT_R32_UINT;
627 case 8: return ISL_FORMAT_R32G32_UINT;
628 case 16: return ISL_FORMAT_R32G32B32A32_UINT;
629 default:
630 unreachable("Not a power-of-two format size");
631 }
632 }
633
634 /**
635 * Returns the greatest common divisor of a and b that is a power of two.
636 */
637 static uint64_t
638 gcd_pow2_u64(uint64_t a, uint64_t b)
639 {
640 assert(a > 0 || b > 0);
641
642 unsigned a_log2 = ffsll(a) - 1;
643 unsigned b_log2 = ffsll(b) - 1;
644
645 /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which
646 * case, the MIN2() will take the other one. If both are 0 then we will
647 * hit the assert above.
648 */
649 return 1 << MIN2(a_log2, b_log2);
650 }
651
652 /* This is maximum possible width/height our HW can handle */
653 #define MAX_SURFACE_DIM (1ull << 14)
654
655 void anv_CmdCopyBuffer(
656 VkCommandBuffer commandBuffer,
657 VkBuffer srcBuffer,
658 VkBuffer dstBuffer,
659 uint32_t regionCount,
660 const VkBufferCopy* pRegions)
661 {
662 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
663 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
664 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
665
666 struct blorp_batch batch;
667 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
668
669 for (unsigned r = 0; r < regionCount; r++) {
670 struct blorp_address src = {
671 .buffer = src_buffer->address.bo,
672 .offset = src_buffer->address.offset + pRegions[r].srcOffset,
673 .mocs = anv_mocs_for_bo(cmd_buffer->device, src_buffer->address.bo),
674 };
675 struct blorp_address dst = {
676 .buffer = dst_buffer->address.bo,
677 .offset = dst_buffer->address.offset + pRegions[r].dstOffset,
678 .mocs = anv_mocs_for_bo(cmd_buffer->device, dst_buffer->address.bo),
679 };
680
681 blorp_buffer_copy(&batch, src, dst, pRegions[r].size);
682 }
683
684 blorp_batch_finish(&batch);
685 }
686
687 void anv_CmdUpdateBuffer(
688 VkCommandBuffer commandBuffer,
689 VkBuffer dstBuffer,
690 VkDeviceSize dstOffset,
691 VkDeviceSize dataSize,
692 const void* pData)
693 {
694 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
695 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
696
697 struct blorp_batch batch;
698 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
699
700 /* We can't quite grab a full block because the state stream needs a
701 * little data at the top to build its linked list.
702 */
703 const uint32_t max_update_size =
704 cmd_buffer->device->dynamic_state_pool.block_size - 64;
705
706 assert(max_update_size < MAX_SURFACE_DIM * 4);
707
708 /* We're about to read data that was written from the CPU. Flush the
709 * texture cache so we don't get anything stale.
710 */
711 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
712
713 while (dataSize) {
714 const uint32_t copy_size = MIN2(dataSize, max_update_size);
715
716 struct anv_state tmp_data =
717 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, copy_size, 64);
718
719 memcpy(tmp_data.map, pData, copy_size);
720
721 anv_state_flush(cmd_buffer->device, tmp_data);
722
723 struct blorp_address src = {
724 .buffer = &cmd_buffer->device->dynamic_state_pool.block_pool.bo,
725 .offset = tmp_data.offset,
726 .mocs = cmd_buffer->device->default_mocs,
727 };
728 struct blorp_address dst = {
729 .buffer = dst_buffer->address.bo,
730 .offset = dst_buffer->address.offset + dstOffset,
731 .mocs = anv_mocs_for_bo(cmd_buffer->device, dst_buffer->address.bo),
732 };
733
734 blorp_buffer_copy(&batch, src, dst, copy_size);
735
736 dataSize -= copy_size;
737 dstOffset += copy_size;
738 pData = (void *)pData + copy_size;
739 }
740
741 blorp_batch_finish(&batch);
742 }
743
744 void anv_CmdFillBuffer(
745 VkCommandBuffer commandBuffer,
746 VkBuffer dstBuffer,
747 VkDeviceSize dstOffset,
748 VkDeviceSize fillSize,
749 uint32_t data)
750 {
751 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
752 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
753 struct blorp_surf surf;
754 struct isl_surf isl_surf;
755
756 struct blorp_batch batch;
757 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
758
759 fillSize = anv_buffer_get_range(dst_buffer, dstOffset, fillSize);
760
761 /* From the Vulkan spec:
762 *
763 * "size is the number of bytes to fill, and must be either a multiple
764 * of 4, or VK_WHOLE_SIZE to fill the range from offset to the end of
765 * the buffer. If VK_WHOLE_SIZE is used and the remaining size of the
766 * buffer is not a multiple of 4, then the nearest smaller multiple is
767 * used."
768 */
769 fillSize &= ~3ull;
770
771 /* First, we compute the biggest format that can be used with the
772 * given offsets and size.
773 */
774 int bs = 16;
775 bs = gcd_pow2_u64(bs, dstOffset);
776 bs = gcd_pow2_u64(bs, fillSize);
777 enum isl_format isl_format = isl_format_for_size(bs);
778
779 union isl_color_value color = {
780 .u32 = { data, data, data, data },
781 };
782
783 const uint64_t max_fill_size = MAX_SURFACE_DIM * MAX_SURFACE_DIM * bs;
784 while (fillSize >= max_fill_size) {
785 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
786 dst_buffer, dstOffset,
787 MAX_SURFACE_DIM, MAX_SURFACE_DIM,
788 MAX_SURFACE_DIM * bs, isl_format,
789 &surf, &isl_surf);
790
791 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
792 0, 0, 1, 0, 0, MAX_SURFACE_DIM, MAX_SURFACE_DIM,
793 color, NULL);
794 fillSize -= max_fill_size;
795 dstOffset += max_fill_size;
796 }
797
798 uint64_t height = fillSize / (MAX_SURFACE_DIM * bs);
799 assert(height < MAX_SURFACE_DIM);
800 if (height != 0) {
801 const uint64_t rect_fill_size = height * MAX_SURFACE_DIM * bs;
802 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
803 dst_buffer, dstOffset,
804 MAX_SURFACE_DIM, height,
805 MAX_SURFACE_DIM * bs, isl_format,
806 &surf, &isl_surf);
807
808 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
809 0, 0, 1, 0, 0, MAX_SURFACE_DIM, height,
810 color, NULL);
811 fillSize -= rect_fill_size;
812 dstOffset += rect_fill_size;
813 }
814
815 if (fillSize != 0) {
816 const uint32_t width = fillSize / bs;
817 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
818 dst_buffer, dstOffset,
819 width, 1,
820 width * bs, isl_format,
821 &surf, &isl_surf);
822
823 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
824 0, 0, 1, 0, 0, width, 1,
825 color, NULL);
826 }
827
828 blorp_batch_finish(&batch);
829 }
830
831 void anv_CmdClearColorImage(
832 VkCommandBuffer commandBuffer,
833 VkImage _image,
834 VkImageLayout imageLayout,
835 const VkClearColorValue* pColor,
836 uint32_t rangeCount,
837 const VkImageSubresourceRange* pRanges)
838 {
839 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
840 ANV_FROM_HANDLE(anv_image, image, _image);
841
842 static const bool color_write_disable[4] = { false, false, false, false };
843
844 struct blorp_batch batch;
845 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
846
847
848 for (unsigned r = 0; r < rangeCount; r++) {
849 if (pRanges[r].aspectMask == 0)
850 continue;
851
852 assert(pRanges[r].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
853
854 struct blorp_surf surf;
855 get_blorp_surf_for_anv_image(cmd_buffer->device,
856 image, pRanges[r].aspectMask,
857 imageLayout, ISL_AUX_USAGE_NONE, &surf);
858
859 struct anv_format_plane src_format =
860 anv_get_format_plane(&cmd_buffer->device->info, image->vk_format,
861 VK_IMAGE_ASPECT_COLOR_BIT, image->tiling);
862
863 unsigned base_layer = pRanges[r].baseArrayLayer;
864 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
865
866 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
867 const unsigned level = pRanges[r].baseMipLevel + i;
868 const unsigned level_width = anv_minify(image->extent.width, level);
869 const unsigned level_height = anv_minify(image->extent.height, level);
870
871 if (image->type == VK_IMAGE_TYPE_3D) {
872 base_layer = 0;
873 layer_count = anv_minify(image->extent.depth, level);
874 }
875
876 anv_cmd_buffer_mark_image_written(cmd_buffer, image,
877 pRanges[r].aspectMask,
878 surf.aux_usage, level,
879 base_layer, layer_count);
880
881 blorp_clear(&batch, &surf,
882 src_format.isl_format, src_format.swizzle,
883 level, base_layer, layer_count,
884 0, 0, level_width, level_height,
885 vk_to_isl_color(*pColor), color_write_disable);
886 }
887 }
888
889 blorp_batch_finish(&batch);
890 }
891
892 void anv_CmdClearDepthStencilImage(
893 VkCommandBuffer commandBuffer,
894 VkImage image_h,
895 VkImageLayout imageLayout,
896 const VkClearDepthStencilValue* pDepthStencil,
897 uint32_t rangeCount,
898 const VkImageSubresourceRange* pRanges)
899 {
900 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
901 ANV_FROM_HANDLE(anv_image, image, image_h);
902
903 struct blorp_batch batch;
904 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
905
906 struct blorp_surf depth, stencil;
907 if (image->aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
908 get_blorp_surf_for_anv_image(cmd_buffer->device,
909 image, VK_IMAGE_ASPECT_DEPTH_BIT,
910 imageLayout, ISL_AUX_USAGE_NONE, &depth);
911 } else {
912 memset(&depth, 0, sizeof(depth));
913 }
914
915 if (image->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
916 get_blorp_surf_for_anv_image(cmd_buffer->device,
917 image, VK_IMAGE_ASPECT_STENCIL_BIT,
918 imageLayout, ISL_AUX_USAGE_NONE, &stencil);
919 } else {
920 memset(&stencil, 0, sizeof(stencil));
921 }
922
923 for (unsigned r = 0; r < rangeCount; r++) {
924 if (pRanges[r].aspectMask == 0)
925 continue;
926
927 bool clear_depth = pRanges[r].aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
928 bool clear_stencil = pRanges[r].aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
929
930 unsigned base_layer = pRanges[r].baseArrayLayer;
931 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
932
933 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
934 const unsigned level = pRanges[r].baseMipLevel + i;
935 const unsigned level_width = anv_minify(image->extent.width, level);
936 const unsigned level_height = anv_minify(image->extent.height, level);
937
938 if (image->type == VK_IMAGE_TYPE_3D)
939 layer_count = anv_minify(image->extent.depth, level);
940
941 blorp_clear_depth_stencil(&batch, &depth, &stencil,
942 level, base_layer, layer_count,
943 0, 0, level_width, level_height,
944 clear_depth, pDepthStencil->depth,
945 clear_stencil ? 0xff : 0,
946 pDepthStencil->stencil);
947 }
948 }
949
950 blorp_batch_finish(&batch);
951 }
952
953 VkResult
954 anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer *cmd_buffer,
955 uint32_t num_entries,
956 uint32_t *state_offset,
957 struct anv_state *bt_state)
958 {
959 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
960 state_offset);
961 if (bt_state->map == NULL) {
962 /* We ran out of space. Grab a new binding table block. */
963 VkResult result = anv_cmd_buffer_new_binding_table_block(cmd_buffer);
964 if (result != VK_SUCCESS)
965 return result;
966
967 /* Re-emit state base addresses so we get the new surface state base
968 * address before we start emitting binding tables etc.
969 */
970 anv_cmd_buffer_emit_state_base_address(cmd_buffer);
971
972 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
973 state_offset);
974 assert(bt_state->map != NULL);
975 }
976
977 return VK_SUCCESS;
978 }
979
980 static VkResult
981 binding_table_for_surface_state(struct anv_cmd_buffer *cmd_buffer,
982 struct anv_state surface_state,
983 uint32_t *bt_offset)
984 {
985 uint32_t state_offset;
986 struct anv_state bt_state;
987
988 VkResult result =
989 anv_cmd_buffer_alloc_blorp_binding_table(cmd_buffer, 1, &state_offset,
990 &bt_state);
991 if (result != VK_SUCCESS)
992 return result;
993
994 uint32_t *bt_map = bt_state.map;
995 bt_map[0] = surface_state.offset + state_offset;
996
997 *bt_offset = bt_state.offset;
998 return VK_SUCCESS;
999 }
1000
1001 static void
1002 clear_color_attachment(struct anv_cmd_buffer *cmd_buffer,
1003 struct blorp_batch *batch,
1004 const VkClearAttachment *attachment,
1005 uint32_t rectCount, const VkClearRect *pRects)
1006 {
1007 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
1008 const uint32_t color_att = attachment->colorAttachment;
1009 const uint32_t att_idx = subpass->color_attachments[color_att].attachment;
1010
1011 if (att_idx == VK_ATTACHMENT_UNUSED)
1012 return;
1013
1014 struct anv_render_pass_attachment *pass_att =
1015 &cmd_buffer->state.pass->attachments[att_idx];
1016 struct anv_attachment_state *att_state =
1017 &cmd_buffer->state.attachments[att_idx];
1018
1019 uint32_t binding_table;
1020 VkResult result =
1021 binding_table_for_surface_state(cmd_buffer, att_state->color.state,
1022 &binding_table);
1023 if (result != VK_SUCCESS)
1024 return;
1025
1026 union isl_color_value clear_color =
1027 vk_to_isl_color(attachment->clearValue.color);
1028
1029 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1030 if (subpass->view_mask) {
1031 uint32_t view_idx;
1032 for_each_bit(view_idx, subpass->view_mask) {
1033 for (uint32_t r = 0; r < rectCount; ++r) {
1034 const VkOffset2D offset = pRects[r].rect.offset;
1035 const VkExtent2D extent = pRects[r].rect.extent;
1036 blorp_clear_attachments(batch, binding_table,
1037 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1038 view_idx, 1,
1039 offset.x, offset.y,
1040 offset.x + extent.width,
1041 offset.y + extent.height,
1042 true, clear_color, false, 0.0f, 0, 0);
1043 }
1044 }
1045 return;
1046 }
1047
1048 for (uint32_t r = 0; r < rectCount; ++r) {
1049 const VkOffset2D offset = pRects[r].rect.offset;
1050 const VkExtent2D extent = pRects[r].rect.extent;
1051 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1052 blorp_clear_attachments(batch, binding_table,
1053 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1054 pRects[r].baseArrayLayer,
1055 pRects[r].layerCount,
1056 offset.x, offset.y,
1057 offset.x + extent.width, offset.y + extent.height,
1058 true, clear_color, false, 0.0f, 0, 0);
1059 }
1060 }
1061
1062 static void
1063 clear_depth_stencil_attachment(struct anv_cmd_buffer *cmd_buffer,
1064 struct blorp_batch *batch,
1065 const VkClearAttachment *attachment,
1066 uint32_t rectCount, const VkClearRect *pRects)
1067 {
1068 static const union isl_color_value color_value = { .u32 = { 0, } };
1069 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
1070 const uint32_t att_idx = subpass->depth_stencil_attachment->attachment;
1071
1072 if (att_idx == VK_ATTACHMENT_UNUSED)
1073 return;
1074
1075 struct anv_render_pass_attachment *pass_att =
1076 &cmd_buffer->state.pass->attachments[att_idx];
1077
1078 bool clear_depth = attachment->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
1079 bool clear_stencil = attachment->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
1080
1081 enum isl_format depth_format = ISL_FORMAT_UNSUPPORTED;
1082 if (clear_depth) {
1083 depth_format = anv_get_isl_format(&cmd_buffer->device->info,
1084 pass_att->format,
1085 VK_IMAGE_ASPECT_DEPTH_BIT,
1086 VK_IMAGE_TILING_OPTIMAL);
1087 }
1088
1089 uint32_t binding_table;
1090 VkResult result =
1091 binding_table_for_surface_state(cmd_buffer,
1092 cmd_buffer->state.null_surface_state,
1093 &binding_table);
1094 if (result != VK_SUCCESS)
1095 return;
1096
1097 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1098 if (subpass->view_mask) {
1099 uint32_t view_idx;
1100 for_each_bit(view_idx, subpass->view_mask) {
1101 for (uint32_t r = 0; r < rectCount; ++r) {
1102 const VkOffset2D offset = pRects[r].rect.offset;
1103 const VkExtent2D extent = pRects[r].rect.extent;
1104 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1105 blorp_clear_attachments(batch, binding_table,
1106 depth_format, pass_att->samples,
1107 view_idx, 1,
1108 offset.x, offset.y,
1109 offset.x + extent.width,
1110 offset.y + extent.height,
1111 false, color_value,
1112 clear_depth, value.depth,
1113 clear_stencil ? 0xff : 0, value.stencil);
1114 }
1115 }
1116 return;
1117 }
1118
1119 for (uint32_t r = 0; r < rectCount; ++r) {
1120 const VkOffset2D offset = pRects[r].rect.offset;
1121 const VkExtent2D extent = pRects[r].rect.extent;
1122 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1123 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1124 blorp_clear_attachments(batch, binding_table,
1125 depth_format, pass_att->samples,
1126 pRects[r].baseArrayLayer,
1127 pRects[r].layerCount,
1128 offset.x, offset.y,
1129 offset.x + extent.width, offset.y + extent.height,
1130 false, color_value,
1131 clear_depth, value.depth,
1132 clear_stencil ? 0xff : 0, value.stencil);
1133 }
1134 }
1135
1136 void anv_CmdClearAttachments(
1137 VkCommandBuffer commandBuffer,
1138 uint32_t attachmentCount,
1139 const VkClearAttachment* pAttachments,
1140 uint32_t rectCount,
1141 const VkClearRect* pRects)
1142 {
1143 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1144
1145 /* Because this gets called within a render pass, we tell blorp not to
1146 * trash our depth and stencil buffers.
1147 */
1148 struct blorp_batch batch;
1149 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1150 BLORP_BATCH_NO_EMIT_DEPTH_STENCIL);
1151
1152 for (uint32_t a = 0; a < attachmentCount; ++a) {
1153 if (pAttachments[a].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
1154 assert(pAttachments[a].aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
1155 clear_color_attachment(cmd_buffer, &batch,
1156 &pAttachments[a],
1157 rectCount, pRects);
1158 } else {
1159 clear_depth_stencil_attachment(cmd_buffer, &batch,
1160 &pAttachments[a],
1161 rectCount, pRects);
1162 }
1163 }
1164
1165 blorp_batch_finish(&batch);
1166 }
1167
1168 enum subpass_stage {
1169 SUBPASS_STAGE_LOAD,
1170 SUBPASS_STAGE_DRAW,
1171 SUBPASS_STAGE_RESOLVE,
1172 };
1173
1174 static void
1175 resolve_surface(struct blorp_batch *batch,
1176 struct blorp_surf *src_surf,
1177 uint32_t src_level, uint32_t src_layer,
1178 struct blorp_surf *dst_surf,
1179 uint32_t dst_level, uint32_t dst_layer,
1180 uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y,
1181 uint32_t width, uint32_t height,
1182 enum blorp_filter filter)
1183 {
1184 blorp_blit(batch,
1185 src_surf, src_level, src_layer,
1186 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1187 dst_surf, dst_level, dst_layer,
1188 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1189 src_x, src_y, src_x + width, src_y + height,
1190 dst_x, dst_y, dst_x + width, dst_y + height,
1191 filter, false, false);
1192 }
1193
1194 static void
1195 resolve_image(struct anv_device *device,
1196 struct blorp_batch *batch,
1197 const struct anv_image *src_image,
1198 VkImageLayout src_image_layout,
1199 uint32_t src_level, uint32_t src_layer,
1200 const struct anv_image *dst_image,
1201 VkImageLayout dst_image_layout,
1202 uint32_t dst_level, uint32_t dst_layer,
1203 VkImageAspectFlags aspect_mask,
1204 uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y,
1205 uint32_t width, uint32_t height)
1206 {
1207 struct anv_cmd_buffer *cmd_buffer = batch->driver_batch;
1208
1209 assert(src_image->type == VK_IMAGE_TYPE_2D);
1210 assert(src_image->samples > 1);
1211 assert(dst_image->type == VK_IMAGE_TYPE_2D);
1212 assert(dst_image->samples == 1);
1213 assert(src_image->n_planes == dst_image->n_planes);
1214
1215 uint32_t aspect_bit;
1216
1217 anv_foreach_image_aspect_bit(aspect_bit, src_image, aspect_mask) {
1218 struct blorp_surf src_surf, dst_surf;
1219 get_blorp_surf_for_anv_image(device, src_image, 1UL << aspect_bit,
1220 src_image_layout, ISL_AUX_USAGE_NONE,
1221 &src_surf);
1222 get_blorp_surf_for_anv_image(device, dst_image, 1UL << aspect_bit,
1223 dst_image_layout, ISL_AUX_USAGE_NONE,
1224 &dst_surf);
1225 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
1226 1UL << aspect_bit,
1227 dst_surf.aux_usage,
1228 dst_level, dst_layer, 1);
1229
1230 enum blorp_filter filter;
1231 if ((src_surf.surf->usage & ISL_SURF_USAGE_DEPTH_BIT) ||
1232 (src_surf.surf->usage & ISL_SURF_USAGE_STENCIL_BIT) ||
1233 isl_format_has_int_channel(src_surf.surf->format)) {
1234 filter = BLORP_FILTER_SAMPLE_0;
1235 } else {
1236 filter = BLORP_FILTER_AVERAGE;
1237 }
1238
1239 assert(!src_image->format->can_ycbcr);
1240 assert(!dst_image->format->can_ycbcr);
1241
1242 resolve_surface(batch,
1243 &src_surf, src_level, src_layer,
1244 &dst_surf, dst_level, dst_layer,
1245 src_x, src_y, dst_x, dst_y, width, height, filter);
1246 }
1247 }
1248
1249 void anv_CmdResolveImage(
1250 VkCommandBuffer commandBuffer,
1251 VkImage srcImage,
1252 VkImageLayout srcImageLayout,
1253 VkImage dstImage,
1254 VkImageLayout dstImageLayout,
1255 uint32_t regionCount,
1256 const VkImageResolve* pRegions)
1257 {
1258 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1259 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
1260 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
1261
1262 struct blorp_batch batch;
1263 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1264
1265 for (uint32_t r = 0; r < regionCount; r++) {
1266 assert(pRegions[r].srcSubresource.aspectMask ==
1267 pRegions[r].dstSubresource.aspectMask);
1268 assert(anv_get_layerCount(src_image, &pRegions[r].srcSubresource) ==
1269 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource));
1270
1271 const uint32_t layer_count =
1272 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource);
1273
1274 VkImageAspectFlags src_mask = pRegions[r].srcSubresource.aspectMask,
1275 dst_mask = pRegions[r].dstSubresource.aspectMask;
1276
1277 assert(anv_image_aspects_compatible(src_mask, dst_mask));
1278
1279 for (uint32_t layer = 0; layer < layer_count; layer++) {
1280 resolve_image(cmd_buffer->device, &batch,
1281 src_image, srcImageLayout,
1282 pRegions[r].srcSubresource.mipLevel,
1283 pRegions[r].srcSubresource.baseArrayLayer + layer,
1284 dst_image, dstImageLayout,
1285 pRegions[r].dstSubresource.mipLevel,
1286 pRegions[r].dstSubresource.baseArrayLayer + layer,
1287 pRegions[r].dstSubresource.aspectMask,
1288 pRegions[r].srcOffset.x, pRegions[r].srcOffset.y,
1289 pRegions[r].dstOffset.x, pRegions[r].dstOffset.y,
1290 pRegions[r].extent.width, pRegions[r].extent.height);
1291 }
1292 }
1293
1294 blorp_batch_finish(&batch);
1295 }
1296
1297 static enum isl_aux_usage
1298 fast_clear_aux_usage(const struct anv_image *image,
1299 VkImageAspectFlagBits aspect)
1300 {
1301 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
1302 if (image->planes[plane].aux_usage == ISL_AUX_USAGE_NONE)
1303 return ISL_AUX_USAGE_CCS_D;
1304 else
1305 return image->planes[plane].aux_usage;
1306 }
1307
1308 void
1309 anv_cmd_buffer_resolve_subpass(struct anv_cmd_buffer *cmd_buffer)
1310 {
1311 struct anv_framebuffer *fb = cmd_buffer->state.framebuffer;
1312 struct anv_subpass *subpass = cmd_buffer->state.subpass;
1313
1314 if (subpass->has_color_resolve) {
1315 struct blorp_batch batch;
1316 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1317
1318 /* We are about to do some MSAA resolves. We need to flush so that the
1319 * result of writes to the MSAA color attachments show up in the sampler
1320 * when we blit to the single-sampled resolve target.
1321 */
1322 cmd_buffer->state.pending_pipe_bits |=
1323 ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT |
1324 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
1325
1326 for (uint32_t i = 0; i < subpass->color_count; ++i) {
1327 uint32_t src_att = subpass->color_attachments[i].attachment;
1328 uint32_t dst_att = subpass->resolve_attachments[i].attachment;
1329
1330 if (dst_att == VK_ATTACHMENT_UNUSED)
1331 continue;
1332
1333 assert(src_att < cmd_buffer->state.pass->attachment_count);
1334 assert(dst_att < cmd_buffer->state.pass->attachment_count);
1335
1336 if (cmd_buffer->state.attachments[dst_att].pending_clear_aspects) {
1337 /* From the Vulkan 1.0 spec:
1338 *
1339 * If the first use of an attachment in a render pass is as a
1340 * resolve attachment, then the loadOp is effectively ignored
1341 * as the resolve is guaranteed to overwrite all pixels in the
1342 * render area.
1343 */
1344 cmd_buffer->state.attachments[dst_att].pending_clear_aspects = 0;
1345 }
1346
1347 struct anv_image_view *src_iview = fb->attachments[src_att];
1348 struct anv_image_view *dst_iview = fb->attachments[dst_att];
1349
1350 enum isl_aux_usage src_aux_usage =
1351 cmd_buffer->state.attachments[src_att].aux_usage;
1352 enum isl_aux_usage dst_aux_usage =
1353 cmd_buffer->state.attachments[dst_att].aux_usage;
1354
1355 const VkRect2D render_area = cmd_buffer->state.render_area;
1356
1357 assert(src_iview->aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT &&
1358 dst_iview->aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT);
1359
1360 enum blorp_filter filter;
1361 if (isl_format_has_int_channel(src_iview->planes[0].isl.format)) {
1362 filter = BLORP_FILTER_SAMPLE_0;
1363 } else {
1364 filter = BLORP_FILTER_AVERAGE;
1365 }
1366
1367 struct blorp_surf src_surf, dst_surf;
1368 get_blorp_surf_for_anv_image(cmd_buffer->device, src_iview->image,
1369 VK_IMAGE_ASPECT_COLOR_BIT,
1370 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1371 src_aux_usage, &src_surf);
1372 if (src_aux_usage == ISL_AUX_USAGE_MCS) {
1373 src_surf.clear_color_addr = anv_to_blorp_address(
1374 anv_image_get_clear_color_addr(cmd_buffer->device,
1375 src_iview->image,
1376 VK_IMAGE_ASPECT_COLOR_BIT));
1377 }
1378 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_iview->image,
1379 VK_IMAGE_ASPECT_COLOR_BIT,
1380 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1381 dst_aux_usage, &dst_surf);
1382
1383 uint32_t base_src_layer = src_iview->planes[0].isl.base_array_layer;
1384 uint32_t base_dst_layer = dst_iview->planes[0].isl.base_array_layer;
1385
1386 assert(src_iview->planes[0].isl.array_len >= fb->layers);
1387 assert(dst_iview->planes[0].isl.array_len >= fb->layers);
1388
1389 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_iview->image,
1390 VK_IMAGE_ASPECT_COLOR_BIT,
1391 dst_surf.aux_usage,
1392 dst_iview->planes[0].isl.base_level,
1393 base_dst_layer, fb->layers);
1394
1395 assert(!src_iview->image->format->can_ycbcr);
1396 assert(!dst_iview->image->format->can_ycbcr);
1397
1398 for (uint32_t i = 0; i < fb->layers; i++) {
1399 resolve_surface(&batch,
1400 &src_surf,
1401 src_iview->planes[0].isl.base_level,
1402 base_src_layer + i,
1403 &dst_surf,
1404 dst_iview->planes[0].isl.base_level,
1405 base_dst_layer + i,
1406 render_area.offset.x, render_area.offset.y,
1407 render_area.offset.x, render_area.offset.y,
1408 render_area.extent.width, render_area.extent.height,
1409 filter);
1410 }
1411 }
1412
1413 blorp_batch_finish(&batch);
1414 }
1415 }
1416
1417 void
1418 anv_image_copy_to_shadow(struct anv_cmd_buffer *cmd_buffer,
1419 const struct anv_image *image,
1420 uint32_t base_level, uint32_t level_count,
1421 uint32_t base_layer, uint32_t layer_count)
1422 {
1423 struct blorp_batch batch;
1424 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1425
1426 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT && image->n_planes == 1);
1427
1428 struct blorp_surf surf;
1429 get_blorp_surf_for_anv_image(cmd_buffer->device,
1430 image, VK_IMAGE_ASPECT_COLOR_BIT,
1431 VK_IMAGE_LAYOUT_GENERAL,
1432 ISL_AUX_USAGE_NONE, &surf);
1433 assert(surf.aux_usage == ISL_AUX_USAGE_NONE);
1434
1435 struct blorp_surf shadow_surf = {
1436 .surf = &image->planes[0].shadow_surface.isl,
1437 .addr = {
1438 .buffer = image->planes[0].address.bo,
1439 .offset = image->planes[0].address.offset +
1440 image->planes[0].shadow_surface.offset,
1441 .mocs = anv_mocs_for_bo(cmd_buffer->device,
1442 image->planes[0].address.bo),
1443 },
1444 };
1445
1446 for (uint32_t l = 0; l < level_count; l++) {
1447 const uint32_t level = base_level + l;
1448
1449 const VkExtent3D extent = {
1450 .width = anv_minify(image->extent.width, level),
1451 .height = anv_minify(image->extent.height, level),
1452 .depth = anv_minify(image->extent.depth, level),
1453 };
1454
1455 if (image->type == VK_IMAGE_TYPE_3D)
1456 layer_count = extent.depth;
1457
1458 for (uint32_t a = 0; a < layer_count; a++) {
1459 const uint32_t layer = base_layer + a;
1460
1461 blorp_copy(&batch, &surf, level, layer,
1462 &shadow_surf, level, layer,
1463 0, 0, 0, 0, extent.width, extent.height);
1464 }
1465 }
1466
1467 blorp_batch_finish(&batch);
1468 }
1469
1470 void
1471 anv_image_clear_color(struct anv_cmd_buffer *cmd_buffer,
1472 const struct anv_image *image,
1473 VkImageAspectFlagBits aspect,
1474 enum isl_aux_usage aux_usage,
1475 enum isl_format format, struct isl_swizzle swizzle,
1476 uint32_t level, uint32_t base_layer, uint32_t layer_count,
1477 VkRect2D area, union isl_color_value clear_color)
1478 {
1479 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1480
1481 /* We don't support planar images with multisampling yet */
1482 assert(image->n_planes == 1);
1483
1484 struct blorp_batch batch;
1485 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1486
1487 struct blorp_surf surf;
1488 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1489 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1490 aux_usage, &surf);
1491 anv_cmd_buffer_mark_image_written(cmd_buffer, image, aspect, aux_usage,
1492 level, base_layer, layer_count);
1493
1494 blorp_clear(&batch, &surf, format, anv_swizzle_for_render(swizzle),
1495 level, base_layer, layer_count,
1496 area.offset.x, area.offset.y,
1497 area.offset.x + area.extent.width,
1498 area.offset.y + area.extent.height,
1499 clear_color, NULL);
1500
1501 blorp_batch_finish(&batch);
1502 }
1503
1504 void
1505 anv_image_clear_depth_stencil(struct anv_cmd_buffer *cmd_buffer,
1506 const struct anv_image *image,
1507 VkImageAspectFlags aspects,
1508 enum isl_aux_usage depth_aux_usage,
1509 uint32_t level,
1510 uint32_t base_layer, uint32_t layer_count,
1511 VkRect2D area,
1512 float depth_value, uint8_t stencil_value)
1513 {
1514 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
1515 VK_IMAGE_ASPECT_STENCIL_BIT));
1516
1517 struct blorp_batch batch;
1518 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1519
1520 struct blorp_surf depth = {};
1521 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1522 get_blorp_surf_for_anv_image(cmd_buffer->device,
1523 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1524 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1525 depth_aux_usage, &depth);
1526 depth.clear_color.f32[0] = ANV_HZ_FC_VAL;
1527 }
1528
1529 struct blorp_surf stencil = {};
1530 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1531 get_blorp_surf_for_anv_image(cmd_buffer->device,
1532 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1533 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1534 ISL_AUX_USAGE_NONE, &stencil);
1535 }
1536
1537 blorp_clear_depth_stencil(&batch, &depth, &stencil,
1538 level, base_layer, layer_count,
1539 area.offset.x, area.offset.y,
1540 area.offset.x + area.extent.width,
1541 area.offset.y + area.extent.height,
1542 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
1543 depth_value,
1544 (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) ? 0xff : 0,
1545 stencil_value);
1546
1547 blorp_batch_finish(&batch);
1548 }
1549
1550 void
1551 anv_image_hiz_op(struct anv_cmd_buffer *cmd_buffer,
1552 const struct anv_image *image,
1553 VkImageAspectFlagBits aspect, uint32_t level,
1554 uint32_t base_layer, uint32_t layer_count,
1555 enum isl_aux_op hiz_op)
1556 {
1557 assert(aspect == VK_IMAGE_ASPECT_DEPTH_BIT);
1558 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, level));
1559 assert(anv_image_aspect_to_plane(image->aspects,
1560 VK_IMAGE_ASPECT_DEPTH_BIT) == 0);
1561
1562 struct blorp_batch batch;
1563 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1564
1565 struct blorp_surf surf;
1566 get_blorp_surf_for_anv_image(cmd_buffer->device,
1567 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1568 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1569 ISL_AUX_USAGE_HIZ, &surf);
1570 surf.clear_color.f32[0] = ANV_HZ_FC_VAL;
1571
1572 blorp_hiz_op(&batch, &surf, level, base_layer, layer_count, hiz_op);
1573
1574 blorp_batch_finish(&batch);
1575 }
1576
1577 void
1578 anv_image_hiz_clear(struct anv_cmd_buffer *cmd_buffer,
1579 const struct anv_image *image,
1580 VkImageAspectFlags aspects,
1581 uint32_t level,
1582 uint32_t base_layer, uint32_t layer_count,
1583 VkRect2D area, uint8_t stencil_value)
1584 {
1585 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
1586 VK_IMAGE_ASPECT_STENCIL_BIT));
1587
1588 struct blorp_batch batch;
1589 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1590
1591 struct blorp_surf depth = {};
1592 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1593 assert(base_layer + layer_count <=
1594 anv_image_aux_layers(image, VK_IMAGE_ASPECT_DEPTH_BIT, level));
1595 get_blorp_surf_for_anv_image(cmd_buffer->device,
1596 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1597 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1598 ISL_AUX_USAGE_HIZ, &depth);
1599 depth.clear_color.f32[0] = ANV_HZ_FC_VAL;
1600 }
1601
1602 struct blorp_surf stencil = {};
1603 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1604 get_blorp_surf_for_anv_image(cmd_buffer->device,
1605 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1606 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1607 ISL_AUX_USAGE_NONE, &stencil);
1608 }
1609
1610 /* From the Sky Lake PRM Volume 7, "Depth Buffer Clear":
1611 *
1612 * "The following is required when performing a depth buffer clear with
1613 * using the WM_STATE or 3DSTATE_WM:
1614 *
1615 * * If other rendering operations have preceded this clear, a
1616 * PIPE_CONTROL with depth cache flush enabled, Depth Stall bit
1617 * enabled must be issued before the rectangle primitive used for
1618 * the depth buffer clear operation.
1619 * * [...]"
1620 *
1621 * Even though the PRM only says that this is required if using 3DSTATE_WM
1622 * and a 3DPRIMITIVE, the GPU appears to also need this to avoid occasional
1623 * hangs when doing a clear with WM_HZ_OP.
1624 */
1625 cmd_buffer->state.pending_pipe_bits |=
1626 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT;
1627
1628 blorp_hiz_clear_depth_stencil(&batch, &depth, &stencil,
1629 level, base_layer, layer_count,
1630 area.offset.x, area.offset.y,
1631 area.offset.x + area.extent.width,
1632 area.offset.y + area.extent.height,
1633 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
1634 ANV_HZ_FC_VAL,
1635 aspects & VK_IMAGE_ASPECT_STENCIL_BIT,
1636 stencil_value);
1637
1638 blorp_batch_finish(&batch);
1639
1640 /* From the SKL PRM, Depth Buffer Clear:
1641 *
1642 * "Depth Buffer Clear Workaround
1643 *
1644 * Depth buffer clear pass using any of the methods (WM_STATE,
1645 * 3DSTATE_WM or 3DSTATE_WM_HZ_OP) must be followed by a PIPE_CONTROL
1646 * command with DEPTH_STALL bit and Depth FLUSH bits “set” before
1647 * starting to render. DepthStall and DepthFlush are not needed between
1648 * consecutive depth clear passes nor is it required if the depth-clear
1649 * pass was done with “full_surf_clear” bit set in the
1650 * 3DSTATE_WM_HZ_OP."
1651 *
1652 * Even though the PRM provides a bunch of conditions under which this is
1653 * supposedly unnecessary, we choose to perform the flush unconditionally
1654 * just to be safe.
1655 */
1656 cmd_buffer->state.pending_pipe_bits |=
1657 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT;
1658 }
1659
1660 void
1661 anv_image_mcs_op(struct anv_cmd_buffer *cmd_buffer,
1662 const struct anv_image *image,
1663 enum isl_format format,
1664 VkImageAspectFlagBits aspect,
1665 uint32_t base_layer, uint32_t layer_count,
1666 enum isl_aux_op mcs_op, union isl_color_value *clear_value,
1667 bool predicate)
1668 {
1669 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1670 assert(image->samples > 1);
1671 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, 0));
1672
1673 /* Multisampling with multi-planar formats is not supported */
1674 assert(image->n_planes == 1);
1675
1676 struct blorp_batch batch;
1677 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1678 predicate ? BLORP_BATCH_PREDICATE_ENABLE : 0);
1679
1680 struct blorp_surf surf;
1681 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1682 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1683 ISL_AUX_USAGE_MCS, &surf);
1684
1685 /* Blorp will store the clear color for us if we provide the clear color
1686 * address and we are doing a fast clear. So we save the clear value into
1687 * the blorp surface. However, in some situations we want to do a fast clear
1688 * without changing the clear value stored in the state buffer. For those
1689 * cases, we set the clear color address pointer to NULL, so blorp will not
1690 * try to store a garbage color.
1691 */
1692 if (mcs_op == ISL_AUX_OP_FAST_CLEAR) {
1693 if (clear_value)
1694 surf.clear_color = *clear_value;
1695 else
1696 surf.clear_color_addr.buffer = NULL;
1697 }
1698
1699 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1700 *
1701 * "After Render target fast clear, pipe-control with color cache
1702 * write-flush must be issued before sending any DRAW commands on
1703 * that render target."
1704 *
1705 * This comment is a bit cryptic and doesn't really tell you what's going
1706 * or what's really needed. It appears that fast clear ops are not
1707 * properly synchronized with other drawing. This means that we cannot
1708 * have a fast clear operation in the pipe at the same time as other
1709 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
1710 * that the contents of the previous draw hit the render target before we
1711 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
1712 * that it is completed before any additional drawing occurs.
1713 */
1714 cmd_buffer->state.pending_pipe_bits |=
1715 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1716
1717 switch (mcs_op) {
1718 case ISL_AUX_OP_FAST_CLEAR:
1719 blorp_fast_clear(&batch, &surf, format,
1720 0, base_layer, layer_count,
1721 0, 0, image->extent.width, image->extent.height);
1722 break;
1723 case ISL_AUX_OP_PARTIAL_RESOLVE:
1724 blorp_mcs_partial_resolve(&batch, &surf, format,
1725 base_layer, layer_count);
1726 break;
1727 case ISL_AUX_OP_FULL_RESOLVE:
1728 case ISL_AUX_OP_AMBIGUATE:
1729 default:
1730 unreachable("Unsupported MCS operation");
1731 }
1732
1733 cmd_buffer->state.pending_pipe_bits |=
1734 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1735
1736 blorp_batch_finish(&batch);
1737 }
1738
1739 void
1740 anv_image_ccs_op(struct anv_cmd_buffer *cmd_buffer,
1741 const struct anv_image *image,
1742 enum isl_format format,
1743 VkImageAspectFlagBits aspect, uint32_t level,
1744 uint32_t base_layer, uint32_t layer_count,
1745 enum isl_aux_op ccs_op, union isl_color_value *clear_value,
1746 bool predicate)
1747 {
1748 assert(image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
1749 assert(image->samples == 1);
1750 assert(level < anv_image_aux_levels(image, aspect));
1751 /* Multi-LOD YcBcR is not allowed */
1752 assert(image->n_planes == 1 || level == 0);
1753 assert(base_layer + layer_count <=
1754 anv_image_aux_layers(image, aspect, level));
1755
1756 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
1757 uint32_t width_div = image->format->planes[plane].denominator_scales[0];
1758 uint32_t height_div = image->format->planes[plane].denominator_scales[1];
1759 uint32_t level_width = anv_minify(image->extent.width, level) / width_div;
1760 uint32_t level_height = anv_minify(image->extent.height, level) / height_div;
1761
1762 struct blorp_batch batch;
1763 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1764 predicate ? BLORP_BATCH_PREDICATE_ENABLE : 0);
1765
1766 struct blorp_surf surf;
1767 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1768 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1769 fast_clear_aux_usage(image, aspect),
1770 &surf);
1771
1772 /* Blorp will store the clear color for us if we provide the clear color
1773 * address and we are doing a fast clear. So we save the clear value into
1774 * the blorp surface. However, in some situations we want to do a fast clear
1775 * without changing the clear value stored in the state buffer. For those
1776 * cases, we set the clear color address pointer to NULL, so blorp will not
1777 * try to store a garbage color.
1778 */
1779 if (ccs_op == ISL_AUX_OP_FAST_CLEAR) {
1780 if (clear_value)
1781 surf.clear_color = *clear_value;
1782 else
1783 surf.clear_color_addr.buffer = NULL;
1784 }
1785
1786 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1787 *
1788 * "After Render target fast clear, pipe-control with color cache
1789 * write-flush must be issued before sending any DRAW commands on
1790 * that render target."
1791 *
1792 * This comment is a bit cryptic and doesn't really tell you what's going
1793 * or what's really needed. It appears that fast clear ops are not
1794 * properly synchronized with other drawing. This means that we cannot
1795 * have a fast clear operation in the pipe at the same time as other
1796 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
1797 * that the contents of the previous draw hit the render target before we
1798 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
1799 * that it is completed before any additional drawing occurs.
1800 */
1801 cmd_buffer->state.pending_pipe_bits |=
1802 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1803
1804 switch (ccs_op) {
1805 case ISL_AUX_OP_FAST_CLEAR:
1806 blorp_fast_clear(&batch, &surf, format,
1807 level, base_layer, layer_count,
1808 0, 0, level_width, level_height);
1809 break;
1810 case ISL_AUX_OP_FULL_RESOLVE:
1811 case ISL_AUX_OP_PARTIAL_RESOLVE:
1812 blorp_ccs_resolve(&batch, &surf, level, base_layer, layer_count,
1813 format, ccs_op);
1814 break;
1815 case ISL_AUX_OP_AMBIGUATE:
1816 for (uint32_t a = 0; a < layer_count; a++) {
1817 const uint32_t layer = base_layer + a;
1818 blorp_ccs_ambiguate(&batch, &surf, level, layer);
1819 }
1820 break;
1821 default:
1822 unreachable("Unsupported CCS operation");
1823 }
1824
1825 cmd_buffer->state.pending_pipe_bits |=
1826 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1827
1828 blorp_batch_finish(&batch);
1829 }