projects / mesa.git / blob
? search:


7d0ded3423ca7d0bb750855ed0e63056142eeb91
[mesa.git] / src / intel / vulkan / anv_blorp.c
1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "anv_private.h"
25
26 static bool
27 lookup_blorp_shader(struct blorp_context *blorp,
28 const void *key, uint32_t key_size,
29 uint32_t *kernel_out, void *prog_data_out)
30 {
31 struct anv_device *device = blorp->driver_ctx;
32
33 /* The blorp cache must be a real cache */
34 assert(device->blorp_shader_cache.cache);
35
36 struct anv_shader_bin *bin =
37 anv_pipeline_cache_search(&device->blorp_shader_cache, key, key_size);
38 if (!bin)
39 return false;
40
41 /* The cache already has a reference and it's not going anywhere so there
42 * is no need to hold a second reference.
43 */
44 anv_shader_bin_unref(device, bin);
45
46 *kernel_out = bin->kernel.offset;
47 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
48
49 return true;
50 }
51
52 static bool
53 upload_blorp_shader(struct blorp_context *blorp,
54 const void *key, uint32_t key_size,
55 const void *kernel, uint32_t kernel_size,
56 const struct brw_stage_prog_data *prog_data,
57 uint32_t prog_data_size,
58 uint32_t *kernel_out, void *prog_data_out)
59 {
60 struct anv_device *device = blorp->driver_ctx;
61
62 /* The blorp cache must be a real cache */
63 assert(device->blorp_shader_cache.cache);
64
65 struct anv_pipeline_bind_map bind_map = {
66 .surface_count = 0,
67 .sampler_count = 0,
68 };
69
70 struct anv_shader_bin *bin =
71 anv_pipeline_cache_upload_kernel(&device->blorp_shader_cache,
72 key, key_size, kernel, kernel_size,
73 prog_data, prog_data_size, &bind_map);
74
75 if (!bin)
76 return false;
77
78 /* The cache already has a reference and it's not going anywhere so there
79 * is no need to hold a second reference.
80 */
81 anv_shader_bin_unref(device, bin);
82
83 *kernel_out = bin->kernel.offset;
84 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
85
86 return true;
87 }
88
89 void
90 anv_device_init_blorp(struct anv_device *device)
91 {
92 anv_pipeline_cache_init(&device->blorp_shader_cache, device, true);
93 blorp_init(&device->blorp, device, &device->isl_dev);
94 device->blorp.compiler = device->instance->physicalDevice.compiler;
95 device->blorp.lookup_shader = lookup_blorp_shader;
96 device->blorp.upload_shader = upload_blorp_shader;
97 switch (device->info.gen) {
98 case 7:
99 if (device->info.is_haswell) {
100 device->blorp.exec = gen75_blorp_exec;
101 } else {
102 device->blorp.exec = gen7_blorp_exec;
103 }
104 break;
105 case 8:
106 device->blorp.exec = gen8_blorp_exec;
107 break;
108 case 9:
109 device->blorp.exec = gen9_blorp_exec;
110 break;
111 case 10:
112 device->blorp.exec = gen10_blorp_exec;
113 break;
114 default:
115 unreachable("Unknown hardware generation");
116 }
117 }
118
119 void
120 anv_device_finish_blorp(struct anv_device *device)
121 {
122 blorp_finish(&device->blorp);
123 anv_pipeline_cache_finish(&device->blorp_shader_cache);
124 }
125
126 static void
127 get_blorp_surf_for_anv_buffer(struct anv_device *device,
128 struct anv_buffer *buffer, uint64_t offset,
129 uint32_t width, uint32_t height,
130 uint32_t row_pitch, enum isl_format format,
131 struct blorp_surf *blorp_surf,
132 struct isl_surf *isl_surf)
133 {
134 const struct isl_format_layout *fmtl =
135 isl_format_get_layout(format);
136 bool ok UNUSED;
137
138 /* ASTC is the only format which doesn't support linear layouts.
139 * Create an equivalently sized surface with ISL to get around this.
140 */
141 if (fmtl->txc == ISL_TXC_ASTC) {
142 /* Use an equivalently sized format */
143 format = ISL_FORMAT_R32G32B32A32_UINT;
144 assert(fmtl->bpb == isl_format_get_layout(format)->bpb);
145
146 /* Shrink the dimensions for the new format */
147 width = DIV_ROUND_UP(width, fmtl->bw);
148 height = DIV_ROUND_UP(height, fmtl->bh);
149 }
150
151 *blorp_surf = (struct blorp_surf) {
152 .surf = isl_surf,
153 .addr = {
154 .buffer = buffer->bo,
155 .offset = buffer->offset + offset,
156 .mocs = device->default_mocs,
157 },
158 };
159
160 ok = isl_surf_init(&device->isl_dev, isl_surf,
161 .dim = ISL_SURF_DIM_2D,
162 .format = format,
163 .width = width,
164 .height = height,
165 .depth = 1,
166 .levels = 1,
167 .array_len = 1,
168 .samples = 1,
169 .row_pitch = row_pitch,
170 .usage = ISL_SURF_USAGE_TEXTURE_BIT |
171 ISL_SURF_USAGE_RENDER_TARGET_BIT,
172 .tiling_flags = ISL_TILING_LINEAR_BIT);
173 assert(ok);
174 }
175
176 #define ANV_AUX_USAGE_DEFAULT ((enum isl_aux_usage)0xff)
177
178 static struct blorp_address
179 anv_to_blorp_address(struct anv_address addr)
180 {
181 return (struct blorp_address) {
182 .buffer = addr.bo,
183 .offset = addr.offset,
184 };
185 }
186
187 static void
188 get_blorp_surf_for_anv_image(const struct anv_device *device,
189 const struct anv_image *image,
190 VkImageAspectFlags aspect,
191 enum isl_aux_usage aux_usage,
192 struct blorp_surf *blorp_surf)
193 {
194 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
195
196 if (aux_usage == ANV_AUX_USAGE_DEFAULT) {
197 aux_usage = image->planes[plane].aux_usage;
198
199 /* Blorp copies and blits can't handle HiZ so disable it by default */
200 if (aux_usage == ISL_AUX_USAGE_HIZ)
201 aux_usage = ISL_AUX_USAGE_NONE;
202 }
203
204 const struct anv_surface *surface = &image->planes[plane].surface;
205 *blorp_surf = (struct blorp_surf) {
206 .surf = &surface->isl,
207 .addr = {
208 .buffer = image->planes[plane].bo,
209 .offset = image->planes[plane].bo_offset + surface->offset,
210 .mocs = device->default_mocs,
211 },
212 };
213
214 if (aux_usage != ISL_AUX_USAGE_NONE) {
215 const struct anv_surface *aux_surface = &image->planes[plane].aux_surface;
216 blorp_surf->aux_surf = &aux_surface->isl,
217 blorp_surf->aux_addr = (struct blorp_address) {
218 .buffer = image->planes[plane].bo,
219 .offset = image->planes[plane].bo_offset + aux_surface->offset,
220 .mocs = device->default_mocs,
221 };
222 blorp_surf->aux_usage = aux_usage;
223 }
224 }
225
226 void anv_CmdCopyImage(
227 VkCommandBuffer commandBuffer,
228 VkImage srcImage,
229 VkImageLayout srcImageLayout,
230 VkImage dstImage,
231 VkImageLayout dstImageLayout,
232 uint32_t regionCount,
233 const VkImageCopy* pRegions)
234 {
235 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
236 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
237 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
238
239 struct blorp_batch batch;
240 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
241
242 for (unsigned r = 0; r < regionCount; r++) {
243 VkOffset3D srcOffset =
244 anv_sanitize_image_offset(src_image->type, pRegions[r].srcOffset);
245 VkOffset3D dstOffset =
246 anv_sanitize_image_offset(dst_image->type, pRegions[r].dstOffset);
247 VkExtent3D extent =
248 anv_sanitize_image_extent(src_image->type, pRegions[r].extent);
249
250 const uint32_t dst_level = pRegions[r].dstSubresource.mipLevel;
251 unsigned dst_base_layer, layer_count;
252 if (dst_image->type == VK_IMAGE_TYPE_3D) {
253 dst_base_layer = pRegions[r].dstOffset.z;
254 layer_count = pRegions[r].extent.depth;
255 } else {
256 dst_base_layer = pRegions[r].dstSubresource.baseArrayLayer;
257 layer_count =
258 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource);
259 }
260
261 const uint32_t src_level = pRegions[r].srcSubresource.mipLevel;
262 unsigned src_base_layer;
263 if (src_image->type == VK_IMAGE_TYPE_3D) {
264 src_base_layer = pRegions[r].srcOffset.z;
265 } else {
266 src_base_layer = pRegions[r].srcSubresource.baseArrayLayer;
267 assert(layer_count ==
268 anv_get_layerCount(src_image, &pRegions[r].srcSubresource));
269 }
270
271 VkImageAspectFlags src_mask = pRegions[r].srcSubresource.aspectMask,
272 dst_mask = pRegions[r].dstSubresource.aspectMask;
273
274 assert(anv_image_aspects_compatible(src_mask, dst_mask));
275
276 if (_mesa_bitcount(src_mask) > 1) {
277 uint32_t aspect_bit;
278 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_mask) {
279 struct blorp_surf src_surf, dst_surf;
280 get_blorp_surf_for_anv_image(cmd_buffer->device,
281 src_image, 1UL << aspect_bit,
282 ANV_AUX_USAGE_DEFAULT, &src_surf);
283 get_blorp_surf_for_anv_image(cmd_buffer->device,
284 dst_image, 1UL << aspect_bit,
285 ANV_AUX_USAGE_DEFAULT, &dst_surf);
286 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
287 1UL << aspect_bit,
288 dst_surf.aux_usage, dst_level,
289 dst_base_layer, layer_count);
290
291 for (unsigned i = 0; i < layer_count; i++) {
292 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
293 &dst_surf, dst_level, dst_base_layer + i,
294 srcOffset.x, srcOffset.y,
295 dstOffset.x, dstOffset.y,
296 extent.width, extent.height);
297 }
298 }
299 } else {
300 struct blorp_surf src_surf, dst_surf;
301 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, src_mask,
302 ANV_AUX_USAGE_DEFAULT, &src_surf);
303 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, dst_mask,
304 ANV_AUX_USAGE_DEFAULT, &dst_surf);
305 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image, dst_mask,
306 dst_surf.aux_usage, dst_level,
307 dst_base_layer, layer_count);
308
309 for (unsigned i = 0; i < layer_count; i++) {
310 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
311 &dst_surf, dst_level, dst_base_layer + i,
312 srcOffset.x, srcOffset.y,
313 dstOffset.x, dstOffset.y,
314 extent.width, extent.height);
315 }
316 }
317 }
318
319 blorp_batch_finish(&batch);
320 }
321
322 static void
323 copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer,
324 struct anv_buffer *anv_buffer,
325 struct anv_image *anv_image,
326 uint32_t regionCount,
327 const VkBufferImageCopy* pRegions,
328 bool buffer_to_image)
329 {
330 struct blorp_batch batch;
331 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
332
333 struct {
334 struct blorp_surf surf;
335 uint32_t level;
336 VkOffset3D offset;
337 } image, buffer, *src, *dst;
338
339 buffer.level = 0;
340 buffer.offset = (VkOffset3D) { 0, 0, 0 };
341
342 if (buffer_to_image) {
343 src = &buffer;
344 dst = &image;
345 } else {
346 src = &image;
347 dst = &buffer;
348 }
349
350 for (unsigned r = 0; r < regionCount; r++) {
351 const VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask;
352
353 get_blorp_surf_for_anv_image(cmd_buffer->device, anv_image, aspect,
354 ANV_AUX_USAGE_DEFAULT, &image.surf);
355 image.offset =
356 anv_sanitize_image_offset(anv_image->type, pRegions[r].imageOffset);
357 image.level = pRegions[r].imageSubresource.mipLevel;
358
359 VkExtent3D extent =
360 anv_sanitize_image_extent(anv_image->type, pRegions[r].imageExtent);
361 if (anv_image->type != VK_IMAGE_TYPE_3D) {
362 image.offset.z = pRegions[r].imageSubresource.baseArrayLayer;
363 extent.depth =
364 anv_get_layerCount(anv_image, &pRegions[r].imageSubresource);
365 }
366
367 const enum isl_format buffer_format =
368 anv_get_isl_format(&cmd_buffer->device->info, anv_image->vk_format,
369 aspect, VK_IMAGE_TILING_LINEAR);
370
371 const VkExtent3D bufferImageExtent = {
372 .width = pRegions[r].bufferRowLength ?
373 pRegions[r].bufferRowLength : extent.width,
374 .height = pRegions[r].bufferImageHeight ?
375 pRegions[r].bufferImageHeight : extent.height,
376 };
377
378 const struct isl_format_layout *buffer_fmtl =
379 isl_format_get_layout(buffer_format);
380
381 const uint32_t buffer_row_pitch =
382 DIV_ROUND_UP(bufferImageExtent.width, buffer_fmtl->bw) *
383 (buffer_fmtl->bpb / 8);
384
385 const uint32_t buffer_layer_stride =
386 DIV_ROUND_UP(bufferImageExtent.height, buffer_fmtl->bh) *
387 buffer_row_pitch;
388
389 struct isl_surf buffer_isl_surf;
390 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
391 anv_buffer, pRegions[r].bufferOffset,
392 extent.width, extent.height,
393 buffer_row_pitch, buffer_format,
394 &buffer.surf, &buffer_isl_surf);
395
396 if (&image == dst) {
397 anv_cmd_buffer_mark_image_written(cmd_buffer, anv_image,
398 aspect, dst->surf.aux_usage,
399 dst->level,
400 dst->offset.z, extent.depth);
401 }
402
403 for (unsigned z = 0; z < extent.depth; z++) {
404 blorp_copy(&batch, &src->surf, src->level, src->offset.z,
405 &dst->surf, dst->level, dst->offset.z,
406 src->offset.x, src->offset.y, dst->offset.x, dst->offset.y,
407 extent.width, extent.height);
408
409 image.offset.z++;
410 buffer.surf.addr.offset += buffer_layer_stride;
411 }
412 }
413
414 blorp_batch_finish(&batch);
415 }
416
417 void anv_CmdCopyBufferToImage(
418 VkCommandBuffer commandBuffer,
419 VkBuffer srcBuffer,
420 VkImage dstImage,
421 VkImageLayout dstImageLayout,
422 uint32_t regionCount,
423 const VkBufferImageCopy* pRegions)
424 {
425 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
426 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
427 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
428
429 copy_buffer_to_image(cmd_buffer, src_buffer, dst_image,
430 regionCount, pRegions, true);
431 }
432
433 void anv_CmdCopyImageToBuffer(
434 VkCommandBuffer commandBuffer,
435 VkImage srcImage,
436 VkImageLayout srcImageLayout,
437 VkBuffer dstBuffer,
438 uint32_t regionCount,
439 const VkBufferImageCopy* pRegions)
440 {
441 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
442 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
443 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
444
445 copy_buffer_to_image(cmd_buffer, dst_buffer, src_image,
446 regionCount, pRegions, false);
447 }
448
449 static bool
450 flip_coords(unsigned *src0, unsigned *src1, unsigned *dst0, unsigned *dst1)
451 {
452 bool flip = false;
453 if (*src0 > *src1) {
454 unsigned tmp = *src0;
455 *src0 = *src1;
456 *src1 = tmp;
457 flip = !flip;
458 }
459
460 if (*dst0 > *dst1) {
461 unsigned tmp = *dst0;
462 *dst0 = *dst1;
463 *dst1 = tmp;
464 flip = !flip;
465 }
466
467 return flip;
468 }
469
470 void anv_CmdBlitImage(
471 VkCommandBuffer commandBuffer,
472 VkImage srcImage,
473 VkImageLayout srcImageLayout,
474 VkImage dstImage,
475 VkImageLayout dstImageLayout,
476 uint32_t regionCount,
477 const VkImageBlit* pRegions,
478 VkFilter filter)
479
480 {
481 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
482 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
483 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
484
485 struct blorp_surf src, dst;
486
487 uint32_t gl_filter;
488 switch (filter) {
489 case VK_FILTER_NEAREST:
490 gl_filter = 0x2600; /* GL_NEAREST */
491 break;
492 case VK_FILTER_LINEAR:
493 gl_filter = 0x2601; /* GL_LINEAR */
494 break;
495 default:
496 unreachable("Invalid filter");
497 }
498
499 struct blorp_batch batch;
500 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
501
502 for (unsigned r = 0; r < regionCount; r++) {
503 const VkImageSubresourceLayers *src_res = &pRegions[r].srcSubresource;
504 const VkImageSubresourceLayers *dst_res = &pRegions[r].dstSubresource;
505
506 get_blorp_surf_for_anv_image(cmd_buffer->device,
507 src_image, src_res->aspectMask,
508 ANV_AUX_USAGE_DEFAULT, &src);
509 get_blorp_surf_for_anv_image(cmd_buffer->device,
510 dst_image, dst_res->aspectMask,
511 ANV_AUX_USAGE_DEFAULT, &dst);
512
513 struct anv_format_plane src_format =
514 anv_get_format_plane(&cmd_buffer->device->info, src_image->vk_format,
515 src_res->aspectMask, src_image->tiling);
516 struct anv_format_plane dst_format =
517 anv_get_format_plane(&cmd_buffer->device->info, dst_image->vk_format,
518 dst_res->aspectMask, dst_image->tiling);
519
520 unsigned dst_start, dst_end;
521 if (dst_image->type == VK_IMAGE_TYPE_3D) {
522 assert(dst_res->baseArrayLayer == 0);
523 dst_start = pRegions[r].dstOffsets[0].z;
524 dst_end = pRegions[r].dstOffsets[1].z;
525 } else {
526 dst_start = dst_res->baseArrayLayer;
527 dst_end = dst_start + anv_get_layerCount(dst_image, dst_res);
528 }
529
530 unsigned src_start, src_end;
531 if (src_image->type == VK_IMAGE_TYPE_3D) {
532 assert(src_res->baseArrayLayer == 0);
533 src_start = pRegions[r].srcOffsets[0].z;
534 src_end = pRegions[r].srcOffsets[1].z;
535 } else {
536 src_start = src_res->baseArrayLayer;
537 src_end = src_start + anv_get_layerCount(src_image, src_res);
538 }
539
540 bool flip_z = flip_coords(&src_start, &src_end, &dst_start, &dst_end);
541 float src_z_step = (float)(src_end + 1 - src_start) /
542 (float)(dst_end + 1 - dst_start);
543
544 if (flip_z) {
545 src_start = src_end;
546 src_z_step *= -1;
547 }
548
549 unsigned src_x0 = pRegions[r].srcOffsets[0].x;
550 unsigned src_x1 = pRegions[r].srcOffsets[1].x;
551 unsigned dst_x0 = pRegions[r].dstOffsets[0].x;
552 unsigned dst_x1 = pRegions[r].dstOffsets[1].x;
553 bool flip_x = flip_coords(&src_x0, &src_x1, &dst_x0, &dst_x1);
554
555 unsigned src_y0 = pRegions[r].srcOffsets[0].y;
556 unsigned src_y1 = pRegions[r].srcOffsets[1].y;
557 unsigned dst_y0 = pRegions[r].dstOffsets[0].y;
558 unsigned dst_y1 = pRegions[r].dstOffsets[1].y;
559 bool flip_y = flip_coords(&src_y0, &src_y1, &dst_y0, &dst_y1);
560
561 const unsigned num_layers = dst_end - dst_start;
562 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
563 dst_res->aspectMask,
564 dst.aux_usage,
565 dst_res->mipLevel,
566 dst_start, num_layers);
567
568 for (unsigned i = 0; i < num_layers; i++) {
569 unsigned dst_z = dst_start + i;
570 unsigned src_z = src_start + i * src_z_step;
571
572 blorp_blit(&batch, &src, src_res->mipLevel, src_z,
573 src_format.isl_format, src_format.swizzle,
574 &dst, dst_res->mipLevel, dst_z,
575 dst_format.isl_format,
576 anv_swizzle_for_render(dst_format.swizzle),
577 src_x0, src_y0, src_x1, src_y1,
578 dst_x0, dst_y0, dst_x1, dst_y1,
579 gl_filter, flip_x, flip_y);
580 }
581
582 }
583
584 blorp_batch_finish(&batch);
585 }
586
587 static enum isl_format
588 isl_format_for_size(unsigned size_B)
589 {
590 switch (size_B) {
591 case 4: return ISL_FORMAT_R32_UINT;
592 case 8: return ISL_FORMAT_R32G32_UINT;
593 case 16: return ISL_FORMAT_R32G32B32A32_UINT;
594 default:
595 unreachable("Not a power-of-two format size");
596 }
597 }
598
599 /**
600 * Returns the greatest common divisor of a and b that is a power of two.
601 */
602 static uint64_t
603 gcd_pow2_u64(uint64_t a, uint64_t b)
604 {
605 assert(a > 0 || b > 0);
606
607 unsigned a_log2 = ffsll(a) - 1;
608 unsigned b_log2 = ffsll(b) - 1;
609
610 /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which
611 * case, the MIN2() will take the other one. If both are 0 then we will
612 * hit the assert above.
613 */
614 return 1 << MIN2(a_log2, b_log2);
615 }
616
617 /* This is maximum possible width/height our HW can handle */
618 #define MAX_SURFACE_DIM (1ull << 14)
619
620 void anv_CmdCopyBuffer(
621 VkCommandBuffer commandBuffer,
622 VkBuffer srcBuffer,
623 VkBuffer dstBuffer,
624 uint32_t regionCount,
625 const VkBufferCopy* pRegions)
626 {
627 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
628 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
629 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
630
631 struct blorp_batch batch;
632 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
633
634 for (unsigned r = 0; r < regionCount; r++) {
635 struct blorp_address src = {
636 .buffer = src_buffer->bo,
637 .offset = src_buffer->offset + pRegions[r].srcOffset,
638 .mocs = cmd_buffer->device->default_mocs,
639 };
640 struct blorp_address dst = {
641 .buffer = dst_buffer->bo,
642 .offset = dst_buffer->offset + pRegions[r].dstOffset,
643 .mocs = cmd_buffer->device->default_mocs,
644 };
645
646 blorp_buffer_copy(&batch, src, dst, pRegions[r].size);
647 }
648
649 blorp_batch_finish(&batch);
650 }
651
652 void anv_CmdUpdateBuffer(
653 VkCommandBuffer commandBuffer,
654 VkBuffer dstBuffer,
655 VkDeviceSize dstOffset,
656 VkDeviceSize dataSize,
657 const void* pData)
658 {
659 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
660 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
661
662 struct blorp_batch batch;
663 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
664
665 /* We can't quite grab a full block because the state stream needs a
666 * little data at the top to build its linked list.
667 */
668 const uint32_t max_update_size =
669 cmd_buffer->device->dynamic_state_pool.block_size - 64;
670
671 assert(max_update_size < MAX_SURFACE_DIM * 4);
672
673 /* We're about to read data that was written from the CPU. Flush the
674 * texture cache so we don't get anything stale.
675 */
676 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
677
678 while (dataSize) {
679 const uint32_t copy_size = MIN2(dataSize, max_update_size);
680
681 struct anv_state tmp_data =
682 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, copy_size, 64);
683
684 memcpy(tmp_data.map, pData, copy_size);
685
686 anv_state_flush(cmd_buffer->device, tmp_data);
687
688 struct blorp_address src = {
689 .buffer = &cmd_buffer->device->dynamic_state_pool.block_pool.bo,
690 .offset = tmp_data.offset,
691 .mocs = cmd_buffer->device->default_mocs,
692 };
693 struct blorp_address dst = {
694 .buffer = dst_buffer->bo,
695 .offset = dst_buffer->offset + dstOffset,
696 .mocs = cmd_buffer->device->default_mocs,
697 };
698
699 blorp_buffer_copy(&batch, src, dst, copy_size);
700
701 dataSize -= copy_size;
702 dstOffset += copy_size;
703 pData = (void *)pData + copy_size;
704 }
705
706 blorp_batch_finish(&batch);
707 }
708
709 void anv_CmdFillBuffer(
710 VkCommandBuffer commandBuffer,
711 VkBuffer dstBuffer,
712 VkDeviceSize dstOffset,
713 VkDeviceSize fillSize,
714 uint32_t data)
715 {
716 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
717 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
718 struct blorp_surf surf;
719 struct isl_surf isl_surf;
720
721 struct blorp_batch batch;
722 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
723
724 fillSize = anv_buffer_get_range(dst_buffer, dstOffset, fillSize);
725
726 /* From the Vulkan spec:
727 *
728 * "size is the number of bytes to fill, and must be either a multiple
729 * of 4, or VK_WHOLE_SIZE to fill the range from offset to the end of
730 * the buffer. If VK_WHOLE_SIZE is used and the remaining size of the
731 * buffer is not a multiple of 4, then the nearest smaller multiple is
732 * used."
733 */
734 fillSize &= ~3ull;
735
736 /* First, we compute the biggest format that can be used with the
737 * given offsets and size.
738 */
739 int bs = 16;
740 bs = gcd_pow2_u64(bs, dstOffset);
741 bs = gcd_pow2_u64(bs, fillSize);
742 enum isl_format isl_format = isl_format_for_size(bs);
743
744 union isl_color_value color = {
745 .u32 = { data, data, data, data },
746 };
747
748 const uint64_t max_fill_size = MAX_SURFACE_DIM * MAX_SURFACE_DIM * bs;
749 while (fillSize >= max_fill_size) {
750 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
751 dst_buffer, dstOffset,
752 MAX_SURFACE_DIM, MAX_SURFACE_DIM,
753 MAX_SURFACE_DIM * bs, isl_format,
754 &surf, &isl_surf);
755
756 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
757 0, 0, 1, 0, 0, MAX_SURFACE_DIM, MAX_SURFACE_DIM,
758 color, NULL);
759 fillSize -= max_fill_size;
760 dstOffset += max_fill_size;
761 }
762
763 uint64_t height = fillSize / (MAX_SURFACE_DIM * bs);
764 assert(height < MAX_SURFACE_DIM);
765 if (height != 0) {
766 const uint64_t rect_fill_size = height * MAX_SURFACE_DIM * bs;
767 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
768 dst_buffer, dstOffset,
769 MAX_SURFACE_DIM, height,
770 MAX_SURFACE_DIM * bs, isl_format,
771 &surf, &isl_surf);
772
773 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
774 0, 0, 1, 0, 0, MAX_SURFACE_DIM, height,
775 color, NULL);
776 fillSize -= rect_fill_size;
777 dstOffset += rect_fill_size;
778 }
779
780 if (fillSize != 0) {
781 const uint32_t width = fillSize / bs;
782 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
783 dst_buffer, dstOffset,
784 width, 1,
785 width * bs, isl_format,
786 &surf, &isl_surf);
787
788 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
789 0, 0, 1, 0, 0, width, 1,
790 color, NULL);
791 }
792
793 blorp_batch_finish(&batch);
794 }
795
796 void anv_CmdClearColorImage(
797 VkCommandBuffer commandBuffer,
798 VkImage _image,
799 VkImageLayout imageLayout,
800 const VkClearColorValue* pColor,
801 uint32_t rangeCount,
802 const VkImageSubresourceRange* pRanges)
803 {
804 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
805 ANV_FROM_HANDLE(anv_image, image, _image);
806
807 static const bool color_write_disable[4] = { false, false, false, false };
808
809 struct blorp_batch batch;
810 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
811
812
813 for (unsigned r = 0; r < rangeCount; r++) {
814 if (pRanges[r].aspectMask == 0)
815 continue;
816
817 assert(pRanges[r].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
818
819 struct blorp_surf surf;
820 get_blorp_surf_for_anv_image(cmd_buffer->device,
821 image, pRanges[r].aspectMask,
822 ANV_AUX_USAGE_DEFAULT, &surf);
823
824 struct anv_format_plane src_format =
825 anv_get_format_plane(&cmd_buffer->device->info, image->vk_format,
826 VK_IMAGE_ASPECT_COLOR_BIT, image->tiling);
827
828 unsigned base_layer = pRanges[r].baseArrayLayer;
829 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
830
831 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
832 const unsigned level = pRanges[r].baseMipLevel + i;
833 const unsigned level_width = anv_minify(image->extent.width, level);
834 const unsigned level_height = anv_minify(image->extent.height, level);
835
836 if (image->type == VK_IMAGE_TYPE_3D) {
837 base_layer = 0;
838 layer_count = anv_minify(image->extent.depth, level);
839 }
840
841 anv_cmd_buffer_mark_image_written(cmd_buffer, image,
842 pRanges[r].aspectMask,
843 surf.aux_usage, level,
844 base_layer, layer_count);
845
846 blorp_clear(&batch, &surf,
847 src_format.isl_format, src_format.swizzle,
848 level, base_layer, layer_count,
849 0, 0, level_width, level_height,
850 vk_to_isl_color(*pColor), color_write_disable);
851 }
852 }
853
854 blorp_batch_finish(&batch);
855 }
856
857 void anv_CmdClearDepthStencilImage(
858 VkCommandBuffer commandBuffer,
859 VkImage image_h,
860 VkImageLayout imageLayout,
861 const VkClearDepthStencilValue* pDepthStencil,
862 uint32_t rangeCount,
863 const VkImageSubresourceRange* pRanges)
864 {
865 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
866 ANV_FROM_HANDLE(anv_image, image, image_h);
867
868 struct blorp_batch batch;
869 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
870
871 struct blorp_surf depth, stencil;
872 if (image->aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
873 get_blorp_surf_for_anv_image(cmd_buffer->device,
874 image, VK_IMAGE_ASPECT_DEPTH_BIT,
875 ISL_AUX_USAGE_NONE, &depth);
876 } else {
877 memset(&depth, 0, sizeof(depth));
878 }
879
880 if (image->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
881 get_blorp_surf_for_anv_image(cmd_buffer->device,
882 image, VK_IMAGE_ASPECT_STENCIL_BIT,
883 ISL_AUX_USAGE_NONE, &stencil);
884 } else {
885 memset(&stencil, 0, sizeof(stencil));
886 }
887
888 for (unsigned r = 0; r < rangeCount; r++) {
889 if (pRanges[r].aspectMask == 0)
890 continue;
891
892 bool clear_depth = pRanges[r].aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
893 bool clear_stencil = pRanges[r].aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
894
895 unsigned base_layer = pRanges[r].baseArrayLayer;
896 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
897
898 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
899 const unsigned level = pRanges[r].baseMipLevel + i;
900 const unsigned level_width = anv_minify(image->extent.width, level);
901 const unsigned level_height = anv_minify(image->extent.height, level);
902
903 if (image->type == VK_IMAGE_TYPE_3D)
904 layer_count = anv_minify(image->extent.depth, level);
905
906 blorp_clear_depth_stencil(&batch, &depth, &stencil,
907 level, base_layer, layer_count,
908 0, 0, level_width, level_height,
909 clear_depth, pDepthStencil->depth,
910 clear_stencil ? 0xff : 0,
911 pDepthStencil->stencil);
912 }
913 }
914
915 blorp_batch_finish(&batch);
916 }
917
918 VkResult
919 anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer *cmd_buffer,
920 uint32_t num_entries,
921 uint32_t *state_offset,
922 struct anv_state *bt_state)
923 {
924 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
925 state_offset);
926 if (bt_state->map == NULL) {
927 /* We ran out of space. Grab a new binding table block. */
928 VkResult result = anv_cmd_buffer_new_binding_table_block(cmd_buffer);
929 if (result != VK_SUCCESS)
930 return result;
931
932 /* Re-emit state base addresses so we get the new surface state base
933 * address before we start emitting binding tables etc.
934 */
935 anv_cmd_buffer_emit_state_base_address(cmd_buffer);
936
937 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
938 state_offset);
939 assert(bt_state->map != NULL);
940 }
941
942 return VK_SUCCESS;
943 }
944
945 static VkResult
946 binding_table_for_surface_state(struct anv_cmd_buffer *cmd_buffer,
947 struct anv_state surface_state,
948 uint32_t *bt_offset)
949 {
950 uint32_t state_offset;
951 struct anv_state bt_state;
952
953 VkResult result =
954 anv_cmd_buffer_alloc_blorp_binding_table(cmd_buffer, 1, &state_offset,
955 &bt_state);
956 if (result != VK_SUCCESS)
957 return result;
958
959 uint32_t *bt_map = bt_state.map;
960 bt_map[0] = surface_state.offset + state_offset;
961
962 *bt_offset = bt_state.offset;
963 return VK_SUCCESS;
964 }
965
966 static void
967 clear_color_attachment(struct anv_cmd_buffer *cmd_buffer,
968 struct blorp_batch *batch,
969 const VkClearAttachment *attachment,
970 uint32_t rectCount, const VkClearRect *pRects)
971 {
972 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
973 const uint32_t color_att = attachment->colorAttachment;
974 const uint32_t att_idx = subpass->color_attachments[color_att].attachment;
975
976 if (att_idx == VK_ATTACHMENT_UNUSED)
977 return;
978
979 struct anv_render_pass_attachment *pass_att =
980 &cmd_buffer->state.pass->attachments[att_idx];
981 struct anv_attachment_state *att_state =
982 &cmd_buffer->state.attachments[att_idx];
983
984 uint32_t binding_table;
985 VkResult result =
986 binding_table_for_surface_state(cmd_buffer, att_state->color.state,
987 &binding_table);
988 if (result != VK_SUCCESS)
989 return;
990
991 union isl_color_value clear_color =
992 vk_to_isl_color(attachment->clearValue.color);
993
994 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
995 if (subpass->view_mask) {
996 uint32_t view_idx;
997 for_each_bit(view_idx, subpass->view_mask) {
998 for (uint32_t r = 0; r < rectCount; ++r) {
999 const VkOffset2D offset = pRects[r].rect.offset;
1000 const VkExtent2D extent = pRects[r].rect.extent;
1001 blorp_clear_attachments(batch, binding_table,
1002 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1003 view_idx, 1,
1004 offset.x, offset.y,
1005 offset.x + extent.width,
1006 offset.y + extent.height,
1007 true, clear_color, false, 0.0f, 0, 0);
1008 }
1009 }
1010 return;
1011 }
1012
1013 for (uint32_t r = 0; r < rectCount; ++r) {
1014 const VkOffset2D offset = pRects[r].rect.offset;
1015 const VkExtent2D extent = pRects[r].rect.extent;
1016 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1017 blorp_clear_attachments(batch, binding_table,
1018 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1019 pRects[r].baseArrayLayer,
1020 pRects[r].layerCount,
1021 offset.x, offset.y,
1022 offset.x + extent.width, offset.y + extent.height,
1023 true, clear_color, false, 0.0f, 0, 0);
1024 }
1025 }
1026
1027 static void
1028 clear_depth_stencil_attachment(struct anv_cmd_buffer *cmd_buffer,
1029 struct blorp_batch *batch,
1030 const VkClearAttachment *attachment,
1031 uint32_t rectCount, const VkClearRect *pRects)
1032 {
1033 static const union isl_color_value color_value = { .u32 = { 0, } };
1034 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
1035 const uint32_t att_idx = subpass->depth_stencil_attachment.attachment;
1036
1037 if (att_idx == VK_ATTACHMENT_UNUSED)
1038 return;
1039
1040 struct anv_render_pass_attachment *pass_att =
1041 &cmd_buffer->state.pass->attachments[att_idx];
1042
1043 bool clear_depth = attachment->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
1044 bool clear_stencil = attachment->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
1045
1046 enum isl_format depth_format = ISL_FORMAT_UNSUPPORTED;
1047 if (clear_depth) {
1048 depth_format = anv_get_isl_format(&cmd_buffer->device->info,
1049 pass_att->format,
1050 VK_IMAGE_ASPECT_DEPTH_BIT,
1051 VK_IMAGE_TILING_OPTIMAL);
1052 }
1053
1054 uint32_t binding_table;
1055 VkResult result =
1056 binding_table_for_surface_state(cmd_buffer,
1057 cmd_buffer->state.null_surface_state,
1058 &binding_table);
1059 if (result != VK_SUCCESS)
1060 return;
1061
1062 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1063 if (subpass->view_mask) {
1064 uint32_t view_idx;
1065 for_each_bit(view_idx, subpass->view_mask) {
1066 for (uint32_t r = 0; r < rectCount; ++r) {
1067 const VkOffset2D offset = pRects[r].rect.offset;
1068 const VkExtent2D extent = pRects[r].rect.extent;
1069 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1070 blorp_clear_attachments(batch, binding_table,
1071 depth_format, pass_att->samples,
1072 view_idx, 1,
1073 offset.x, offset.y,
1074 offset.x + extent.width,
1075 offset.y + extent.height,
1076 false, color_value,
1077 clear_depth, value.depth,
1078 clear_stencil ? 0xff : 0, value.stencil);
1079 }
1080 }
1081 return;
1082 }
1083
1084 for (uint32_t r = 0; r < rectCount; ++r) {
1085 const VkOffset2D offset = pRects[r].rect.offset;
1086 const VkExtent2D extent = pRects[r].rect.extent;
1087 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1088 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1089 blorp_clear_attachments(batch, binding_table,
1090 depth_format, pass_att->samples,
1091 pRects[r].baseArrayLayer,
1092 pRects[r].layerCount,
1093 offset.x, offset.y,
1094 offset.x + extent.width, offset.y + extent.height,
1095 false, color_value,
1096 clear_depth, value.depth,
1097 clear_stencil ? 0xff : 0, value.stencil);
1098 }
1099 }
1100
1101 void anv_CmdClearAttachments(
1102 VkCommandBuffer commandBuffer,
1103 uint32_t attachmentCount,
1104 const VkClearAttachment* pAttachments,
1105 uint32_t rectCount,
1106 const VkClearRect* pRects)
1107 {
1108 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1109
1110 /* Because this gets called within a render pass, we tell blorp not to
1111 * trash our depth and stencil buffers.
1112 */
1113 struct blorp_batch batch;
1114 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1115 BLORP_BATCH_NO_EMIT_DEPTH_STENCIL);
1116
1117 for (uint32_t a = 0; a < attachmentCount; ++a) {
1118 if (pAttachments[a].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
1119 assert(pAttachments[a].aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
1120 clear_color_attachment(cmd_buffer, &batch,
1121 &pAttachments[a],
1122 rectCount, pRects);
1123 } else {
1124 clear_depth_stencil_attachment(cmd_buffer, &batch,
1125 &pAttachments[a],
1126 rectCount, pRects);
1127 }
1128 }
1129
1130 blorp_batch_finish(&batch);
1131 }
1132
1133 enum subpass_stage {
1134 SUBPASS_STAGE_LOAD,
1135 SUBPASS_STAGE_DRAW,
1136 SUBPASS_STAGE_RESOLVE,
1137 };
1138
1139 static bool
1140 subpass_needs_clear(const struct anv_cmd_buffer *cmd_buffer)
1141 {
1142 const struct anv_cmd_state *cmd_state = &cmd_buffer->state;
1143 uint32_t ds = cmd_state->subpass->depth_stencil_attachment.attachment;
1144
1145 for (uint32_t i = 0; i < cmd_state->subpass->color_count; ++i) {
1146 uint32_t a = cmd_state->subpass->color_attachments[i].attachment;
1147 if (a == VK_ATTACHMENT_UNUSED)
1148 continue;
1149
1150 assert(a < cmd_state->pass->attachment_count);
1151 if (cmd_state->attachments[a].pending_clear_aspects) {
1152 return true;
1153 }
1154 }
1155
1156 if (ds != VK_ATTACHMENT_UNUSED) {
1157 assert(ds < cmd_state->pass->attachment_count);
1158 if (cmd_state->attachments[ds].pending_clear_aspects)
1159 return true;
1160 }
1161
1162 return false;
1163 }
1164
1165 void
1166 anv_cmd_buffer_clear_subpass(struct anv_cmd_buffer *cmd_buffer)
1167 {
1168 const struct anv_cmd_state *cmd_state = &cmd_buffer->state;
1169 const VkRect2D render_area = cmd_buffer->state.render_area;
1170
1171
1172 if (!subpass_needs_clear(cmd_buffer))
1173 return;
1174
1175 /* Because this gets called within a render pass, we tell blorp not to
1176 * trash our depth and stencil buffers.
1177 */
1178 struct blorp_batch batch;
1179 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1180 BLORP_BATCH_NO_EMIT_DEPTH_STENCIL);
1181
1182 VkClearRect clear_rect = {
1183 .rect = cmd_buffer->state.render_area,
1184 .baseArrayLayer = 0,
1185 .layerCount = cmd_buffer->state.framebuffer->layers,
1186 };
1187
1188 struct anv_framebuffer *fb = cmd_buffer->state.framebuffer;
1189 for (uint32_t i = 0; i < cmd_state->subpass->color_count; ++i) {
1190 const uint32_t a = cmd_state->subpass->color_attachments[i].attachment;
1191 if (a == VK_ATTACHMENT_UNUSED)
1192 continue;
1193
1194 assert(a < cmd_state->pass->attachment_count);
1195 struct anv_attachment_state *att_state = &cmd_state->attachments[a];
1196
1197 if (!att_state->pending_clear_aspects)
1198 continue;
1199
1200 assert(att_state->pending_clear_aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1201
1202 struct anv_image_view *iview = fb->attachments[a];
1203 const struct anv_image *image = iview->image;
1204 struct blorp_surf surf;
1205 get_blorp_surf_for_anv_image(cmd_buffer->device,
1206 image, VK_IMAGE_ASPECT_COLOR_BIT,
1207 att_state->aux_usage, &surf);
1208
1209 uint32_t base_layer = iview->planes[0].isl.base_array_layer;
1210 uint32_t layer_count = fb->layers;
1211
1212 if (att_state->fast_clear) {
1213 surf.clear_color = vk_to_isl_color(att_state->clear_value.color);
1214
1215 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1216 *
1217 * "After Render target fast clear, pipe-control with color cache
1218 * write-flush must be issued before sending any DRAW commands on
1219 * that render target."
1220 *
1221 * This comment is a bit cryptic and doesn't really tell you what's
1222 * going or what's really needed. It appears that fast clear ops are
1223 * not properly synchronized with other drawing. This means that we
1224 * cannot have a fast clear operation in the pipe at the same time as
1225 * other regular drawing operations. We need to use a PIPE_CONTROL
1226 * to ensure that the contents of the previous draw hit the render
1227 * target before we resolve and then use a second PIPE_CONTROL after
1228 * the resolve to ensure that it is completed before any additional
1229 * drawing occurs.
1230 */
1231 cmd_buffer->state.pending_pipe_bits |=
1232 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1233
1234 /* We only support fast-clears on the first layer */
1235 assert(iview->planes[0].isl.base_level == 0);
1236 assert(iview->planes[0].isl.base_array_layer == 0);
1237
1238 assert(image->n_planes == 1);
1239 blorp_fast_clear(&batch, &surf, iview->planes[0].isl.format, 0, 0, 1,
1240 render_area.offset.x, render_area.offset.y,
1241 render_area.offset.x + render_area.extent.width,
1242 render_area.offset.y + render_area.extent.height);
1243 base_layer++;
1244 layer_count--;
1245
1246 cmd_buffer->state.pending_pipe_bits |=
1247 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1248 }
1249
1250 if (layer_count > 0) {
1251 assert(image->n_planes == 1);
1252 anv_cmd_buffer_mark_image_written(cmd_buffer, image,
1253 VK_IMAGE_ASPECT_COLOR_BIT,
1254 att_state->aux_usage,
1255 iview->planes[0].isl.base_level,
1256 base_layer, layer_count);
1257
1258 blorp_clear(&batch, &surf, iview->planes[0].isl.format,
1259 anv_swizzle_for_render(iview->planes[0].isl.swizzle),
1260 iview->planes[0].isl.base_level, base_layer, layer_count,
1261 render_area.offset.x, render_area.offset.y,
1262 render_area.offset.x + render_area.extent.width,
1263 render_area.offset.y + render_area.extent.height,
1264 vk_to_isl_color(att_state->clear_value.color), NULL);
1265 }
1266
1267 att_state->pending_clear_aspects = 0;
1268 }
1269
1270 const uint32_t ds = cmd_state->subpass->depth_stencil_attachment.attachment;
1271 assert(ds == VK_ATTACHMENT_UNUSED || ds < cmd_state->pass->attachment_count);
1272
1273 if (ds != VK_ATTACHMENT_UNUSED &&
1274 cmd_state->attachments[ds].pending_clear_aspects) {
1275
1276 VkClearAttachment clear_att = {
1277 .aspectMask = cmd_state->attachments[ds].pending_clear_aspects,
1278 .clearValue = cmd_state->attachments[ds].clear_value,
1279 };
1280
1281
1282 const uint8_t gen = cmd_buffer->device->info.gen;
1283 bool clear_with_hiz = gen >= 8 && cmd_state->attachments[ds].aux_usage ==
1284 ISL_AUX_USAGE_HIZ;
1285 const struct anv_image_view *iview = fb->attachments[ds];
1286
1287 if (clear_with_hiz) {
1288 const bool clear_depth = clear_att.aspectMask &
1289 VK_IMAGE_ASPECT_DEPTH_BIT;
1290 const bool clear_stencil = clear_att.aspectMask &
1291 VK_IMAGE_ASPECT_STENCIL_BIT;
1292
1293 /* Check against restrictions for depth buffer clearing. A great GPU
1294 * performance benefit isn't expected when using the HZ sequence for
1295 * stencil-only clears. Therefore, we don't emit a HZ op sequence for
1296 * a stencil clear in addition to using the BLORP-fallback for depth.
1297 */
1298 if (clear_depth) {
1299 if (!blorp_can_hiz_clear_depth(gen, iview->planes[0].isl.format,
1300 iview->image->samples,
1301 render_area.offset.x,
1302 render_area.offset.y,
1303 render_area.offset.x +
1304 render_area.extent.width,
1305 render_area.offset.y +
1306 render_area.extent.height)) {
1307 clear_with_hiz = false;
1308 } else if (clear_att.clearValue.depthStencil.depth !=
1309 ANV_HZ_FC_VAL) {
1310 /* Don't enable fast depth clears for any color not equal to
1311 * ANV_HZ_FC_VAL.
1312 */
1313 clear_with_hiz = false;
1314 } else if (gen == 8 &&
1315 anv_can_sample_with_hiz(&cmd_buffer->device->info,
1316 iview->image)) {
1317 /* Only gen9+ supports returning ANV_HZ_FC_VAL when sampling a
1318 * fast-cleared portion of a HiZ buffer. Testing has revealed
1319 * that Gen8 only supports returning 0.0f. Gens prior to gen8 do
1320 * not support this feature at all.
1321 */
1322 clear_with_hiz = false;
1323 }
1324 }
1325
1326 if (clear_with_hiz) {
1327 blorp_gen8_hiz_clear_attachments(&batch, iview->image->samples,
1328 render_area.offset.x,
1329 render_area.offset.y,
1330 render_area.offset.x +
1331 render_area.extent.width,
1332 render_area.offset.y +
1333 render_area.extent.height,
1334 clear_depth, clear_stencil,
1335 clear_att.clearValue.
1336 depthStencil.stencil);
1337
1338 /* From the SKL PRM, Depth Buffer Clear:
1339 *
1340 * Depth Buffer Clear Workaround
1341 * Depth buffer clear pass using any of the methods (WM_STATE,
1342 * 3DSTATE_WM or 3DSTATE_WM_HZ_OP) must be followed by a
1343 * PIPE_CONTROL command with DEPTH_STALL bit and Depth FLUSH bits
1344 * “set” before starting to render. DepthStall and DepthFlush are
1345 * not needed between consecutive depth clear passes nor is it
1346 * required if the depth-clear pass was done with “full_surf_clear”
1347 * bit set in the 3DSTATE_WM_HZ_OP.
1348 */
1349 if (clear_depth) {
1350 cmd_buffer->state.pending_pipe_bits |=
1351 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT;
1352 }
1353 }
1354 }
1355
1356 if (!clear_with_hiz) {
1357 clear_depth_stencil_attachment(cmd_buffer, &batch,
1358 &clear_att, 1, &clear_rect);
1359 }
1360
1361 cmd_state->attachments[ds].pending_clear_aspects = 0;
1362 }
1363
1364 blorp_batch_finish(&batch);
1365 }
1366
1367 static void
1368 resolve_surface(struct blorp_batch *batch,
1369 struct blorp_surf *src_surf,
1370 uint32_t src_level, uint32_t src_layer,
1371 struct blorp_surf *dst_surf,
1372 uint32_t dst_level, uint32_t dst_layer,
1373 uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y,
1374 uint32_t width, uint32_t height)
1375 {
1376 blorp_blit(batch,
1377 src_surf, src_level, src_layer,
1378 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1379 dst_surf, dst_level, dst_layer,
1380 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1381 src_x, src_y, src_x + width, src_y + height,
1382 dst_x, dst_y, dst_x + width, dst_y + height,
1383 0x2600 /* GL_NEAREST */, false, false);
1384 }
1385
1386 static void
1387 resolve_image(struct anv_device *device,
1388 struct blorp_batch *batch,
1389 const struct anv_image *src_image,
1390 uint32_t src_level, uint32_t src_layer,
1391 const struct anv_image *dst_image,
1392 uint32_t dst_level, uint32_t dst_layer,
1393 VkImageAspectFlags aspect_mask,
1394 uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y,
1395 uint32_t width, uint32_t height)
1396 {
1397 struct anv_cmd_buffer *cmd_buffer = batch->driver_batch;
1398
1399 assert(src_image->type == VK_IMAGE_TYPE_2D);
1400 assert(src_image->samples > 1);
1401 assert(dst_image->type == VK_IMAGE_TYPE_2D);
1402 assert(dst_image->samples == 1);
1403 assert(src_image->n_planes == dst_image->n_planes);
1404
1405 uint32_t aspect_bit;
1406
1407 anv_foreach_image_aspect_bit(aspect_bit, src_image, aspect_mask) {
1408 struct blorp_surf src_surf, dst_surf;
1409 get_blorp_surf_for_anv_image(device, src_image, 1UL << aspect_bit,
1410 ANV_AUX_USAGE_DEFAULT, &src_surf);
1411 get_blorp_surf_for_anv_image(device, dst_image, 1UL << aspect_bit,
1412 ANV_AUX_USAGE_DEFAULT, &dst_surf);
1413 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
1414 1UL << aspect_bit,
1415 dst_surf.aux_usage,
1416 dst_level, dst_layer, 1);
1417
1418 assert(!src_image->format->can_ycbcr);
1419 assert(!dst_image->format->can_ycbcr);
1420
1421 resolve_surface(batch,
1422 &src_surf, src_level, src_layer,
1423 &dst_surf, dst_level, dst_layer,
1424 src_x, src_y, dst_x, dst_y, width, height);
1425 }
1426 }
1427
1428 void anv_CmdResolveImage(
1429 VkCommandBuffer commandBuffer,
1430 VkImage srcImage,
1431 VkImageLayout srcImageLayout,
1432 VkImage dstImage,
1433 VkImageLayout dstImageLayout,
1434 uint32_t regionCount,
1435 const VkImageResolve* pRegions)
1436 {
1437 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1438 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
1439 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
1440
1441 struct blorp_batch batch;
1442 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1443
1444 for (uint32_t r = 0; r < regionCount; r++) {
1445 assert(pRegions[r].srcSubresource.aspectMask ==
1446 pRegions[r].dstSubresource.aspectMask);
1447 assert(anv_get_layerCount(src_image, &pRegions[r].srcSubresource) ==
1448 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource));
1449
1450 const uint32_t layer_count =
1451 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource);
1452
1453 VkImageAspectFlags src_mask = pRegions[r].srcSubresource.aspectMask,
1454 dst_mask = pRegions[r].dstSubresource.aspectMask;
1455
1456 assert(anv_image_aspects_compatible(src_mask, dst_mask));
1457
1458 for (uint32_t layer = 0; layer < layer_count; layer++) {
1459 resolve_image(cmd_buffer->device, &batch,
1460 src_image,
1461 pRegions[r].srcSubresource.mipLevel,
1462 pRegions[r].srcSubresource.baseArrayLayer + layer,
1463 dst_image,
1464 pRegions[r].dstSubresource.mipLevel,
1465 pRegions[r].dstSubresource.baseArrayLayer + layer,
1466 pRegions[r].dstSubresource.aspectMask,
1467 pRegions[r].srcOffset.x, pRegions[r].srcOffset.y,
1468 pRegions[r].dstOffset.x, pRegions[r].dstOffset.y,
1469 pRegions[r].extent.width, pRegions[r].extent.height);
1470 }
1471 }
1472
1473 blorp_batch_finish(&batch);
1474 }
1475
1476 static enum isl_aux_usage
1477 fast_clear_aux_usage(const struct anv_image *image,
1478 VkImageAspectFlagBits aspect)
1479 {
1480 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
1481 if (image->planes[plane].aux_usage == ISL_AUX_USAGE_NONE)
1482 return ISL_AUX_USAGE_CCS_D;
1483 else
1484 return image->planes[plane].aux_usage;
1485 }
1486
1487 void
1488 anv_cmd_buffer_resolve_subpass(struct anv_cmd_buffer *cmd_buffer)
1489 {
1490 struct anv_framebuffer *fb = cmd_buffer->state.framebuffer;
1491 struct anv_subpass *subpass = cmd_buffer->state.subpass;
1492
1493 if (subpass->has_resolve) {
1494 struct blorp_batch batch;
1495 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1496
1497 /* We are about to do some MSAA resolves. We need to flush so that the
1498 * result of writes to the MSAA color attachments show up in the sampler
1499 * when we blit to the single-sampled resolve target.
1500 */
1501 cmd_buffer->state.pending_pipe_bits |=
1502 ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT |
1503 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
1504
1505 for (uint32_t i = 0; i < subpass->color_count; ++i) {
1506 uint32_t src_att = subpass->color_attachments[i].attachment;
1507 uint32_t dst_att = subpass->resolve_attachments[i].attachment;
1508
1509 if (dst_att == VK_ATTACHMENT_UNUSED)
1510 continue;
1511
1512 assert(src_att < cmd_buffer->state.pass->attachment_count);
1513 assert(dst_att < cmd_buffer->state.pass->attachment_count);
1514
1515 if (cmd_buffer->state.attachments[dst_att].pending_clear_aspects) {
1516 /* From the Vulkan 1.0 spec:
1517 *
1518 * If the first use of an attachment in a render pass is as a
1519 * resolve attachment, then the loadOp is effectively ignored
1520 * as the resolve is guaranteed to overwrite all pixels in the
1521 * render area.
1522 */
1523 cmd_buffer->state.attachments[dst_att].pending_clear_aspects = 0;
1524 }
1525
1526 struct anv_image_view *src_iview = fb->attachments[src_att];
1527 struct anv_image_view *dst_iview = fb->attachments[dst_att];
1528
1529 enum isl_aux_usage src_aux_usage =
1530 cmd_buffer->state.attachments[src_att].aux_usage;
1531 enum isl_aux_usage dst_aux_usage =
1532 cmd_buffer->state.attachments[dst_att].aux_usage;
1533
1534 const VkRect2D render_area = cmd_buffer->state.render_area;
1535
1536 assert(src_iview->aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT &&
1537 dst_iview->aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT);
1538
1539 struct blorp_surf src_surf, dst_surf;
1540 get_blorp_surf_for_anv_image(cmd_buffer->device, src_iview->image,
1541 VK_IMAGE_ASPECT_COLOR_BIT,
1542 src_aux_usage, &src_surf);
1543 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_iview->image,
1544 VK_IMAGE_ASPECT_COLOR_BIT,
1545 dst_aux_usage, &dst_surf);
1546 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_iview->image,
1547 VK_IMAGE_ASPECT_COLOR_BIT,
1548 dst_surf.aux_usage,
1549 dst_iview->planes[0].isl.base_level,
1550 dst_iview->planes[0].isl.base_array_layer, 1);
1551
1552 assert(!src_iview->image->format->can_ycbcr);
1553 assert(!dst_iview->image->format->can_ycbcr);
1554
1555 resolve_surface(&batch,
1556 &src_surf,
1557 src_iview->planes[0].isl.base_level,
1558 src_iview->planes[0].isl.base_array_layer,
1559 &dst_surf,
1560 dst_iview->planes[0].isl.base_level,
1561 dst_iview->planes[0].isl.base_array_layer,
1562 render_area.offset.x, render_area.offset.y,
1563 render_area.offset.x, render_area.offset.y,
1564 render_area.extent.width, render_area.extent.height);
1565 }
1566
1567 blorp_batch_finish(&batch);
1568 }
1569 }
1570
1571 void
1572 anv_image_copy_to_shadow(struct anv_cmd_buffer *cmd_buffer,
1573 const struct anv_image *image,
1574 uint32_t base_level, uint32_t level_count,
1575 uint32_t base_layer, uint32_t layer_count)
1576 {
1577 struct blorp_batch batch;
1578 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1579
1580 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT && image->n_planes == 1);
1581
1582 struct blorp_surf surf;
1583 get_blorp_surf_for_anv_image(cmd_buffer->device,
1584 image, VK_IMAGE_ASPECT_COLOR_BIT,
1585 ISL_AUX_USAGE_NONE, &surf);
1586
1587 struct blorp_surf shadow_surf = {
1588 .surf = &image->planes[0].shadow_surface.isl,
1589 .addr = {
1590 .buffer = image->planes[0].bo,
1591 .offset = image->planes[0].bo_offset +
1592 image->planes[0].shadow_surface.offset,
1593 .mocs = cmd_buffer->device->default_mocs,
1594 },
1595 };
1596
1597 for (uint32_t l = 0; l < level_count; l++) {
1598 const uint32_t level = base_level + l;
1599
1600 const VkExtent3D extent = {
1601 .width = anv_minify(image->extent.width, level),
1602 .height = anv_minify(image->extent.height, level),
1603 .depth = anv_minify(image->extent.depth, level),
1604 };
1605
1606 if (image->type == VK_IMAGE_TYPE_3D)
1607 layer_count = extent.depth;
1608
1609 for (uint32_t a = 0; a < layer_count; a++) {
1610 const uint32_t layer = base_layer + a;
1611
1612 blorp_copy(&batch, &surf, level, layer,
1613 &shadow_surf, level, layer,
1614 0, 0, 0, 0, extent.width, extent.height);
1615 }
1616 }
1617
1618 blorp_batch_finish(&batch);
1619 }
1620
1621 static enum blorp_hiz_op
1622 isl_to_blorp_hiz_op(enum isl_aux_op isl_op)
1623 {
1624 switch (isl_op) {
1625 case ISL_AUX_OP_FAST_CLEAR: return BLORP_HIZ_OP_DEPTH_CLEAR;
1626 case ISL_AUX_OP_FULL_RESOLVE: return BLORP_HIZ_OP_DEPTH_RESOLVE;
1627 case ISL_AUX_OP_AMBIGUATE: return BLORP_HIZ_OP_HIZ_RESOLVE;
1628 default:
1629 unreachable("Unsupported HiZ aux op");
1630 }
1631 }
1632
1633 void
1634 anv_image_hiz_op(struct anv_cmd_buffer *cmd_buffer,
1635 const struct anv_image *image,
1636 VkImageAspectFlagBits aspect, uint32_t level,
1637 uint32_t base_layer, uint32_t layer_count,
1638 enum isl_aux_op hiz_op)
1639 {
1640 assert(aspect == VK_IMAGE_ASPECT_DEPTH_BIT);
1641 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, level));
1642 assert(anv_image_aspect_to_plane(image->aspects,
1643 VK_IMAGE_ASPECT_DEPTH_BIT) == 0);
1644
1645 struct blorp_batch batch;
1646 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1647
1648 struct blorp_surf surf;
1649 get_blorp_surf_for_anv_image(cmd_buffer->device,
1650 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1651 ISL_AUX_USAGE_HIZ, &surf);
1652 surf.clear_color.f32[0] = ANV_HZ_FC_VAL;
1653
1654 blorp_hiz_op(&batch, &surf, level, base_layer, layer_count,
1655 isl_to_blorp_hiz_op(hiz_op));
1656
1657 blorp_batch_finish(&batch);
1658 }
1659
1660 void
1661 anv_image_mcs_op(struct anv_cmd_buffer *cmd_buffer,
1662 const struct anv_image *image,
1663 VkImageAspectFlagBits aspect,
1664 uint32_t base_layer, uint32_t layer_count,
1665 enum isl_aux_op mcs_op, bool predicate)
1666 {
1667 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1668 assert(image->samples > 1);
1669 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, 0));
1670
1671 /* Multisampling with multi-planar formats is not supported */
1672 assert(image->n_planes == 1);
1673
1674 struct blorp_batch batch;
1675 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1676 predicate ? BLORP_BATCH_PREDICATE_ENABLE : 0);
1677
1678 struct blorp_surf surf;
1679 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1680 ANV_AUX_USAGE_DEFAULT, &surf);
1681
1682 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1683 *
1684 * "After Render target fast clear, pipe-control with color cache
1685 * write-flush must be issued before sending any DRAW commands on
1686 * that render target."
1687 *
1688 * This comment is a bit cryptic and doesn't really tell you what's going
1689 * or what's really needed. It appears that fast clear ops are not
1690 * properly synchronized with other drawing. This means that we cannot
1691 * have a fast clear operation in the pipe at the same time as other
1692 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
1693 * that the contents of the previous draw hit the render target before we
1694 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
1695 * that it is completed before any additional drawing occurs.
1696 */
1697 cmd_buffer->state.pending_pipe_bits |=
1698 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1699
1700 switch (mcs_op) {
1701 case ISL_AUX_OP_FAST_CLEAR:
1702 blorp_fast_clear(&batch, &surf, surf.surf->format,
1703 0, base_layer, layer_count,
1704 0, 0, image->extent.width, image->extent.height);
1705 break;
1706 case ISL_AUX_OP_FULL_RESOLVE:
1707 case ISL_AUX_OP_PARTIAL_RESOLVE:
1708 case ISL_AUX_OP_AMBIGUATE:
1709 default:
1710 unreachable("Unsupported MCS operation");
1711 }
1712
1713 cmd_buffer->state.pending_pipe_bits |=
1714 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1715
1716 blorp_batch_finish(&batch);
1717 }
1718
1719 void
1720 anv_image_ccs_op(struct anv_cmd_buffer *cmd_buffer,
1721 const struct anv_image *image,
1722 VkImageAspectFlagBits aspect, uint32_t level,
1723 uint32_t base_layer, uint32_t layer_count,
1724 enum isl_aux_op ccs_op, bool predicate)
1725 {
1726 assert(image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
1727 assert(image->samples == 1);
1728 assert(level < anv_image_aux_levels(image, aspect));
1729 /* Multi-LOD YcBcR is not allowed */
1730 assert(image->n_planes == 1 || level == 0);
1731 assert(base_layer + layer_count <=
1732 anv_image_aux_layers(image, aspect, level));
1733
1734 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
1735 uint32_t width_div = image->format->planes[plane].denominator_scales[0];
1736 uint32_t height_div = image->format->planes[plane].denominator_scales[1];
1737 uint32_t level_width = anv_minify(image->extent.width, level) / width_div;
1738 uint32_t level_height = anv_minify(image->extent.height, level) / height_div;
1739
1740 struct blorp_batch batch;
1741 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1742 predicate ? BLORP_BATCH_PREDICATE_ENABLE : 0);
1743
1744 struct blorp_surf surf;
1745 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1746 fast_clear_aux_usage(image, aspect),
1747 &surf);
1748
1749 if (ccs_op == ISL_AUX_OP_FULL_RESOLVE ||
1750 ccs_op == ISL_AUX_OP_PARTIAL_RESOLVE) {
1751 /* If we're doing a resolve operation, then we need the indirect clear
1752 * color. The clear and ambiguate operations just stomp the CCS to a
1753 * particular value and don't care about format or clear value.
1754 */
1755 const struct anv_address clear_color_addr =
1756 anv_image_get_clear_color_addr(cmd_buffer->device, image, aspect);
1757 surf.clear_color_addr = anv_to_blorp_address(clear_color_addr);
1758 }
1759
1760 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1761 *
1762 * "After Render target fast clear, pipe-control with color cache
1763 * write-flush must be issued before sending any DRAW commands on
1764 * that render target."
1765 *
1766 * This comment is a bit cryptic and doesn't really tell you what's going
1767 * or what's really needed. It appears that fast clear ops are not
1768 * properly synchronized with other drawing. This means that we cannot
1769 * have a fast clear operation in the pipe at the same time as other
1770 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
1771 * that the contents of the previous draw hit the render target before we
1772 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
1773 * that it is completed before any additional drawing occurs.
1774 */
1775 cmd_buffer->state.pending_pipe_bits |=
1776 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1777
1778 switch (ccs_op) {
1779 case ISL_AUX_OP_FAST_CLEAR:
1780 blorp_fast_clear(&batch, &surf, surf.surf->format,
1781 level, base_layer, layer_count,
1782 0, 0, level_width, level_height);
1783 break;
1784 case ISL_AUX_OP_FULL_RESOLVE:
1785 case ISL_AUX_OP_PARTIAL_RESOLVE:
1786 blorp_ccs_resolve(&batch, &surf, level, base_layer, layer_count,
1787 surf.surf->format, ccs_op);
1788 break;
1789 case ISL_AUX_OP_AMBIGUATE:
1790 for (uint32_t a = 0; a < layer_count; a++) {
1791 const uint32_t layer = base_layer + a;
1792 blorp_ccs_ambiguate(&batch, &surf, level, layer);
1793 }
1794 break;
1795 default:
1796 unreachable("Unsupported CCS operation");
1797 }
1798
1799 cmd_buffer->state.pending_pipe_bits |=
1800 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1801
1802 blorp_batch_finish(&batch);
1803 }