projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
anv: Use an address for each anv_image plane
[mesa.git] / src / intel / vulkan / anv_blorp.c
1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "anv_private.h"
25
26 static bool
27 lookup_blorp_shader(struct blorp_context *blorp,
28 const void *key, uint32_t key_size,
29 uint32_t *kernel_out, void *prog_data_out)
30 {
31 struct anv_device *device = blorp->driver_ctx;
32
33 /* The blorp cache must be a real cache */
34 assert(device->blorp_shader_cache.cache);
35
36 struct anv_shader_bin *bin =
37 anv_pipeline_cache_search(&device->blorp_shader_cache, key, key_size);
38 if (!bin)
39 return false;
40
41 /* The cache already has a reference and it's not going anywhere so there
42 * is no need to hold a second reference.
43 */
44 anv_shader_bin_unref(device, bin);
45
46 *kernel_out = bin->kernel.offset;
47 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
48
49 return true;
50 }
51
52 static bool
53 upload_blorp_shader(struct blorp_context *blorp,
54 const void *key, uint32_t key_size,
55 const void *kernel, uint32_t kernel_size,
56 const struct brw_stage_prog_data *prog_data,
57 uint32_t prog_data_size,
58 uint32_t *kernel_out, void *prog_data_out)
59 {
60 struct anv_device *device = blorp->driver_ctx;
61
62 /* The blorp cache must be a real cache */
63 assert(device->blorp_shader_cache.cache);
64
65 struct anv_pipeline_bind_map bind_map = {
66 .surface_count = 0,
67 .sampler_count = 0,
68 };
69
70 struct anv_shader_bin *bin =
71 anv_pipeline_cache_upload_kernel(&device->blorp_shader_cache,
72 key, key_size, kernel, kernel_size,
73 prog_data, prog_data_size, &bind_map);
74
75 if (!bin)
76 return false;
77
78 /* The cache already has a reference and it's not going anywhere so there
79 * is no need to hold a second reference.
80 */
81 anv_shader_bin_unref(device, bin);
82
83 *kernel_out = bin->kernel.offset;
84 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
85
86 return true;
87 }
88
89 void
90 anv_device_init_blorp(struct anv_device *device)
91 {
92 anv_pipeline_cache_init(&device->blorp_shader_cache, device, true);
93 blorp_init(&device->blorp, device, &device->isl_dev);
94 device->blorp.compiler = device->instance->physicalDevice.compiler;
95 device->blorp.lookup_shader = lookup_blorp_shader;
96 device->blorp.upload_shader = upload_blorp_shader;
97 switch (device->info.gen) {
98 case 7:
99 if (device->info.is_haswell) {
100 device->blorp.exec = gen75_blorp_exec;
101 } else {
102 device->blorp.exec = gen7_blorp_exec;
103 }
104 break;
105 case 8:
106 device->blorp.exec = gen8_blorp_exec;
107 break;
108 case 9:
109 device->blorp.exec = gen9_blorp_exec;
110 break;
111 case 10:
112 device->blorp.exec = gen10_blorp_exec;
113 break;
114 case 11:
115 device->blorp.exec = gen11_blorp_exec;
116 break;
117 default:
118 unreachable("Unknown hardware generation");
119 }
120 }
121
122 void
123 anv_device_finish_blorp(struct anv_device *device)
124 {
125 blorp_finish(&device->blorp);
126 anv_pipeline_cache_finish(&device->blorp_shader_cache);
127 }
128
129 static void
130 get_blorp_surf_for_anv_buffer(struct anv_device *device,
131 struct anv_buffer *buffer, uint64_t offset,
132 uint32_t width, uint32_t height,
133 uint32_t row_pitch, enum isl_format format,
134 struct blorp_surf *blorp_surf,
135 struct isl_surf *isl_surf)
136 {
137 const struct isl_format_layout *fmtl =
138 isl_format_get_layout(format);
139 bool ok UNUSED;
140
141 /* ASTC is the only format which doesn't support linear layouts.
142 * Create an equivalently sized surface with ISL to get around this.
143 */
144 if (fmtl->txc == ISL_TXC_ASTC) {
145 /* Use an equivalently sized format */
146 format = ISL_FORMAT_R32G32B32A32_UINT;
147 assert(fmtl->bpb == isl_format_get_layout(format)->bpb);
148
149 /* Shrink the dimensions for the new format */
150 width = DIV_ROUND_UP(width, fmtl->bw);
151 height = DIV_ROUND_UP(height, fmtl->bh);
152 }
153
154 *blorp_surf = (struct blorp_surf) {
155 .surf = isl_surf,
156 .addr = {
157 .buffer = buffer->address.bo,
158 .offset = buffer->address.offset + offset,
159 .mocs = device->default_mocs,
160 },
161 };
162
163 ok = isl_surf_init(&device->isl_dev, isl_surf,
164 .dim = ISL_SURF_DIM_2D,
165 .format = format,
166 .width = width,
167 .height = height,
168 .depth = 1,
169 .levels = 1,
170 .array_len = 1,
171 .samples = 1,
172 .row_pitch = row_pitch,
173 .usage = ISL_SURF_USAGE_TEXTURE_BIT |
174 ISL_SURF_USAGE_RENDER_TARGET_BIT,
175 .tiling_flags = ISL_TILING_LINEAR_BIT);
176 assert(ok);
177 }
178
179 /* Pick something high enough that it won't be used in core and low enough it
180 * will never map to an extension.
181 */
182 #define ANV_IMAGE_LAYOUT_EXPLICIT_AUX (VkImageLayout)10000000
183
184 static struct blorp_address
185 anv_to_blorp_address(struct anv_address addr)
186 {
187 return (struct blorp_address) {
188 .buffer = addr.bo,
189 .offset = addr.offset,
190 };
191 }
192
193 static void
194 get_blorp_surf_for_anv_image(const struct anv_device *device,
195 const struct anv_image *image,
196 VkImageAspectFlags aspect,
197 VkImageLayout layout,
198 enum isl_aux_usage aux_usage,
199 struct blorp_surf *blorp_surf)
200 {
201 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
202
203 if (layout != ANV_IMAGE_LAYOUT_EXPLICIT_AUX)
204 aux_usage = anv_layout_to_aux_usage(&device->info, image, aspect, layout);
205
206 const struct anv_surface *surface = &image->planes[plane].surface;
207 *blorp_surf = (struct blorp_surf) {
208 .surf = &surface->isl,
209 .addr = {
210 .buffer = image->planes[plane].address.bo,
211 .offset = image->planes[plane].address.offset + surface->offset,
212 .mocs = device->default_mocs,
213 },
214 };
215
216 if (aux_usage != ISL_AUX_USAGE_NONE) {
217 const struct anv_surface *aux_surface = &image->planes[plane].aux_surface;
218 blorp_surf->aux_surf = &aux_surface->isl,
219 blorp_surf->aux_addr = (struct blorp_address) {
220 .buffer = image->planes[plane].address.bo,
221 .offset = image->planes[plane].address.offset + aux_surface->offset,
222 .mocs = device->default_mocs,
223 };
224 blorp_surf->aux_usage = aux_usage;
225
226 /* If we're doing a partial resolve, then we need the indirect clear
227 * color. If we are doing a fast clear and want to store/update the
228 * clear color, we also pass the address to blorp, otherwise it will only
229 * stomp the CCS to a particular value and won't care about format or
230 * clear value
231 */
232 if (aspect & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
233 const struct anv_address clear_color_addr =
234 anv_image_get_clear_color_addr(device, image, aspect);
235 blorp_surf->clear_color_addr = anv_to_blorp_address(clear_color_addr);
236 } else if (aspect & VK_IMAGE_ASPECT_DEPTH_BIT
237 && device->info.gen >= 10) {
238 /* Vulkan always clears to 1.0. On gen < 10, we set that directly in
239 * the state packet. For gen >= 10, must provide the clear value in a
240 * buffer. We have a single global buffer that stores the 1.0 value.
241 */
242 const struct anv_address clear_color_addr = (struct anv_address) {
243 .bo = (struct anv_bo *)&device->hiz_clear_bo
244 };
245 blorp_surf->clear_color_addr = anv_to_blorp_address(clear_color_addr);
246 }
247 }
248 }
249
250 void anv_CmdCopyImage(
251 VkCommandBuffer commandBuffer,
252 VkImage srcImage,
253 VkImageLayout srcImageLayout,
254 VkImage dstImage,
255 VkImageLayout dstImageLayout,
256 uint32_t regionCount,
257 const VkImageCopy* pRegions)
258 {
259 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
260 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
261 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
262
263 struct blorp_batch batch;
264 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
265
266 for (unsigned r = 0; r < regionCount; r++) {
267 VkOffset3D srcOffset =
268 anv_sanitize_image_offset(src_image->type, pRegions[r].srcOffset);
269 VkOffset3D dstOffset =
270 anv_sanitize_image_offset(dst_image->type, pRegions[r].dstOffset);
271 VkExtent3D extent =
272 anv_sanitize_image_extent(src_image->type, pRegions[r].extent);
273
274 const uint32_t dst_level = pRegions[r].dstSubresource.mipLevel;
275 unsigned dst_base_layer, layer_count;
276 if (dst_image->type == VK_IMAGE_TYPE_3D) {
277 dst_base_layer = pRegions[r].dstOffset.z;
278 layer_count = pRegions[r].extent.depth;
279 } else {
280 dst_base_layer = pRegions[r].dstSubresource.baseArrayLayer;
281 layer_count =
282 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource);
283 }
284
285 const uint32_t src_level = pRegions[r].srcSubresource.mipLevel;
286 unsigned src_base_layer;
287 if (src_image->type == VK_IMAGE_TYPE_3D) {
288 src_base_layer = pRegions[r].srcOffset.z;
289 } else {
290 src_base_layer = pRegions[r].srcSubresource.baseArrayLayer;
291 assert(layer_count ==
292 anv_get_layerCount(src_image, &pRegions[r].srcSubresource));
293 }
294
295 VkImageAspectFlags src_mask = pRegions[r].srcSubresource.aspectMask,
296 dst_mask = pRegions[r].dstSubresource.aspectMask;
297
298 assert(anv_image_aspects_compatible(src_mask, dst_mask));
299
300 if (_mesa_bitcount(src_mask) > 1) {
301 uint32_t aspect_bit;
302 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_mask) {
303 struct blorp_surf src_surf, dst_surf;
304 get_blorp_surf_for_anv_image(cmd_buffer->device,
305 src_image, 1UL << aspect_bit,
306 srcImageLayout, ISL_AUX_USAGE_NONE,
307 &src_surf);
308 get_blorp_surf_for_anv_image(cmd_buffer->device,
309 dst_image, 1UL << aspect_bit,
310 dstImageLayout, ISL_AUX_USAGE_NONE,
311 &dst_surf);
312 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
313 1UL << aspect_bit,
314 dst_surf.aux_usage, dst_level,
315 dst_base_layer, layer_count);
316
317 for (unsigned i = 0; i < layer_count; i++) {
318 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
319 &dst_surf, dst_level, dst_base_layer + i,
320 srcOffset.x, srcOffset.y,
321 dstOffset.x, dstOffset.y,
322 extent.width, extent.height);
323 }
324 }
325 } else {
326 struct blorp_surf src_surf, dst_surf;
327 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, src_mask,
328 srcImageLayout, ISL_AUX_USAGE_NONE,
329 &src_surf);
330 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, dst_mask,
331 dstImageLayout, ISL_AUX_USAGE_NONE,
332 &dst_surf);
333 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image, dst_mask,
334 dst_surf.aux_usage, dst_level,
335 dst_base_layer, layer_count);
336
337 for (unsigned i = 0; i < layer_count; i++) {
338 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
339 &dst_surf, dst_level, dst_base_layer + i,
340 srcOffset.x, srcOffset.y,
341 dstOffset.x, dstOffset.y,
342 extent.width, extent.height);
343 }
344 }
345 }
346
347 blorp_batch_finish(&batch);
348 }
349
350 static void
351 copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer,
352 struct anv_buffer *anv_buffer,
353 struct anv_image *anv_image,
354 VkImageLayout image_layout,
355 uint32_t regionCount,
356 const VkBufferImageCopy* pRegions,
357 bool buffer_to_image)
358 {
359 struct blorp_batch batch;
360 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
361
362 struct {
363 struct blorp_surf surf;
364 uint32_t level;
365 VkOffset3D offset;
366 } image, buffer, *src, *dst;
367
368 buffer.level = 0;
369 buffer.offset = (VkOffset3D) { 0, 0, 0 };
370
371 if (buffer_to_image) {
372 src = &buffer;
373 dst = &image;
374 } else {
375 src = &image;
376 dst = &buffer;
377 }
378
379 for (unsigned r = 0; r < regionCount; r++) {
380 const VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask;
381
382 get_blorp_surf_for_anv_image(cmd_buffer->device, anv_image, aspect,
383 image_layout, ISL_AUX_USAGE_NONE,
384 &image.surf);
385 image.offset =
386 anv_sanitize_image_offset(anv_image->type, pRegions[r].imageOffset);
387 image.level = pRegions[r].imageSubresource.mipLevel;
388
389 VkExtent3D extent =
390 anv_sanitize_image_extent(anv_image->type, pRegions[r].imageExtent);
391 if (anv_image->type != VK_IMAGE_TYPE_3D) {
392 image.offset.z = pRegions[r].imageSubresource.baseArrayLayer;
393 extent.depth =
394 anv_get_layerCount(anv_image, &pRegions[r].imageSubresource);
395 }
396
397 const enum isl_format buffer_format =
398 anv_get_isl_format(&cmd_buffer->device->info, anv_image->vk_format,
399 aspect, VK_IMAGE_TILING_LINEAR);
400
401 const VkExtent3D bufferImageExtent = {
402 .width = pRegions[r].bufferRowLength ?
403 pRegions[r].bufferRowLength : extent.width,
404 .height = pRegions[r].bufferImageHeight ?
405 pRegions[r].bufferImageHeight : extent.height,
406 };
407
408 const struct isl_format_layout *buffer_fmtl =
409 isl_format_get_layout(buffer_format);
410
411 const uint32_t buffer_row_pitch =
412 DIV_ROUND_UP(bufferImageExtent.width, buffer_fmtl->bw) *
413 (buffer_fmtl->bpb / 8);
414
415 const uint32_t buffer_layer_stride =
416 DIV_ROUND_UP(bufferImageExtent.height, buffer_fmtl->bh) *
417 buffer_row_pitch;
418
419 struct isl_surf buffer_isl_surf;
420 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
421 anv_buffer, pRegions[r].bufferOffset,
422 extent.width, extent.height,
423 buffer_row_pitch, buffer_format,
424 &buffer.surf, &buffer_isl_surf);
425
426 if (&image == dst) {
427 anv_cmd_buffer_mark_image_written(cmd_buffer, anv_image,
428 aspect, dst->surf.aux_usage,
429 dst->level,
430 dst->offset.z, extent.depth);
431 }
432
433 for (unsigned z = 0; z < extent.depth; z++) {
434 blorp_copy(&batch, &src->surf, src->level, src->offset.z,
435 &dst->surf, dst->level, dst->offset.z,
436 src->offset.x, src->offset.y, dst->offset.x, dst->offset.y,
437 extent.width, extent.height);
438
439 image.offset.z++;
440 buffer.surf.addr.offset += buffer_layer_stride;
441 }
442 }
443
444 blorp_batch_finish(&batch);
445 }
446
447 void anv_CmdCopyBufferToImage(
448 VkCommandBuffer commandBuffer,
449 VkBuffer srcBuffer,
450 VkImage dstImage,
451 VkImageLayout dstImageLayout,
452 uint32_t regionCount,
453 const VkBufferImageCopy* pRegions)
454 {
455 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
456 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
457 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
458
459 copy_buffer_to_image(cmd_buffer, src_buffer, dst_image, dstImageLayout,
460 regionCount, pRegions, true);
461 }
462
463 void anv_CmdCopyImageToBuffer(
464 VkCommandBuffer commandBuffer,
465 VkImage srcImage,
466 VkImageLayout srcImageLayout,
467 VkBuffer dstBuffer,
468 uint32_t regionCount,
469 const VkBufferImageCopy* pRegions)
470 {
471 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
472 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
473 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
474
475 copy_buffer_to_image(cmd_buffer, dst_buffer, src_image, srcImageLayout,
476 regionCount, pRegions, false);
477 }
478
479 static bool
480 flip_coords(unsigned *src0, unsigned *src1, unsigned *dst0, unsigned *dst1)
481 {
482 bool flip = false;
483 if (*src0 > *src1) {
484 unsigned tmp = *src0;
485 *src0 = *src1;
486 *src1 = tmp;
487 flip = !flip;
488 }
489
490 if (*dst0 > *dst1) {
491 unsigned tmp = *dst0;
492 *dst0 = *dst1;
493 *dst1 = tmp;
494 flip = !flip;
495 }
496
497 return flip;
498 }
499
500 void anv_CmdBlitImage(
501 VkCommandBuffer commandBuffer,
502 VkImage srcImage,
503 VkImageLayout srcImageLayout,
504 VkImage dstImage,
505 VkImageLayout dstImageLayout,
506 uint32_t regionCount,
507 const VkImageBlit* pRegions,
508 VkFilter filter)
509
510 {
511 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
512 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
513 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
514
515 struct blorp_surf src, dst;
516
517 uint32_t gl_filter;
518 switch (filter) {
519 case VK_FILTER_NEAREST:
520 gl_filter = 0x2600; /* GL_NEAREST */
521 break;
522 case VK_FILTER_LINEAR:
523 gl_filter = 0x2601; /* GL_LINEAR */
524 break;
525 default:
526 unreachable("Invalid filter");
527 }
528
529 struct blorp_batch batch;
530 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
531
532 for (unsigned r = 0; r < regionCount; r++) {
533 const VkImageSubresourceLayers *src_res = &pRegions[r].srcSubresource;
534 const VkImageSubresourceLayers *dst_res = &pRegions[r].dstSubresource;
535
536 get_blorp_surf_for_anv_image(cmd_buffer->device,
537 src_image, src_res->aspectMask,
538 srcImageLayout, ISL_AUX_USAGE_NONE, &src);
539 get_blorp_surf_for_anv_image(cmd_buffer->device,
540 dst_image, dst_res->aspectMask,
541 dstImageLayout, ISL_AUX_USAGE_NONE, &dst);
542
543 struct anv_format_plane src_format =
544 anv_get_format_plane(&cmd_buffer->device->info, src_image->vk_format,
545 src_res->aspectMask, src_image->tiling);
546 struct anv_format_plane dst_format =
547 anv_get_format_plane(&cmd_buffer->device->info, dst_image->vk_format,
548 dst_res->aspectMask, dst_image->tiling);
549
550 unsigned dst_start, dst_end;
551 if (dst_image->type == VK_IMAGE_TYPE_3D) {
552 assert(dst_res->baseArrayLayer == 0);
553 dst_start = pRegions[r].dstOffsets[0].z;
554 dst_end = pRegions[r].dstOffsets[1].z;
555 } else {
556 dst_start = dst_res->baseArrayLayer;
557 dst_end = dst_start + anv_get_layerCount(dst_image, dst_res);
558 }
559
560 unsigned src_start, src_end;
561 if (src_image->type == VK_IMAGE_TYPE_3D) {
562 assert(src_res->baseArrayLayer == 0);
563 src_start = pRegions[r].srcOffsets[0].z;
564 src_end = pRegions[r].srcOffsets[1].z;
565 } else {
566 src_start = src_res->baseArrayLayer;
567 src_end = src_start + anv_get_layerCount(src_image, src_res);
568 }
569
570 bool flip_z = flip_coords(&src_start, &src_end, &dst_start, &dst_end);
571 float src_z_step = (float)(src_end + 1 - src_start) /
572 (float)(dst_end + 1 - dst_start);
573
574 if (flip_z) {
575 src_start = src_end;
576 src_z_step *= -1;
577 }
578
579 unsigned src_x0 = pRegions[r].srcOffsets[0].x;
580 unsigned src_x1 = pRegions[r].srcOffsets[1].x;
581 unsigned dst_x0 = pRegions[r].dstOffsets[0].x;
582 unsigned dst_x1 = pRegions[r].dstOffsets[1].x;
583 bool flip_x = flip_coords(&src_x0, &src_x1, &dst_x0, &dst_x1);
584
585 unsigned src_y0 = pRegions[r].srcOffsets[0].y;
586 unsigned src_y1 = pRegions[r].srcOffsets[1].y;
587 unsigned dst_y0 = pRegions[r].dstOffsets[0].y;
588 unsigned dst_y1 = pRegions[r].dstOffsets[1].y;
589 bool flip_y = flip_coords(&src_y0, &src_y1, &dst_y0, &dst_y1);
590
591 const unsigned num_layers = dst_end - dst_start;
592 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
593 dst_res->aspectMask,
594 dst.aux_usage,
595 dst_res->mipLevel,
596 dst_start, num_layers);
597
598 for (unsigned i = 0; i < num_layers; i++) {
599 unsigned dst_z = dst_start + i;
600 unsigned src_z = src_start + i * src_z_step;
601
602 blorp_blit(&batch, &src, src_res->mipLevel, src_z,
603 src_format.isl_format, src_format.swizzle,
604 &dst, dst_res->mipLevel, dst_z,
605 dst_format.isl_format, dst_format.swizzle,
606 src_x0, src_y0, src_x1, src_y1,
607 dst_x0, dst_y0, dst_x1, dst_y1,
608 gl_filter, flip_x, flip_y);
609 }
610
611 }
612
613 blorp_batch_finish(&batch);
614 }
615
616 static enum isl_format
617 isl_format_for_size(unsigned size_B)
618 {
619 switch (size_B) {
620 case 4: return ISL_FORMAT_R32_UINT;
621 case 8: return ISL_FORMAT_R32G32_UINT;
622 case 16: return ISL_FORMAT_R32G32B32A32_UINT;
623 default:
624 unreachable("Not a power-of-two format size");
625 }
626 }
627
628 /**
629 * Returns the greatest common divisor of a and b that is a power of two.
630 */
631 static uint64_t
632 gcd_pow2_u64(uint64_t a, uint64_t b)
633 {
634 assert(a > 0 || b > 0);
635
636 unsigned a_log2 = ffsll(a) - 1;
637 unsigned b_log2 = ffsll(b) - 1;
638
639 /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which
640 * case, the MIN2() will take the other one. If both are 0 then we will
641 * hit the assert above.
642 */
643 return 1 << MIN2(a_log2, b_log2);
644 }
645
646 /* This is maximum possible width/height our HW can handle */
647 #define MAX_SURFACE_DIM (1ull << 14)
648
649 void anv_CmdCopyBuffer(
650 VkCommandBuffer commandBuffer,
651 VkBuffer srcBuffer,
652 VkBuffer dstBuffer,
653 uint32_t regionCount,
654 const VkBufferCopy* pRegions)
655 {
656 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
657 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
658 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
659
660 struct blorp_batch batch;
661 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
662
663 for (unsigned r = 0; r < regionCount; r++) {
664 struct blorp_address src = {
665 .buffer = src_buffer->address.bo,
666 .offset = src_buffer->address.offset + pRegions[r].srcOffset,
667 .mocs = cmd_buffer->device->default_mocs,
668 };
669 struct blorp_address dst = {
670 .buffer = dst_buffer->address.bo,
671 .offset = dst_buffer->address.offset + pRegions[r].dstOffset,
672 .mocs = cmd_buffer->device->default_mocs,
673 };
674
675 blorp_buffer_copy(&batch, src, dst, pRegions[r].size);
676 }
677
678 blorp_batch_finish(&batch);
679 }
680
681 void anv_CmdUpdateBuffer(
682 VkCommandBuffer commandBuffer,
683 VkBuffer dstBuffer,
684 VkDeviceSize dstOffset,
685 VkDeviceSize dataSize,
686 const void* pData)
687 {
688 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
689 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
690
691 struct blorp_batch batch;
692 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
693
694 /* We can't quite grab a full block because the state stream needs a
695 * little data at the top to build its linked list.
696 */
697 const uint32_t max_update_size =
698 cmd_buffer->device->dynamic_state_pool.block_size - 64;
699
700 assert(max_update_size < MAX_SURFACE_DIM * 4);
701
702 /* We're about to read data that was written from the CPU. Flush the
703 * texture cache so we don't get anything stale.
704 */
705 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
706
707 while (dataSize) {
708 const uint32_t copy_size = MIN2(dataSize, max_update_size);
709
710 struct anv_state tmp_data =
711 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, copy_size, 64);
712
713 memcpy(tmp_data.map, pData, copy_size);
714
715 anv_state_flush(cmd_buffer->device, tmp_data);
716
717 struct blorp_address src = {
718 .buffer = &cmd_buffer->device->dynamic_state_pool.block_pool.bo,
719 .offset = tmp_data.offset,
720 .mocs = cmd_buffer->device->default_mocs,
721 };
722 struct blorp_address dst = {
723 .buffer = dst_buffer->address.bo,
724 .offset = dst_buffer->address.offset + dstOffset,
725 .mocs = cmd_buffer->device->default_mocs,
726 };
727
728 blorp_buffer_copy(&batch, src, dst, copy_size);
729
730 dataSize -= copy_size;
731 dstOffset += copy_size;
732 pData = (void *)pData + copy_size;
733 }
734
735 blorp_batch_finish(&batch);
736 }
737
738 void anv_CmdFillBuffer(
739 VkCommandBuffer commandBuffer,
740 VkBuffer dstBuffer,
741 VkDeviceSize dstOffset,
742 VkDeviceSize fillSize,
743 uint32_t data)
744 {
745 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
746 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
747 struct blorp_surf surf;
748 struct isl_surf isl_surf;
749
750 struct blorp_batch batch;
751 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
752
753 fillSize = anv_buffer_get_range(dst_buffer, dstOffset, fillSize);
754
755 /* From the Vulkan spec:
756 *
757 * "size is the number of bytes to fill, and must be either a multiple
758 * of 4, or VK_WHOLE_SIZE to fill the range from offset to the end of
759 * the buffer. If VK_WHOLE_SIZE is used and the remaining size of the
760 * buffer is not a multiple of 4, then the nearest smaller multiple is
761 * used."
762 */
763 fillSize &= ~3ull;
764
765 /* First, we compute the biggest format that can be used with the
766 * given offsets and size.
767 */
768 int bs = 16;
769 bs = gcd_pow2_u64(bs, dstOffset);
770 bs = gcd_pow2_u64(bs, fillSize);
771 enum isl_format isl_format = isl_format_for_size(bs);
772
773 union isl_color_value color = {
774 .u32 = { data, data, data, data },
775 };
776
777 const uint64_t max_fill_size = MAX_SURFACE_DIM * MAX_SURFACE_DIM * bs;
778 while (fillSize >= max_fill_size) {
779 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
780 dst_buffer, dstOffset,
781 MAX_SURFACE_DIM, MAX_SURFACE_DIM,
782 MAX_SURFACE_DIM * bs, isl_format,
783 &surf, &isl_surf);
784
785 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
786 0, 0, 1, 0, 0, MAX_SURFACE_DIM, MAX_SURFACE_DIM,
787 color, NULL);
788 fillSize -= max_fill_size;
789 dstOffset += max_fill_size;
790 }
791
792 uint64_t height = fillSize / (MAX_SURFACE_DIM * bs);
793 assert(height < MAX_SURFACE_DIM);
794 if (height != 0) {
795 const uint64_t rect_fill_size = height * MAX_SURFACE_DIM * bs;
796 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
797 dst_buffer, dstOffset,
798 MAX_SURFACE_DIM, height,
799 MAX_SURFACE_DIM * bs, isl_format,
800 &surf, &isl_surf);
801
802 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
803 0, 0, 1, 0, 0, MAX_SURFACE_DIM, height,
804 color, NULL);
805 fillSize -= rect_fill_size;
806 dstOffset += rect_fill_size;
807 }
808
809 if (fillSize != 0) {
810 const uint32_t width = fillSize / bs;
811 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
812 dst_buffer, dstOffset,
813 width, 1,
814 width * bs, isl_format,
815 &surf, &isl_surf);
816
817 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
818 0, 0, 1, 0, 0, width, 1,
819 color, NULL);
820 }
821
822 blorp_batch_finish(&batch);
823 }
824
825 void anv_CmdClearColorImage(
826 VkCommandBuffer commandBuffer,
827 VkImage _image,
828 VkImageLayout imageLayout,
829 const VkClearColorValue* pColor,
830 uint32_t rangeCount,
831 const VkImageSubresourceRange* pRanges)
832 {
833 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
834 ANV_FROM_HANDLE(anv_image, image, _image);
835
836 static const bool color_write_disable[4] = { false, false, false, false };
837
838 struct blorp_batch batch;
839 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
840
841
842 for (unsigned r = 0; r < rangeCount; r++) {
843 if (pRanges[r].aspectMask == 0)
844 continue;
845
846 assert(pRanges[r].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
847
848 struct blorp_surf surf;
849 get_blorp_surf_for_anv_image(cmd_buffer->device,
850 image, pRanges[r].aspectMask,
851 imageLayout, ISL_AUX_USAGE_NONE, &surf);
852
853 struct anv_format_plane src_format =
854 anv_get_format_plane(&cmd_buffer->device->info, image->vk_format,
855 VK_IMAGE_ASPECT_COLOR_BIT, image->tiling);
856
857 unsigned base_layer = pRanges[r].baseArrayLayer;
858 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
859
860 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
861 const unsigned level = pRanges[r].baseMipLevel + i;
862 const unsigned level_width = anv_minify(image->extent.width, level);
863 const unsigned level_height = anv_minify(image->extent.height, level);
864
865 if (image->type == VK_IMAGE_TYPE_3D) {
866 base_layer = 0;
867 layer_count = anv_minify(image->extent.depth, level);
868 }
869
870 anv_cmd_buffer_mark_image_written(cmd_buffer, image,
871 pRanges[r].aspectMask,
872 surf.aux_usage, level,
873 base_layer, layer_count);
874
875 blorp_clear(&batch, &surf,
876 src_format.isl_format, src_format.swizzle,
877 level, base_layer, layer_count,
878 0, 0, level_width, level_height,
879 vk_to_isl_color(*pColor), color_write_disable);
880 }
881 }
882
883 blorp_batch_finish(&batch);
884 }
885
886 void anv_CmdClearDepthStencilImage(
887 VkCommandBuffer commandBuffer,
888 VkImage image_h,
889 VkImageLayout imageLayout,
890 const VkClearDepthStencilValue* pDepthStencil,
891 uint32_t rangeCount,
892 const VkImageSubresourceRange* pRanges)
893 {
894 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
895 ANV_FROM_HANDLE(anv_image, image, image_h);
896
897 struct blorp_batch batch;
898 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
899
900 struct blorp_surf depth, stencil;
901 if (image->aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
902 get_blorp_surf_for_anv_image(cmd_buffer->device,
903 image, VK_IMAGE_ASPECT_DEPTH_BIT,
904 imageLayout, ISL_AUX_USAGE_NONE, &depth);
905 } else {
906 memset(&depth, 0, sizeof(depth));
907 }
908
909 if (image->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
910 get_blorp_surf_for_anv_image(cmd_buffer->device,
911 image, VK_IMAGE_ASPECT_STENCIL_BIT,
912 imageLayout, ISL_AUX_USAGE_NONE, &stencil);
913 } else {
914 memset(&stencil, 0, sizeof(stencil));
915 }
916
917 for (unsigned r = 0; r < rangeCount; r++) {
918 if (pRanges[r].aspectMask == 0)
919 continue;
920
921 bool clear_depth = pRanges[r].aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
922 bool clear_stencil = pRanges[r].aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
923
924 unsigned base_layer = pRanges[r].baseArrayLayer;
925 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
926
927 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
928 const unsigned level = pRanges[r].baseMipLevel + i;
929 const unsigned level_width = anv_minify(image->extent.width, level);
930 const unsigned level_height = anv_minify(image->extent.height, level);
931
932 if (image->type == VK_IMAGE_TYPE_3D)
933 layer_count = anv_minify(image->extent.depth, level);
934
935 blorp_clear_depth_stencil(&batch, &depth, &stencil,
936 level, base_layer, layer_count,
937 0, 0, level_width, level_height,
938 clear_depth, pDepthStencil->depth,
939 clear_stencil ? 0xff : 0,
940 pDepthStencil->stencil);
941 }
942 }
943
944 blorp_batch_finish(&batch);
945 }
946
947 VkResult
948 anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer *cmd_buffer,
949 uint32_t num_entries,
950 uint32_t *state_offset,
951 struct anv_state *bt_state)
952 {
953 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
954 state_offset);
955 if (bt_state->map == NULL) {
956 /* We ran out of space. Grab a new binding table block. */
957 VkResult result = anv_cmd_buffer_new_binding_table_block(cmd_buffer);
958 if (result != VK_SUCCESS)
959 return result;
960
961 /* Re-emit state base addresses so we get the new surface state base
962 * address before we start emitting binding tables etc.
963 */
964 anv_cmd_buffer_emit_state_base_address(cmd_buffer);
965
966 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
967 state_offset);
968 assert(bt_state->map != NULL);
969 }
970
971 return VK_SUCCESS;
972 }
973
974 static VkResult
975 binding_table_for_surface_state(struct anv_cmd_buffer *cmd_buffer,
976 struct anv_state surface_state,
977 uint32_t *bt_offset)
978 {
979 uint32_t state_offset;
980 struct anv_state bt_state;
981
982 VkResult result =
983 anv_cmd_buffer_alloc_blorp_binding_table(cmd_buffer, 1, &state_offset,
984 &bt_state);
985 if (result != VK_SUCCESS)
986 return result;
987
988 uint32_t *bt_map = bt_state.map;
989 bt_map[0] = surface_state.offset + state_offset;
990
991 *bt_offset = bt_state.offset;
992 return VK_SUCCESS;
993 }
994
995 static void
996 clear_color_attachment(struct anv_cmd_buffer *cmd_buffer,
997 struct blorp_batch *batch,
998 const VkClearAttachment *attachment,
999 uint32_t rectCount, const VkClearRect *pRects)
1000 {
1001 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
1002 const uint32_t color_att = attachment->colorAttachment;
1003 const uint32_t att_idx = subpass->color_attachments[color_att].attachment;
1004
1005 if (att_idx == VK_ATTACHMENT_UNUSED)
1006 return;
1007
1008 struct anv_render_pass_attachment *pass_att =
1009 &cmd_buffer->state.pass->attachments[att_idx];
1010 struct anv_attachment_state *att_state =
1011 &cmd_buffer->state.attachments[att_idx];
1012
1013 uint32_t binding_table;
1014 VkResult result =
1015 binding_table_for_surface_state(cmd_buffer, att_state->color.state,
1016 &binding_table);
1017 if (result != VK_SUCCESS)
1018 return;
1019
1020 union isl_color_value clear_color =
1021 vk_to_isl_color(attachment->clearValue.color);
1022
1023 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1024 if (subpass->view_mask) {
1025 uint32_t view_idx;
1026 for_each_bit(view_idx, subpass->view_mask) {
1027 for (uint32_t r = 0; r < rectCount; ++r) {
1028 const VkOffset2D offset = pRects[r].rect.offset;
1029 const VkExtent2D extent = pRects[r].rect.extent;
1030 blorp_clear_attachments(batch, binding_table,
1031 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1032 view_idx, 1,
1033 offset.x, offset.y,
1034 offset.x + extent.width,
1035 offset.y + extent.height,
1036 true, clear_color, false, 0.0f, 0, 0);
1037 }
1038 }
1039 return;
1040 }
1041
1042 for (uint32_t r = 0; r < rectCount; ++r) {
1043 const VkOffset2D offset = pRects[r].rect.offset;
1044 const VkExtent2D extent = pRects[r].rect.extent;
1045 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1046 blorp_clear_attachments(batch, binding_table,
1047 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1048 pRects[r].baseArrayLayer,
1049 pRects[r].layerCount,
1050 offset.x, offset.y,
1051 offset.x + extent.width, offset.y + extent.height,
1052 true, clear_color, false, 0.0f, 0, 0);
1053 }
1054 }
1055
1056 static void
1057 clear_depth_stencil_attachment(struct anv_cmd_buffer *cmd_buffer,
1058 struct blorp_batch *batch,
1059 const VkClearAttachment *attachment,
1060 uint32_t rectCount, const VkClearRect *pRects)
1061 {
1062 static const union isl_color_value color_value = { .u32 = { 0, } };
1063 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
1064 const uint32_t att_idx = subpass->depth_stencil_attachment.attachment;
1065
1066 if (att_idx == VK_ATTACHMENT_UNUSED)
1067 return;
1068
1069 struct anv_render_pass_attachment *pass_att =
1070 &cmd_buffer->state.pass->attachments[att_idx];
1071
1072 bool clear_depth = attachment->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
1073 bool clear_stencil = attachment->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
1074
1075 enum isl_format depth_format = ISL_FORMAT_UNSUPPORTED;
1076 if (clear_depth) {
1077 depth_format = anv_get_isl_format(&cmd_buffer->device->info,
1078 pass_att->format,
1079 VK_IMAGE_ASPECT_DEPTH_BIT,
1080 VK_IMAGE_TILING_OPTIMAL);
1081 }
1082
1083 uint32_t binding_table;
1084 VkResult result =
1085 binding_table_for_surface_state(cmd_buffer,
1086 cmd_buffer->state.null_surface_state,
1087 &binding_table);
1088 if (result != VK_SUCCESS)
1089 return;
1090
1091 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1092 if (subpass->view_mask) {
1093 uint32_t view_idx;
1094 for_each_bit(view_idx, subpass->view_mask) {
1095 for (uint32_t r = 0; r < rectCount; ++r) {
1096 const VkOffset2D offset = pRects[r].rect.offset;
1097 const VkExtent2D extent = pRects[r].rect.extent;
1098 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1099 blorp_clear_attachments(batch, binding_table,
1100 depth_format, pass_att->samples,
1101 view_idx, 1,
1102 offset.x, offset.y,
1103 offset.x + extent.width,
1104 offset.y + extent.height,
1105 false, color_value,
1106 clear_depth, value.depth,
1107 clear_stencil ? 0xff : 0, value.stencil);
1108 }
1109 }
1110 return;
1111 }
1112
1113 for (uint32_t r = 0; r < rectCount; ++r) {
1114 const VkOffset2D offset = pRects[r].rect.offset;
1115 const VkExtent2D extent = pRects[r].rect.extent;
1116 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1117 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1118 blorp_clear_attachments(batch, binding_table,
1119 depth_format, pass_att->samples,
1120 pRects[r].baseArrayLayer,
1121 pRects[r].layerCount,
1122 offset.x, offset.y,
1123 offset.x + extent.width, offset.y + extent.height,
1124 false, color_value,
1125 clear_depth, value.depth,
1126 clear_stencil ? 0xff : 0, value.stencil);
1127 }
1128 }
1129
1130 void anv_CmdClearAttachments(
1131 VkCommandBuffer commandBuffer,
1132 uint32_t attachmentCount,
1133 const VkClearAttachment* pAttachments,
1134 uint32_t rectCount,
1135 const VkClearRect* pRects)
1136 {
1137 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1138
1139 /* Because this gets called within a render pass, we tell blorp not to
1140 * trash our depth and stencil buffers.
1141 */
1142 struct blorp_batch batch;
1143 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1144 BLORP_BATCH_NO_EMIT_DEPTH_STENCIL);
1145
1146 for (uint32_t a = 0; a < attachmentCount; ++a) {
1147 if (pAttachments[a].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
1148 assert(pAttachments[a].aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
1149 clear_color_attachment(cmd_buffer, &batch,
1150 &pAttachments[a],
1151 rectCount, pRects);
1152 } else {
1153 clear_depth_stencil_attachment(cmd_buffer, &batch,
1154 &pAttachments[a],
1155 rectCount, pRects);
1156 }
1157 }
1158
1159 blorp_batch_finish(&batch);
1160 }
1161
1162 enum subpass_stage {
1163 SUBPASS_STAGE_LOAD,
1164 SUBPASS_STAGE_DRAW,
1165 SUBPASS_STAGE_RESOLVE,
1166 };
1167
1168 static void
1169 resolve_surface(struct blorp_batch *batch,
1170 struct blorp_surf *src_surf,
1171 uint32_t src_level, uint32_t src_layer,
1172 struct blorp_surf *dst_surf,
1173 uint32_t dst_level, uint32_t dst_layer,
1174 uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y,
1175 uint32_t width, uint32_t height)
1176 {
1177 blorp_blit(batch,
1178 src_surf, src_level, src_layer,
1179 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1180 dst_surf, dst_level, dst_layer,
1181 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1182 src_x, src_y, src_x + width, src_y + height,
1183 dst_x, dst_y, dst_x + width, dst_y + height,
1184 0x2600 /* GL_NEAREST */, false, false);
1185 }
1186
1187 static void
1188 resolve_image(struct anv_device *device,
1189 struct blorp_batch *batch,
1190 const struct anv_image *src_image,
1191 VkImageLayout src_image_layout,
1192 uint32_t src_level, uint32_t src_layer,
1193 const struct anv_image *dst_image,
1194 VkImageLayout dst_image_layout,
1195 uint32_t dst_level, uint32_t dst_layer,
1196 VkImageAspectFlags aspect_mask,
1197 uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y,
1198 uint32_t width, uint32_t height)
1199 {
1200 struct anv_cmd_buffer *cmd_buffer = batch->driver_batch;
1201
1202 assert(src_image->type == VK_IMAGE_TYPE_2D);
1203 assert(src_image->samples > 1);
1204 assert(dst_image->type == VK_IMAGE_TYPE_2D);
1205 assert(dst_image->samples == 1);
1206 assert(src_image->n_planes == dst_image->n_planes);
1207
1208 uint32_t aspect_bit;
1209
1210 anv_foreach_image_aspect_bit(aspect_bit, src_image, aspect_mask) {
1211 struct blorp_surf src_surf, dst_surf;
1212 get_blorp_surf_for_anv_image(device, src_image, 1UL << aspect_bit,
1213 src_image_layout, ISL_AUX_USAGE_NONE,
1214 &src_surf);
1215 get_blorp_surf_for_anv_image(device, dst_image, 1UL << aspect_bit,
1216 dst_image_layout, ISL_AUX_USAGE_NONE,
1217 &dst_surf);
1218 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
1219 1UL << aspect_bit,
1220 dst_surf.aux_usage,
1221 dst_level, dst_layer, 1);
1222
1223 assert(!src_image->format->can_ycbcr);
1224 assert(!dst_image->format->can_ycbcr);
1225
1226 resolve_surface(batch,
1227 &src_surf, src_level, src_layer,
1228 &dst_surf, dst_level, dst_layer,
1229 src_x, src_y, dst_x, dst_y, width, height);
1230 }
1231 }
1232
1233 void anv_CmdResolveImage(
1234 VkCommandBuffer commandBuffer,
1235 VkImage srcImage,
1236 VkImageLayout srcImageLayout,
1237 VkImage dstImage,
1238 VkImageLayout dstImageLayout,
1239 uint32_t regionCount,
1240 const VkImageResolve* pRegions)
1241 {
1242 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1243 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
1244 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
1245
1246 struct blorp_batch batch;
1247 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1248
1249 for (uint32_t r = 0; r < regionCount; r++) {
1250 assert(pRegions[r].srcSubresource.aspectMask ==
1251 pRegions[r].dstSubresource.aspectMask);
1252 assert(anv_get_layerCount(src_image, &pRegions[r].srcSubresource) ==
1253 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource));
1254
1255 const uint32_t layer_count =
1256 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource);
1257
1258 VkImageAspectFlags src_mask = pRegions[r].srcSubresource.aspectMask,
1259 dst_mask = pRegions[r].dstSubresource.aspectMask;
1260
1261 assert(anv_image_aspects_compatible(src_mask, dst_mask));
1262
1263 for (uint32_t layer = 0; layer < layer_count; layer++) {
1264 resolve_image(cmd_buffer->device, &batch,
1265 src_image, srcImageLayout,
1266 pRegions[r].srcSubresource.mipLevel,
1267 pRegions[r].srcSubresource.baseArrayLayer + layer,
1268 dst_image, dstImageLayout,
1269 pRegions[r].dstSubresource.mipLevel,
1270 pRegions[r].dstSubresource.baseArrayLayer + layer,
1271 pRegions[r].dstSubresource.aspectMask,
1272 pRegions[r].srcOffset.x, pRegions[r].srcOffset.y,
1273 pRegions[r].dstOffset.x, pRegions[r].dstOffset.y,
1274 pRegions[r].extent.width, pRegions[r].extent.height);
1275 }
1276 }
1277
1278 blorp_batch_finish(&batch);
1279 }
1280
1281 static enum isl_aux_usage
1282 fast_clear_aux_usage(const struct anv_image *image,
1283 VkImageAspectFlagBits aspect)
1284 {
1285 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
1286 if (image->planes[plane].aux_usage == ISL_AUX_USAGE_NONE)
1287 return ISL_AUX_USAGE_CCS_D;
1288 else
1289 return image->planes[plane].aux_usage;
1290 }
1291
1292 void
1293 anv_cmd_buffer_resolve_subpass(struct anv_cmd_buffer *cmd_buffer)
1294 {
1295 struct anv_framebuffer *fb = cmd_buffer->state.framebuffer;
1296 struct anv_subpass *subpass = cmd_buffer->state.subpass;
1297
1298 if (subpass->has_resolve) {
1299 struct blorp_batch batch;
1300 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1301
1302 /* We are about to do some MSAA resolves. We need to flush so that the
1303 * result of writes to the MSAA color attachments show up in the sampler
1304 * when we blit to the single-sampled resolve target.
1305 */
1306 cmd_buffer->state.pending_pipe_bits |=
1307 ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT |
1308 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
1309
1310 for (uint32_t i = 0; i < subpass->color_count; ++i) {
1311 uint32_t src_att = subpass->color_attachments[i].attachment;
1312 uint32_t dst_att = subpass->resolve_attachments[i].attachment;
1313
1314 if (dst_att == VK_ATTACHMENT_UNUSED)
1315 continue;
1316
1317 assert(src_att < cmd_buffer->state.pass->attachment_count);
1318 assert(dst_att < cmd_buffer->state.pass->attachment_count);
1319
1320 if (cmd_buffer->state.attachments[dst_att].pending_clear_aspects) {
1321 /* From the Vulkan 1.0 spec:
1322 *
1323 * If the first use of an attachment in a render pass is as a
1324 * resolve attachment, then the loadOp is effectively ignored
1325 * as the resolve is guaranteed to overwrite all pixels in the
1326 * render area.
1327 */
1328 cmd_buffer->state.attachments[dst_att].pending_clear_aspects = 0;
1329 }
1330
1331 struct anv_image_view *src_iview = fb->attachments[src_att];
1332 struct anv_image_view *dst_iview = fb->attachments[dst_att];
1333
1334 enum isl_aux_usage src_aux_usage =
1335 cmd_buffer->state.attachments[src_att].aux_usage;
1336 enum isl_aux_usage dst_aux_usage =
1337 cmd_buffer->state.attachments[dst_att].aux_usage;
1338
1339 const VkRect2D render_area = cmd_buffer->state.render_area;
1340
1341 assert(src_iview->aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT &&
1342 dst_iview->aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT);
1343
1344 struct blorp_surf src_surf, dst_surf;
1345 get_blorp_surf_for_anv_image(cmd_buffer->device, src_iview->image,
1346 VK_IMAGE_ASPECT_COLOR_BIT,
1347 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1348 src_aux_usage, &src_surf);
1349 if (src_aux_usage == ISL_AUX_USAGE_MCS) {
1350 src_surf.clear_color_addr = anv_to_blorp_address(
1351 anv_image_get_clear_color_addr(cmd_buffer->device,
1352 src_iview->image,
1353 VK_IMAGE_ASPECT_COLOR_BIT));
1354 }
1355 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_iview->image,
1356 VK_IMAGE_ASPECT_COLOR_BIT,
1357 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1358 dst_aux_usage, &dst_surf);
1359
1360 uint32_t base_src_layer = src_iview->planes[0].isl.base_array_layer;
1361 uint32_t base_dst_layer = dst_iview->planes[0].isl.base_array_layer;
1362
1363 assert(src_iview->planes[0].isl.array_len >= fb->layers);
1364 assert(dst_iview->planes[0].isl.array_len >= fb->layers);
1365
1366 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_iview->image,
1367 VK_IMAGE_ASPECT_COLOR_BIT,
1368 dst_surf.aux_usage,
1369 dst_iview->planes[0].isl.base_level,
1370 base_dst_layer, fb->layers);
1371
1372 assert(!src_iview->image->format->can_ycbcr);
1373 assert(!dst_iview->image->format->can_ycbcr);
1374
1375 for (uint32_t i = 0; i < fb->layers; i++) {
1376 resolve_surface(&batch,
1377 &src_surf,
1378 src_iview->planes[0].isl.base_level,
1379 base_src_layer + i,
1380 &dst_surf,
1381 dst_iview->planes[0].isl.base_level,
1382 base_dst_layer + i,
1383 render_area.offset.x, render_area.offset.y,
1384 render_area.offset.x, render_area.offset.y,
1385 render_area.extent.width, render_area.extent.height);
1386 }
1387 }
1388
1389 blorp_batch_finish(&batch);
1390 }
1391 }
1392
1393 void
1394 anv_image_copy_to_shadow(struct anv_cmd_buffer *cmd_buffer,
1395 const struct anv_image *image,
1396 uint32_t base_level, uint32_t level_count,
1397 uint32_t base_layer, uint32_t layer_count)
1398 {
1399 struct blorp_batch batch;
1400 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1401
1402 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT && image->n_planes == 1);
1403
1404 struct blorp_surf surf;
1405 get_blorp_surf_for_anv_image(cmd_buffer->device,
1406 image, VK_IMAGE_ASPECT_COLOR_BIT,
1407 VK_IMAGE_LAYOUT_GENERAL,
1408 ISL_AUX_USAGE_NONE, &surf);
1409 assert(surf.aux_usage == ISL_AUX_USAGE_NONE);
1410
1411 struct blorp_surf shadow_surf = {
1412 .surf = &image->planes[0].shadow_surface.isl,
1413 .addr = {
1414 .buffer = image->planes[0].address.bo,
1415 .offset = image->planes[0].address.offset +
1416 image->planes[0].shadow_surface.offset,
1417 .mocs = cmd_buffer->device->default_mocs,
1418 },
1419 };
1420
1421 for (uint32_t l = 0; l < level_count; l++) {
1422 const uint32_t level = base_level + l;
1423
1424 const VkExtent3D extent = {
1425 .width = anv_minify(image->extent.width, level),
1426 .height = anv_minify(image->extent.height, level),
1427 .depth = anv_minify(image->extent.depth, level),
1428 };
1429
1430 if (image->type == VK_IMAGE_TYPE_3D)
1431 layer_count = extent.depth;
1432
1433 for (uint32_t a = 0; a < layer_count; a++) {
1434 const uint32_t layer = base_layer + a;
1435
1436 blorp_copy(&batch, &surf, level, layer,
1437 &shadow_surf, level, layer,
1438 0, 0, 0, 0, extent.width, extent.height);
1439 }
1440 }
1441
1442 blorp_batch_finish(&batch);
1443 }
1444
1445 void
1446 anv_image_clear_color(struct anv_cmd_buffer *cmd_buffer,
1447 const struct anv_image *image,
1448 VkImageAspectFlagBits aspect,
1449 enum isl_aux_usage aux_usage,
1450 enum isl_format format, struct isl_swizzle swizzle,
1451 uint32_t level, uint32_t base_layer, uint32_t layer_count,
1452 VkRect2D area, union isl_color_value clear_color)
1453 {
1454 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1455
1456 /* We don't support planar images with multisampling yet */
1457 assert(image->n_planes == 1);
1458
1459 struct blorp_batch batch;
1460 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1461
1462 struct blorp_surf surf;
1463 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1464 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1465 aux_usage, &surf);
1466 anv_cmd_buffer_mark_image_written(cmd_buffer, image, aspect, aux_usage,
1467 level, base_layer, layer_count);
1468
1469 blorp_clear(&batch, &surf, format, anv_swizzle_for_render(swizzle),
1470 level, base_layer, layer_count,
1471 area.offset.x, area.offset.y,
1472 area.offset.x + area.extent.width,
1473 area.offset.y + area.extent.height,
1474 clear_color, NULL);
1475
1476 blorp_batch_finish(&batch);
1477 }
1478
1479 void
1480 anv_image_clear_depth_stencil(struct anv_cmd_buffer *cmd_buffer,
1481 const struct anv_image *image,
1482 VkImageAspectFlags aspects,
1483 enum isl_aux_usage depth_aux_usage,
1484 uint32_t level,
1485 uint32_t base_layer, uint32_t layer_count,
1486 VkRect2D area,
1487 float depth_value, uint8_t stencil_value)
1488 {
1489 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
1490 VK_IMAGE_ASPECT_STENCIL_BIT));
1491
1492 struct blorp_batch batch;
1493 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1494
1495 struct blorp_surf depth = {};
1496 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1497 get_blorp_surf_for_anv_image(cmd_buffer->device,
1498 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1499 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1500 depth_aux_usage, &depth);
1501 depth.clear_color.f32[0] = ANV_HZ_FC_VAL;
1502 }
1503
1504 struct blorp_surf stencil = {};
1505 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1506 get_blorp_surf_for_anv_image(cmd_buffer->device,
1507 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1508 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1509 ISL_AUX_USAGE_NONE, &stencil);
1510 }
1511
1512 blorp_clear_depth_stencil(&batch, &depth, &stencil,
1513 level, base_layer, layer_count,
1514 area.offset.x, area.offset.y,
1515 area.offset.x + area.extent.width,
1516 area.offset.y + area.extent.height,
1517 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
1518 depth_value,
1519 (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) ? 0xff : 0,
1520 stencil_value);
1521
1522 blorp_batch_finish(&batch);
1523 }
1524
1525 void
1526 anv_image_hiz_op(struct anv_cmd_buffer *cmd_buffer,
1527 const struct anv_image *image,
1528 VkImageAspectFlagBits aspect, uint32_t level,
1529 uint32_t base_layer, uint32_t layer_count,
1530 enum isl_aux_op hiz_op)
1531 {
1532 assert(aspect == VK_IMAGE_ASPECT_DEPTH_BIT);
1533 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, level));
1534 assert(anv_image_aspect_to_plane(image->aspects,
1535 VK_IMAGE_ASPECT_DEPTH_BIT) == 0);
1536
1537 struct blorp_batch batch;
1538 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1539
1540 struct blorp_surf surf;
1541 get_blorp_surf_for_anv_image(cmd_buffer->device,
1542 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1543 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1544 ISL_AUX_USAGE_HIZ, &surf);
1545 surf.clear_color.f32[0] = ANV_HZ_FC_VAL;
1546
1547 blorp_hiz_op(&batch, &surf, level, base_layer, layer_count, hiz_op);
1548
1549 blorp_batch_finish(&batch);
1550 }
1551
1552 void
1553 anv_image_hiz_clear(struct anv_cmd_buffer *cmd_buffer,
1554 const struct anv_image *image,
1555 VkImageAspectFlags aspects,
1556 uint32_t level,
1557 uint32_t base_layer, uint32_t layer_count,
1558 VkRect2D area, uint8_t stencil_value)
1559 {
1560 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
1561 VK_IMAGE_ASPECT_STENCIL_BIT));
1562
1563 struct blorp_batch batch;
1564 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1565
1566 struct blorp_surf depth = {};
1567 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1568 assert(base_layer + layer_count <=
1569 anv_image_aux_layers(image, VK_IMAGE_ASPECT_DEPTH_BIT, level));
1570 get_blorp_surf_for_anv_image(cmd_buffer->device,
1571 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1572 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1573 ISL_AUX_USAGE_HIZ, &depth);
1574 depth.clear_color.f32[0] = ANV_HZ_FC_VAL;
1575 }
1576
1577 struct blorp_surf stencil = {};
1578 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1579 get_blorp_surf_for_anv_image(cmd_buffer->device,
1580 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1581 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1582 ISL_AUX_USAGE_NONE, &stencil);
1583 }
1584
1585 blorp_hiz_clear_depth_stencil(&batch, &depth, &stencil,
1586 level, base_layer, layer_count,
1587 area.offset.x, area.offset.y,
1588 area.offset.x + area.extent.width,
1589 area.offset.y + area.extent.height,
1590 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
1591 ANV_HZ_FC_VAL,
1592 aspects & VK_IMAGE_ASPECT_STENCIL_BIT,
1593 stencil_value);
1594
1595 blorp_batch_finish(&batch);
1596
1597 /* From the SKL PRM, Depth Buffer Clear:
1598 *
1599 * Depth Buffer Clear Workaround
1600 * Depth buffer clear pass using any of the methods (WM_STATE, 3DSTATE_WM
1601 * or 3DSTATE_WM_HZ_OP) must be followed by a PIPE_CONTROL command with
1602 * DEPTH_STALL bit and Depth FLUSH bits “set” before starting to render.
1603 * DepthStall and DepthFlush are not needed between consecutive depth clear
1604 * passes nor is it required if the depth-clear pass was done with
1605 * “full_surf_clear” bit set in the 3DSTATE_WM_HZ_OP.
1606 */
1607 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1608 cmd_buffer->state.pending_pipe_bits |=
1609 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT;
1610 }
1611 }
1612
1613 void
1614 anv_image_mcs_op(struct anv_cmd_buffer *cmd_buffer,
1615 const struct anv_image *image,
1616 VkImageAspectFlagBits aspect,
1617 uint32_t base_layer, uint32_t layer_count,
1618 enum isl_aux_op mcs_op, union isl_color_value *clear_value,
1619 bool predicate)
1620 {
1621 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1622 assert(image->samples > 1);
1623 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, 0));
1624
1625 /* Multisampling with multi-planar formats is not supported */
1626 assert(image->n_planes == 1);
1627
1628 struct blorp_batch batch;
1629 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1630 predicate ? BLORP_BATCH_PREDICATE_ENABLE : 0);
1631
1632 struct blorp_surf surf;
1633 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1634 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1635 ISL_AUX_USAGE_MCS, &surf);
1636
1637 /* Blorp will store the clear color for us if we provide the clear color
1638 * address and we are doing a fast clear. So we save the clear value into
1639 * the blorp surface. However, in some situations we want to do a fast clear
1640 * without changing the clear value stored in the state buffer. For those
1641 * cases, we set the clear color address pointer to NULL, so blorp will not
1642 * try to store a garbage color.
1643 */
1644 if (mcs_op == ISL_AUX_OP_FAST_CLEAR) {
1645 if (clear_value)
1646 surf.clear_color = *clear_value;
1647 else
1648 surf.clear_color_addr.buffer = NULL;
1649 }
1650
1651 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1652 *
1653 * "After Render target fast clear, pipe-control with color cache
1654 * write-flush must be issued before sending any DRAW commands on
1655 * that render target."
1656 *
1657 * This comment is a bit cryptic and doesn't really tell you what's going
1658 * or what's really needed. It appears that fast clear ops are not
1659 * properly synchronized with other drawing. This means that we cannot
1660 * have a fast clear operation in the pipe at the same time as other
1661 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
1662 * that the contents of the previous draw hit the render target before we
1663 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
1664 * that it is completed before any additional drawing occurs.
1665 */
1666 cmd_buffer->state.pending_pipe_bits |=
1667 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1668
1669 switch (mcs_op) {
1670 case ISL_AUX_OP_FAST_CLEAR:
1671 blorp_fast_clear(&batch, &surf, surf.surf->format,
1672 0, base_layer, layer_count,
1673 0, 0, image->extent.width, image->extent.height);
1674 break;
1675 case ISL_AUX_OP_PARTIAL_RESOLVE:
1676 blorp_mcs_partial_resolve(&batch, &surf, surf.surf->format,
1677 base_layer, layer_count);
1678 break;
1679 case ISL_AUX_OP_FULL_RESOLVE:
1680 case ISL_AUX_OP_AMBIGUATE:
1681 default:
1682 unreachable("Unsupported MCS operation");
1683 }
1684
1685 cmd_buffer->state.pending_pipe_bits |=
1686 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1687
1688 blorp_batch_finish(&batch);
1689 }
1690
1691 void
1692 anv_image_ccs_op(struct anv_cmd_buffer *cmd_buffer,
1693 const struct anv_image *image,
1694 VkImageAspectFlagBits aspect, uint32_t level,
1695 uint32_t base_layer, uint32_t layer_count,
1696 enum isl_aux_op ccs_op, union isl_color_value *clear_value,
1697 bool predicate)
1698 {
1699 assert(image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
1700 assert(image->samples == 1);
1701 assert(level < anv_image_aux_levels(image, aspect));
1702 /* Multi-LOD YcBcR is not allowed */
1703 assert(image->n_planes == 1 || level == 0);
1704 assert(base_layer + layer_count <=
1705 anv_image_aux_layers(image, aspect, level));
1706
1707 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
1708 uint32_t width_div = image->format->planes[plane].denominator_scales[0];
1709 uint32_t height_div = image->format->planes[plane].denominator_scales[1];
1710 uint32_t level_width = anv_minify(image->extent.width, level) / width_div;
1711 uint32_t level_height = anv_minify(image->extent.height, level) / height_div;
1712
1713 struct blorp_batch batch;
1714 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1715 predicate ? BLORP_BATCH_PREDICATE_ENABLE : 0);
1716
1717 struct blorp_surf surf;
1718 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1719 ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1720 fast_clear_aux_usage(image, aspect),
1721 &surf);
1722
1723 /* Blorp will store the clear color for us if we provide the clear color
1724 * address and we are doing a fast clear. So we save the clear value into
1725 * the blorp surface. However, in some situations we want to do a fast clear
1726 * without changing the clear value stored in the state buffer. For those
1727 * cases, we set the clear color address pointer to NULL, so blorp will not
1728 * try to store a garbage color.
1729 */
1730 if (ccs_op == ISL_AUX_OP_FAST_CLEAR) {
1731 if (clear_value)
1732 surf.clear_color = *clear_value;
1733 else
1734 surf.clear_color_addr.buffer = NULL;
1735 }
1736
1737 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1738 *
1739 * "After Render target fast clear, pipe-control with color cache
1740 * write-flush must be issued before sending any DRAW commands on
1741 * that render target."
1742 *
1743 * This comment is a bit cryptic and doesn't really tell you what's going
1744 * or what's really needed. It appears that fast clear ops are not
1745 * properly synchronized with other drawing. This means that we cannot
1746 * have a fast clear operation in the pipe at the same time as other
1747 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
1748 * that the contents of the previous draw hit the render target before we
1749 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
1750 * that it is completed before any additional drawing occurs.
1751 */
1752 cmd_buffer->state.pending_pipe_bits |=
1753 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1754
1755 switch (ccs_op) {
1756 case ISL_AUX_OP_FAST_CLEAR:
1757 blorp_fast_clear(&batch, &surf, surf.surf->format,
1758 level, base_layer, layer_count,
1759 0, 0, level_width, level_height);
1760 break;
1761 case ISL_AUX_OP_FULL_RESOLVE:
1762 case ISL_AUX_OP_PARTIAL_RESOLVE:
1763 blorp_ccs_resolve(&batch, &surf, level, base_layer, layer_count,
1764 surf.surf->format, ccs_op);
1765 break;
1766 case ISL_AUX_OP_AMBIGUATE:
1767 for (uint32_t a = 0; a < layer_count; a++) {
1768 const uint32_t layer = base_layer + a;
1769 blorp_ccs_ambiguate(&batch, &surf, level, layer);
1770 }
1771 break;
1772 default:
1773 unreachable("Unsupported CCS operation");
1774 }
1775
1776 cmd_buffer->state.pending_pipe_bits |=
1777 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1778
1779 blorp_batch_finish(&batch);
1780 }