projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
anv: Keep the shader stage in anv_shader_bin
[mesa.git] / src / intel / vulkan / anv_blorp.c
1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "anv_private.h"
25
26 static bool
27 lookup_blorp_shader(struct blorp_batch *batch,
28 const void *key, uint32_t key_size,
29 uint32_t *kernel_out, void *prog_data_out)
30 {
31 struct blorp_context *blorp = batch->blorp;
32 struct anv_device *device = blorp->driver_ctx;
33
34 /* The default cache must be a real cache */
35 assert(device->default_pipeline_cache.cache);
36
37 struct anv_shader_bin *bin =
38 anv_pipeline_cache_search(&device->default_pipeline_cache, key, key_size);
39 if (!bin)
40 return false;
41
42 /* The cache already has a reference and it's not going anywhere so there
43 * is no need to hold a second reference.
44 */
45 anv_shader_bin_unref(device, bin);
46
47 *kernel_out = bin->kernel.offset;
48 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
49
50 return true;
51 }
52
53 static bool
54 upload_blorp_shader(struct blorp_batch *batch,
55 const void *key, uint32_t key_size,
56 const void *kernel, uint32_t kernel_size,
57 const struct brw_stage_prog_data *prog_data,
58 uint32_t prog_data_size,
59 uint32_t *kernel_out, void *prog_data_out)
60 {
61 struct blorp_context *blorp = batch->blorp;
62 struct anv_device *device = blorp->driver_ctx;
63
64 /* The blorp cache must be a real cache */
65 assert(device->default_pipeline_cache.cache);
66
67 struct anv_pipeline_bind_map bind_map = {
68 .surface_count = 0,
69 .sampler_count = 0,
70 };
71
72 /* The stage in anv_shader_bin is only useful for pipeline shaders, so just pass
73 * a dummy value here so we don't need to change BLORP API.
74 *
75 * TODO: Plumb the stage information to BLORP.
76 */
77 gl_shader_stage stage = MESA_SHADER_NONE;
78 struct anv_shader_bin *bin =
79 anv_pipeline_cache_upload_kernel(&device->default_pipeline_cache, stage,
80 key, key_size, kernel, kernel_size,
81 NULL, 0,
82 prog_data, prog_data_size,
83 NULL, 0, NULL, &bind_map);
84
85 if (!bin)
86 return false;
87
88 /* The cache already has a reference and it's not going anywhere so there
89 * is no need to hold a second reference.
90 */
91 anv_shader_bin_unref(device, bin);
92
93 *kernel_out = bin->kernel.offset;
94 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data;
95
96 return true;
97 }
98
99 void
100 anv_device_init_blorp(struct anv_device *device)
101 {
102 blorp_init(&device->blorp, device, &device->isl_dev);
103 device->blorp.compiler = device->physical->compiler;
104 device->blorp.lookup_shader = lookup_blorp_shader;
105 device->blorp.upload_shader = upload_blorp_shader;
106 switch (device->info.gen) {
107 case 7:
108 if (device->info.is_haswell) {
109 device->blorp.exec = gen75_blorp_exec;
110 } else {
111 device->blorp.exec = gen7_blorp_exec;
112 }
113 break;
114 case 8:
115 device->blorp.exec = gen8_blorp_exec;
116 break;
117 case 9:
118 device->blorp.exec = gen9_blorp_exec;
119 break;
120 case 10:
121 device->blorp.exec = gen10_blorp_exec;
122 break;
123 case 11:
124 device->blorp.exec = gen11_blorp_exec;
125 break;
126 case 12:
127 device->blorp.exec = gen12_blorp_exec;
128 break;
129 default:
130 unreachable("Unknown hardware generation");
131 }
132 }
133
134 void
135 anv_device_finish_blorp(struct anv_device *device)
136 {
137 blorp_finish(&device->blorp);
138 }
139
140 static void
141 get_blorp_surf_for_anv_buffer(struct anv_device *device,
142 struct anv_buffer *buffer, uint64_t offset,
143 uint32_t width, uint32_t height,
144 uint32_t row_pitch, enum isl_format format,
145 struct blorp_surf *blorp_surf,
146 struct isl_surf *isl_surf)
147 {
148 const struct isl_format_layout *fmtl =
149 isl_format_get_layout(format);
150 bool ok UNUSED;
151
152 /* ASTC is the only format which doesn't support linear layouts.
153 * Create an equivalently sized surface with ISL to get around this.
154 */
155 if (fmtl->txc == ISL_TXC_ASTC) {
156 /* Use an equivalently sized format */
157 format = ISL_FORMAT_R32G32B32A32_UINT;
158 assert(fmtl->bpb == isl_format_get_layout(format)->bpb);
159
160 /* Shrink the dimensions for the new format */
161 width = DIV_ROUND_UP(width, fmtl->bw);
162 height = DIV_ROUND_UP(height, fmtl->bh);
163 }
164
165 *blorp_surf = (struct blorp_surf) {
166 .surf = isl_surf,
167 .addr = {
168 .buffer = buffer->address.bo,
169 .offset = buffer->address.offset + offset,
170 .mocs = anv_mocs_for_bo(device, buffer->address.bo),
171 },
172 };
173
174 ok = isl_surf_init(&device->isl_dev, isl_surf,
175 .dim = ISL_SURF_DIM_2D,
176 .format = format,
177 .width = width,
178 .height = height,
179 .depth = 1,
180 .levels = 1,
181 .array_len = 1,
182 .samples = 1,
183 .row_pitch_B = row_pitch,
184 .usage = ISL_SURF_USAGE_TEXTURE_BIT |
185 ISL_SURF_USAGE_RENDER_TARGET_BIT,
186 .tiling_flags = ISL_TILING_LINEAR_BIT);
187 assert(ok);
188 }
189
190 /* Pick something high enough that it won't be used in core and low enough it
191 * will never map to an extension.
192 */
193 #define ANV_IMAGE_LAYOUT_EXPLICIT_AUX (VkImageLayout)10000000
194
195 static struct blorp_address
196 anv_to_blorp_address(struct anv_address addr)
197 {
198 return (struct blorp_address) {
199 .buffer = addr.bo,
200 .offset = addr.offset,
201 };
202 }
203
204 static void
205 get_blorp_surf_for_anv_image(const struct anv_device *device,
206 const struct anv_image *image,
207 VkImageAspectFlags aspect,
208 VkImageUsageFlags usage,
209 VkImageLayout layout,
210 enum isl_aux_usage aux_usage,
211 struct blorp_surf *blorp_surf)
212 {
213 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
214
215 if (layout != ANV_IMAGE_LAYOUT_EXPLICIT_AUX) {
216 assert(usage != 0);
217 aux_usage = anv_layout_to_aux_usage(&device->info, image,
218 aspect, usage, layout);
219 }
220
221 const struct anv_surface *surface = &image->planes[plane].surface;
222 *blorp_surf = (struct blorp_surf) {
223 .surf = &surface->isl,
224 .addr = {
225 .buffer = image->planes[plane].address.bo,
226 .offset = image->planes[plane].address.offset + surface->offset,
227 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
228 },
229 };
230
231 if (aux_usage != ISL_AUX_USAGE_NONE) {
232 const struct anv_surface *aux_surface = &image->planes[plane].aux_surface;
233 blorp_surf->aux_surf = &aux_surface->isl,
234 blorp_surf->aux_addr = (struct blorp_address) {
235 .buffer = image->planes[plane].address.bo,
236 .offset = image->planes[plane].address.offset + aux_surface->offset,
237 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
238 };
239 blorp_surf->aux_usage = aux_usage;
240
241 /* If we're doing a partial resolve, then we need the indirect clear
242 * color. If we are doing a fast clear and want to store/update the
243 * clear color, we also pass the address to blorp, otherwise it will only
244 * stomp the CCS to a particular value and won't care about format or
245 * clear value
246 */
247 if (aspect & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
248 const struct anv_address clear_color_addr =
249 anv_image_get_clear_color_addr(device, image, aspect);
250 blorp_surf->clear_color_addr = anv_to_blorp_address(clear_color_addr);
251 } else if (aspect & VK_IMAGE_ASPECT_DEPTH_BIT) {
252 if (device->info.gen >= 10) {
253 /* Vulkan always clears to 1.0. On gen < 10, we set that directly
254 * in the state packet. For gen >= 10, must provide the clear
255 * value in a buffer. We have a single global buffer that stores
256 * the 1.0 value.
257 */
258 const struct anv_address clear_color_addr = (struct anv_address) {
259 .bo = device->hiz_clear_bo,
260 };
261 blorp_surf->clear_color_addr =
262 anv_to_blorp_address(clear_color_addr);
263 } else {
264 blorp_surf->clear_color = (union isl_color_value) {
265 .f32 = { ANV_HZ_FC_VAL },
266 };
267 }
268 }
269 }
270 }
271
272 static bool
273 get_blorp_surf_for_anv_shadow_image(const struct anv_device *device,
274 const struct anv_image *image,
275 VkImageAspectFlags aspect,
276 struct blorp_surf *blorp_surf)
277 {
278
279 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
280 if (image->planes[plane].shadow_surface.isl.size_B == 0)
281 return false;
282
283 *blorp_surf = (struct blorp_surf) {
284 .surf = &image->planes[plane].shadow_surface.isl,
285 .addr = {
286 .buffer = image->planes[plane].address.bo,
287 .offset = image->planes[plane].address.offset +
288 image->planes[plane].shadow_surface.offset,
289 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo),
290 },
291 };
292
293 return true;
294 }
295
296 void anv_CmdCopyImage(
297 VkCommandBuffer commandBuffer,
298 VkImage srcImage,
299 VkImageLayout srcImageLayout,
300 VkImage dstImage,
301 VkImageLayout dstImageLayout,
302 uint32_t regionCount,
303 const VkImageCopy* pRegions)
304 {
305 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
306 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
307 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
308
309 struct blorp_batch batch;
310 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
311
312 for (unsigned r = 0; r < regionCount; r++) {
313 VkOffset3D srcOffset =
314 anv_sanitize_image_offset(src_image->type, pRegions[r].srcOffset);
315 VkOffset3D dstOffset =
316 anv_sanitize_image_offset(dst_image->type, pRegions[r].dstOffset);
317 VkExtent3D extent =
318 anv_sanitize_image_extent(src_image->type, pRegions[r].extent);
319
320 const uint32_t dst_level = pRegions[r].dstSubresource.mipLevel;
321 unsigned dst_base_layer, layer_count;
322 if (dst_image->type == VK_IMAGE_TYPE_3D) {
323 dst_base_layer = pRegions[r].dstOffset.z;
324 layer_count = pRegions[r].extent.depth;
325 } else {
326 dst_base_layer = pRegions[r].dstSubresource.baseArrayLayer;
327 layer_count =
328 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource);
329 }
330
331 const uint32_t src_level = pRegions[r].srcSubresource.mipLevel;
332 unsigned src_base_layer;
333 if (src_image->type == VK_IMAGE_TYPE_3D) {
334 src_base_layer = pRegions[r].srcOffset.z;
335 } else {
336 src_base_layer = pRegions[r].srcSubresource.baseArrayLayer;
337 assert(layer_count ==
338 anv_get_layerCount(src_image, &pRegions[r].srcSubresource));
339 }
340
341 VkImageAspectFlags src_mask = pRegions[r].srcSubresource.aspectMask,
342 dst_mask = pRegions[r].dstSubresource.aspectMask;
343
344 assert(anv_image_aspects_compatible(src_mask, dst_mask));
345
346 if (util_bitcount(src_mask) > 1) {
347 uint32_t aspect_bit;
348 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_mask) {
349 struct blorp_surf src_surf, dst_surf;
350 get_blorp_surf_for_anv_image(cmd_buffer->device,
351 src_image, 1UL << aspect_bit,
352 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
353 srcImageLayout, ISL_AUX_USAGE_NONE,
354 &src_surf);
355 get_blorp_surf_for_anv_image(cmd_buffer->device,
356 dst_image, 1UL << aspect_bit,
357 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
358 dstImageLayout, ISL_AUX_USAGE_NONE,
359 &dst_surf);
360 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
361 1UL << aspect_bit,
362 dst_surf.aux_usage, dst_level,
363 dst_base_layer, layer_count);
364
365 for (unsigned i = 0; i < layer_count; i++) {
366 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
367 &dst_surf, dst_level, dst_base_layer + i,
368 srcOffset.x, srcOffset.y,
369 dstOffset.x, dstOffset.y,
370 extent.width, extent.height);
371 }
372
373 struct blorp_surf dst_shadow_surf;
374 if (get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
375 dst_image,
376 1UL << aspect_bit,
377 &dst_shadow_surf)) {
378 for (unsigned i = 0; i < layer_count; i++) {
379 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
380 &dst_shadow_surf, dst_level, dst_base_layer + i,
381 srcOffset.x, srcOffset.y,
382 dstOffset.x, dstOffset.y,
383 extent.width, extent.height);
384 }
385 }
386 }
387 } else {
388 struct blorp_surf src_surf, dst_surf;
389 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, src_mask,
390 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
391 srcImageLayout, ISL_AUX_USAGE_NONE,
392 &src_surf);
393 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, dst_mask,
394 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
395 dstImageLayout, ISL_AUX_USAGE_NONE,
396 &dst_surf);
397 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image, dst_mask,
398 dst_surf.aux_usage, dst_level,
399 dst_base_layer, layer_count);
400
401 for (unsigned i = 0; i < layer_count; i++) {
402 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
403 &dst_surf, dst_level, dst_base_layer + i,
404 srcOffset.x, srcOffset.y,
405 dstOffset.x, dstOffset.y,
406 extent.width, extent.height);
407 }
408
409 struct blorp_surf dst_shadow_surf;
410 if (get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
411 dst_image, dst_mask,
412 &dst_shadow_surf)) {
413 for (unsigned i = 0; i < layer_count; i++) {
414 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i,
415 &dst_shadow_surf, dst_level, dst_base_layer + i,
416 srcOffset.x, srcOffset.y,
417 dstOffset.x, dstOffset.y,
418 extent.width, extent.height);
419 }
420 }
421 }
422 }
423
424 blorp_batch_finish(&batch);
425 }
426
427 static enum isl_format
428 isl_format_for_size(unsigned size_B)
429 {
430 /* Prefer 32-bit per component formats for CmdFillBuffer */
431 switch (size_B) {
432 case 1: return ISL_FORMAT_R8_UINT;
433 case 2: return ISL_FORMAT_R16_UINT;
434 case 3: return ISL_FORMAT_R8G8B8_UINT;
435 case 4: return ISL_FORMAT_R32_UINT;
436 case 6: return ISL_FORMAT_R16G16B16_UINT;
437 case 8: return ISL_FORMAT_R32G32_UINT;
438 case 12: return ISL_FORMAT_R32G32B32_UINT;
439 case 16: return ISL_FORMAT_R32G32B32A32_UINT;
440 default:
441 unreachable("Unknown format size");
442 }
443 }
444
445 static void
446 copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer,
447 struct anv_buffer *anv_buffer,
448 struct anv_image *anv_image,
449 VkImageLayout image_layout,
450 uint32_t regionCount,
451 const VkBufferImageCopy* pRegions,
452 bool buffer_to_image)
453 {
454 struct blorp_batch batch;
455 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
456
457 struct {
458 struct blorp_surf surf;
459 uint32_t level;
460 VkOffset3D offset;
461 } image, buffer, *src, *dst;
462
463 buffer.level = 0;
464 buffer.offset = (VkOffset3D) { 0, 0, 0 };
465
466 if (buffer_to_image) {
467 src = &buffer;
468 dst = &image;
469 } else {
470 src = &image;
471 dst = &buffer;
472 }
473
474 for (unsigned r = 0; r < regionCount; r++) {
475 const VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask;
476
477 get_blorp_surf_for_anv_image(cmd_buffer->device, anv_image, aspect,
478 buffer_to_image ?
479 VK_IMAGE_USAGE_TRANSFER_DST_BIT :
480 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
481 image_layout, ISL_AUX_USAGE_NONE,
482 &image.surf);
483 image.offset =
484 anv_sanitize_image_offset(anv_image->type, pRegions[r].imageOffset);
485 image.level = pRegions[r].imageSubresource.mipLevel;
486
487 VkExtent3D extent =
488 anv_sanitize_image_extent(anv_image->type, pRegions[r].imageExtent);
489 if (anv_image->type != VK_IMAGE_TYPE_3D) {
490 image.offset.z = pRegions[r].imageSubresource.baseArrayLayer;
491 extent.depth =
492 anv_get_layerCount(anv_image, &pRegions[r].imageSubresource);
493 }
494
495 const enum isl_format linear_format =
496 anv_get_isl_format(&cmd_buffer->device->info, anv_image->vk_format,
497 aspect, VK_IMAGE_TILING_LINEAR);
498 const struct isl_format_layout *linear_fmtl =
499 isl_format_get_layout(linear_format);
500
501 const uint32_t buffer_row_length =
502 pRegions[r].bufferRowLength ?
503 pRegions[r].bufferRowLength : extent.width;
504
505 const uint32_t buffer_image_height =
506 pRegions[r].bufferImageHeight ?
507 pRegions[r].bufferImageHeight : extent.height;
508
509 const uint32_t buffer_row_pitch =
510 DIV_ROUND_UP(buffer_row_length, linear_fmtl->bw) *
511 (linear_fmtl->bpb / 8);
512
513 const uint32_t buffer_layer_stride =
514 DIV_ROUND_UP(buffer_image_height, linear_fmtl->bh) *
515 buffer_row_pitch;
516
517 /* Some formats have additional restrictions which may cause ISL to
518 * fail to create a surface for us. Some examples include:
519 *
520 * 1. ASTC formats are not allowed to be LINEAR and must be tiled
521 * 2. YCbCr formats have to have 2-pixel aligned strides
522 *
523 * To avoid these issues, we always bind the buffer as if it's a
524 * "normal" format like RGBA32_UINT. Since we're using blorp_copy,
525 * the format doesn't matter as long as it has the right bpb.
526 */
527 const VkExtent2D buffer_extent = {
528 .width = DIV_ROUND_UP(extent.width, linear_fmtl->bw),
529 .height = DIV_ROUND_UP(extent.height, linear_fmtl->bh),
530 };
531 const enum isl_format buffer_format =
532 isl_format_for_size(linear_fmtl->bpb / 8);
533
534 struct isl_surf buffer_isl_surf;
535 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
536 anv_buffer, pRegions[r].bufferOffset,
537 buffer_extent.width, buffer_extent.height,
538 buffer_row_pitch, buffer_format,
539 &buffer.surf, &buffer_isl_surf);
540
541 bool dst_has_shadow = false;
542 struct blorp_surf dst_shadow_surf;
543 if (&image == dst) {
544 /* In this case, the source is the buffer and, since blorp takes its
545 * copy dimensions in terms of the source format, we have to use the
546 * scaled down version for compressed textures because the source
547 * format is an RGB format.
548 */
549 extent.width = buffer_extent.width;
550 extent.height = buffer_extent.height;
551
552 anv_cmd_buffer_mark_image_written(cmd_buffer, anv_image,
553 aspect, dst->surf.aux_usage,
554 dst->level,
555 dst->offset.z, extent.depth);
556
557 dst_has_shadow =
558 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
559 anv_image, aspect,
560 &dst_shadow_surf);
561 }
562
563 for (unsigned z = 0; z < extent.depth; z++) {
564 blorp_copy(&batch, &src->surf, src->level, src->offset.z,
565 &dst->surf, dst->level, dst->offset.z,
566 src->offset.x, src->offset.y, dst->offset.x, dst->offset.y,
567 extent.width, extent.height);
568
569 if (dst_has_shadow) {
570 blorp_copy(&batch, &src->surf, src->level, src->offset.z,
571 &dst_shadow_surf, dst->level, dst->offset.z,
572 src->offset.x, src->offset.y,
573 dst->offset.x, dst->offset.y,
574 extent.width, extent.height);
575 }
576
577 image.offset.z++;
578 buffer.surf.addr.offset += buffer_layer_stride;
579 }
580 }
581
582 blorp_batch_finish(&batch);
583 }
584
585 void anv_CmdCopyBufferToImage(
586 VkCommandBuffer commandBuffer,
587 VkBuffer srcBuffer,
588 VkImage dstImage,
589 VkImageLayout dstImageLayout,
590 uint32_t regionCount,
591 const VkBufferImageCopy* pRegions)
592 {
593 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
594 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
595 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
596
597 copy_buffer_to_image(cmd_buffer, src_buffer, dst_image, dstImageLayout,
598 regionCount, pRegions, true);
599 }
600
601 void anv_CmdCopyImageToBuffer(
602 VkCommandBuffer commandBuffer,
603 VkImage srcImage,
604 VkImageLayout srcImageLayout,
605 VkBuffer dstBuffer,
606 uint32_t regionCount,
607 const VkBufferImageCopy* pRegions)
608 {
609 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
610 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
611 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
612
613 copy_buffer_to_image(cmd_buffer, dst_buffer, src_image, srcImageLayout,
614 regionCount, pRegions, false);
615
616 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
617 }
618
619 static bool
620 flip_coords(unsigned *src0, unsigned *src1, unsigned *dst0, unsigned *dst1)
621 {
622 bool flip = false;
623 if (*src0 > *src1) {
624 unsigned tmp = *src0;
625 *src0 = *src1;
626 *src1 = tmp;
627 flip = !flip;
628 }
629
630 if (*dst0 > *dst1) {
631 unsigned tmp = *dst0;
632 *dst0 = *dst1;
633 *dst1 = tmp;
634 flip = !flip;
635 }
636
637 return flip;
638 }
639
640 void anv_CmdBlitImage(
641 VkCommandBuffer commandBuffer,
642 VkImage srcImage,
643 VkImageLayout srcImageLayout,
644 VkImage dstImage,
645 VkImageLayout dstImageLayout,
646 uint32_t regionCount,
647 const VkImageBlit* pRegions,
648 VkFilter filter)
649
650 {
651 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
652 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
653 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
654
655 struct blorp_surf src, dst;
656
657 enum blorp_filter blorp_filter;
658 switch (filter) {
659 case VK_FILTER_NEAREST:
660 blorp_filter = BLORP_FILTER_NEAREST;
661 break;
662 case VK_FILTER_LINEAR:
663 blorp_filter = BLORP_FILTER_BILINEAR;
664 break;
665 default:
666 unreachable("Invalid filter");
667 }
668
669 struct blorp_batch batch;
670 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
671
672 for (unsigned r = 0; r < regionCount; r++) {
673 const VkImageSubresourceLayers *src_res = &pRegions[r].srcSubresource;
674 const VkImageSubresourceLayers *dst_res = &pRegions[r].dstSubresource;
675
676 assert(anv_image_aspects_compatible(src_res->aspectMask,
677 dst_res->aspectMask));
678
679 uint32_t aspect_bit;
680 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_res->aspectMask) {
681 get_blorp_surf_for_anv_image(cmd_buffer->device,
682 src_image, 1U << aspect_bit,
683 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
684 srcImageLayout, ISL_AUX_USAGE_NONE, &src);
685 get_blorp_surf_for_anv_image(cmd_buffer->device,
686 dst_image, 1U << aspect_bit,
687 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
688 dstImageLayout, ISL_AUX_USAGE_NONE, &dst);
689
690 struct anv_format_plane src_format =
691 anv_get_format_plane(&cmd_buffer->device->info, src_image->vk_format,
692 1U << aspect_bit, src_image->tiling);
693 struct anv_format_plane dst_format =
694 anv_get_format_plane(&cmd_buffer->device->info, dst_image->vk_format,
695 1U << aspect_bit, dst_image->tiling);
696
697 unsigned dst_start, dst_end;
698 if (dst_image->type == VK_IMAGE_TYPE_3D) {
699 assert(dst_res->baseArrayLayer == 0);
700 dst_start = pRegions[r].dstOffsets[0].z;
701 dst_end = pRegions[r].dstOffsets[1].z;
702 } else {
703 dst_start = dst_res->baseArrayLayer;
704 dst_end = dst_start + anv_get_layerCount(dst_image, dst_res);
705 }
706
707 unsigned src_start, src_end;
708 if (src_image->type == VK_IMAGE_TYPE_3D) {
709 assert(src_res->baseArrayLayer == 0);
710 src_start = pRegions[r].srcOffsets[0].z;
711 src_end = pRegions[r].srcOffsets[1].z;
712 } else {
713 src_start = src_res->baseArrayLayer;
714 src_end = src_start + anv_get_layerCount(src_image, src_res);
715 }
716
717 bool flip_z = flip_coords(&src_start, &src_end, &dst_start, &dst_end);
718 float src_z_step = (float)(src_end + 1 - src_start) /
719 (float)(dst_end + 1 - dst_start);
720
721 if (flip_z) {
722 src_start = src_end;
723 src_z_step *= -1;
724 }
725
726 unsigned src_x0 = pRegions[r].srcOffsets[0].x;
727 unsigned src_x1 = pRegions[r].srcOffsets[1].x;
728 unsigned dst_x0 = pRegions[r].dstOffsets[0].x;
729 unsigned dst_x1 = pRegions[r].dstOffsets[1].x;
730 bool flip_x = flip_coords(&src_x0, &src_x1, &dst_x0, &dst_x1);
731
732 unsigned src_y0 = pRegions[r].srcOffsets[0].y;
733 unsigned src_y1 = pRegions[r].srcOffsets[1].y;
734 unsigned dst_y0 = pRegions[r].dstOffsets[0].y;
735 unsigned dst_y1 = pRegions[r].dstOffsets[1].y;
736 bool flip_y = flip_coords(&src_y0, &src_y1, &dst_y0, &dst_y1);
737
738 const unsigned num_layers = dst_end - dst_start;
739 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
740 1U << aspect_bit,
741 dst.aux_usage,
742 dst_res->mipLevel,
743 dst_start, num_layers);
744
745 for (unsigned i = 0; i < num_layers; i++) {
746 unsigned dst_z = dst_start + i;
747 unsigned src_z = src_start + i * src_z_step;
748
749 blorp_blit(&batch, &src, src_res->mipLevel, src_z,
750 src_format.isl_format, src_format.swizzle,
751 &dst, dst_res->mipLevel, dst_z,
752 dst_format.isl_format, dst_format.swizzle,
753 src_x0, src_y0, src_x1, src_y1,
754 dst_x0, dst_y0, dst_x1, dst_y1,
755 blorp_filter, flip_x, flip_y);
756 }
757 }
758 }
759
760 blorp_batch_finish(&batch);
761 }
762
763 /**
764 * Returns the greatest common divisor of a and b that is a power of two.
765 */
766 static uint64_t
767 gcd_pow2_u64(uint64_t a, uint64_t b)
768 {
769 assert(a > 0 || b > 0);
770
771 unsigned a_log2 = ffsll(a) - 1;
772 unsigned b_log2 = ffsll(b) - 1;
773
774 /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which
775 * case, the MIN2() will take the other one. If both are 0 then we will
776 * hit the assert above.
777 */
778 return 1 << MIN2(a_log2, b_log2);
779 }
780
781 /* This is maximum possible width/height our HW can handle */
782 #define MAX_SURFACE_DIM (1ull << 14)
783
784 void anv_CmdCopyBuffer(
785 VkCommandBuffer commandBuffer,
786 VkBuffer srcBuffer,
787 VkBuffer dstBuffer,
788 uint32_t regionCount,
789 const VkBufferCopy* pRegions)
790 {
791 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
792 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
793 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
794
795 struct blorp_batch batch;
796 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
797
798 for (unsigned r = 0; r < regionCount; r++) {
799 struct blorp_address src = {
800 .buffer = src_buffer->address.bo,
801 .offset = src_buffer->address.offset + pRegions[r].srcOffset,
802 .mocs = anv_mocs_for_bo(cmd_buffer->device, src_buffer->address.bo),
803 };
804 struct blorp_address dst = {
805 .buffer = dst_buffer->address.bo,
806 .offset = dst_buffer->address.offset + pRegions[r].dstOffset,
807 .mocs = anv_mocs_for_bo(cmd_buffer->device, dst_buffer->address.bo),
808 };
809
810 blorp_buffer_copy(&batch, src, dst, pRegions[r].size);
811 }
812
813 blorp_batch_finish(&batch);
814
815 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
816 }
817
818 void anv_CmdUpdateBuffer(
819 VkCommandBuffer commandBuffer,
820 VkBuffer dstBuffer,
821 VkDeviceSize dstOffset,
822 VkDeviceSize dataSize,
823 const void* pData)
824 {
825 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
826 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
827
828 struct blorp_batch batch;
829 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
830
831 /* We can't quite grab a full block because the state stream needs a
832 * little data at the top to build its linked list.
833 */
834 const uint32_t max_update_size =
835 cmd_buffer->device->dynamic_state_pool.block_size - 64;
836
837 assert(max_update_size < MAX_SURFACE_DIM * 4);
838
839 /* We're about to read data that was written from the CPU. Flush the
840 * texture cache so we don't get anything stale.
841 */
842 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
843
844 while (dataSize) {
845 const uint32_t copy_size = MIN2(dataSize, max_update_size);
846
847 struct anv_state tmp_data =
848 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, copy_size, 64);
849
850 memcpy(tmp_data.map, pData, copy_size);
851
852 struct blorp_address src = {
853 .buffer = cmd_buffer->device->dynamic_state_pool.block_pool.bo,
854 .offset = tmp_data.offset,
855 .mocs = cmd_buffer->device->isl_dev.mocs.internal,
856 };
857 struct blorp_address dst = {
858 .buffer = dst_buffer->address.bo,
859 .offset = dst_buffer->address.offset + dstOffset,
860 .mocs = anv_mocs_for_bo(cmd_buffer->device, dst_buffer->address.bo),
861 };
862
863 blorp_buffer_copy(&batch, src, dst, copy_size);
864
865 dataSize -= copy_size;
866 dstOffset += copy_size;
867 pData = (void *)pData + copy_size;
868 }
869
870 blorp_batch_finish(&batch);
871
872 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
873 }
874
875 void anv_CmdFillBuffer(
876 VkCommandBuffer commandBuffer,
877 VkBuffer dstBuffer,
878 VkDeviceSize dstOffset,
879 VkDeviceSize fillSize,
880 uint32_t data)
881 {
882 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
883 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
884 struct blorp_surf surf;
885 struct isl_surf isl_surf;
886
887 struct blorp_batch batch;
888 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
889
890 fillSize = anv_buffer_get_range(dst_buffer, dstOffset, fillSize);
891
892 /* From the Vulkan spec:
893 *
894 * "size is the number of bytes to fill, and must be either a multiple
895 * of 4, or VK_WHOLE_SIZE to fill the range from offset to the end of
896 * the buffer. If VK_WHOLE_SIZE is used and the remaining size of the
897 * buffer is not a multiple of 4, then the nearest smaller multiple is
898 * used."
899 */
900 fillSize &= ~3ull;
901
902 /* First, we compute the biggest format that can be used with the
903 * given offsets and size.
904 */
905 int bs = 16;
906 bs = gcd_pow2_u64(bs, dstOffset);
907 bs = gcd_pow2_u64(bs, fillSize);
908 enum isl_format isl_format = isl_format_for_size(bs);
909
910 union isl_color_value color = {
911 .u32 = { data, data, data, data },
912 };
913
914 const uint64_t max_fill_size = MAX_SURFACE_DIM * MAX_SURFACE_DIM * bs;
915 while (fillSize >= max_fill_size) {
916 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
917 dst_buffer, dstOffset,
918 MAX_SURFACE_DIM, MAX_SURFACE_DIM,
919 MAX_SURFACE_DIM * bs, isl_format,
920 &surf, &isl_surf);
921
922 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
923 0, 0, 1, 0, 0, MAX_SURFACE_DIM, MAX_SURFACE_DIM,
924 color, NULL);
925 fillSize -= max_fill_size;
926 dstOffset += max_fill_size;
927 }
928
929 uint64_t height = fillSize / (MAX_SURFACE_DIM * bs);
930 assert(height < MAX_SURFACE_DIM);
931 if (height != 0) {
932 const uint64_t rect_fill_size = height * MAX_SURFACE_DIM * bs;
933 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
934 dst_buffer, dstOffset,
935 MAX_SURFACE_DIM, height,
936 MAX_SURFACE_DIM * bs, isl_format,
937 &surf, &isl_surf);
938
939 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
940 0, 0, 1, 0, 0, MAX_SURFACE_DIM, height,
941 color, NULL);
942 fillSize -= rect_fill_size;
943 dstOffset += rect_fill_size;
944 }
945
946 if (fillSize != 0) {
947 const uint32_t width = fillSize / bs;
948 get_blorp_surf_for_anv_buffer(cmd_buffer->device,
949 dst_buffer, dstOffset,
950 width, 1,
951 width * bs, isl_format,
952 &surf, &isl_surf);
953
954 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY,
955 0, 0, 1, 0, 0, width, 1,
956 color, NULL);
957 }
958
959 blorp_batch_finish(&batch);
960
961 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES;
962 }
963
964 void anv_CmdClearColorImage(
965 VkCommandBuffer commandBuffer,
966 VkImage _image,
967 VkImageLayout imageLayout,
968 const VkClearColorValue* pColor,
969 uint32_t rangeCount,
970 const VkImageSubresourceRange* pRanges)
971 {
972 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
973 ANV_FROM_HANDLE(anv_image, image, _image);
974
975 static const bool color_write_disable[4] = { false, false, false, false };
976
977 struct blorp_batch batch;
978 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
979
980
981 for (unsigned r = 0; r < rangeCount; r++) {
982 if (pRanges[r].aspectMask == 0)
983 continue;
984
985 assert(pRanges[r].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
986
987 struct blorp_surf surf;
988 get_blorp_surf_for_anv_image(cmd_buffer->device,
989 image, pRanges[r].aspectMask,
990 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
991 imageLayout, ISL_AUX_USAGE_NONE, &surf);
992
993 struct anv_format_plane src_format =
994 anv_get_format_plane(&cmd_buffer->device->info, image->vk_format,
995 VK_IMAGE_ASPECT_COLOR_BIT, image->tiling);
996
997 unsigned base_layer = pRanges[r].baseArrayLayer;
998 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
999
1000 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
1001 const unsigned level = pRanges[r].baseMipLevel + i;
1002 const unsigned level_width = anv_minify(image->extent.width, level);
1003 const unsigned level_height = anv_minify(image->extent.height, level);
1004
1005 if (image->type == VK_IMAGE_TYPE_3D) {
1006 base_layer = 0;
1007 layer_count = anv_minify(image->extent.depth, level);
1008 }
1009
1010 anv_cmd_buffer_mark_image_written(cmd_buffer, image,
1011 pRanges[r].aspectMask,
1012 surf.aux_usage, level,
1013 base_layer, layer_count);
1014
1015 blorp_clear(&batch, &surf,
1016 src_format.isl_format, src_format.swizzle,
1017 level, base_layer, layer_count,
1018 0, 0, level_width, level_height,
1019 vk_to_isl_color(*pColor), color_write_disable);
1020 }
1021 }
1022
1023 blorp_batch_finish(&batch);
1024 }
1025
1026 void anv_CmdClearDepthStencilImage(
1027 VkCommandBuffer commandBuffer,
1028 VkImage image_h,
1029 VkImageLayout imageLayout,
1030 const VkClearDepthStencilValue* pDepthStencil,
1031 uint32_t rangeCount,
1032 const VkImageSubresourceRange* pRanges)
1033 {
1034 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1035 ANV_FROM_HANDLE(anv_image, image, image_h);
1036
1037 struct blorp_batch batch;
1038 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1039
1040 struct blorp_surf depth, stencil, stencil_shadow;
1041 if (image->aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1042 get_blorp_surf_for_anv_image(cmd_buffer->device,
1043 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1044 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
1045 imageLayout, ISL_AUX_USAGE_NONE, &depth);
1046 } else {
1047 memset(&depth, 0, sizeof(depth));
1048 }
1049
1050 bool has_stencil_shadow = false;
1051 if (image->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1052 get_blorp_surf_for_anv_image(cmd_buffer->device,
1053 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1054 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
1055 imageLayout, ISL_AUX_USAGE_NONE, &stencil);
1056
1057 has_stencil_shadow =
1058 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device, image,
1059 VK_IMAGE_ASPECT_STENCIL_BIT,
1060 &stencil_shadow);
1061 } else {
1062 memset(&stencil, 0, sizeof(stencil));
1063 }
1064
1065 for (unsigned r = 0; r < rangeCount; r++) {
1066 if (pRanges[r].aspectMask == 0)
1067 continue;
1068
1069 bool clear_depth = pRanges[r].aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
1070 bool clear_stencil = pRanges[r].aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
1071
1072 unsigned base_layer = pRanges[r].baseArrayLayer;
1073 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]);
1074
1075 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) {
1076 const unsigned level = pRanges[r].baseMipLevel + i;
1077 const unsigned level_width = anv_minify(image->extent.width, level);
1078 const unsigned level_height = anv_minify(image->extent.height, level);
1079
1080 if (image->type == VK_IMAGE_TYPE_3D)
1081 layer_count = anv_minify(image->extent.depth, level);
1082
1083 blorp_clear_depth_stencil(&batch, &depth, &stencil,
1084 level, base_layer, layer_count,
1085 0, 0, level_width, level_height,
1086 clear_depth, pDepthStencil->depth,
1087 clear_stencil ? 0xff : 0,
1088 pDepthStencil->stencil);
1089
1090 if (clear_stencil && has_stencil_shadow) {
1091 union isl_color_value stencil_color = {
1092 .u32 = { pDepthStencil->stencil, },
1093 };
1094 blorp_clear(&batch, &stencil_shadow,
1095 ISL_FORMAT_R8_UINT, ISL_SWIZZLE_IDENTITY,
1096 level, base_layer, layer_count,
1097 0, 0, level_width, level_height,
1098 stencil_color, NULL);
1099 }
1100 }
1101 }
1102
1103 blorp_batch_finish(&batch);
1104 }
1105
1106 VkResult
1107 anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer *cmd_buffer,
1108 uint32_t num_entries,
1109 uint32_t *state_offset,
1110 struct anv_state *bt_state)
1111 {
1112 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
1113 state_offset);
1114 if (bt_state->map == NULL) {
1115 /* We ran out of space. Grab a new binding table block. */
1116 VkResult result = anv_cmd_buffer_new_binding_table_block(cmd_buffer);
1117 if (result != VK_SUCCESS)
1118 return result;
1119
1120 /* Re-emit state base addresses so we get the new surface state base
1121 * address before we start emitting binding tables etc.
1122 */
1123 anv_cmd_buffer_emit_state_base_address(cmd_buffer);
1124
1125 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries,
1126 state_offset);
1127 assert(bt_state->map != NULL);
1128 }
1129
1130 return VK_SUCCESS;
1131 }
1132
1133 static VkResult
1134 binding_table_for_surface_state(struct anv_cmd_buffer *cmd_buffer,
1135 struct anv_state surface_state,
1136 uint32_t *bt_offset)
1137 {
1138 uint32_t state_offset;
1139 struct anv_state bt_state;
1140
1141 VkResult result =
1142 anv_cmd_buffer_alloc_blorp_binding_table(cmd_buffer, 1, &state_offset,
1143 &bt_state);
1144 if (result != VK_SUCCESS)
1145 return result;
1146
1147 uint32_t *bt_map = bt_state.map;
1148 bt_map[0] = surface_state.offset + state_offset;
1149
1150 *bt_offset = bt_state.offset;
1151 return VK_SUCCESS;
1152 }
1153
1154 static void
1155 clear_color_attachment(struct anv_cmd_buffer *cmd_buffer,
1156 struct blorp_batch *batch,
1157 const VkClearAttachment *attachment,
1158 uint32_t rectCount, const VkClearRect *pRects)
1159 {
1160 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
1161 const uint32_t color_att = attachment->colorAttachment;
1162 assert(color_att < subpass->color_count);
1163 const uint32_t att_idx = subpass->color_attachments[color_att].attachment;
1164
1165 if (att_idx == VK_ATTACHMENT_UNUSED)
1166 return;
1167
1168 struct anv_render_pass_attachment *pass_att =
1169 &cmd_buffer->state.pass->attachments[att_idx];
1170 struct anv_attachment_state *att_state =
1171 &cmd_buffer->state.attachments[att_idx];
1172
1173 uint32_t binding_table;
1174 VkResult result =
1175 binding_table_for_surface_state(cmd_buffer, att_state->color.state,
1176 &binding_table);
1177 if (result != VK_SUCCESS)
1178 return;
1179
1180 union isl_color_value clear_color =
1181 vk_to_isl_color(attachment->clearValue.color);
1182
1183 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1184 if (subpass->view_mask) {
1185 uint32_t view_idx;
1186 for_each_bit(view_idx, subpass->view_mask) {
1187 for (uint32_t r = 0; r < rectCount; ++r) {
1188 const VkOffset2D offset = pRects[r].rect.offset;
1189 const VkExtent2D extent = pRects[r].rect.extent;
1190 blorp_clear_attachments(batch, binding_table,
1191 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1192 view_idx, 1,
1193 offset.x, offset.y,
1194 offset.x + extent.width,
1195 offset.y + extent.height,
1196 true, clear_color, false, 0.0f, 0, 0);
1197 }
1198 }
1199 return;
1200 }
1201
1202 for (uint32_t r = 0; r < rectCount; ++r) {
1203 const VkOffset2D offset = pRects[r].rect.offset;
1204 const VkExtent2D extent = pRects[r].rect.extent;
1205 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1206 blorp_clear_attachments(batch, binding_table,
1207 ISL_FORMAT_UNSUPPORTED, pass_att->samples,
1208 pRects[r].baseArrayLayer,
1209 pRects[r].layerCount,
1210 offset.x, offset.y,
1211 offset.x + extent.width, offset.y + extent.height,
1212 true, clear_color, false, 0.0f, 0, 0);
1213 }
1214 }
1215
1216 static void
1217 clear_depth_stencil_attachment(struct anv_cmd_buffer *cmd_buffer,
1218 struct blorp_batch *batch,
1219 const VkClearAttachment *attachment,
1220 uint32_t rectCount, const VkClearRect *pRects)
1221 {
1222 static const union isl_color_value color_value = { .u32 = { 0, } };
1223 const struct anv_subpass *subpass = cmd_buffer->state.subpass;
1224 if (!subpass->depth_stencil_attachment)
1225 return;
1226
1227 const uint32_t att_idx = subpass->depth_stencil_attachment->attachment;
1228 assert(att_idx != VK_ATTACHMENT_UNUSED);
1229 struct anv_render_pass_attachment *pass_att =
1230 &cmd_buffer->state.pass->attachments[att_idx];
1231
1232 bool clear_depth = attachment->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT;
1233 bool clear_stencil = attachment->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT;
1234
1235 enum isl_format depth_format = ISL_FORMAT_UNSUPPORTED;
1236 if (clear_depth) {
1237 depth_format = anv_get_isl_format(&cmd_buffer->device->info,
1238 pass_att->format,
1239 VK_IMAGE_ASPECT_DEPTH_BIT,
1240 VK_IMAGE_TILING_OPTIMAL);
1241 }
1242
1243 uint32_t binding_table;
1244 VkResult result =
1245 binding_table_for_surface_state(cmd_buffer,
1246 cmd_buffer->state.null_surface_state,
1247 &binding_table);
1248 if (result != VK_SUCCESS)
1249 return;
1250
1251 /* If multiview is enabled we ignore baseArrayLayer and layerCount */
1252 if (subpass->view_mask) {
1253 uint32_t view_idx;
1254 for_each_bit(view_idx, subpass->view_mask) {
1255 for (uint32_t r = 0; r < rectCount; ++r) {
1256 const VkOffset2D offset = pRects[r].rect.offset;
1257 const VkExtent2D extent = pRects[r].rect.extent;
1258 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1259 blorp_clear_attachments(batch, binding_table,
1260 depth_format, pass_att->samples,
1261 view_idx, 1,
1262 offset.x, offset.y,
1263 offset.x + extent.width,
1264 offset.y + extent.height,
1265 false, color_value,
1266 clear_depth, value.depth,
1267 clear_stencil ? 0xff : 0, value.stencil);
1268 }
1269 }
1270 return;
1271 }
1272
1273 for (uint32_t r = 0; r < rectCount; ++r) {
1274 const VkOffset2D offset = pRects[r].rect.offset;
1275 const VkExtent2D extent = pRects[r].rect.extent;
1276 VkClearDepthStencilValue value = attachment->clearValue.depthStencil;
1277 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS);
1278 blorp_clear_attachments(batch, binding_table,
1279 depth_format, pass_att->samples,
1280 pRects[r].baseArrayLayer,
1281 pRects[r].layerCount,
1282 offset.x, offset.y,
1283 offset.x + extent.width, offset.y + extent.height,
1284 false, color_value,
1285 clear_depth, value.depth,
1286 clear_stencil ? 0xff : 0, value.stencil);
1287 }
1288 }
1289
1290 void anv_CmdClearAttachments(
1291 VkCommandBuffer commandBuffer,
1292 uint32_t attachmentCount,
1293 const VkClearAttachment* pAttachments,
1294 uint32_t rectCount,
1295 const VkClearRect* pRects)
1296 {
1297 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1298
1299 /* Because this gets called within a render pass, we tell blorp not to
1300 * trash our depth and stencil buffers.
1301 */
1302 struct blorp_batch batch;
1303 enum blorp_batch_flags flags = BLORP_BATCH_NO_EMIT_DEPTH_STENCIL;
1304 if (cmd_buffer->state.conditional_render_enabled) {
1305 anv_cmd_emit_conditional_render_predicate(cmd_buffer);
1306 flags |= BLORP_BATCH_PREDICATE_ENABLE;
1307 }
1308 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, flags);
1309
1310 for (uint32_t a = 0; a < attachmentCount; ++a) {
1311 if (pAttachments[a].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
1312 assert(pAttachments[a].aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
1313 clear_color_attachment(cmd_buffer, &batch,
1314 &pAttachments[a],
1315 rectCount, pRects);
1316 } else {
1317 clear_depth_stencil_attachment(cmd_buffer, &batch,
1318 &pAttachments[a],
1319 rectCount, pRects);
1320 }
1321 }
1322
1323 blorp_batch_finish(&batch);
1324 }
1325
1326 enum subpass_stage {
1327 SUBPASS_STAGE_LOAD,
1328 SUBPASS_STAGE_DRAW,
1329 SUBPASS_STAGE_RESOLVE,
1330 };
1331
1332 void
1333 anv_image_msaa_resolve(struct anv_cmd_buffer *cmd_buffer,
1334 const struct anv_image *src_image,
1335 enum isl_aux_usage src_aux_usage,
1336 uint32_t src_level, uint32_t src_base_layer,
1337 const struct anv_image *dst_image,
1338 enum isl_aux_usage dst_aux_usage,
1339 uint32_t dst_level, uint32_t dst_base_layer,
1340 VkImageAspectFlagBits aspect,
1341 uint32_t src_x, uint32_t src_y,
1342 uint32_t dst_x, uint32_t dst_y,
1343 uint32_t width, uint32_t height,
1344 uint32_t layer_count,
1345 enum blorp_filter filter)
1346 {
1347 struct blorp_batch batch;
1348 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1349
1350 assert(src_image->type == VK_IMAGE_TYPE_2D);
1351 assert(src_image->samples > 1);
1352 assert(dst_image->type == VK_IMAGE_TYPE_2D);
1353 assert(dst_image->samples == 1);
1354 assert(src_image->n_planes == dst_image->n_planes);
1355 assert(!src_image->format->can_ycbcr);
1356 assert(!dst_image->format->can_ycbcr);
1357
1358 struct blorp_surf src_surf, dst_surf;
1359 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, aspect,
1360 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1361 src_aux_usage, &src_surf);
1362 if (src_aux_usage == ISL_AUX_USAGE_MCS) {
1363 src_surf.clear_color_addr = anv_to_blorp_address(
1364 anv_image_get_clear_color_addr(cmd_buffer->device, src_image,
1365 VK_IMAGE_ASPECT_COLOR_BIT));
1366 }
1367 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, aspect,
1368 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1369 dst_aux_usage, &dst_surf);
1370 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image,
1371 aspect, dst_aux_usage,
1372 dst_level, dst_base_layer, layer_count);
1373
1374 if (filter == BLORP_FILTER_NONE) {
1375 /* If no explicit filter is provided, then it's implied by the type of
1376 * the source image.
1377 */
1378 if ((src_surf.surf->usage & ISL_SURF_USAGE_DEPTH_BIT) ||
1379 (src_surf.surf->usage & ISL_SURF_USAGE_STENCIL_BIT) ||
1380 isl_format_has_int_channel(src_surf.surf->format)) {
1381 filter = BLORP_FILTER_SAMPLE_0;
1382 } else {
1383 filter = BLORP_FILTER_AVERAGE;
1384 }
1385 }
1386
1387 for (uint32_t l = 0; l < layer_count; l++) {
1388 blorp_blit(&batch,
1389 &src_surf, src_level, src_base_layer + l,
1390 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1391 &dst_surf, dst_level, dst_base_layer + l,
1392 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
1393 src_x, src_y, src_x + width, src_y + height,
1394 dst_x, dst_y, dst_x + width, dst_y + height,
1395 filter, false, false);
1396 }
1397
1398 blorp_batch_finish(&batch);
1399 }
1400
1401 void anv_CmdResolveImage(
1402 VkCommandBuffer commandBuffer,
1403 VkImage srcImage,
1404 VkImageLayout srcImageLayout,
1405 VkImage dstImage,
1406 VkImageLayout dstImageLayout,
1407 uint32_t regionCount,
1408 const VkImageResolve* pRegions)
1409 {
1410 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1411 ANV_FROM_HANDLE(anv_image, src_image, srcImage);
1412 ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
1413
1414 assert(!src_image->format->can_ycbcr);
1415
1416 for (uint32_t r = 0; r < regionCount; r++) {
1417 assert(pRegions[r].srcSubresource.aspectMask ==
1418 pRegions[r].dstSubresource.aspectMask);
1419 assert(anv_get_layerCount(src_image, &pRegions[r].srcSubresource) ==
1420 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource));
1421
1422 const uint32_t layer_count =
1423 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource);
1424
1425 uint32_t aspect_bit;
1426 anv_foreach_image_aspect_bit(aspect_bit, src_image,
1427 pRegions[r].srcSubresource.aspectMask) {
1428 enum isl_aux_usage src_aux_usage =
1429 anv_layout_to_aux_usage(&cmd_buffer->device->info, src_image,
1430 (1 << aspect_bit),
1431 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
1432 srcImageLayout);
1433 enum isl_aux_usage dst_aux_usage =
1434 anv_layout_to_aux_usage(&cmd_buffer->device->info, dst_image,
1435 (1 << aspect_bit),
1436 VK_IMAGE_USAGE_TRANSFER_DST_BIT,
1437 dstImageLayout);
1438
1439 anv_image_msaa_resolve(cmd_buffer,
1440 src_image, src_aux_usage,
1441 pRegions[r].srcSubresource.mipLevel,
1442 pRegions[r].srcSubresource.baseArrayLayer,
1443 dst_image, dst_aux_usage,
1444 pRegions[r].dstSubresource.mipLevel,
1445 pRegions[r].dstSubresource.baseArrayLayer,
1446 (1 << aspect_bit),
1447 pRegions[r].srcOffset.x,
1448 pRegions[r].srcOffset.y,
1449 pRegions[r].dstOffset.x,
1450 pRegions[r].dstOffset.y,
1451 pRegions[r].extent.width,
1452 pRegions[r].extent.height,
1453 layer_count, BLORP_FILTER_NONE);
1454 }
1455 }
1456 }
1457
1458 void
1459 anv_image_copy_to_shadow(struct anv_cmd_buffer *cmd_buffer,
1460 const struct anv_image *image,
1461 VkImageAspectFlagBits aspect,
1462 uint32_t base_level, uint32_t level_count,
1463 uint32_t base_layer, uint32_t layer_count)
1464 {
1465 struct blorp_batch batch;
1466 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1467
1468 /* We don't know who touched the main surface last so flush a bunch of
1469 * caches to ensure we get good data.
1470 */
1471 cmd_buffer->state.pending_pipe_bits |=
1472 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT |
1473 ANV_PIPE_DATA_CACHE_FLUSH_BIT |
1474 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT |
1475 ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
1476
1477 struct blorp_surf surf;
1478 get_blorp_surf_for_anv_image(cmd_buffer->device,
1479 image, aspect,
1480 VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
1481 VK_IMAGE_LAYOUT_GENERAL,
1482 ISL_AUX_USAGE_NONE, &surf);
1483 assert(surf.aux_usage == ISL_AUX_USAGE_NONE);
1484
1485 struct blorp_surf shadow_surf;
1486 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device,
1487 image, aspect, &shadow_surf);
1488
1489 for (uint32_t l = 0; l < level_count; l++) {
1490 const uint32_t level = base_level + l;
1491
1492 const VkExtent3D extent = {
1493 .width = anv_minify(image->extent.width, level),
1494 .height = anv_minify(image->extent.height, level),
1495 .depth = anv_minify(image->extent.depth, level),
1496 };
1497
1498 if (image->type == VK_IMAGE_TYPE_3D)
1499 layer_count = extent.depth;
1500
1501 for (uint32_t a = 0; a < layer_count; a++) {
1502 const uint32_t layer = base_layer + a;
1503
1504 blorp_copy(&batch, &surf, level, layer,
1505 &shadow_surf, level, layer,
1506 0, 0, 0, 0, extent.width, extent.height);
1507 }
1508 }
1509
1510 /* We just wrote to the buffer with the render cache. Flush it. */
1511 cmd_buffer->state.pending_pipe_bits |=
1512 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
1513
1514 blorp_batch_finish(&batch);
1515 }
1516
1517 void
1518 anv_image_clear_color(struct anv_cmd_buffer *cmd_buffer,
1519 const struct anv_image *image,
1520 VkImageAspectFlagBits aspect,
1521 enum isl_aux_usage aux_usage,
1522 enum isl_format format, struct isl_swizzle swizzle,
1523 uint32_t level, uint32_t base_layer, uint32_t layer_count,
1524 VkRect2D area, union isl_color_value clear_color)
1525 {
1526 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1527
1528 /* We don't support planar images with multisampling yet */
1529 assert(image->n_planes == 1);
1530
1531 struct blorp_batch batch;
1532 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1533
1534 struct blorp_surf surf;
1535 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1536 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1537 aux_usage, &surf);
1538 anv_cmd_buffer_mark_image_written(cmd_buffer, image, aspect, aux_usage,
1539 level, base_layer, layer_count);
1540
1541 blorp_clear(&batch, &surf, format, anv_swizzle_for_render(swizzle),
1542 level, base_layer, layer_count,
1543 area.offset.x, area.offset.y,
1544 area.offset.x + area.extent.width,
1545 area.offset.y + area.extent.height,
1546 clear_color, NULL);
1547
1548 blorp_batch_finish(&batch);
1549 }
1550
1551 void
1552 anv_image_clear_depth_stencil(struct anv_cmd_buffer *cmd_buffer,
1553 const struct anv_image *image,
1554 VkImageAspectFlags aspects,
1555 enum isl_aux_usage depth_aux_usage,
1556 uint32_t level,
1557 uint32_t base_layer, uint32_t layer_count,
1558 VkRect2D area,
1559 float depth_value, uint8_t stencil_value)
1560 {
1561 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
1562 VK_IMAGE_ASPECT_STENCIL_BIT));
1563
1564 struct blorp_batch batch;
1565 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1566
1567 struct blorp_surf depth = {};
1568 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1569 get_blorp_surf_for_anv_image(cmd_buffer->device,
1570 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1571 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1572 depth_aux_usage, &depth);
1573 depth.clear_color.f32[0] = ANV_HZ_FC_VAL;
1574 }
1575
1576 struct blorp_surf stencil = {};
1577 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1578 get_blorp_surf_for_anv_image(cmd_buffer->device,
1579 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1580 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1581 ISL_AUX_USAGE_NONE, &stencil);
1582 }
1583
1584 /* Blorp may choose to clear stencil using RGBA32_UINT for better
1585 * performance. If it does this, we need to flush it out of the depth
1586 * cache before rendering to it.
1587 */
1588 cmd_buffer->state.pending_pipe_bits |=
1589 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_END_OF_PIPE_SYNC_BIT;
1590
1591 blorp_clear_depth_stencil(&batch, &depth, &stencil,
1592 level, base_layer, layer_count,
1593 area.offset.x, area.offset.y,
1594 area.offset.x + area.extent.width,
1595 area.offset.y + area.extent.height,
1596 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
1597 depth_value,
1598 (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) ? 0xff : 0,
1599 stencil_value);
1600
1601 /* Blorp may choose to clear stencil using RGBA32_UINT for better
1602 * performance. If it does this, we need to flush it out of the render
1603 * cache before someone starts trying to do stencil on it.
1604 */
1605 cmd_buffer->state.pending_pipe_bits |=
1606 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_END_OF_PIPE_SYNC_BIT;
1607
1608 struct blorp_surf stencil_shadow;
1609 if ((aspects & VK_IMAGE_ASPECT_STENCIL_BIT) &&
1610 get_blorp_surf_for_anv_shadow_image(cmd_buffer->device, image,
1611 VK_IMAGE_ASPECT_STENCIL_BIT,
1612 &stencil_shadow)) {
1613 union isl_color_value stencil_color = {
1614 .u32 = { stencil_value },
1615 };
1616 blorp_clear(&batch, &stencil_shadow,
1617 ISL_FORMAT_R8_UINT, ISL_SWIZZLE_IDENTITY,
1618 level, base_layer, layer_count,
1619 area.offset.x, area.offset.y,
1620 area.offset.x + area.extent.width,
1621 area.offset.y + area.extent.height,
1622 stencil_color, NULL);
1623 }
1624
1625 blorp_batch_finish(&batch);
1626 }
1627
1628 void
1629 anv_image_hiz_op(struct anv_cmd_buffer *cmd_buffer,
1630 const struct anv_image *image,
1631 VkImageAspectFlagBits aspect, uint32_t level,
1632 uint32_t base_layer, uint32_t layer_count,
1633 enum isl_aux_op hiz_op)
1634 {
1635 assert(aspect == VK_IMAGE_ASPECT_DEPTH_BIT);
1636 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, level));
1637 assert(anv_image_aspect_to_plane(image->aspects,
1638 VK_IMAGE_ASPECT_DEPTH_BIT) == 0);
1639
1640 struct blorp_batch batch;
1641 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1642
1643 struct blorp_surf surf;
1644 get_blorp_surf_for_anv_image(cmd_buffer->device,
1645 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1646 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1647 ISL_AUX_USAGE_HIZ, &surf);
1648 surf.clear_color.f32[0] = ANV_HZ_FC_VAL;
1649
1650 blorp_hiz_op(&batch, &surf, level, base_layer, layer_count, hiz_op);
1651
1652 blorp_batch_finish(&batch);
1653 }
1654
1655 void
1656 anv_image_hiz_clear(struct anv_cmd_buffer *cmd_buffer,
1657 const struct anv_image *image,
1658 VkImageAspectFlags aspects,
1659 uint32_t level,
1660 uint32_t base_layer, uint32_t layer_count,
1661 VkRect2D area, uint8_t stencil_value)
1662 {
1663 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
1664 VK_IMAGE_ASPECT_STENCIL_BIT));
1665
1666 struct blorp_batch batch;
1667 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0);
1668
1669 struct blorp_surf depth = {};
1670 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1671 assert(base_layer + layer_count <=
1672 anv_image_aux_layers(image, VK_IMAGE_ASPECT_DEPTH_BIT, level));
1673 get_blorp_surf_for_anv_image(cmd_buffer->device,
1674 image, VK_IMAGE_ASPECT_DEPTH_BIT,
1675 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1676 ISL_AUX_USAGE_HIZ, &depth);
1677 depth.clear_color.f32[0] = ANV_HZ_FC_VAL;
1678 }
1679
1680 struct blorp_surf stencil = {};
1681 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1682 get_blorp_surf_for_anv_image(cmd_buffer->device,
1683 image, VK_IMAGE_ASPECT_STENCIL_BIT,
1684 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1685 ISL_AUX_USAGE_NONE, &stencil);
1686 }
1687
1688 /* From the Sky Lake PRM Volume 7, "Depth Buffer Clear":
1689 *
1690 * "The following is required when performing a depth buffer clear with
1691 * using the WM_STATE or 3DSTATE_WM:
1692 *
1693 * * If other rendering operations have preceded this clear, a
1694 * PIPE_CONTROL with depth cache flush enabled, Depth Stall bit
1695 * enabled must be issued before the rectangle primitive used for
1696 * the depth buffer clear operation.
1697 * * [...]"
1698 *
1699 * Even though the PRM only says that this is required if using 3DSTATE_WM
1700 * and a 3DPRIMITIVE, the GPU appears to also need this to avoid occasional
1701 * hangs when doing a clear with WM_HZ_OP.
1702 */
1703 cmd_buffer->state.pending_pipe_bits |=
1704 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT;
1705
1706 blorp_hiz_clear_depth_stencil(&batch, &depth, &stencil,
1707 level, base_layer, layer_count,
1708 area.offset.x, area.offset.y,
1709 area.offset.x + area.extent.width,
1710 area.offset.y + area.extent.height,
1711 aspects & VK_IMAGE_ASPECT_DEPTH_BIT,
1712 ANV_HZ_FC_VAL,
1713 aspects & VK_IMAGE_ASPECT_STENCIL_BIT,
1714 stencil_value);
1715
1716 blorp_batch_finish(&batch);
1717
1718 /* From the SKL PRM, Depth Buffer Clear:
1719 *
1720 * "Depth Buffer Clear Workaround
1721 *
1722 * Depth buffer clear pass using any of the methods (WM_STATE,
1723 * 3DSTATE_WM or 3DSTATE_WM_HZ_OP) must be followed by a PIPE_CONTROL
1724 * command with DEPTH_STALL bit and Depth FLUSH bits “set” before
1725 * starting to render. DepthStall and DepthFlush are not needed between
1726 * consecutive depth clear passes nor is it required if the depth-clear
1727 * pass was done with “full_surf_clear” bit set in the
1728 * 3DSTATE_WM_HZ_OP."
1729 *
1730 * Even though the PRM provides a bunch of conditions under which this is
1731 * supposedly unnecessary, we choose to perform the flush unconditionally
1732 * just to be safe.
1733 */
1734 cmd_buffer->state.pending_pipe_bits |=
1735 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT;
1736 }
1737
1738 void
1739 anv_image_mcs_op(struct anv_cmd_buffer *cmd_buffer,
1740 const struct anv_image *image,
1741 enum isl_format format,
1742 VkImageAspectFlagBits aspect,
1743 uint32_t base_layer, uint32_t layer_count,
1744 enum isl_aux_op mcs_op, union isl_color_value *clear_value,
1745 bool predicate)
1746 {
1747 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1748 assert(image->samples > 1);
1749 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, 0));
1750
1751 /* Multisampling with multi-planar formats is not supported */
1752 assert(image->n_planes == 1);
1753
1754 struct blorp_batch batch;
1755 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1756 BLORP_BATCH_PREDICATE_ENABLE * predicate +
1757 BLORP_BATCH_NO_UPDATE_CLEAR_COLOR * !clear_value);
1758
1759 struct blorp_surf surf;
1760 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1761 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1762 ISL_AUX_USAGE_MCS, &surf);
1763
1764 /* Blorp will store the clear color for us if we provide the clear color
1765 * address and we are doing a fast clear. So we save the clear value into
1766 * the blorp surface.
1767 */
1768 if (clear_value)
1769 surf.clear_color = *clear_value;
1770
1771 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1772 *
1773 * "After Render target fast clear, pipe-control with color cache
1774 * write-flush must be issued before sending any DRAW commands on
1775 * that render target."
1776 *
1777 * This comment is a bit cryptic and doesn't really tell you what's going
1778 * or what's really needed. It appears that fast clear ops are not
1779 * properly synchronized with other drawing. This means that we cannot
1780 * have a fast clear operation in the pipe at the same time as other
1781 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
1782 * that the contents of the previous draw hit the render target before we
1783 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
1784 * that it is completed before any additional drawing occurs.
1785 */
1786 cmd_buffer->state.pending_pipe_bits |=
1787 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_END_OF_PIPE_SYNC_BIT;
1788
1789 switch (mcs_op) {
1790 case ISL_AUX_OP_FAST_CLEAR:
1791 blorp_fast_clear(&batch, &surf, format,
1792 0, base_layer, layer_count,
1793 0, 0, image->extent.width, image->extent.height);
1794 break;
1795 case ISL_AUX_OP_PARTIAL_RESOLVE:
1796 blorp_mcs_partial_resolve(&batch, &surf, format,
1797 base_layer, layer_count);
1798 break;
1799 case ISL_AUX_OP_FULL_RESOLVE:
1800 case ISL_AUX_OP_AMBIGUATE:
1801 default:
1802 unreachable("Unsupported MCS operation");
1803 }
1804
1805 cmd_buffer->state.pending_pipe_bits |=
1806 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_END_OF_PIPE_SYNC_BIT;
1807
1808 blorp_batch_finish(&batch);
1809 }
1810
1811 void
1812 anv_image_ccs_op(struct anv_cmd_buffer *cmd_buffer,
1813 const struct anv_image *image,
1814 enum isl_format format,
1815 VkImageAspectFlagBits aspect, uint32_t level,
1816 uint32_t base_layer, uint32_t layer_count,
1817 enum isl_aux_op ccs_op, union isl_color_value *clear_value,
1818 bool predicate)
1819 {
1820 assert(image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
1821 assert(image->samples == 1);
1822 assert(level < anv_image_aux_levels(image, aspect));
1823 /* Multi-LOD YcBcR is not allowed */
1824 assert(image->n_planes == 1 || level == 0);
1825 assert(base_layer + layer_count <=
1826 anv_image_aux_layers(image, aspect, level));
1827
1828 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
1829 uint32_t width_div = image->format->planes[plane].denominator_scales[0];
1830 uint32_t height_div = image->format->planes[plane].denominator_scales[1];
1831 uint32_t level_width = anv_minify(image->extent.width, level) / width_div;
1832 uint32_t level_height = anv_minify(image->extent.height, level) / height_div;
1833
1834 struct blorp_batch batch;
1835 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer,
1836 BLORP_BATCH_PREDICATE_ENABLE * predicate +
1837 BLORP_BATCH_NO_UPDATE_CLEAR_COLOR * !clear_value);
1838
1839 struct blorp_surf surf;
1840 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect,
1841 0, ANV_IMAGE_LAYOUT_EXPLICIT_AUX,
1842 image->planes[plane].aux_usage,
1843 &surf);
1844
1845 /* Blorp will store the clear color for us if we provide the clear color
1846 * address and we are doing a fast clear. So we save the clear value into
1847 * the blorp surface.
1848 */
1849 if (clear_value)
1850 surf.clear_color = *clear_value;
1851
1852 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear":
1853 *
1854 * "After Render target fast clear, pipe-control with color cache
1855 * write-flush must be issued before sending any DRAW commands on
1856 * that render target."
1857 *
1858 * This comment is a bit cryptic and doesn't really tell you what's going
1859 * or what's really needed. It appears that fast clear ops are not
1860 * properly synchronized with other drawing. This means that we cannot
1861 * have a fast clear operation in the pipe at the same time as other
1862 * regular drawing operations. We need to use a PIPE_CONTROL to ensure
1863 * that the contents of the previous draw hit the render target before we
1864 * resolve and then use a second PIPE_CONTROL after the resolve to ensure
1865 * that it is completed before any additional drawing occurs.
1866 */
1867 cmd_buffer->state.pending_pipe_bits |=
1868 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_END_OF_PIPE_SYNC_BIT;
1869
1870 switch (ccs_op) {
1871 case ISL_AUX_OP_FAST_CLEAR:
1872 blorp_fast_clear(&batch, &surf, format,
1873 level, base_layer, layer_count,
1874 0, 0, level_width, level_height);
1875 break;
1876 case ISL_AUX_OP_FULL_RESOLVE:
1877 case ISL_AUX_OP_PARTIAL_RESOLVE:
1878 blorp_ccs_resolve(&batch, &surf, level, base_layer, layer_count,
1879 format, ccs_op);
1880 break;
1881 case ISL_AUX_OP_AMBIGUATE:
1882 for (uint32_t a = 0; a < layer_count; a++) {
1883 const uint32_t layer = base_layer + a;
1884 blorp_ccs_ambiguate(&batch, &surf, level, layer);
1885 }
1886 break;
1887 default:
1888 unreachable("Unsupported CCS operation");
1889 }
1890
1891 cmd_buffer->state.pending_pipe_bits |=
1892 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_END_OF_PIPE_SYNC_BIT;
1893
1894 blorp_batch_finish(&batch);
1895 }