2 * Copyright © 2017 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
24 * @file iris_resolve.c
26 * This file handles resolve tracking for main and auxiliary surfaces.
28 * It also handles our cache tracking. We have sets for the render cache,
29 * depth cache, and so on. If a BO is in a cache's set, then it may have
30 * data in that cache. The helpers take care of emitting flushes for
31 * render-to-texture, format reinterpretation issues, and other situations.
34 #include "util/hash_table.h"
36 #include "iris_context.h"
37 #include "compiler/nir/nir.h"
40 * Disable auxiliary buffers if a renderbuffer is also bound as a texture
41 * or shader image. This causes a self-dependency, where both rendering
42 * and sampling may concurrently read or write the CCS buffer, causing
46 disable_rb_aux_buffer(struct iris_context
*ice
,
47 bool *draw_aux_buffer_disabled
,
48 struct iris_resource
*tex_res
,
49 unsigned min_level
, unsigned num_levels
,
52 struct pipe_framebuffer_state
*cso_fb
= &ice
->state
.framebuffer
;
55 /* We only need to worry about color compression and fast clears. */
56 if (tex_res
->aux
.usage
!= ISL_AUX_USAGE_CCS_D
&&
57 tex_res
->aux
.usage
!= ISL_AUX_USAGE_CCS_E
)
60 for (unsigned i
= 0; i
< cso_fb
->nr_cbufs
; i
++) {
61 struct iris_surface
*surf
= (void *) cso_fb
->cbufs
[i
];
65 struct iris_resource
*rb_res
= (void *) surf
->base
.texture
;
67 if (rb_res
->bo
== tex_res
->bo
&&
68 surf
->base
.u
.tex
.level
>= min_level
&&
69 surf
->base
.u
.tex
.level
< min_level
+ num_levels
) {
70 found
= draw_aux_buffer_disabled
[i
] = true;
76 "Disabling CCS because a renderbuffer is also bound %s.\n",
84 resolve_sampler_views(struct iris_context
*ice
,
85 struct iris_batch
*batch
,
86 struct iris_shader_state
*shs
,
87 const struct shader_info
*info
,
88 bool *draw_aux_buffer_disabled
,
89 bool consider_framebuffer
)
91 uint32_t views
= info
? (shs
->bound_sampler_views
& info
->textures_used
) : 0;
94 const int i
= u_bit_scan(&views
);
95 struct iris_sampler_view
*isv
= shs
->textures
[i
];
96 struct iris_resource
*res
= (void *) isv
->base
.texture
;
98 if (res
->base
.target
!= PIPE_BUFFER
) {
99 if (consider_framebuffer
) {
100 disable_rb_aux_buffer(ice
, draw_aux_buffer_disabled
,
101 res
, isv
->view
.base_level
, isv
->view
.levels
,
105 iris_resource_prepare_texture(ice
, res
, isv
->view
.format
,
106 isv
->view
.base_level
, isv
->view
.levels
,
107 isv
->view
.base_array_layer
,
108 isv
->view
.array_len
);
111 iris_cache_flush_for_read(batch
, res
->bo
);
116 resolve_image_views(struct iris_context
*ice
,
117 struct iris_batch
*batch
,
118 struct iris_shader_state
*shs
,
119 const struct shader_info
*info
,
120 bool *draw_aux_buffer_disabled
,
121 bool consider_framebuffer
)
123 uint32_t views
= info
? (shs
->bound_image_views
& info
->images_used
) : 0;
126 const int i
= u_bit_scan(&views
);
127 struct pipe_image_view
*pview
= &shs
->image
[i
].base
;
128 struct iris_resource
*res
= (void *) pview
->resource
;
130 if (res
->base
.target
!= PIPE_BUFFER
) {
131 if (consider_framebuffer
) {
132 disable_rb_aux_buffer(ice
, draw_aux_buffer_disabled
,
133 res
, pview
->u
.tex
.level
, 1,
134 "as a shader image");
137 unsigned num_layers
=
138 pview
->u
.tex
.last_layer
- pview
->u
.tex
.first_layer
+ 1;
140 enum isl_aux_usage aux_usage
=
141 iris_image_view_aux_usage(ice
, pview
, info
);
143 iris_resource_prepare_access(ice
, res
,
144 pview
->u
.tex
.level
, 1,
145 pview
->u
.tex
.first_layer
, num_layers
,
149 iris_cache_flush_for_read(batch
, res
->bo
);
155 * \brief Resolve buffers before drawing.
157 * Resolve the depth buffer's HiZ buffer, resolve the depth buffer of each
158 * enabled depth texture, and flush the render cache for any dirty textures.
161 iris_predraw_resolve_inputs(struct iris_context
*ice
,
162 struct iris_batch
*batch
,
163 bool *draw_aux_buffer_disabled
,
164 gl_shader_stage stage
,
165 bool consider_framebuffer
)
167 struct iris_shader_state
*shs
= &ice
->state
.shaders
[stage
];
168 const struct shader_info
*info
= iris_get_shader_info(ice
, stage
);
170 uint64_t stage_dirty
= (IRIS_STAGE_DIRTY_BINDINGS_VS
<< stage
) |
171 (consider_framebuffer
? IRIS_STAGE_DIRTY_BINDINGS_FS
: 0);
173 if (ice
->state
.stage_dirty
& stage_dirty
) {
174 resolve_sampler_views(ice
, batch
, shs
, info
, draw_aux_buffer_disabled
,
175 consider_framebuffer
);
176 resolve_image_views(ice
, batch
, shs
, info
, draw_aux_buffer_disabled
,
177 consider_framebuffer
);
182 iris_predraw_resolve_framebuffer(struct iris_context
*ice
,
183 struct iris_batch
*batch
,
184 bool *draw_aux_buffer_disabled
)
186 struct pipe_framebuffer_state
*cso_fb
= &ice
->state
.framebuffer
;
187 struct iris_screen
*screen
= (void *) ice
->ctx
.screen
;
188 struct gen_device_info
*devinfo
= &screen
->devinfo
;
189 struct iris_uncompiled_shader
*ish
=
190 ice
->shaders
.uncompiled
[MESA_SHADER_FRAGMENT
];
191 const nir_shader
*nir
= ish
->nir
;
193 if (ice
->state
.dirty
& IRIS_DIRTY_DEPTH_BUFFER
) {
194 struct pipe_surface
*zs_surf
= cso_fb
->zsbuf
;
197 struct iris_resource
*z_res
, *s_res
;
198 iris_get_depth_stencil_resources(zs_surf
->texture
, &z_res
, &s_res
);
199 unsigned num_layers
=
200 zs_surf
->u
.tex
.last_layer
- zs_surf
->u
.tex
.first_layer
+ 1;
203 iris_resource_prepare_depth(ice
, batch
, z_res
,
204 zs_surf
->u
.tex
.level
,
205 zs_surf
->u
.tex
.first_layer
,
207 iris_cache_flush_for_depth(batch
, z_res
->bo
);
211 iris_cache_flush_for_depth(batch
, s_res
->bo
);
216 if (devinfo
->gen
== 8 && nir
->info
.outputs_read
!= 0) {
217 for (unsigned i
= 0; i
< cso_fb
->nr_cbufs
; i
++) {
218 if (cso_fb
->cbufs
[i
]) {
219 struct iris_surface
*surf
= (void *) cso_fb
->cbufs
[i
];
220 struct iris_resource
*res
= (void *) cso_fb
->cbufs
[i
]->texture
;
222 iris_resource_prepare_texture(ice
, res
, surf
->view
.format
,
223 surf
->view
.base_level
, 1,
224 surf
->view
.base_array_layer
,
225 surf
->view
.array_len
);
230 if ((ice
->state
.dirty
& IRIS_DIRTY_BLEND_STATE
) ||
231 (ice
->state
.stage_dirty
& IRIS_STAGE_DIRTY_BINDINGS_FS
)) {
232 for (unsigned i
= 0; i
< cso_fb
->nr_cbufs
; i
++) {
233 struct iris_surface
*surf
= (void *) cso_fb
->cbufs
[i
];
237 struct iris_resource
*res
= (void *) surf
->base
.texture
;
239 enum isl_aux_usage aux_usage
=
240 iris_resource_render_aux_usage(ice
, res
, surf
->view
.format
,
241 ice
->state
.blend_enables
& (1u << i
),
242 draw_aux_buffer_disabled
[i
]);
244 if (ice
->state
.draw_aux_usage
[i
] != aux_usage
) {
245 ice
->state
.draw_aux_usage
[i
] = aux_usage
;
246 /* XXX: Need to track which bindings to make dirty */
247 ice
->state
.dirty
|= IRIS_DIRTY_RENDER_BUFFER
;
248 ice
->state
.stage_dirty
|= IRIS_ALL_STAGE_DIRTY_BINDINGS
;
251 iris_resource_prepare_render(ice
, batch
, res
, surf
->view
.base_level
,
252 surf
->view
.base_array_layer
,
253 surf
->view
.array_len
,
256 iris_cache_flush_for_render(batch
, res
->bo
, surf
->view
.format
,
263 * \brief Call this after drawing to mark which buffers need resolving
265 * If the depth buffer was written to and if it has an accompanying HiZ
266 * buffer, then mark that it needs a depth resolve.
268 * If the color buffer is a multisample window system buffer, then
269 * mark that it needs a downsample.
271 * Also mark any render targets which will be textured as needing a render
275 iris_postdraw_update_resolve_tracking(struct iris_context
*ice
,
276 struct iris_batch
*batch
)
278 struct pipe_framebuffer_state
*cso_fb
= &ice
->state
.framebuffer
;
280 // XXX: front buffer drawing?
282 bool may_have_resolved_depth
=
283 ice
->state
.dirty
& (IRIS_DIRTY_DEPTH_BUFFER
|
284 IRIS_DIRTY_WM_DEPTH_STENCIL
);
286 struct pipe_surface
*zs_surf
= cso_fb
->zsbuf
;
288 struct iris_resource
*z_res
, *s_res
;
289 iris_get_depth_stencil_resources(zs_surf
->texture
, &z_res
, &s_res
);
290 unsigned num_layers
=
291 zs_surf
->u
.tex
.last_layer
- zs_surf
->u
.tex
.first_layer
+ 1;
294 if (may_have_resolved_depth
) {
295 iris_resource_finish_depth(ice
, z_res
, zs_surf
->u
.tex
.level
,
296 zs_surf
->u
.tex
.first_layer
, num_layers
,
297 ice
->state
.depth_writes_enabled
);
302 if (may_have_resolved_depth
&& ice
->state
.stencil_writes_enabled
) {
303 iris_resource_finish_write(ice
, s_res
, zs_surf
->u
.tex
.level
,
304 zs_surf
->u
.tex
.first_layer
, num_layers
,
310 bool may_have_resolved_color
=
311 (ice
->state
.dirty
& IRIS_DIRTY_BLEND_STATE
) ||
312 (ice
->state
.stage_dirty
& IRIS_STAGE_DIRTY_BINDINGS_FS
);
314 for (unsigned i
= 0; i
< cso_fb
->nr_cbufs
; i
++) {
315 struct iris_surface
*surf
= (void *) cso_fb
->cbufs
[i
];
319 struct iris_resource
*res
= (void *) surf
->base
.texture
;
320 enum isl_aux_usage aux_usage
= ice
->state
.draw_aux_usage
[i
];
322 if (may_have_resolved_color
) {
323 union pipe_surface_desc
*desc
= &surf
->base
.u
;
324 unsigned num_layers
=
325 desc
->tex
.last_layer
- desc
->tex
.first_layer
+ 1;
326 iris_resource_finish_render(ice
, res
, desc
->tex
.level
,
327 desc
->tex
.first_layer
, num_layers
,
334 * Emits an appropriate flush for a BO if it has been rendered to within the
335 * same batchbuffer as a read that's about to be emitted.
337 * The GPU has separate, incoherent caches for the render cache and the
338 * sampler cache, along with other caches. Usually data in the different
339 * caches don't interact (e.g. we don't render to our driver-generated
340 * immediate constant data), but for render-to-texture in FBOs we definitely
341 * do. When a batchbuffer is flushed, the kernel will ensure that everything
342 * necessary is flushed before another use of that BO, but for reuse from
343 * different caches within a batchbuffer, it's all our responsibility.
346 iris_flush_depth_and_render_caches(struct iris_batch
*batch
)
348 iris_emit_pipe_control_flush(batch
,
349 "cache tracker: render-to-texture",
350 PIPE_CONTROL_DEPTH_CACHE_FLUSH
|
351 PIPE_CONTROL_RENDER_TARGET_FLUSH
|
352 PIPE_CONTROL_CS_STALL
);
354 iris_emit_pipe_control_flush(batch
,
355 "cache tracker: render-to-texture",
356 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE
|
357 PIPE_CONTROL_CONST_CACHE_INVALIDATE
);
361 iris_cache_flush_for_read(struct iris_batch
*batch
,
364 iris_emit_buffer_barrier_for(batch
, bo
, IRIS_DOMAIN_OTHER_READ
);
368 format_aux_tuple(enum isl_format format
, enum isl_aux_usage aux_usage
)
370 return (void *)(uintptr_t)((uint32_t)format
<< 8 | aux_usage
);
374 iris_cache_flush_for_render(struct iris_batch
*batch
,
376 enum isl_format format
,
377 enum isl_aux_usage aux_usage
)
379 iris_emit_buffer_barrier_for(batch
, bo
, IRIS_DOMAIN_RENDER_WRITE
);
381 /* Check to see if this bo has been used by a previous rendering operation
382 * but with a different format or aux usage. If it has, flush the render
383 * cache so we ensure that it's only in there with one format or aux usage
386 * Even though it's not obvious, this can easily happen in practice.
387 * Suppose a client is blending on a surface with sRGB encode enabled on
388 * gen9. This implies that you get AUX_USAGE_CCS_D at best. If the client
389 * then disables sRGB decode and continues blending we will flip on
390 * AUX_USAGE_CCS_E without doing any sort of resolve in-between (this is
391 * perfectly valid since CCS_E is a subset of CCS_D). However, this means
392 * that we have fragments in-flight which are rendering with UNORM+CCS_E
393 * and other fragments in-flight with SRGB+CCS_D on the same surface at the
394 * same time and the pixel scoreboard and color blender are trying to sort
395 * it all out. This ends badly (i.e. GPU hangs).
397 * To date, we have never observed GPU hangs or even corruption to be
398 * associated with switching the format, only the aux usage. However,
399 * there are comments in various docs which indicate that the render cache
400 * isn't 100% resilient to format changes. We may as well be conservative
401 * and flush on format changes too. We can always relax this later if we
402 * find it to be a performance problem.
404 struct hash_entry
*entry
=
405 _mesa_hash_table_search_pre_hashed(batch
->cache
.render
, bo
->hash
, bo
);
407 _mesa_hash_table_insert_pre_hashed(batch
->cache
.render
, bo
->hash
, bo
,
408 format_aux_tuple(format
, aux_usage
));
409 } else if (entry
->data
!= format_aux_tuple(format
, aux_usage
)) {
410 iris_flush_depth_and_render_caches(batch
);
411 entry
->data
= format_aux_tuple(format
, aux_usage
);
416 iris_cache_flush_for_depth(struct iris_batch
*batch
,
419 iris_emit_buffer_barrier_for(batch
, bo
, IRIS_DOMAIN_DEPTH_WRITE
);
423 iris_resolve_color(struct iris_context
*ice
,
424 struct iris_batch
*batch
,
425 struct iris_resource
*res
,
426 unsigned level
, unsigned layer
,
427 enum isl_aux_op resolve_op
)
429 //DBG("%s to mt %p level %u layer %u\n", __FUNCTION__, mt, level, layer);
431 struct blorp_surf surf
;
432 iris_blorp_surf_for_resource(&batch
->screen
->isl_dev
, &surf
,
433 &res
->base
, res
->aux
.usage
, level
, true);
435 iris_batch_maybe_flush(batch
, 1500);
437 /* Ivybridge PRM Vol 2, Part 1, "11.7 MCS Buffer for Render Target(s)":
439 * "Any transition from any value in {Clear, Render, Resolve} to a
440 * different value in {Clear, Render, Resolve} requires end of pipe
443 * In other words, fast clear ops are not properly synchronized with
444 * other drawing. We need to use a PIPE_CONTROL to ensure that the
445 * contents of the previous draw hit the render target before we resolve
446 * and again afterwards to ensure that the resolve is complete before we
447 * do any more regular drawing.
449 iris_emit_end_of_pipe_sync(batch
, "color resolve: pre-flush",
450 PIPE_CONTROL_RENDER_TARGET_FLUSH
);
452 iris_batch_sync_region_start(batch
);
453 struct blorp_batch blorp_batch
;
454 blorp_batch_init(&ice
->blorp
, &blorp_batch
, batch
, 0);
455 /* On Gen >= 12, Stencil buffer with lossless compression needs to be
456 * resolve with WM_HZ_OP packet.
458 if (res
->aux
.usage
== ISL_AUX_USAGE_STC_CCS
) {
459 blorp_hiz_stencil_op(&blorp_batch
, &surf
, level
, layer
,
462 blorp_ccs_resolve(&blorp_batch
, &surf
, level
, layer
, 1,
463 isl_format_srgb_to_linear(res
->surf
.format
),
466 blorp_batch_finish(&blorp_batch
);
468 /* See comment above */
469 iris_emit_end_of_pipe_sync(batch
, "color resolve: post-flush",
470 PIPE_CONTROL_RENDER_TARGET_FLUSH
);
471 iris_batch_sync_region_end(batch
);
475 iris_mcs_partial_resolve(struct iris_context
*ice
,
476 struct iris_batch
*batch
,
477 struct iris_resource
*res
,
478 uint32_t start_layer
,
481 //DBG("%s to mt %p layers %u-%u\n", __FUNCTION__, mt,
482 //start_layer, start_layer + num_layers - 1);
484 assert(isl_aux_usage_has_mcs(res
->aux
.usage
));
486 struct blorp_surf surf
;
487 iris_blorp_surf_for_resource(&batch
->screen
->isl_dev
, &surf
,
488 &res
->base
, res
->aux
.usage
, 0, true);
490 struct blorp_batch blorp_batch
;
491 iris_batch_sync_region_start(batch
);
492 blorp_batch_init(&ice
->blorp
, &blorp_batch
, batch
, 0);
493 blorp_mcs_partial_resolve(&blorp_batch
, &surf
,
494 isl_format_srgb_to_linear(res
->surf
.format
),
495 start_layer
, num_layers
);
496 blorp_batch_finish(&blorp_batch
);
497 iris_batch_sync_region_end(batch
);
502 * Return true if the format that will be used to access the resource is
503 * CCS_E-compatible with the resource's linear/non-sRGB format.
505 * Why use the linear format? Well, although the resourcemay be specified
506 * with an sRGB format, the usage of that color space/format can be toggled.
507 * Since our HW tends to support more linear formats than sRGB ones, we use
508 * this format variant for check for CCS_E compatibility.
511 format_ccs_e_compat_with_resource(const struct gen_device_info
*devinfo
,
512 const struct iris_resource
*res
,
513 enum isl_format access_format
)
515 assert(res
->aux
.usage
== ISL_AUX_USAGE_CCS_E
);
517 enum isl_format isl_format
= isl_format_srgb_to_linear(res
->surf
.format
);
518 return isl_formats_are_ccs_e_compatible(devinfo
, isl_format
, access_format
);
522 iris_sample_with_depth_aux(const struct gen_device_info
*devinfo
,
523 const struct iris_resource
*res
)
525 switch (res
->aux
.usage
) {
526 case ISL_AUX_USAGE_HIZ
:
527 if (devinfo
->has_sample_with_hiz
)
530 case ISL_AUX_USAGE_HIZ_CCS
:
532 case ISL_AUX_USAGE_HIZ_CCS_WT
:
538 /* It seems the hardware won't fallback to the depth buffer if some of the
539 * mipmap levels aren't available in the HiZ buffer. So we need all levels
540 * of the texture to be HiZ enabled.
542 for (unsigned level
= 0; level
< res
->surf
.levels
; ++level
) {
543 if (!iris_resource_level_has_hiz(res
, level
))
547 /* If compressed multisampling is enabled, then we use it for the auxiliary
550 * From the BDW PRM (Volume 2d: Command Reference: Structures
551 * RENDER_SURFACE_STATE.AuxiliarySurfaceMode):
553 * "If this field is set to AUX_HIZ, Number of Multisamples must be
554 * MULTISAMPLECOUNT_1, and Surface Type cannot be SURFTYPE_3D.
556 * There is no such blurb for 1D textures, but there is sufficient evidence
557 * that this is broken on SKL+.
559 // XXX: i965 disables this for arrays too, is that reasonable?
560 return res
->surf
.samples
== 1 && res
->surf
.dim
== ISL_SURF_DIM_2D
;
564 * Perform a HiZ or depth resolve operation.
566 * For an overview of HiZ ops, see the following sections of the Sandy Bridge
567 * PRM, Volume 1, Part 2:
568 * - 7.5.3.1 Depth Buffer Clear
569 * - 7.5.3.2 Depth Buffer Resolve
570 * - 7.5.3.3 Hierarchical Depth Buffer Resolve
573 iris_hiz_exec(struct iris_context
*ice
,
574 struct iris_batch
*batch
,
575 struct iris_resource
*res
,
576 unsigned int level
, unsigned int start_layer
,
577 unsigned int num_layers
, enum isl_aux_op op
,
578 bool update_clear_depth
)
580 assert(iris_resource_level_has_hiz(res
, level
));
581 assert(op
!= ISL_AUX_OP_NONE
);
582 UNUSED
const char *name
= NULL
;
585 case ISL_AUX_OP_FULL_RESOLVE
:
586 name
= "depth resolve";
588 case ISL_AUX_OP_AMBIGUATE
:
589 name
= "hiz ambiguate";
591 case ISL_AUX_OP_FAST_CLEAR
:
592 name
= "depth clear";
594 case ISL_AUX_OP_PARTIAL_RESOLVE
:
595 case ISL_AUX_OP_NONE
:
596 unreachable("Invalid HiZ op");
599 //DBG("%s %s to mt %p level %d layers %d-%d\n",
600 //__func__, name, mt, level, start_layer, start_layer + num_layers - 1);
602 /* The following stalls and flushes are only documented to be required
603 * for HiZ clear operations. However, they also seem to be required for
604 * resolve operations.
606 * From the Ivybridge PRM, volume 2, "Depth Buffer Clear":
608 * "If other rendering operations have preceded this clear, a
609 * PIPE_CONTROL with depth cache flush enabled, Depth Stall bit
610 * enabled must be issued before the rectangle primitive used for
611 * the depth buffer clear operation."
613 * Same applies for Gen8 and Gen9.
615 * In addition, from the Ivybridge PRM, volume 2, 1.10.4.1
616 * PIPE_CONTROL, Depth Cache Flush Enable:
618 * "This bit must not be set when Depth Stall Enable bit is set in
621 * This is confirmed to hold for real, Haswell gets immediate gpu hangs.
623 * Therefore issue two pipe control flushes, one for cache flush and
624 * another for depth stall.
626 iris_emit_pipe_control_flush(batch
,
627 "hiz op: pre-flushes (1/2)",
628 PIPE_CONTROL_DEPTH_CACHE_FLUSH
|
629 PIPE_CONTROL_CS_STALL
);
631 iris_emit_pipe_control_flush(batch
, "hiz op: pre-flushes (2/2)",
632 PIPE_CONTROL_DEPTH_STALL
);
634 assert(isl_aux_usage_has_hiz(res
->aux
.usage
) && res
->aux
.bo
);
636 iris_batch_maybe_flush(batch
, 1500);
638 iris_batch_sync_region_start(batch
);
640 struct blorp_surf surf
;
641 iris_blorp_surf_for_resource(&batch
->screen
->isl_dev
, &surf
,
642 &res
->base
, res
->aux
.usage
, level
, true);
644 struct blorp_batch blorp_batch
;
645 enum blorp_batch_flags flags
= 0;
646 flags
|= update_clear_depth
? 0 : BLORP_BATCH_NO_UPDATE_CLEAR_COLOR
;
647 blorp_batch_init(&ice
->blorp
, &blorp_batch
, batch
, flags
);
648 blorp_hiz_op(&blorp_batch
, &surf
, level
, start_layer
, num_layers
, op
);
649 blorp_batch_finish(&blorp_batch
);
651 /* The following stalls and flushes are only documented to be required
652 * for HiZ clear operations. However, they also seem to be required for
653 * resolve operations.
655 * From the Broadwell PRM, volume 7, "Depth Buffer Clear":
657 * "Depth buffer clear pass using any of the methods (WM_STATE,
658 * 3DSTATE_WM or 3DSTATE_WM_HZ_OP) must be followed by a
659 * PIPE_CONTROL command with DEPTH_STALL bit and Depth FLUSH bits
660 * "set" before starting to render. DepthStall and DepthFlush are
661 * not needed between consecutive depth clear passes nor is it
662 * required if the depth clear pass was done with
663 * 'full_surf_clear' bit set in the 3DSTATE_WM_HZ_OP."
665 * TODO: Such as the spec says, this could be conditional.
667 iris_emit_pipe_control_flush(batch
,
668 "hiz op: post flush",
669 PIPE_CONTROL_DEPTH_CACHE_FLUSH
|
670 PIPE_CONTROL_DEPTH_STALL
);
672 iris_batch_sync_region_end(batch
);
676 level_has_aux(const struct iris_resource
*res
, uint32_t level
)
678 return isl_aux_usage_has_hiz(res
->aux
.usage
) ?
679 iris_resource_level_has_hiz(res
, level
) :
680 res
->aux
.usage
!= ISL_AUX_USAGE_NONE
;
684 * Does the resource's slice have hiz enabled?
687 iris_resource_level_has_hiz(const struct iris_resource
*res
, uint32_t level
)
689 iris_resource_check_level_layer(res
, level
, 0);
690 return res
->aux
.has_hiz
& 1 << level
;
693 /** \brief Assert that the level and layer are valid for the resource. */
695 iris_resource_check_level_layer(UNUSED
const struct iris_resource
*res
,
696 UNUSED
uint32_t level
, UNUSED
uint32_t layer
)
698 assert(level
< res
->surf
.levels
);
699 assert(layer
< util_num_layers(&res
->base
, level
));
702 static inline uint32_t
703 miptree_level_range_length(const struct iris_resource
*res
,
704 uint32_t start_level
, uint32_t num_levels
)
706 assert(start_level
< res
->surf
.levels
);
708 if (num_levels
== INTEL_REMAINING_LAYERS
)
709 num_levels
= res
->surf
.levels
;
711 /* Check for overflow */
712 assert(start_level
+ num_levels
>= start_level
);
713 assert(start_level
+ num_levels
<= res
->surf
.levels
);
718 static inline uint32_t
719 miptree_layer_range_length(const struct iris_resource
*res
, uint32_t level
,
720 uint32_t start_layer
, uint32_t num_layers
)
722 assert(level
<= res
->base
.last_level
);
724 const uint32_t total_num_layers
= iris_get_num_logical_layers(res
, level
);
725 assert(start_layer
< total_num_layers
);
726 if (num_layers
== INTEL_REMAINING_LAYERS
)
727 num_layers
= total_num_layers
- start_layer
;
728 /* Check for overflow */
729 assert(start_layer
+ num_layers
>= start_layer
);
730 assert(start_layer
+ num_layers
<= total_num_layers
);
736 iris_has_color_unresolved(const struct iris_resource
*res
,
737 unsigned start_level
, unsigned num_levels
,
738 unsigned start_layer
, unsigned num_layers
)
743 /* Clamp the level range to fit the resource */
744 num_levels
= miptree_level_range_length(res
, start_level
, num_levels
);
746 for (uint32_t l
= 0; l
< num_levels
; l
++) {
747 const uint32_t level
= start_level
+ l
;
748 const uint32_t level_layers
=
749 miptree_layer_range_length(res
, level
, start_layer
, num_layers
);
750 for (unsigned a
= 0; a
< level_layers
; a
++) {
751 enum isl_aux_state aux_state
=
752 iris_resource_get_aux_state(res
, level
, start_layer
+ a
);
753 assert(aux_state
!= ISL_AUX_STATE_AUX_INVALID
);
754 if (aux_state
!= ISL_AUX_STATE_PASS_THROUGH
)
763 iris_resource_prepare_access(struct iris_context
*ice
,
764 struct iris_resource
*res
,
765 uint32_t start_level
, uint32_t num_levels
,
766 uint32_t start_layer
, uint32_t num_layers
,
767 enum isl_aux_usage aux_usage
,
768 bool fast_clear_supported
)
770 /* We can't do resolves on the compute engine, so awkwardly, we have to
771 * do them on the render batch...
773 struct iris_batch
*batch
= &ice
->batches
[IRIS_BATCH_RENDER
];
775 const uint32_t clamped_levels
=
776 miptree_level_range_length(res
, start_level
, num_levels
);
777 for (uint32_t l
= 0; l
< clamped_levels
; l
++) {
778 const uint32_t level
= start_level
+ l
;
779 if (!level_has_aux(res
, level
))
782 const uint32_t level_layers
=
783 miptree_layer_range_length(res
, level
, start_layer
, num_layers
);
784 for (uint32_t a
= 0; a
< level_layers
; a
++) {
785 const uint32_t layer
= start_layer
+ a
;
786 const enum isl_aux_state aux_state
=
787 iris_resource_get_aux_state(res
, level
, layer
);
788 const enum isl_aux_op aux_op
=
789 isl_aux_prepare_access(aux_state
, aux_usage
, fast_clear_supported
);
791 if (aux_op
== ISL_AUX_OP_NONE
) {
792 /* Nothing to do here. */
793 } else if (isl_aux_usage_has_mcs(res
->aux
.usage
)) {
794 assert(aux_op
== ISL_AUX_OP_PARTIAL_RESOLVE
);
795 iris_mcs_partial_resolve(ice
, batch
, res
, layer
, 1);
796 } else if (isl_aux_usage_has_hiz(res
->aux
.usage
)) {
797 iris_hiz_exec(ice
, batch
, res
, level
, layer
, 1, aux_op
, false);
799 assert(isl_aux_usage_has_ccs(res
->aux
.usage
));
800 iris_resolve_color(ice
, batch
, res
, level
, layer
, aux_op
);
803 const enum isl_aux_state new_state
=
804 isl_aux_state_transition_aux_op(aux_state
, res
->aux
.usage
, aux_op
);
805 iris_resource_set_aux_state(ice
, res
, level
, layer
, 1, new_state
);
811 iris_resource_finish_write(struct iris_context
*ice
,
812 struct iris_resource
*res
, uint32_t level
,
813 uint32_t start_layer
, uint32_t num_layers
,
814 enum isl_aux_usage aux_usage
)
816 if (!level_has_aux(res
, level
))
819 const uint32_t level_layers
=
820 miptree_layer_range_length(res
, level
, start_layer
, num_layers
);
822 for (uint32_t a
= 0; a
< level_layers
; a
++) {
823 const uint32_t layer
= start_layer
+ a
;
824 const enum isl_aux_state aux_state
=
825 iris_resource_get_aux_state(res
, level
, layer
);
826 const enum isl_aux_state new_aux_state
=
827 isl_aux_state_transition_write(aux_state
, aux_usage
, false);
828 iris_resource_set_aux_state(ice
, res
, level
, layer
, 1, new_aux_state
);
833 iris_resource_get_aux_state(const struct iris_resource
*res
,
834 uint32_t level
, uint32_t layer
)
836 iris_resource_check_level_layer(res
, level
, layer
);
838 if (res
->surf
.usage
& ISL_SURF_USAGE_DEPTH_BIT
) {
839 assert(iris_resource_level_has_hiz(res
, level
));
841 assert(res
->surf
.samples
== 1 ||
842 res
->surf
.msaa_layout
== ISL_MSAA_LAYOUT_ARRAY
);
845 return res
->aux
.state
[level
][layer
];
849 iris_resource_set_aux_state(struct iris_context
*ice
,
850 struct iris_resource
*res
, uint32_t level
,
851 uint32_t start_layer
, uint32_t num_layers
,
852 enum isl_aux_state aux_state
)
854 num_layers
= miptree_layer_range_length(res
, level
, start_layer
, num_layers
);
856 if (res
->surf
.usage
& ISL_SURF_USAGE_DEPTH_BIT
) {
857 assert(iris_resource_level_has_hiz(res
, level
));
859 assert(res
->surf
.samples
== 1 ||
860 res
->surf
.msaa_layout
== ISL_MSAA_LAYOUT_ARRAY
);
863 for (unsigned a
= 0; a
< num_layers
; a
++) {
864 if (res
->aux
.state
[level
][start_layer
+ a
] != aux_state
) {
865 res
->aux
.state
[level
][start_layer
+ a
] = aux_state
;
866 /* XXX: Need to track which bindings to make dirty */
867 ice
->state
.dirty
|= IRIS_DIRTY_RENDER_BUFFER
;
868 ice
->state
.stage_dirty
|= IRIS_ALL_STAGE_DIRTY_BINDINGS
;
873 /* On Gen9 color buffers may be compressed by the hardware (lossless
874 * compression). There are, however, format restrictions and care needs to be
875 * taken that the sampler engine is capable for re-interpreting a buffer with
876 * format different the buffer was originally written with.
878 * For example, SRGB formats are not compressible and the sampler engine isn't
879 * capable of treating RGBA_UNORM as SRGB_ALPHA. In such a case the underlying
880 * color buffer needs to be resolved so that the sampling surface can be
881 * sampled as non-compressed (i.e., without the auxiliary MCS buffer being
885 can_texture_with_ccs(const struct gen_device_info
*devinfo
,
886 struct pipe_debug_callback
*dbg
,
887 const struct iris_resource
*res
,
888 enum isl_format view_format
)
890 if (res
->aux
.usage
!= ISL_AUX_USAGE_CCS_E
)
893 if (!format_ccs_e_compat_with_resource(devinfo
, res
, view_format
)) {
894 const struct isl_format_layout
*res_fmtl
=
895 isl_format_get_layout(res
->surf
.format
);
896 const struct isl_format_layout
*view_fmtl
=
897 isl_format_get_layout(view_format
);
899 perf_debug(dbg
, "Incompatible sampling format (%s) for CCS (%s)\n",
900 view_fmtl
->name
, res_fmtl
->name
);
909 iris_resource_texture_aux_usage(struct iris_context
*ice
,
910 const struct iris_resource
*res
,
911 enum isl_format view_format
)
913 struct iris_screen
*screen
= (void *) ice
->ctx
.screen
;
914 struct gen_device_info
*devinfo
= &screen
->devinfo
;
916 switch (res
->aux
.usage
) {
917 case ISL_AUX_USAGE_HIZ
:
918 if (iris_sample_with_depth_aux(devinfo
, res
))
919 return ISL_AUX_USAGE_HIZ
;
922 case ISL_AUX_USAGE_HIZ_CCS
:
923 assert(!iris_sample_with_depth_aux(devinfo
, res
));
924 return ISL_AUX_USAGE_NONE
;
926 case ISL_AUX_USAGE_HIZ_CCS_WT
:
927 if (iris_sample_with_depth_aux(devinfo
, res
))
928 return ISL_AUX_USAGE_HIZ_CCS_WT
;
931 case ISL_AUX_USAGE_MCS
:
932 case ISL_AUX_USAGE_MCS_CCS
:
933 case ISL_AUX_USAGE_STC_CCS
:
934 return res
->aux
.usage
;
936 case ISL_AUX_USAGE_CCS_D
:
937 case ISL_AUX_USAGE_CCS_E
:
938 /* If we don't have any unresolved color, report an aux usage of
939 * ISL_AUX_USAGE_NONE. This way, texturing won't even look at the
940 * aux surface and we can save some bandwidth.
942 if (!iris_has_color_unresolved(res
, 0, INTEL_REMAINING_LEVELS
,
943 0, INTEL_REMAINING_LAYERS
))
944 return ISL_AUX_USAGE_NONE
;
946 if (can_texture_with_ccs(devinfo
, &ice
->dbg
, res
, view_format
))
947 return ISL_AUX_USAGE_CCS_E
;
954 return ISL_AUX_USAGE_NONE
;
958 iris_image_view_aux_usage(struct iris_context
*ice
,
959 const struct pipe_image_view
*pview
,
960 const struct shader_info
*info
)
963 return ISL_AUX_USAGE_NONE
;
965 struct iris_screen
*screen
= (void *) ice
->ctx
.screen
;
966 const struct gen_device_info
*devinfo
= &screen
->devinfo
;
967 struct iris_resource
*res
= (void *) pview
->resource
;
969 enum isl_format view_format
= iris_image_view_get_format(ice
, pview
);
970 enum isl_aux_usage aux_usage
=
971 iris_resource_texture_aux_usage(ice
, res
, view_format
);
973 bool uses_atomic_load_store
=
974 ice
->shaders
.uncompiled
[info
->stage
]->uses_atomic_load_store
;
976 if ((devinfo
->gen
== 12 && aux_usage
== ISL_AUX_USAGE_CCS_E
) &&
977 !uses_atomic_load_store
)
978 return ISL_AUX_USAGE_CCS_E
;
980 return ISL_AUX_USAGE_NONE
;
984 isl_formats_are_fast_clear_compatible(enum isl_format a
, enum isl_format b
)
986 /* On gen8 and earlier, the hardware was only capable of handling 0/1 clear
987 * values so sRGB curve application was a no-op for all fast-clearable
990 * On gen9+, the hardware supports arbitrary clear values. For sRGB clear
991 * values, the hardware interprets the floats, not as what would be
992 * returned from the sampler (or written by the shader), but as being
993 * between format conversion and sRGB curve application. This means that
994 * we can switch between sRGB and UNORM without having to whack the clear
997 return isl_format_srgb_to_linear(a
) == isl_format_srgb_to_linear(b
);
1001 iris_resource_prepare_texture(struct iris_context
*ice
,
1002 struct iris_resource
*res
,
1003 enum isl_format view_format
,
1004 uint32_t start_level
, uint32_t num_levels
,
1005 uint32_t start_layer
, uint32_t num_layers
)
1007 enum isl_aux_usage aux_usage
=
1008 iris_resource_texture_aux_usage(ice
, res
, view_format
);
1010 bool clear_supported
= isl_aux_usage_has_fast_clears(aux_usage
);
1012 /* Clear color is specified as ints or floats and the conversion is done by
1013 * the sampler. If we have a texture view, we would have to perform the
1014 * clear color conversion manually. Just disable clear color.
1016 if (!isl_formats_are_fast_clear_compatible(res
->surf
.format
, view_format
))
1017 clear_supported
= false;
1019 iris_resource_prepare_access(ice
, res
, start_level
, num_levels
,
1020 start_layer
, num_layers
,
1021 aux_usage
, clear_supported
);
1025 iris_resource_render_aux_usage(struct iris_context
*ice
,
1026 struct iris_resource
*res
,
1027 enum isl_format render_format
,
1029 bool draw_aux_disabled
)
1031 struct iris_screen
*screen
= (void *) ice
->ctx
.screen
;
1032 struct gen_device_info
*devinfo
= &screen
->devinfo
;
1034 if (draw_aux_disabled
)
1035 return ISL_AUX_USAGE_NONE
;
1037 switch (res
->aux
.usage
) {
1038 case ISL_AUX_USAGE_MCS
:
1039 case ISL_AUX_USAGE_MCS_CCS
:
1040 return res
->aux
.usage
;
1042 case ISL_AUX_USAGE_CCS_D
:
1043 case ISL_AUX_USAGE_CCS_E
:
1044 /* Gen9+ hardware technically supports non-0/1 clear colors with sRGB
1045 * formats. However, there are issues with blending where it doesn't
1046 * properly apply the sRGB curve to the clear color when blending.
1048 if (devinfo
->gen
>= 9 && blend_enabled
&&
1049 isl_format_is_srgb(render_format
) &&
1050 !isl_color_value_is_zero_one(res
->aux
.clear_color
, render_format
))
1051 return ISL_AUX_USAGE_NONE
;
1053 if (res
->aux
.usage
== ISL_AUX_USAGE_CCS_E
&&
1054 format_ccs_e_compat_with_resource(devinfo
, res
, render_format
))
1055 return ISL_AUX_USAGE_CCS_E
;
1057 /* Otherwise, we try to fall back to CCS_D */
1058 if (isl_format_supports_ccs_d(devinfo
, render_format
))
1059 return ISL_AUX_USAGE_CCS_D
;
1062 return ISL_AUX_USAGE_NONE
;
1067 iris_resource_prepare_render(struct iris_context
*ice
,
1068 struct iris_batch
*batch
,
1069 struct iris_resource
*res
, uint32_t level
,
1070 uint32_t start_layer
, uint32_t layer_count
,
1071 enum isl_aux_usage aux_usage
)
1073 iris_resource_prepare_access(ice
, res
, level
, 1, start_layer
,
1074 layer_count
, aux_usage
,
1075 isl_aux_usage_has_fast_clears(aux_usage
));
1079 iris_resource_finish_render(struct iris_context
*ice
,
1080 struct iris_resource
*res
, uint32_t level
,
1081 uint32_t start_layer
, uint32_t layer_count
,
1082 enum isl_aux_usage aux_usage
)
1084 iris_resource_finish_write(ice
, res
, level
, start_layer
, layer_count
,
1089 iris_resource_prepare_depth(struct iris_context
*ice
,
1090 struct iris_batch
*batch
,
1091 struct iris_resource
*res
, uint32_t level
,
1092 uint32_t start_layer
, uint32_t layer_count
)
1094 iris_resource_prepare_access(ice
, res
, level
, 1, start_layer
,
1095 layer_count
, res
->aux
.usage
, !!res
->aux
.bo
);
1099 iris_resource_finish_depth(struct iris_context
*ice
,
1100 struct iris_resource
*res
, uint32_t level
,
1101 uint32_t start_layer
, uint32_t layer_count
,
1104 if (depth_written
) {
1105 iris_resource_finish_write(ice
, res
, level
, start_layer
, layer_count
,