iris: properly free resources on BO allocation failure
[mesa.git] / src / gallium / drivers / iris / iris_resolve.c
1 /*
2 * Copyright © 2017 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23 /**
24 * @file iris_resolve.c
25 *
26 * This file handles resolve tracking for main and auxiliary surfaces.
27 *
28 * It also handles our cache tracking. We have sets for the render cache,
29 * depth cache, and so on. If a BO is in a cache's set, then it may have
30 * data in that cache. The helpers take care of emitting flushes for
31 * render-to-texture, format reinterpretation issues, and other situations.
32 */
33
34 #include "util/hash_table.h"
35 #include "util/set.h"
36 #include "iris_context.h"
37 #include "compiler/nir/nir.h"
38
39 /**
40 * Disable auxiliary buffers if a renderbuffer is also bound as a texture
41 * or shader image. This causes a self-dependency, where both rendering
42 * and sampling may concurrently read or write the CCS buffer, causing
43 * incorrect pixels.
44 */
45 static bool
46 disable_rb_aux_buffer(struct iris_context *ice,
47 bool *draw_aux_buffer_disabled,
48 struct iris_resource *tex_res,
49 unsigned min_level, unsigned num_levels,
50 const char *usage)
51 {
52 struct pipe_framebuffer_state *cso_fb = &ice->state.framebuffer;
53 bool found = false;
54
55 /* We only need to worry about color compression and fast clears. */
56 if (tex_res->aux.usage != ISL_AUX_USAGE_CCS_D &&
57 tex_res->aux.usage != ISL_AUX_USAGE_CCS_E)
58 return false;
59
60 for (unsigned i = 0; i < cso_fb->nr_cbufs; i++) {
61 struct iris_surface *surf = (void *) cso_fb->cbufs[i];
62 if (!surf)
63 continue;
64
65 struct iris_resource *rb_res = (void *) surf->base.texture;
66
67 if (rb_res->bo == tex_res->bo &&
68 surf->base.u.tex.level >= min_level &&
69 surf->base.u.tex.level < min_level + num_levels) {
70 found = draw_aux_buffer_disabled[i] = true;
71 }
72 }
73
74 if (found) {
75 perf_debug(&ice->dbg,
76 "Disabling CCS because a renderbuffer is also bound %s.\n",
77 usage);
78 }
79
80 return found;
81 }
82
83 static void
84 resolve_sampler_views(struct iris_context *ice,
85 struct iris_batch *batch,
86 struct iris_shader_state *shs,
87 const struct shader_info *info,
88 bool *draw_aux_buffer_disabled,
89 bool consider_framebuffer)
90 {
91 uint32_t views = info ? (shs->bound_sampler_views & info->textures_used) : 0;
92
93 while (views) {
94 const int i = u_bit_scan(&views);
95 struct iris_sampler_view *isv = shs->textures[i];
96 struct iris_resource *res = (void *) isv->base.texture;
97
98 if (res->base.target != PIPE_BUFFER) {
99 if (consider_framebuffer) {
100 disable_rb_aux_buffer(ice, draw_aux_buffer_disabled,
101 res, isv->view.base_level, isv->view.levels,
102 "for sampling");
103 }
104
105 iris_resource_prepare_texture(ice, batch, res, isv->view.format,
106 isv->view.base_level, isv->view.levels,
107 isv->view.base_array_layer,
108 isv->view.array_len);
109 }
110
111 iris_cache_flush_for_read(batch, res->bo);
112 }
113 }
114
115 static void
116 resolve_image_views(struct iris_context *ice,
117 struct iris_batch *batch,
118 struct iris_shader_state *shs,
119 const struct shader_info *info,
120 bool *draw_aux_buffer_disabled,
121 bool consider_framebuffer)
122 {
123 uint32_t views = info ? (shs->bound_image_views & info->images_used) : 0;
124
125 while (views) {
126 const int i = u_bit_scan(&views);
127 struct pipe_image_view *pview = &shs->image[i].base;
128 struct iris_resource *res = (void *) pview->resource;
129
130 if (res->base.target != PIPE_BUFFER) {
131 if (consider_framebuffer) {
132 disable_rb_aux_buffer(ice, draw_aux_buffer_disabled,
133 res, pview->u.tex.level, 1,
134 "as a shader image");
135 }
136
137 unsigned num_layers =
138 pview->u.tex.last_layer - pview->u.tex.first_layer + 1;
139
140 enum isl_aux_usage aux_usage =
141 iris_image_view_aux_usage(ice, pview, info);
142
143 iris_resource_prepare_access(ice, batch, res,
144 pview->u.tex.level, 1,
145 pview->u.tex.first_layer, num_layers,
146 aux_usage, false);
147 }
148
149 iris_cache_flush_for_read(batch, res->bo);
150 }
151 }
152
153
154 /**
155 * \brief Resolve buffers before drawing.
156 *
157 * Resolve the depth buffer's HiZ buffer, resolve the depth buffer of each
158 * enabled depth texture, and flush the render cache for any dirty textures.
159 */
160 void
161 iris_predraw_resolve_inputs(struct iris_context *ice,
162 struct iris_batch *batch,
163 bool *draw_aux_buffer_disabled,
164 gl_shader_stage stage,
165 bool consider_framebuffer)
166 {
167 struct iris_shader_state *shs = &ice->state.shaders[stage];
168 const struct shader_info *info = iris_get_shader_info(ice, stage);
169
170 uint64_t dirty = (IRIS_DIRTY_BINDINGS_VS << stage) |
171 (consider_framebuffer ? IRIS_DIRTY_BINDINGS_FS : 0);
172
173 if (ice->state.dirty & dirty) {
174 resolve_sampler_views(ice, batch, shs, info, draw_aux_buffer_disabled,
175 consider_framebuffer);
176 resolve_image_views(ice, batch, shs, info, draw_aux_buffer_disabled,
177 consider_framebuffer);
178 }
179 }
180
181 void
182 iris_predraw_resolve_framebuffer(struct iris_context *ice,
183 struct iris_batch *batch,
184 bool *draw_aux_buffer_disabled)
185 {
186 struct pipe_framebuffer_state *cso_fb = &ice->state.framebuffer;
187 struct iris_screen *screen = (void *) ice->ctx.screen;
188 struct gen_device_info *devinfo = &screen->devinfo;
189 struct iris_uncompiled_shader *ish =
190 ice->shaders.uncompiled[MESA_SHADER_FRAGMENT];
191 const nir_shader *nir = ish->nir;
192
193 if (ice->state.dirty & IRIS_DIRTY_DEPTH_BUFFER) {
194 struct pipe_surface *zs_surf = cso_fb->zsbuf;
195
196 if (zs_surf) {
197 struct iris_resource *z_res, *s_res;
198 iris_get_depth_stencil_resources(zs_surf->texture, &z_res, &s_res);
199 unsigned num_layers =
200 zs_surf->u.tex.last_layer - zs_surf->u.tex.first_layer + 1;
201
202 if (z_res) {
203 iris_resource_prepare_depth(ice, batch, z_res,
204 zs_surf->u.tex.level,
205 zs_surf->u.tex.first_layer,
206 num_layers);
207 iris_cache_flush_for_depth(batch, z_res->bo);
208 }
209
210 if (s_res) {
211 iris_cache_flush_for_depth(batch, s_res->bo);
212 }
213 }
214 }
215
216 if (devinfo->gen == 8 && nir->info.outputs_read != 0) {
217 for (unsigned i = 0; i < cso_fb->nr_cbufs; i++) {
218 if (cso_fb->cbufs[i]) {
219 struct iris_surface *surf = (void *) cso_fb->cbufs[i];
220 struct iris_resource *res = (void *) cso_fb->cbufs[i]->texture;
221
222 iris_resource_prepare_texture(ice, batch, res, surf->view.format,
223 surf->view.base_level, 1,
224 surf->view.base_array_layer,
225 surf->view.array_len);
226 }
227 }
228 }
229
230 if (ice->state.dirty & (IRIS_DIRTY_BINDINGS_FS | IRIS_DIRTY_BLEND_STATE)) {
231 for (unsigned i = 0; i < cso_fb->nr_cbufs; i++) {
232 struct iris_surface *surf = (void *) cso_fb->cbufs[i];
233 if (!surf)
234 continue;
235
236 struct iris_resource *res = (void *) surf->base.texture;
237
238 enum isl_aux_usage aux_usage =
239 iris_resource_render_aux_usage(ice, res, surf->view.format,
240 ice->state.blend_enables & (1u << i),
241 draw_aux_buffer_disabled[i]);
242
243 if (ice->state.draw_aux_usage[i] != aux_usage) {
244 ice->state.draw_aux_usage[i] = aux_usage;
245 /* XXX: Need to track which bindings to make dirty */
246 ice->state.dirty |= IRIS_ALL_DIRTY_BINDINGS;
247 }
248
249 iris_resource_prepare_render(ice, batch, res, surf->view.base_level,
250 surf->view.base_array_layer,
251 surf->view.array_len,
252 aux_usage);
253
254 iris_cache_flush_for_render(batch, res->bo, surf->view.format,
255 aux_usage);
256 }
257 }
258 }
259
260 /**
261 * \brief Call this after drawing to mark which buffers need resolving
262 *
263 * If the depth buffer was written to and if it has an accompanying HiZ
264 * buffer, then mark that it needs a depth resolve.
265 *
266 * If the color buffer is a multisample window system buffer, then
267 * mark that it needs a downsample.
268 *
269 * Also mark any render targets which will be textured as needing a render
270 * cache flush.
271 */
272 void
273 iris_postdraw_update_resolve_tracking(struct iris_context *ice,
274 struct iris_batch *batch)
275 {
276 struct pipe_framebuffer_state *cso_fb = &ice->state.framebuffer;
277
278 // XXX: front buffer drawing?
279
280 bool may_have_resolved_depth =
281 ice->state.dirty & (IRIS_DIRTY_DEPTH_BUFFER |
282 IRIS_DIRTY_WM_DEPTH_STENCIL);
283
284 struct pipe_surface *zs_surf = cso_fb->zsbuf;
285 if (zs_surf) {
286 struct iris_resource *z_res, *s_res;
287 iris_get_depth_stencil_resources(zs_surf->texture, &z_res, &s_res);
288 unsigned num_layers =
289 zs_surf->u.tex.last_layer - zs_surf->u.tex.first_layer + 1;
290
291 if (z_res) {
292 if (may_have_resolved_depth) {
293 iris_resource_finish_depth(ice, z_res, zs_surf->u.tex.level,
294 zs_surf->u.tex.first_layer, num_layers,
295 ice->state.depth_writes_enabled);
296 }
297
298 if (ice->state.depth_writes_enabled)
299 iris_depth_cache_add_bo(batch, z_res->bo);
300 }
301
302 if (s_res) {
303 if (may_have_resolved_depth && ice->state.stencil_writes_enabled) {
304 iris_resource_finish_write(ice, s_res, zs_surf->u.tex.level,
305 zs_surf->u.tex.first_layer, num_layers,
306 s_res->aux.usage);
307 }
308
309 if (ice->state.stencil_writes_enabled)
310 iris_depth_cache_add_bo(batch, s_res->bo);
311 }
312 }
313
314 bool may_have_resolved_color =
315 ice->state.dirty & (IRIS_DIRTY_BINDINGS_FS | IRIS_DIRTY_BLEND_STATE);
316
317 for (unsigned i = 0; i < cso_fb->nr_cbufs; i++) {
318 struct iris_surface *surf = (void *) cso_fb->cbufs[i];
319 if (!surf)
320 continue;
321
322 struct iris_resource *res = (void *) surf->base.texture;
323 enum isl_aux_usage aux_usage = ice->state.draw_aux_usage[i];
324
325 iris_render_cache_add_bo(batch, res->bo, surf->view.format,
326 aux_usage);
327
328 if (may_have_resolved_color) {
329 union pipe_surface_desc *desc = &surf->base.u;
330 unsigned num_layers =
331 desc->tex.last_layer - desc->tex.first_layer + 1;
332 iris_resource_finish_render(ice, res, desc->tex.level,
333 desc->tex.first_layer, num_layers,
334 aux_usage);
335 }
336 }
337 }
338
339 /**
340 * Clear the cache-tracking sets.
341 */
342 void
343 iris_cache_sets_clear(struct iris_batch *batch)
344 {
345 hash_table_foreach(batch->cache.render, render_entry)
346 _mesa_hash_table_remove(batch->cache.render, render_entry);
347
348 set_foreach(batch->cache.depth, depth_entry)
349 _mesa_set_remove(batch->cache.depth, depth_entry);
350 }
351
352 /**
353 * Emits an appropriate flush for a BO if it has been rendered to within the
354 * same batchbuffer as a read that's about to be emitted.
355 *
356 * The GPU has separate, incoherent caches for the render cache and the
357 * sampler cache, along with other caches. Usually data in the different
358 * caches don't interact (e.g. we don't render to our driver-generated
359 * immediate constant data), but for render-to-texture in FBOs we definitely
360 * do. When a batchbuffer is flushed, the kernel will ensure that everything
361 * necessary is flushed before another use of that BO, but for reuse from
362 * different caches within a batchbuffer, it's all our responsibility.
363 */
364 void
365 iris_flush_depth_and_render_caches(struct iris_batch *batch)
366 {
367 iris_emit_pipe_control_flush(batch,
368 "cache tracker: render-to-texture",
369 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
370 PIPE_CONTROL_RENDER_TARGET_FLUSH |
371 PIPE_CONTROL_CS_STALL);
372
373 iris_emit_pipe_control_flush(batch,
374 "cache tracker: render-to-texture",
375 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
376 PIPE_CONTROL_CONST_CACHE_INVALIDATE);
377
378 iris_cache_sets_clear(batch);
379 }
380
381 void
382 iris_cache_flush_for_read(struct iris_batch *batch,
383 struct iris_bo *bo)
384 {
385 if (_mesa_hash_table_search_pre_hashed(batch->cache.render, bo->hash, bo) ||
386 _mesa_set_search_pre_hashed(batch->cache.depth, bo->hash, bo))
387 iris_flush_depth_and_render_caches(batch);
388 }
389
390 static void *
391 format_aux_tuple(enum isl_format format, enum isl_aux_usage aux_usage)
392 {
393 return (void *)(uintptr_t)((uint32_t)format << 8 | aux_usage);
394 }
395
396 void
397 iris_cache_flush_for_render(struct iris_batch *batch,
398 struct iris_bo *bo,
399 enum isl_format format,
400 enum isl_aux_usage aux_usage)
401 {
402 if (_mesa_set_search_pre_hashed(batch->cache.depth, bo->hash, bo))
403 iris_flush_depth_and_render_caches(batch);
404
405 /* Check to see if this bo has been used by a previous rendering operation
406 * but with a different format or aux usage. If it has, flush the render
407 * cache so we ensure that it's only in there with one format or aux usage
408 * at a time.
409 *
410 * Even though it's not obvious, this can easily happen in practice.
411 * Suppose a client is blending on a surface with sRGB encode enabled on
412 * gen9. This implies that you get AUX_USAGE_CCS_D at best. If the client
413 * then disables sRGB decode and continues blending we will flip on
414 * AUX_USAGE_CCS_E without doing any sort of resolve in-between (this is
415 * perfectly valid since CCS_E is a subset of CCS_D). However, this means
416 * that we have fragments in-flight which are rendering with UNORM+CCS_E
417 * and other fragments in-flight with SRGB+CCS_D on the same surface at the
418 * same time and the pixel scoreboard and color blender are trying to sort
419 * it all out. This ends badly (i.e. GPU hangs).
420 *
421 * To date, we have never observed GPU hangs or even corruption to be
422 * associated with switching the format, only the aux usage. However,
423 * there are comments in various docs which indicate that the render cache
424 * isn't 100% resilient to format changes. We may as well be conservative
425 * and flush on format changes too. We can always relax this later if we
426 * find it to be a performance problem.
427 */
428 struct hash_entry *entry =
429 _mesa_hash_table_search_pre_hashed(batch->cache.render, bo->hash, bo);
430 if (entry && entry->data != format_aux_tuple(format, aux_usage))
431 iris_flush_depth_and_render_caches(batch);
432 }
433
434 void
435 iris_render_cache_add_bo(struct iris_batch *batch,
436 struct iris_bo *bo,
437 enum isl_format format,
438 enum isl_aux_usage aux_usage)
439 {
440 #ifndef NDEBUG
441 struct hash_entry *entry =
442 _mesa_hash_table_search_pre_hashed(batch->cache.render, bo->hash, bo);
443 if (entry) {
444 /* Otherwise, someone didn't do a flush_for_render and that would be
445 * very bad indeed.
446 */
447 assert(entry->data == format_aux_tuple(format, aux_usage));
448 }
449 #endif
450
451 _mesa_hash_table_insert_pre_hashed(batch->cache.render, bo->hash, bo,
452 format_aux_tuple(format, aux_usage));
453 }
454
455 void
456 iris_cache_flush_for_depth(struct iris_batch *batch,
457 struct iris_bo *bo)
458 {
459 if (_mesa_hash_table_search_pre_hashed(batch->cache.render, bo->hash, bo))
460 iris_flush_depth_and_render_caches(batch);
461 }
462
463 void
464 iris_depth_cache_add_bo(struct iris_batch *batch, struct iris_bo *bo)
465 {
466 _mesa_set_add_pre_hashed(batch->cache.depth, bo->hash, bo);
467 }
468
469 static void
470 iris_resolve_color(struct iris_context *ice,
471 struct iris_batch *batch,
472 struct iris_resource *res,
473 unsigned level, unsigned layer,
474 enum isl_aux_op resolve_op)
475 {
476 //DBG("%s to mt %p level %u layer %u\n", __FUNCTION__, mt, level, layer);
477
478 struct blorp_surf surf;
479 iris_blorp_surf_for_resource(&ice->vtbl, &batch->screen->isl_dev, &surf,
480 &res->base, res->aux.usage, level, true);
481
482 iris_batch_maybe_flush(batch, 1500);
483
484 /* Ivybridge PRM Vol 2, Part 1, "11.7 MCS Buffer for Render Target(s)":
485 *
486 * "Any transition from any value in {Clear, Render, Resolve} to a
487 * different value in {Clear, Render, Resolve} requires end of pipe
488 * synchronization."
489 *
490 * In other words, fast clear ops are not properly synchronized with
491 * other drawing. We need to use a PIPE_CONTROL to ensure that the
492 * contents of the previous draw hit the render target before we resolve
493 * and again afterwards to ensure that the resolve is complete before we
494 * do any more regular drawing.
495 */
496 iris_emit_end_of_pipe_sync(batch, "color resolve: pre-flush",
497 PIPE_CONTROL_RENDER_TARGET_FLUSH);
498
499 struct blorp_batch blorp_batch;
500 blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0);
501 /* On Gen >= 12, Stencil buffer with lossless compression needs to be
502 * resolve with WM_HZ_OP packet.
503 */
504 if (res->aux.usage == ISL_AUX_USAGE_STC_CCS) {
505 blorp_hiz_stencil_op(&blorp_batch, &surf, level, layer,
506 1, resolve_op);
507 } else {
508 blorp_ccs_resolve(&blorp_batch, &surf, level, layer, 1,
509 isl_format_srgb_to_linear(res->surf.format),
510 resolve_op);
511 }
512 blorp_batch_finish(&blorp_batch);
513
514 /* See comment above */
515 iris_emit_end_of_pipe_sync(batch, "color resolve: post-flush",
516 PIPE_CONTROL_RENDER_TARGET_FLUSH);
517 }
518
519 static void
520 iris_mcs_partial_resolve(struct iris_context *ice,
521 struct iris_batch *batch,
522 struct iris_resource *res,
523 uint32_t start_layer,
524 uint32_t num_layers)
525 {
526 //DBG("%s to mt %p layers %u-%u\n", __FUNCTION__, mt,
527 //start_layer, start_layer + num_layers - 1);
528
529 assert(isl_aux_usage_has_mcs(res->aux.usage));
530
531 struct blorp_surf surf;
532 iris_blorp_surf_for_resource(&ice->vtbl, &batch->screen->isl_dev, &surf,
533 &res->base, res->aux.usage, 0, true);
534
535 struct blorp_batch blorp_batch;
536 blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0);
537 blorp_mcs_partial_resolve(&blorp_batch, &surf,
538 isl_format_srgb_to_linear(res->surf.format),
539 start_layer, num_layers);
540 blorp_batch_finish(&blorp_batch);
541 }
542
543
544 /**
545 * Return true if the format that will be used to access the resource is
546 * CCS_E-compatible with the resource's linear/non-sRGB format.
547 *
548 * Why use the linear format? Well, although the resourcemay be specified
549 * with an sRGB format, the usage of that color space/format can be toggled.
550 * Since our HW tends to support more linear formats than sRGB ones, we use
551 * this format variant for check for CCS_E compatibility.
552 */
553 static bool
554 format_ccs_e_compat_with_resource(const struct gen_device_info *devinfo,
555 const struct iris_resource *res,
556 enum isl_format access_format)
557 {
558 assert(res->aux.usage == ISL_AUX_USAGE_CCS_E);
559
560 enum isl_format isl_format = isl_format_srgb_to_linear(res->surf.format);
561 return isl_formats_are_ccs_e_compatible(devinfo, isl_format, access_format);
562 }
563
564 bool
565 iris_sample_with_depth_aux(const struct gen_device_info *devinfo,
566 const struct iris_resource *res)
567 {
568 switch (res->aux.usage) {
569 case ISL_AUX_USAGE_HIZ:
570 if (devinfo->has_sample_with_hiz)
571 break;
572 return false;
573 case ISL_AUX_USAGE_HIZ_CCS:
574 return false;
575 case ISL_AUX_USAGE_HIZ_CCS_WT:
576 break;
577 default:
578 return false;
579 }
580
581 /* It seems the hardware won't fallback to the depth buffer if some of the
582 * mipmap levels aren't available in the HiZ buffer. So we need all levels
583 * of the texture to be HiZ enabled.
584 */
585 for (unsigned level = 0; level < res->surf.levels; ++level) {
586 if (!iris_resource_level_has_hiz(res, level))
587 return false;
588 }
589
590 /* If compressed multisampling is enabled, then we use it for the auxiliary
591 * buffer instead.
592 *
593 * From the BDW PRM (Volume 2d: Command Reference: Structures
594 * RENDER_SURFACE_STATE.AuxiliarySurfaceMode):
595 *
596 * "If this field is set to AUX_HIZ, Number of Multisamples must be
597 * MULTISAMPLECOUNT_1, and Surface Type cannot be SURFTYPE_3D.
598 *
599 * There is no such blurb for 1D textures, but there is sufficient evidence
600 * that this is broken on SKL+.
601 */
602 // XXX: i965 disables this for arrays too, is that reasonable?
603 return res->surf.samples == 1 && res->surf.dim == ISL_SURF_DIM_2D;
604 }
605
606 /**
607 * Perform a HiZ or depth resolve operation.
608 *
609 * For an overview of HiZ ops, see the following sections of the Sandy Bridge
610 * PRM, Volume 1, Part 2:
611 * - 7.5.3.1 Depth Buffer Clear
612 * - 7.5.3.2 Depth Buffer Resolve
613 * - 7.5.3.3 Hierarchical Depth Buffer Resolve
614 */
615 void
616 iris_hiz_exec(struct iris_context *ice,
617 struct iris_batch *batch,
618 struct iris_resource *res,
619 unsigned int level, unsigned int start_layer,
620 unsigned int num_layers, enum isl_aux_op op,
621 bool update_clear_depth)
622 {
623 assert(iris_resource_level_has_hiz(res, level));
624 assert(op != ISL_AUX_OP_NONE);
625 UNUSED const char *name = NULL;
626
627 switch (op) {
628 case ISL_AUX_OP_FULL_RESOLVE:
629 name = "depth resolve";
630 break;
631 case ISL_AUX_OP_AMBIGUATE:
632 name = "hiz ambiguate";
633 break;
634 case ISL_AUX_OP_FAST_CLEAR:
635 name = "depth clear";
636 break;
637 case ISL_AUX_OP_PARTIAL_RESOLVE:
638 case ISL_AUX_OP_NONE:
639 unreachable("Invalid HiZ op");
640 }
641
642 //DBG("%s %s to mt %p level %d layers %d-%d\n",
643 //__func__, name, mt, level, start_layer, start_layer + num_layers - 1);
644
645 /* The following stalls and flushes are only documented to be required
646 * for HiZ clear operations. However, they also seem to be required for
647 * resolve operations.
648 *
649 * From the Ivybridge PRM, volume 2, "Depth Buffer Clear":
650 *
651 * "If other rendering operations have preceded this clear, a
652 * PIPE_CONTROL with depth cache flush enabled, Depth Stall bit
653 * enabled must be issued before the rectangle primitive used for
654 * the depth buffer clear operation."
655 *
656 * Same applies for Gen8 and Gen9.
657 *
658 * In addition, from the Ivybridge PRM, volume 2, 1.10.4.1
659 * PIPE_CONTROL, Depth Cache Flush Enable:
660 *
661 * "This bit must not be set when Depth Stall Enable bit is set in
662 * this packet."
663 *
664 * This is confirmed to hold for real, Haswell gets immediate gpu hangs.
665 *
666 * Therefore issue two pipe control flushes, one for cache flush and
667 * another for depth stall.
668 */
669 iris_emit_pipe_control_flush(batch,
670 "hiz op: pre-flushes (1/2)",
671 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
672 PIPE_CONTROL_CS_STALL);
673
674 iris_emit_pipe_control_flush(batch, "hiz op: pre-flushes (2/2)",
675 PIPE_CONTROL_DEPTH_STALL);
676
677 assert(isl_aux_usage_has_hiz(res->aux.usage) && res->aux.bo);
678
679 iris_batch_maybe_flush(batch, 1500);
680
681 struct blorp_surf surf;
682 iris_blorp_surf_for_resource(&ice->vtbl, &batch->screen->isl_dev, &surf,
683 &res->base, res->aux.usage, level, true);
684
685 struct blorp_batch blorp_batch;
686 enum blorp_batch_flags flags = 0;
687 flags |= update_clear_depth ? 0 : BLORP_BATCH_NO_UPDATE_CLEAR_COLOR;
688 blorp_batch_init(&ice->blorp, &blorp_batch, batch, flags);
689 blorp_hiz_op(&blorp_batch, &surf, level, start_layer, num_layers, op);
690 blorp_batch_finish(&blorp_batch);
691
692 /* The following stalls and flushes are only documented to be required
693 * for HiZ clear operations. However, they also seem to be required for
694 * resolve operations.
695 *
696 * From the Broadwell PRM, volume 7, "Depth Buffer Clear":
697 *
698 * "Depth buffer clear pass using any of the methods (WM_STATE,
699 * 3DSTATE_WM or 3DSTATE_WM_HZ_OP) must be followed by a
700 * PIPE_CONTROL command with DEPTH_STALL bit and Depth FLUSH bits
701 * "set" before starting to render. DepthStall and DepthFlush are
702 * not needed between consecutive depth clear passes nor is it
703 * required if the depth clear pass was done with
704 * 'full_surf_clear' bit set in the 3DSTATE_WM_HZ_OP."
705 *
706 * TODO: Such as the spec says, this could be conditional.
707 */
708 iris_emit_pipe_control_flush(batch,
709 "hiz op: post flush",
710 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
711 PIPE_CONTROL_DEPTH_STALL);
712 }
713
714 static bool
715 level_has_aux(const struct iris_resource *res, uint32_t level)
716 {
717 return isl_aux_usage_has_hiz(res->aux.usage) ?
718 iris_resource_level_has_hiz(res, level) :
719 res->aux.usage != ISL_AUX_USAGE_NONE;
720 }
721
722 /**
723 * Does the resource's slice have hiz enabled?
724 */
725 bool
726 iris_resource_level_has_hiz(const struct iris_resource *res, uint32_t level)
727 {
728 iris_resource_check_level_layer(res, level, 0);
729 return res->aux.has_hiz & 1 << level;
730 }
731
732 /** \brief Assert that the level and layer are valid for the resource. */
733 void
734 iris_resource_check_level_layer(UNUSED const struct iris_resource *res,
735 UNUSED uint32_t level, UNUSED uint32_t layer)
736 {
737 assert(level < res->surf.levels);
738 assert(layer < util_num_layers(&res->base, level));
739 }
740
741 static inline uint32_t
742 miptree_level_range_length(const struct iris_resource *res,
743 uint32_t start_level, uint32_t num_levels)
744 {
745 assert(start_level < res->surf.levels);
746
747 if (num_levels == INTEL_REMAINING_LAYERS)
748 num_levels = res->surf.levels;
749
750 /* Check for overflow */
751 assert(start_level + num_levels >= start_level);
752 assert(start_level + num_levels <= res->surf.levels);
753
754 return num_levels;
755 }
756
757 static inline uint32_t
758 miptree_layer_range_length(const struct iris_resource *res, uint32_t level,
759 uint32_t start_layer, uint32_t num_layers)
760 {
761 assert(level <= res->base.last_level);
762
763 const uint32_t total_num_layers = iris_get_num_logical_layers(res, level);
764 assert(start_layer < total_num_layers);
765 if (num_layers == INTEL_REMAINING_LAYERS)
766 num_layers = total_num_layers - start_layer;
767 /* Check for overflow */
768 assert(start_layer + num_layers >= start_layer);
769 assert(start_layer + num_layers <= total_num_layers);
770
771 return num_layers;
772 }
773
774 bool
775 iris_has_color_unresolved(const struct iris_resource *res,
776 unsigned start_level, unsigned num_levels,
777 unsigned start_layer, unsigned num_layers)
778 {
779 if (!res->aux.bo)
780 return false;
781
782 /* Clamp the level range to fit the resource */
783 num_levels = miptree_level_range_length(res, start_level, num_levels);
784
785 for (uint32_t l = 0; l < num_levels; l++) {
786 const uint32_t level = start_level + l;
787 const uint32_t level_layers =
788 miptree_layer_range_length(res, level, start_layer, num_layers);
789 for (unsigned a = 0; a < level_layers; a++) {
790 enum isl_aux_state aux_state =
791 iris_resource_get_aux_state(res, level, start_layer + a);
792 assert(aux_state != ISL_AUX_STATE_AUX_INVALID);
793 if (aux_state != ISL_AUX_STATE_PASS_THROUGH)
794 return true;
795 }
796 }
797
798 return false;
799 }
800
801 void
802 iris_resource_prepare_access(struct iris_context *ice,
803 struct iris_batch *batch,
804 struct iris_resource *res,
805 uint32_t start_level, uint32_t num_levels,
806 uint32_t start_layer, uint32_t num_layers,
807 enum isl_aux_usage aux_usage,
808 bool fast_clear_supported)
809 {
810 const uint32_t clamped_levels =
811 miptree_level_range_length(res, start_level, num_levels);
812 for (uint32_t l = 0; l < clamped_levels; l++) {
813 const uint32_t level = start_level + l;
814 if (!level_has_aux(res, level))
815 continue;
816
817 const uint32_t level_layers =
818 miptree_layer_range_length(res, level, start_layer, num_layers);
819 for (uint32_t a = 0; a < level_layers; a++) {
820 const uint32_t layer = start_layer + a;
821 const enum isl_aux_state aux_state =
822 iris_resource_get_aux_state(res, level, layer);
823 const enum isl_aux_op aux_op =
824 isl_aux_prepare_access(aux_state, aux_usage, fast_clear_supported);
825
826 if (aux_op == ISL_AUX_OP_NONE) {
827 /* Nothing to do here. */
828 } else if (isl_aux_usage_has_mcs(res->aux.usage)) {
829 assert(aux_op == ISL_AUX_OP_PARTIAL_RESOLVE);
830 iris_mcs_partial_resolve(ice, batch, res, layer, 1);
831 } else if (isl_aux_usage_has_hiz(res->aux.usage)) {
832 iris_hiz_exec(ice, batch, res, level, layer, 1, aux_op, false);
833 } else {
834 assert(isl_aux_usage_has_ccs(res->aux.usage));
835 iris_resolve_color(ice, batch, res, level, layer, aux_op);
836 }
837
838 const enum isl_aux_state new_state =
839 isl_aux_state_transition_aux_op(aux_state, res->aux.usage, aux_op);
840 iris_resource_set_aux_state(ice, res, level, layer, 1, new_state);
841 }
842 }
843 }
844
845 void
846 iris_resource_finish_write(struct iris_context *ice,
847 struct iris_resource *res, uint32_t level,
848 uint32_t start_layer, uint32_t num_layers,
849 enum isl_aux_usage aux_usage)
850 {
851 if (!level_has_aux(res, level))
852 return;
853
854 const uint32_t level_layers =
855 miptree_layer_range_length(res, level, start_layer, num_layers);
856
857 for (uint32_t a = 0; a < level_layers; a++) {
858 const uint32_t layer = start_layer + a;
859 const enum isl_aux_state aux_state =
860 iris_resource_get_aux_state(res, level, layer);
861 const enum isl_aux_state new_aux_state =
862 isl_aux_state_transition_write(aux_state, aux_usage, false);
863 iris_resource_set_aux_state(ice, res, level, layer, 1, new_aux_state);
864 }
865 }
866
867 enum isl_aux_state
868 iris_resource_get_aux_state(const struct iris_resource *res,
869 uint32_t level, uint32_t layer)
870 {
871 iris_resource_check_level_layer(res, level, layer);
872
873 if (res->surf.usage & ISL_SURF_USAGE_DEPTH_BIT) {
874 assert(iris_resource_level_has_hiz(res, level));
875 } else {
876 assert(res->surf.samples == 1 ||
877 res->surf.msaa_layout == ISL_MSAA_LAYOUT_ARRAY);
878 }
879
880 return res->aux.state[level][layer];
881 }
882
883 void
884 iris_resource_set_aux_state(struct iris_context *ice,
885 struct iris_resource *res, uint32_t level,
886 uint32_t start_layer, uint32_t num_layers,
887 enum isl_aux_state aux_state)
888 {
889 num_layers = miptree_layer_range_length(res, level, start_layer, num_layers);
890
891 if (res->surf.usage & ISL_SURF_USAGE_DEPTH_BIT) {
892 assert(iris_resource_level_has_hiz(res, level));
893 } else {
894 assert(res->surf.samples == 1 ||
895 res->surf.msaa_layout == ISL_MSAA_LAYOUT_ARRAY);
896 }
897
898 for (unsigned a = 0; a < num_layers; a++) {
899 if (res->aux.state[level][start_layer + a] != aux_state) {
900 res->aux.state[level][start_layer + a] = aux_state;
901 /* XXX: Need to track which bindings to make dirty */
902 ice->state.dirty |= IRIS_ALL_DIRTY_BINDINGS;
903 }
904 }
905 }
906
907 /* On Gen9 color buffers may be compressed by the hardware (lossless
908 * compression). There are, however, format restrictions and care needs to be
909 * taken that the sampler engine is capable for re-interpreting a buffer with
910 * format different the buffer was originally written with.
911 *
912 * For example, SRGB formats are not compressible and the sampler engine isn't
913 * capable of treating RGBA_UNORM as SRGB_ALPHA. In such a case the underlying
914 * color buffer needs to be resolved so that the sampling surface can be
915 * sampled as non-compressed (i.e., without the auxiliary MCS buffer being
916 * set).
917 */
918 static bool
919 can_texture_with_ccs(const struct gen_device_info *devinfo,
920 struct pipe_debug_callback *dbg,
921 const struct iris_resource *res,
922 enum isl_format view_format)
923 {
924 if (res->aux.usage != ISL_AUX_USAGE_CCS_E)
925 return false;
926
927 if (!format_ccs_e_compat_with_resource(devinfo, res, view_format)) {
928 const struct isl_format_layout *res_fmtl =
929 isl_format_get_layout(res->surf.format);
930 const struct isl_format_layout *view_fmtl =
931 isl_format_get_layout(view_format);
932
933 perf_debug(dbg, "Incompatible sampling format (%s) for CCS (%s)\n",
934 view_fmtl->name, res_fmtl->name);
935
936 return false;
937 }
938
939 return true;
940 }
941
942 enum isl_aux_usage
943 iris_resource_texture_aux_usage(struct iris_context *ice,
944 const struct iris_resource *res,
945 enum isl_format view_format)
946 {
947 struct iris_screen *screen = (void *) ice->ctx.screen;
948 struct gen_device_info *devinfo = &screen->devinfo;
949
950 switch (res->aux.usage) {
951 case ISL_AUX_USAGE_HIZ:
952 if (iris_sample_with_depth_aux(devinfo, res))
953 return ISL_AUX_USAGE_HIZ;
954 break;
955
956 case ISL_AUX_USAGE_HIZ_CCS:
957 assert(!iris_sample_with_depth_aux(devinfo, res));
958 return ISL_AUX_USAGE_NONE;
959
960 case ISL_AUX_USAGE_HIZ_CCS_WT:
961 if (iris_sample_with_depth_aux(devinfo, res))
962 return ISL_AUX_USAGE_HIZ_CCS_WT;
963 break;
964
965 case ISL_AUX_USAGE_MCS:
966 case ISL_AUX_USAGE_MCS_CCS:
967 case ISL_AUX_USAGE_STC_CCS:
968 return res->aux.usage;
969
970 case ISL_AUX_USAGE_CCS_D:
971 case ISL_AUX_USAGE_CCS_E:
972 /* If we don't have any unresolved color, report an aux usage of
973 * ISL_AUX_USAGE_NONE. This way, texturing won't even look at the
974 * aux surface and we can save some bandwidth.
975 */
976 if (!iris_has_color_unresolved(res, 0, INTEL_REMAINING_LEVELS,
977 0, INTEL_REMAINING_LAYERS))
978 return ISL_AUX_USAGE_NONE;
979
980 if (can_texture_with_ccs(devinfo, &ice->dbg, res, view_format))
981 return ISL_AUX_USAGE_CCS_E;
982 break;
983
984 default:
985 break;
986 }
987
988 return ISL_AUX_USAGE_NONE;
989 }
990
991 enum isl_aux_usage
992 iris_image_view_aux_usage(struct iris_context *ice,
993 const struct pipe_image_view *pview,
994 const struct shader_info *info)
995 {
996 if (!info)
997 return ISL_AUX_USAGE_NONE;
998
999 struct iris_screen *screen = (void *) ice->ctx.screen;
1000 const struct gen_device_info *devinfo = &screen->devinfo;
1001 struct iris_resource *res = (void *) pview->resource;
1002
1003 enum isl_format view_format = iris_image_view_get_format(ice, pview);
1004 enum isl_aux_usage aux_usage =
1005 iris_resource_texture_aux_usage(ice, res, view_format);
1006
1007 bool uses_atomic_load_store =
1008 ice->shaders.uncompiled[info->stage]->uses_atomic_load_store;
1009
1010 if ((devinfo->gen == 12 && aux_usage == ISL_AUX_USAGE_CCS_E) &&
1011 !uses_atomic_load_store)
1012 return ISL_AUX_USAGE_CCS_E;
1013
1014 return ISL_AUX_USAGE_NONE;
1015 }
1016
1017 static bool
1018 isl_formats_are_fast_clear_compatible(enum isl_format a, enum isl_format b)
1019 {
1020 /* On gen8 and earlier, the hardware was only capable of handling 0/1 clear
1021 * values so sRGB curve application was a no-op for all fast-clearable
1022 * formats.
1023 *
1024 * On gen9+, the hardware supports arbitrary clear values. For sRGB clear
1025 * values, the hardware interprets the floats, not as what would be
1026 * returned from the sampler (or written by the shader), but as being
1027 * between format conversion and sRGB curve application. This means that
1028 * we can switch between sRGB and UNORM without having to whack the clear
1029 * color.
1030 */
1031 return isl_format_srgb_to_linear(a) == isl_format_srgb_to_linear(b);
1032 }
1033
1034 void
1035 iris_resource_prepare_texture(struct iris_context *ice,
1036 struct iris_batch *batch,
1037 struct iris_resource *res,
1038 enum isl_format view_format,
1039 uint32_t start_level, uint32_t num_levels,
1040 uint32_t start_layer, uint32_t num_layers)
1041 {
1042 enum isl_aux_usage aux_usage =
1043 iris_resource_texture_aux_usage(ice, res, view_format);
1044
1045 bool clear_supported = isl_aux_usage_has_fast_clears(aux_usage);
1046
1047 /* Clear color is specified as ints or floats and the conversion is done by
1048 * the sampler. If we have a texture view, we would have to perform the
1049 * clear color conversion manually. Just disable clear color.
1050 */
1051 if (!isl_formats_are_fast_clear_compatible(res->surf.format, view_format))
1052 clear_supported = false;
1053
1054 iris_resource_prepare_access(ice, batch, res, start_level, num_levels,
1055 start_layer, num_layers,
1056 aux_usage, clear_supported);
1057 }
1058
1059 enum isl_aux_usage
1060 iris_resource_render_aux_usage(struct iris_context *ice,
1061 struct iris_resource *res,
1062 enum isl_format render_format,
1063 bool blend_enabled,
1064 bool draw_aux_disabled)
1065 {
1066 struct iris_screen *screen = (void *) ice->ctx.screen;
1067 struct gen_device_info *devinfo = &screen->devinfo;
1068
1069 if (draw_aux_disabled)
1070 return ISL_AUX_USAGE_NONE;
1071
1072 switch (res->aux.usage) {
1073 case ISL_AUX_USAGE_MCS:
1074 case ISL_AUX_USAGE_MCS_CCS:
1075 return res->aux.usage;
1076
1077 case ISL_AUX_USAGE_CCS_D:
1078 case ISL_AUX_USAGE_CCS_E:
1079 /* Gen9+ hardware technically supports non-0/1 clear colors with sRGB
1080 * formats. However, there are issues with blending where it doesn't
1081 * properly apply the sRGB curve to the clear color when blending.
1082 */
1083 if (devinfo->gen >= 9 && blend_enabled &&
1084 isl_format_is_srgb(render_format) &&
1085 !isl_color_value_is_zero_one(res->aux.clear_color, render_format))
1086 return ISL_AUX_USAGE_NONE;
1087
1088 if (res->aux.usage == ISL_AUX_USAGE_CCS_E &&
1089 format_ccs_e_compat_with_resource(devinfo, res, render_format))
1090 return ISL_AUX_USAGE_CCS_E;
1091
1092 /* Otherwise, we try to fall back to CCS_D */
1093 if (isl_format_supports_ccs_d(devinfo, render_format))
1094 return ISL_AUX_USAGE_CCS_D;
1095
1096 default:
1097 return ISL_AUX_USAGE_NONE;
1098 }
1099 }
1100
1101 void
1102 iris_resource_prepare_render(struct iris_context *ice,
1103 struct iris_batch *batch,
1104 struct iris_resource *res, uint32_t level,
1105 uint32_t start_layer, uint32_t layer_count,
1106 enum isl_aux_usage aux_usage)
1107 {
1108 iris_resource_prepare_access(ice, batch, res, level, 1, start_layer,
1109 layer_count, aux_usage,
1110 isl_aux_usage_has_fast_clears(aux_usage));
1111 }
1112
1113 void
1114 iris_resource_finish_render(struct iris_context *ice,
1115 struct iris_resource *res, uint32_t level,
1116 uint32_t start_layer, uint32_t layer_count,
1117 enum isl_aux_usage aux_usage)
1118 {
1119 iris_resource_finish_write(ice, res, level, start_layer, layer_count,
1120 aux_usage);
1121 }
1122
1123 void
1124 iris_resource_prepare_depth(struct iris_context *ice,
1125 struct iris_batch *batch,
1126 struct iris_resource *res, uint32_t level,
1127 uint32_t start_layer, uint32_t layer_count)
1128 {
1129 iris_resource_prepare_access(ice, batch, res, level, 1, start_layer,
1130 layer_count, res->aux.usage, !!res->aux.bo);
1131 }
1132
1133 void
1134 iris_resource_finish_depth(struct iris_context *ice,
1135 struct iris_resource *res, uint32_t level,
1136 uint32_t start_layer, uint32_t layer_count,
1137 bool depth_written)
1138 {
1139 if (depth_written) {
1140 iris_resource_finish_write(ice, res, level, start_layer, layer_count,
1141 res->aux.usage);
1142 }
1143 }