src/gallium/drivers/iris/iris_resolve.c

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included
  12  * in all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  15  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  19  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  20  * DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 /**
  24  * @file iris_resolve.c
  25  *
  26  * This file handles resolve tracking for main and auxiliary surfaces.
  27  *
  28  * It also handles our cache tracking.  We have sets for the render cache,
  29  * depth cache, and so on.  If a BO is in a cache's set, then it may have
  30  * data in that cache.  The helpers take care of emitting flushes for
  31  * render-to-texture, format reinterpretation issues, and other situations.
  32  */
  33
  34 #include "util/hash_table.h"
  35 #include "util/set.h"
  36 #include "iris_context.h"
  37
  38 static void
  39 resolve_sampler_views(struct iris_batch *batch,
  40                       struct iris_shader_state *shs)
  41 {
  42    uint32_t views = shs->bound_sampler_views;
  43
  44    while (views) {
  45       const int i = u_bit_scan(&views);
  46       struct iris_sampler_view *isv = shs->textures[i];
  47       struct iris_resource *res = (void *) isv->base.texture;
  48
  49       // XXX: aux tracking
  50       iris_cache_flush_for_read(batch, res->bo);
  51    }
  52 }
  53
  54 static void
  55 resolve_image_views(struct iris_batch *batch,
  56                     struct iris_shader_state *shs)
  57 {
  58    uint32_t views = shs->bound_image_views;
  59
  60    while (views) {
  61       const int i = u_bit_scan(&views);
  62       struct pipe_resource *res = shs->image[i].res;
  63
  64       // XXX: aux tracking
  65       iris_cache_flush_for_read(batch, iris_resource_bo(res));
  66    }
  67 }
  68
  69
  70 /**
  71  * \brief Resolve buffers before drawing.
  72  *
  73  * Resolve the depth buffer's HiZ buffer, resolve the depth buffer of each
  74  * enabled depth texture, and flush the render cache for any dirty textures.
  75  */
  76 void
  77 iris_predraw_resolve_inputs(struct iris_batch *batch,
  78                             struct iris_shader_state *shs)
  79 {
  80    resolve_sampler_views(batch, shs);
  81    resolve_image_views(batch, shs);
  82 }
  83
  84 void
  85 iris_predraw_resolve_framebuffer(struct iris_context *ice,
  86                                  struct iris_batch *batch)
  87 {
  88    struct pipe_framebuffer_state *cso_fb = &ice->state.framebuffer;
  89    struct pipe_surface *zs_surf = cso_fb->zsbuf;
  90
  91    if (zs_surf) {
  92       // XXX: HiZ resolves
  93    }
  94
  95    for (unsigned i = 0; i < cso_fb->nr_cbufs; i++) {
  96       struct iris_surface *surf = (void *) cso_fb->cbufs[i];
  97       if (!surf)
  98          continue;
  99
 100       struct iris_resource *res = (void *) surf->base.texture;
 101
 102       // XXX: aux tracking
 103
 104       iris_cache_flush_for_render(batch, res->bo, surf->view.format,
 105                                   ISL_AUX_USAGE_NONE);
 106    }
 107 }
 108
 109 /**
 110  * \brief Call this after drawing to mark which buffers need resolving
 111  *
 112  * If the depth buffer was written to and if it has an accompanying HiZ
 113  * buffer, then mark that it needs a depth resolve.
 114  *
 115  * If the color buffer is a multisample window system buffer, then
 116  * mark that it needs a downsample.
 117  *
 118  * Also mark any render targets which will be textured as needing a render
 119  * cache flush.
 120  */
 121 void
 122 iris_postdraw_update_resolve_tracking(struct iris_context *ice,
 123                                       struct iris_batch *batch)
 124 {
 125    struct pipe_framebuffer_state *cso_fb = &ice->state.framebuffer;
 126    struct pipe_surface *zs_surf = cso_fb->zsbuf;
 127
 128    // XXX: front buffer drawing?
 129
 130    if (zs_surf) {
 131       struct iris_resource *z_res, *s_res;
 132       iris_get_depth_stencil_resources(zs_surf->texture, &z_res, &s_res);
 133
 134       if (z_res) {
 135          // XXX: aux tracking
 136
 137          if (ice->state.depth_writes_enabled)
 138             iris_depth_cache_add_bo(batch, z_res->bo);
 139       }
 140
 141       if (s_res) {
 142          // XXX: aux tracking
 143
 144          if (ice->state.stencil_writes_enabled)
 145             iris_depth_cache_add_bo(batch, s_res->bo);
 146       }
 147    }
 148
 149    for (unsigned i = 0; i < cso_fb->nr_cbufs; i++) {
 150       struct iris_surface *surf = (void *) cso_fb->cbufs[i];
 151       if (!surf)
 152          continue;
 153
 154       struct iris_resource *res = (void *) surf->base.texture;
 155
 156       // XXX: aux tracking
 157       iris_render_cache_add_bo(batch, res->bo, surf->view.format,
 158                                ISL_AUX_USAGE_NONE);
 159    }
 160 }
 161
 162 /**
 163  * Clear the cache-tracking sets.
 164  */
 165 void
 166 iris_cache_sets_clear(struct iris_batch *batch)
 167 {
 168    hash_table_foreach(batch->cache.render, render_entry)
 169       _mesa_hash_table_remove(batch->cache.render, render_entry);
 170
 171    set_foreach(batch->cache.depth, depth_entry)
 172       _mesa_set_remove(batch->cache.depth, depth_entry);
 173 }
 174
 175 /**
 176  * Emits an appropriate flush for a BO if it has been rendered to within the
 177  * same batchbuffer as a read that's about to be emitted.
 178  *
 179  * The GPU has separate, incoherent caches for the render cache and the
 180  * sampler cache, along with other caches.  Usually data in the different
 181  * caches don't interact (e.g. we don't render to our driver-generated
 182  * immediate constant data), but for render-to-texture in FBOs we definitely
 183  * do.  When a batchbuffer is flushed, the kernel will ensure that everything
 184  * necessary is flushed before another use of that BO, but for reuse from
 185  * different caches within a batchbuffer, it's all our responsibility.
 186  */
 187 void
 188 iris_flush_depth_and_render_caches(struct iris_batch *batch)
 189 {
 190    iris_emit_pipe_control_flush(batch,
 191                                 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
 192                                 PIPE_CONTROL_RENDER_TARGET_FLUSH |
 193                                 PIPE_CONTROL_CS_STALL);
 194
 195    iris_emit_pipe_control_flush(batch,
 196                                 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
 197                                 PIPE_CONTROL_CONST_CACHE_INVALIDATE);
 198
 199    iris_cache_sets_clear(batch);
 200 }
 201
 202 void
 203 iris_cache_flush_for_read(struct iris_batch *batch,
 204                           struct iris_bo *bo)
 205 {
 206    if (_mesa_hash_table_search_pre_hashed(batch->cache.render, bo->hash, bo) ||
 207        _mesa_set_search_pre_hashed(batch->cache.depth, bo->hash, bo))
 208       iris_flush_depth_and_render_caches(batch);
 209 }
 210
 211 static void *
 212 format_aux_tuple(enum isl_format format, enum isl_aux_usage aux_usage)
 213 {
 214    return (void *)(uintptr_t)((uint32_t)format << 8 | aux_usage);
 215 }
 216
 217 void
 218 iris_cache_flush_for_render(struct iris_batch *batch,
 219                             struct iris_bo *bo,
 220                             enum isl_format format,
 221                             enum isl_aux_usage aux_usage)
 222 {
 223    if (_mesa_set_search_pre_hashed(batch->cache.depth, bo->hash, bo))
 224       iris_flush_depth_and_render_caches(batch);
 225
 226    /* Check to see if this bo has been used by a previous rendering operation
 227     * but with a different format or aux usage.  If it has, flush the render
 228     * cache so we ensure that it's only in there with one format or aux usage
 229     * at a time.
 230     *
 231     * Even though it's not obvious, this can easily happen in practice.
 232     * Suppose a client is blending on a surface with sRGB encode enabled on
 233     * gen9.  This implies that you get AUX_USAGE_CCS_D at best.  If the client
 234     * then disables sRGB decode and continues blending we will flip on
 235     * AUX_USAGE_CCS_E without doing any sort of resolve in-between (this is
 236     * perfectly valid since CCS_E is a subset of CCS_D).  However, this means
 237     * that we have fragments in-flight which are rendering with UNORM+CCS_E
 238     * and other fragments in-flight with SRGB+CCS_D on the same surface at the
 239     * same time and the pixel scoreboard and color blender are trying to sort
 240     * it all out.  This ends badly (i.e. GPU hangs).
 241     *
 242     * To date, we have never observed GPU hangs or even corruption to be
 243     * associated with switching the format, only the aux usage.  However,
 244     * there are comments in various docs which indicate that the render cache
 245     * isn't 100% resilient to format changes.  We may as well be conservative
 246     * and flush on format changes too.  We can always relax this later if we
 247     * find it to be a performance problem.
 248     */
 249    struct hash_entry *entry =
 250       _mesa_hash_table_search_pre_hashed(batch->cache.render, bo->hash, bo);
 251    if (entry && entry->data != format_aux_tuple(format, aux_usage))
 252       iris_flush_depth_and_render_caches(batch);
 253 }
 254
 255 void
 256 iris_render_cache_add_bo(struct iris_batch *batch,
 257                          struct iris_bo *bo,
 258                          enum isl_format format,
 259                          enum isl_aux_usage aux_usage)
 260 {
 261 #ifndef NDEBUG
 262    struct hash_entry *entry =
 263       _mesa_hash_table_search_pre_hashed(batch->cache.render, bo->hash, bo);
 264    if (entry) {
 265       /* Otherwise, someone didn't do a flush_for_render and that would be
 266        * very bad indeed.
 267        */
 268       assert(entry->data == format_aux_tuple(format, aux_usage));
 269    }
 270 #endif
 271
 272    _mesa_hash_table_insert_pre_hashed(batch->cache.render, bo->hash, bo,
 273                                       format_aux_tuple(format, aux_usage));
 274 }
 275
 276 void
 277 iris_cache_flush_for_depth(struct iris_batch *batch,
 278                            struct iris_bo *bo)
 279 {
 280    if (_mesa_hash_table_search_pre_hashed(batch->cache.render, bo->hash, bo))
 281       iris_flush_depth_and_render_caches(batch);
 282 }
 283
 284 void
 285 iris_depth_cache_add_bo(struct iris_batch *batch, struct iris_bo *bo)
 286 {
 287    _mesa_set_add_pre_hashed(batch->cache.depth, bo->hash, bo);
 288 }