src/gallium/drivers/iris/iris_pipe_control.c

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included
  12  * in all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  15  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  19  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  20  * DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 /**
  24  * @file iris_pipe_control.c
  25  *
  26  * PIPE_CONTROL is the main flushing and synchronization primitive on Intel
  27  * GPUs.  It can invalidate caches, stall until rendering reaches various
  28  * stages of completion, write to memory, and other things.  In a way, it's
  29  * a swiss army knife command - it has all kinds of capabilities, but some
  30  * significant limitations as well.
  31  *
  32  * Unfortunately, it's notoriously complicated and difficult to use.  Many
  33  * sub-commands can't be used together.  Some are meant to be used at the
  34  * top of the pipeline (invalidating caches before drawing), while some are
  35  * meant to be used at the end (stalling or flushing after drawing).
  36  *
  37  * Also, there's a list of restrictions a mile long, which vary by generation.
  38  * Do this before doing that, or suffer the consequences (usually a GPU hang).
  39  *
  40  * This file contains helpers for emitting them safely.  You can simply call
  41  * iris_emit_pipe_control_flush() with the desired operations (as logical
  42  * PIPE_CONTROL_* bits), and it will take care of splitting it into multiple
  43  * PIPE_CONTROL commands as necessary.  The per-generation workarounds are
  44  * applied in iris_emit_raw_pipe_control() in iris_state.c.
  45  */
  46
  47 #include "iris_context.h"
  48 #include "util/hash_table.h"
  49 #include "util/set.h"
  50
  51 /**
  52  * Emit a PIPE_CONTROL with various flushing flags.
  53  *
  54  * The caller is responsible for deciding what flags are appropriate for the
  55  * given generation.
  56  */
  57 void
  58 iris_emit_pipe_control_flush(struct iris_batch *batch,
  59                              const char *reason,
  60                              uint32_t flags)
  61 {
  62    if ((flags & PIPE_CONTROL_CACHE_FLUSH_BITS) &&
  63        (flags & PIPE_CONTROL_CACHE_INVALIDATE_BITS)) {
  64       /* A pipe control command with flush and invalidate bits set
  65        * simultaneously is an inherently racy operation on Gen6+ if the
  66        * contents of the flushed caches were intended to become visible from
  67        * any of the invalidated caches.  Split it in two PIPE_CONTROLs, the
  68        * first one should stall the pipeline to make sure that the flushed R/W
  69        * caches are coherent with memory once the specified R/O caches are
  70        * invalidated.  On pre-Gen6 hardware the (implicit) R/O cache
  71        * invalidation seems to happen at the bottom of the pipeline together
  72        * with any write cache flush, so this shouldn't be a concern.  In order
  73        * to ensure a full stall, we do an end-of-pipe sync.
  74        */
  75       iris_emit_end_of_pipe_sync(batch, reason,
  76                                  flags & PIPE_CONTROL_CACHE_FLUSH_BITS);
  77       flags &= ~(PIPE_CONTROL_CACHE_FLUSH_BITS | PIPE_CONTROL_CS_STALL);
  78    }
  79
  80    batch->vtbl->emit_raw_pipe_control(batch, reason, flags, NULL, 0, 0);
  81 }
  82
  83 /**
  84  * Emit a PIPE_CONTROL that writes to a buffer object.
  85  *
  86  * \p flags should contain one of the following items:
  87  *  - PIPE_CONTROL_WRITE_IMMEDIATE
  88  *  - PIPE_CONTROL_WRITE_TIMESTAMP
  89  *  - PIPE_CONTROL_WRITE_DEPTH_COUNT
  90  */
  91 void
  92 iris_emit_pipe_control_write(struct iris_batch *batch,
  93                              const char *reason, uint32_t flags,
  94                              struct iris_bo *bo, uint32_t offset,
  95                              uint64_t imm)
  96 {
  97    batch->vtbl->emit_raw_pipe_control(batch, reason, flags, bo, offset, imm);
  98 }
  99
 100 /*
 101  * From Sandybridge PRM, volume 2, "1.7.2 End-of-Pipe Synchronization":
 102  *
 103  *  Write synchronization is a special case of end-of-pipe
 104  *  synchronization that requires that the render cache and/or depth
 105  *  related caches are flushed to memory, where the data will become
 106  *  globally visible. This type of synchronization is required prior to
 107  *  SW (CPU) actually reading the result data from memory, or initiating
 108  *  an operation that will use as a read surface (such as a texture
 109  *  surface) a previous render target and/or depth/stencil buffer
 110  *
 111  * From Haswell PRM, volume 2, part 1, "End-of-Pipe Synchronization":
 112  *
 113  *  Exercising the write cache flush bits (Render Target Cache Flush
 114  *  Enable, Depth Cache Flush Enable, DC Flush) in PIPE_CONTROL only
 115  *  ensures the write caches are flushed and doesn't guarantee the data
 116  *  is globally visible.
 117  *
 118  *  SW can track the completion of the end-of-pipe-synchronization by
 119  *  using "Notify Enable" and "PostSync Operation - Write Immediate
 120  *  Data" in the PIPE_CONTROL command.
 121  */
 122 void
 123 iris_emit_end_of_pipe_sync(struct iris_batch *batch,
 124                            const char *reason, uint32_t flags)
 125 {
 126    /* From Sandybridge PRM, volume 2, "1.7.3.1 Writing a Value to Memory":
 127     *
 128     *    "The most common action to perform upon reaching a synchronization
 129     *    point is to write a value out to memory. An immediate value
 130     *    (included with the synchronization command) may be written."
 131     *
 132     * From Broadwell PRM, volume 7, "End-of-Pipe Synchronization":
 133     *
 134     *    "In case the data flushed out by the render engine is to be read
 135     *    back in to the render engine in coherent manner, then the render
 136     *    engine has to wait for the fence completion before accessing the
 137     *    flushed data. This can be achieved by following means on various
 138     *    products: PIPE_CONTROL command with CS Stall and the required
 139     *    write caches flushed with Post-Sync-Operation as Write Immediate
 140     *    Data.
 141     *
 142     *    Example:
 143     *       - Workload-1 (3D/GPGPU/MEDIA)
 144     *       - PIPE_CONTROL (CS Stall, Post-Sync-Operation Write Immediate
 145     *         Data, Required Write Cache Flush bits set)
 146     *       - Workload-2 (Can use the data produce or output by Workload-1)
 147     */
 148    iris_emit_pipe_control_write(batch, reason,
 149                                 flags | PIPE_CONTROL_CS_STALL |
 150                                 PIPE_CONTROL_WRITE_IMMEDIATE,
 151                                 batch->screen->workaround_bo, 0, 0);
 152 }
 153
 154 /**
 155  * Flush and invalidate all caches (for debugging purposes).
 156  */
 157 void
 158 iris_flush_all_caches(struct iris_batch *batch)
 159 {
 160    iris_emit_pipe_control_flush(batch, "debug: flush all caches",
 161                                 PIPE_CONTROL_CS_STALL |
 162                                 PIPE_CONTROL_DATA_CACHE_FLUSH |
 163                                 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
 164                                 PIPE_CONTROL_RENDER_TARGET_FLUSH |
 165                                 PIPE_CONTROL_VF_CACHE_INVALIDATE |
 166                                 PIPE_CONTROL_INSTRUCTION_INVALIDATE |
 167                                 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
 168                                 PIPE_CONTROL_CONST_CACHE_INVALIDATE |
 169                                 PIPE_CONTROL_STATE_CACHE_INVALIDATE);
 170 }
 171
 172 static void
 173 iris_texture_barrier(struct pipe_context *ctx, unsigned flags)
 174 {
 175    struct iris_context *ice = (void *) ctx;
 176    struct iris_batch *render_batch = &ice->batches[IRIS_BATCH_RENDER];
 177    struct iris_batch *compute_batch = &ice->batches[IRIS_BATCH_COMPUTE];
 178
 179    if (render_batch->contains_draw ||
 180        render_batch->cache.render->entries ||
 181        render_batch->cache.depth->entries) {
 182       iris_batch_maybe_flush(render_batch, 48);
 183       iris_emit_pipe_control_flush(render_batch,
 184                                    "API: texture barrier (1/2)",
 185                                    PIPE_CONTROL_DEPTH_CACHE_FLUSH |
 186                                    PIPE_CONTROL_RENDER_TARGET_FLUSH |
 187                                    PIPE_CONTROL_CS_STALL);
 188       iris_emit_pipe_control_flush(render_batch,
 189                                    "API: texture barrier (2/2)",
 190                                    PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
 191    }
 192
 193    if (compute_batch->contains_draw) {
 194       iris_batch_maybe_flush(compute_batch, 48);
 195       iris_emit_pipe_control_flush(compute_batch,
 196                                    "API: texture barrier (1/2)",
 197                                    PIPE_CONTROL_CS_STALL);
 198       iris_emit_pipe_control_flush(compute_batch,
 199                                    "API: texture barrier (2/2)",
 200                                    PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
 201    }
 202 }
 203
 204 static void
 205 iris_memory_barrier(struct pipe_context *ctx, unsigned flags)
 206 {
 207    struct iris_context *ice = (void *) ctx;
 208    unsigned bits = PIPE_CONTROL_DATA_CACHE_FLUSH | PIPE_CONTROL_CS_STALL;
 209
 210    if (flags & (PIPE_BARRIER_VERTEX_BUFFER |
 211                 PIPE_BARRIER_INDEX_BUFFER |
 212                 PIPE_BARRIER_INDIRECT_BUFFER)) {
 213       bits |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 214    }
 215
 216    if (flags & PIPE_BARRIER_CONSTANT_BUFFER) {
 217       bits |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
 218               PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 219    }
 220
 221    if (flags & (PIPE_BARRIER_TEXTURE | PIPE_BARRIER_FRAMEBUFFER)) {
 222       bits |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
 223               PIPE_CONTROL_RENDER_TARGET_FLUSH;
 224    }
 225
 226    for (int i = 0; i < IRIS_BATCH_COUNT; i++) {
 227       if (ice->batches[i].contains_draw ||
 228           ice->batches[i].cache.render->entries) {
 229          iris_batch_maybe_flush(&ice->batches[i], 24);
 230          iris_emit_pipe_control_flush(&ice->batches[i], "API: memory barrier",
 231                                       bits);
 232       }
 233    }
 234 }
 235
 236 void
 237 iris_init_flush_functions(struct pipe_context *ctx)
 238 {
 239    ctx->memory_barrier = iris_memory_barrier;
 240    ctx->texture_barrier = iris_texture_barrier;
 241 }