src/mesa/drivers/dri/i965/brw_pipe_control.c

   1 /*
   2  * Copyright © 2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "brw_context.h"
  25 #include "brw_defines.h"
  26 #include "intel_batchbuffer.h"
  27 #include "intel_fbo.h"
  28
  29 /**
  30  * According to the latest documentation, any PIPE_CONTROL with the
  31  * "Command Streamer Stall" bit set must also have another bit set,
  32  * with five different options:
  33  *
  34  *  - Render Target Cache Flush
  35  *  - Depth Cache Flush
  36  *  - Stall at Pixel Scoreboard
  37  *  - Post-Sync Operation
  38  *  - Depth Stall
  39  *  - DC Flush Enable
  40  *
  41  * I chose "Stall at Pixel Scoreboard" since we've used it effectively
  42  * in the past, but the choice is fairly arbitrary.
  43  */
  44 static void
  45 gen8_add_cs_stall_workaround_bits(uint32_t *flags)
  46 {
  47    uint32_t wa_bits = PIPE_CONTROL_RENDER_TARGET_FLUSH |
  48                       PIPE_CONTROL_DEPTH_CACHE_FLUSH |
  49                       PIPE_CONTROL_WRITE_IMMEDIATE |
  50                       PIPE_CONTROL_WRITE_DEPTH_COUNT |
  51                       PIPE_CONTROL_WRITE_TIMESTAMP |
  52                       PIPE_CONTROL_STALL_AT_SCOREBOARD |
  53                       PIPE_CONTROL_DEPTH_STALL |
  54                       PIPE_CONTROL_DATA_CACHE_FLUSH;
  55
  56    /* If we're doing a CS stall, and don't already have one of the
  57     * workaround bits set, add "Stall at Pixel Scoreboard."
  58     */
  59    if ((*flags & PIPE_CONTROL_CS_STALL) != 0 && (*flags & wa_bits) == 0)
  60       *flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD;
  61 }
  62
  63 /* Implement the WaCsStallAtEveryFourthPipecontrol workaround on IVB, BYT:
  64  *
  65  * "Every 4th PIPE_CONTROL command, not counting the PIPE_CONTROL with
  66  *  only read-cache-invalidate bit(s) set, must have a CS_STALL bit set."
  67  *
  68  * Note that the kernel does CS stalls between batches, so we only need
  69  * to count them within a batch.
  70  */
  71 static uint32_t
  72 gen7_cs_stall_every_four_pipe_controls(struct brw_context *brw, uint32_t flags)
  73 {
  74    const struct gen_device_info *devinfo = &brw->screen->devinfo;
  75
  76    if (devinfo->gen == 7 && !devinfo->is_haswell) {
  77       if (flags & PIPE_CONTROL_CS_STALL) {
  78          /* If we're doing a CS stall, reset the counter and carry on. */
  79          brw->pipe_controls_since_last_cs_stall = 0;
  80          return 0;
  81       }
  82
  83       /* If this is the fourth pipe control without a CS stall, do one now. */
  84       if (++brw->pipe_controls_since_last_cs_stall == 4) {
  85          brw->pipe_controls_since_last_cs_stall = 0;
  86          return PIPE_CONTROL_CS_STALL;
  87       }
  88    }
  89    return 0;
  90 }
  91
  92 /* #1130 from gen10 workarounds page in h/w specs:
  93  * "Enable Depth Stall on every Post Sync Op if Render target Cache Flush is
  94  *  not enabled in same PIPE CONTROL and Enable Pixel score board stall if
  95  *  Render target cache flush is enabled."
  96  *
  97  * Applicable to CNL B0 and C0 steppings only.
  98  */
  99 static void
 100 gen10_add_rcpfe_workaround_bits(uint32_t *flags)
 101 {
 102    if (*flags & PIPE_CONTROL_RENDER_TARGET_FLUSH) {
 103       *flags = *flags | PIPE_CONTROL_STALL_AT_SCOREBOARD;
 104    } else if (*flags &
 105              (PIPE_CONTROL_WRITE_IMMEDIATE |
 106               PIPE_CONTROL_WRITE_DEPTH_COUNT |
 107               PIPE_CONTROL_WRITE_TIMESTAMP)) {
 108       *flags = *flags | PIPE_CONTROL_DEPTH_STALL;
 109    }
 110 }
 111
 112 static void
 113 brw_emit_pipe_control(struct brw_context *brw, uint32_t flags,
 114                       struct brw_bo *bo, uint32_t offset, uint64_t imm)
 115 {
 116    const struct gen_device_info *devinfo = &brw->screen->devinfo;
 117
 118    if (devinfo->gen >= 8) {
 119       if (devinfo->gen == 8)
 120          gen8_add_cs_stall_workaround_bits(&flags);
 121
 122       if (devinfo->gen == 9 &&
 123           (flags & PIPE_CONTROL_VF_CACHE_INVALIDATE)) {
 124          /* Hardware workaround: SKL
 125           *
 126           * Emit Pipe Control with all bits set to zero before emitting
 127           * a Pipe Control with VF Cache Invalidate set.
 128           */
 129          brw_emit_pipe_control_flush(brw, 0);
 130       }
 131
 132       if (devinfo->gen == 10)
 133          gen10_add_rcpfe_workaround_bits(&flags);
 134
 135       BEGIN_BATCH(6);
 136       OUT_BATCH(_3DSTATE_PIPE_CONTROL | (6 - 2));
 137       OUT_BATCH(flags);
 138       if (bo) {
 139          OUT_RELOC64(bo, RELOC_WRITE, offset);
 140       } else {
 141          OUT_BATCH(0);
 142          OUT_BATCH(0);
 143       }
 144       OUT_BATCH(imm);
 145       OUT_BATCH(imm >> 32);
 146       ADVANCE_BATCH();
 147    } else if (devinfo->gen >= 6) {
 148       if (devinfo->gen == 6 &&
 149           (flags & PIPE_CONTROL_RENDER_TARGET_FLUSH)) {
 150          /* Hardware workaround: SNB B-Spec says:
 151           *
 152           *   [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush
 153           *   Enable = 1, a PIPE_CONTROL with any non-zero post-sync-op is
 154           *   required.
 155           */
 156          brw_emit_post_sync_nonzero_flush(brw);
 157       }
 158
 159       flags |= gen7_cs_stall_every_four_pipe_controls(brw, flags);
 160
 161       /* PPGTT/GGTT is selected by DW2 bit 2 on Sandybridge, but DW1 bit 24
 162        * on later platforms.  We always use PPGTT on Gen7+.
 163        */
 164       unsigned gen6_gtt = devinfo->gen == 6 ? PIPE_CONTROL_GLOBAL_GTT_WRITE : 0;
 165
 166       BEGIN_BATCH(5);
 167       OUT_BATCH(_3DSTATE_PIPE_CONTROL | (5 - 2));
 168       OUT_BATCH(flags);
 169       if (bo) {
 170          OUT_RELOC(bo, RELOC_WRITE | RELOC_NEEDS_GGTT, gen6_gtt | offset);
 171       } else {
 172          OUT_BATCH(0);
 173       }
 174       OUT_BATCH(imm);
 175       OUT_BATCH(imm >> 32);
 176       ADVANCE_BATCH();
 177    } else {
 178       BEGIN_BATCH(4);
 179       OUT_BATCH(_3DSTATE_PIPE_CONTROL | flags | (4 - 2));
 180       if (bo) {
 181          OUT_RELOC(bo, RELOC_WRITE, PIPE_CONTROL_GLOBAL_GTT_WRITE | offset);
 182       } else {
 183          OUT_BATCH(0);
 184       }
 185       OUT_BATCH(imm);
 186       OUT_BATCH(imm >> 32);
 187       ADVANCE_BATCH();
 188    }
 189 }
 190
 191 /**
 192  * Emit a PIPE_CONTROL with various flushing flags.
 193  *
 194  * The caller is responsible for deciding what flags are appropriate for the
 195  * given generation.
 196  */
 197 void
 198 brw_emit_pipe_control_flush(struct brw_context *brw, uint32_t flags)
 199 {
 200    const struct gen_device_info *devinfo = &brw->screen->devinfo;
 201
 202    if (devinfo->gen >= 6 &&
 203        (flags & PIPE_CONTROL_CACHE_FLUSH_BITS) &&
 204        (flags & PIPE_CONTROL_CACHE_INVALIDATE_BITS)) {
 205       /* A pipe control command with flush and invalidate bits set
 206        * simultaneously is an inherently racy operation on Gen6+ if the
 207        * contents of the flushed caches were intended to become visible from
 208        * any of the invalidated caches.  Split it in two PIPE_CONTROLs, the
 209        * first one should stall the pipeline to make sure that the flushed R/W
 210        * caches are coherent with memory once the specified R/O caches are
 211        * invalidated.  On pre-Gen6 hardware the (implicit) R/O cache
 212        * invalidation seems to happen at the bottom of the pipeline together
 213        * with any write cache flush, so this shouldn't be a concern.  In order
 214        * to ensure a full stall, we do an end-of-pipe sync.
 215        */
 216       brw_emit_end_of_pipe_sync(brw, (flags & PIPE_CONTROL_CACHE_FLUSH_BITS));
 217       flags &= ~(PIPE_CONTROL_CACHE_FLUSH_BITS | PIPE_CONTROL_CS_STALL);
 218    }
 219
 220    brw_emit_pipe_control(brw, flags, NULL, 0, 0);
 221 }
 222
 223 /**
 224  * Emit a PIPE_CONTROL that writes to a buffer object.
 225  *
 226  * \p flags should contain one of the following items:
 227  *  - PIPE_CONTROL_WRITE_IMMEDIATE
 228  *  - PIPE_CONTROL_WRITE_TIMESTAMP
 229  *  - PIPE_CONTROL_WRITE_DEPTH_COUNT
 230  */
 231 void
 232 brw_emit_pipe_control_write(struct brw_context *brw, uint32_t flags,
 233                             struct brw_bo *bo, uint32_t offset,
 234                             uint64_t imm)
 235 {
 236    brw_emit_pipe_control(brw, flags, bo, offset, imm);
 237 }
 238
 239 /**
 240  * Restriction [DevSNB, DevIVB]:
 241  *
 242  * Prior to changing Depth/Stencil Buffer state (i.e. any combination of
 243  * 3DSTATE_DEPTH_BUFFER, 3DSTATE_CLEAR_PARAMS, 3DSTATE_STENCIL_BUFFER,
 244  * 3DSTATE_HIER_DEPTH_BUFFER) SW must first issue a pipelined depth stall
 245  * (PIPE_CONTROL with Depth Stall bit set), followed by a pipelined depth
 246  * cache flush (PIPE_CONTROL with Depth Flush Bit set), followed by
 247  * another pipelined depth stall (PIPE_CONTROL with Depth Stall bit set),
 248  * unless SW can otherwise guarantee that the pipeline from WM onwards is
 249  * already flushed (e.g., via a preceding MI_FLUSH).
 250  */
 251 void
 252 brw_emit_depth_stall_flushes(struct brw_context *brw)
 253 {
 254    const struct gen_device_info *devinfo = &brw->screen->devinfo;
 255
 256    assert(devinfo->gen >= 6);
 257
 258    /* Starting on BDW, these pipe controls are unnecessary.
 259     *
 260     *   WM HW will internally manage the draining pipe and flushing of the caches
 261     *   when this command is issued. The PIPE_CONTROL restrictions are removed.
 262     */
 263    if (devinfo->gen >= 8)
 264       return;
 265
 266    brw_emit_pipe_control_flush(brw, PIPE_CONTROL_DEPTH_STALL);
 267    brw_emit_pipe_control_flush(brw, PIPE_CONTROL_DEPTH_CACHE_FLUSH);
 268    brw_emit_pipe_control_flush(brw, PIPE_CONTROL_DEPTH_STALL);
 269 }
 270
 271 /**
 272  * From the Ivybridge PRM, Volume 2 Part 1, Section 3.2 (VS Stage Input):
 273  * "A PIPE_CONTROL with Post-Sync Operation set to 1h and a depth
 274  *  stall needs to be sent just prior to any 3DSTATE_VS, 3DSTATE_URB_VS,
 275  *  3DSTATE_CONSTANT_VS, 3DSTATE_BINDING_TABLE_POINTER_VS,
 276  *  3DSTATE_SAMPLER_STATE_POINTER_VS command.  Only one PIPE_CONTROL needs
 277  *  to be sent before any combination of VS associated 3DSTATE."
 278  */
 279 void
 280 gen7_emit_vs_workaround_flush(struct brw_context *brw)
 281 {
 282    const struct gen_device_info *devinfo = &brw->screen->devinfo;
 283
 284    assert(devinfo->gen == 7);
 285    brw_emit_pipe_control_write(brw,
 286                                PIPE_CONTROL_WRITE_IMMEDIATE
 287                                | PIPE_CONTROL_DEPTH_STALL,
 288                                brw->workaround_bo, 0, 0);
 289 }
 290
 291
 292 /**
 293  * Emit a PIPE_CONTROL command for gen7 with the CS Stall bit set.
 294  */
 295 void
 296 gen7_emit_cs_stall_flush(struct brw_context *brw)
 297 {
 298    brw_emit_pipe_control_write(brw,
 299                                PIPE_CONTROL_CS_STALL
 300                                | PIPE_CONTROL_WRITE_IMMEDIATE,
 301                                brw->workaround_bo, 0, 0);
 302 }
 303
 304 /**
 305  * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
 306  * implementing two workarounds on gen6.  From section 1.4.7.1
 307  * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
 308  *
 309  * [DevSNB-C+{W/A}] Before any depth stall flush (including those
 310  * produced by non-pipelined state commands), software needs to first
 311  * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
 312  * 0.
 313  *
 314  * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
 315  * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
 316  *
 317  * And the workaround for these two requires this workaround first:
 318  *
 319  * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
 320  * BEFORE the pipe-control with a post-sync op and no write-cache
 321  * flushes.
 322  *
 323  * And this last workaround is tricky because of the requirements on
 324  * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
 325  * volume 2 part 1:
 326  *
 327  *     "1 of the following must also be set:
 328  *      - Render Target Cache Flush Enable ([12] of DW1)
 329  *      - Depth Cache Flush Enable ([0] of DW1)
 330  *      - Stall at Pixel Scoreboard ([1] of DW1)
 331  *      - Depth Stall ([13] of DW1)
 332  *      - Post-Sync Operation ([13] of DW1)
 333  *      - Notify Enable ([8] of DW1)"
 334  *
 335  * The cache flushes require the workaround flush that triggered this
 336  * one, so we can't use it.  Depth stall would trigger the same.
 337  * Post-sync nonzero is what triggered this second workaround, so we
 338  * can't use that one either.  Notify enable is IRQs, which aren't
 339  * really our business.  That leaves only stall at scoreboard.
 340  */
 341 void
 342 brw_emit_post_sync_nonzero_flush(struct brw_context *brw)
 343 {
 344    brw_emit_pipe_control_flush(brw,
 345                                PIPE_CONTROL_CS_STALL |
 346                                PIPE_CONTROL_STALL_AT_SCOREBOARD);
 347
 348    brw_emit_pipe_control_write(brw, PIPE_CONTROL_WRITE_IMMEDIATE,
 349                                brw->workaround_bo, 0, 0);
 350 }
 351
 352 /*
 353  * From Sandybridge PRM, volume 2, "1.7.2 End-of-Pipe Synchronization":
 354  *
 355  *  Write synchronization is a special case of end-of-pipe
 356  *  synchronization that requires that the render cache and/or depth
 357  *  related caches are flushed to memory, where the data will become
 358  *  globally visible. This type of synchronization is required prior to
 359  *  SW (CPU) actually reading the result data from memory, or initiating
 360  *  an operation that will use as a read surface (such as a texture
 361  *  surface) a previous render target and/or depth/stencil buffer
 362  *
 363  *
 364  * From Haswell PRM, volume 2, part 1, "End-of-Pipe Synchronization":
 365  *
 366  *  Exercising the write cache flush bits (Render Target Cache Flush
 367  *  Enable, Depth Cache Flush Enable, DC Flush) in PIPE_CONTROL only
 368  *  ensures the write caches are flushed and doesn't guarantee the data
 369  *  is globally visible.
 370  *
 371  *  SW can track the completion of the end-of-pipe-synchronization by
 372  *  using "Notify Enable" and "PostSync Operation - Write Immediate
 373  *  Data" in the PIPE_CONTROL command.
 374  */
 375 void
 376 brw_emit_end_of_pipe_sync(struct brw_context *brw, uint32_t flags)
 377 {
 378    const struct gen_device_info *devinfo = &brw->screen->devinfo;
 379
 380    if (devinfo->gen >= 6) {
 381       /* From Sandybridge PRM, volume 2, "1.7.3.1 Writing a Value to Memory":
 382        *
 383        *    "The most common action to perform upon reaching a synchronization
 384        *    point is to write a value out to memory. An immediate value
 385        *    (included with the synchronization command) may be written."
 386        *
 387        *
 388        * From Broadwell PRM, volume 7, "End-of-Pipe Synchronization":
 389        *
 390        *    "In case the data flushed out by the render engine is to be read
 391        *    back in to the render engine in coherent manner, then the render
 392        *    engine has to wait for the fence completion before accessing the
 393        *    flushed data. This can be achieved by following means on various
 394        *    products: PIPE_CONTROL command with CS Stall and the required
 395        *    write caches flushed with Post-Sync-Operation as Write Immediate
 396        *    Data.
 397        *
 398        *    Example:
 399        *       - Workload-1 (3D/GPGPU/MEDIA)
 400        *       - PIPE_CONTROL (CS Stall, Post-Sync-Operation Write Immediate
 401        *         Data, Required Write Cache Flush bits set)
 402        *       - Workload-2 (Can use the data produce or output by Workload-1)
 403        */
 404       brw_emit_pipe_control_write(brw,
 405                                   flags | PIPE_CONTROL_CS_STALL |
 406                                   PIPE_CONTROL_WRITE_IMMEDIATE,
 407                                   brw->workaround_bo, 0, 0);
 408
 409       if (devinfo->is_haswell) {
 410          /* Haswell needs addition work-arounds:
 411           *
 412           * From Haswell PRM, volume 2, part 1, "End-of-Pipe Synchronization":
 413           *
 414           *    Option 1:
 415           *    PIPE_CONTROL command with the CS Stall and the required write
 416           *    caches flushed with Post-SyncOperation as Write Immediate Data
 417           *    followed by eight dummy MI_STORE_DATA_IMM (write to scratch
 418           *    spce) commands.
 419           *
 420           *    Example:
 421           *       - Workload-1
 422           *       - PIPE_CONTROL (CS Stall, Post-Sync-Operation Write
 423           *         Immediate Data, Required Write Cache Flush bits set)
 424           *       - MI_STORE_DATA_IMM (8 times) (Dummy data, Scratch Address)
 425           *       - Workload-2 (Can use the data produce or output by
 426           *         Workload-1)
 427           *
 428           * Unfortunately, both the PRMs and the internal docs are a bit
 429           * out-of-date in this regard.  What the windows driver does (and
 430           * this appears to actually work) is to emit a register read from the
 431           * memory address written by the pipe control above.
 432           *
 433           * What register we load into doesn't matter.  We choose an indirect
 434           * rendering register because we know it always exists and it's one
 435           * of the first registers the command parser allows us to write.  If
 436           * you don't have command parser support in your kernel (pre-4.2),
 437           * this will get turned into MI_NOOP and you won't get the
 438           * workaround.  Unfortunately, there's just not much we can do in
 439           * that case.  This register is perfectly safe to write since we
 440           * always re-load all of the indirect draw registers right before
 441           * 3DPRIMITIVE when needed anyway.
 442           */
 443          brw_load_register_mem(brw, GEN7_3DPRIM_START_INSTANCE,
 444                                brw->workaround_bo, 0);
 445       }
 446    } else {
 447       /* On gen4-5, a regular pipe control seems to suffice. */
 448       brw_emit_pipe_control_flush(brw, flags);
 449    }
 450 }
 451
 452 /* Emit a pipelined flush to either flush render and texture cache for
 453  * reading from a FBO-drawn texture, or flush so that frontbuffer
 454  * render appears on the screen in DRI1.
 455  *
 456  * This is also used for the always_flush_cache driconf debug option.
 457  */
 458 void
 459 brw_emit_mi_flush(struct brw_context *brw)
 460 {
 461    const struct gen_device_info *devinfo = &brw->screen->devinfo;
 462
 463    if (brw->batch.ring == BLT_RING && devinfo->gen >= 6) {
 464       const unsigned n_dwords = devinfo->gen >= 8 ? 5 : 4;
 465       BEGIN_BATCH_BLT(n_dwords);
 466       OUT_BATCH(MI_FLUSH_DW | (n_dwords - 2));
 467       OUT_BATCH(0);
 468       OUT_BATCH(0);
 469       OUT_BATCH(0);
 470       if (n_dwords == 5)
 471          OUT_BATCH(0);
 472       ADVANCE_BATCH();
 473    } else {
 474       int flags = PIPE_CONTROL_NO_WRITE | PIPE_CONTROL_RENDER_TARGET_FLUSH;
 475       if (devinfo->gen >= 6) {
 476          flags |= PIPE_CONTROL_INSTRUCTION_INVALIDATE |
 477                   PIPE_CONTROL_CONST_CACHE_INVALIDATE |
 478                   PIPE_CONTROL_DATA_CACHE_FLUSH |
 479                   PIPE_CONTROL_DEPTH_CACHE_FLUSH |
 480                   PIPE_CONTROL_VF_CACHE_INVALIDATE |
 481                   PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
 482                   PIPE_CONTROL_CS_STALL;
 483       }
 484       brw_emit_pipe_control_flush(brw, flags);
 485    }
 486 }
 487
 488 int
 489 brw_init_pipe_control(struct brw_context *brw,
 490                       const struct gen_device_info *devinfo)
 491 {
 492    if (devinfo->gen < 6)
 493       return 0;
 494
 495    /* We can't just use brw_state_batch to get a chunk of space for
 496     * the gen6 workaround because it involves actually writing to
 497     * the buffer, and the kernel doesn't let us write to the batch.
 498     */
 499    brw->workaround_bo = brw_bo_alloc(brw->bufmgr,
 500                                      "pipe_control workaround",
 501                                      4096, 4096);
 502    if (brw->workaround_bo == NULL)
 503       return -ENOMEM;
 504
 505    brw->pipe_controls_since_last_cs_stall = 0;
 506
 507    return 0;
 508 }
 509
 510 void
 511 brw_fini_pipe_control(struct brw_context *brw)
 512 {
 513    brw_bo_unreference(brw->workaround_bo);
 514 }