2 * Copyright © 2010 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 #include "brw_context.h"
25 #include "brw_defines.h"
26 #include "intel_batchbuffer.h"
27 #include "intel_fbo.h"
30 * According to the latest documentation, any PIPE_CONTROL with the
31 * "Command Streamer Stall" bit set must also have another bit set,
32 * with five different options:
34 * - Render Target Cache Flush
36 * - Stall at Pixel Scoreboard
37 * - Post-Sync Operation
41 * I chose "Stall at Pixel Scoreboard" since we've used it effectively
42 * in the past, but the choice is fairly arbitrary.
45 gen8_add_cs_stall_workaround_bits(uint32_t *flags
)
47 uint32_t wa_bits
= PIPE_CONTROL_RENDER_TARGET_FLUSH
|
48 PIPE_CONTROL_DEPTH_CACHE_FLUSH
|
49 PIPE_CONTROL_WRITE_IMMEDIATE
|
50 PIPE_CONTROL_WRITE_DEPTH_COUNT
|
51 PIPE_CONTROL_WRITE_TIMESTAMP
|
52 PIPE_CONTROL_STALL_AT_SCOREBOARD
|
53 PIPE_CONTROL_DEPTH_STALL
|
54 PIPE_CONTROL_DATA_CACHE_FLUSH
;
56 /* If we're doing a CS stall, and don't already have one of the
57 * workaround bits set, add "Stall at Pixel Scoreboard."
59 if ((*flags
& PIPE_CONTROL_CS_STALL
) != 0 && (*flags
& wa_bits
) == 0)
60 *flags
|= PIPE_CONTROL_STALL_AT_SCOREBOARD
;
63 /* Implement the WaCsStallAtEveryFourthPipecontrol workaround on IVB, BYT:
65 * "Every 4th PIPE_CONTROL command, not counting the PIPE_CONTROL with
66 * only read-cache-invalidate bit(s) set, must have a CS_STALL bit set."
68 * Note that the kernel does CS stalls between batches, so we only need
69 * to count them within a batch.
72 gen7_cs_stall_every_four_pipe_controls(struct brw_context
*brw
, uint32_t flags
)
74 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
76 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
) {
77 if (flags
& PIPE_CONTROL_CS_STALL
) {
78 /* If we're doing a CS stall, reset the counter and carry on. */
79 brw
->pipe_controls_since_last_cs_stall
= 0;
83 /* If this is the fourth pipe control without a CS stall, do one now. */
84 if (++brw
->pipe_controls_since_last_cs_stall
== 4) {
85 brw
->pipe_controls_since_last_cs_stall
= 0;
86 return PIPE_CONTROL_CS_STALL
;
92 /* #1130 from gen10 workarounds page in h/w specs:
93 * "Enable Depth Stall on every Post Sync Op if Render target Cache Flush is
94 * not enabled in same PIPE CONTROL and Enable Pixel score board stall if
95 * Render target cache flush is enabled."
97 * Applicable to CNL B0 and C0 steppings only.
100 gen10_add_rcpfe_workaround_bits(uint32_t *flags
)
102 if (*flags
& PIPE_CONTROL_RENDER_TARGET_FLUSH
) {
103 *flags
= *flags
| PIPE_CONTROL_STALL_AT_SCOREBOARD
;
105 (PIPE_CONTROL_WRITE_IMMEDIATE
|
106 PIPE_CONTROL_WRITE_DEPTH_COUNT
|
107 PIPE_CONTROL_WRITE_TIMESTAMP
)) {
108 *flags
= *flags
| PIPE_CONTROL_DEPTH_STALL
;
113 brw_emit_pipe_control(struct brw_context
*brw
, uint32_t flags
,
114 struct brw_bo
*bo
, uint32_t offset
, uint64_t imm
)
116 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
118 if (devinfo
->gen
>= 8) {
119 if (devinfo
->gen
== 8)
120 gen8_add_cs_stall_workaround_bits(&flags
);
122 if (flags
& PIPE_CONTROL_VF_CACHE_INVALIDATE
) {
123 if (devinfo
->gen
== 9) {
124 /* The PIPE_CONTROL "VF Cache Invalidation Enable" bit description
125 * lists several workarounds:
127 * "Project: SKL, KBL, BXT
129 * If the VF Cache Invalidation Enable is set to a 1 in a
130 * PIPE_CONTROL, a separate Null PIPE_CONTROL, all bitfields
131 * sets to 0, with the VF Cache Invalidation Enable set to 0
132 * needs to be sent prior to the PIPE_CONTROL with VF Cache
133 * Invalidation Enable set to a 1."
135 brw_emit_pipe_control_flush(brw
, 0);
138 if (devinfo
->gen
>= 9) {
139 /* THE PIPE_CONTROL "VF Cache Invalidation Enable" docs continue:
143 * When VF Cache Invalidate is set “Post Sync Operation” must
144 * be enabled to “Write Immediate Data” or “Write PS Depth
145 * Count” or “Write Timestamp”."
147 * If there's a BO, we're already doing some kind of write.
148 * If not, add a write to the workaround BO.
150 * XXX: This causes GPU hangs on Broadwell, so restrict it to
151 * Gen9+ for now...see this bug for more information:
152 * https://bugs.freedesktop.org/show_bug.cgi?id=103787
155 flags
|= PIPE_CONTROL_WRITE_IMMEDIATE
;
156 bo
= brw
->workaround_bo
;
161 if (devinfo
->gen
== 10)
162 gen10_add_rcpfe_workaround_bits(&flags
);
165 OUT_BATCH(_3DSTATE_PIPE_CONTROL
| (6 - 2));
168 OUT_RELOC64(bo
, RELOC_WRITE
, offset
);
174 OUT_BATCH(imm
>> 32);
176 } else if (devinfo
->gen
>= 6) {
177 if (devinfo
->gen
== 6 &&
178 (flags
& PIPE_CONTROL_RENDER_TARGET_FLUSH
)) {
179 /* Hardware workaround: SNB B-Spec says:
181 * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush
182 * Enable = 1, a PIPE_CONTROL with any non-zero post-sync-op is
185 brw_emit_post_sync_nonzero_flush(brw
);
188 flags
|= gen7_cs_stall_every_four_pipe_controls(brw
, flags
);
190 /* PPGTT/GGTT is selected by DW2 bit 2 on Sandybridge, but DW1 bit 24
191 * on later platforms. We always use PPGTT on Gen7+.
193 unsigned gen6_gtt
= devinfo
->gen
== 6 ? PIPE_CONTROL_GLOBAL_GTT_WRITE
: 0;
196 OUT_BATCH(_3DSTATE_PIPE_CONTROL
| (5 - 2));
199 OUT_RELOC(bo
, RELOC_WRITE
| RELOC_NEEDS_GGTT
, gen6_gtt
| offset
);
204 OUT_BATCH(imm
>> 32);
208 OUT_BATCH(_3DSTATE_PIPE_CONTROL
| flags
| (4 - 2));
210 OUT_RELOC(bo
, RELOC_WRITE
, PIPE_CONTROL_GLOBAL_GTT_WRITE
| offset
);
215 OUT_BATCH(imm
>> 32);
221 * Emit a PIPE_CONTROL with various flushing flags.
223 * The caller is responsible for deciding what flags are appropriate for the
227 brw_emit_pipe_control_flush(struct brw_context
*brw
, uint32_t flags
)
229 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
231 if (devinfo
->gen
>= 6 &&
232 (flags
& PIPE_CONTROL_CACHE_FLUSH_BITS
) &&
233 (flags
& PIPE_CONTROL_CACHE_INVALIDATE_BITS
)) {
234 /* A pipe control command with flush and invalidate bits set
235 * simultaneously is an inherently racy operation on Gen6+ if the
236 * contents of the flushed caches were intended to become visible from
237 * any of the invalidated caches. Split it in two PIPE_CONTROLs, the
238 * first one should stall the pipeline to make sure that the flushed R/W
239 * caches are coherent with memory once the specified R/O caches are
240 * invalidated. On pre-Gen6 hardware the (implicit) R/O cache
241 * invalidation seems to happen at the bottom of the pipeline together
242 * with any write cache flush, so this shouldn't be a concern. In order
243 * to ensure a full stall, we do an end-of-pipe sync.
245 brw_emit_end_of_pipe_sync(brw
, (flags
& PIPE_CONTROL_CACHE_FLUSH_BITS
));
246 flags
&= ~(PIPE_CONTROL_CACHE_FLUSH_BITS
| PIPE_CONTROL_CS_STALL
);
249 brw_emit_pipe_control(brw
, flags
, NULL
, 0, 0);
253 * Emit a PIPE_CONTROL that writes to a buffer object.
255 * \p flags should contain one of the following items:
256 * - PIPE_CONTROL_WRITE_IMMEDIATE
257 * - PIPE_CONTROL_WRITE_TIMESTAMP
258 * - PIPE_CONTROL_WRITE_DEPTH_COUNT
261 brw_emit_pipe_control_write(struct brw_context
*brw
, uint32_t flags
,
262 struct brw_bo
*bo
, uint32_t offset
,
265 brw_emit_pipe_control(brw
, flags
, bo
, offset
, imm
);
269 * Restriction [DevSNB, DevIVB]:
271 * Prior to changing Depth/Stencil Buffer state (i.e. any combination of
272 * 3DSTATE_DEPTH_BUFFER, 3DSTATE_CLEAR_PARAMS, 3DSTATE_STENCIL_BUFFER,
273 * 3DSTATE_HIER_DEPTH_BUFFER) SW must first issue a pipelined depth stall
274 * (PIPE_CONTROL with Depth Stall bit set), followed by a pipelined depth
275 * cache flush (PIPE_CONTROL with Depth Flush Bit set), followed by
276 * another pipelined depth stall (PIPE_CONTROL with Depth Stall bit set),
277 * unless SW can otherwise guarantee that the pipeline from WM onwards is
278 * already flushed (e.g., via a preceding MI_FLUSH).
281 brw_emit_depth_stall_flushes(struct brw_context
*brw
)
283 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
285 assert(devinfo
->gen
>= 6);
287 /* Starting on BDW, these pipe controls are unnecessary.
289 * WM HW will internally manage the draining pipe and flushing of the caches
290 * when this command is issued. The PIPE_CONTROL restrictions are removed.
292 if (devinfo
->gen
>= 8)
295 brw_emit_pipe_control_flush(brw
, PIPE_CONTROL_DEPTH_STALL
);
296 brw_emit_pipe_control_flush(brw
, PIPE_CONTROL_DEPTH_CACHE_FLUSH
);
297 brw_emit_pipe_control_flush(brw
, PIPE_CONTROL_DEPTH_STALL
);
301 * From the Ivybridge PRM, Volume 2 Part 1, Section 3.2 (VS Stage Input):
302 * "A PIPE_CONTROL with Post-Sync Operation set to 1h and a depth
303 * stall needs to be sent just prior to any 3DSTATE_VS, 3DSTATE_URB_VS,
304 * 3DSTATE_CONSTANT_VS, 3DSTATE_BINDING_TABLE_POINTER_VS,
305 * 3DSTATE_SAMPLER_STATE_POINTER_VS command. Only one PIPE_CONTROL needs
306 * to be sent before any combination of VS associated 3DSTATE."
309 gen7_emit_vs_workaround_flush(struct brw_context
*brw
)
311 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
313 assert(devinfo
->gen
== 7);
314 brw_emit_pipe_control_write(brw
,
315 PIPE_CONTROL_WRITE_IMMEDIATE
316 | PIPE_CONTROL_DEPTH_STALL
,
317 brw
->workaround_bo
, 0, 0);
322 * Emit a PIPE_CONTROL command for gen7 with the CS Stall bit set.
325 gen7_emit_cs_stall_flush(struct brw_context
*brw
)
327 brw_emit_pipe_control_write(brw
,
328 PIPE_CONTROL_CS_STALL
329 | PIPE_CONTROL_WRITE_IMMEDIATE
,
330 brw
->workaround_bo
, 0, 0);
334 * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
335 * implementing two workarounds on gen6. From section 1.4.7.1
336 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
338 * [DevSNB-C+{W/A}] Before any depth stall flush (including those
339 * produced by non-pipelined state commands), software needs to first
340 * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
343 * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
344 * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
346 * And the workaround for these two requires this workaround first:
348 * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
349 * BEFORE the pipe-control with a post-sync op and no write-cache
352 * And this last workaround is tricky because of the requirements on
353 * that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
356 * "1 of the following must also be set:
357 * - Render Target Cache Flush Enable ([12] of DW1)
358 * - Depth Cache Flush Enable ([0] of DW1)
359 * - Stall at Pixel Scoreboard ([1] of DW1)
360 * - Depth Stall ([13] of DW1)
361 * - Post-Sync Operation ([13] of DW1)
362 * - Notify Enable ([8] of DW1)"
364 * The cache flushes require the workaround flush that triggered this
365 * one, so we can't use it. Depth stall would trigger the same.
366 * Post-sync nonzero is what triggered this second workaround, so we
367 * can't use that one either. Notify enable is IRQs, which aren't
368 * really our business. That leaves only stall at scoreboard.
371 brw_emit_post_sync_nonzero_flush(struct brw_context
*brw
)
373 brw_emit_pipe_control_flush(brw
,
374 PIPE_CONTROL_CS_STALL
|
375 PIPE_CONTROL_STALL_AT_SCOREBOARD
);
377 brw_emit_pipe_control_write(brw
, PIPE_CONTROL_WRITE_IMMEDIATE
,
378 brw
->workaround_bo
, 0, 0);
382 * From Sandybridge PRM, volume 2, "1.7.2 End-of-Pipe Synchronization":
384 * Write synchronization is a special case of end-of-pipe
385 * synchronization that requires that the render cache and/or depth
386 * related caches are flushed to memory, where the data will become
387 * globally visible. This type of synchronization is required prior to
388 * SW (CPU) actually reading the result data from memory, or initiating
389 * an operation that will use as a read surface (such as a texture
390 * surface) a previous render target and/or depth/stencil buffer
393 * From Haswell PRM, volume 2, part 1, "End-of-Pipe Synchronization":
395 * Exercising the write cache flush bits (Render Target Cache Flush
396 * Enable, Depth Cache Flush Enable, DC Flush) in PIPE_CONTROL only
397 * ensures the write caches are flushed and doesn't guarantee the data
398 * is globally visible.
400 * SW can track the completion of the end-of-pipe-synchronization by
401 * using "Notify Enable" and "PostSync Operation - Write Immediate
402 * Data" in the PIPE_CONTROL command.
405 brw_emit_end_of_pipe_sync(struct brw_context
*brw
, uint32_t flags
)
407 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
409 if (devinfo
->gen
>= 6) {
410 /* From Sandybridge PRM, volume 2, "1.7.3.1 Writing a Value to Memory":
412 * "The most common action to perform upon reaching a synchronization
413 * point is to write a value out to memory. An immediate value
414 * (included with the synchronization command) may be written."
417 * From Broadwell PRM, volume 7, "End-of-Pipe Synchronization":
419 * "In case the data flushed out by the render engine is to be read
420 * back in to the render engine in coherent manner, then the render
421 * engine has to wait for the fence completion before accessing the
422 * flushed data. This can be achieved by following means on various
423 * products: PIPE_CONTROL command with CS Stall and the required
424 * write caches flushed with Post-Sync-Operation as Write Immediate
428 * - Workload-1 (3D/GPGPU/MEDIA)
429 * - PIPE_CONTROL (CS Stall, Post-Sync-Operation Write Immediate
430 * Data, Required Write Cache Flush bits set)
431 * - Workload-2 (Can use the data produce or output by Workload-1)
433 brw_emit_pipe_control_write(brw
,
434 flags
| PIPE_CONTROL_CS_STALL
|
435 PIPE_CONTROL_WRITE_IMMEDIATE
,
436 brw
->workaround_bo
, 0, 0);
438 if (devinfo
->is_haswell
) {
439 /* Haswell needs addition work-arounds:
441 * From Haswell PRM, volume 2, part 1, "End-of-Pipe Synchronization":
444 * PIPE_CONTROL command with the CS Stall and the required write
445 * caches flushed with Post-SyncOperation as Write Immediate Data
446 * followed by eight dummy MI_STORE_DATA_IMM (write to scratch
451 * - PIPE_CONTROL (CS Stall, Post-Sync-Operation Write
452 * Immediate Data, Required Write Cache Flush bits set)
453 * - MI_STORE_DATA_IMM (8 times) (Dummy data, Scratch Address)
454 * - Workload-2 (Can use the data produce or output by
457 * Unfortunately, both the PRMs and the internal docs are a bit
458 * out-of-date in this regard. What the windows driver does (and
459 * this appears to actually work) is to emit a register read from the
460 * memory address written by the pipe control above.
462 * What register we load into doesn't matter. We choose an indirect
463 * rendering register because we know it always exists and it's one
464 * of the first registers the command parser allows us to write. If
465 * you don't have command parser support in your kernel (pre-4.2),
466 * this will get turned into MI_NOOP and you won't get the
467 * workaround. Unfortunately, there's just not much we can do in
468 * that case. This register is perfectly safe to write since we
469 * always re-load all of the indirect draw registers right before
470 * 3DPRIMITIVE when needed anyway.
472 brw_load_register_mem(brw
, GEN7_3DPRIM_START_INSTANCE
,
473 brw
->workaround_bo
, 0);
476 /* On gen4-5, a regular pipe control seems to suffice. */
477 brw_emit_pipe_control_flush(brw
, flags
);
481 /* Emit a pipelined flush to either flush render and texture cache for
482 * reading from a FBO-drawn texture, or flush so that frontbuffer
483 * render appears on the screen in DRI1.
485 * This is also used for the always_flush_cache driconf debug option.
488 brw_emit_mi_flush(struct brw_context
*brw
)
490 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
492 if (brw
->batch
.ring
== BLT_RING
&& devinfo
->gen
>= 6) {
493 const unsigned n_dwords
= devinfo
->gen
>= 8 ? 5 : 4;
494 BEGIN_BATCH_BLT(n_dwords
);
495 OUT_BATCH(MI_FLUSH_DW
| (n_dwords
- 2));
503 int flags
= PIPE_CONTROL_NO_WRITE
| PIPE_CONTROL_RENDER_TARGET_FLUSH
;
504 if (devinfo
->gen
>= 6) {
505 flags
|= PIPE_CONTROL_INSTRUCTION_INVALIDATE
|
506 PIPE_CONTROL_CONST_CACHE_INVALIDATE
|
507 PIPE_CONTROL_DATA_CACHE_FLUSH
|
508 PIPE_CONTROL_DEPTH_CACHE_FLUSH
|
509 PIPE_CONTROL_VF_CACHE_INVALIDATE
|
510 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE
|
511 PIPE_CONTROL_CS_STALL
;
513 brw_emit_pipe_control_flush(brw
, flags
);
518 brw_init_pipe_control(struct brw_context
*brw
,
519 const struct gen_device_info
*devinfo
)
521 if (devinfo
->gen
< 6)
524 /* We can't just use brw_state_batch to get a chunk of space for
525 * the gen6 workaround because it involves actually writing to
526 * the buffer, and the kernel doesn't let us write to the batch.
528 brw
->workaround_bo
= brw_bo_alloc(brw
->bufmgr
,
529 "pipe_control workaround",
531 if (brw
->workaround_bo
== NULL
)
534 brw
->pipe_controls_since_last_cs_stall
= 0;
540 brw_fini_pipe_control(struct brw_context
*brw
)
542 brw_bo_unreference(brw
->workaround_bo
);