* - Stall at Pixel Scoreboard
* - Post-Sync Operation
* - Depth Stall
+ * - DC Flush Enable
*
* I chose "Stall at Pixel Scoreboard" since we've used it effectively
* in the past, but the choice is fairly arbitrary.
PIPE_CONTROL_WRITE_DEPTH_COUNT |
PIPE_CONTROL_WRITE_TIMESTAMP |
PIPE_CONTROL_STALL_AT_SCOREBOARD |
- PIPE_CONTROL_DEPTH_STALL;
+ PIPE_CONTROL_DEPTH_STALL |
+ PIPE_CONTROL_DATA_CACHE_FLUSH;
/* If we're doing a CS stall, and don't already have one of the
* workaround bits set, add "Stall at Pixel Scoreboard."
if (brw->gen == 7 && !brw->is_haswell) {
if (flags & PIPE_CONTROL_CS_STALL) {
/* If we're doing a CS stall, reset the counter and carry on. */
- brw->batch.pipe_controls_since_last_cs_stall = 0;
+ brw->pipe_controls_since_last_cs_stall = 0;
return 0;
}
/* If this is the fourth pipe control without a CS stall, do one now. */
- if (++brw->batch.pipe_controls_since_last_cs_stall == 4) {
- brw->batch.pipe_controls_since_last_cs_stall = 0;
+ if (++brw->pipe_controls_since_last_cs_stall == 4) {
+ brw->pipe_controls_since_last_cs_stall = 0;
return PIPE_CONTROL_CS_STALL;
}
}
void
brw_emit_pipe_control_flush(struct brw_context *brw, uint32_t flags)
{
+ if (brw->gen >= 6 &&
+ (flags & PIPE_CONTROL_CACHE_FLUSH_BITS) &&
+ (flags & PIPE_CONTROL_CACHE_INVALIDATE_BITS)) {
+ /* A pipe control command with flush and invalidate bits set
+ * simultaneously is an inherently racy operation on Gen6+ if the
+ * contents of the flushed caches were intended to become visible from
+ * any of the invalidated caches. Split it in two PIPE_CONTROLs, the
+ * first one should stall the pipeline to make sure that the flushed R/W
+ * caches are coherent with memory once the specified R/O caches are
+ * invalidated. On pre-Gen6 hardware the (implicit) R/O cache
+ * invalidation seems to happen at the bottom of the pipeline together
+ * with any write cache flush, so this shouldn't be a concern.
+ */
+ brw_emit_pipe_control_flush(brw, (flags & PIPE_CONTROL_CACHE_FLUSH_BITS) |
+ PIPE_CONTROL_CS_STALL);
+ flags &= ~(PIPE_CONTROL_CACHE_FLUSH_BITS | PIPE_CONTROL_CS_STALL);
+ }
+
if (brw->gen >= 8) {
- gen8_add_cs_stall_workaround_bits(&flags);
+ if (brw->gen == 8)
+ gen8_add_cs_stall_workaround_bits(&flags);
+
+ if (brw->gen == 9 &&
+ (flags & PIPE_CONTROL_VF_CACHE_INVALIDATE)) {
+ /* Hardware workaround: SKL
+ *
+ * Emit Pipe Control with all bits set to zero before emitting
+ * a Pipe Control with VF Cache Invalidate set.
+ */
+ brw_emit_pipe_control_flush(brw, 0);
+ }
BEGIN_BATCH(6);
OUT_BATCH(_3DSTATE_PIPE_CONTROL | (6 - 2));
OUT_BATCH(0);
ADVANCE_BATCH();
} else if (brw->gen >= 6) {
+ if (brw->gen == 6 &&
+ (flags & PIPE_CONTROL_RENDER_TARGET_FLUSH)) {
+ /* Hardware workaround: SNB B-Spec says:
+ *
+ * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush
+ * Enable = 1, a PIPE_CONTROL with any non-zero post-sync-op is
+ * required.
+ */
+ brw_emit_post_sync_nonzero_flush(brw);
+ }
+
flags |= gen7_cs_stall_every_four_pipe_controls(brw, flags);
BEGIN_BATCH(5);
uint32_t imm_lower, uint32_t imm_upper)
{
if (brw->gen >= 8) {
- gen8_add_cs_stall_workaround_bits(&flags);
+ if (brw->gen == 8)
+ gen8_add_cs_stall_workaround_bits(&flags);
BEGIN_BATCH(6);
OUT_BATCH(_3DSTATE_PIPE_CONTROL | (6 - 2));
* already flushed (e.g., via a preceding MI_FLUSH).
*/
void
-intel_emit_depth_stall_flushes(struct brw_context *brw)
+brw_emit_depth_stall_flushes(struct brw_context *brw)
{
assert(brw->gen >= 6 && brw->gen <= 9);
+ /* Starting on BDW, these pipe controls are unnecessary.
+ *
+ * WM HW will internally manage the draining pipe and flushing of the caches
+ * when this command is issued. The PIPE_CONTROL restrictions are removed.
+ */
+ if (brw->gen >= 8)
+ return;
+
brw_emit_pipe_control_flush(brw, PIPE_CONTROL_DEPTH_STALL);
brw_emit_pipe_control_flush(brw, PIPE_CONTROL_DEPTH_CACHE_FLUSH);
brw_emit_pipe_control_flush(brw, PIPE_CONTROL_DEPTH_STALL);
brw_emit_pipe_control_write(brw,
PIPE_CONTROL_WRITE_IMMEDIATE
| PIPE_CONTROL_DEPTH_STALL,
- brw->batch.workaround_bo, 0,
+ brw->workaround_bo, 0,
0, 0);
}
brw_emit_pipe_control_write(brw,
PIPE_CONTROL_CS_STALL
| PIPE_CONTROL_WRITE_IMMEDIATE,
- brw->batch.workaround_bo, 0,
+ brw->workaround_bo, 0,
0, 0);
}
* really our business. That leaves only stall at scoreboard.
*/
void
-intel_emit_post_sync_nonzero_flush(struct brw_context *brw)
+brw_emit_post_sync_nonzero_flush(struct brw_context *brw)
{
brw_emit_pipe_control_flush(brw,
PIPE_CONTROL_CS_STALL |
PIPE_CONTROL_STALL_AT_SCOREBOARD);
brw_emit_pipe_control_write(brw, PIPE_CONTROL_WRITE_IMMEDIATE,
- brw->batch.workaround_bo, 0, 0, 0);
+ brw->workaround_bo, 0, 0, 0);
}
/* Emit a pipelined flush to either flush render and texture cache for
* This is also used for the always_flush_cache driconf debug option.
*/
void
-intel_batchbuffer_emit_mi_flush(struct brw_context *brw)
+brw_emit_mi_flush(struct brw_context *brw)
{
if (brw->batch.ring == BLT_RING && brw->gen >= 6) {
BEGIN_BATCH_BLT(4);
} else {
int flags = PIPE_CONTROL_NO_WRITE | PIPE_CONTROL_RENDER_TARGET_FLUSH;
if (brw->gen >= 6) {
- if (brw->gen == 9) {
- /* Hardware workaround: SKL
- *
- * Emit Pipe Control with all bits set to zero before emitting
- * a Pipe Control with VF Cache Invalidate set.
- */
- brw_emit_pipe_control_flush(brw, 0);
- }
-
flags |= PIPE_CONTROL_INSTRUCTION_INVALIDATE |
PIPE_CONTROL_DEPTH_CACHE_FLUSH |
PIPE_CONTROL_VF_CACHE_INVALIDATE |
PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
PIPE_CONTROL_CS_STALL;
-
- if (brw->gen == 6) {
- /* Hardware workaround: SNB B-Spec says:
- *
- * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache
- * Flush Enable =1, a PIPE_CONTROL with any non-zero
- * post-sync-op is required.
- */
- intel_emit_post_sync_nonzero_flush(brw);
- }
}
brw_emit_pipe_control_flush(brw, flags);
}
+}
+
+int
+brw_init_pipe_control(struct brw_context *brw,
+ const struct brw_device_info *devinfo)
+{
+ if (devinfo->gen < 6)
+ return 0;
+
+ /* We can't just use brw_state_batch to get a chunk of space for
+ * the gen6 workaround because it involves actually writing to
+ * the buffer, and the kernel doesn't let us write to the batch.
+ */
+ brw->workaround_bo = drm_intel_bo_alloc(brw->bufmgr,
+ "pipe_control workaround",
+ 4096, 4096);
+ if (brw->workaround_bo == NULL)
+ return -ENOMEM;
+
+ brw->pipe_controls_since_last_cs_stall = 0;
- brw_render_cache_set_clear(brw);
+ return 0;
+}
+
+void
+brw_fini_pipe_control(struct brw_context *brw)
+{
+ drm_intel_bo_unreference(brw->workaround_bo);
}
projects / mesa.git / blobdiff