2 * Copyright © 2010 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 #include "brw_context.h"
25 #include "intel_batchbuffer.h"
26 #include "intel_fbo.h"
27 #include "intel_reg.h"
30 * According to the latest documentation, any PIPE_CONTROL with the
31 * "Command Streamer Stall" bit set must also have another bit set,
32 * with five different options:
34 * - Render Target Cache Flush
36 * - Stall at Pixel Scoreboard
37 * - Post-Sync Operation
40 * I chose "Stall at Pixel Scoreboard" since we've used it effectively
41 * in the past, but the choice is fairly arbitrary.
44 gen8_add_cs_stall_workaround_bits(uint32_t *flags
)
46 uint32_t wa_bits
= PIPE_CONTROL_RENDER_TARGET_FLUSH
|
47 PIPE_CONTROL_DEPTH_CACHE_FLUSH
|
48 PIPE_CONTROL_WRITE_IMMEDIATE
|
49 PIPE_CONTROL_WRITE_DEPTH_COUNT
|
50 PIPE_CONTROL_WRITE_TIMESTAMP
|
51 PIPE_CONTROL_STALL_AT_SCOREBOARD
|
52 PIPE_CONTROL_DEPTH_STALL
;
54 /* If we're doing a CS stall, and don't already have one of the
55 * workaround bits set, add "Stall at Pixel Scoreboard."
57 if ((*flags
& PIPE_CONTROL_CS_STALL
) != 0 && (*flags
& wa_bits
) == 0)
58 *flags
|= PIPE_CONTROL_STALL_AT_SCOREBOARD
;
61 /* Implement the WaCsStallAtEveryFourthPipecontrol workaround on IVB, BYT:
63 * "Every 4th PIPE_CONTROL command, not counting the PIPE_CONTROL with
64 * only read-cache-invalidate bit(s) set, must have a CS_STALL bit set."
66 * Note that the kernel does CS stalls between batches, so we only need
67 * to count them within a batch.
70 gen7_cs_stall_every_four_pipe_controls(struct brw_context
*brw
, uint32_t flags
)
72 if (brw
->gen
== 7 && !brw
->is_haswell
) {
73 if (flags
& PIPE_CONTROL_CS_STALL
) {
74 /* If we're doing a CS stall, reset the counter and carry on. */
75 brw
->pipe_controls_since_last_cs_stall
= 0;
79 /* If this is the fourth pipe control without a CS stall, do one now. */
80 if (++brw
->pipe_controls_since_last_cs_stall
== 4) {
81 brw
->pipe_controls_since_last_cs_stall
= 0;
82 return PIPE_CONTROL_CS_STALL
;
89 * Emit a PIPE_CONTROL with various flushing flags.
91 * The caller is responsible for deciding what flags are appropriate for the
95 brw_emit_pipe_control_flush(struct brw_context
*brw
, uint32_t flags
)
98 gen8_add_cs_stall_workaround_bits(&flags
);
101 OUT_BATCH(_3DSTATE_PIPE_CONTROL
| (6 - 2));
108 } else if (brw
->gen
>= 6) {
109 flags
|= gen7_cs_stall_every_four_pipe_controls(brw
, flags
);
112 OUT_BATCH(_3DSTATE_PIPE_CONTROL
| (5 - 2));
120 OUT_BATCH(_3DSTATE_PIPE_CONTROL
| flags
| (4 - 2));
129 * Emit a PIPE_CONTROL that writes to a buffer object.
131 * \p flags should contain one of the following items:
132 * - PIPE_CONTROL_WRITE_IMMEDIATE
133 * - PIPE_CONTROL_WRITE_TIMESTAMP
134 * - PIPE_CONTROL_WRITE_DEPTH_COUNT
137 brw_emit_pipe_control_write(struct brw_context
*brw
, uint32_t flags
,
138 drm_intel_bo
*bo
, uint32_t offset
,
139 uint32_t imm_lower
, uint32_t imm_upper
)
142 gen8_add_cs_stall_workaround_bits(&flags
);
145 OUT_BATCH(_3DSTATE_PIPE_CONTROL
| (6 - 2));
147 OUT_RELOC64(bo
, I915_GEM_DOMAIN_INSTRUCTION
, I915_GEM_DOMAIN_INSTRUCTION
,
149 OUT_BATCH(imm_lower
);
150 OUT_BATCH(imm_upper
);
152 } else if (brw
->gen
>= 6) {
153 flags
|= gen7_cs_stall_every_four_pipe_controls(brw
, flags
);
155 /* PPGTT/GGTT is selected by DW2 bit 2 on Sandybridge, but DW1 bit 24
156 * on later platforms. We always use PPGTT on Gen7+.
158 unsigned gen6_gtt
= brw
->gen
== 6 ? PIPE_CONTROL_GLOBAL_GTT_WRITE
: 0;
161 OUT_BATCH(_3DSTATE_PIPE_CONTROL
| (5 - 2));
163 OUT_RELOC(bo
, I915_GEM_DOMAIN_INSTRUCTION
, I915_GEM_DOMAIN_INSTRUCTION
,
165 OUT_BATCH(imm_lower
);
166 OUT_BATCH(imm_upper
);
170 OUT_BATCH(_3DSTATE_PIPE_CONTROL
| flags
| (4 - 2));
171 OUT_RELOC(bo
, I915_GEM_DOMAIN_INSTRUCTION
, I915_GEM_DOMAIN_INSTRUCTION
,
172 PIPE_CONTROL_GLOBAL_GTT_WRITE
| offset
);
173 OUT_BATCH(imm_lower
);
174 OUT_BATCH(imm_upper
);
180 * Restriction [DevSNB, DevIVB]:
182 * Prior to changing Depth/Stencil Buffer state (i.e. any combination of
183 * 3DSTATE_DEPTH_BUFFER, 3DSTATE_CLEAR_PARAMS, 3DSTATE_STENCIL_BUFFER,
184 * 3DSTATE_HIER_DEPTH_BUFFER) SW must first issue a pipelined depth stall
185 * (PIPE_CONTROL with Depth Stall bit set), followed by a pipelined depth
186 * cache flush (PIPE_CONTROL with Depth Flush Bit set), followed by
187 * another pipelined depth stall (PIPE_CONTROL with Depth Stall bit set),
188 * unless SW can otherwise guarantee that the pipeline from WM onwards is
189 * already flushed (e.g., via a preceding MI_FLUSH).
192 brw_emit_depth_stall_flushes(struct brw_context
*brw
)
194 assert(brw
->gen
>= 6 && brw
->gen
<= 9);
196 brw_emit_pipe_control_flush(brw
, PIPE_CONTROL_DEPTH_STALL
);
197 brw_emit_pipe_control_flush(brw
, PIPE_CONTROL_DEPTH_CACHE_FLUSH
);
198 brw_emit_pipe_control_flush(brw
, PIPE_CONTROL_DEPTH_STALL
);
202 * From the Ivybridge PRM, Volume 2 Part 1, Section 3.2 (VS Stage Input):
203 * "A PIPE_CONTROL with Post-Sync Operation set to 1h and a depth
204 * stall needs to be sent just prior to any 3DSTATE_VS, 3DSTATE_URB_VS,
205 * 3DSTATE_CONSTANT_VS, 3DSTATE_BINDING_TABLE_POINTER_VS,
206 * 3DSTATE_SAMPLER_STATE_POINTER_VS command. Only one PIPE_CONTROL needs
207 * to be sent before any combination of VS associated 3DSTATE."
210 gen7_emit_vs_workaround_flush(struct brw_context
*brw
)
212 assert(brw
->gen
== 7);
213 brw_emit_pipe_control_write(brw
,
214 PIPE_CONTROL_WRITE_IMMEDIATE
215 | PIPE_CONTROL_DEPTH_STALL
,
216 brw
->workaround_bo
, 0,
222 * Emit a PIPE_CONTROL command for gen7 with the CS Stall bit set.
225 gen7_emit_cs_stall_flush(struct brw_context
*brw
)
227 brw_emit_pipe_control_write(brw
,
228 PIPE_CONTROL_CS_STALL
229 | PIPE_CONTROL_WRITE_IMMEDIATE
,
230 brw
->workaround_bo
, 0,
236 * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
237 * implementing two workarounds on gen6. From section 1.4.7.1
238 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
240 * [DevSNB-C+{W/A}] Before any depth stall flush (including those
241 * produced by non-pipelined state commands), software needs to first
242 * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
245 * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
246 * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
248 * And the workaround for these two requires this workaround first:
250 * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
251 * BEFORE the pipe-control with a post-sync op and no write-cache
254 * And this last workaround is tricky because of the requirements on
255 * that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
258 * "1 of the following must also be set:
259 * - Render Target Cache Flush Enable ([12] of DW1)
260 * - Depth Cache Flush Enable ([0] of DW1)
261 * - Stall at Pixel Scoreboard ([1] of DW1)
262 * - Depth Stall ([13] of DW1)
263 * - Post-Sync Operation ([13] of DW1)
264 * - Notify Enable ([8] of DW1)"
266 * The cache flushes require the workaround flush that triggered this
267 * one, so we can't use it. Depth stall would trigger the same.
268 * Post-sync nonzero is what triggered this second workaround, so we
269 * can't use that one either. Notify enable is IRQs, which aren't
270 * really our business. That leaves only stall at scoreboard.
273 brw_emit_post_sync_nonzero_flush(struct brw_context
*brw
)
275 brw_emit_pipe_control_flush(brw
,
276 PIPE_CONTROL_CS_STALL
|
277 PIPE_CONTROL_STALL_AT_SCOREBOARD
);
279 brw_emit_pipe_control_write(brw
, PIPE_CONTROL_WRITE_IMMEDIATE
,
280 brw
->workaround_bo
, 0, 0, 0);
283 /* Emit a pipelined flush to either flush render and texture cache for
284 * reading from a FBO-drawn texture, or flush so that frontbuffer
285 * render appears on the screen in DRI1.
287 * This is also used for the always_flush_cache driconf debug option.
290 brw_emit_mi_flush(struct brw_context
*brw
)
292 if (brw
->batch
.ring
== BLT_RING
&& brw
->gen
>= 6) {
294 OUT_BATCH(MI_FLUSH_DW
);
300 int flags
= PIPE_CONTROL_NO_WRITE
| PIPE_CONTROL_RENDER_TARGET_FLUSH
;
303 /* Hardware workaround: SKL
305 * Emit Pipe Control with all bits set to zero before emitting
306 * a Pipe Control with VF Cache Invalidate set.
308 brw_emit_pipe_control_flush(brw
, 0);
311 flags
|= PIPE_CONTROL_INSTRUCTION_INVALIDATE
|
312 PIPE_CONTROL_DEPTH_CACHE_FLUSH
|
313 PIPE_CONTROL_VF_CACHE_INVALIDATE
|
314 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE
|
315 PIPE_CONTROL_CS_STALL
;
318 /* Hardware workaround: SNB B-Spec says:
320 * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache
321 * Flush Enable =1, a PIPE_CONTROL with any non-zero
322 * post-sync-op is required.
324 brw_emit_post_sync_nonzero_flush(brw
);
327 brw_emit_pipe_control_flush(brw
, flags
);
330 brw_render_cache_set_clear(brw
);
334 brw_init_pipe_control(struct brw_context
*brw
,
335 const struct brw_device_info
*devinfo
)
337 if (devinfo
->gen
< 6)
340 /* We can't just use brw_state_batch to get a chunk of space for
341 * the gen6 workaround because it involves actually writing to
342 * the buffer, and the kernel doesn't let us write to the batch.
344 brw
->workaround_bo
= drm_intel_bo_alloc(brw
->bufmgr
,
345 "pipe_control workaround",
347 if (brw
->workaround_bo
== NULL
)
350 brw
->pipe_controls_since_last_cs_stall
= 0;
356 brw_fini_pipe_control(struct brw_context
*brw
)
358 drm_intel_bo_unreference(brw
->workaround_bo
);