2 * Copyright © 2017 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
24 * @file iris_pipe_control.c
26 * PIPE_CONTROL is the main flushing and synchronization primitive on Intel
27 * GPUs. It can invalidate caches, stall until rendering reaches various
28 * stages of completion, write to memory, and other things. In a way, it's
29 * a swiss army knife command - it has all kinds of capabilities, but some
30 * significant limitations as well.
32 * Unfortunately, it's notoriously complicated and difficult to use. Many
33 * sub-commands can't be used together. Some are meant to be used at the
34 * top of the pipeline (invalidating caches before drawing), while some are
35 * meant to be used at the end (stalling or flushing after drawing).
37 * Also, there's a list of restrictions a mile long, which vary by generation.
38 * Do this before doing that, or suffer the consequences (usually a GPU hang).
40 * This file contains helpers for emitting them safely. You can simply call
41 * iris_emit_pipe_control_flush() with the desired operations (as logical
42 * PIPE_CONTROL_* bits), and it will take care of splitting it into multiple
43 * PIPE_CONTROL commands as necessary. The per-generation workarounds are
44 * applied in iris_emit_raw_pipe_control() in iris_state.c.
47 #include "iris_context.h"
48 #include "util/hash_table.h"
52 * Emit a PIPE_CONTROL with various flushing flags.
54 * The caller is responsible for deciding what flags are appropriate for the
58 iris_emit_pipe_control_flush(struct iris_batch
*batch
, uint32_t flags
)
60 if ((flags
& PIPE_CONTROL_CACHE_FLUSH_BITS
) &&
61 (flags
& PIPE_CONTROL_CACHE_INVALIDATE_BITS
)) {
62 /* A pipe control command with flush and invalidate bits set
63 * simultaneously is an inherently racy operation on Gen6+ if the
64 * contents of the flushed caches were intended to become visible from
65 * any of the invalidated caches. Split it in two PIPE_CONTROLs, the
66 * first one should stall the pipeline to make sure that the flushed R/W
67 * caches are coherent with memory once the specified R/O caches are
68 * invalidated. On pre-Gen6 hardware the (implicit) R/O cache
69 * invalidation seems to happen at the bottom of the pipeline together
70 * with any write cache flush, so this shouldn't be a concern. In order
71 * to ensure a full stall, we do an end-of-pipe sync.
73 iris_emit_end_of_pipe_sync(batch
, flags
& PIPE_CONTROL_CACHE_FLUSH_BITS
);
74 flags
&= ~(PIPE_CONTROL_CACHE_FLUSH_BITS
| PIPE_CONTROL_CS_STALL
);
77 batch
->vtbl
->emit_raw_pipe_control(batch
, flags
, NULL
, 0, 0);
81 * Emit a PIPE_CONTROL that writes to a buffer object.
83 * \p flags should contain one of the following items:
84 * - PIPE_CONTROL_WRITE_IMMEDIATE
85 * - PIPE_CONTROL_WRITE_TIMESTAMP
86 * - PIPE_CONTROL_WRITE_DEPTH_COUNT
89 iris_emit_pipe_control_write(struct iris_batch
*batch
, uint32_t flags
,
90 struct iris_bo
*bo
, uint32_t offset
,
93 batch
->vtbl
->emit_raw_pipe_control(batch
, flags
, bo
, offset
, imm
);
97 * From Sandybridge PRM, volume 2, "1.7.2 End-of-Pipe Synchronization":
99 * Write synchronization is a special case of end-of-pipe
100 * synchronization that requires that the render cache and/or depth
101 * related caches are flushed to memory, where the data will become
102 * globally visible. This type of synchronization is required prior to
103 * SW (CPU) actually reading the result data from memory, or initiating
104 * an operation that will use as a read surface (such as a texture
105 * surface) a previous render target and/or depth/stencil buffer
107 * From Haswell PRM, volume 2, part 1, "End-of-Pipe Synchronization":
109 * Exercising the write cache flush bits (Render Target Cache Flush
110 * Enable, Depth Cache Flush Enable, DC Flush) in PIPE_CONTROL only
111 * ensures the write caches are flushed and doesn't guarantee the data
112 * is globally visible.
114 * SW can track the completion of the end-of-pipe-synchronization by
115 * using "Notify Enable" and "PostSync Operation - Write Immediate
116 * Data" in the PIPE_CONTROL command.
119 iris_emit_end_of_pipe_sync(struct iris_batch
*batch
, uint32_t flags
)
121 /* From Sandybridge PRM, volume 2, "1.7.3.1 Writing a Value to Memory":
123 * "The most common action to perform upon reaching a synchronization
124 * point is to write a value out to memory. An immediate value
125 * (included with the synchronization command) may be written."
127 * From Broadwell PRM, volume 7, "End-of-Pipe Synchronization":
129 * "In case the data flushed out by the render engine is to be read
130 * back in to the render engine in coherent manner, then the render
131 * engine has to wait for the fence completion before accessing the
132 * flushed data. This can be achieved by following means on various
133 * products: PIPE_CONTROL command with CS Stall and the required
134 * write caches flushed with Post-Sync-Operation as Write Immediate
138 * - Workload-1 (3D/GPGPU/MEDIA)
139 * - PIPE_CONTROL (CS Stall, Post-Sync-Operation Write Immediate
140 * Data, Required Write Cache Flush bits set)
141 * - Workload-2 (Can use the data produce or output by Workload-1)
143 iris_emit_pipe_control_write(batch
, flags
| PIPE_CONTROL_CS_STALL
|
144 PIPE_CONTROL_WRITE_IMMEDIATE
,
145 batch
->screen
->workaround_bo
, 0, 0);
149 iris_texture_barrier(struct pipe_context
*ctx
, unsigned flags
)
151 struct iris_context
*ice
= (void *) ctx
;
153 if (ice
->batches
[IRIS_BATCH_RENDER
].contains_draw
) {
154 iris_emit_pipe_control_flush(&ice
->batches
[IRIS_BATCH_RENDER
],
155 PIPE_CONTROL_DEPTH_CACHE_FLUSH
|
156 PIPE_CONTROL_RENDER_TARGET_FLUSH
|
157 PIPE_CONTROL_CS_STALL
);
158 iris_emit_pipe_control_flush(&ice
->batches
[IRIS_BATCH_RENDER
],
159 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE
);
162 if (ice
->batches
[IRIS_BATCH_COMPUTE
].contains_draw
) {
163 iris_emit_pipe_control_flush(&ice
->batches
[IRIS_BATCH_COMPUTE
],
164 PIPE_CONTROL_CS_STALL
);
165 iris_emit_pipe_control_flush(&ice
->batches
[IRIS_BATCH_COMPUTE
],
166 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE
);
171 iris_memory_barrier(struct pipe_context
*ctx
, unsigned flags
)
173 struct iris_context
*ice
= (void *) ctx
;
174 unsigned bits
= PIPE_CONTROL_DATA_CACHE_FLUSH
| PIPE_CONTROL_CS_STALL
;
176 if (flags
& (PIPE_BARRIER_VERTEX_BUFFER
|
177 PIPE_BARRIER_INDEX_BUFFER
|
178 PIPE_BARRIER_INDIRECT_BUFFER
)) {
179 bits
|= PIPE_CONTROL_VF_CACHE_INVALIDATE
;
182 if (flags
& PIPE_BARRIER_CONSTANT_BUFFER
) {
183 bits
|= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE
|
184 PIPE_CONTROL_CONST_CACHE_INVALIDATE
;
187 if (flags
& (PIPE_BARRIER_TEXTURE
| PIPE_BARRIER_FRAMEBUFFER
)) {
188 bits
|= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE
|
189 PIPE_CONTROL_RENDER_TARGET_FLUSH
;
192 // XXX: don't unconditionally emit flushes in both engines, we don't
193 // even know if we're even using e.g. the compute engine...
195 for (int i
= 0; i
< IRIS_BATCH_COUNT
; i
++) {
196 if (ice
->batches
[i
].contains_draw
)
197 iris_emit_pipe_control_flush(&ice
->batches
[i
], bits
);
202 iris_init_flush_functions(struct pipe_context
*ctx
)
204 ctx
->memory_barrier
= iris_memory_barrier
;
205 ctx
->texture_barrier
= iris_texture_barrier
;