2 * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
3 * Copyright 2018 Advanced Micro Devices, Inc.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * on the rights to use, copy, modify, merge, publish, distribute, sub
10 * license, and/or sell copies of the Software, and to permit persons to whom
11 * the Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
21 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
22 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
23 * USE OR OTHER DEALINGS IN THE SOFTWARE.
28 #include "util/os_time.h"
31 void si_need_gfx_cs_space(struct si_context
*ctx
)
33 struct radeon_cmdbuf
*cs
= ctx
->gfx_cs
;
35 /* There is no need to flush the DMA IB here, because
36 * r600_need_dma_space always flushes the GFX IB if there is
37 * a conflict, which means any unflushed DMA commands automatically
38 * precede the GFX IB (= they had no dependency on the GFX IB when
39 * they were submitted).
42 /* There are two memory usage counters in the winsys for all buffers
43 * that have been added (cs_add_buffer) and two counters in the pipe
44 * driver for those that haven't been added yet.
46 if (unlikely(!radeon_cs_memory_below_limit(ctx
->screen
, ctx
->gfx_cs
,
47 ctx
->vram
, ctx
->gtt
))) {
50 si_flush_gfx_cs(ctx
, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW
, NULL
);
56 /* If the IB is sufficiently large, don't count the space needed
57 * and just flush if there is not enough space left.
59 * Also reserve space for stopping queries at the end of IB, because
60 * the number of active queries is mostly unlimited.
62 unsigned need_dwords
= 2048 + ctx
->num_cs_dw_queries_suspend
;
63 if (!ctx
->ws
->cs_check_space(cs
, need_dwords
))
64 si_flush_gfx_cs(ctx
, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW
, NULL
);
67 void si_flush_gfx_cs(struct si_context
*ctx
, unsigned flags
,
68 struct pipe_fence_handle
**fence
)
70 struct radeon_cmdbuf
*cs
= ctx
->gfx_cs
;
71 struct radeon_winsys
*ws
= ctx
->ws
;
72 unsigned wait_flags
= 0;
74 if (ctx
->gfx_flush_in_progress
)
77 if (!ctx
->screen
->info
.kernel_flushes_tc_l2_after_ib
) {
78 wait_flags
|= SI_CONTEXT_PS_PARTIAL_FLUSH
|
79 SI_CONTEXT_CS_PARTIAL_FLUSH
|
80 SI_CONTEXT_INV_GLOBAL_L2
;
81 } else if (ctx
->chip_class
== SI
) {
82 /* The kernel flushes L2 before shaders are finished. */
83 wait_flags
|= SI_CONTEXT_PS_PARTIAL_FLUSH
|
84 SI_CONTEXT_CS_PARTIAL_FLUSH
;
85 } else if (!(flags
& RADEON_FLUSH_START_NEXT_GFX_IB_NOW
)) {
86 wait_flags
|= SI_CONTEXT_PS_PARTIAL_FLUSH
|
87 SI_CONTEXT_CS_PARTIAL_FLUSH
;
90 /* Drop this flush if it's a no-op. */
91 if (!radeon_emitted(cs
, ctx
->initial_gfx_cs_size
) &&
92 (!wait_flags
|| !ctx
->gfx_last_ib_is_busy
))
95 if (si_check_device_reset(ctx
))
98 if (ctx
->screen
->debug_flags
& DBG(CHECK_VM
))
99 flags
&= ~PIPE_FLUSH_ASYNC
;
101 /* If the state tracker is flushing the GFX IB, si_flush_from_st is
102 * responsible for flushing the DMA IB and merging the fences from both.
103 * This code is only needed when the driver flushes the GFX IB
104 * internally, and it never asks for a fence handle.
106 if (radeon_emitted(ctx
->dma_cs
, 0)) {
107 assert(fence
== NULL
); /* internal flushes only */
108 si_flush_dma_cs(ctx
, flags
, NULL
);
111 ctx
->gfx_flush_in_progress
= true;
113 if (!LIST_IS_EMPTY(&ctx
->active_queries
))
114 si_suspend_queries(ctx
);
116 ctx
->streamout
.suspended
= false;
117 if (ctx
->streamout
.begin_emitted
) {
118 si_emit_streamout_end(ctx
);
119 ctx
->streamout
.suspended
= true;
122 /* Make sure CP DMA is idle at the end of IBs after L2 prefetches
123 * because the kernel doesn't wait for it. */
124 if (ctx
->chip_class
>= CIK
)
125 si_cp_dma_wait_for_idle(ctx
);
127 /* Wait for draw calls to finish if needed. */
129 ctx
->flags
|= wait_flags
;
130 si_emit_cache_flush(ctx
);
132 ctx
->gfx_last_ib_is_busy
= wait_flags
== 0;
134 if (ctx
->current_saved_cs
) {
137 /* Save the IB for debug contexts. */
138 si_save_cs(ws
, cs
, &ctx
->current_saved_cs
->gfx
, true);
139 ctx
->current_saved_cs
->flushed
= true;
140 ctx
->current_saved_cs
->time_flush
= os_time_get_nano();
142 si_log_hw_flush(ctx
);
146 ws
->cs_flush(cs
, flags
, &ctx
->last_gfx_fence
);
148 ws
->fence_reference(fence
, ctx
->last_gfx_fence
);
150 /* This must be after cs_flush returns, since the context's API
151 * thread can concurrently read this value in si_fence_finish. */
152 ctx
->num_gfx_cs_flushes
++;
154 /* Check VM faults if needed. */
155 if (ctx
->screen
->debug_flags
& DBG(CHECK_VM
)) {
156 /* Use conservative timeout 800ms, after which we won't wait any
157 * longer and assume the GPU is hung.
159 ctx
->ws
->fence_wait(ctx
->ws
, ctx
->last_gfx_fence
, 800*1000*1000);
161 si_check_vm_faults(ctx
, &ctx
->current_saved_cs
->gfx
, RING_GFX
);
164 if (ctx
->current_saved_cs
)
165 si_saved_cs_reference(&ctx
->current_saved_cs
, NULL
);
167 si_begin_new_gfx_cs(ctx
);
168 ctx
->gfx_flush_in_progress
= false;
171 static void si_begin_gfx_cs_debug(struct si_context
*ctx
)
173 static const uint32_t zeros
[1];
174 assert(!ctx
->current_saved_cs
);
176 ctx
->current_saved_cs
= calloc(1, sizeof(*ctx
->current_saved_cs
));
177 if (!ctx
->current_saved_cs
)
180 pipe_reference_init(&ctx
->current_saved_cs
->reference
, 1);
182 ctx
->current_saved_cs
->trace_buf
= r600_resource(
183 pipe_buffer_create(ctx
->b
.screen
, 0, PIPE_USAGE_STAGING
, 8));
184 if (!ctx
->current_saved_cs
->trace_buf
) {
185 free(ctx
->current_saved_cs
);
186 ctx
->current_saved_cs
= NULL
;
190 pipe_buffer_write_nooverlap(&ctx
->b
, &ctx
->current_saved_cs
->trace_buf
->b
.b
,
191 0, sizeof(zeros
), zeros
);
192 ctx
->current_saved_cs
->trace_id
= 0;
196 radeon_add_to_buffer_list(ctx
, ctx
->gfx_cs
, ctx
->current_saved_cs
->trace_buf
,
197 RADEON_USAGE_READWRITE
, RADEON_PRIO_TRACE
);
200 void si_begin_new_gfx_cs(struct si_context
*ctx
)
203 si_begin_gfx_cs_debug(ctx
);
205 /* Always invalidate caches at the beginning of IBs, because external
206 * users (e.g. BO evictions and SDMA/UVD/VCE IBs) can modify our
209 * Note that the cache flush done by the kernel at the end of GFX IBs
210 * isn't useful here, because that flush can finish after the following
213 * TODO: Do we also need to invalidate CB & DB caches?
215 ctx
->flags
|= SI_CONTEXT_INV_ICACHE
|
216 SI_CONTEXT_INV_SMEM_L1
|
217 SI_CONTEXT_INV_VMEM_L1
|
218 SI_CONTEXT_INV_GLOBAL_L2
|
219 SI_CONTEXT_START_PIPELINE_STATS
;
221 /* set all valid group as dirty so they get reemited on
224 si_pm4_reset_emitted(ctx
);
226 /* The CS initialization should be emitted before everything else. */
227 si_pm4_emit(ctx
, ctx
->init_config
);
228 if (ctx
->init_config_gs_rings
)
229 si_pm4_emit(ctx
, ctx
->init_config_gs_rings
);
231 if (ctx
->queued
.named
.ls
)
232 ctx
->prefetch_L2_mask
|= SI_PREFETCH_LS
;
233 if (ctx
->queued
.named
.hs
)
234 ctx
->prefetch_L2_mask
|= SI_PREFETCH_HS
;
235 if (ctx
->queued
.named
.es
)
236 ctx
->prefetch_L2_mask
|= SI_PREFETCH_ES
;
237 if (ctx
->queued
.named
.gs
)
238 ctx
->prefetch_L2_mask
|= SI_PREFETCH_GS
;
239 if (ctx
->queued
.named
.vs
)
240 ctx
->prefetch_L2_mask
|= SI_PREFETCH_VS
;
241 if (ctx
->queued
.named
.ps
)
242 ctx
->prefetch_L2_mask
|= SI_PREFETCH_PS
;
243 if (ctx
->vb_descriptors_buffer
&& ctx
->vertex_elements
)
244 ctx
->prefetch_L2_mask
|= SI_PREFETCH_VBO_DESCRIPTORS
;
246 /* CLEAR_STATE disables all colorbuffers, so only enable bound ones. */
247 bool has_clear_state
= ctx
->screen
->has_clear_state
;
248 if (has_clear_state
) {
249 ctx
->framebuffer
.dirty_cbufs
=
250 u_bit_consecutive(0, ctx
->framebuffer
.state
.nr_cbufs
);
251 /* CLEAR_STATE disables the zbuffer, so only enable it if it's bound. */
252 ctx
->framebuffer
.dirty_zsbuf
= ctx
->framebuffer
.state
.zsbuf
!= NULL
;
254 ctx
->framebuffer
.dirty_cbufs
= u_bit_consecutive(0, 8);
255 ctx
->framebuffer
.dirty_zsbuf
= true;
257 /* This should always be marked as dirty to set the framebuffer scissor
259 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.framebuffer
);
261 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.clip_regs
);
262 /* CLEAR_STATE sets zeros. */
263 if (!has_clear_state
|| ctx
->clip_state
.any_nonzeros
)
264 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.clip_state
);
265 ctx
->sample_locs_num_samples
= 0;
266 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.msaa_sample_locs
);
267 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.msaa_config
);
268 /* CLEAR_STATE sets 0xffff. */
269 if (!has_clear_state
|| ctx
->sample_mask
!= 0xffff)
270 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.sample_mask
);
271 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.cb_render_state
);
272 /* CLEAR_STATE sets zeros. */
273 if (!has_clear_state
|| ctx
->blend_color
.any_nonzeros
)
274 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.blend_color
);
275 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.db_render_state
);
276 if (ctx
->chip_class
>= GFX9
)
277 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.dpbb_state
);
278 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.stencil_ref
);
279 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.spi_map
);
280 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.streamout_enable
);
281 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.render_cond
);
282 /* CLEAR_STATE disables all window rectangles. */
283 if (!has_clear_state
|| ctx
->num_window_rectangles
> 0)
284 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.window_rectangles
);
285 si_all_descriptors_begin_new_cs(ctx
);
286 si_all_resident_buffers_begin_new_cs(ctx
);
288 ctx
->scissors
.dirty_mask
= (1 << SI_MAX_VIEWPORTS
) - 1;
289 ctx
->viewports
.dirty_mask
= (1 << SI_MAX_VIEWPORTS
) - 1;
290 ctx
->viewports
.depth_range_dirty_mask
= (1 << SI_MAX_VIEWPORTS
) - 1;
291 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.guardband
);
292 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.scissors
);
293 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.viewports
);
295 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.scratch_state
);
296 if (ctx
->scratch_buffer
) {
297 si_context_add_resource_size(ctx
, &ctx
->scratch_buffer
->b
.b
);
300 if (ctx
->streamout
.suspended
) {
301 ctx
->streamout
.append_bitmask
= ctx
->streamout
.enabled_mask
;
302 si_streamout_buffers_dirty(ctx
);
305 if (!LIST_IS_EMPTY(&ctx
->active_queries
))
306 si_resume_queries(ctx
);
308 assert(!ctx
->gfx_cs
->prev_dw
);
309 ctx
->initial_gfx_cs_size
= ctx
->gfx_cs
->current
.cdw
;
311 /* Invalidate various draw states so that they are emitted before
312 * the first draw call. */
313 si_invalidate_draw_sh_constants(ctx
);
314 ctx
->last_index_size
= -1;
315 ctx
->last_primitive_restart_en
= -1;
316 ctx
->last_restart_index
= SI_RESTART_INDEX_UNKNOWN
;
318 ctx
->last_multi_vgt_param
= -1;
319 ctx
->last_rast_prim
= -1;
320 ctx
->last_sc_line_stipple
= ~0;
321 ctx
->last_vs_state
= ~0;
323 ctx
->last_tcs
= NULL
;
324 ctx
->last_tes_sh_base
= -1;
325 ctx
->last_num_tcs_input_cp
= -1;
326 ctx
->last_ls_hs_config
= -1; /* impossible value */
328 ctx
->cs_shader_state
.initialized
= false;
330 if (has_clear_state
) {
331 ctx
->tracked_regs
.reg_value
[SI_TRACKED_DB_RENDER_CONTROL
] = 0x00000000;
332 ctx
->tracked_regs
.reg_value
[SI_TRACKED_DB_COUNT_CONTROL
] = 0x00000000;
333 ctx
->tracked_regs
.reg_value
[SI_TRACKED_DB_RENDER_OVERRIDE2
] = 0x00000000;
334 ctx
->tracked_regs
.reg_value
[SI_TRACKED_DB_SHADER_CONTROL
] = 0x00000000;
335 ctx
->tracked_regs
.reg_value
[SI_TRACKED_CB_TARGET_MASK
] = 0xffffffff;
336 ctx
->tracked_regs
.reg_value
[SI_TRACKED_CB_DCC_CONTROL
] = 0x00000000;
337 ctx
->tracked_regs
.reg_value
[SI_TRACKED_SX_PS_DOWNCONVERT
] = 0x00000000;
338 ctx
->tracked_regs
.reg_value
[SI_TRACKED_SX_BLEND_OPT_EPSILON
] = 0x00000000;
339 ctx
->tracked_regs
.reg_value
[SI_TRACKED_SX_BLEND_OPT_CONTROL
] = 0x00000000;
340 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_SC_LINE_CNTL
] = 0x00001000;
341 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_SC_AA_CONFIG
] = 0x00000000;
342 ctx
->tracked_regs
.reg_value
[SI_TRACKED_DB_EQAA
] = 0x00000000;
343 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_SC_MODE_CNTL_1
] = 0x00000000;
344 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_SU_SMALL_PRIM_FILTER_CNTL
] = 0x00000000;
345 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_CL_VS_OUT_CNTL
] = 0x00000000;
346 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_CL_CLIP_CNTL
] = 0x00090000;
347 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_SC_BINNER_CNTL_0
] = 0x00000003;
348 ctx
->tracked_regs
.reg_value
[SI_TRACKED_DB_DFSM_CONTROL
] = 0x00000000;
349 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_CL_GB_VERT_CLIP_ADJ
] = 0x3f800000;
350 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_CL_GB_VERT_DISC_ADJ
] = 0x3f800000;
351 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_CL_GB_HORZ_CLIP_ADJ
] = 0x3f800000;
352 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_CL_GB_HORZ_DISC_ADJ
] = 0x3f800000;
353 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_SC_CLIPRECT_RULE
] = 0xffff;
355 /* Set all saved registers state to saved. */
356 ctx
->tracked_regs
.reg_saved
= 0xffffffff;
358 /* Set all saved registers state to unknown. */
359 ctx
->tracked_regs
.reg_saved
= 0;
362 /* 0xffffffff is a impossible value to register SPI_PS_INPUT_CNTL_n */
363 memset(ctx
->tracked_regs
.spi_ps_input_cntl
, 0xff, sizeof(uint32_t) * 32);