2 * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
3 * Copyright 2018 Advanced Micro Devices, Inc.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * on the rights to use, copy, modify, merge, publish, distribute, sub
10 * license, and/or sell copies of the Software, and to permit persons to whom
11 * the Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
21 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
22 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
23 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 #include "si_build_pm4.h"
29 #include "util/os_time.h"
30 #include "util/u_upload_mgr.h"
33 void si_need_gfx_cs_space(struct si_context
*ctx
)
35 struct radeon_cmdbuf
*cs
= ctx
->gfx_cs
;
37 /* There is no need to flush the DMA IB here, because
38 * si_need_dma_space always flushes the GFX IB if there is
39 * a conflict, which means any unflushed DMA commands automatically
40 * precede the GFX IB (= they had no dependency on the GFX IB when
41 * they were submitted).
44 /* There are two memory usage counters in the winsys for all buffers
45 * that have been added (cs_add_buffer) and two counters in the pipe
46 * driver for those that haven't been added yet.
48 if (unlikely(!radeon_cs_memory_below_limit(ctx
->screen
, ctx
->gfx_cs
, ctx
->vram
, ctx
->gtt
))) {
51 si_flush_gfx_cs(ctx
, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW
, NULL
);
57 unsigned need_dwords
= si_get_minimum_num_gfx_cs_dwords(ctx
);
58 if (!ctx
->ws
->cs_check_space(cs
, need_dwords
, false))
59 si_flush_gfx_cs(ctx
, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW
, NULL
);
62 void si_unref_sdma_uploads(struct si_context
*sctx
)
64 for (unsigned i
= 0; i
< sctx
->num_sdma_uploads
; i
++) {
65 si_resource_reference(&sctx
->sdma_uploads
[i
].dst
, NULL
);
66 si_resource_reference(&sctx
->sdma_uploads
[i
].src
, NULL
);
68 sctx
->num_sdma_uploads
= 0;
71 void si_flush_gfx_cs(struct si_context
*ctx
, unsigned flags
, struct pipe_fence_handle
**fence
)
73 struct radeon_cmdbuf
*cs
= ctx
->gfx_cs
;
74 struct radeon_winsys
*ws
= ctx
->ws
;
75 struct si_screen
*sscreen
= ctx
->screen
;
76 const unsigned wait_ps_cs
= SI_CONTEXT_PS_PARTIAL_FLUSH
| SI_CONTEXT_CS_PARTIAL_FLUSH
;
77 unsigned wait_flags
= 0;
79 if (ctx
->gfx_flush_in_progress
)
82 /* The amdgpu kernel driver synchronizes execution for shared DMABUFs between
83 * processes on DRM >= 3.39.0, so we don't have to wait at the end of IBs to
84 * make sure everything is idle.
86 * The amdgpu winsys synchronizes execution for buffers shared by different
87 * contexts within the same process.
89 * Interop with AMDVLK, RADV, or OpenCL within the same process requires
90 * explicit fences or glFinish.
92 if (sscreen
->info
.is_amdgpu
&& sscreen
->info
.drm_minor
>= 39)
93 flags
|= RADEON_FLUSH_START_NEXT_GFX_IB_NOW
;
95 if (!sscreen
->info
.kernel_flushes_tc_l2_after_ib
) {
96 wait_flags
|= wait_ps_cs
| SI_CONTEXT_INV_L2
;
97 } else if (ctx
->chip_class
== GFX6
) {
98 /* The kernel flushes L2 before shaders are finished. */
99 wait_flags
|= wait_ps_cs
;
100 } else if (!(flags
& RADEON_FLUSH_START_NEXT_GFX_IB_NOW
)) {
101 wait_flags
|= wait_ps_cs
;
104 /* Drop this flush if it's a no-op. */
105 if (!radeon_emitted(cs
, ctx
->initial_gfx_cs_size
) && (!wait_flags
|| !ctx
->gfx_last_ib_is_busy
))
108 if (ctx
->b
.get_device_reset_status(&ctx
->b
) != PIPE_NO_RESET
)
111 if (sscreen
->debug_flags
& DBG(CHECK_VM
))
112 flags
&= ~PIPE_FLUSH_ASYNC
;
114 ctx
->gfx_flush_in_progress
= true;
116 /* If the gallium frontend is flushing the GFX IB, si_flush_from_st is
117 * responsible for flushing the DMA IB and merging the fences from both.
118 * If the driver flushes the GFX IB internally, and it should never ask
119 * for a fence handle.
121 assert(!radeon_emitted(ctx
->sdma_cs
, 0) || fence
== NULL
);
123 /* Update the sdma_uploads list by flushing the uploader. */
124 u_upload_unmap(ctx
->b
.const_uploader
);
126 /* Execute SDMA uploads. */
127 ctx
->sdma_uploads_in_progress
= true;
128 for (unsigned i
= 0; i
< ctx
->num_sdma_uploads
; i
++) {
129 struct si_sdma_upload
*up
= &ctx
->sdma_uploads
[i
];
131 assert(up
->src_offset
% 4 == 0 && up
->dst_offset
% 4 == 0 && up
->size
% 4 == 0);
133 si_sdma_copy_buffer(ctx
, &up
->dst
->b
.b
, &up
->src
->b
.b
, up
->dst_offset
, up
->src_offset
,
136 ctx
->sdma_uploads_in_progress
= false;
137 si_unref_sdma_uploads(ctx
);
139 /* Flush SDMA (preamble IB). */
140 if (radeon_emitted(ctx
->sdma_cs
, 0))
141 si_flush_dma_cs(ctx
, flags
, NULL
);
143 if (radeon_emitted(ctx
->prim_discard_compute_cs
, 0)) {
144 struct radeon_cmdbuf
*compute_cs
= ctx
->prim_discard_compute_cs
;
145 si_compute_signal_gfx(ctx
);
147 /* Make sure compute shaders are idle before leaving the IB, so that
148 * the next IB doesn't overwrite GDS that might be in use. */
149 radeon_emit(compute_cs
, PKT3(PKT3_EVENT_WRITE
, 0, 0));
150 radeon_emit(compute_cs
, EVENT_TYPE(V_028A90_CS_PARTIAL_FLUSH
) | EVENT_INDEX(4));
152 /* Save the GDS prim restart counter if needed. */
153 if (ctx
->preserve_prim_restart_gds_at_flush
) {
154 si_cp_copy_data(ctx
, compute_cs
, COPY_DATA_DST_MEM
, ctx
->wait_mem_scratch
, 4,
155 COPY_DATA_GDS
, NULL
, 4);
159 if (ctx
->has_graphics
) {
160 if (!list_is_empty(&ctx
->active_queries
))
161 si_suspend_queries(ctx
);
163 ctx
->streamout
.suspended
= false;
164 if (ctx
->streamout
.begin_emitted
) {
165 si_emit_streamout_end(ctx
);
166 ctx
->streamout
.suspended
= true;
168 /* Since NGG streamout uses GDS, we need to make GDS
169 * idle when we leave the IB, otherwise another process
170 * might overwrite it while our shaders are busy.
172 if (sscreen
->use_ngg_streamout
)
173 wait_flags
|= SI_CONTEXT_PS_PARTIAL_FLUSH
;
177 /* Make sure CP DMA is idle at the end of IBs after L2 prefetches
178 * because the kernel doesn't wait for it. */
179 if (ctx
->chip_class
>= GFX7
)
180 si_cp_dma_wait_for_idle(ctx
);
182 /* Wait for draw calls to finish if needed. */
184 ctx
->flags
|= wait_flags
;
185 ctx
->emit_cache_flush(ctx
);
187 ctx
->gfx_last_ib_is_busy
= (wait_flags
& wait_ps_cs
) != wait_ps_cs
;
189 if (ctx
->current_saved_cs
) {
192 /* Save the IB for debug contexts. */
193 si_save_cs(ws
, cs
, &ctx
->current_saved_cs
->gfx
, true);
194 ctx
->current_saved_cs
->flushed
= true;
195 ctx
->current_saved_cs
->time_flush
= os_time_get_nano();
197 si_log_hw_flush(ctx
);
200 if (si_compute_prim_discard_enabled(ctx
)) {
201 /* The compute IB can start after the previous gfx IB starts. */
202 if (radeon_emitted(ctx
->prim_discard_compute_cs
, 0) && ctx
->last_gfx_fence
) {
203 ctx
->ws
->cs_add_fence_dependency(
204 ctx
->gfx_cs
, ctx
->last_gfx_fence
,
205 RADEON_DEPENDENCY_PARALLEL_COMPUTE_ONLY
| RADEON_DEPENDENCY_START_FENCE
);
208 /* Remember the last execution barrier. It's in the IB.
209 * It will signal the start of the next compute IB.
211 if (flags
& RADEON_FLUSH_START_NEXT_GFX_IB_NOW
&& ctx
->last_pkt3_write_data
) {
212 *ctx
->last_pkt3_write_data
= PKT3(PKT3_WRITE_DATA
, 3, 0);
213 ctx
->last_pkt3_write_data
= NULL
;
215 si_resource_reference(&ctx
->last_ib_barrier_buf
, ctx
->barrier_buf
);
216 ctx
->last_ib_barrier_buf_offset
= ctx
->barrier_buf_offset
;
217 si_resource_reference(&ctx
->barrier_buf
, NULL
);
219 ws
->fence_reference(&ctx
->last_ib_barrier_fence
, NULL
);
224 ws
->cs_flush(cs
, flags
, &ctx
->last_gfx_fence
);
226 ws
->fence_reference(fence
, ctx
->last_gfx_fence
);
228 ctx
->num_gfx_cs_flushes
++;
230 if (si_compute_prim_discard_enabled(ctx
)) {
231 /* Remember the last execution barrier, which is the last fence
234 if (!(flags
& RADEON_FLUSH_START_NEXT_GFX_IB_NOW
)) {
235 ctx
->last_pkt3_write_data
= NULL
;
236 si_resource_reference(&ctx
->last_ib_barrier_buf
, NULL
);
237 ws
->fence_reference(&ctx
->last_ib_barrier_fence
, ctx
->last_gfx_fence
);
241 /* Check VM faults if needed. */
242 if (sscreen
->debug_flags
& DBG(CHECK_VM
)) {
243 /* Use conservative timeout 800ms, after which we won't wait any
244 * longer and assume the GPU is hung.
246 ctx
->ws
->fence_wait(ctx
->ws
, ctx
->last_gfx_fence
, 800 * 1000 * 1000);
248 si_check_vm_faults(ctx
, &ctx
->current_saved_cs
->gfx
, RING_GFX
);
251 if (ctx
->current_saved_cs
)
252 si_saved_cs_reference(&ctx
->current_saved_cs
, NULL
);
254 si_begin_new_gfx_cs(ctx
);
255 ctx
->gfx_flush_in_progress
= false;
258 static void si_begin_gfx_cs_debug(struct si_context
*ctx
)
260 static const uint32_t zeros
[1];
261 assert(!ctx
->current_saved_cs
);
263 ctx
->current_saved_cs
= calloc(1, sizeof(*ctx
->current_saved_cs
));
264 if (!ctx
->current_saved_cs
)
267 pipe_reference_init(&ctx
->current_saved_cs
->reference
, 1);
269 ctx
->current_saved_cs
->trace_buf
=
270 si_resource(pipe_buffer_create(ctx
->b
.screen
, 0, PIPE_USAGE_STAGING
, 8));
271 if (!ctx
->current_saved_cs
->trace_buf
) {
272 free(ctx
->current_saved_cs
);
273 ctx
->current_saved_cs
= NULL
;
277 pipe_buffer_write_nooverlap(&ctx
->b
, &ctx
->current_saved_cs
->trace_buf
->b
.b
, 0, sizeof(zeros
),
279 ctx
->current_saved_cs
->trace_id
= 0;
283 radeon_add_to_buffer_list(ctx
, ctx
->gfx_cs
, ctx
->current_saved_cs
->trace_buf
,
284 RADEON_USAGE_READWRITE
, RADEON_PRIO_TRACE
);
287 static void si_add_gds_to_buffer_list(struct si_context
*sctx
)
290 sctx
->ws
->cs_add_buffer(sctx
->gfx_cs
, sctx
->gds
, RADEON_USAGE_READWRITE
, 0, 0);
292 sctx
->ws
->cs_add_buffer(sctx
->gfx_cs
, sctx
->gds_oa
, RADEON_USAGE_READWRITE
, 0, 0);
297 void si_allocate_gds(struct si_context
*sctx
)
299 struct radeon_winsys
*ws
= sctx
->ws
;
304 assert(sctx
->screen
->use_ngg_streamout
);
306 /* 4 streamout GDS counters.
307 * We need 256B (64 dw) of GDS, otherwise streamout hangs.
309 sctx
->gds
= ws
->buffer_create(ws
, 256, 4, RADEON_DOMAIN_GDS
, 0);
310 sctx
->gds_oa
= ws
->buffer_create(ws
, 4, 1, RADEON_DOMAIN_OA
, 0);
312 assert(sctx
->gds
&& sctx
->gds_oa
);
313 si_add_gds_to_buffer_list(sctx
);
316 void si_set_tracked_regs_to_clear_state(struct si_context
*ctx
)
318 STATIC_ASSERT(SI_NUM_TRACKED_REGS
<= sizeof(ctx
->tracked_regs
.reg_saved
) * 8);
320 ctx
->tracked_regs
.reg_value
[SI_TRACKED_DB_RENDER_CONTROL
] = 0x00000000;
321 ctx
->tracked_regs
.reg_value
[SI_TRACKED_DB_COUNT_CONTROL
] = 0x00000000;
322 ctx
->tracked_regs
.reg_value
[SI_TRACKED_DB_RENDER_OVERRIDE2
] = 0x00000000;
323 ctx
->tracked_regs
.reg_value
[SI_TRACKED_DB_SHADER_CONTROL
] = 0x00000000;
324 ctx
->tracked_regs
.reg_value
[SI_TRACKED_CB_TARGET_MASK
] = 0xffffffff;
325 ctx
->tracked_regs
.reg_value
[SI_TRACKED_CB_DCC_CONTROL
] = 0x00000000;
326 ctx
->tracked_regs
.reg_value
[SI_TRACKED_SX_PS_DOWNCONVERT
] = 0x00000000;
327 ctx
->tracked_regs
.reg_value
[SI_TRACKED_SX_BLEND_OPT_EPSILON
] = 0x00000000;
328 ctx
->tracked_regs
.reg_value
[SI_TRACKED_SX_BLEND_OPT_CONTROL
] = 0x00000000;
329 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_SC_LINE_CNTL
] = 0x00001000;
330 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_SC_AA_CONFIG
] = 0x00000000;
331 ctx
->tracked_regs
.reg_value
[SI_TRACKED_DB_EQAA
] = 0x00000000;
332 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_SC_MODE_CNTL_1
] = 0x00000000;
333 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_SU_PRIM_FILTER_CNTL
] = 0;
334 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_SU_SMALL_PRIM_FILTER_CNTL
] = 0x00000000;
335 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_CL_VS_OUT_CNTL__VS
] = 0x00000000;
336 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_CL_VS_OUT_CNTL__CL
] = 0x00000000;
337 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_CL_CLIP_CNTL
] = 0x00090000;
338 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_SC_BINNER_CNTL_0
] = 0x00000003;
339 ctx
->tracked_regs
.reg_value
[SI_TRACKED_DB_DFSM_CONTROL
] = 0x00000000;
340 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_CL_GB_VERT_CLIP_ADJ
] = 0x3f800000;
341 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_CL_GB_VERT_DISC_ADJ
] = 0x3f800000;
342 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_CL_GB_HORZ_CLIP_ADJ
] = 0x3f800000;
343 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_CL_GB_HORZ_DISC_ADJ
] = 0x3f800000;
344 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_SU_HARDWARE_SCREEN_OFFSET
] = 0;
345 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_SU_VTX_CNTL
] = 0x00000005;
346 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_SC_CLIPRECT_RULE
] = 0xffff;
347 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_SC_LINE_STIPPLE
] = 0;
348 ctx
->tracked_regs
.reg_value
[SI_TRACKED_VGT_ESGS_RING_ITEMSIZE
] = 0x00000000;
349 ctx
->tracked_regs
.reg_value
[SI_TRACKED_VGT_GSVS_RING_OFFSET_1
] = 0x00000000;
350 ctx
->tracked_regs
.reg_value
[SI_TRACKED_VGT_GSVS_RING_OFFSET_2
] = 0x00000000;
351 ctx
->tracked_regs
.reg_value
[SI_TRACKED_VGT_GSVS_RING_OFFSET_3
] = 0x00000000;
352 ctx
->tracked_regs
.reg_value
[SI_TRACKED_VGT_GSVS_RING_ITEMSIZE
] = 0x00000000;
353 ctx
->tracked_regs
.reg_value
[SI_TRACKED_VGT_GS_MAX_VERT_OUT
] = 0x00000000;
354 ctx
->tracked_regs
.reg_value
[SI_TRACKED_VGT_GS_VERT_ITEMSIZE
] = 0x00000000;
355 ctx
->tracked_regs
.reg_value
[SI_TRACKED_VGT_GS_VERT_ITEMSIZE_1
] = 0x00000000;
356 ctx
->tracked_regs
.reg_value
[SI_TRACKED_VGT_GS_VERT_ITEMSIZE_2
] = 0x00000000;
357 ctx
->tracked_regs
.reg_value
[SI_TRACKED_VGT_GS_VERT_ITEMSIZE_3
] = 0x00000000;
358 ctx
->tracked_regs
.reg_value
[SI_TRACKED_VGT_GS_INSTANCE_CNT
] = 0x00000000;
359 ctx
->tracked_regs
.reg_value
[SI_TRACKED_VGT_GS_ONCHIP_CNTL
] = 0x00000000;
360 ctx
->tracked_regs
.reg_value
[SI_TRACKED_VGT_GS_MAX_PRIMS_PER_SUBGROUP
] = 0x00000000;
361 ctx
->tracked_regs
.reg_value
[SI_TRACKED_VGT_GS_MODE
] = 0x00000000;
362 ctx
->tracked_regs
.reg_value
[SI_TRACKED_VGT_PRIMITIVEID_EN
] = 0x00000000;
363 ctx
->tracked_regs
.reg_value
[SI_TRACKED_VGT_REUSE_OFF
] = 0x00000000;
364 ctx
->tracked_regs
.reg_value
[SI_TRACKED_SPI_VS_OUT_CONFIG
] = 0x00000000;
365 ctx
->tracked_regs
.reg_value
[SI_TRACKED_GE_MAX_OUTPUT_PER_SUBGROUP
] = 0x00000000;
366 ctx
->tracked_regs
.reg_value
[SI_TRACKED_GE_NGG_SUBGRP_CNTL
] = 0x00000000;
367 ctx
->tracked_regs
.reg_value
[SI_TRACKED_SPI_SHADER_IDX_FORMAT
] = 0x00000000;
368 ctx
->tracked_regs
.reg_value
[SI_TRACKED_SPI_SHADER_POS_FORMAT
] = 0x00000000;
369 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_CL_VTE_CNTL
] = 0x00000000;
370 ctx
->tracked_regs
.reg_value
[SI_TRACKED_PA_CL_NGG_CNTL
] = 0x00000000;
371 ctx
->tracked_regs
.reg_value
[SI_TRACKED_SPI_PS_INPUT_ENA
] = 0x00000000;
372 ctx
->tracked_regs
.reg_value
[SI_TRACKED_SPI_PS_INPUT_ADDR
] = 0x00000000;
373 ctx
->tracked_regs
.reg_value
[SI_TRACKED_SPI_BARYC_CNTL
] = 0x00000000;
374 ctx
->tracked_regs
.reg_value
[SI_TRACKED_SPI_PS_IN_CONTROL
] = 0x00000002;
375 ctx
->tracked_regs
.reg_value
[SI_TRACKED_SPI_SHADER_Z_FORMAT
] = 0x00000000;
376 ctx
->tracked_regs
.reg_value
[SI_TRACKED_SPI_SHADER_COL_FORMAT
] = 0x00000000;
377 ctx
->tracked_regs
.reg_value
[SI_TRACKED_CB_SHADER_MASK
] = 0xffffffff;
378 ctx
->tracked_regs
.reg_value
[SI_TRACKED_VGT_TF_PARAM
] = 0x00000000;
379 ctx
->tracked_regs
.reg_value
[SI_TRACKED_VGT_VERTEX_REUSE_BLOCK_CNTL
] = 0x0000001e; /* From GFX8 */
381 /* Set all cleared context registers to saved. */
382 ctx
->tracked_regs
.reg_saved
= ~(1ull << SI_TRACKED_GE_PC_ALLOC
); /* uconfig reg */
383 ctx
->last_gs_out_prim
= 0; /* cleared by CLEAR_STATE */
386 void si_begin_new_gfx_cs(struct si_context
*ctx
)
389 si_begin_gfx_cs_debug(ctx
);
391 si_add_gds_to_buffer_list(ctx
);
393 /* Always invalidate caches at the beginning of IBs, because external
394 * users (e.g. BO evictions and SDMA/UVD/VCE IBs) can modify our
397 * Note that the cache flush done by the kernel at the end of GFX IBs
398 * isn't useful here, because that flush can finish after the following
401 * TODO: Do we also need to invalidate CB & DB caches?
403 ctx
->flags
|= SI_CONTEXT_INV_ICACHE
| SI_CONTEXT_INV_SCACHE
| SI_CONTEXT_INV_VCACHE
|
404 SI_CONTEXT_INV_L2
| SI_CONTEXT_START_PIPELINE_STATS
;
406 radeon_add_to_buffer_list(ctx
, ctx
->gfx_cs
, ctx
->border_color_buffer
,
407 RADEON_USAGE_READ
, RADEON_PRIO_BORDER_COLORS
);
409 ctx
->cs_shader_state
.initialized
= false;
410 si_add_all_descriptors_to_bo_list(ctx
);
411 si_shader_pointers_mark_dirty(ctx
);
413 if (!ctx
->has_graphics
) {
414 ctx
->initial_gfx_cs_size
= ctx
->gfx_cs
->current
.cdw
;
418 if (ctx
->tess_rings
) {
419 radeon_add_to_buffer_list(ctx
, ctx
->gfx_cs
, si_resource(ctx
->tess_rings
),
420 RADEON_USAGE_READWRITE
, RADEON_PRIO_SHADER_RINGS
);
423 /* set all valid group as dirty so they get reemited on
426 si_pm4_reset_emitted(ctx
);
428 /* The CS initialization should be emitted before everything else. */
429 si_pm4_emit(ctx
, ctx
->cs_preamble_state
);
430 if (ctx
->cs_preamble_gs_rings
)
431 si_pm4_emit(ctx
, ctx
->cs_preamble_gs_rings
);
433 if (ctx
->queued
.named
.ls
)
434 ctx
->prefetch_L2_mask
|= SI_PREFETCH_LS
;
435 if (ctx
->queued
.named
.hs
)
436 ctx
->prefetch_L2_mask
|= SI_PREFETCH_HS
;
437 if (ctx
->queued
.named
.es
)
438 ctx
->prefetch_L2_mask
|= SI_PREFETCH_ES
;
439 if (ctx
->queued
.named
.gs
)
440 ctx
->prefetch_L2_mask
|= SI_PREFETCH_GS
;
441 if (ctx
->queued
.named
.vs
)
442 ctx
->prefetch_L2_mask
|= SI_PREFETCH_VS
;
443 if (ctx
->queued
.named
.ps
)
444 ctx
->prefetch_L2_mask
|= SI_PREFETCH_PS
;
445 if (ctx
->vb_descriptors_buffer
&& ctx
->vertex_elements
)
446 ctx
->prefetch_L2_mask
|= SI_PREFETCH_VBO_DESCRIPTORS
;
448 /* CLEAR_STATE disables all colorbuffers, so only enable bound ones. */
449 bool has_clear_state
= ctx
->screen
->info
.has_clear_state
;
450 if (has_clear_state
) {
451 ctx
->framebuffer
.dirty_cbufs
= u_bit_consecutive(0, ctx
->framebuffer
.state
.nr_cbufs
);
452 /* CLEAR_STATE disables the zbuffer, so only enable it if it's bound. */
453 ctx
->framebuffer
.dirty_zsbuf
= ctx
->framebuffer
.state
.zsbuf
!= NULL
;
455 ctx
->framebuffer
.dirty_cbufs
= u_bit_consecutive(0, 8);
456 ctx
->framebuffer
.dirty_zsbuf
= true;
458 /* This should always be marked as dirty to set the framebuffer scissor
460 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.framebuffer
);
462 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.clip_regs
);
463 /* CLEAR_STATE sets zeros. */
464 if (!has_clear_state
|| ctx
->clip_state
.any_nonzeros
)
465 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.clip_state
);
466 ctx
->sample_locs_num_samples
= 0;
467 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.msaa_sample_locs
);
468 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.msaa_config
);
469 /* CLEAR_STATE sets 0xffff. */
470 if (!has_clear_state
|| ctx
->sample_mask
!= 0xffff)
471 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.sample_mask
);
472 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.cb_render_state
);
473 /* CLEAR_STATE sets zeros. */
474 if (!has_clear_state
|| ctx
->blend_color
.any_nonzeros
)
475 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.blend_color
);
476 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.db_render_state
);
477 if (ctx
->chip_class
>= GFX9
)
478 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.dpbb_state
);
479 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.stencil_ref
);
480 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.spi_map
);
481 if (!ctx
->screen
->use_ngg_streamout
)
482 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.streamout_enable
);
483 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.render_cond
);
484 /* CLEAR_STATE disables all window rectangles. */
485 if (!has_clear_state
|| ctx
->num_window_rectangles
> 0)
486 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.window_rectangles
);
488 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.guardband
);
489 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.scissors
);
490 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.viewports
);
492 si_mark_atom_dirty(ctx
, &ctx
->atoms
.s
.scratch_state
);
493 if (ctx
->scratch_buffer
) {
494 si_context_add_resource_size(ctx
, &ctx
->scratch_buffer
->b
.b
);
497 if (ctx
->streamout
.suspended
) {
498 ctx
->streamout
.append_bitmask
= ctx
->streamout
.enabled_mask
;
499 si_streamout_buffers_dirty(ctx
);
502 if (!list_is_empty(&ctx
->active_queries
))
503 si_resume_queries(ctx
);
505 assert(!ctx
->gfx_cs
->prev_dw
);
506 ctx
->initial_gfx_cs_size
= ctx
->gfx_cs
->current
.cdw
;
508 /* Invalidate various draw states so that they are emitted before
509 * the first draw call. */
510 si_invalidate_draw_sh_constants(ctx
);
511 ctx
->last_index_size
= -1;
512 ctx
->last_primitive_restart_en
= -1;
513 ctx
->last_restart_index
= SI_RESTART_INDEX_UNKNOWN
;
515 ctx
->last_multi_vgt_param
= -1;
516 ctx
->last_vs_state
= ~0;
518 ctx
->last_tcs
= NULL
;
519 ctx
->last_tes_sh_base
= -1;
520 ctx
->last_num_tcs_input_cp
= -1;
521 ctx
->last_ls_hs_config
= -1; /* impossible value */
522 ctx
->last_binning_enabled
= -1;
523 ctx
->small_prim_cull_info_dirty
= ctx
->small_prim_cull_info_buf
!= NULL
;
525 ctx
->prim_discard_compute_ib_initialized
= false;
527 /* Compute-based primitive discard:
528 * The index ring is divided into 2 halves. Switch between the halves
529 * in the same fashion as doublebuffering.
531 if (ctx
->index_ring_base
)
532 ctx
->index_ring_base
= 0;
534 ctx
->index_ring_base
= ctx
->index_ring_size_per_ib
;
536 ctx
->index_ring_offset
= 0;
538 if (has_clear_state
) {
539 si_set_tracked_regs_to_clear_state(ctx
);
541 /* Set all register values to unknown. */
542 ctx
->tracked_regs
.reg_saved
= 0;
543 ctx
->last_gs_out_prim
= -1; /* unknown */
546 /* 0xffffffff is a impossible value to register SPI_PS_INPUT_CNTL_n */
547 memset(ctx
->tracked_regs
.spi_ps_input_cntl
, 0xff, sizeof(uint32_t) * 32);