src/gallium/drivers/radeonsi/si_gfx_cs.c

   1 /*
   2  * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
   3  * Copyright 2018 Advanced Micro Devices, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * on the rights to use, copy, modify, merge, publish, distribute, sub
  10  * license, and/or sell copies of the Software, and to permit persons to whom
  11  * the Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  20  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  21  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  22  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  23  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  24  */
  25
  26 #include "si_build_pm4.h"
  27 #include "si_pipe.h"
  28 #include "sid.h"
  29 #include "util/os_time.h"
  30 #include "util/u_upload_mgr.h"
  31
  32 /* initialize */
  33 void si_need_gfx_cs_space(struct si_context *ctx)
  34 {
  35    struct radeon_cmdbuf *cs = ctx->gfx_cs;
  36
  37    /* There is no need to flush the DMA IB here, because
  38     * si_need_dma_space always flushes the GFX IB if there is
  39     * a conflict, which means any unflushed DMA commands automatically
  40     * precede the GFX IB (= they had no dependency on the GFX IB when
  41     * they were submitted).
  42     */
  43
  44    /* There are two memory usage counters in the winsys for all buffers
  45     * that have been added (cs_add_buffer) and two counters in the pipe
  46     * driver for those that haven't been added yet.
  47     */
  48    if (unlikely(!radeon_cs_memory_below_limit(ctx->screen, ctx->gfx_cs, ctx->vram, ctx->gtt))) {
  49       ctx->gtt = 0;
  50       ctx->vram = 0;
  51       si_flush_gfx_cs(ctx, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW, NULL);
  52       return;
  53    }
  54    ctx->gtt = 0;
  55    ctx->vram = 0;
  56
  57    unsigned need_dwords = si_get_minimum_num_gfx_cs_dwords(ctx);
  58    if (!ctx->ws->cs_check_space(cs, need_dwords, false))
  59       si_flush_gfx_cs(ctx, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW, NULL);
  60 }
  61
  62 void si_unref_sdma_uploads(struct si_context *sctx)
  63 {
  64    for (unsigned i = 0; i < sctx->num_sdma_uploads; i++) {
  65       si_resource_reference(&sctx->sdma_uploads[i].dst, NULL);
  66       si_resource_reference(&sctx->sdma_uploads[i].src, NULL);
  67    }
  68    sctx->num_sdma_uploads = 0;
  69 }
  70
  71 void si_flush_gfx_cs(struct si_context *ctx, unsigned flags, struct pipe_fence_handle **fence)
  72 {
  73    struct radeon_cmdbuf *cs = ctx->gfx_cs;
  74    struct radeon_winsys *ws = ctx->ws;
  75    const unsigned wait_ps_cs = SI_CONTEXT_PS_PARTIAL_FLUSH | SI_CONTEXT_CS_PARTIAL_FLUSH;
  76    unsigned wait_flags = 0;
  77
  78    if (ctx->gfx_flush_in_progress)
  79       return;
  80
  81    if (!ctx->screen->info.kernel_flushes_tc_l2_after_ib) {
  82       wait_flags |= wait_ps_cs | SI_CONTEXT_INV_L2;
  83    } else if (ctx->chip_class == GFX6) {
  84       /* The kernel flushes L2 before shaders are finished. */
  85       wait_flags |= wait_ps_cs;
  86    } else if (!(flags & RADEON_FLUSH_START_NEXT_GFX_IB_NOW)) {
  87       wait_flags |= wait_ps_cs;
  88    }
  89
  90    /* Drop this flush if it's a no-op. */
  91    if (!radeon_emitted(cs, ctx->initial_gfx_cs_size) && (!wait_flags || !ctx->gfx_last_ib_is_busy))
  92       return;
  93
  94    if (ctx->b.get_device_reset_status(&ctx->b) != PIPE_NO_RESET)
  95       return;
  96
  97    if (ctx->screen->debug_flags & DBG(CHECK_VM))
  98       flags &= ~PIPE_FLUSH_ASYNC;
  99
 100    ctx->gfx_flush_in_progress = true;
 101
 102         /* If the gallium frontend is flushing the GFX IB, si_flush_from_st is
 103     * responsible for flushing the DMA IB and merging the fences from both.
 104     * If the driver flushes the GFX IB internally, and it should never ask
 105     * for a fence handle.
 106     */
 107    assert(!radeon_emitted(ctx->sdma_cs, 0) || fence == NULL);
 108
 109    /* Update the sdma_uploads list by flushing the uploader. */
 110    u_upload_unmap(ctx->b.const_uploader);
 111
 112    /* Execute SDMA uploads. */
 113    ctx->sdma_uploads_in_progress = true;
 114    for (unsigned i = 0; i < ctx->num_sdma_uploads; i++) {
 115       struct si_sdma_upload *up = &ctx->sdma_uploads[i];
 116
 117       assert(up->src_offset % 4 == 0 && up->dst_offset % 4 == 0 && up->size % 4 == 0);
 118
 119       si_sdma_copy_buffer(ctx, &up->dst->b.b, &up->src->b.b, up->dst_offset, up->src_offset,
 120                           up->size);
 121    }
 122    ctx->sdma_uploads_in_progress = false;
 123    si_unref_sdma_uploads(ctx);
 124
 125    /* Flush SDMA (preamble IB). */
 126    if (radeon_emitted(ctx->sdma_cs, 0))
 127       si_flush_dma_cs(ctx, flags, NULL);
 128
 129    if (radeon_emitted(ctx->prim_discard_compute_cs, 0)) {
 130       struct radeon_cmdbuf *compute_cs = ctx->prim_discard_compute_cs;
 131       si_compute_signal_gfx(ctx);
 132
 133       /* Make sure compute shaders are idle before leaving the IB, so that
 134        * the next IB doesn't overwrite GDS that might be in use. */
 135       radeon_emit(compute_cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
 136       radeon_emit(compute_cs, EVENT_TYPE(V_028A90_CS_PARTIAL_FLUSH) | EVENT_INDEX(4));
 137
 138       /* Save the GDS prim restart counter if needed. */
 139       if (ctx->preserve_prim_restart_gds_at_flush) {
 140          si_cp_copy_data(ctx, compute_cs, COPY_DATA_DST_MEM, ctx->wait_mem_scratch, 4,
 141                          COPY_DATA_GDS, NULL, 4);
 142       }
 143    }
 144
 145    if (ctx->has_graphics) {
 146       if (!list_is_empty(&ctx->active_queries))
 147          si_suspend_queries(ctx);
 148
 149       ctx->streamout.suspended = false;
 150       if (ctx->streamout.begin_emitted) {
 151          si_emit_streamout_end(ctx);
 152          ctx->streamout.suspended = true;
 153
 154          /* Since NGG streamout uses GDS, we need to make GDS
 155           * idle when we leave the IB, otherwise another process
 156           * might overwrite it while our shaders are busy.
 157           */
 158          if (ctx->screen->use_ngg_streamout)
 159             wait_flags |= SI_CONTEXT_PS_PARTIAL_FLUSH;
 160       }
 161    }
 162
 163    /* Make sure CP DMA is idle at the end of IBs after L2 prefetches
 164     * because the kernel doesn't wait for it. */
 165    if (ctx->chip_class >= GFX7)
 166       si_cp_dma_wait_for_idle(ctx);
 167
 168    /* Wait for draw calls to finish if needed. */
 169    if (wait_flags) {
 170       ctx->flags |= wait_flags;
 171       ctx->emit_cache_flush(ctx);
 172    }
 173    ctx->gfx_last_ib_is_busy = (wait_flags & wait_ps_cs) != wait_ps_cs;
 174
 175    if (ctx->current_saved_cs) {
 176       si_trace_emit(ctx);
 177
 178       /* Save the IB for debug contexts. */
 179       si_save_cs(ws, cs, &ctx->current_saved_cs->gfx, true);
 180       ctx->current_saved_cs->flushed = true;
 181       ctx->current_saved_cs->time_flush = os_time_get_nano();
 182
 183       si_log_hw_flush(ctx);
 184    }
 185
 186    if (si_compute_prim_discard_enabled(ctx)) {
 187       /* The compute IB can start after the previous gfx IB starts. */
 188       if (radeon_emitted(ctx->prim_discard_compute_cs, 0) && ctx->last_gfx_fence) {
 189          ctx->ws->cs_add_fence_dependency(
 190             ctx->gfx_cs, ctx->last_gfx_fence,
 191             RADEON_DEPENDENCY_PARALLEL_COMPUTE_ONLY | RADEON_DEPENDENCY_START_FENCE);
 192       }
 193
 194       /* Remember the last execution barrier. It's in the IB.
 195        * It will signal the start of the next compute IB.
 196        */
 197       if (flags & RADEON_FLUSH_START_NEXT_GFX_IB_NOW && ctx->last_pkt3_write_data) {
 198          *ctx->last_pkt3_write_data = PKT3(PKT3_WRITE_DATA, 3, 0);
 199          ctx->last_pkt3_write_data = NULL;
 200
 201          si_resource_reference(&ctx->last_ib_barrier_buf, ctx->barrier_buf);
 202          ctx->last_ib_barrier_buf_offset = ctx->barrier_buf_offset;
 203          si_resource_reference(&ctx->barrier_buf, NULL);
 204
 205          ws->fence_reference(&ctx->last_ib_barrier_fence, NULL);
 206       }
 207    }
 208
 209    /* Flush the CS. */
 210    ws->cs_flush(cs, flags, &ctx->last_gfx_fence);
 211    if (fence)
 212       ws->fence_reference(fence, ctx->last_gfx_fence);
 213
 214    ctx->num_gfx_cs_flushes++;
 215
 216    if (si_compute_prim_discard_enabled(ctx)) {
 217       /* Remember the last execution barrier, which is the last fence
 218        * in this case.
 219        */
 220       if (!(flags & RADEON_FLUSH_START_NEXT_GFX_IB_NOW)) {
 221          ctx->last_pkt3_write_data = NULL;
 222          si_resource_reference(&ctx->last_ib_barrier_buf, NULL);
 223          ws->fence_reference(&ctx->last_ib_barrier_fence, ctx->last_gfx_fence);
 224       }
 225    }
 226
 227    /* Check VM faults if needed. */
 228    if (ctx->screen->debug_flags & DBG(CHECK_VM)) {
 229       /* Use conservative timeout 800ms, after which we won't wait any
 230        * longer and assume the GPU is hung.
 231        */
 232       ctx->ws->fence_wait(ctx->ws, ctx->last_gfx_fence, 800 * 1000 * 1000);
 233
 234       si_check_vm_faults(ctx, &ctx->current_saved_cs->gfx, RING_GFX);
 235    }
 236
 237    if (ctx->current_saved_cs)
 238       si_saved_cs_reference(&ctx->current_saved_cs, NULL);
 239
 240    si_begin_new_gfx_cs(ctx);
 241    ctx->gfx_flush_in_progress = false;
 242 }
 243
 244 static void si_begin_gfx_cs_debug(struct si_context *ctx)
 245 {
 246    static const uint32_t zeros[1];
 247    assert(!ctx->current_saved_cs);
 248
 249    ctx->current_saved_cs = calloc(1, sizeof(*ctx->current_saved_cs));
 250    if (!ctx->current_saved_cs)
 251       return;
 252
 253    pipe_reference_init(&ctx->current_saved_cs->reference, 1);
 254
 255    ctx->current_saved_cs->trace_buf =
 256       si_resource(pipe_buffer_create(ctx->b.screen, 0, PIPE_USAGE_STAGING, 8));
 257    if (!ctx->current_saved_cs->trace_buf) {
 258       free(ctx->current_saved_cs);
 259       ctx->current_saved_cs = NULL;
 260       return;
 261    }
 262
 263    pipe_buffer_write_nooverlap(&ctx->b, &ctx->current_saved_cs->trace_buf->b.b, 0, sizeof(zeros),
 264                                zeros);
 265    ctx->current_saved_cs->trace_id = 0;
 266
 267    si_trace_emit(ctx);
 268
 269    radeon_add_to_buffer_list(ctx, ctx->gfx_cs, ctx->current_saved_cs->trace_buf,
 270                              RADEON_USAGE_READWRITE, RADEON_PRIO_TRACE);
 271 }
 272
 273 static void si_add_gds_to_buffer_list(struct si_context *sctx)
 274 {
 275    if (sctx->gds) {
 276       sctx->ws->cs_add_buffer(sctx->gfx_cs, sctx->gds, RADEON_USAGE_READWRITE, 0, 0);
 277       if (sctx->gds_oa) {
 278          sctx->ws->cs_add_buffer(sctx->gfx_cs, sctx->gds_oa, RADEON_USAGE_READWRITE, 0, 0);
 279       }
 280    }
 281 }
 282
 283 void si_allocate_gds(struct si_context *sctx)
 284 {
 285    struct radeon_winsys *ws = sctx->ws;
 286
 287    if (sctx->gds)
 288       return;
 289
 290    assert(sctx->screen->use_ngg_streamout);
 291
 292    /* 4 streamout GDS counters.
 293     * We need 256B (64 dw) of GDS, otherwise streamout hangs.
 294     */
 295    sctx->gds = ws->buffer_create(ws, 256, 4, RADEON_DOMAIN_GDS, 0);
 296    sctx->gds_oa = ws->buffer_create(ws, 4, 1, RADEON_DOMAIN_OA, 0);
 297
 298    assert(sctx->gds && sctx->gds_oa);
 299    si_add_gds_to_buffer_list(sctx);
 300 }
 301
 302 void si_begin_new_gfx_cs(struct si_context *ctx)
 303 {
 304    if (ctx->is_debug)
 305       si_begin_gfx_cs_debug(ctx);
 306
 307    si_add_gds_to_buffer_list(ctx);
 308
 309    /* Always invalidate caches at the beginning of IBs, because external
 310     * users (e.g. BO evictions and SDMA/UVD/VCE IBs) can modify our
 311     * buffers.
 312     *
 313     * Note that the cache flush done by the kernel at the end of GFX IBs
 314     * isn't useful here, because that flush can finish after the following
 315     * IB starts drawing.
 316     *
 317     * TODO: Do we also need to invalidate CB & DB caches?
 318     */
 319    ctx->flags |= SI_CONTEXT_INV_ICACHE | SI_CONTEXT_INV_SCACHE | SI_CONTEXT_INV_VCACHE |
 320                  SI_CONTEXT_INV_L2 | SI_CONTEXT_START_PIPELINE_STATS;
 321
 322    ctx->cs_shader_state.initialized = false;
 323    si_all_descriptors_begin_new_cs(ctx);
 324
 325    if (!ctx->has_graphics) {
 326       ctx->initial_gfx_cs_size = ctx->gfx_cs->current.cdw;
 327       return;
 328    }
 329
 330    /* set all valid group as dirty so they get reemited on
 331     * next draw command
 332     */
 333    si_pm4_reset_emitted(ctx);
 334
 335    /* The CS initialization should be emitted before everything else. */
 336    si_pm4_emit(ctx, ctx->init_config);
 337    if (ctx->init_config_gs_rings)
 338       si_pm4_emit(ctx, ctx->init_config_gs_rings);
 339
 340    if (ctx->queued.named.ls)
 341       ctx->prefetch_L2_mask |= SI_PREFETCH_LS;
 342    if (ctx->queued.named.hs)
 343       ctx->prefetch_L2_mask |= SI_PREFETCH_HS;
 344    if (ctx->queued.named.es)
 345       ctx->prefetch_L2_mask |= SI_PREFETCH_ES;
 346    if (ctx->queued.named.gs)
 347       ctx->prefetch_L2_mask |= SI_PREFETCH_GS;
 348    if (ctx->queued.named.vs)
 349       ctx->prefetch_L2_mask |= SI_PREFETCH_VS;
 350    if (ctx->queued.named.ps)
 351       ctx->prefetch_L2_mask |= SI_PREFETCH_PS;
 352    if (ctx->vb_descriptors_buffer && ctx->vertex_elements)
 353       ctx->prefetch_L2_mask |= SI_PREFETCH_VBO_DESCRIPTORS;
 354
 355    /* CLEAR_STATE disables all colorbuffers, so only enable bound ones. */
 356    bool has_clear_state = ctx->screen->info.has_clear_state;
 357    if (has_clear_state) {
 358       ctx->framebuffer.dirty_cbufs = u_bit_consecutive(0, ctx->framebuffer.state.nr_cbufs);
 359       /* CLEAR_STATE disables the zbuffer, so only enable it if it's bound. */
 360       ctx->framebuffer.dirty_zsbuf = ctx->framebuffer.state.zsbuf != NULL;
 361    } else {
 362       ctx->framebuffer.dirty_cbufs = u_bit_consecutive(0, 8);
 363       ctx->framebuffer.dirty_zsbuf = true;
 364    }
 365    /* This should always be marked as dirty to set the framebuffer scissor
 366     * at least. */
 367    si_mark_atom_dirty(ctx, &ctx->atoms.s.framebuffer);
 368
 369    si_mark_atom_dirty(ctx, &ctx->atoms.s.clip_regs);
 370    /* CLEAR_STATE sets zeros. */
 371    if (!has_clear_state || ctx->clip_state.any_nonzeros)
 372       si_mark_atom_dirty(ctx, &ctx->atoms.s.clip_state);
 373    ctx->sample_locs_num_samples = 0;
 374    si_mark_atom_dirty(ctx, &ctx->atoms.s.msaa_sample_locs);
 375    si_mark_atom_dirty(ctx, &ctx->atoms.s.msaa_config);
 376    /* CLEAR_STATE sets 0xffff. */
 377    if (!has_clear_state || ctx->sample_mask != 0xffff)
 378       si_mark_atom_dirty(ctx, &ctx->atoms.s.sample_mask);
 379    si_mark_atom_dirty(ctx, &ctx->atoms.s.cb_render_state);
 380    /* CLEAR_STATE sets zeros. */
 381    if (!has_clear_state || ctx->blend_color.any_nonzeros)
 382       si_mark_atom_dirty(ctx, &ctx->atoms.s.blend_color);
 383    si_mark_atom_dirty(ctx, &ctx->atoms.s.db_render_state);
 384    if (ctx->chip_class >= GFX9)
 385       si_mark_atom_dirty(ctx, &ctx->atoms.s.dpbb_state);
 386    si_mark_atom_dirty(ctx, &ctx->atoms.s.stencil_ref);
 387    si_mark_atom_dirty(ctx, &ctx->atoms.s.spi_map);
 388    if (!ctx->screen->use_ngg_streamout)
 389       si_mark_atom_dirty(ctx, &ctx->atoms.s.streamout_enable);
 390    si_mark_atom_dirty(ctx, &ctx->atoms.s.render_cond);
 391    /* CLEAR_STATE disables all window rectangles. */
 392    if (!has_clear_state || ctx->num_window_rectangles > 0)
 393       si_mark_atom_dirty(ctx, &ctx->atoms.s.window_rectangles);
 394
 395    si_mark_atom_dirty(ctx, &ctx->atoms.s.guardband);
 396    si_mark_atom_dirty(ctx, &ctx->atoms.s.scissors);
 397    si_mark_atom_dirty(ctx, &ctx->atoms.s.viewports);
 398
 399    si_mark_atom_dirty(ctx, &ctx->atoms.s.scratch_state);
 400    if (ctx->scratch_buffer) {
 401       si_context_add_resource_size(ctx, &ctx->scratch_buffer->b.b);
 402    }
 403
 404    if (ctx->streamout.suspended) {
 405       ctx->streamout.append_bitmask = ctx->streamout.enabled_mask;
 406       si_streamout_buffers_dirty(ctx);
 407    }
 408
 409    if (!list_is_empty(&ctx->active_queries))
 410       si_resume_queries(ctx);
 411
 412    assert(!ctx->gfx_cs->prev_dw);
 413    ctx->initial_gfx_cs_size = ctx->gfx_cs->current.cdw;
 414
 415    /* Invalidate various draw states so that they are emitted before
 416     * the first draw call. */
 417    si_invalidate_draw_sh_constants(ctx);
 418    ctx->last_index_size = -1;
 419    ctx->last_primitive_restart_en = -1;
 420    ctx->last_restart_index = SI_RESTART_INDEX_UNKNOWN;
 421    ctx->last_prim = -1;
 422    ctx->last_multi_vgt_param = -1;
 423    ctx->last_vs_state = ~0;
 424    ctx->last_ls = NULL;
 425    ctx->last_tcs = NULL;
 426    ctx->last_tes_sh_base = -1;
 427    ctx->last_num_tcs_input_cp = -1;
 428    ctx->last_ls_hs_config = -1; /* impossible value */
 429    ctx->last_binning_enabled = -1;
 430    ctx->small_prim_cull_info_dirty = ctx->small_prim_cull_info_buf != NULL;
 431
 432    ctx->prim_discard_compute_ib_initialized = false;
 433
 434    /* Compute-based primitive discard:
 435     *   The index ring is divided into 2 halves. Switch between the halves
 436     *   in the same fashion as doublebuffering.
 437     */
 438    if (ctx->index_ring_base)
 439       ctx->index_ring_base = 0;
 440    else
 441       ctx->index_ring_base = ctx->index_ring_size_per_ib;
 442
 443    ctx->index_ring_offset = 0;
 444
 445    STATIC_ASSERT(SI_NUM_TRACKED_REGS <= sizeof(ctx->tracked_regs.reg_saved) * 8);
 446
 447    if (has_clear_state) {
 448       ctx->tracked_regs.reg_value[SI_TRACKED_DB_RENDER_CONTROL] = 0x00000000;
 449       ctx->tracked_regs.reg_value[SI_TRACKED_DB_COUNT_CONTROL] = 0x00000000;
 450       ctx->tracked_regs.reg_value[SI_TRACKED_DB_RENDER_OVERRIDE2] = 0x00000000;
 451       ctx->tracked_regs.reg_value[SI_TRACKED_DB_SHADER_CONTROL] = 0x00000000;
 452       ctx->tracked_regs.reg_value[SI_TRACKED_CB_TARGET_MASK] = 0xffffffff;
 453       ctx->tracked_regs.reg_value[SI_TRACKED_CB_DCC_CONTROL] = 0x00000000;
 454       ctx->tracked_regs.reg_value[SI_TRACKED_SX_PS_DOWNCONVERT] = 0x00000000;
 455       ctx->tracked_regs.reg_value[SI_TRACKED_SX_BLEND_OPT_EPSILON] = 0x00000000;
 456       ctx->tracked_regs.reg_value[SI_TRACKED_SX_BLEND_OPT_CONTROL] = 0x00000000;
 457       ctx->tracked_regs.reg_value[SI_TRACKED_PA_SC_LINE_CNTL] = 0x00001000;
 458       ctx->tracked_regs.reg_value[SI_TRACKED_PA_SC_AA_CONFIG] = 0x00000000;
 459       ctx->tracked_regs.reg_value[SI_TRACKED_DB_EQAA] = 0x00000000;
 460       ctx->tracked_regs.reg_value[SI_TRACKED_PA_SC_MODE_CNTL_1] = 0x00000000;
 461       ctx->tracked_regs.reg_value[SI_TRACKED_PA_SU_PRIM_FILTER_CNTL] = 0;
 462       ctx->tracked_regs.reg_value[SI_TRACKED_PA_SU_SMALL_PRIM_FILTER_CNTL] = 0x00000000;
 463       ctx->tracked_regs.reg_value[SI_TRACKED_PA_CL_VS_OUT_CNTL__VS] = 0x00000000;
 464       ctx->tracked_regs.reg_value[SI_TRACKED_PA_CL_VS_OUT_CNTL__CL] = 0x00000000;
 465       ctx->tracked_regs.reg_value[SI_TRACKED_PA_CL_CLIP_CNTL] = 0x00090000;
 466       ctx->tracked_regs.reg_value[SI_TRACKED_PA_SC_BINNER_CNTL_0] = 0x00000003;
 467       ctx->tracked_regs.reg_value[SI_TRACKED_DB_DFSM_CONTROL] = 0x00000000;
 468       ctx->tracked_regs.reg_value[SI_TRACKED_PA_CL_GB_VERT_CLIP_ADJ] = 0x3f800000;
 469       ctx->tracked_regs.reg_value[SI_TRACKED_PA_CL_GB_VERT_DISC_ADJ] = 0x3f800000;
 470       ctx->tracked_regs.reg_value[SI_TRACKED_PA_CL_GB_HORZ_CLIP_ADJ] = 0x3f800000;
 471       ctx->tracked_regs.reg_value[SI_TRACKED_PA_CL_GB_HORZ_DISC_ADJ] = 0x3f800000;
 472       ctx->tracked_regs.reg_value[SI_TRACKED_PA_SU_HARDWARE_SCREEN_OFFSET] = 0;
 473       ctx->tracked_regs.reg_value[SI_TRACKED_PA_SU_VTX_CNTL] = 0x00000005;
 474       ctx->tracked_regs.reg_value[SI_TRACKED_PA_SC_CLIPRECT_RULE] = 0xffff;
 475       ctx->tracked_regs.reg_value[SI_TRACKED_PA_SC_LINE_STIPPLE] = 0;
 476       ctx->tracked_regs.reg_value[SI_TRACKED_VGT_ESGS_RING_ITEMSIZE] = 0x00000000;
 477       ctx->tracked_regs.reg_value[SI_TRACKED_VGT_GSVS_RING_OFFSET_1] = 0x00000000;
 478       ctx->tracked_regs.reg_value[SI_TRACKED_VGT_GSVS_RING_OFFSET_2] = 0x00000000;
 479       ctx->tracked_regs.reg_value[SI_TRACKED_VGT_GSVS_RING_OFFSET_3] = 0x00000000;
 480       ctx->tracked_regs.reg_value[SI_TRACKED_VGT_GSVS_RING_ITEMSIZE] = 0x00000000;
 481       ctx->tracked_regs.reg_value[SI_TRACKED_VGT_GS_MAX_VERT_OUT] = 0x00000000;
 482       ctx->tracked_regs.reg_value[SI_TRACKED_VGT_GS_VERT_ITEMSIZE] = 0x00000000;
 483       ctx->tracked_regs.reg_value[SI_TRACKED_VGT_GS_VERT_ITEMSIZE_1] = 0x00000000;
 484       ctx->tracked_regs.reg_value[SI_TRACKED_VGT_GS_VERT_ITEMSIZE_2] = 0x00000000;
 485       ctx->tracked_regs.reg_value[SI_TRACKED_VGT_GS_VERT_ITEMSIZE_3] = 0x00000000;
 486       ctx->tracked_regs.reg_value[SI_TRACKED_VGT_GS_INSTANCE_CNT] = 0x00000000;
 487       ctx->tracked_regs.reg_value[SI_TRACKED_VGT_GS_ONCHIP_CNTL] = 0x00000000;
 488       ctx->tracked_regs.reg_value[SI_TRACKED_VGT_GS_MAX_PRIMS_PER_SUBGROUP] = 0x00000000;
 489       ctx->tracked_regs.reg_value[SI_TRACKED_VGT_GS_MODE] = 0x00000000;
 490       ctx->tracked_regs.reg_value[SI_TRACKED_VGT_PRIMITIVEID_EN] = 0x00000000;
 491       ctx->tracked_regs.reg_value[SI_TRACKED_VGT_REUSE_OFF] = 0x00000000;
 492       ctx->tracked_regs.reg_value[SI_TRACKED_SPI_VS_OUT_CONFIG] = 0x00000000;
 493       ctx->tracked_regs.reg_value[SI_TRACKED_GE_MAX_OUTPUT_PER_SUBGROUP] = 0x00000000;
 494       ctx->tracked_regs.reg_value[SI_TRACKED_GE_NGG_SUBGRP_CNTL] = 0x00000000;
 495       ctx->tracked_regs.reg_value[SI_TRACKED_SPI_SHADER_IDX_FORMAT] = 0x00000000;
 496       ctx->tracked_regs.reg_value[SI_TRACKED_SPI_SHADER_POS_FORMAT] = 0x00000000;
 497       ctx->tracked_regs.reg_value[SI_TRACKED_PA_CL_VTE_CNTL] = 0x00000000;
 498       ctx->tracked_regs.reg_value[SI_TRACKED_PA_CL_NGG_CNTL] = 0x00000000;
 499       ctx->tracked_regs.reg_value[SI_TRACKED_SPI_PS_INPUT_ENA] = 0x00000000;
 500       ctx->tracked_regs.reg_value[SI_TRACKED_SPI_PS_INPUT_ADDR] = 0x00000000;
 501       ctx->tracked_regs.reg_value[SI_TRACKED_SPI_BARYC_CNTL] = 0x00000000;
 502       ctx->tracked_regs.reg_value[SI_TRACKED_SPI_PS_IN_CONTROL] = 0x00000002;
 503       ctx->tracked_regs.reg_value[SI_TRACKED_SPI_SHADER_Z_FORMAT] = 0x00000000;
 504       ctx->tracked_regs.reg_value[SI_TRACKED_SPI_SHADER_COL_FORMAT] = 0x00000000;
 505       ctx->tracked_regs.reg_value[SI_TRACKED_CB_SHADER_MASK] = 0xffffffff;
 506       ctx->tracked_regs.reg_value[SI_TRACKED_VGT_TF_PARAM] = 0x00000000;
 507       ctx->tracked_regs.reg_value[SI_TRACKED_VGT_VERTEX_REUSE_BLOCK_CNTL] =
 508          0x0000001e; /* From GFX8 */
 509
 510       /* Set all cleared context registers to saved. */
 511       ctx->tracked_regs.reg_saved = ~(1ull << SI_TRACKED_GE_PC_ALLOC); /* uconfig reg */
 512       ctx->last_gs_out_prim = 0;                                       /* cleared by CLEAR_STATE */
 513    } else {
 514       /* Set all register values to unknown. */
 515       ctx->tracked_regs.reg_saved = 0;
 516       ctx->last_gs_out_prim = -1; /* unknown */
 517    }
 518
 519    /* 0xffffffff is a impossible value to register SPI_PS_INPUT_CNTL_n */
 520    memset(ctx->tracked_regs.spi_ps_input_cntl, 0xff, sizeof(uint32_t) * 32);
 521 }