amd/rtld: layout and relocate LDS symbols
[mesa.git] / src / gallium / drivers / radeonsi / si_state_shaders.c
index fb0a2f1d190166aaa5899e253f63fe00cefef4b5..e90884c898a9e5cd789c3bb9c4415dfe113f720c 100644 (file)
@@ -23,7 +23,7 @@
  */
 
 #include "si_build_pm4.h"
-#include "gfx9d.h"
+#include "sid.h"
 
 #include "compiler/nir/nir_serialize.h"
 #include "tgsi/tgsi_parse.h"
@@ -127,21 +127,21 @@ static uint32_t *read_chunk(uint32_t *ptr, void **data, unsigned *size)
 static void *si_get_shader_binary(struct si_shader *shader)
 {
        /* There is always a size of data followed by the data itself. */
-       unsigned relocs_size = shader->binary.reloc_count *
-                              sizeof(shader->binary.relocs[0]);
-       unsigned disasm_size = shader->binary.disasm_string ?
-                              strlen(shader->binary.disasm_string) + 1 : 0;
        unsigned llvm_ir_size = shader->binary.llvm_ir_string ?
                                strlen(shader->binary.llvm_ir_string) + 1 : 0;
+
+       /* Refuse to allocate overly large buffers and guard against integer
+        * overflow. */
+       if (shader->binary.elf_size > UINT_MAX / 4 ||
+           llvm_ir_size > UINT_MAX / 4)
+               return NULL;
+
        unsigned size =
                4 + /* total size */
                4 + /* CRC32 of the data below */
                align(sizeof(shader->config), 4) +
                align(sizeof(shader->info), 4) +
-               4 + align(shader->binary.code_size, 4) +
-               4 + align(shader->binary.rodata_size, 4) +
-               4 + align(relocs_size, 4) +
-               4 + align(disasm_size, 4) +
+               4 + align(shader->binary.elf_size, 4) +
                4 + align(llvm_ir_size, 4);
        void *buffer = CALLOC(1, size);
        uint32_t *ptr = (uint32_t*)buffer;
@@ -154,10 +154,7 @@ static void *si_get_shader_binary(struct si_shader *shader)
 
        ptr = write_data(ptr, &shader->config, sizeof(shader->config));
        ptr = write_data(ptr, &shader->info, sizeof(shader->info));
-       ptr = write_chunk(ptr, shader->binary.code, shader->binary.code_size);
-       ptr = write_chunk(ptr, shader->binary.rodata, shader->binary.rodata_size);
-       ptr = write_chunk(ptr, shader->binary.relocs, relocs_size);
-       ptr = write_chunk(ptr, shader->binary.disasm_string, disasm_size);
+       ptr = write_chunk(ptr, shader->binary.elf_buffer, shader->binary.elf_size);
        ptr = write_chunk(ptr, shader->binary.llvm_ir_string, llvm_ir_size);
        assert((char *)ptr - (char *)buffer == size);
 
@@ -175,6 +172,7 @@ static bool si_load_shader_binary(struct si_shader *shader, void *binary)
        uint32_t size = *ptr++;
        uint32_t crc32 = *ptr++;
        unsigned chunk_size;
+       unsigned elf_size;
 
        if (util_hash_crc32(ptr, size - 8) != crc32) {
                fprintf(stderr, "radeonsi: binary shader has invalid CRC32\n");
@@ -183,13 +181,9 @@ static bool si_load_shader_binary(struct si_shader *shader, void *binary)
 
        ptr = read_data(ptr, &shader->config, sizeof(shader->config));
        ptr = read_data(ptr, &shader->info, sizeof(shader->info));
-       ptr = read_chunk(ptr, (void**)&shader->binary.code,
-                        &shader->binary.code_size);
-       ptr = read_chunk(ptr, (void**)&shader->binary.rodata,
-                        &shader->binary.rodata_size);
-       ptr = read_chunk(ptr, (void**)&shader->binary.relocs, &chunk_size);
-       shader->binary.reloc_count = chunk_size / sizeof(shader->binary.relocs[0]);
-       ptr = read_chunk(ptr, (void**)&shader->binary.disasm_string, &chunk_size);
+       ptr = read_chunk(ptr, (void**)&shader->binary.elf_buffer,
+                        &elf_size);
+       shader->binary.elf_size = elf_size;
        ptr = read_chunk(ptr, (void**)&shader->binary.llvm_ir_string, &chunk_size);
 
        return true;
@@ -337,10 +331,10 @@ void si_destroy_shader_cache(struct si_screen *sscreen)
 /* SHADER STATES */
 
 static void si_set_tesseval_regs(struct si_screen *sscreen,
-                                struct si_shader_selector *tes,
+                                const struct si_shader_selector *tes,
                                 struct si_pm4_state *pm4)
 {
-       struct tgsi_shader_info *info = &tes->info;
+       const struct tgsi_shader_info *info = &tes->info;
        unsigned tes_prim_mode = info->properties[TGSI_PROPERTY_TES_PRIM_MODE];
        unsigned tes_spacing = info->properties[TGSI_PROPERTY_TES_SPACING];
        bool tes_vertex_order_cw = info->properties[TGSI_PROPERTY_TES_VERTEX_ORDER_CW];
@@ -396,11 +390,11 @@ static void si_set_tesseval_regs(struct si_screen *sscreen,
        } else
                distribution_mode = V_028B6C_DISTRIBUTION_MODE_NO_DIST;
 
-       si_pm4_set_reg(pm4, R_028B6C_VGT_TF_PARAM,
-                      S_028B6C_TYPE(type) |
-                      S_028B6C_PARTITIONING(partitioning) |
-                      S_028B6C_TOPOLOGY(topology) |
-                      S_028B6C_DISTRIBUTION_MODE(distribution_mode));
+       assert(pm4->shader);
+       pm4->shader->vgt_tf_param = S_028B6C_TYPE(type) |
+                                   S_028B6C_PARTITIONING(partitioning) |
+                                   S_028B6C_TOPOLOGY(topology) |
+                                   S_028B6C_DISTRIBUTION_MODE(distribution_mode);
 }
 
 /* Polaris needs different VTX_REUSE_DEPTH settings depending on
@@ -440,8 +434,8 @@ static void polaris_set_vgt_vertex_reuse(struct si_screen *sscreen,
                    PIPE_TESS_SPACING_FRACTIONAL_ODD)
                        vtx_reuse_depth = 14;
 
-               si_pm4_set_reg(pm4, R_028C58_VGT_VERTEX_REUSE_BLOCK_CNTL,
-                              vtx_reuse_depth);
+               assert(pm4->shader);
+               pm4->shader->vgt_vertex_reuse_block_cntl = vtx_reuse_depth;
        }
 }
 
@@ -464,12 +458,7 @@ static struct si_pm4_state *si_get_shader_pm4_state(struct si_shader *shader)
 static unsigned si_get_num_vs_user_sgprs(unsigned num_always_on_user_sgprs)
 {
        /* Add the pointer to VBO descriptors. */
-       if (HAVE_32BIT_POINTERS) {
-               return num_always_on_user_sgprs + 1;
-       } else {
-               assert(num_always_on_user_sgprs % 2 == 0);
-               return num_always_on_user_sgprs + 2;
-       }
+       return num_always_on_user_sgprs + 1;
 }
 
 static void si_shader_ls(struct si_screen *sscreen, struct si_shader *shader)
@@ -478,7 +467,7 @@ static void si_shader_ls(struct si_screen *sscreen, struct si_shader *shader)
        unsigned vgpr_comp_cnt;
        uint64_t va;
 
-       assert(sscreen->info.chip_class <= VI);
+       assert(sscreen->info.chip_class <= GFX8);
 
        pm4 = si_get_shader_pm4_state(shader);
        if (!pm4)
@@ -552,7 +541,7 @@ static void si_shader_hs(struct si_screen *sscreen, struct si_shader *shader)
                       S_00B428_FLOAT_MODE(shader->config.float_mode) |
                       S_00B428_LS_VGPR_COMP_CNT(ls_vgpr_comp_cnt));
 
-       if (sscreen->info.chip_class <= VI) {
+       if (sscreen->info.chip_class <= GFX8) {
                si_pm4_set_reg(pm4, R_00B42C_SPI_SHADER_PGM_RSRC2_HS,
                               shader->config.rsrc2);
        }
@@ -561,6 +550,7 @@ static void si_shader_hs(struct si_screen *sscreen, struct si_shader *shader)
 static void si_emit_shader_es(struct si_context *sctx)
 {
        struct si_shader *shader = sctx->queued.named.es->shader;
+       unsigned initial_cdw = sctx->gfx_cs->current.cdw;
 
        if (!shader)
                return;
@@ -568,6 +558,19 @@ static void si_emit_shader_es(struct si_context *sctx)
        radeon_opt_set_context_reg(sctx, R_028AAC_VGT_ESGS_RING_ITEMSIZE,
                                   SI_TRACKED_VGT_ESGS_RING_ITEMSIZE,
                                   shader->selector->esgs_itemsize / 4);
+
+       if (shader->selector->type == PIPE_SHADER_TESS_EVAL)
+               radeon_opt_set_context_reg(sctx, R_028B6C_VGT_TF_PARAM,
+                                          SI_TRACKED_VGT_TF_PARAM,
+                                          shader->vgt_tf_param);
+
+       if (shader->vgt_vertex_reuse_block_cntl)
+               radeon_opt_set_context_reg(sctx, R_028C58_VGT_VERTEX_REUSE_BLOCK_CNTL,
+                                          SI_TRACKED_VGT_VERTEX_REUSE_BLOCK_CNTL,
+                                          shader->vgt_vertex_reuse_block_cntl);
+
+       if (initial_cdw != sctx->gfx_cs->current.cdw)
+               sctx->context_roll = true;
 }
 
 static void si_shader_es(struct si_screen *sscreen, struct si_shader *shader)
@@ -578,7 +581,7 @@ static void si_shader_es(struct si_screen *sscreen, struct si_shader *shader)
        uint64_t va;
        unsigned oc_lds_en;
 
-       assert(sscreen->info.chip_class <= VI);
+       assert(sscreen->info.chip_class <= GFX8);
 
        pm4 = si_get_shader_pm4_state(shader);
        if (!pm4)
@@ -749,6 +752,76 @@ static void gfx9_get_gs_info(struct si_shader_selector *es,
        assert(out->max_prims_per_subgroup <= max_out_prims);
 }
 
+static void si_emit_shader_gs(struct si_context *sctx)
+{
+       struct si_shader *shader = sctx->queued.named.gs->shader;
+       unsigned initial_cdw = sctx->gfx_cs->current.cdw;
+
+       if (!shader)
+               return;
+
+       /* R_028A60_VGT_GSVS_RING_OFFSET_1, R_028A64_VGT_GSVS_RING_OFFSET_2
+        * R_028A68_VGT_GSVS_RING_OFFSET_3, R_028A6C_VGT_GS_OUT_PRIM_TYPE */
+       radeon_opt_set_context_reg4(sctx, R_028A60_VGT_GSVS_RING_OFFSET_1,
+                                   SI_TRACKED_VGT_GSVS_RING_OFFSET_1,
+                                   shader->ctx_reg.gs.vgt_gsvs_ring_offset_1,
+                                   shader->ctx_reg.gs.vgt_gsvs_ring_offset_2,
+                                   shader->ctx_reg.gs.vgt_gsvs_ring_offset_3,
+                                   shader->ctx_reg.gs.vgt_gs_out_prim_type);
+
+
+       /* R_028AB0_VGT_GSVS_RING_ITEMSIZE */
+       radeon_opt_set_context_reg(sctx, R_028AB0_VGT_GSVS_RING_ITEMSIZE,
+                                  SI_TRACKED_VGT_GSVS_RING_ITEMSIZE,
+                                  shader->ctx_reg.gs.vgt_gsvs_ring_itemsize);
+
+       /* R_028B38_VGT_GS_MAX_VERT_OUT */
+       radeon_opt_set_context_reg(sctx, R_028B38_VGT_GS_MAX_VERT_OUT,
+                                  SI_TRACKED_VGT_GS_MAX_VERT_OUT,
+                                  shader->ctx_reg.gs.vgt_gs_max_vert_out);
+
+       /* R_028B5C_VGT_GS_VERT_ITEMSIZE, R_028B60_VGT_GS_VERT_ITEMSIZE_1
+        * R_028B64_VGT_GS_VERT_ITEMSIZE_2, R_028B68_VGT_GS_VERT_ITEMSIZE_3 */
+       radeon_opt_set_context_reg4(sctx, R_028B5C_VGT_GS_VERT_ITEMSIZE,
+                                   SI_TRACKED_VGT_GS_VERT_ITEMSIZE,
+                                   shader->ctx_reg.gs.vgt_gs_vert_itemsize,
+                                   shader->ctx_reg.gs.vgt_gs_vert_itemsize_1,
+                                   shader->ctx_reg.gs.vgt_gs_vert_itemsize_2,
+                                   shader->ctx_reg.gs.vgt_gs_vert_itemsize_3);
+
+       /* R_028B90_VGT_GS_INSTANCE_CNT */
+       radeon_opt_set_context_reg(sctx, R_028B90_VGT_GS_INSTANCE_CNT,
+                                  SI_TRACKED_VGT_GS_INSTANCE_CNT,
+                                  shader->ctx_reg.gs.vgt_gs_instance_cnt);
+
+       if (sctx->chip_class >= GFX9) {
+               /* R_028A44_VGT_GS_ONCHIP_CNTL */
+               radeon_opt_set_context_reg(sctx, R_028A44_VGT_GS_ONCHIP_CNTL,
+                                          SI_TRACKED_VGT_GS_ONCHIP_CNTL,
+                                          shader->ctx_reg.gs.vgt_gs_onchip_cntl);
+               /* R_028A94_VGT_GS_MAX_PRIMS_PER_SUBGROUP */
+               radeon_opt_set_context_reg(sctx, R_028A94_VGT_GS_MAX_PRIMS_PER_SUBGROUP,
+                                          SI_TRACKED_VGT_GS_MAX_PRIMS_PER_SUBGROUP,
+                                          shader->ctx_reg.gs.vgt_gs_max_prims_per_subgroup);
+               /* R_028AAC_VGT_ESGS_RING_ITEMSIZE */
+               radeon_opt_set_context_reg(sctx, R_028AAC_VGT_ESGS_RING_ITEMSIZE,
+                                          SI_TRACKED_VGT_ESGS_RING_ITEMSIZE,
+                                          shader->ctx_reg.gs.vgt_esgs_ring_itemsize);
+
+               if (shader->key.part.gs.es->type == PIPE_SHADER_TESS_EVAL)
+                       radeon_opt_set_context_reg(sctx, R_028B6C_VGT_TF_PARAM,
+                                                  SI_TRACKED_VGT_TF_PARAM,
+                                                  shader->vgt_tf_param);
+               if (shader->vgt_vertex_reuse_block_cntl)
+                       radeon_opt_set_context_reg(sctx, R_028C58_VGT_VERTEX_REUSE_BLOCK_CNTL,
+                                                  SI_TRACKED_VGT_VERTEX_REUSE_BLOCK_CNTL,
+                                                  shader->vgt_vertex_reuse_block_cntl);
+       }
+
+       if (initial_cdw != sctx->gfx_cs->current.cdw)
+               sctx->context_roll = true;
+}
+
 static void si_shader_gs(struct si_screen *sscreen, struct si_shader *shader)
 {
        struct si_shader_selector *sel = shader->selector;
@@ -763,33 +836,38 @@ static void si_shader_gs(struct si_screen *sscreen, struct si_shader *shader)
        if (!pm4)
                return;
 
+       pm4->atom.emit = si_emit_shader_gs;
+
        offset = num_components[0] * sel->gs_max_out_vertices;
-       si_pm4_set_reg(pm4, R_028A60_VGT_GSVS_RING_OFFSET_1, offset);
+       shader->ctx_reg.gs.vgt_gsvs_ring_offset_1 = offset;
+
        if (max_stream >= 1)
                offset += num_components[1] * sel->gs_max_out_vertices;
-       si_pm4_set_reg(pm4, R_028A64_VGT_GSVS_RING_OFFSET_2, offset);
+       shader->ctx_reg.gs.vgt_gsvs_ring_offset_2 = offset;
+
        if (max_stream >= 2)
                offset += num_components[2] * sel->gs_max_out_vertices;
-       si_pm4_set_reg(pm4, R_028A68_VGT_GSVS_RING_OFFSET_3, offset);
-       si_pm4_set_reg(pm4, R_028A6C_VGT_GS_OUT_PRIM_TYPE,
-                      si_conv_prim_to_gs_out(sel->gs_output_prim));
+       shader->ctx_reg.gs.vgt_gsvs_ring_offset_3 = offset;
+
+       shader->ctx_reg.gs.vgt_gs_out_prim_type =
+               si_conv_prim_to_gs_out(sel->gs_output_prim);
+
        if (max_stream >= 3)
                offset += num_components[3] * sel->gs_max_out_vertices;
-       si_pm4_set_reg(pm4, R_028AB0_VGT_GSVS_RING_ITEMSIZE, offset);
+       shader->ctx_reg.gs.vgt_gsvs_ring_itemsize = offset;
 
        /* The GSVS_RING_ITEMSIZE register takes 15 bits */
        assert(offset < (1 << 15));
 
-       si_pm4_set_reg(pm4, R_028B38_VGT_GS_MAX_VERT_OUT, sel->gs_max_out_vertices);
+       shader->ctx_reg.gs.vgt_gs_max_vert_out = sel->gs_max_out_vertices;
 
-       si_pm4_set_reg(pm4, R_028B5C_VGT_GS_VERT_ITEMSIZE, num_components[0]);
-       si_pm4_set_reg(pm4, R_028B60_VGT_GS_VERT_ITEMSIZE_1, (max_stream >= 1) ? num_components[1] : 0);
-       si_pm4_set_reg(pm4, R_028B64_VGT_GS_VERT_ITEMSIZE_2, (max_stream >= 2) ? num_components[2] : 0);
-       si_pm4_set_reg(pm4, R_028B68_VGT_GS_VERT_ITEMSIZE_3, (max_stream >= 3) ? num_components[3] : 0);
+       shader->ctx_reg.gs.vgt_gs_vert_itemsize = num_components[0];
+       shader->ctx_reg.gs.vgt_gs_vert_itemsize_1 = (max_stream >= 1) ? num_components[1] : 0;
+       shader->ctx_reg.gs.vgt_gs_vert_itemsize_2 = (max_stream >= 2) ? num_components[2] : 0;
+       shader->ctx_reg.gs.vgt_gs_vert_itemsize_3 = (max_stream >= 3) ? num_components[3] : 0;
 
-       si_pm4_set_reg(pm4, R_028B90_VGT_GS_INSTANCE_CNT,
-                      S_028B90_CNT(MIN2(gs_num_invocations, 127)) |
-                      S_028B90_ENABLE(gs_num_invocations > 0));
+       shader->ctx_reg.gs.vgt_gs_instance_cnt = S_028B90_CNT(MIN2(gs_num_invocations, 127)) |
+                                                S_028B90_ENABLE(gs_num_invocations > 0);
 
        va = shader->bo->gpu_address;
        si_pm4_add_bo(pm4, shader->bo, RADEON_USAGE_READ, RADEON_PRIO_SHADER_BINARY);
@@ -845,14 +923,14 @@ static void si_shader_gs(struct si_screen *sscreen, struct si_shader *shader)
                               S_00B22C_LDS_SIZE(gs_info.lds_size) |
                               S_00B22C_SCRATCH_EN(shader->config.scratch_bytes_per_wave > 0));
 
-               si_pm4_set_reg(pm4, R_028A44_VGT_GS_ONCHIP_CNTL,
-                              S_028A44_ES_VERTS_PER_SUBGRP(gs_info.es_verts_per_subgroup) |
-                              S_028A44_GS_PRIMS_PER_SUBGRP(gs_info.gs_prims_per_subgroup) |
-                              S_028A44_GS_INST_PRIMS_IN_SUBGRP(gs_info.gs_inst_prims_in_subgroup));
-               si_pm4_set_reg(pm4, R_028A94_VGT_GS_MAX_PRIMS_PER_SUBGROUP,
-                              S_028A94_MAX_PRIMS_PER_SUBGROUP(gs_info.max_prims_per_subgroup));
-               si_pm4_set_reg(pm4, R_028AAC_VGT_ESGS_RING_ITEMSIZE,
-                              shader->key.part.gs.es->esgs_itemsize / 4);
+               shader->ctx_reg.gs.vgt_gs_onchip_cntl =
+                       S_028A44_ES_VERTS_PER_SUBGRP(gs_info.es_verts_per_subgroup) |
+                       S_028A44_GS_PRIMS_PER_SUBGRP(gs_info.gs_prims_per_subgroup) |
+                       S_028A44_GS_INST_PRIMS_IN_SUBGRP(gs_info.gs_inst_prims_in_subgroup);
+               shader->ctx_reg.gs.vgt_gs_max_prims_per_subgroup =
+                       S_028A94_MAX_PRIMS_PER_SUBGROUP(gs_info.max_prims_per_subgroup);
+               shader->ctx_reg.gs.vgt_esgs_ring_itemsize =
+                       shader->key.part.gs.es->esgs_itemsize / 4;
 
                if (es_type == PIPE_SHADER_TESS_EVAL)
                        si_set_tesseval_regs(sscreen, shader->key.part.gs.es, pm4);
@@ -874,6 +952,53 @@ static void si_shader_gs(struct si_screen *sscreen, struct si_shader *shader)
        }
 }
 
+static void si_emit_shader_vs(struct si_context *sctx)
+{
+       struct si_shader *shader = sctx->queued.named.vs->shader;
+       unsigned initial_cdw = sctx->gfx_cs->current.cdw;
+
+       if (!shader)
+               return;
+
+       radeon_opt_set_context_reg(sctx, R_028A40_VGT_GS_MODE,
+                                  SI_TRACKED_VGT_GS_MODE,
+                                  shader->ctx_reg.vs.vgt_gs_mode);
+       radeon_opt_set_context_reg(sctx, R_028A84_VGT_PRIMITIVEID_EN,
+                                  SI_TRACKED_VGT_PRIMITIVEID_EN,
+                                  shader->ctx_reg.vs.vgt_primitiveid_en);
+
+       if (sctx->chip_class <= GFX8) {
+               radeon_opt_set_context_reg(sctx, R_028AB4_VGT_REUSE_OFF,
+                                          SI_TRACKED_VGT_REUSE_OFF,
+                                          shader->ctx_reg.vs.vgt_reuse_off);
+       }
+
+       radeon_opt_set_context_reg(sctx, R_0286C4_SPI_VS_OUT_CONFIG,
+                                  SI_TRACKED_SPI_VS_OUT_CONFIG,
+                                  shader->ctx_reg.vs.spi_vs_out_config);
+
+       radeon_opt_set_context_reg(sctx, R_02870C_SPI_SHADER_POS_FORMAT,
+                                  SI_TRACKED_SPI_SHADER_POS_FORMAT,
+                                  shader->ctx_reg.vs.spi_shader_pos_format);
+
+       radeon_opt_set_context_reg(sctx, R_028818_PA_CL_VTE_CNTL,
+                                  SI_TRACKED_PA_CL_VTE_CNTL,
+                                  shader->ctx_reg.vs.pa_cl_vte_cntl);
+
+       if (shader->selector->type == PIPE_SHADER_TESS_EVAL)
+               radeon_opt_set_context_reg(sctx, R_028B6C_VGT_TF_PARAM,
+                                          SI_TRACKED_VGT_TF_PARAM,
+                                          shader->vgt_tf_param);
+
+       if (shader->vgt_vertex_reuse_block_cntl)
+               radeon_opt_set_context_reg(sctx, R_028C58_VGT_VERTEX_REUSE_BLOCK_CNTL,
+                                          SI_TRACKED_VGT_VERTEX_REUSE_BLOCK_CNTL,
+                                          shader->vgt_vertex_reuse_block_cntl);
+
+       if (initial_cdw != sctx->gfx_cs->current.cdw)
+               sctx->context_roll = true;
+}
+
 /**
  * Compute the state for \p shader, which will run as a vertex shader on the
  * hardware.
@@ -886,18 +1011,19 @@ static void si_shader_vs(struct si_screen *sscreen, struct si_shader *shader,
 {
        const struct tgsi_shader_info *info = &shader->selector->info;
        struct si_pm4_state *pm4;
-       unsigned num_user_sgprs;
-       unsigned nparams, vgpr_comp_cnt;
+       unsigned num_user_sgprs, vgpr_comp_cnt;
        uint64_t va;
-       unsigned oc_lds_en;
+       unsigned nparams, oc_lds_en;
        unsigned window_space =
-          info->properties[TGSI_PROPERTY_VS_WINDOW_SPACE_POSITION];
+               info->properties[TGSI_PROPERTY_VS_WINDOW_SPACE_POSITION];
        bool enable_prim_id = shader->key.mono.u.vs_export_prim_id || info->uses_primid;
 
        pm4 = si_get_shader_pm4_state(shader);
        if (!pm4)
                return;
 
+       pm4->atom.emit = si_emit_shader_vs;
+
        /* We always write VGT_GS_MODE in the VS state, because every switch
         * between different shader pipelines involving a different GS or no
         * GS at all involves a switch of the VS (different GS use different
@@ -912,19 +1038,18 @@ static void si_shader_vs(struct si_screen *sscreen, struct si_shader *shader,
                if (enable_prim_id)
                        mode = V_028A40_GS_SCENARIO_A;
 
-               si_pm4_set_reg(pm4, R_028A40_VGT_GS_MODE, S_028A40_MODE(mode));
-               si_pm4_set_reg(pm4, R_028A84_VGT_PRIMITIVEID_EN, enable_prim_id);
+               shader->ctx_reg.vs.vgt_gs_mode = S_028A40_MODE(mode);
+               shader->ctx_reg.vs.vgt_primitiveid_en = enable_prim_id;
        } else {
-               si_pm4_set_reg(pm4, R_028A40_VGT_GS_MODE,
-                              ac_vgt_gs_mode(gs->gs_max_out_vertices,
-                                             sscreen->info.chip_class));
-               si_pm4_set_reg(pm4, R_028A84_VGT_PRIMITIVEID_EN, 0);
+               shader->ctx_reg.vs.vgt_gs_mode = ac_vgt_gs_mode(gs->gs_max_out_vertices,
+                                                               sscreen->info.chip_class);
+               shader->ctx_reg.vs.vgt_primitiveid_en = 0;
        }
 
-       if (sscreen->info.chip_class <= VI) {
+       if (sscreen->info.chip_class <= GFX8) {
                /* Reuse needs to be set off if we write oViewport. */
-               si_pm4_set_reg(pm4, R_028AB4_VGT_REUSE_OFF,
-                              S_028AB4_REUSE_OFF(info->writes_viewport_index));
+               shader->ctx_reg.vs.vgt_reuse_off =
+                               S_028AB4_REUSE_OFF(info->writes_viewport_index);
        }
 
        va = shader->bo->gpu_address;
@@ -954,20 +1079,19 @@ static void si_shader_vs(struct si_screen *sscreen, struct si_shader *shader,
 
        /* VS is required to export at least one param. */
        nparams = MAX2(shader->info.nr_param_exports, 1);
-       si_pm4_set_reg(pm4, R_0286C4_SPI_VS_OUT_CONFIG,
-                      S_0286C4_VS_EXPORT_COUNT(nparams - 1));
-
-       si_pm4_set_reg(pm4, R_02870C_SPI_SHADER_POS_FORMAT,
-                      S_02870C_POS0_EXPORT_FORMAT(V_02870C_SPI_SHADER_4COMP) |
-                      S_02870C_POS1_EXPORT_FORMAT(shader->info.nr_pos_exports > 1 ?
-                                                  V_02870C_SPI_SHADER_4COMP :
-                                                  V_02870C_SPI_SHADER_NONE) |
-                      S_02870C_POS2_EXPORT_FORMAT(shader->info.nr_pos_exports > 2 ?
-                                                  V_02870C_SPI_SHADER_4COMP :
-                                                  V_02870C_SPI_SHADER_NONE) |
-                      S_02870C_POS3_EXPORT_FORMAT(shader->info.nr_pos_exports > 3 ?
-                                                  V_02870C_SPI_SHADER_4COMP :
-                                                  V_02870C_SPI_SHADER_NONE));
+       shader->ctx_reg.vs.spi_vs_out_config = S_0286C4_VS_EXPORT_COUNT(nparams - 1);
+
+       shader->ctx_reg.vs.spi_shader_pos_format =
+                       S_02870C_POS0_EXPORT_FORMAT(V_02870C_SPI_SHADER_4COMP) |
+                       S_02870C_POS1_EXPORT_FORMAT(shader->info.nr_pos_exports > 1 ?
+                                                   V_02870C_SPI_SHADER_4COMP :
+                                                   V_02870C_SPI_SHADER_NONE) |
+                       S_02870C_POS2_EXPORT_FORMAT(shader->info.nr_pos_exports > 2 ?
+                                                   V_02870C_SPI_SHADER_4COMP :
+                                                   V_02870C_SPI_SHADER_NONE) |
+                       S_02870C_POS3_EXPORT_FORMAT(shader->info.nr_pos_exports > 3 ?
+                                                   V_02870C_SPI_SHADER_4COMP :
+                                                   V_02870C_SPI_SHADER_NONE);
 
        oc_lds_en = shader->selector->type == PIPE_SHADER_TESS_EVAL ? 1 : 0;
 
@@ -988,15 +1112,16 @@ static void si_shader_vs(struct si_screen *sscreen, struct si_shader *shader,
                       S_00B12C_SO_BASE3_EN(!!shader->selector->so.stride[3]) |
                       S_00B12C_SO_EN(!!shader->selector->so.num_outputs) |
                       S_00B12C_SCRATCH_EN(shader->config.scratch_bytes_per_wave > 0));
+
        if (window_space)
-               si_pm4_set_reg(pm4, R_028818_PA_CL_VTE_CNTL,
-                              S_028818_VTX_XY_FMT(1) | S_028818_VTX_Z_FMT(1));
+               shader->ctx_reg.vs.pa_cl_vte_cntl =
+                               S_028818_VTX_XY_FMT(1) | S_028818_VTX_Z_FMT(1);
        else
-               si_pm4_set_reg(pm4, R_028818_PA_CL_VTE_CNTL,
-                              S_028818_VTX_W0_FMT(1) |
-                              S_028818_VPORT_X_SCALE_ENA(1) | S_028818_VPORT_X_OFFSET_ENA(1) |
-                              S_028818_VPORT_Y_SCALE_ENA(1) | S_028818_VPORT_Y_OFFSET_ENA(1) |
-                              S_028818_VPORT_Z_SCALE_ENA(1) | S_028818_VPORT_Z_OFFSET_ENA(1));
+               shader->ctx_reg.vs.pa_cl_vte_cntl =
+                               S_028818_VTX_W0_FMT(1) |
+                               S_028818_VPORT_X_SCALE_ENA(1) | S_028818_VPORT_X_OFFSET_ENA(1) |
+                               S_028818_VPORT_Y_SCALE_ENA(1) | S_028818_VPORT_Y_OFFSET_ENA(1) |
+                               S_028818_VPORT_Z_SCALE_ENA(1) | S_028818_VPORT_Z_OFFSET_ENA(1);
 
        if (shader->selector->type == PIPE_SHADER_TESS_EVAL)
                si_set_tesseval_regs(sscreen, shader->selector, pm4);
@@ -1031,6 +1156,41 @@ static unsigned si_get_spi_shader_col_format(struct si_shader *shader)
        return value;
 }
 
+static void si_emit_shader_ps(struct si_context *sctx)
+{
+       struct si_shader *shader = sctx->queued.named.ps->shader;
+       unsigned initial_cdw = sctx->gfx_cs->current.cdw;
+
+       if (!shader)
+               return;
+
+       /* R_0286CC_SPI_PS_INPUT_ENA, R_0286D0_SPI_PS_INPUT_ADDR*/
+       radeon_opt_set_context_reg2(sctx, R_0286CC_SPI_PS_INPUT_ENA,
+                                   SI_TRACKED_SPI_PS_INPUT_ENA,
+                                   shader->ctx_reg.ps.spi_ps_input_ena,
+                                   shader->ctx_reg.ps.spi_ps_input_addr);
+
+       radeon_opt_set_context_reg(sctx, R_0286E0_SPI_BARYC_CNTL,
+                                  SI_TRACKED_SPI_BARYC_CNTL,
+                                  shader->ctx_reg.ps.spi_baryc_cntl);
+       radeon_opt_set_context_reg(sctx, R_0286D8_SPI_PS_IN_CONTROL,
+                                  SI_TRACKED_SPI_PS_IN_CONTROL,
+                                  shader->ctx_reg.ps.spi_ps_in_control);
+
+       /* R_028710_SPI_SHADER_Z_FORMAT, R_028714_SPI_SHADER_COL_FORMAT */
+       radeon_opt_set_context_reg2(sctx, R_028710_SPI_SHADER_Z_FORMAT,
+                                   SI_TRACKED_SPI_SHADER_Z_FORMAT,
+                                   shader->ctx_reg.ps.spi_shader_z_format,
+                                   shader->ctx_reg.ps.spi_shader_col_format);
+
+       radeon_opt_set_context_reg(sctx, R_02823C_CB_SHADER_MASK,
+                                  SI_TRACKED_CB_SHADER_MASK,
+                                  shader->ctx_reg.ps.cb_shader_mask);
+
+       if (initial_cdw != sctx->gfx_cs->current.cdw)
+               sctx->context_roll = true;
+}
+
 static void si_shader_ps(struct si_shader *shader)
 {
        struct tgsi_shader_info *info = &shader->selector->info;
@@ -1088,6 +1248,8 @@ static void si_shader_ps(struct si_shader *shader)
        if (!pm4)
                return;
 
+       pm4->atom.emit = si_emit_shader_ps;
+
        /* SPI_BARYC_CNTL.POS_FLOAT_LOCATION
         * Possible vaules:
         * 0 -> Position = pixel center
@@ -1130,24 +1292,20 @@ static void si_shader_ps(struct si_shader *shader)
            !info->writes_z && !info->writes_stencil && !info->writes_samplemask)
                spi_shader_col_format = V_028714_SPI_SHADER_32_R;
 
-       si_pm4_set_reg(pm4, R_0286CC_SPI_PS_INPUT_ENA, input_ena);
-       si_pm4_set_reg(pm4, R_0286D0_SPI_PS_INPUT_ADDR,
-                      shader->config.spi_ps_input_addr);
+       shader->ctx_reg.ps.spi_ps_input_ena = input_ena;
+       shader->ctx_reg.ps.spi_ps_input_addr = shader->config.spi_ps_input_addr;
 
        /* Set interpolation controls. */
        spi_ps_in_control = S_0286D8_NUM_INTERP(si_get_ps_num_interp(shader));
 
-       /* Set registers. */
-       si_pm4_set_reg(pm4, R_0286E0_SPI_BARYC_CNTL, spi_baryc_cntl);
-       si_pm4_set_reg(pm4, R_0286D8_SPI_PS_IN_CONTROL, spi_ps_in_control);
-
-       si_pm4_set_reg(pm4, R_028710_SPI_SHADER_Z_FORMAT,
-                      ac_get_spi_shader_z_format(info->writes_z,
-                                                 info->writes_stencil,
-                                                 info->writes_samplemask));
-
-       si_pm4_set_reg(pm4, R_028714_SPI_SHADER_COL_FORMAT, spi_shader_col_format);
-       si_pm4_set_reg(pm4, R_02823C_CB_SHADER_MASK, cb_shader_mask);
+       shader->ctx_reg.ps.spi_baryc_cntl = spi_baryc_cntl;
+       shader->ctx_reg.ps.spi_ps_in_control = spi_ps_in_control;
+       shader->ctx_reg.ps.spi_shader_z_format =
+                       ac_get_spi_shader_z_format(info->writes_z,
+                                                  info->writes_stencil,
+                                                  info->writes_samplemask);
+       shader->ctx_reg.ps.spi_shader_col_format = spi_shader_col_format;
+       shader->ctx_reg.ps.cb_shader_mask = cb_shader_mask;
 
        va = shader->bo->gpu_address;
        si_pm4_add_bo(pm4, shader->bo, RADEON_USAGE_READ, RADEON_PRIO_SHADER_BINARY);
@@ -1206,28 +1364,53 @@ static unsigned si_get_alpha_test_func(struct si_context *sctx)
        return PIPE_FUNC_ALWAYS;
 }
 
-static void si_shader_selector_key_vs(struct si_context *sctx,
-                                     struct si_shader_selector *vs,
-                                     struct si_shader_key *key,
-                                     struct si_vs_prolog_bits *prolog_key)
+void si_shader_selector_key_vs(struct si_context *sctx,
+                              struct si_shader_selector *vs,
+                              struct si_shader_key *key,
+                              struct si_vs_prolog_bits *prolog_key)
 {
        if (!sctx->vertex_elements ||
            vs->info.properties[TGSI_PROPERTY_VS_BLIT_SGPRS])
                return;
 
-       prolog_key->instance_divisor_is_one =
-               sctx->vertex_elements->instance_divisor_is_one;
-       prolog_key->instance_divisor_is_fetched =
-               sctx->vertex_elements->instance_divisor_is_fetched;
+       struct si_vertex_elements *elts = sctx->vertex_elements;
+
+       prolog_key->instance_divisor_is_one = elts->instance_divisor_is_one;
+       prolog_key->instance_divisor_is_fetched = elts->instance_divisor_is_fetched;
+       prolog_key->unpack_instance_id_from_vertex_id =
+               sctx->prim_discard_cs_instancing;
 
        /* Prefer a monolithic shader to allow scheduling divisions around
         * VBO loads. */
        if (prolog_key->instance_divisor_is_fetched)
                key->opt.prefer_mono = 1;
 
-       unsigned count = MIN2(vs->info.num_inputs,
-                             sctx->vertex_elements->count);
-       memcpy(key->mono.vs_fix_fetch, sctx->vertex_elements->fix_fetch, count);
+       unsigned count = MIN2(vs->info.num_inputs, elts->count);
+       unsigned count_mask = (1 << count) - 1;
+       unsigned fix = elts->fix_fetch_always & count_mask;
+       unsigned opencode = elts->fix_fetch_opencode & count_mask;
+
+       if (sctx->vertex_buffer_unaligned & elts->vb_alignment_check_mask) {
+               uint32_t mask = elts->fix_fetch_unaligned & count_mask;
+               while (mask) {
+                       unsigned i = u_bit_scan(&mask);
+                       unsigned log_hw_load_size = 1 + ((elts->hw_load_is_dword >> i) & 1);
+                       unsigned vbidx = elts->vertex_buffer_index[i];
+                       struct pipe_vertex_buffer *vb = &sctx->vertex_buffer[vbidx];
+                       unsigned align_mask = (1 << log_hw_load_size) - 1;
+                       if (vb->buffer_offset & align_mask ||
+                           vb->stride & align_mask) {
+                               fix |= 1 << i;
+                               opencode |= 1 << i;
+                       }
+               }
+       }
+
+       while (fix) {
+               unsigned i = u_bit_scan(&fix);
+               key->mono.vs_fix_fetch[i].bits = elts->fix_fetch[i];
+       }
+       key->mono.vs_fetch_opencode = opencode;
 }
 
 static void si_shader_selector_key_hw_vs(struct si_context *sctx,
@@ -1415,11 +1598,11 @@ static inline void si_shader_selector_key(struct pipe_context *ctx,
                    blend && blend->alpha_to_coverage)
                        key->part.ps.epilog.spi_shader_col_format |= V_028710_SPI_SHADER_32_AR;
 
-               /* On SI and CIK except Hawaii, the CB doesn't clamp outputs
+               /* On GFX6 and GFX7 except Hawaii, the CB doesn't clamp outputs
                 * to the range supported by the type if a channel has less
                 * than 16 bits and the export format is 16_ABGR.
                 */
-               if (sctx->chip_class <= CIK && sctx->family != CHIP_HAWAII) {
+               if (sctx->chip_class <= GFX7 && sctx->family != CHIP_HAWAII) {
                        key->part.ps.epilog.color_is_int8 = sctx->framebuffer.color_is_int8;
                        key->part.ps.epilog.color_is_int10 = sctx->framebuffer.color_is_int10;
                }
@@ -1493,7 +1676,7 @@ static inline void si_shader_selector_key(struct pipe_context *ctx,
                key->part.ps.epilog.alpha_func = si_get_alpha_test_func(sctx);
 
                /* ps_uses_fbfetch is true only if the color buffer is bound. */
-               if (sctx->ps_uses_fbfetch) {
+               if (sctx->ps_uses_fbfetch && !sctx->blitter->running) {
                        struct pipe_surface *cb0 = sctx->framebuffer.state.cbufs[0];
                        struct pipe_resource *tex = cb0->texture;
 
@@ -1526,7 +1709,6 @@ static void si_build_shader_variant(struct si_shader *shader,
        struct si_screen *sscreen = sel->screen;
        struct ac_llvm_compiler *compiler;
        struct pipe_debug_callback *debug = &shader->compiler_ctx_state.debug;
-       int r;
 
        if (thread_index >= 0) {
                if (low_priority) {
@@ -1543,10 +1725,9 @@ static void si_build_shader_variant(struct si_shader *shader,
                compiler = shader->compiler_ctx_state.compiler;
        }
 
-       r = si_shader_create(sscreen, compiler, shader, debug);
-       if (unlikely(r)) {
-               PRINT_ERR("Failed to build shader variant (type=%u) %d\n",
-                        sel->type, r);
+       if (unlikely(!si_shader_create(sscreen, compiler, shader, debug))) {
+               PRINT_ERR("Failed to build shader variant (type=%u)\n",
+                         sel->type);
                shader->compilation_failed = true;
                return;
        }
@@ -1607,12 +1788,19 @@ static bool si_check_missing_main_part(struct si_screen *sscreen,
        return true;
 }
 
-/* Select the hw shader variant depending on the current state. */
-static int si_shader_select_with_key(struct si_screen *sscreen,
-                                    struct si_shader_ctx_state *state,
-                                    struct si_compiler_ctx_state *compiler_state,
-                                    struct si_shader_key *key,
-                                    int thread_index)
+/**
+ * Select a shader variant according to the shader key.
+ *
+ * \param optimized_or_none  If the key describes an optimized shader variant and
+ *                           the compilation isn't finished, don't select any
+ *                           shader and return an error.
+ */
+int si_shader_select_with_key(struct si_screen *sscreen,
+                             struct si_shader_ctx_state *state,
+                             struct si_compiler_ctx_state *compiler_state,
+                             struct si_shader_key *key,
+                             int thread_index,
+                             bool optimized_or_none)
 {
        struct si_shader_selector *sel = state->cso;
        struct si_shader_selector *previous_stage_sel = NULL;
@@ -1628,6 +1816,9 @@ again:
                   memcmp(&current->key, key, sizeof(*key)) == 0)) {
                if (unlikely(!util_queue_fence_is_signalled(&current->ready))) {
                        if (current->is_optimized) {
+                               if (optimized_or_none)
+                                       return -1;
+
                                memset(&key->opt, 0, sizeof(key->opt));
                                goto current_not_ready;
                        }
@@ -1664,6 +1855,8 @@ current_not_ready:
                                 * shader so as not to cause a stall due to compilation.
                                 */
                                if (iter->is_optimized) {
+                                       if (optimized_or_none)
+                                               return -1;
                                        memset(&key->opt, 0, sizeof(key->opt));
                                        goto again;
                                }
@@ -1705,13 +1898,17 @@ current_not_ready:
                        util_queue_fence_wait(&previous_stage_sel->ready);
        }
 
-       /* Compile the main shader part if it doesn't exist. This can happen
-        * if the initial guess was wrong. */
        bool is_pure_monolithic =
                sscreen->use_monolithic_shaders ||
                memcmp(&key->mono, &zeroed.mono, sizeof(key->mono)) != 0;
 
-       if (!is_pure_monolithic) {
+       /* Compile the main shader part if it doesn't exist. This can happen
+        * if the initial guess was wrong.
+        *
+        * The prim discard CS doesn't need the main shader part.
+        */
+       if (!is_pure_monolithic &&
+           !key->opt.vs_as_prim_discard_cs) {
                bool ok;
 
                /* Make sure the main shader part is present. This is needed
@@ -1762,14 +1959,13 @@ current_not_ready:
                is_pure_monolithic ||
                memcmp(&key->opt, &zeroed.opt, sizeof(key->opt)) != 0;
 
+       /* The prim discard CS is always optimized. */
        shader->is_optimized =
-               !is_pure_monolithic &&
-               memcmp(&key->opt, &zeroed.opt, sizeof(key->opt)) != 0;
+               (!is_pure_monolithic || key->opt.vs_as_prim_discard_cs) &&
+                memcmp(&key->opt, &zeroed.opt, sizeof(key->opt)) != 0;
 
        /* If it's an optimized shader, compile it asynchronously. */
-       if (shader->is_optimized &&
-           !is_pure_monolithic &&
-           thread_index < 0) {
+       if (shader->is_optimized && thread_index < 0) {
                /* Compile it asynchronously. */
                util_queue_add_job(&sscreen->shader_compiler_queue_low_priority,
                                   shader, &shader->ready,
@@ -1788,6 +1984,12 @@ current_not_ready:
                /* Use the default (unoptimized) shader for now. */
                memset(&key->opt, 0, sizeof(key->opt));
                mtx_unlock(&sel->mutex);
+
+               if (sscreen->options.sync_compile)
+                       util_queue_fence_wait(&shader->ready);
+
+               if (optimized_or_none)
+                       return -1;
                goto again;
        }
 
@@ -1824,7 +2026,7 @@ static int si_shader_select(struct pipe_context *ctx,
 
        si_shader_selector_key(ctx, state->cso, &key);
        return si_shader_select_with_key(sctx->screen, state, compiler_state,
-                                        &key, -1);
+                                        &key, -1, false);
 }
 
 static void si_parse_next_shader_property(const struct tgsi_shader_info *info,
@@ -1879,6 +2081,9 @@ static void si_init_shader_selector_async(void *job, int thread_index)
        assert(thread_index < ARRAY_SIZE(sscreen->compiler));
        compiler = &sscreen->compiler[thread_index];
 
+       if (sel->nir)
+               si_lower_nir(sel);
+
        /* Compile the main shader part for use with a prolog and/or epilog.
         * If this fails, the driver will try to compile a monolithic shader
         * on demand.
@@ -1911,7 +2116,7 @@ static void si_init_shader_selector_async(void *job, int thread_index)
                if (ir_binary &&
                    si_shader_cache_load_shader(sscreen, ir_binary, shader)) {
                        mtx_unlock(&sscreen->shader_cache_mutex);
-                       si_shader_dump_stats_for_shader_db(shader, debug);
+                       si_shader_dump_stats_for_shader_db(sscreen, shader, debug);
                } else {
                        mtx_unlock(&sscreen->shader_cache_mutex);
 
@@ -1997,12 +2202,12 @@ void si_schedule_initial_compile(struct si_context *sctx, unsigned processor,
        util_queue_fence_init(ready_fence);
 
        struct util_async_debug_callback async_debug;
-       bool wait =
+       bool debug =
                (sctx->debug.debug_message && !sctx->debug.async) ||
                sctx->is_debug ||
                si_can_dump_shader(sctx->screen, processor);
 
-       if (wait) {
+       if (debug) {
                u_async_debug_init(&async_debug);
                compiler_ctx_state->debug = async_debug.base;
        }
@@ -2010,11 +2215,14 @@ void si_schedule_initial_compile(struct si_context *sctx, unsigned processor,
        util_queue_add_job(&sctx->screen->shader_compiler_queue, job,
                           ready_fence, execute, NULL);
 
-       if (wait) {
+       if (debug) {
                util_queue_fence_wait(ready_fence);
                u_async_debug_drain(&async_debug, &sctx->debug);
                u_async_debug_cleanup(&async_debug);
        }
+
+       if (sctx->screen->options.sync_compile)
+               util_queue_fence_wait(ready_fence);
 }
 
 /* Return descriptor slot usage masks from the given shader info. */
@@ -2073,10 +2281,9 @@ static void *si_create_shader_selector(struct pipe_context *ctx,
 
                sel->nir = state->ir.nir;
 
+               si_nir_opts(sel->nir);
                si_nir_scan_shader(sel->nir, &sel->info);
-               si_nir_scan_tess_ctrl(sel->nir, &sel->info, &sel->tcs_info);
-
-               si_lower_nir(sel);
+               si_nir_scan_tess_ctrl(sel->nir, &sel->tcs_info);
        }
 
        sel->type = sel->info.processor;
@@ -2103,6 +2310,15 @@ static void *si_create_shader_selector(struct pipe_context *ctx,
                sel->info.uses_kill &&
                sctx->screen->debug_flags & DBG(FS_CORRECT_DERIVS_AFTER_KILL);
 
+       sel->prim_discard_cs_allowed =
+               sel->type == PIPE_SHADER_VERTEX &&
+               !sel->info.uses_bindless_images &&
+               !sel->info.uses_bindless_samplers &&
+               !sel->info.writes_memory &&
+               !sel->info.writes_viewport_index &&
+               !sel->info.properties[TGSI_PROPERTY_VS_WINDOW_SPACE_POSITION] &&
+               !sel->so.num_outputs;
+
        /* Set which opcode uses which (i,j) pair. */
        if (sel->info.uses_persp_opcode_interp_centroid)
                sel->info.uses_persp_centroid = true;
@@ -2510,10 +2726,10 @@ static void si_delete_shader(struct si_context *sctx, struct si_shader *shader)
                switch (shader->selector->type) {
                case PIPE_SHADER_VERTEX:
                        if (shader->key.as_ls) {
-                               assert(sctx->chip_class <= VI);
+                               assert(sctx->chip_class <= GFX8);
                                si_pm4_delete_state(sctx, ls, shader->pm4);
                        } else if (shader->key.as_es) {
-                               assert(sctx->chip_class <= VI);
+                               assert(sctx->chip_class <= GFX8);
                                si_pm4_delete_state(sctx, es, shader->pm4);
                        } else {
                                si_pm4_delete_state(sctx, vs, shader->pm4);
@@ -2524,7 +2740,7 @@ static void si_delete_shader(struct si_context *sctx, struct si_shader *shader)
                        break;
                case PIPE_SHADER_TESS_EVAL:
                        if (shader->key.as_es) {
-                               assert(sctx->chip_class <= VI);
+                               assert(sctx->chip_class <= GFX8);
                                si_pm4_delete_state(sctx, es, shader->pm4);
                        } else {
                                si_pm4_delete_state(sctx, vs, shader->pm4);
@@ -2604,7 +2820,8 @@ static unsigned si_get_ps_input_cntl(struct si_context *sctx,
        unsigned j, offset, ps_input_cntl = 0;
 
        if (interpolate == TGSI_INTERPOLATE_CONSTANT ||
-           (interpolate == TGSI_INTERPOLATE_COLOR && sctx->flatshade))
+           (interpolate == TGSI_INTERPOLATE_COLOR && sctx->flatshade) ||
+           name == TGSI_SEMANTIC_PRIMID)
                ps_input_cntl |= S_028644_FLAT_SHADE(1);
 
        if (name == TGSI_SEMANTIC_PCOORD ||
@@ -2699,9 +2916,13 @@ static void si_emit_spi_map(struct si_context *sctx)
        /* R_028644_SPI_PS_INPUT_CNTL_0 */
        /* Dota 2: Only ~16% of SPI map updates set different values. */
        /* Talos: Only ~9% of SPI map updates set different values. */
+       unsigned initial_cdw = sctx->gfx_cs->current.cdw;
        radeon_opt_set_context_regn(sctx, R_028644_SPI_PS_INPUT_CNTL_0,
                                    spi_ps_input_cntl,
                                    sctx->tracked_regs.spi_ps_input_cntl, num_interp);
+
+       if (initial_cdw != sctx->gfx_cs->current.cdw)
+               sctx->context_roll = true;
 }
 
 /**
@@ -2737,10 +2958,10 @@ static bool si_update_gs_ring_buffers(struct si_context *sctx)
        unsigned num_se = sctx->screen->info.max_se;
        unsigned wave_size = 64;
        unsigned max_gs_waves = 32 * num_se; /* max 32 per SE on GCN */
-       /* On SI-CI, the value comes from VGT_GS_VERTEX_REUSE = 16.
-        * On VI+, the value comes from VGT_VERTEX_REUSE_BLOCK_CNTL = 30 (+2).
+       /* On GFX6-GFX7, the value comes from VGT_GS_VERTEX_REUSE = 16.
+        * On GFX8+, the value comes from VGT_VERTEX_REUSE_BLOCK_CNTL = 30 (+2).
         */
-       unsigned gs_vertex_reuse = (sctx->chip_class >= VI ? 32 : 16) * num_se;
+       unsigned gs_vertex_reuse = (sctx->chip_class >= GFX8 ? 32 : 16) * num_se;
        unsigned alignment = 256 * num_se;
        /* The maximum size is 63.999 MB per SE. */
        unsigned max_size = ((unsigned)(63.999 * 1024 * 1024) & ~255) * num_se;
@@ -2767,7 +2988,7 @@ static bool si_update_gs_ring_buffers(struct si_context *sctx)
         *
         * GFX9 doesn't have the ESGS ring.
         */
-       bool update_esgs = sctx->chip_class <= VI &&
+       bool update_esgs = sctx->chip_class <= GFX8 &&
                           esgs_ring_size &&
                           (!sctx->esgs_ring ||
                            sctx->esgs_ring->width0 < esgs_ring_size);
@@ -2805,9 +3026,9 @@ static bool si_update_gs_ring_buffers(struct si_context *sctx)
        if (!pm4)
                return false;
 
-       if (sctx->chip_class >= CIK) {
+       if (sctx->chip_class >= GFX7) {
                if (sctx->esgs_ring) {
-                       assert(sctx->chip_class <= VI);
+                       assert(sctx->chip_class <= GFX8);
                        si_pm4_set_reg(pm4, R_030900_VGT_ESGS_RING_SIZE,
                                       sctx->esgs_ring->width0 / 256);
                }
@@ -2839,7 +3060,7 @@ static bool si_update_gs_ring_buffers(struct si_context *sctx)
 
        /* Set ring bindings. */
        if (sctx->esgs_ring) {
-               assert(sctx->chip_class <= VI);
+               assert(sctx->chip_class <= GFX8);
                si_set_ring_buffer(sctx, SI_ES_RING_ESGS,
                                   sctx->esgs_ring, 0, sctx->esgs_ring->width0,
                                   true, true, 4, 64, 0);
@@ -2881,7 +3102,6 @@ static int si_update_scratch_buffer(struct si_context *sctx,
                                    struct si_shader *shader)
 {
        uint64_t scratch_va = sctx->scratch_buffer->gpu_address;
-       int r;
 
        if (!shader)
                return 0;
@@ -2906,22 +3126,16 @@ static int si_update_scratch_buffer(struct si_context *sctx,
 
        assert(sctx->scratch_buffer);
 
-       if (shader->previous_stage)
-               si_shader_apply_scratch_relocs(shader->previous_stage, scratch_va);
-
-       si_shader_apply_scratch_relocs(shader, scratch_va);
-
        /* Replace the shader bo with a new bo that has the relocs applied. */
-       r = si_shader_binary_upload(sctx->screen, shader);
-       if (r) {
+       if (!si_shader_binary_upload(sctx->screen, shader, scratch_va)) {
                si_shader_unlock(shader);
-               return r;
+               return -1;
        }
 
        /* Update the shader state to use the new shader bo. */
        si_shader_init_pm4_state(sctx->screen, shader);
 
-       r600_resource_reference(&shader->scratch_bo, sctx->scratch_buffer);
+       si_resource_reference(&shader->scratch_bo, sctx->scratch_buffer);
 
        si_shader_unlock(shader);
        return 1;
@@ -3031,7 +3245,7 @@ static bool si_update_spi_tmpring_size(struct si_context *sctx)
        if (scratch_needed_size > 0) {
                if (scratch_needed_size > current_scratch_buffer_size) {
                        /* Create a bigger scratch buffer */
-                       r600_resource_reference(&sctx->scratch_buffer, NULL);
+                       si_resource_reference(&sctx->scratch_buffer, NULL);
 
                        sctx->scratch_buffer =
                                si_aligned_buffer_create(&sctx->screen->b,
@@ -3081,14 +3295,14 @@ static void si_init_tess_factor_ring(struct si_context *sctx)
 
        si_init_config_add_vgt_flush(sctx);
 
-       si_pm4_add_bo(sctx->init_config, r600_resource(sctx->tess_rings),
+       si_pm4_add_bo(sctx->init_config, si_resource(sctx->tess_rings),
                      RADEON_USAGE_READWRITE, RADEON_PRIO_SHADER_RINGS);
 
-       uint64_t factor_va = r600_resource(sctx->tess_rings)->gpu_address +
+       uint64_t factor_va = si_resource(sctx->tess_rings)->gpu_address +
                             sctx->screen->tess_offchip_ring_size;
 
        /* Append these registers to the init config state. */
-       if (sctx->chip_class >= CIK) {
+       if (sctx->chip_class >= GFX7) {
                si_pm4_set_reg(sctx->init_config, R_030938_VGT_TF_RING_SIZE,
                               S_030938_SIZE(sctx->screen->tess_factor_ring_size / 4));
                si_pm4_set_reg(sctx->init_config, R_030940_VGT_TF_MEMORY_BASE,
@@ -3176,7 +3390,7 @@ bool si_update_shaders(struct si_context *sctx)
                }
 
                /* VS as LS */
-               if (sctx->chip_class <= VI) {
+               if (sctx->chip_class <= GFX8) {
                        r = si_shader_select(ctx, &sctx->vs_shader,
                                             &compiler_state);
                        if (r)
@@ -3208,7 +3422,7 @@ bool si_update_shaders(struct si_context *sctx)
 
                if (sctx->gs_shader.cso) {
                        /* TES as ES */
-                       if (sctx->chip_class <= VI) {
+                       if (sctx->chip_class <= GFX8) {
                                r = si_shader_select(ctx, &sctx->tes_shader,
                                                     &compiler_state);
                                if (r)
@@ -3224,7 +3438,7 @@ bool si_update_shaders(struct si_context *sctx)
                        si_pm4_bind_state(sctx, vs, sctx->tes_shader.current->pm4);
                }
        } else if (sctx->gs_shader.cso) {
-               if (sctx->chip_class <= VI) {
+               if (sctx->chip_class <= GFX8) {
                        /* VS as ES */
                        r = si_shader_select(ctx, &sctx->vs_shader,
                                             &compiler_state);
@@ -3257,7 +3471,7 @@ bool si_update_shaders(struct si_context *sctx)
                        return false;
        } else {
                si_pm4_bind_state(sctx, gs, NULL);
-               if (sctx->chip_class <= VI)
+               if (sctx->chip_class <= GFX8)
                        si_pm4_bind_state(sctx, es, NULL);
        }
 
@@ -3304,7 +3518,7 @@ bool si_update_shaders(struct si_context *sctx)
                        sctx->smoothing_enabled = sctx->ps_shader.current->key.part.ps.epilog.poly_line_smoothing;
                        si_mark_atom_dirty(sctx, &sctx->atoms.s.msaa_config);
 
-                       if (sctx->chip_class == SI)
+                       if (sctx->chip_class == GFX6)
                                si_mark_atom_dirty(sctx, &sctx->atoms.s.db_render_state);
 
                        if (sctx->framebuffer.nr_samples <= 1)
@@ -3322,7 +3536,7 @@ bool si_update_shaders(struct si_context *sctx)
                        return false;
        }
 
-       if (sctx->chip_class >= CIK) {
+       if (sctx->chip_class >= GFX7) {
                if (si_pm4_state_enabled_and_changed(sctx, ls))
                        sctx->prefetch_L2_mask |= SI_PREFETCH_LS;
                else if (!sctx->queued.named.ls)