radeonsi: prevent a gfx10_ngg_calculate_subgroup_info failure for TES+NGG GS

[mesa.git] / src / gallium / drivers / radeonsi / si_state_shaders.c

diff --git a/src/gallium/drivers/radeonsi/si_state_shaders.c b/src/gallium/drivers/radeonsi/si_state_shaders.c
index 396737bd2c22dd7d3a16cff2982b906eb2855970..01ed224b282eb874c42599f5a078a29ea417cec0 100644 (file)
--- a/src/gallium/drivers/radeonsi/si_state_shaders.c
+++ b/src/gallium/drivers/radeonsi/si_state_shaders.c
@@ -69,7 +69,7 @@ void si_get_ir_cache_key(struct si_shader_selector *sel, bool ngg, bool es,
       shader_variant_flags |= 1 << 0;
    if (sel->nir)
       shader_variant_flags |= 1 << 1;
-   if (si_get_wave_size(sel->screen, sel->type, ngg, es, false) == 32)
+   if (si_get_wave_size(sel->screen, sel->type, ngg, es, false, false) == 32)
       shader_variant_flags |= 1 << 2;
    if (sel->type == PIPE_SHADER_FRAGMENT && sel->info.uses_derivatives && sel->info.uses_kill &&
        sel->screen->debug_flags & DBG(FS_CORRECT_DERIVS_AFTER_KILL))
@@ -479,8 +479,6 @@ static void si_shader_ls(struct si_screen *sscreen, struct si_shader *shader)
       return;
 
    va = shader->bo->gpu_address;
-   si_pm4_add_bo(pm4, shader->bo, RADEON_USAGE_READ, RADEON_PRIO_SHADER_BINARY);
-
    si_pm4_set_reg(pm4, R_00B520_SPI_SHADER_PGM_LO_LS, va >> 8);
    si_pm4_set_reg(pm4, R_00B524_SPI_SHADER_PGM_HI_LS, S_00B524_MEM_BASE(va >> 40));
 
@@ -503,7 +501,6 @@ static void si_shader_hs(struct si_screen *sscreen, struct si_shader *shader)
       return;
 
    va = shader->bo->gpu_address;
-   si_pm4_add_bo(pm4, shader->bo, RADEON_USAGE_READ, RADEON_PRIO_SHADER_BINARY);
 
    if (sscreen->info.chip_class >= GFX9) {
       if (sscreen->info.chip_class >= GFX10) {
@@ -589,7 +586,6 @@ static void si_shader_es(struct si_screen *sscreen, struct si_shader *shader)
 
    pm4->atom.emit = si_emit_shader_es;
    va = shader->bo->gpu_address;
-   si_pm4_add_bo(pm4, shader->bo, RADEON_USAGE_READ, RADEON_PRIO_SHADER_BINARY);
 
    if (shader->selector->type == PIPE_SHADER_VERTEX) {
       vgpr_comp_cnt = si_get_vs_vgpr_comp_cnt(sscreen, shader, false);
@@ -818,7 +814,6 @@ static void si_shader_gs(struct si_screen *sscreen, struct si_shader *shader)
       S_028B90_CNT(MIN2(gs_num_invocations, 127)) | S_028B90_ENABLE(gs_num_invocations > 0);
 
    va = shader->bo->gpu_address;
-   si_pm4_add_bo(pm4, shader->bo, RADEON_USAGE_READ, RADEON_PRIO_SHADER_BINARY);
 
    if (sscreen->info.chip_class >= GFX9) {
       unsigned input_prim = sel->info.properties[TGSI_PROPERTY_GS_INPUT_PRIM];
@@ -1088,7 +1083,6 @@ static void gfx10_shader_ngg(struct si_screen *sscreen, struct si_shader *shader
    }
 
    va = shader->bo->gpu_address;
-   si_pm4_add_bo(pm4, shader->bo, RADEON_USAGE_READ, RADEON_PRIO_SHADER_BINARY);
 
    if (es_type == PIPE_SHADER_VERTEX) {
       es_vgpr_comp_cnt = si_get_vs_vgpr_comp_cnt(sscreen, shader, false);
@@ -1126,11 +1120,13 @@ static void gfx10_shader_ngg(struct si_screen *sscreen, struct si_shader *shader
    else
       gs_vgpr_comp_cnt = 0; /* VGPR0 contains offsets 0, 1 */
 
+   unsigned wave_size = si_get_shader_wave_size(shader);
+
    si_pm4_set_reg(pm4, R_00B320_SPI_SHADER_PGM_LO_ES, va >> 8);
    si_pm4_set_reg(pm4, R_00B324_SPI_SHADER_PGM_HI_ES, va >> 40);
    si_pm4_set_reg(
       pm4, R_00B228_SPI_SHADER_PGM_RSRC1_GS,
-      S_00B228_VGPRS((shader->config.num_vgprs - 1) / (sscreen->ge_wave_size == 32 ? 8 : 4)) |
+      S_00B228_VGPRS((shader->config.num_vgprs - 1) / (wave_size == 32 ? 8 : 4)) |
          S_00B228_FLOAT_MODE(shader->config.float_mode) | S_00B228_DX10_CLAMP(1) |
          S_00B228_MEM_ORDERED(1) | S_00B228_WGP_MODE(1) |
          S_00B228_GS_VGPR_COMP_CNT(gs_vgpr_comp_cnt));
@@ -1388,7 +1384,6 @@ static void si_shader_vs(struct si_screen *sscreen, struct si_shader *shader,
    }
 
    va = shader->bo->gpu_address;
-   si_pm4_add_bo(pm4, shader->bo, RADEON_USAGE_READ, RADEON_PRIO_SHADER_BINARY);
 
    if (gs) {
       vgpr_comp_cnt = 0; /* only VertexID is needed for GS-COPY. */
@@ -1641,7 +1636,6 @@ static void si_shader_ps(struct si_screen *sscreen, struct si_shader *shader)
    shader->ctx_reg.ps.cb_shader_mask = cb_shader_mask;
 
    va = shader->bo->gpu_address;
-   si_pm4_add_bo(pm4, shader->bo, RADEON_USAGE_READ, RADEON_PRIO_SHADER_BINARY);
    si_pm4_set_reg(pm4, R_00B020_SPI_SHADER_PGM_LO_PS, va >> 8);
    si_pm4_set_reg(pm4, R_00B024_SPI_SHADER_PGM_HI_PS, S_00B024_MEM_BASE(va >> 40));
 
@@ -2550,7 +2544,7 @@ void si_schedule_initial_compile(struct si_context *sctx, unsigned processor,
 }
 
 /* Return descriptor slot usage masks from the given shader info. */
-void si_get_active_slot_masks(const struct si_shader_info *info, uint32_t *const_and_shader_buffers,
+void si_get_active_slot_masks(const struct si_shader_info *info, uint64_t *const_and_shader_buffers,
                               uint64_t *samplers_and_images)
 {
    unsigned start, num_shaderbufs, num_constbufs, num_images, num_msaa_images, num_samplers;
@@ -2564,7 +2558,7 @@ void si_get_active_slot_masks(const struct si_shader_info *info, uint32_t *const
 
    /* The layout is: sb[last] ... sb[0], cb[0] ... cb[last] */
    start = si_get_shaderbuf_slot(num_shaderbufs - 1);
-   *const_and_shader_buffers = u_bit_consecutive(start, num_shaderbufs + num_constbufs);
+   *const_and_shader_buffers = u_bit_consecutive64(start, num_shaderbufs + num_constbufs);
 
    /* The layout is:
     *   - fmask[last] ... fmask[0]     go to [15-last .. 15]
@@ -2657,9 +2651,15 @@ static void *si_create_shader_selector(struct pipe_context *ctx,
       sel->gs_input_verts_per_prim =
          u_vertices_per_prim(sel->info.properties[TGSI_PROPERTY_GS_INPUT_PRIM]);
 
-      /* EN_MAX_VERT_OUT_PER_GS_INSTANCE does not work with tesselation. */
+      /* EN_MAX_VERT_OUT_PER_GS_INSTANCE does not work with tesselation so
+       * we can't split workgroups. Disable ngg if any of the following conditions is true:
+       * - num_invocations * gs_max_out_vertices > 256
+       * - LDS usage is too high
+       */
       sel->tess_turns_off_ngg = sscreen->info.chip_class >= GFX10 &&
-                                sel->gs_num_invocations * sel->gs_max_out_vertices > 256;
+                                (sel->gs_num_invocations * sel->gs_max_out_vertices > 256 ||
+                                 sel->gs_num_invocations * sel->gs_max_out_vertices *
+                                 (sel->info.num_outputs * 4 + 1) > 6500 /* max dw per GS primitive */);
       break;
 
    case PIPE_SHADER_TESS_CTRL:
@@ -2758,9 +2758,10 @@ static void *si_create_shader_selector(struct pipe_context *ctx,
       sscreen->info.chip_class >= GFX10 &&
       sscreen->info.has_dedicated_vram &&
       sscreen->use_ngg_culling &&
-      /* Disallow TES by default, because TessMark results are mixed. */
       (sel->type == PIPE_SHADER_VERTEX ||
-       (sscreen->always_use_ngg_culling && sel->type == PIPE_SHADER_TESS_EVAL)) &&
+       (sel->type == PIPE_SHADER_TESS_EVAL &&
+        (sscreen->always_use_ngg_culling_all ||
+         sscreen->always_use_ngg_culling_tess))) &&
       sel->info.writes_position &&
       !sel->info.writes_viewport_index && /* cull only against viewport 0 */
       !sel->info.writes_memory && !sel->so.num_outputs &&
@@ -3301,14 +3302,12 @@ static void si_cs_preamble_add_vgt_flush(struct si_context *sctx)
       return;
 
    /* Done by Vulkan before VGT_FLUSH. */
-   si_pm4_cmd_begin(sctx->cs_preamble_state, PKT3_EVENT_WRITE);
+   si_pm4_cmd_add(sctx->cs_preamble_state, PKT3(PKT3_EVENT_WRITE, 0, 0));
    si_pm4_cmd_add(sctx->cs_preamble_state, EVENT_TYPE(V_028A90_VS_PARTIAL_FLUSH) | EVENT_INDEX(4));
-   si_pm4_cmd_end(sctx->cs_preamble_state, false);
 
    /* VGT_FLUSH is required even if VGT is idle. It resets VGT pointers. */
-   si_pm4_cmd_begin(sctx->cs_preamble_state, PKT3_EVENT_WRITE);
+   si_pm4_cmd_add(sctx->cs_preamble_state, PKT3(PKT3_EVENT_WRITE, 0, 0));
    si_pm4_cmd_add(sctx->cs_preamble_state, EVENT_TYPE(V_028A90_VGT_FLUSH) | EVENT_INDEX(0));
-   si_pm4_cmd_end(sctx->cs_preamble_state, false);
    sctx->cs_preamble_has_vgt_flush = true;
 }
 
@@ -3640,9 +3639,6 @@ static void si_init_tess_factor_ring(struct si_context *sctx)
 
    si_cs_preamble_add_vgt_flush(sctx);
 
-   si_pm4_add_bo(sctx->cs_preamble_state, si_resource(sctx->tess_rings), RADEON_USAGE_READWRITE,
-                 RADEON_PRIO_SHADER_RINGS);
-
    uint64_t factor_va =
       si_resource(sctx->tess_rings)->gpu_address + sctx->screen->tess_offchip_ring_size;
 
@@ -3705,7 +3701,9 @@ static struct si_pm4_state *si_build_vgt_shader_config(struct si_screen *screen,
    if (screen->info.chip_class >= GFX9)
       stages |= S_028B54_MAX_PRIMGRP_IN_WAVE(2);
 
-   if (screen->info.chip_class >= GFX10 && screen->ge_wave_size == 32) {
+   if (screen->info.chip_class >= GFX10 &&
+       /* GS fast launch hangs with Wave64, so always use Wave32. */
+       (screen->ge_wave_size == 32 || (key.u.ngg && key.u.ngg_gs_fast_launch))) {
       stages |= S_028B54_HS_W32_EN(1) |
                 S_028B54_GS_W32_EN(key.u.ngg) | /* legacy GS only supports Wave64 */
                 S_028B54_VS_W32_EN(1);