aco: fix vgpr alloc granule with wave32

We still need to increase the number of physical vgprs

Totals from affected shaders:
SGPRS: 671976 -> 675288 (0.49 %)
VGPRS: 550112 -> 562596 (2.27 %)
Spilled SGPRs: 0 -> 0 (0.00 %)
Spilled VGPRs: 0 -> 0 (0.00 %)
Code Size: 27621660 -> 27606532 (-0.05 %) bytes
Max Waves: 81083 -> 87833 (8.32 %)
Instructions: 5391560 -> 5389031 (-0.05 %)

Signed-off-by: Rhys Perry <pendingchaos02@gmail.com>
Reviewed-by: Daniel Schürmann <daniel@schuermann.dev>

diff --git a/src/amd/compiler/aco_instruction_selection_setup.cpp b/src/amd/compiler/aco_instruction_selection_setup.cpp
index 4d4ff44101c4de242ec542837cbe1acd78dbe372..d55d4105d393c5a20411e6d7dea2f1b6ffbba998 100644 (file)
--- a/src/amd/compiler/aco_instruction_selection_setup.cpp
+++ b/src/amd/compiler/aco_instruction_selection_setup.cpp
@@ -800,11 +800,13 @@ setup_isel_context(Program* program,
    program->lds_alloc_granule = args->options->chip_class >= GFX7 ? 512 : 256;
    program->lds_limit = args->options->chip_class >= GFX7 ? 65536 : 32768;
    program->vgpr_limit = 256;
+   program->vgpr_alloc_granule = 3;
 
    if (args->options->chip_class >= GFX10) {
       program->physical_sgprs = 2560; /* doesn't matter as long as it's at least 128 * 20 */
       program->sgpr_alloc_granule = 127;
       program->sgpr_limit = 106;
+      program->vgpr_alloc_granule = program->wave_size == 32 ? 7 : 3;
    } else if (program->chip_class >= GFX8) {
       program->physical_sgprs = 800;
       program->sgpr_alloc_granule = 15;

diff --git a/src/amd/compiler/aco_ir.h b/src/amd/compiler/aco_ir.h
index 2928d831c8b7484294c16c30666b4facb743c6e4..bc71a95641aa8220a3ed39b7b2dc05c235b3a682 100644 (file)
--- a/src/amd/compiler/aco_ir.h
+++ b/src/amd/compiler/aco_ir.h
@@ -1161,6 +1161,7 @@ public:
    uint16_t sgpr_limit;
    uint16_t physical_sgprs;
    uint16_t sgpr_alloc_granule; /* minus one. must be power of two */
+   uint16_t vgpr_alloc_granule; /* minus one. must be power of two */
 
    bool needs_vcc = false;
    bool needs_xnack_mask = false;
@@ -1248,11 +1249,13 @@ void aco_print_program(Program *program, FILE *output);
 /* number of sgprs that need to be allocated but might notbe addressable as s0-s105 */
 uint16_t get_extra_sgprs(Program *program);
 
-/* get number of sgprs allocated required to address a number of sgprs */
+/* get number of sgprs/vgprs allocated required to address a number of sgprs/vgprs */
 uint16_t get_sgpr_alloc(Program *program, uint16_t addressable_sgprs);
+uint16_t get_vgpr_alloc(Program *program, uint16_t addressable_vgprs);
 
-/* return number of addressable SGPRs for max_waves */
+/* return number of addressable sgprs/vgprs for max_waves */
 uint16_t get_addr_sgpr_from_waves(Program *program, uint16_t max_waves);
+uint16_t get_addr_vgpr_from_waves(Program *program, uint16_t max_waves);
 
 typedef struct {
    const int16_t opcode_gfx7[static_cast<int>(aco_opcode::num_opcodes)];

diff --git a/src/amd/compiler/aco_live_var_analysis.cpp b/src/amd/compiler/aco_live_var_analysis.cpp
index 4255d56173be7e4ceb9f6efc40e967771066ac38..2841ba208f667304963c48f261e949f085e46ee1 100644 (file)
--- a/src/amd/compiler/aco_live_var_analysis.cpp
+++ b/src/amd/compiler/aco_live_var_analysis.cpp
@@ -264,6 +264,13 @@ uint16_t get_sgpr_alloc(Program *program, uint16_t addressable_sgprs)
    return align(std::max(sgprs, granule), granule);
 }
 
+uint16_t get_vgpr_alloc(Program *program, uint16_t addressable_vgprs)
+{
+   assert(addressable_vgprs <= program->vgpr_limit);
+   uint16_t granule = program->vgpr_alloc_granule + 1;
+   return align(std::max(addressable_vgprs, granule), granule);
+}
+
 uint16_t get_addr_sgpr_from_waves(Program *program, uint16_t max_waves)
 {
     uint16_t sgprs = program->physical_sgprs / max_waves & ~program->sgpr_alloc_granule;
@@ -271,6 +278,12 @@ uint16_t get_addr_sgpr_from_waves(Program *program, uint16_t max_waves)
     return std::min(sgprs, program->sgpr_limit);
 }
 
+uint16_t get_addr_vgpr_from_waves(Program *program, uint16_t max_waves)
+{
+    uint16_t vgprs = 256 / max_waves & ~program->vgpr_alloc_granule;
+    return std::min(vgprs, program->vgpr_limit);
+}
+
 void update_vgpr_sgpr_demand(Program* program, const RegisterDemand new_demand)
 {
    /* TODO: max_waves_per_simd, simd_per_cu and the number of physical vgprs for Navi */
@@ -281,14 +294,13 @@ void update_vgpr_sgpr_demand(Program* program, const RegisterDemand new_demand)
    unsigned simd_per_cu_wgp = wgp ? simd_per_cu * 2 : simd_per_cu;
    unsigned lds_limit = wgp ? program->lds_limit * 2 : program->lds_limit;
 
-   const int16_t vgpr_alloc = std::max<int16_t>(4, (new_demand.vgpr + 3) & ~3);
    /* this won't compile, register pressure reduction necessary */
    if (new_demand.vgpr > program->vgpr_limit || new_demand.sgpr > program->sgpr_limit) {
       program->num_waves = 0;
       program->max_reg_demand = new_demand;
    } else {
       program->num_waves = program->physical_sgprs / get_sgpr_alloc(program, new_demand.sgpr);
-      program->num_waves = std::min<uint16_t>(program->num_waves, 256 / vgpr_alloc);
+      program->num_waves = std::min<uint16_t>(program->num_waves, 256 / get_vgpr_alloc(program, new_demand.vgpr));
       program->max_waves = max_waves_per_simd;
 
       /* adjust max_waves for workgroup and LDS limits */
@@ -314,7 +326,7 @@ void update_vgpr_sgpr_demand(Program* program, const RegisterDemand new_demand)
 
       /* incorporate max_waves and calculate max_reg_demand */
       program->num_waves = std::min<uint16_t>(program->num_waves, program->max_waves);
-      program->max_reg_demand.vgpr = int16_t((256 / program->num_waves) & ~3);
+      program->max_reg_demand.vgpr = get_addr_vgpr_from_waves(program, program->num_waves);
       program->max_reg_demand.sgpr = get_addr_sgpr_from_waves(program, program->num_waves);
    }
 }

diff --git a/src/amd/compiler/aco_scheduler.cpp b/src/amd/compiler/aco_scheduler.cpp
index 5c164703ebff6da29e1dab4bb55b8708ceaa1d1b..665fcb2db6f03ec39fd6eace565dbaf1731990c2 100644 (file)
--- a/src/amd/compiler/aco_scheduler.cpp
+++ b/src/amd/compiler/aco_scheduler.cpp
@@ -934,7 +934,8 @@ void schedule_program(Program *program, live& live_vars)
       ctx.num_waves = 8;
 
    assert(ctx.num_waves > 0 && ctx.num_waves <= program->num_waves);
-   ctx.max_registers = { int16_t(((256 / ctx.num_waves) & ~3) - 2), int16_t(get_addr_sgpr_from_waves(program, ctx.num_waves))};
+   ctx.max_registers = { int16_t(get_addr_vgpr_from_waves(program, ctx.num_waves) - 2),
+                         int16_t(get_addr_sgpr_from_waves(program, ctx.num_waves))};
 
    for (Block& block : program->blocks)
       schedule_block(ctx, program, &block, live_vars);