aco: add missing add_to_hazard_query

[mesa.git] / src / amd / compiler / aco_register_allocation.cpp

diff --git a/src/amd/compiler/aco_register_allocation.cpp b/src/amd/compiler/aco_register_allocation.cpp
index 096fe80f6ea87977e319dc697f0e822efbf18600..862b00d859b6d97f4553057aa9673f5391f4ec86 100644 (file)
--- a/src/amd/compiler/aco_register_allocation.cpp
+++ b/src/amd/compiler/aco_register_allocation.cpp
@@ -185,6 +185,13 @@ public:
       return false;
    }
 
+   bool is_empty_or_blocked(PhysReg start) {
+      if (regs[start] == 0xF0000000) {
+         return subdword_regs[start][start.byte()] + 1 <= 1;
+      }
+      return regs[start] + 1 <= 1;
+   }
+
    void clear(PhysReg start, RegClass rc) {
       if (rc.is_subdword())
          fill_subdword(start, rc.bytes(), 0);
@@ -311,6 +318,9 @@ void print_regs(ra_ctx& ctx, bool vgprs, RegisterFile& reg_file)
 
 unsigned get_subdword_operand_stride(chip_class chip, const aco_ptr<Instruction>& instr, unsigned idx, RegClass rc)
 {
+   /* v_readfirstlane_b32 cannot use SDWA */
+   if (instr->opcode == aco_opcode::p_as_uniform)
+      return 4;
    if (instr->format == Format::PSEUDO && chip >= GFX8)
       return rc.bytes() % 2 == 0 ? 2 : 1;
 
@@ -584,7 +594,7 @@ std::pair<PhysReg, bool> get_reg_simple(ra_ctx& ctx,
    RegClass rc = info.rc;
 
    if (stride == 1) {
-
+      info.rc = RegClass(rc.type(), size);
       for (unsigned stride = 8; stride > 1; stride /= 2) {
          if (size % stride)
             continue;
@@ -770,12 +780,13 @@ bool get_regs_for_copies(ra_ctx& ctx,
             PhysReg reg(def_reg_lo);
             for (unsigned i = 0; i < instr->operands.size(); i++) {
                if (instr->operands[i].isTemp() && instr->operands[i].tempId() == id) {
-                  assert(!reg_file.test(reg, var.rc.bytes()));
-                  res = {reg, !var.rc.is_subdword() || (reg.byte() % info.stride == 0)};
+                  res = {reg, (!var.rc.is_subdword() || (reg.byte() % info.stride == 0)) && !reg_file.test(reg, var.rc.bytes())};
                   break;
                }
                reg.reg_b += instr->operands[i].bytes();
             }
+            if (!res.second)
+               res = {var.reg, !reg_file.test(var.reg, var.rc.bytes())};
          } else {
             info.lb = def_reg_lo;
             info.ub = def_reg_hi + 1;
@@ -946,9 +957,11 @@ std::pair<PhysReg, bool> get_reg_impl(ra_ctx& ctx,
    unsigned reg_hi = lb + size - 1;
    for (reg_lo = lb, reg_hi = lb + size - 1; reg_hi < ub; reg_lo += stride, reg_hi += stride) {
       /* first check the edges: this is what we have to fix to allow for num_moves > size */
-      if (reg_lo > lb && reg_file[reg_lo] != 0 && reg_file.get_id(PhysReg(reg_lo)) == reg_file.get_id(PhysReg(reg_lo).advance(-1)))
+      if (reg_lo > lb && !reg_file.is_empty_or_blocked(PhysReg(reg_lo)) &&
+          reg_file.get_id(PhysReg(reg_lo)) == reg_file.get_id(PhysReg(reg_lo).advance(-1)))
          continue;
-      if (reg_hi < ub - 1 && reg_file[reg_hi] != 0 && reg_file.get_id(PhysReg(reg_hi).advance(3)) == reg_file.get_id(PhysReg(reg_hi).advance(4)))
+      if (reg_hi < ub - 1 && !reg_file.is_empty_or_blocked(PhysReg(reg_hi).advance(3)) &&
+          reg_file.get_id(PhysReg(reg_hi).advance(3)) == reg_file.get_id(PhysReg(reg_hi).advance(4)))
          continue;
 
       /* second, check that we have at most k=num_moves elements in the window
@@ -1364,19 +1377,21 @@ PhysReg get_reg_create_vector(ra_ctx& ctx,
    /* collect variables to be moved */
    std::set<std::pair<unsigned, unsigned>> vars = collect_vars(ctx, reg_file, PhysReg{best_pos}, size);
 
-   /* GFX9+: move killed operands which aren't yet at the correct position
-    * Moving all killed operands generally leads to more register swaps.
-    * This is only done on GFX9+ because of the cheap v_swap instruction.
-    */
-   if (ctx.program->chip_class >= GFX9) {
-      for (unsigned i = 0, offset = 0; i < instr->operands.size(); offset += instr->operands[i].bytes(), i++) {
-         if (instr->operands[i].isTemp() &&
-             instr->operands[i].isFirstKillBeforeDef() &&
-             instr->operands[i].getTemp().type() == rc.type() &&
-             instr->operands[i].physReg().reg_b != best_pos * 4 + offset) {
-            vars.emplace(instr->operands[i].bytes(), instr->operands[i].tempId());
-            reg_file.clear(instr->operands[i]);
-         }
+   for (unsigned i = 0, offset = 0; i < instr->operands.size(); offset += instr->operands[i].bytes(), i++) {
+      if (!instr->operands[i].isTemp() || !instr->operands[i].isFirstKillBeforeDef() ||
+          instr->operands[i].getTemp().type() != rc.type())
+         continue;
+      bool correct_pos = instr->operands[i].physReg().reg_b == best_pos * 4 + offset;
+      /* GFX9+: move killed operands which aren't yet at the correct position
+       * Moving all killed operands generally leads to more register swaps.
+       * This is only done on GFX9+ because of the cheap v_swap instruction.
+       */
+      if (ctx.program->chip_class >= GFX9 && !correct_pos) {
+         vars.emplace(instr->operands[i].bytes(), instr->operands[i].tempId());
+         reg_file.clear(instr->operands[i]);
+      /* fill operands which are in the correct position to avoid overwriting */
+      } else if (correct_pos) {
+         reg_file.fill(instr->operands[i]);
       }
    }
    ASSERTED bool success = false;