aco: fix assembly of FLAT/GLOBAL atomics

[mesa.git] / src / amd / compiler / aco_assembler.cpp

diff --git a/src/amd/compiler/aco_assembler.cpp b/src/amd/compiler/aco_assembler.cpp
index d569d5ca25fdfcab4d73be05140189760ebb8bd2..c9b5ca1068447a351264e9aee50de56891731a85 100644 (file)
--- a/src/amd/compiler/aco_assembler.cpp
+++ b/src/amd/compiler/aco_assembler.cpp
@@ -105,7 +105,7 @@ void emit_instruction(asm_context& ctx, std::vector<uint32_t>& out, Instruction*
       encoding |=
          !instr->definitions.empty() && !(instr->definitions[0].physReg() == scc) ?
          instr->definitions[0].physReg() << 16 :
-         !instr->operands.empty() && !(instr->operands[0].physReg() == scc) ?
+         !instr->operands.empty() && instr->operands[0].physReg() <= 127 ?
          instr->operands[0].physReg() << 16 : 0;
       encoding |= sopk->imm;
       out.push_back(encoding);
@@ -170,10 +170,10 @@ void emit_instruction(asm_context& ctx, std::vector<uint32_t>& out, Instruction*
       }
 
       if (is_load || instr->operands.size() >= 3) { /* SDATA */
-         encoding |= (is_load ? instr->definitions[0].physReg().reg : instr->operands[2].physReg().reg) << 6;
+         encoding |= (is_load ? instr->definitions[0].physReg() : instr->operands[2].physReg()) << 6;
       }
       if (instr->operands.size() >= 1) { /* SBASE */
-         encoding |= instr->operands[0].physReg().reg >> 1;
+         encoding |= instr->operands[0].physReg() >> 1;
       }
 
       out.push_back(encoding);
@@ -213,27 +213,27 @@ void emit_instruction(asm_context& ctx, std::vector<uint32_t>& out, Instruction*
    case Format::VOP2: {
       uint32_t encoding = 0;
       encoding |= opcode << 25;
-      encoding |= (0xFF & instr->definitions[0].physReg().reg) << 17;
-      encoding |= (0xFF & instr->operands[1].physReg().reg) << 9;
-      encoding |= instr->operands[0].physReg().reg;
+      encoding |= (0xFF & instr->definitions[0].physReg()) << 17;
+      encoding |= (0xFF & instr->operands[1].physReg()) << 9;
+      encoding |= instr->operands[0].physReg();
       out.push_back(encoding);
       break;
    }
    case Format::VOP1: {
       uint32_t encoding = (0b0111111 << 25);
       if (!instr->definitions.empty())
-         encoding |= (0xFF & instr->definitions[0].physReg().reg) << 17;
+         encoding |= (0xFF & instr->definitions[0].physReg()) << 17;
       encoding |= opcode << 9;
       if (!instr->operands.empty())
-         encoding |= instr->operands[0].physReg().reg;
+         encoding |= instr->operands[0].physReg();
       out.push_back(encoding);
       break;
    }
    case Format::VOPC: {
       uint32_t encoding = (0b0111110 << 25);
       encoding |= opcode << 17;
-      encoding |= (0xFF & instr->operands[1].physReg().reg) << 9;
-      encoding |= instr->operands[0].physReg().reg;
+      encoding |= (0xFF & instr->operands[1].physReg()) << 9;
+      encoding |= instr->operands[0].physReg();
       out.push_back(encoding);
       break;
    }
@@ -248,14 +248,14 @@ void emit_instruction(asm_context& ctx, std::vector<uint32_t>& out, Instruction*
       }
 
       assert(encoding);
-      encoding |= (0xFF & instr->definitions[0].physReg().reg) << 18;
+      encoding |= (0xFF & instr->definitions[0].physReg()) << 18;
       encoding |= opcode << 16;
       encoding |= interp->attribute << 10;
       encoding |= interp->component << 8;
       if (instr->opcode == aco_opcode::v_interp_mov_f32)
          encoding |= (0x3 & instr->operands[0].constantValue());
       else
-         encoding |= (0xFF & instr->operands[0].physReg().reg);
+         encoding |= (0xFF & instr->operands[0].physReg());
       out.push_back(encoding);
       break;
    }
@@ -279,7 +279,7 @@ void emit_instruction(asm_context& ctx, std::vector<uint32_t>& out, Instruction*
       encoding |= (0xFF & reg) << 16;
       reg = instr->operands.size() >= 2 && !(instr->operands[1].physReg() == m0) ? instr->operands[1].physReg() : 0;
       encoding |= (0xFF & reg) << 8;
-      encoding |= (0xFF & instr->operands[0].physReg().reg);
+      encoding |= (0xFF & instr->operands[0].physReg());
       out.push_back(encoding);
       break;
    }
@@ -306,9 +306,9 @@ void emit_instruction(asm_context& ctx, std::vector<uint32_t>& out, Instruction*
       encoding |= instr->operands[2].physReg() << 24;
       encoding |= (mubuf->tfe ? 1 : 0) << 23;
       encoding |= (instr->operands[1].physReg() >> 2) << 16;
-      unsigned reg = instr->operands.size() > 3 ? instr->operands[3].physReg() : instr->definitions[0].physReg().reg;
+      unsigned reg = instr->operands.size() > 3 ? instr->operands[3].physReg() : instr->definitions[0].physReg();
       encoding |= (0xFF & reg) << 8;
-      encoding |= (0xFF & instr->operands[0].physReg().reg);
+      encoding |= (0xFF & instr->operands[0].physReg());
       out.push_back(encoding);
       break;
    }
@@ -317,6 +317,7 @@ void emit_instruction(asm_context& ctx, std::vector<uint32_t>& out, Instruction*
 
       uint32_t img_format = ac_get_tbuffer_format(ctx.chip_class, mtbuf->dfmt, mtbuf->nfmt);
       uint32_t encoding = (0b111010 << 26);
+      assert(img_format <= 0x7F);
       assert(!mtbuf->dlc || ctx.chip_class >= GFX10);
       encoding |= (mtbuf->dlc ? 1 : 0) << 15; /* DLC bit replaces one bit of the OPCODE on GFX10 */
       encoding |= (mtbuf->glc ? 1 : 0) << 14;
@@ -334,13 +335,13 @@ void emit_instruction(asm_context& ctx, std::vector<uint32_t>& out, Instruction*
       out.push_back(encoding);
       encoding = 0;
 
-      encoding |= instr->operands[2].physReg().reg << 24;
+      encoding |= instr->operands[2].physReg() << 24;
       encoding |= (mtbuf->tfe ? 1 : 0) << 23;
       encoding |= (mtbuf->slc ? 1 : 0) << 22;
-      encoding |= (instr->operands[1].physReg().reg >> 2) << 16;
-      unsigned reg = instr->operands.size() > 3 ? instr->operands[3].physReg().reg : instr->definitions[0].physReg().reg;
+      encoding |= (instr->operands[1].physReg() >> 2) << 16;
+      unsigned reg = instr->operands.size() > 3 ? instr->operands[3].physReg() : instr->definitions[0].physReg();
       encoding |= (0xFF & reg) << 8;
-      encoding |= (0xFF & instr->operands[0].physReg().reg);
+      encoding |= (0xFF & instr->operands[0].physReg());
 
       if (ctx.chip_class >= GFX10) {
          encoding |= (((opcode & 0x08) >> 4) << 21); /* MSB of 4-bit OPCODE */
@@ -370,11 +371,11 @@ void emit_instruction(asm_context& ctx, std::vector<uint32_t>& out, Instruction*
       }
       encoding |= (0xF & mimg->dmask) << 8;
       out.push_back(encoding);
-      encoding = (0xFF & instr->operands[0].physReg().reg); /* VADDR */
+      encoding = (0xFF & instr->operands[0].physReg()); /* VADDR */
       if (!instr->definitions.empty()) {
-         encoding |= (0xFF & instr->definitions[0].physReg().reg) << 8; /* VDATA */
+         encoding |= (0xFF & instr->definitions[0].physReg()) << 8; /* VDATA */
       } else if (instr->operands.size() == 4) {
-         encoding |= (0xFF & instr->operands[3].physReg().reg) << 8; /* VDATA */
+         encoding |= (0xFF & instr->operands[3].physReg()) << 8; /* VDATA */
       }
       encoding |= (0x1F & (instr->operands[1].physReg() >> 2)) << 16; /* T# (resource) */
       if (instr->operands.size() > 2)
@@ -398,9 +399,14 @@ void emit_instruction(asm_context& ctx, std::vector<uint32_t>& out, Instruction*
       if (ctx.chip_class <= GFX9) {
          assert(flat->offset <= 0x1fff);
          encoding |= flat->offset & 0x1fff;
+      } else if (instr->format == Format::FLAT) {
+         /* GFX10 has a 12-bit immediate OFFSET field,
+          * but it has a hw bug: it ignores the offset, called FlatSegmentOffsetBug
+          */
+         assert(flat->offset == 0);
       } else {
-         assert(flat->offset <= 0x0fff);
-         encoding |= flat->offset & 0x0fff;
+         assert(flat->offset <= 0xfff);
+         encoding |= flat->offset & 0xfff;
       }
       if (instr->format == Format::SCRATCH)
          encoding |= 1 << 14;
@@ -419,7 +425,7 @@ void emit_instruction(asm_context& ctx, std::vector<uint32_t>& out, Instruction*
       encoding = (0xFF & instr->operands[0].physReg());
       if (!instr->definitions.empty())
          encoding |= (0xFF & instr->definitions[0].physReg()) << 24;
-      else
+      if (instr->operands.size() >= 3)
          encoding |= (0xFF & instr->operands[2].physReg()) << 8;
       if (!instr->operands[1].isUndefined()) {
          assert(ctx.chip_class >= GFX10 || instr->operands[1].physReg() != 0x7F);
@@ -450,10 +456,10 @@ void emit_instruction(asm_context& ctx, std::vector<uint32_t>& out, Instruction*
       encoding |= exp->dest << 4;
       encoding |= exp->enabled_mask;
       out.push_back(encoding);
-      encoding = 0xFF & exp->operands[0].physReg().reg;
-      encoding |= (0xFF & exp->operands[1].physReg().reg) << 8;
-      encoding |= (0xFF & exp->operands[2].physReg().reg) << 16;
-      encoding |= (0xFF & exp->operands[3].physReg().reg) << 24;
+      encoding = 0xFF & exp->operands[0].physReg();
+      encoding |= (0xFF & exp->operands[1].physReg()) << 8;
+      encoding |= (0xFF & exp->operands[2].physReg()) << 16;
+      encoding |= (0xFF & exp->operands[3].physReg()) << 24;
       out.push_back(encoding);
       break;
    }
@@ -494,7 +500,7 @@ void emit_instruction(asm_context& ctx, std::vector<uint32_t>& out, Instruction*
             encoding |= vop3->opsel[i] << (11+i);
          if (instr->definitions.size() == 2)
             encoding |= instr->definitions[1].physReg() << 8;
-         encoding |= (0xFF & instr->definitions[0].physReg().reg);
+         encoding |= (0xFF & instr->definitions[0].physReg());
          out.push_back(encoding);
          encoding = 0;
          if (instr->opcode == aco_opcode::v_interp_mov_f32) {
@@ -523,7 +529,7 @@ void emit_instruction(asm_context& ctx, std::vector<uint32_t>& out, Instruction*
          encoding |= dpp->neg[0] << 20;
          encoding |= dpp->bound_ctrl << 19;
          encoding |= dpp->dpp_ctrl << 8;
-         encoding |= (0xFF) & dpp_op.physReg().reg;
+         encoding |= (0xFF) & dpp_op.physReg();
          out.push_back(encoding);
          return;
       } else {
@@ -624,7 +630,8 @@ static void fix_branches_gfx10(asm_context& ctx, std::vector<uint32_t>& out)
       if (gfx10_3f_bug) {
          /* Insert an s_nop after the branch */
          constexpr uint32_t s_nop_0 = 0xbf800000u;
-         auto out_pos = std::next(out.begin(), buggy_branch_it->first + 1);
+         int s_nop_pos = buggy_branch_it->first + 1;
+         auto out_pos = std::next(out.begin(), s_nop_pos);
          out.insert(out_pos, s_nop_0);
 
          /* Update the offset of each affected block */
@@ -636,6 +643,16 @@ static void fix_branches_gfx10(asm_context& ctx, std::vector<uint32_t>& out)
          /* Update the branches following the current one */
          for (auto branch_it = std::next(buggy_branch_it); branch_it != ctx.branches.end(); ++branch_it)
             branch_it->first++;
+
+         /* Find first constant address after the inserted instruction */
+         auto caddr_it = std::find_if(ctx.constaddrs.begin(), ctx.constaddrs.end(), [s_nop_pos](const int &caddr_pos) -> bool {
+            return caddr_pos >= s_nop_pos;
+         });
+
+         /* Update the locations of constant addresses */
+         for (; caddr_it != ctx.constaddrs.end(); ++caddr_it)
+            (*caddr_it)++;
+
       }
    } while (gfx10_3f_bug);
 }