X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Famd%2Fcompiler%2Faco_register_allocation.cpp;h=ca7a0fa4815745a3db6906f60bf0fcbcd013183f;hb=0654a9e82307b854824fc1092a8b79632a2703c0;hp=e702e09059dcc06db9091ced8c6090c1660029fb;hpb=99ca96fbf58975d49e4ad131f907c5b01e12db85;p=mesa.git

diff --git a/src/amd/compiler/aco_register_allocation.cpp b/src/amd/compiler/aco_register_allocation.cpp
index e702e09059d..ca7a0fa4815 100644
--- a/src/amd/compiler/aco_register_allocation.cpp
+++ b/src/amd/compiler/aco_register_allocation.cpp
@@ -38,6 +38,11 @@
 namespace aco {
 namespace {
 
+unsigned get_subdword_operand_stride(chip_class chip, const aco_ptr<Instruction>& instr, unsigned idx, RegClass rc);
+void add_subdword_operand(chip_class chip, aco_ptr<Instruction>& instr, unsigned idx, unsigned byte, RegClass rc);
+std::pair<unsigned, unsigned> get_subdword_definition_info(Program *program, const aco_ptr<Instruction>& instr, RegClass rc);
+void add_subdword_definition(Program *program, aco_ptr<Instruction>& instr, unsigned idx, PhysReg reg, bool is_partial);
+
 struct assignment {
    PhysReg reg;
    RegClass rc;
@@ -64,6 +69,7 @@ struct ra_ctx {
    std::unordered_map<unsigned, phi_info> phi_map;
    std::unordered_map<unsigned, unsigned> affinities;
    std::unordered_map<unsigned, Instruction*> vectors;
+   std::unordered_map<unsigned, Instruction*> split_vectors;
    aco_ptr<Instruction> pseudo_dummy;
    unsigned max_used_sgpr = 0;
    unsigned max_used_vgpr = 0;
@@ -80,11 +86,6 @@ struct ra_ctx {
    }
 };
 
-bool instr_can_access_subdword(aco_ptr<Instruction>& instr)
-{
-   return instr->isSDWA() || instr->format == Format::PSEUDO;
-}
-
 struct DefInfo {
    uint16_t lb;
    uint16_t ub;
@@ -92,7 +93,7 @@ struct DefInfo {
    uint8_t stride;
    RegClass rc;
 
-   DefInfo(ra_ctx& ctx, aco_ptr<Instruction>& instr, RegClass rc) : rc(rc) {
+   DefInfo(ra_ctx& ctx, aco_ptr<Instruction>& instr, RegClass rc_, int operand) : rc(rc_) {
       size = rc.size();
       stride = 1;
 
@@ -108,14 +109,23 @@ struct DefInfo {
             stride = 4;
       }
 
-      if (rc.is_subdword()) {
+      if (rc.is_subdword() && operand >= 0) {
          /* stride in bytes */
-         if(!instr_can_access_subdword(instr))
-            stride = 4;
-         else if (rc.bytes() % 4 == 0)
-            stride = 4;
-         else if (rc.bytes() % 2 == 0)
-            stride = 2;
+         stride = get_subdword_operand_stride(ctx.program->chip_class, instr, operand, rc);
+      } else if (rc.is_subdword()) {
+         std::pair<unsigned, unsigned> info = get_subdword_definition_info(ctx.program, instr, rc);
+         stride = info.first;
+         if (info.second > rc.bytes()) {
+            rc = RegClass::get(rc.type(), info.second);
+            size = rc.size();
+            /* we might still be able to put the definition in the high half,
+             * but that's only useful for affinities and this information isn't
+             * used for them */
+            stride = align(stride, info.second);
+            if (!rc.is_subdword())
+               stride = DIV_ROUND_UP(stride, 4);
+         }
+         assert(stride > 0);
       }
    }
 };
@@ -175,6 +185,13 @@ public:
       return false;
    }
 
+   bool is_empty_or_blocked(PhysReg start) {
+      if (regs[start] == 0xF0000000) {
+         return subdword_regs[start][start.byte()] + 1 <= 1;
+      }
+      return regs[start] + 1 <= 1;
+   }
+
    void clear(PhysReg start, RegClass rc) {
       if (rc.is_subdword())
          fill_subdword(start, rc.bytes(), 0);
@@ -204,6 +221,10 @@ public:
       clear(def.physReg(), def.regClass());
    }
 
+   unsigned get_id(PhysReg reg) {
+      return regs[reg] == 0xF0000000 ? subdword_regs[reg][reg.byte()] : regs[reg];
+   }
+
 private:
    void fill(PhysReg start, unsigned size, uint32_t val) {
       for (unsigned i = 0; i < size; i++)
@@ -295,6 +316,204 @@ void print_regs(ra_ctx& ctx, bool vgprs, RegisterFile& reg_file)
 #endif
 
 
+unsigned get_subdword_operand_stride(chip_class chip, const aco_ptr<Instruction>& instr, unsigned idx, RegClass rc)
+{
+   /* v_readfirstlane_b32 cannot use SDWA */
+   if (instr->opcode == aco_opcode::p_as_uniform)
+      return 4;
+   if (instr->format == Format::PSEUDO && chip >= GFX8)
+      return rc.bytes() % 2 == 0 ? 2 : 1;
+
+   if (instr->opcode == aco_opcode::v_cvt_f32_ubyte0) {
+      return 1;
+   } else if (can_use_SDWA(chip, instr)) {
+      return rc.bytes() % 2 == 0 ? 2 : 1;
+   } else if (rc.bytes() == 2 && can_use_opsel(chip, instr->opcode, idx, 1)) {
+      return 2;
+   }
+
+   switch (instr->opcode) {
+   case aco_opcode::ds_write_b8:
+   case aco_opcode::ds_write_b16:
+      return chip >= GFX8 ? 2 : 4;
+   case aco_opcode::buffer_store_byte:
+   case aco_opcode::buffer_store_short:
+   case aco_opcode::flat_store_byte:
+   case aco_opcode::flat_store_short:
+   case aco_opcode::scratch_store_byte:
+   case aco_opcode::scratch_store_short:
+   case aco_opcode::global_store_byte:
+   case aco_opcode::global_store_short:
+      return chip >= GFX9 ? 2 : 4;
+   default:
+      break;
+   }
+
+   return 4;
+}
+
+void add_subdword_operand(chip_class chip, aco_ptr<Instruction>& instr, unsigned idx, unsigned byte, RegClass rc)
+{
+   if (instr->format == Format::PSEUDO || byte == 0)
+      return;
+
+   assert(rc.bytes() <= 2);
+
+   if (!instr->usesModifiers() && instr->opcode == aco_opcode::v_cvt_f32_ubyte0) {
+      switch (byte) {
+      case 0:
+         instr->opcode = aco_opcode::v_cvt_f32_ubyte0;
+         break;
+      case 1:
+         instr->opcode = aco_opcode::v_cvt_f32_ubyte1;
+         break;
+      case 2:
+         instr->opcode = aco_opcode::v_cvt_f32_ubyte2;
+         break;
+      case 3:
+         instr->opcode = aco_opcode::v_cvt_f32_ubyte3;
+         break;
+      }
+      return;
+   } else if (can_use_SDWA(chip, instr)) {
+      convert_to_SDWA(chip, instr);
+      return;
+   } else if (rc.bytes() == 2 && can_use_opsel(chip, instr->opcode, idx, byte / 2)) {
+      VOP3A_instruction *vop3 = static_cast<VOP3A_instruction *>(instr.get());
+      vop3->opsel |= (byte / 2) << idx;
+      return;
+   }
+
+   if (chip >= GFX8 && instr->opcode == aco_opcode::ds_write_b8 && byte == 2) {
+      instr->opcode = aco_opcode::ds_write_b8_d16_hi;
+      return;
+   }
+   if (chip >= GFX8 && instr->opcode == aco_opcode::ds_write_b16 && byte == 2) {
+      instr->opcode = aco_opcode::ds_write_b16_d16_hi;
+      return;
+   }
+
+   if (chip >= GFX9 && byte == 2) {
+      if (instr->opcode == aco_opcode::buffer_store_byte)
+         instr->opcode = aco_opcode::buffer_store_byte_d16_hi;
+      else if (instr->opcode == aco_opcode::buffer_store_short)
+         instr->opcode = aco_opcode::buffer_store_short_d16_hi;
+      else if (instr->opcode == aco_opcode::flat_store_byte)
+         instr->opcode = aco_opcode::flat_store_byte_d16_hi;
+      else if (instr->opcode == aco_opcode::flat_store_short)
+         instr->opcode = aco_opcode::flat_store_short_d16_hi;
+      else if (instr->opcode == aco_opcode::scratch_store_byte)
+         instr->opcode = aco_opcode::scratch_store_byte_d16_hi;
+      else if (instr->opcode == aco_opcode::scratch_store_short)
+         instr->opcode = aco_opcode::scratch_store_short_d16_hi;
+      else if (instr->opcode == aco_opcode::global_store_byte)
+         instr->opcode = aco_opcode::global_store_byte_d16_hi;
+      else if (instr->opcode == aco_opcode::global_store_short)
+         instr->opcode = aco_opcode::global_store_short_d16_hi;
+      else
+         unreachable("Something went wrong: Impossible register assignment.");
+   }
+}
+
+/* minimum_stride, bytes_written */
+std::pair<unsigned, unsigned> get_subdword_definition_info(Program *program, const aco_ptr<Instruction>& instr, RegClass rc)
+{
+   chip_class chip = program->chip_class;
+
+   if (instr->format == Format::PSEUDO && chip >= GFX8)
+      return std::make_pair(rc.bytes() % 2 == 0 ? 2 : 1, rc.bytes());
+   else if (instr->format == Format::PSEUDO)
+      return std::make_pair(4, rc.size() * 4u);
+
+   unsigned bytes_written = chip >= GFX10 ? rc.bytes() : 4u;
+   switch (instr->opcode) {
+   case aco_opcode::v_mad_f16:
+   case aco_opcode::v_mad_u16:
+   case aco_opcode::v_mad_i16:
+   case aco_opcode::v_fma_f16:
+   case aco_opcode::v_div_fixup_f16:
+   case aco_opcode::v_interp_p2_f16:
+      bytes_written = chip >= GFX9 ? rc.bytes() : 4u;
+      break;
+   default:
+      break;
+   }
+   bytes_written = MAX2(bytes_written, instr_info.definition_size[(int)instr->opcode] / 8u);
+
+   if (can_use_SDWA(chip, instr)) {
+      return std::make_pair(rc.bytes(), rc.bytes());
+   } else if (rc.bytes() == 2 && can_use_opsel(chip, instr->opcode, -1, 1)) {
+      return std::make_pair(2u, bytes_written);
+   }
+
+   switch (instr->opcode) {
+   case aco_opcode::buffer_load_ubyte_d16:
+   case aco_opcode::buffer_load_short_d16:
+   case aco_opcode::flat_load_ubyte_d16:
+   case aco_opcode::flat_load_short_d16:
+   case aco_opcode::scratch_load_ubyte_d16:
+   case aco_opcode::scratch_load_short_d16:
+   case aco_opcode::global_load_ubyte_d16:
+   case aco_opcode::global_load_short_d16:
+   case aco_opcode::ds_read_u8_d16:
+   case aco_opcode::ds_read_u16_d16:
+      if (chip >= GFX9 && !program->sram_ecc_enabled)
+         return std::make_pair(2u, 2u);
+      else
+         return std::make_pair(2u, 4u);
+   default:
+      break;
+   }
+
+   return std::make_pair(4u, bytes_written);
+}
+
+void add_subdword_definition(Program *program, aco_ptr<Instruction>& instr, unsigned idx, PhysReg reg, bool is_partial)
+{
+   RegClass rc = instr->definitions[idx].regClass();
+   chip_class chip = program->chip_class;
+
+   instr->definitions[idx].setFixed(reg);
+
+   if (instr->format == Format::PSEUDO) {
+      return;
+   } else if (can_use_SDWA(chip, instr)) {
+      if (reg.byte() || (is_partial && chip < GFX10))
+         convert_to_SDWA(chip, instr);
+      return;
+   } else if (reg.byte() && rc.bytes() == 2 && can_use_opsel(chip, instr->opcode, -1, reg.byte() / 2)) {
+      VOP3A_instruction *vop3 = static_cast<VOP3A_instruction *>(instr.get());
+      if (reg.byte() == 2)
+         vop3->opsel |= (1 << 3); /* dst in high half */
+      return;
+   }
+
+   if (reg.byte() == 2) {
+      if (instr->opcode == aco_opcode::buffer_load_ubyte_d16)
+         instr->opcode = aco_opcode::buffer_load_ubyte_d16_hi;
+      else if (instr->opcode == aco_opcode::buffer_load_short_d16)
+         instr->opcode = aco_opcode::buffer_load_short_d16_hi;
+      else if (instr->opcode == aco_opcode::flat_load_ubyte_d16)
+         instr->opcode = aco_opcode::flat_load_ubyte_d16_hi;
+      else if (instr->opcode == aco_opcode::flat_load_short_d16)
+         instr->opcode = aco_opcode::flat_load_short_d16_hi;
+      else if (instr->opcode == aco_opcode::scratch_load_ubyte_d16)
+         instr->opcode = aco_opcode::scratch_load_ubyte_d16_hi;
+      else if (instr->opcode == aco_opcode::scratch_load_short_d16)
+         instr->opcode = aco_opcode::scratch_load_short_d16_hi;
+      else if (instr->opcode == aco_opcode::global_load_ubyte_d16)
+         instr->opcode = aco_opcode::global_load_ubyte_d16_hi;
+      else if (instr->opcode == aco_opcode::global_load_short_d16)
+         instr->opcode = aco_opcode::global_load_short_d16_hi;
+      else if (instr->opcode == aco_opcode::ds_read_u8_d16)
+         instr->opcode = aco_opcode::ds_read_u8_d16_hi;
+      else if (instr->opcode == aco_opcode::ds_read_u16_d16)
+         instr->opcode = aco_opcode::ds_read_u16_d16_hi;
+      else
+         unreachable("Something went wrong: Impossible register assignment.");
+   }
+}
+
 void adjust_max_used_regs(ra_ctx& ctx, RegClass rc, unsigned reg)
 {
    unsigned max_addressible_sgpr = ctx.program->sgpr_limit;
@@ -312,7 +531,7 @@ void adjust_max_used_regs(ra_ctx& ctx, RegClass rc, unsigned reg)
 
 void update_renames(ra_ctx& ctx, RegisterFile& reg_file,
                     std::vector<std::pair<Operand, Definition>>& parallelcopies,
-                    aco_ptr<Instruction>& instr)
+                    aco_ptr<Instruction>& instr, bool rename_not_killed_ops)
 {
    /* allocate id's and rename operands: this is done transparently here */
    for (std::pair<Operand, Definition>& copy : parallelcopies) {
@@ -336,19 +555,27 @@ void update_renames(ra_ctx& ctx, RegisterFile& reg_file,
       reg_file.fill(copy.second);
 
       /* check if we moved an operand */
-      for (Operand& op : instr->operands) {
+      bool first = true;
+      for (unsigned i = 0; i < instr->operands.size(); i++) {
+         Operand& op = instr->operands[i];
          if (!op.isTemp())
             continue;
          if (op.tempId() == copy.first.tempId()) {
-            bool omit_renaming = instr->opcode == aco_opcode::p_create_vector && !op.isKillBeforeDef();
+            bool omit_renaming = !rename_not_killed_ops && !op.isKillBeforeDef();
             for (std::pair<Operand, Definition>& pc : parallelcopies) {
                PhysReg def_reg = pc.second.physReg();
                omit_renaming &= def_reg > copy.first.physReg() ?
                                 (copy.first.physReg() + copy.first.size() <= def_reg.reg()) :
                                 (def_reg + pc.second.size() <= copy.first.physReg().reg());
             }
-            if (omit_renaming)
+            if (omit_renaming) {
+               if (first)
+                  op.setFirstKill(true);
+               else
+                  op.setKill(true);
+               first = false;
                continue;
+            }
             op.setTemp(copy.second.getTemp());
             op.setFixed(copy.second.physReg());
          }
@@ -363,39 +590,11 @@ std::pair<PhysReg, bool> get_reg_simple(ra_ctx& ctx,
    uint32_t lb = info.lb;
    uint32_t ub = info.ub;
    uint32_t size = info.size;
-   uint32_t stride = info.stride;
+   uint32_t stride = info.rc.is_subdword() ? DIV_ROUND_UP(info.stride, 4) : info.stride;
    RegClass rc = info.rc;
 
-   if (rc.is_subdword()) {
-      for (std::pair<uint32_t, std::array<uint32_t, 4>> entry : reg_file.subdword_regs) {
-         assert(reg_file[entry.first] == 0xF0000000);
-         if (lb > entry.first || entry.first >= ub)
-            continue;
-
-         for (unsigned i = 0; i < 4; i+= stride) {
-            if (entry.second[i] != 0)
-               continue;
-
-            bool reg_found = true;
-            for (unsigned j = 1; reg_found && i + j < 4 && j < rc.bytes(); j++)
-               reg_found &= entry.second[i + j] == 0;
-
-            /* check neighboring reg if needed */
-            reg_found &= ((int)i <= 4 - (int)rc.bytes() || reg_file[entry.first + 1] == 0);
-            if (reg_found) {
-               PhysReg res{entry.first};
-               res.reg_b += i;
-               return {res, true};
-            }
-         }
-      }
-
-      stride = 1; /* stride in full registers */
-      rc = info.rc = RegClass(RegType::vgpr, size);
-   }
-
    if (stride == 1) {
-
+      info.rc = RegClass(rc.type(), size);
       for (unsigned stride = 8; stride > 1; stride /= 2) {
          if (size % stride)
             continue;
@@ -484,6 +683,35 @@ std::pair<PhysReg, bool> get_reg_simple(ra_ctx& ctx,
       reg_lo += stride;
    }
 
+   /* do this late because using the upper bytes of a register can require
+    * larger instruction encodings or copies
+    * TODO: don't do this in situations where it doesn't benefit */
+   if (rc.is_subdword()) {
+      for (std::pair<uint32_t, std::array<uint32_t, 4>> entry : reg_file.subdword_regs) {
+         assert(reg_file[entry.first] == 0xF0000000);
+         if (lb > entry.first || entry.first >= ub)
+            continue;
+
+         for (unsigned i = 0; i < 4; i+= info.stride) {
+            if (entry.second[i] != 0)
+               continue;
+
+            bool reg_found = true;
+            for (unsigned j = 1; reg_found && i + j < 4 && j < rc.bytes(); j++)
+               reg_found &= entry.second[i + j] == 0;
+
+            /* check neighboring reg if needed */
+            reg_found &= ((int)i <= 4 - (int)rc.bytes() || reg_file[entry.first + 1] == 0);
+            if (reg_found) {
+               PhysReg res{entry.first};
+               res.reg_b += i;
+               adjust_max_used_regs(ctx, rc, entry.first);
+               return {res, true};
+            }
+         }
+      }
+   }
+
    return {{}, false};
 }
 
@@ -531,28 +759,34 @@ bool get_regs_for_copies(ra_ctx& ctx,
    for (std::set<std::pair<unsigned, unsigned>>::const_reverse_iterator it = vars.rbegin(); it != vars.rend(); ++it) {
       unsigned id = it->second;
       assignment& var = ctx.assignments[id];
-      DefInfo info = DefInfo(ctx, ctx.pseudo_dummy, var.rc);
+      DefInfo info = DefInfo(ctx, ctx.pseudo_dummy, var.rc, -1);
       uint32_t size = info.size;
 
-      /* check if this is a dead operand, then we can re-use the space from the definition */
+      /* check if this is a dead operand, then we can re-use the space from the definition
+       * also use the correct stride for sub-dword operands */
       bool is_dead_operand = false;
-      for (unsigned i = 0; !is_phi(instr) && !is_dead_operand && (i < instr->operands.size()); i++) {
-         if (instr->operands[i].isTemp() && instr->operands[i].isKillBeforeDef() && instr->operands[i].tempId() == id)
-            is_dead_operand = true;
+      for (unsigned i = 0; !is_phi(instr) && i < instr->operands.size(); i++) {
+         if (instr->operands[i].isTemp() && instr->operands[i].tempId() == id) {
+            if (instr->operands[i].isKillBeforeDef())
+               is_dead_operand = true;
+            info = DefInfo(ctx, instr, var.rc, i);
+            break;
+         }
       }
 
       std::pair<PhysReg, bool> res;
       if (is_dead_operand) {
          if (instr->opcode == aco_opcode::p_create_vector) {
-            for (unsigned i = 0, offset = 0; i < instr->operands.size(); offset += instr->operands[i].bytes(), i++) {
+            PhysReg reg(def_reg_lo);
+            for (unsigned i = 0; i < instr->operands.size(); i++) {
                if (instr->operands[i].isTemp() && instr->operands[i].tempId() == id) {
-                  PhysReg reg(def_reg_lo);
-                  reg.reg_b += offset;
-                  assert(!reg_file.test(reg, var.rc.bytes()));
-                  res = {reg, true};
+                  res = {reg, (!var.rc.is_subdword() || (reg.byte() % info.stride == 0)) && !reg_file.test(reg, var.rc.bytes())};
                   break;
                }
+               reg.reg_b += instr->operands[i].bytes();
             }
+            if (!res.second)
+               res = {var.reg, !reg_file.test(var.reg, var.rc.bytes())};
          } else {
             info.lb = def_reg_lo;
             info.ub = def_reg_hi + 1;
@@ -560,9 +794,9 @@ bool get_regs_for_copies(ra_ctx& ctx,
          }
       } else {
          info.lb = lb;
-         info.ub = def_reg_lo;
+         info.ub = MIN2(def_reg_lo, ub);
          res = get_reg_simple(ctx, reg_file, info);
-         if (!res.second) {
+         if (!res.second && def_reg_hi < ub) {
             info.lb = (def_reg_hi + info.stride) & ~(info.stride - 1);
             info.ub = ub;
             res = get_reg_simple(ctx, reg_file, info);
@@ -698,9 +932,9 @@ std::pair<PhysReg, bool> get_reg_impl(ra_ctx& ctx,
       if (instr->operands[j].isTemp() &&
           instr->operands[j].isFirstKillBeforeDef() &&
           instr->operands[j].physReg() >= lb &&
-          instr->operands[j].physReg() < ub) {
+          instr->operands[j].physReg() < ub &&
+          !reg_file.test(instr->operands[j].physReg(), instr->operands[j].bytes())) {
          assert(instr->operands[j].isFixed());
-         assert(!reg_file.test(instr->operands[j].physReg(), instr->operands[j].bytes()));
          reg_file.block(instr->operands[j].physReg(), instr->operands[j].regClass());
          killed_ops += instr->operands[j].getTemp().size();
       }
@@ -723,9 +957,11 @@ std::pair<PhysReg, bool> get_reg_impl(ra_ctx& ctx,
    unsigned reg_hi = lb + size - 1;
    for (reg_lo = lb, reg_hi = lb + size - 1; reg_hi < ub; reg_lo += stride, reg_hi += stride) {
       /* first check the edges: this is what we have to fix to allow for num_moves > size */
-      if (reg_lo > lb && reg_file[reg_lo] != 0 && reg_file[reg_lo] == reg_file[reg_lo - 1])
+      if (reg_lo > lb && !reg_file.is_empty_or_blocked(PhysReg(reg_lo)) &&
+          reg_file.get_id(PhysReg(reg_lo)) == reg_file.get_id(PhysReg(reg_lo).advance(-1)))
          continue;
-      if (reg_hi < ub - 1 && reg_file[reg_hi] != 0 && reg_file[reg_hi] == reg_file[reg_hi + 1])
+      if (reg_hi < ub - 1 && !reg_file.is_empty_or_blocked(PhysReg(reg_hi).advance(3)) &&
+          reg_file.get_id(PhysReg(reg_hi).advance(3)) == reg_file.get_id(PhysReg(reg_hi).advance(4)))
          continue;
 
       /* second, check that we have at most k=num_moves elements in the window
@@ -805,24 +1041,29 @@ std::pair<PhysReg, bool> get_reg_impl(ra_ctx& ctx,
    std::set<std::pair<unsigned, unsigned>> vars = collect_vars(ctx, reg_file, PhysReg{best_pos}, size);
 
    if (instr->opcode == aco_opcode::p_create_vector) {
-      /* move killed operands which aren't yet at the correct position */
-      for (unsigned i = 0, offset = 0; i < instr->operands.size(); offset += instr->operands[i].size(), i++) {
-         if (instr->operands[i].isTemp() && instr->operands[i].isFirstKillBeforeDef() &&
-             instr->operands[i].getTemp().type() == rc.type()) {
-
-            if (instr->operands[i].physReg() != best_pos + offset) {
-               vars.emplace(instr->operands[i].bytes(), instr->operands[i].tempId());
-               reg_file.clear(instr->operands[i]);
+      /* move killed operands which aren't yet at the correct position (GFX9+)
+       * or which are in the definition space */
+      PhysReg reg = PhysReg{best_pos};
+      for (Operand& op : instr->operands) {
+         if (op.isTemp() && op.isFirstKillBeforeDef() &&
+             op.getTemp().type() == rc.type()) {
+            if (op.physReg() != reg &&
+                (ctx.program->chip_class >= GFX9 ||
+                 (op.physReg().advance(op.bytes()) > PhysReg{best_pos} &&
+                  op.physReg() < PhysReg{best_pos + size}))) {
+               vars.emplace(op.bytes(), op.tempId());
+               reg_file.clear(op);
             } else {
-               reg_file.fill(instr->operands[i]);
+               reg_file.fill(op);
             }
          }
+         reg.reg_b += op.bytes();
       }
-   } else {
-      /* re-enable the killed operands */
-      for (unsigned j = 0; !is_phi(instr) && j < instr->operands.size(); j++) {
-         if (instr->operands[j].isTemp() && instr->operands[j].isFirstKill())
-            reg_file.fill(instr->operands[j]);
+   } else if (!is_phi(instr)) {
+      /* re-enable killed operands */
+      for (Operand& op : instr->operands) {
+         if (op.isTemp() && op.isFirstKillBeforeDef())
+            reg_file.fill(op);
       }
    }
 
@@ -849,7 +1090,7 @@ std::pair<PhysReg, bool> get_reg_impl(ra_ctx& ctx,
    /* we set the definition regs == 0. the actual caller is responsible for correct setting */
    reg_file.clear(PhysReg{best_pos}, rc);
 
-   update_renames(ctx, reg_file, parallelcopies, instr);
+   update_renames(ctx, reg_file, parallelcopies, instr, instr->opcode != aco_opcode::p_create_vector);
 
    /* remove killed operands from reg_file once again */
    for (unsigned i = 0; !is_phi(instr) && i < instr->operands.size(); i++) {
@@ -876,7 +1117,11 @@ bool get_reg_specified(ra_ctx& ctx,
                        aco_ptr<Instruction>& instr,
                        PhysReg reg)
 {
-   if (rc.is_subdword() && reg.byte() && !instr_can_access_subdword(instr))
+   std::pair<unsigned, unsigned> sdw_def_info;
+   if (rc.is_subdword())
+      sdw_def_info = get_subdword_definition_info(ctx.program, instr, rc);
+
+   if (rc.is_subdword() && reg.byte() % sdw_def_info.first)
       return false;
    if (!rc.is_subdword() && reg.byte())
       return false;
@@ -905,8 +1150,15 @@ bool get_reg_specified(ra_ctx& ctx,
    if (reg_lo < lb || reg_hi >= ub || reg_lo > reg_hi)
       return false;
 
-   if (reg_file.test(reg, rc.bytes()))
-      return false;
+   if (rc.is_subdword()) {
+      PhysReg test_reg;
+      test_reg.reg_b = reg.reg_b & ~(sdw_def_info.second - 1);
+      if (reg_file.test(test_reg, sdw_def_info.second))
+         return false;
+   } else {
+      if (reg_file.test(reg, rc.bytes()))
+         return false;
+   }
 
    adjust_max_used_regs(ctx, rc, reg_lo);
    return true;
@@ -916,8 +1168,24 @@ PhysReg get_reg(ra_ctx& ctx,
                 RegisterFile& reg_file,
                 Temp temp,
                 std::vector<std::pair<Operand, Definition>>& parallelcopies,
-                aco_ptr<Instruction>& instr)
+                aco_ptr<Instruction>& instr,
+                int operand_index=-1)
 {
+   auto split_vec = ctx.split_vectors.find(temp.id());
+   if (split_vec != ctx.split_vectors.end()) {
+      unsigned offset = 0;
+      for (Definition def : split_vec->second->definitions) {
+         auto affinity_it = ctx.affinities.find(def.tempId());
+         if (affinity_it != ctx.affinities.end() && ctx.assignments[affinity_it->second].assigned) {
+            PhysReg reg = ctx.assignments[affinity_it->second].reg;
+            reg.reg_b -= offset;
+            if (get_reg_specified(ctx, reg_file, temp.regClass(), parallelcopies, instr, reg))
+               return reg;
+         }
+         offset += def.bytes();
+      }
+   }
+
    if (ctx.affinities.find(temp.id()) != ctx.affinities.end() &&
        ctx.assignments[ctx.affinities[temp.id()]].assigned) {
       PhysReg reg = ctx.assignments[ctx.affinities[temp.id()]].reg;
@@ -948,7 +1216,7 @@ PhysReg get_reg(ra_ctx& ctx,
          k += op.bytes();
       }
 
-      DefInfo info(ctx, ctx.pseudo_dummy, vec->definitions[0].regClass());
+      DefInfo info(ctx, ctx.pseudo_dummy, vec->definitions[0].regClass(), -1);
       std::pair<PhysReg, bool> res = get_reg_simple(ctx, reg_file, info);
       PhysReg reg = res.first;
       if (res.second) {
@@ -959,7 +1227,7 @@ PhysReg get_reg(ra_ctx& ctx,
       }
    }
 
-   DefInfo info(ctx, instr, temp.regClass());
+   DefInfo info(ctx, instr, temp.regClass(), operand_index);
 
    /* try to find space without live-range splits */
    std::pair<PhysReg, bool> res = get_reg_simple(ctx, reg_file, info);
@@ -983,15 +1251,15 @@ PhysReg get_reg(ra_ctx& ctx,
    uint16_t max_addressible_vgpr = ctx.program->vgpr_limit;
    if (info.rc.type() == RegType::vgpr && ctx.program->max_reg_demand.vgpr < max_addressible_vgpr) {
       update_vgpr_sgpr_demand(ctx.program, RegisterDemand(ctx.program->max_reg_demand.vgpr + 1, ctx.program->max_reg_demand.sgpr));
-      return get_reg(ctx, reg_file, temp, parallelcopies, instr);
+      return get_reg(ctx, reg_file, temp, parallelcopies, instr, operand_index);
    } else if (info.rc.type() == RegType::sgpr && ctx.program->max_reg_demand.sgpr < max_addressible_sgpr) {
       update_vgpr_sgpr_demand(ctx.program,  RegisterDemand(ctx.program->max_reg_demand.vgpr, ctx.program->max_reg_demand.sgpr + 1));
-      return get_reg(ctx, reg_file, temp, parallelcopies, instr);
+      return get_reg(ctx, reg_file, temp, parallelcopies, instr, operand_index);
    }
 
    //FIXME: if nothing helps, shift-rotate the registers to make space
 
-   fprintf(stderr, "ACO: failed to allocate registers during shader compilation\n");
+   aco_err(ctx.program, "Failed to allocate registers during shader compilation.");
    abort();
 }
 
@@ -1049,9 +1317,9 @@ PhysReg get_reg_create_vector(ra_ctx& ctx,
       // TODO: this can be improved */
       if (reg_lo < lb || reg_hi >= ub || reg_lo % stride != 0)
          continue;
-      if (reg_lo > lb && reg_file[reg_lo] != 0 && reg_file[reg_lo] == reg_file[reg_lo - 1])
+      if (reg_lo > lb && reg_file[reg_lo] != 0 && reg_file.get_id(PhysReg(reg_lo)) == reg_file.get_id(PhysReg(reg_lo).advance(-1)))
          continue;
-      if (reg_hi < ub - 1 && reg_file[reg_hi] != 0 && reg_file[reg_hi] == reg_file[reg_hi + 1])
+      if (reg_hi < ub - 1 && reg_file[reg_hi] != 0 && reg_file.get_id(PhysReg(reg_hi).advance(3)) == reg_file.get_id(PhysReg(reg_hi).advance(4)))
          continue;
 
       /* count variables to be moved and check war_hint */
@@ -1098,32 +1366,48 @@ PhysReg get_reg_create_vector(ra_ctx& ctx,
    if (num_moves >= bytes)
       return get_reg(ctx, reg_file, temp, parallelcopies, instr);
 
+   /* re-enable killed operands which are in the wrong position */
+   for (unsigned i = 0, offset = 0; i < instr->operands.size(); offset += instr->operands[i].bytes(), i++) {
+      if (instr->operands[i].isTemp() &&
+          instr->operands[i].isFirstKillBeforeDef() &&
+          instr->operands[i].physReg().reg_b != best_pos * 4 + offset)
+         reg_file.fill(instr->operands[i]);
+   }
+
    /* collect variables to be moved */
    std::set<std::pair<unsigned, unsigned>> vars = collect_vars(ctx, reg_file, PhysReg{best_pos}, size);
 
-   /* move killed operands which aren't yet at the correct position */
-   uint64_t moved_operand_mask = 0;
    for (unsigned i = 0, offset = 0; i < instr->operands.size(); offset += instr->operands[i].bytes(), i++) {
-      if (instr->operands[i].isTemp() &&
-          instr->operands[i].isFirstKillBeforeDef() &&
-          instr->operands[i].getTemp().type() == rc.type() &&
-          instr->operands[i].physReg().reg_b != best_pos * 4 + offset) {
+      if (!instr->operands[i].isTemp() || !instr->operands[i].isFirstKillBeforeDef() ||
+          instr->operands[i].getTemp().type() != rc.type())
+         continue;
+      bool correct_pos = instr->operands[i].physReg().reg_b == best_pos * 4 + offset;
+      /* GFX9+: move killed operands which aren't yet at the correct position
+       * Moving all killed operands generally leads to more register swaps.
+       * This is only done on GFX9+ because of the cheap v_swap instruction.
+       */
+      if (ctx.program->chip_class >= GFX9 && !correct_pos) {
          vars.emplace(instr->operands[i].bytes(), instr->operands[i].tempId());
-         moved_operand_mask |= (uint64_t)1 << i;
+         reg_file.clear(instr->operands[i]);
+      /* fill operands which are in the correct position to avoid overwriting */
+      } else if (correct_pos) {
+         reg_file.fill(instr->operands[i]);
       }
    }
-
    ASSERTED bool success = false;
    success = get_regs_for_copies(ctx, reg_file, parallelcopies, vars, lb, ub, instr, best_pos, best_pos + size - 1);
    assert(success);
 
-   update_renames(ctx, reg_file, parallelcopies, instr);
+   update_renames(ctx, reg_file, parallelcopies, instr, false);
    adjust_max_used_regs(ctx, rc, best_pos);
 
-   while (moved_operand_mask) {
-      unsigned i = u_bit_scan64(&moved_operand_mask);
-      assert(instr->operands[i].isFirstKillBeforeDef());
-      reg_file.clear(instr->operands[i]);
+   /* remove killed operands from reg_file once again */
+   for (unsigned i = 0; i < instr->operands.size(); i++) {
+      if (!instr->operands[i].isTemp() || !instr->operands[i].isFixed())
+         continue;
+      assert(!instr->operands[i].isUndefined());
+      if (instr->operands[i].isFirstKillBeforeDef())
+         reg_file.clear(instr->operands[i]);
    }
 
    return PhysReg{best_pos};
@@ -1158,13 +1442,18 @@ void handle_pseudo(ra_ctx& ctx,
    }
    /* if all operands are constant, no need to care either */
    bool reads_sgpr = false;
+   bool reads_subdword = false;
    for (Operand& op : instr->operands) {
       if (op.isTemp() && op.getTemp().type() == RegType::sgpr) {
          reads_sgpr = true;
          break;
       }
+      if (op.isTemp() && op.regClass().is_subdword())
+         reads_subdword = true;
    }
-   if (!(writes_sgpr && reads_sgpr))
+   bool needs_scratch_reg = (writes_sgpr && reads_sgpr) ||
+                            (ctx.program->chip_class <= GFX7 && reads_subdword);
+   if (!needs_scratch_reg)
       return;
 
    Pseudo_instruction *pi = (Pseudo_instruction *)instr;
@@ -1178,7 +1467,10 @@ void handle_pseudo(ra_ctx& ctx,
          reg = ctx.max_used_sgpr + 1;
          for (; reg < ctx.program->max_reg_demand.sgpr && reg_file[reg]; reg++)
             ;
-         assert(reg < ctx.program->max_reg_demand.sgpr);
+         if (reg == ctx.program->max_reg_demand.sgpr) {
+            assert(reads_subdword && reg_file[m0] == 0);
+            reg = m0;
+         }
       }
 
       adjust_max_used_regs(ctx, s1, reg);
@@ -1188,13 +1480,16 @@ void handle_pseudo(ra_ctx& ctx,
    }
 }
 
-bool operand_can_use_reg(aco_ptr<Instruction>& instr, unsigned idx, PhysReg reg)
+bool operand_can_use_reg(chip_class chip, aco_ptr<Instruction>& instr, unsigned idx, PhysReg reg, RegClass rc)
 {
    if (instr->operands[idx].isFixed())
       return instr->operands[idx].physReg() == reg;
 
-   if (!instr_can_access_subdword(instr) && reg.byte())
-      return false;
+   if (reg.byte()) {
+      unsigned stride = get_subdword_operand_stride(chip, instr, idx, rc);
+      if (reg.byte() % stride)
+         return false;
+   }
 
    switch (instr->format) {
    case Format::SMEM:
@@ -1210,7 +1505,7 @@ bool operand_can_use_reg(aco_ptr<Instruction>& instr, unsigned idx, PhysReg reg)
 
 void get_reg_for_operand(ra_ctx& ctx, RegisterFile& register_file,
                          std::vector<std::pair<Operand, Definition>>& parallelcopy,
-                         aco_ptr<Instruction>& instr, Operand& operand)
+                         aco_ptr<Instruction>& instr, Operand& operand, unsigned operand_index)
 {
    /* check if the operand is fixed */
    PhysReg dst;
@@ -1234,7 +1529,7 @@ void get_reg_for_operand(ra_ctx& ctx, RegisterFile& register_file,
       dst = operand.physReg();
 
    } else {
-      dst = get_reg(ctx, register_file, operand.getTemp(), parallelcopy, instr);
+      dst = get_reg(ctx, register_file, operand.getTemp(), parallelcopy, instr, operand_index);
    }
 
    Operand pc_op = operand;
@@ -1242,7 +1537,7 @@ void get_reg_for_operand(ra_ctx& ctx, RegisterFile& register_file,
    Definition pc_def = Definition(dst, pc_op.regClass());
    register_file.clear(pc_op);
    parallelcopy.emplace_back(pc_op, pc_def);
-   update_renames(ctx, register_file, parallelcopy, instr);
+   update_renames(ctx, register_file, parallelcopy, instr, true);
 }
 
 Temp read_variable(ra_ctx& ctx, Temp val, unsigned block_idx)
@@ -1409,9 +1704,10 @@ void register_allocation(Program *program, std::vector<TempSet>& live_out_per_bl
       for (rit = block.instructions.rbegin(); rit != block.instructions.rend(); ++rit) {
          aco_ptr<Instruction>& instr = *rit;
          if (is_phi(instr)) {
-            live.erase(instr->definitions[0].getTemp());
-            if (instr->definitions[0].isKill() || instr->definitions[0].isFixed())
+            if (instr->definitions[0].isKill() || instr->definitions[0].isFixed()) {
+               live.erase(instr->definitions[0].getTemp());
                continue;
+            }
             /* collect information about affinity-related temporaries */
             std::vector<Temp> affinity_related;
             /* affinity_related[0] is the last seen affinity-related temp */
@@ -1424,23 +1720,25 @@ void register_allocation(Program *program, std::vector<TempSet>& live_out_per_bl
                }
             }
             phi_ressources.emplace_back(std::move(affinity_related));
-            continue;
-         }
+         } else {
+            /* add vector affinities */
+            if (instr->opcode == aco_opcode::p_create_vector) {
+               for (const Operand& op : instr->operands) {
+                  if (op.isTemp() && op.isFirstKill() && op.getTemp().type() == instr->definitions[0].getTemp().type())
+                     ctx.vectors[op.tempId()] = instr.get();
+               }
+            }
 
-         /* add vector affinities */
-         if (instr->opcode == aco_opcode::p_create_vector) {
+            if (instr->opcode == aco_opcode::p_split_vector && instr->operands[0].isFirstKillBeforeDef())
+               ctx.split_vectors[instr->operands[0].tempId()] = instr.get();
+
+            /* add operands to live variables */
             for (const Operand& op : instr->operands) {
-               if (op.isTemp() && op.isFirstKill() && op.getTemp().type() == instr->definitions[0].getTemp().type())
-                  ctx.vectors[op.tempId()] = instr.get();
+               if (op.isTemp())
+                  live.emplace(op.getTemp());
             }
          }
 
-         /* add operands to live variables */
-         for (const Operand& op : instr->operands) {
-            if (op.isTemp())
-               live.emplace(op.getTemp());
-         }
-
          /* erase definitions from live */
          for (unsigned i = 0; i < instr->definitions.size(); i++) {
             const Definition& def = instr->definitions[i];
@@ -1455,7 +1753,11 @@ void register_allocation(Program *program, std::vector<TempSet>& live_out_per_bl
                Operand op = Operand();
                if (!def.isFixed() && instr->opcode == aco_opcode::p_parallelcopy)
                   op = instr->operands[i];
-               else if (instr->opcode == aco_opcode::v_mad_f32 && !instr->usesModifiers())
+               else if ((instr->opcode == aco_opcode::v_mad_f32 ||
+                        (instr->opcode == aco_opcode::v_fma_f32 && program->chip_class >= GFX10) ||
+                        instr->opcode == aco_opcode::v_mad_f16 ||
+                        instr->opcode == aco_opcode::v_mad_legacy_f16 ||
+                        (instr->opcode == aco_opcode::v_fma_f16 && program->chip_class >= GFX10)) && !instr->usesModifiers())
                   op = instr->operands[2];
 
                if (op.isTemp() && op.isFirstKillBeforeDef() && def.regClass() == op.regClass()) {
@@ -1706,10 +2008,10 @@ void register_allocation(Program *program, std::vector<TempSet>& live_out_per_bl
             assert(ctx.assignments[operand.tempId()].assigned);
 
             PhysReg reg = ctx.assignments[operand.tempId()].reg;
-            if (operand_can_use_reg(instr, i, reg))
+            if (operand_can_use_reg(program->chip_class, instr, i, reg, operand.regClass()))
                operand.setFixed(reg);
             else
-               get_reg_for_operand(ctx, register_file, parallelcopy, instr, operand);
+               get_reg_for_operand(ctx, register_file, parallelcopy, instr, operand, i);
 
             if (instr->format == Format::EXP ||
                 (instr->isVMEM() && i == 3 && ctx.program->chip_class == GFX6) ||
@@ -1730,20 +2032,52 @@ void register_allocation(Program *program, std::vector<TempSet>& live_out_per_bl
          }
 
          /* try to optimize v_mad_f32 -> v_mac_f32 */
-         if (instr->opcode == aco_opcode::v_mad_f32 &&
+         if ((instr->opcode == aco_opcode::v_mad_f32 ||
+              (instr->opcode == aco_opcode::v_fma_f32 && program->chip_class >= GFX10) ||
+              instr->opcode == aco_opcode::v_mad_f16 ||
+              instr->opcode == aco_opcode::v_mad_legacy_f16 ||
+              (instr->opcode == aco_opcode::v_fma_f16 && program->chip_class >= GFX10)) &&
              instr->operands[2].isTemp() &&
              instr->operands[2].isKillBeforeDef() &&
              instr->operands[2].getTemp().type() == RegType::vgpr &&
              instr->operands[1].isTemp() &&
              instr->operands[1].getTemp().type() == RegType::vgpr &&
-             !instr->usesModifiers()) {
-            instr->format = Format::VOP2;
-            instr->opcode = aco_opcode::v_mac_f32;
+             !instr->usesModifiers() &&
+             instr->operands[0].physReg().byte() == 0 &&
+             instr->operands[1].physReg().byte() == 0 &&
+             instr->operands[2].physReg().byte() == 0) {
+            unsigned def_id = instr->definitions[0].tempId();
+            auto it = ctx.affinities.find(def_id);
+            if (it == ctx.affinities.end() || !ctx.assignments[it->second].assigned ||
+                instr->operands[2].physReg() == ctx.assignments[it->second].reg ||
+                register_file.test(ctx.assignments[it->second].reg, instr->operands[2].bytes())) {
+               instr->format = Format::VOP2;
+               switch (instr->opcode) {
+               case aco_opcode::v_mad_f32:
+                  instr->opcode = aco_opcode::v_mac_f32;
+                  break;
+               case aco_opcode::v_fma_f32:
+                  instr->opcode = aco_opcode::v_fmac_f32;
+                  break;
+               case aco_opcode::v_mad_f16:
+               case aco_opcode::v_mad_legacy_f16:
+                  instr->opcode = aco_opcode::v_mac_f16;
+                  break;
+               case aco_opcode::v_fma_f16:
+                  instr->opcode = aco_opcode::v_fmac_f16;
+                  break;
+               default:
+                  break;
+               }
+            }
          }
 
          /* handle definitions which must have the same register as an operand */
          if (instr->opcode == aco_opcode::v_interp_p2_f32 ||
              instr->opcode == aco_opcode::v_mac_f32 ||
+             instr->opcode == aco_opcode::v_fmac_f32 ||
+             instr->opcode == aco_opcode::v_mac_f16 ||
+             instr->opcode == aco_opcode::v_fmac_f16 ||
              instr->opcode == aco_opcode::v_writelane_b32 ||
              instr->opcode == aco_opcode::v_writelane_b32_e64) {
             instr->definitions[0].setFixed(instr->operands[2].physReg());
@@ -1770,23 +2104,26 @@ void register_allocation(Program *program, std::vector<TempSet>& live_out_per_bl
 
             adjust_max_used_regs(ctx, definition.regClass(), definition.physReg());
             /* check if the target register is blocked */
-            if (register_file[definition.physReg().reg()] != 0) {
-               /* create parallelcopy pair to move blocking var */
-               Temp tmp = {register_file[definition.physReg()], ctx.assignments[register_file[definition.physReg()]].rc};
-               Operand pc_op = Operand(tmp);
-               pc_op.setFixed(ctx.assignments[register_file[definition.physReg().reg()]].reg);
-               RegClass rc = pc_op.regClass();
-               tmp = Temp{program->allocateId(), rc};
-               Definition pc_def = Definition(tmp);
-
-               /* re-enable the killed operands, so that we don't move the blocking var there */
+            if (register_file.test(definition.physReg(), definition.bytes())) {
+               /* create parallelcopy pair to move blocking vars */
+               std::set<std::pair<unsigned, unsigned>> vars = collect_vars(ctx, register_file, definition.physReg(), definition.size());
+
+               /* re-enable the killed operands, so that we don't move the blocking vars there */
                for (const Operand& op : instr->operands) {
                   if (op.isTemp() && op.isFirstKillBeforeDef())
                      register_file.fill(op);
                }
 
-               /* find a new register for the blocking variable */
-               PhysReg reg = get_reg(ctx, register_file, pc_op.getTemp(), parallelcopy, instr);
+               ASSERTED bool success = false;
+               DefInfo info(ctx, instr, definition.regClass(), -1);
+               success = get_regs_for_copies(ctx, register_file, parallelcopy,
+                                             vars, info.lb, info.ub, instr,
+                                             definition.physReg(),
+                                             definition.physReg() + definition.size() - 1);
+               assert(success);
+
+               update_renames(ctx, register_file, parallelcopy, instr, false);
+
                /* once again, disable killed operands */
                for (const Operand& op : instr->operands) {
                   if (op.isTemp() && op.isFirstKillBeforeDef())
@@ -1796,16 +2133,6 @@ void register_allocation(Program *program, std::vector<TempSet>& live_out_per_bl
                   if (instr->definitions[k].isTemp() && ctx.defs_done.test(k) && !instr->definitions[k].isKill())
                      register_file.fill(instr->definitions[k]);
                }
-               pc_def.setFixed(reg);
-
-               /* finish assignment of parallelcopy */
-               ctx.assignments.emplace_back(reg, pc_def.regClass());
-               assert(ctx.assignments.size() == ctx.program->peekAllocationId());
-               parallelcopy.emplace_back(pc_op, pc_def);
-
-               /* add changes to reg_file */
-               register_file.clear(pc_op);
-               register_file.fill(pc_def);
             }
             ctx.defs_done.set(i);
 
@@ -1822,76 +2149,78 @@ void register_allocation(Program *program, std::vector<TempSet>& live_out_per_bl
 
          /* handle all other definitions */
          for (unsigned i = 0; i < instr->definitions.size(); ++i) {
-            auto& definition = instr->definitions[i];
+            Definition *definition = &instr->definitions[i];
 
-            if (definition.isFixed() || !definition.isTemp())
+            if (definition->isFixed() || !definition->isTemp())
                continue;
 
             /* find free reg */
-            if (definition.hasHint() && register_file[definition.physReg().reg()] == 0)
-               definition.setFixed(definition.physReg());
+            if (definition->hasHint() && register_file[definition->physReg().reg()] == 0)
+               definition->setFixed(definition->physReg());
             else if (instr->opcode == aco_opcode::p_split_vector) {
                PhysReg reg = instr->operands[0].physReg();
                for (unsigned j = 0; j < i; j++)
                   reg.reg_b += instr->definitions[j].bytes();
-               if (get_reg_specified(ctx, register_file, definition.regClass(), parallelcopy, instr, reg))
-                  definition.setFixed(reg);
-            } else if (instr->opcode == aco_opcode::p_wqm) {
-               PhysReg reg;
-               if (instr->operands[0].isKillBeforeDef() && instr->operands[0].getTemp().type() == definition.getTemp().type()) {
-                  reg = instr->operands[0].physReg();
-                  definition.setFixed(reg);
-                  assert(register_file[reg.reg()] == 0);
-               }
+               if (get_reg_specified(ctx, register_file, definition->regClass(), parallelcopy, instr, reg))
+                  definition->setFixed(reg);
+            } else if (instr->opcode == aco_opcode::p_wqm || instr->opcode == aco_opcode::p_parallelcopy) {
+               PhysReg reg = instr->operands[i].physReg();
+               if (instr->operands[i].isTemp() &&
+                   instr->operands[i].getTemp().type() == definition->getTemp().type() &&
+                   !register_file.test(reg, definition->bytes()))
+                  definition->setFixed(reg);
             } else if (instr->opcode == aco_opcode::p_extract_vector) {
                PhysReg reg;
                if (instr->operands[0].isKillBeforeDef() &&
-                   instr->operands[0].getTemp().type() == definition.getTemp().type()) {
+                   instr->operands[0].getTemp().type() == definition->getTemp().type()) {
                   reg = instr->operands[0].physReg();
-                  reg.reg_b += definition.bytes() * instr->operands[1].constantValue();
-                  assert(!register_file.test(reg, definition.bytes()));
-                  definition.setFixed(reg);
+                  reg.reg_b += definition->bytes() * instr->operands[1].constantValue();
+                  assert(!register_file.test(reg, definition->bytes()));
+                  definition->setFixed(reg);
                }
             } else if (instr->opcode == aco_opcode::p_create_vector) {
-               PhysReg reg = get_reg_create_vector(ctx, register_file, definition.getTemp(),
+               PhysReg reg = get_reg_create_vector(ctx, register_file, definition->getTemp(),
                                                    parallelcopy, instr);
-               definition.setFixed(reg);
+               definition->setFixed(reg);
             }
 
-            if (!definition.isFixed()) {
-               Temp tmp = definition.getTemp();
-               /* subdword instructions before RDNA write full registers */
-               if (tmp.regClass().is_subdword() &&
-                   !instr_can_access_subdword(instr) &&
-                   ctx.program->chip_class <= GFX9) {
-                  assert(tmp.bytes() <= 4);
-                  tmp = Temp(definition.tempId(), v1);
+            if (!definition->isFixed()) {
+               Temp tmp = definition->getTemp();
+               if (definition->regClass().is_subdword() && definition->bytes() < 4) {
+                  PhysReg reg = get_reg(ctx, register_file, tmp, parallelcopy, instr);
+                  bool partial = !(tmp.bytes() <= 4 && reg.byte() == 0 && !register_file.test(reg, 4));
+                  add_subdword_definition(program, instr, i, reg, partial);
+                  definition = &instr->definitions[i]; /* add_subdword_definition can invalidate the reference */
+               } else {
+                  definition->setFixed(get_reg(ctx, register_file, tmp, parallelcopy, instr));
                }
-               definition.setFixed(get_reg(ctx, register_file, tmp, parallelcopy, instr));
             }
 
-            assert(definition.isFixed() && ((definition.getTemp().type() == RegType::vgpr && definition.physReg() >= 256) ||
-                                            (definition.getTemp().type() != RegType::vgpr && definition.physReg() < 256)));
+            assert(definition->isFixed() && ((definition->getTemp().type() == RegType::vgpr && definition->physReg() >= 256) ||
+                                             (definition->getTemp().type() != RegType::vgpr && definition->physReg() < 256)));
             ctx.defs_done.set(i);
 
             /* set live if it has a kill point */
-            if (!definition.isKill())
-               live.emplace(definition.getTemp());
+            if (!definition->isKill())
+               live.emplace(definition->getTemp());
 
-            ctx.assignments[definition.tempId()] = {definition.physReg(), definition.regClass()};
-            register_file.fill(definition);
+            ctx.assignments[definition->tempId()] = {definition->physReg(), definition->regClass()};
+            register_file.fill(*definition);
          }
 
          handle_pseudo(ctx, register_file, instr.get());
 
-         /* kill definitions and late-kill operands */
+         /* kill definitions and late-kill operands and ensure that sub-dword operands can actually be read */
          for (const Definition& def : instr->definitions) {
              if (def.isTemp() && def.isKill())
                 register_file.clear(def);
          }
-         for (const Operand& op : instr->operands) {
+         for (unsigned i = 0; i < instr->operands.size(); i++) {
+            const Operand& op = instr->operands[i];
             if (op.isTemp() && op.isFirstKill() && op.isLateKill())
                register_file.clear(op);
+            if (op.isTemp() && op.physReg().byte() != 0)
+               add_subdword_operand(program->chip_class, instr, i, op.physReg().byte(), op.regClass());
          }
 
          /* emit parallelcopy */
@@ -2038,6 +2367,7 @@ void register_allocation(Program *program, std::vector<TempSet>& live_out_per_bl
             }
             std::copy(tmp->definitions.begin(), tmp->definitions.end(), instr->definitions.begin());
          }
+
          instructions.emplace_back(std::move(*it));
 
       } /* end for Instr */