X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Famd%2Fcompiler%2Faco_register_allocation.cpp;h=862b00d859b6d97f4553057aa9673f5391f4ec86;hb=1bbb64f300f66fbb292078072f89f04231ffd541;hp=410d8c14d8c815e4d4a2338fd78b7e882514f094;hpb=2ae27b96efca6473e9671a22d60f6b9496001413;p=mesa.git diff --git a/src/amd/compiler/aco_register_allocation.cpp b/src/amd/compiler/aco_register_allocation.cpp index 410d8c14d8c..862b00d859b 100644 --- a/src/amd/compiler/aco_register_allocation.cpp +++ b/src/amd/compiler/aco_register_allocation.cpp @@ -30,7 +30,6 @@ #include #include #include -#include #include "aco_ir.h" #include "sid.h" @@ -39,6 +38,11 @@ namespace aco { namespace { +unsigned get_subdword_operand_stride(chip_class chip, const aco_ptr& instr, unsigned idx, RegClass rc); +void add_subdword_operand(chip_class chip, aco_ptr& instr, unsigned idx, unsigned byte, RegClass rc); +std::pair get_subdword_definition_info(Program *program, const aco_ptr& instr, RegClass rc); +void add_subdword_definition(Program *program, aco_ptr& instr, unsigned idx, PhysReg reg, bool is_partial); + struct assignment { PhysReg reg; RegClass rc; @@ -47,16 +51,83 @@ struct assignment { assignment(PhysReg reg, RegClass rc) : reg(reg), rc(rc), assigned(-1) {} }; +struct phi_info { + Instruction* phi; + unsigned block_idx; + std::set uses; +}; + struct ra_ctx { std::bitset<512> war_hint; Program* program; std::vector assignments; + std::vector> renames; + std::vector> incomplete_phis; + std::vector filled; + std::vector sealed; std::unordered_map orig_names; + std::unordered_map phi_map; + std::unordered_map affinities; + std::unordered_map vectors; + std::unordered_map split_vectors; + aco_ptr pseudo_dummy; unsigned max_used_sgpr = 0; unsigned max_used_vgpr = 0; std::bitset<64> defs_done; /* see MAX_ARGS in aco_instruction_selection_setup.cpp */ - ra_ctx(Program* program) : program(program), assignments(program->peekAllocationId()) {} + ra_ctx(Program* program) : program(program), + assignments(program->peekAllocationId()), + renames(program->blocks.size()), + incomplete_phis(program->blocks.size()), + filled(program->blocks.size()), + sealed(program->blocks.size()) + { + pseudo_dummy.reset(create_instruction(aco_opcode::p_parallelcopy, Format::PSEUDO, 0, 0)); + } +}; + +struct DefInfo { + uint16_t lb; + uint16_t ub; + uint8_t size; + uint8_t stride; + RegClass rc; + + DefInfo(ra_ctx& ctx, aco_ptr& instr, RegClass rc_, int operand) : rc(rc_) { + size = rc.size(); + stride = 1; + + if (rc.type() == RegType::vgpr) { + lb = 256; + ub = 256 + ctx.program->max_reg_demand.vgpr; + } else { + lb = 0; + ub = ctx.program->max_reg_demand.sgpr; + if (size == 2) + stride = 2; + else if (size >= 4) + stride = 4; + } + + if (rc.is_subdword() && operand >= 0) { + /* stride in bytes */ + stride = get_subdword_operand_stride(ctx.program->chip_class, instr, operand, rc); + } else if (rc.is_subdword()) { + std::pair info = get_subdword_definition_info(ctx.program, instr, rc); + stride = info.first; + if (info.second > rc.bytes()) { + rc = RegClass::get(rc.type(), info.second); + size = rc.size(); + /* we might still be able to put the definition in the high half, + * but that's only useful for affinities and this information isn't + * used for them */ + stride = align(stride, info.second); + if (!rc.is_subdword()) + stride = DIV_ROUND_UP(stride, 4); + } + assert(stride > 0); + } + } }; class RegisterFile { @@ -96,31 +167,11 @@ public: return false; } - void fill(PhysReg start, unsigned size, uint32_t val) { - for (unsigned i = 0; i < size; i++) - regs[start + i] = val; - } - - void fill_subdword(PhysReg start, unsigned num_bytes, uint32_t val) { - fill(start, DIV_ROUND_UP(num_bytes, 4), 0xF0000000); - for (PhysReg i = start; i.reg_b < start.reg_b + num_bytes; i = PhysReg(i + 1)) { - /* emplace or get */ - std::array& sub = subdword_regs.emplace(i, std::array{0, 0, 0, 0}).first->second; - for (unsigned j = i.byte(); i * 4 + j < start.reg_b + num_bytes && j < 4; j++) - sub[j] = val; - - if (sub == std::array{0, 0, 0, 0}) { - subdword_regs.erase(i); - regs[i] = 0; - } - } - } - - void block(PhysReg start, unsigned num_bytes) { - if (start.byte() || num_bytes % 4) - fill_subdword(start, num_bytes, 0xFFFFFFFF); + void block(PhysReg start, RegClass rc) { + if (rc.is_subdword()) + fill_subdword(start, rc.bytes(), 0xFFFFFFFF); else - fill(start, num_bytes / 4, 0xFFFFFFFF); + fill(start, rc.size(), 0xFFFFFFFF); } bool is_blocked(PhysReg start) { @@ -134,6 +185,13 @@ public: return false; } + bool is_empty_or_blocked(PhysReg start) { + if (regs[start] == 0xF0000000) { + return subdword_regs[start][start.byte()] + 1 <= 1; + } + return regs[start] + 1 <= 1; + } + void clear(PhysReg start, RegClass rc) { if (rc.is_subdword()) fill_subdword(start, rc.bytes(), 0); @@ -162,6 +220,31 @@ public: void clear(Definition def) { clear(def.physReg(), def.regClass()); } + + unsigned get_id(PhysReg reg) { + return regs[reg] == 0xF0000000 ? subdword_regs[reg][reg.byte()] : regs[reg]; + } + +private: + void fill(PhysReg start, unsigned size, uint32_t val) { + for (unsigned i = 0; i < size; i++) + regs[start + i] = val; + } + + void fill_subdword(PhysReg start, unsigned num_bytes, uint32_t val) { + fill(start, DIV_ROUND_UP(num_bytes, 4), 0xF0000000); + for (PhysReg i = start; i.reg_b < start.reg_b + num_bytes; i = PhysReg(i + 1)) { + /* emplace or get */ + std::array& sub = subdword_regs.emplace(i, std::array{0, 0, 0, 0}).first->second; + for (unsigned j = i.byte(); i * 4 + j < start.reg_b + num_bytes && j < 4; j++) + sub[j] = val; + + if (sub == std::array{0, 0, 0, 0}) { + subdword_regs.erase(i); + regs[i] = 0; + } + } + } }; @@ -233,6 +316,204 @@ void print_regs(ra_ctx& ctx, bool vgprs, RegisterFile& reg_file) #endif +unsigned get_subdword_operand_stride(chip_class chip, const aco_ptr& instr, unsigned idx, RegClass rc) +{ + /* v_readfirstlane_b32 cannot use SDWA */ + if (instr->opcode == aco_opcode::p_as_uniform) + return 4; + if (instr->format == Format::PSEUDO && chip >= GFX8) + return rc.bytes() % 2 == 0 ? 2 : 1; + + if (instr->opcode == aco_opcode::v_cvt_f32_ubyte0) { + return 1; + } else if (can_use_SDWA(chip, instr)) { + return rc.bytes() % 2 == 0 ? 2 : 1; + } else if (rc.bytes() == 2 && can_use_opsel(chip, instr->opcode, idx, 1)) { + return 2; + } + + switch (instr->opcode) { + case aco_opcode::ds_write_b8: + case aco_opcode::ds_write_b16: + return chip >= GFX8 ? 2 : 4; + case aco_opcode::buffer_store_byte: + case aco_opcode::buffer_store_short: + case aco_opcode::flat_store_byte: + case aco_opcode::flat_store_short: + case aco_opcode::scratch_store_byte: + case aco_opcode::scratch_store_short: + case aco_opcode::global_store_byte: + case aco_opcode::global_store_short: + return chip >= GFX9 ? 2 : 4; + default: + break; + } + + return 4; +} + +void add_subdword_operand(chip_class chip, aco_ptr& instr, unsigned idx, unsigned byte, RegClass rc) +{ + if (instr->format == Format::PSEUDO || byte == 0) + return; + + assert(rc.bytes() <= 2); + + if (!instr->usesModifiers() && instr->opcode == aco_opcode::v_cvt_f32_ubyte0) { + switch (byte) { + case 0: + instr->opcode = aco_opcode::v_cvt_f32_ubyte0; + break; + case 1: + instr->opcode = aco_opcode::v_cvt_f32_ubyte1; + break; + case 2: + instr->opcode = aco_opcode::v_cvt_f32_ubyte2; + break; + case 3: + instr->opcode = aco_opcode::v_cvt_f32_ubyte3; + break; + } + return; + } else if (can_use_SDWA(chip, instr)) { + convert_to_SDWA(chip, instr); + return; + } else if (rc.bytes() == 2 && can_use_opsel(chip, instr->opcode, idx, byte / 2)) { + VOP3A_instruction *vop3 = static_cast(instr.get()); + vop3->opsel |= (byte / 2) << idx; + return; + } + + if (chip >= GFX8 && instr->opcode == aco_opcode::ds_write_b8 && byte == 2) { + instr->opcode = aco_opcode::ds_write_b8_d16_hi; + return; + } + if (chip >= GFX8 && instr->opcode == aco_opcode::ds_write_b16 && byte == 2) { + instr->opcode = aco_opcode::ds_write_b16_d16_hi; + return; + } + + if (chip >= GFX9 && byte == 2) { + if (instr->opcode == aco_opcode::buffer_store_byte) + instr->opcode = aco_opcode::buffer_store_byte_d16_hi; + else if (instr->opcode == aco_opcode::buffer_store_short) + instr->opcode = aco_opcode::buffer_store_short_d16_hi; + else if (instr->opcode == aco_opcode::flat_store_byte) + instr->opcode = aco_opcode::flat_store_byte_d16_hi; + else if (instr->opcode == aco_opcode::flat_store_short) + instr->opcode = aco_opcode::flat_store_short_d16_hi; + else if (instr->opcode == aco_opcode::scratch_store_byte) + instr->opcode = aco_opcode::scratch_store_byte_d16_hi; + else if (instr->opcode == aco_opcode::scratch_store_short) + instr->opcode = aco_opcode::scratch_store_short_d16_hi; + else if (instr->opcode == aco_opcode::global_store_byte) + instr->opcode = aco_opcode::global_store_byte_d16_hi; + else if (instr->opcode == aco_opcode::global_store_short) + instr->opcode = aco_opcode::global_store_short_d16_hi; + else + unreachable("Something went wrong: Impossible register assignment."); + } +} + +/* minimum_stride, bytes_written */ +std::pair get_subdword_definition_info(Program *program, const aco_ptr& instr, RegClass rc) +{ + chip_class chip = program->chip_class; + + if (instr->format == Format::PSEUDO && chip >= GFX8) + return std::make_pair(rc.bytes() % 2 == 0 ? 2 : 1, rc.bytes()); + else if (instr->format == Format::PSEUDO) + return std::make_pair(4, rc.size() * 4u); + + unsigned bytes_written = chip >= GFX10 ? rc.bytes() : 4u; + switch (instr->opcode) { + case aco_opcode::v_mad_f16: + case aco_opcode::v_mad_u16: + case aco_opcode::v_mad_i16: + case aco_opcode::v_fma_f16: + case aco_opcode::v_div_fixup_f16: + case aco_opcode::v_interp_p2_f16: + bytes_written = chip >= GFX9 ? rc.bytes() : 4u; + break; + default: + break; + } + bytes_written = MAX2(bytes_written, instr_info.definition_size[(int)instr->opcode] / 8u); + + if (can_use_SDWA(chip, instr)) { + return std::make_pair(rc.bytes(), rc.bytes()); + } else if (rc.bytes() == 2 && can_use_opsel(chip, instr->opcode, -1, 1)) { + return std::make_pair(2u, bytes_written); + } + + switch (instr->opcode) { + case aco_opcode::buffer_load_ubyte_d16: + case aco_opcode::buffer_load_short_d16: + case aco_opcode::flat_load_ubyte_d16: + case aco_opcode::flat_load_short_d16: + case aco_opcode::scratch_load_ubyte_d16: + case aco_opcode::scratch_load_short_d16: + case aco_opcode::global_load_ubyte_d16: + case aco_opcode::global_load_short_d16: + case aco_opcode::ds_read_u8_d16: + case aco_opcode::ds_read_u16_d16: + if (chip >= GFX9 && !program->sram_ecc_enabled) + return std::make_pair(2u, 2u); + else + return std::make_pair(2u, 4u); + default: + break; + } + + return std::make_pair(4u, bytes_written); +} + +void add_subdword_definition(Program *program, aco_ptr& instr, unsigned idx, PhysReg reg, bool is_partial) +{ + RegClass rc = instr->definitions[idx].regClass(); + chip_class chip = program->chip_class; + + instr->definitions[idx].setFixed(reg); + + if (instr->format == Format::PSEUDO) { + return; + } else if (can_use_SDWA(chip, instr)) { + if (reg.byte() || (is_partial && chip < GFX10)) + convert_to_SDWA(chip, instr); + return; + } else if (reg.byte() && rc.bytes() == 2 && can_use_opsel(chip, instr->opcode, -1, reg.byte() / 2)) { + VOP3A_instruction *vop3 = static_cast(instr.get()); + if (reg.byte() == 2) + vop3->opsel |= (1 << 3); /* dst in high half */ + return; + } + + if (reg.byte() == 2) { + if (instr->opcode == aco_opcode::buffer_load_ubyte_d16) + instr->opcode = aco_opcode::buffer_load_ubyte_d16_hi; + else if (instr->opcode == aco_opcode::buffer_load_short_d16) + instr->opcode = aco_opcode::buffer_load_short_d16_hi; + else if (instr->opcode == aco_opcode::flat_load_ubyte_d16) + instr->opcode = aco_opcode::flat_load_ubyte_d16_hi; + else if (instr->opcode == aco_opcode::flat_load_short_d16) + instr->opcode = aco_opcode::flat_load_short_d16_hi; + else if (instr->opcode == aco_opcode::scratch_load_ubyte_d16) + instr->opcode = aco_opcode::scratch_load_ubyte_d16_hi; + else if (instr->opcode == aco_opcode::scratch_load_short_d16) + instr->opcode = aco_opcode::scratch_load_short_d16_hi; + else if (instr->opcode == aco_opcode::global_load_ubyte_d16) + instr->opcode = aco_opcode::global_load_ubyte_d16_hi; + else if (instr->opcode == aco_opcode::global_load_short_d16) + instr->opcode = aco_opcode::global_load_short_d16_hi; + else if (instr->opcode == aco_opcode::ds_read_u8_d16) + instr->opcode = aco_opcode::ds_read_u8_d16_hi; + else if (instr->opcode == aco_opcode::ds_read_u16_d16) + instr->opcode = aco_opcode::ds_read_u16_d16_hi; + else + unreachable("Something went wrong: Impossible register assignment."); + } +} + void adjust_max_used_regs(ra_ctx& ctx, RegClass rc, unsigned reg) { unsigned max_addressible_sgpr = ctx.program->sgpr_limit; @@ -250,7 +531,7 @@ void adjust_max_used_regs(ra_ctx& ctx, RegClass rc, unsigned reg) void update_renames(ra_ctx& ctx, RegisterFile& reg_file, std::vector>& parallelcopies, - aco_ptr& instr) + aco_ptr& instr, bool rename_not_killed_ops) { /* allocate id's and rename operands: this is done transparently here */ for (std::pair& copy : parallelcopies) { @@ -274,19 +555,27 @@ void update_renames(ra_ctx& ctx, RegisterFile& reg_file, reg_file.fill(copy.second); /* check if we moved an operand */ - for (Operand& op : instr->operands) { + bool first = true; + for (unsigned i = 0; i < instr->operands.size(); i++) { + Operand& op = instr->operands[i]; if (!op.isTemp()) continue; if (op.tempId() == copy.first.tempId()) { - bool omit_renaming = instr->opcode == aco_opcode::p_create_vector && !op.isKillBeforeDef(); + bool omit_renaming = !rename_not_killed_ops && !op.isKillBeforeDef(); for (std::pair& pc : parallelcopies) { PhysReg def_reg = pc.second.physReg(); omit_renaming &= def_reg > copy.first.physReg() ? (copy.first.physReg() + copy.first.size() <= def_reg.reg()) : (def_reg + pc.second.size() <= copy.first.physReg().reg()); } - if (omit_renaming) + if (omit_renaming) { + if (first) + op.setFirstKill(true); + else + op.setKill(true); + first = false; continue; + } op.setTemp(copy.second.getTemp()); op.setFixed(copy.second.physReg()); } @@ -294,86 +583,82 @@ void update_renames(ra_ctx& ctx, RegisterFile& reg_file, } } -bool instr_can_access_subdword(aco_ptr& instr) -{ - return instr->isSDWA() || instr->format == Format::PSEUDO; -} - std::pair get_reg_simple(ra_ctx& ctx, RegisterFile& reg_file, - uint32_t lb, uint32_t ub, - uint32_t size, uint32_t stride, - RegClass rc) + DefInfo info) { - if (rc.is_subdword()) { - for (std::pair> entry : reg_file.subdword_regs) { - assert(reg_file[entry.first] == 0xF0000000); - if (lb > entry.first || entry.first >= ub) - continue; + uint32_t lb = info.lb; + uint32_t ub = info.ub; + uint32_t size = info.size; + uint32_t stride = info.rc.is_subdword() ? DIV_ROUND_UP(info.stride, 4) : info.stride; + RegClass rc = info.rc; - for (unsigned i = 0; i < 4; i+= stride) { - if (entry.second[i] != 0) - continue; - - bool reg_found = true; - for (unsigned j = 1; reg_found && i + j < 4 && j < rc.bytes(); j++) - reg_found &= entry.second[i + j] == 0; - - /* check neighboring reg if needed */ - reg_found &= (i <= 4 - rc.bytes() || reg_file[entry.first + 1] == 0); - if (reg_found) { - PhysReg res{entry.first}; - res.reg_b += i; - return {res, true}; - } - } + if (stride == 1) { + info.rc = RegClass(rc.type(), size); + for (unsigned stride = 8; stride > 1; stride /= 2) { + if (size % stride) + continue; + info.stride = stride; + std::pair res = get_reg_simple(ctx, reg_file, info); + if (res.second) + return res; } - stride = 1; /* stride in full registers */ - } - - /* best fit algorithm: find the smallest gap to fit in the variable */ - if (stride == 1) { + /* best fit algorithm: find the smallest gap to fit in the variable */ unsigned best_pos = 0xFFFF; unsigned gap_size = 0xFFFF; - unsigned next_pos = 0xFFFF; + unsigned last_pos = 0xFFFF; for (unsigned current_reg = lb; current_reg < ub; current_reg++) { - if (reg_file[current_reg] != 0 || ctx.war_hint[current_reg]) { - if (next_pos == 0xFFFF) - continue; - /* check if the variable fits */ - if (next_pos + size > current_reg) { - next_pos = 0xFFFF; + if (reg_file[current_reg] == 0 && !ctx.war_hint[current_reg]) { + if (last_pos == 0xFFFF) + last_pos = current_reg; + + /* stop searching after max_used_gpr */ + if (current_reg == ctx.max_used_sgpr + 1 || current_reg == 256 + ctx.max_used_vgpr + 1) + break; + else continue; - } + } - /* check if the tested gap is smaller */ - if (current_reg - next_pos < gap_size) { - best_pos = next_pos; - gap_size = current_reg - next_pos; - } - next_pos = 0xFFFF; + if (last_pos == 0xFFFF) continue; + + /* early return on exact matches */ + if (last_pos + size == current_reg) { + adjust_max_used_regs(ctx, rc, last_pos); + return {PhysReg{last_pos}, true}; } - if (next_pos == 0xFFFF) - next_pos = current_reg; + /* check if it fits and the gap size is smaller */ + if (last_pos + size < current_reg && current_reg - last_pos < gap_size) { + best_pos = last_pos; + gap_size = current_reg - last_pos; + } + last_pos = 0xFFFF; } /* final check */ - if (next_pos != 0xFFFF && - next_pos + size <= ub && - ub - next_pos < gap_size) { - best_pos = next_pos; - gap_size = ub - next_pos; + if (last_pos + size <= ub && ub - last_pos < gap_size) { + best_pos = last_pos; + gap_size = ub - last_pos; } - if (best_pos != 0xFFFF) { - adjust_max_used_regs(ctx, rc, best_pos); - return {PhysReg{best_pos}, true}; + + if (best_pos == 0xFFFF) + return {{}, false}; + + /* find best position within gap by leaving a good stride for other variables*/ + unsigned buffer = gap_size - size; + if (buffer > 1) { + if (((best_pos + size) % 8 != 0 && (best_pos + buffer) % 8 == 0) || + ((best_pos + size) % 4 != 0 && (best_pos + buffer) % 4 == 0) || + ((best_pos + size) % 2 != 0 && (best_pos + buffer) % 2 == 0)) + best_pos = best_pos + buffer; } - return {{}, false}; + + adjust_max_used_regs(ctx, rc, best_pos); + return {PhysReg{best_pos}, true}; } bool found = false; @@ -398,6 +683,35 @@ std::pair get_reg_simple(ra_ctx& ctx, reg_lo += stride; } + /* do this late because using the upper bytes of a register can require + * larger instruction encodings or copies + * TODO: don't do this in situations where it doesn't benefit */ + if (rc.is_subdword()) { + for (std::pair> entry : reg_file.subdword_regs) { + assert(reg_file[entry.first] == 0xF0000000); + if (lb > entry.first || entry.first >= ub) + continue; + + for (unsigned i = 0; i < 4; i+= info.stride) { + if (entry.second[i] != 0) + continue; + + bool reg_found = true; + for (unsigned j = 1; reg_found && i + j < 4 && j < rc.bytes(); j++) + reg_found &= entry.second[i + j] == 0; + + /* check neighboring reg if needed */ + reg_found &= ((int)i <= 4 - (int)rc.bytes() || reg_file[entry.first + 1] == 0); + if (reg_found) { + PhysReg res{entry.first}; + res.reg_b += i; + adjust_max_used_regs(ctx, rc, entry.first); + return {res, true}; + } + } + } + } + return {{}, false}; } @@ -445,47 +759,53 @@ bool get_regs_for_copies(ra_ctx& ctx, for (std::set>::const_reverse_iterator it = vars.rbegin(); it != vars.rend(); ++it) { unsigned id = it->second; assignment& var = ctx.assignments[id]; - uint32_t size = var.rc.size(); - uint32_t stride = 1; - if (var.rc.type() == RegType::sgpr) { - if (size == 2) - stride = 2; - if (size > 3) - stride = 4; - } + DefInfo info = DefInfo(ctx, ctx.pseudo_dummy, var.rc, -1); + uint32_t size = info.size; - /* check if this is a dead operand, then we can re-use the space from the definition */ + /* check if this is a dead operand, then we can re-use the space from the definition + * also use the correct stride for sub-dword operands */ bool is_dead_operand = false; - for (unsigned i = 0; !is_phi(instr) && !is_dead_operand && (i < instr->operands.size()); i++) { - if (instr->operands[i].isTemp() && instr->operands[i].isKillBeforeDef() && instr->operands[i].tempId() == id) - is_dead_operand = true; + for (unsigned i = 0; !is_phi(instr) && i < instr->operands.size(); i++) { + if (instr->operands[i].isTemp() && instr->operands[i].tempId() == id) { + if (instr->operands[i].isKillBeforeDef()) + is_dead_operand = true; + info = DefInfo(ctx, instr, var.rc, i); + break; + } } std::pair res; if (is_dead_operand) { if (instr->opcode == aco_opcode::p_create_vector) { - for (unsigned i = 0, offset = 0; i < instr->operands.size(); offset += instr->operands[i].size(), i++) { + PhysReg reg(def_reg_lo); + for (unsigned i = 0; i < instr->operands.size(); i++) { if (instr->operands[i].isTemp() && instr->operands[i].tempId() == id) { - for (unsigned j = 0; j < size; j++) - assert(reg_file[def_reg_lo + offset + j] == 0); - res = {PhysReg{def_reg_lo + offset}, true}; + res = {reg, (!var.rc.is_subdword() || (reg.byte() % info.stride == 0)) && !reg_file.test(reg, var.rc.bytes())}; break; } + reg.reg_b += instr->operands[i].bytes(); } + if (!res.second) + res = {var.reg, !reg_file.test(var.reg, var.rc.bytes())}; } else { - res = get_reg_simple(ctx, reg_file, def_reg_lo, def_reg_hi + 1, size, stride, var.rc); + info.lb = def_reg_lo; + info.ub = def_reg_hi + 1; + res = get_reg_simple(ctx, reg_file, info); } } else { - res = get_reg_simple(ctx, reg_file, lb, def_reg_lo, size, stride, var.rc); - if (!res.second) { - unsigned lb = (def_reg_hi + stride) & ~(stride - 1); - res = get_reg_simple(ctx, reg_file, lb, ub, size, stride, var.rc); + info.lb = lb; + info.ub = MIN2(def_reg_lo, ub); + res = get_reg_simple(ctx, reg_file, info); + if (!res.second && def_reg_hi < ub) { + info.lb = (def_reg_hi + info.stride) & ~(info.stride - 1); + info.ub = ub; + res = get_reg_simple(ctx, reg_file, info); } } if (res.second) { /* mark the area as blocked */ - reg_file.block(res.first, var.rc.bytes()); + reg_file.block(res.first, var.rc); /* create parallelcopy pair (without definition id) */ Temp tmp = Temp(id, var.rc); @@ -503,6 +823,7 @@ bool get_regs_for_copies(ra_ctx& ctx, /* we use a sliding window to find potential positions */ unsigned reg_lo = lb; unsigned reg_hi = lb + size - 1; + unsigned stride = var.rc.is_subdword() ? 1 : info.stride; for (reg_lo = lb, reg_hi = lb + size - 1; reg_hi < ub; reg_lo += stride, reg_hi += stride) { if (!is_dead_operand && ((reg_lo >= def_reg_lo && reg_lo <= def_reg_hi) || (reg_hi >= def_reg_lo && reg_hi <= def_reg_hi))) @@ -571,7 +892,7 @@ bool get_regs_for_copies(ra_ctx& ctx, std::set> new_vars = collect_vars(ctx, reg_file, PhysReg{reg_lo}, size); /* mark the area as blocked */ - reg_file.block(PhysReg{reg_lo}, size * 4); + reg_file.block(PhysReg{reg_lo}, var.rc); if (!get_regs_for_copies(ctx, reg_file, parallelcopies, new_vars, lb, ub, instr, def_reg_lo, def_reg_hi)) return false; @@ -593,11 +914,15 @@ bool get_regs_for_copies(ra_ctx& ctx, std::pair get_reg_impl(ra_ctx& ctx, RegisterFile& reg_file, std::vector>& parallelcopies, - uint32_t lb, uint32_t ub, - uint32_t size, uint32_t stride, - RegClass rc, + DefInfo info, aco_ptr& instr) { + uint32_t lb = info.lb; + uint32_t ub = info.ub; + uint32_t size = info.size; + uint32_t stride = info.stride; + RegClass rc = info.rc; + /* check how many free regs we have */ unsigned regs_free = reg_file.count_zero(PhysReg{lb}, ub-lb); @@ -607,10 +932,10 @@ std::pair get_reg_impl(ra_ctx& ctx, if (instr->operands[j].isTemp() && instr->operands[j].isFirstKillBeforeDef() && instr->operands[j].physReg() >= lb && - instr->operands[j].physReg() < ub) { + instr->operands[j].physReg() < ub && + !reg_file.test(instr->operands[j].physReg(), instr->operands[j].bytes())) { assert(instr->operands[j].isFixed()); - assert(!reg_file.test(instr->operands[j].physReg(), instr->operands[j].bytes())); - reg_file.block(instr->operands[j].physReg(), instr->operands[j].bytes()); + reg_file.block(instr->operands[j].physReg(), instr->operands[j].regClass()); killed_ops += instr->operands[j].getTemp().size(); } } @@ -632,9 +957,11 @@ std::pair get_reg_impl(ra_ctx& ctx, unsigned reg_hi = lb + size - 1; for (reg_lo = lb, reg_hi = lb + size - 1; reg_hi < ub; reg_lo += stride, reg_hi += stride) { /* first check the edges: this is what we have to fix to allow for num_moves > size */ - if (reg_lo > lb && reg_file[reg_lo] != 0 && reg_file[reg_lo] == reg_file[reg_lo - 1]) + if (reg_lo > lb && !reg_file.is_empty_or_blocked(PhysReg(reg_lo)) && + reg_file.get_id(PhysReg(reg_lo)) == reg_file.get_id(PhysReg(reg_lo).advance(-1))) continue; - if (reg_hi < ub - 1 && reg_file[reg_hi] != 0 && reg_file[reg_hi] == reg_file[reg_hi + 1]) + if (reg_hi < ub - 1 && !reg_file.is_empty_or_blocked(PhysReg(reg_hi).advance(3)) && + reg_file.get_id(PhysReg(reg_hi).advance(3)) == reg_file.get_id(PhysReg(reg_hi).advance(4))) continue; /* second, check that we have at most k=num_moves elements in the window @@ -714,24 +1041,29 @@ std::pair get_reg_impl(ra_ctx& ctx, std::set> vars = collect_vars(ctx, reg_file, PhysReg{best_pos}, size); if (instr->opcode == aco_opcode::p_create_vector) { - /* move killed operands which aren't yet at the correct position */ - for (unsigned i = 0, offset = 0; i < instr->operands.size(); offset += instr->operands[i].size(), i++) { - if (instr->operands[i].isTemp() && instr->operands[i].isFirstKillBeforeDef() && - instr->operands[i].getTemp().type() == rc.type()) { - - if (instr->operands[i].physReg() != best_pos + offset) { - vars.emplace(instr->operands[i].bytes(), instr->operands[i].tempId()); - reg_file.clear(instr->operands[i]); + /* move killed operands which aren't yet at the correct position (GFX9+) + * or which are in the definition space */ + PhysReg reg = PhysReg{best_pos}; + for (Operand& op : instr->operands) { + if (op.isTemp() && op.isFirstKillBeforeDef() && + op.getTemp().type() == rc.type()) { + if (op.physReg() != reg && + (ctx.program->chip_class >= GFX9 || + (op.physReg().advance(op.bytes()) > PhysReg{best_pos} && + op.physReg() < PhysReg{best_pos + size}))) { + vars.emplace(op.bytes(), op.tempId()); + reg_file.clear(op); } else { - reg_file.fill(instr->operands[i]); + reg_file.fill(op); } } + reg.reg_b += op.bytes(); } - } else { - /* re-enable the killed operands */ - for (unsigned j = 0; !is_phi(instr) && j < instr->operands.size(); j++) { - if (instr->operands[j].isTemp() && instr->operands[j].isFirstKill()) - reg_file.fill(instr->operands[j]); + } else if (!is_phi(instr)) { + /* re-enable killed operands */ + for (Operand& op : instr->operands) { + if (op.isTemp() && op.isFirstKillBeforeDef()) + reg_file.fill(op); } } @@ -758,7 +1090,7 @@ std::pair get_reg_impl(ra_ctx& ctx, /* we set the definition regs == 0. the actual caller is responsible for correct setting */ reg_file.clear(PhysReg{best_pos}, rc); - update_renames(ctx, reg_file, parallelcopies, instr); + update_renames(ctx, reg_file, parallelcopies, instr, instr->opcode != aco_opcode::p_create_vector); /* remove killed operands from reg_file once again */ for (unsigned i = 0; !is_phi(instr) && i < instr->operands.size(); i++) { @@ -778,48 +1110,133 @@ std::pair get_reg_impl(ra_ctx& ctx, return {PhysReg{best_pos}, true}; } -PhysReg get_reg(ra_ctx& ctx, - RegisterFile& reg_file, - RegClass rc, - std::vector>& parallelcopies, - aco_ptr& instr) +bool get_reg_specified(ra_ctx& ctx, + RegisterFile& reg_file, + RegClass rc, + std::vector>& parallelcopies, + aco_ptr& instr, + PhysReg reg) { + std::pair sdw_def_info; + if (rc.is_subdword()) + sdw_def_info = get_subdword_definition_info(ctx.program, instr, rc); + + if (rc.is_subdword() && reg.byte() % sdw_def_info.first) + return false; + if (!rc.is_subdword() && reg.byte()) + return false; + uint32_t size = rc.size(); uint32_t stride = 1; uint32_t lb, ub; + if (rc.type() == RegType::vgpr) { lb = 256; ub = 256 + ctx.program->max_reg_demand.vgpr; } else { - lb = 0; - ub = ctx.program->max_reg_demand.sgpr; if (size == 2) stride = 2; else if (size >= 4) stride = 4; + if (reg % stride != 0) + return false; + lb = 0; + ub = ctx.program->max_reg_demand.sgpr; } + uint32_t reg_lo = reg.reg(); + uint32_t reg_hi = reg + (size - 1); + + if (reg_lo < lb || reg_hi >= ub || reg_lo > reg_hi) + return false; + if (rc.is_subdword()) { - /* stride in bytes */ - if(!instr_can_access_subdword(instr)) - stride = 4; - else if (rc.bytes() % 4 == 0) - stride = 4; - else if (rc.bytes() % 2 == 0) - stride = 2; + PhysReg test_reg; + test_reg.reg_b = reg.reg_b & ~(sdw_def_info.second - 1); + if (reg_file.test(test_reg, sdw_def_info.second)) + return false; + } else { + if (reg_file.test(reg, rc.bytes())) + return false; + } + + adjust_max_used_regs(ctx, rc, reg_lo); + return true; +} + +PhysReg get_reg(ra_ctx& ctx, + RegisterFile& reg_file, + Temp temp, + std::vector>& parallelcopies, + aco_ptr& instr, + int operand_index=-1) +{ + auto split_vec = ctx.split_vectors.find(temp.id()); + if (split_vec != ctx.split_vectors.end()) { + unsigned offset = 0; + for (Definition def : split_vec->second->definitions) { + auto affinity_it = ctx.affinities.find(def.tempId()); + if (affinity_it != ctx.affinities.end() && ctx.assignments[affinity_it->second].assigned) { + PhysReg reg = ctx.assignments[affinity_it->second].reg; + reg.reg_b -= offset; + if (get_reg_specified(ctx, reg_file, temp.regClass(), parallelcopies, instr, reg)) + return reg; + } + offset += def.bytes(); + } + } + + if (ctx.affinities.find(temp.id()) != ctx.affinities.end() && + ctx.assignments[ctx.affinities[temp.id()]].assigned) { + PhysReg reg = ctx.assignments[ctx.affinities[temp.id()]].reg; + if (get_reg_specified(ctx, reg_file, temp.regClass(), parallelcopies, instr, reg)) + return reg; } - std::pair res = {{}, false}; + if (ctx.vectors.find(temp.id()) != ctx.vectors.end()) { + Instruction* vec = ctx.vectors[temp.id()]; + unsigned byte_offset = 0; + for (const Operand& op : vec->operands) { + if (op.isTemp() && op.tempId() == temp.id()) + break; + else + byte_offset += op.bytes(); + } + unsigned k = 0; + for (const Operand& op : vec->operands) { + if (op.isTemp() && + op.tempId() != temp.id() && + op.getTemp().type() == temp.type() && + ctx.assignments[op.tempId()].assigned) { + PhysReg reg = ctx.assignments[op.tempId()].reg; + reg.reg_b += (byte_offset - k); + if (get_reg_specified(ctx, reg_file, temp.regClass(), parallelcopies, instr, reg)) + return reg; + } + k += op.bytes(); + } + + DefInfo info(ctx, ctx.pseudo_dummy, vec->definitions[0].regClass(), -1); + std::pair res = get_reg_simple(ctx, reg_file, info); + PhysReg reg = res.first; + if (res.second) { + reg.reg_b += byte_offset; + /* make sure to only use byte offset if the instruction supports it */ + if (get_reg_specified(ctx, reg_file, temp.regClass(), parallelcopies, instr, reg)) + return reg; + } + } + + DefInfo info(ctx, instr, temp.regClass(), operand_index); + /* try to find space without live-range splits */ - if (rc.type() == RegType::vgpr && (size == 4 || size == 8)) - res = get_reg_simple(ctx, reg_file, lb, ub, size, 4, rc); - if (!res.second) - res = get_reg_simple(ctx, reg_file, lb, ub, size, stride, rc); + std::pair res = get_reg_simple(ctx, reg_file, info); + if (res.second) return res.first; /* try to find space with live-range splits */ - res = get_reg_impl(ctx, reg_file, parallelcopies, lb, ub, size, stride, rc, instr); + res = get_reg_impl(ctx, reg_file, parallelcopies, info, instr); if (res.second) return res.first; @@ -828,54 +1245,34 @@ PhysReg get_reg(ra_ctx& ctx, /* We should only fail here because keeping under the limit would require * too many moves. */ - assert(reg_file.count_zero(PhysReg{lb}, ub-lb) >= size); + assert(reg_file.count_zero(PhysReg{info.lb}, info.ub-info.lb) >= info.size); uint16_t max_addressible_sgpr = ctx.program->sgpr_limit; uint16_t max_addressible_vgpr = ctx.program->vgpr_limit; - if (rc.type() == RegType::vgpr && ctx.program->max_reg_demand.vgpr < max_addressible_vgpr) { + if (info.rc.type() == RegType::vgpr && ctx.program->max_reg_demand.vgpr < max_addressible_vgpr) { update_vgpr_sgpr_demand(ctx.program, RegisterDemand(ctx.program->max_reg_demand.vgpr + 1, ctx.program->max_reg_demand.sgpr)); - return get_reg(ctx, reg_file, rc, parallelcopies, instr); - } else if (rc.type() == RegType::sgpr && ctx.program->max_reg_demand.sgpr < max_addressible_sgpr) { + return get_reg(ctx, reg_file, temp, parallelcopies, instr, operand_index); + } else if (info.rc.type() == RegType::sgpr && ctx.program->max_reg_demand.sgpr < max_addressible_sgpr) { update_vgpr_sgpr_demand(ctx.program, RegisterDemand(ctx.program->max_reg_demand.vgpr, ctx.program->max_reg_demand.sgpr + 1)); - return get_reg(ctx, reg_file, rc, parallelcopies, instr); + return get_reg(ctx, reg_file, temp, parallelcopies, instr, operand_index); } //FIXME: if nothing helps, shift-rotate the registers to make space - unreachable("did not find a register"); + fprintf(stderr, "ACO: failed to allocate registers during shader compilation\n"); + abort(); } - -std::pair get_reg_vec(ra_ctx& ctx, - RegisterFile& reg_file, - RegClass rc) -{ - uint32_t size = rc.size(); - uint32_t stride = 1; - uint32_t lb, ub; - if (rc.type() == RegType::vgpr) { - lb = 256; - ub = 256 + ctx.program->max_reg_demand.vgpr; - } else { - lb = 0; - ub = ctx.program->max_reg_demand.sgpr; - if (size == 2) - stride = 2; - else if (size >= 4) - stride = 4; - } - return get_reg_simple(ctx, reg_file, lb, ub, size, stride, rc); -} - - PhysReg get_reg_create_vector(ra_ctx& ctx, RegisterFile& reg_file, - RegClass rc, + Temp temp, std::vector>& parallelcopies, aco_ptr& instr) { + RegClass rc = temp.regClass(); /* create_vector instructions have different costs w.r.t. register coalescing */ uint32_t size = rc.size(); + uint32_t bytes = rc.bytes(); uint32_t stride = 1; uint32_t lb, ub; if (rc.type() == RegType::vgpr) { @@ -890,20 +1287,25 @@ PhysReg get_reg_create_vector(ra_ctx& ctx, stride = 4; } + //TODO: improve p_create_vector for sub-dword vectors + unsigned best_pos = -1; unsigned num_moves = 0xFF; bool best_war_hint = true; /* test for each operand which definition placement causes the least shuffle instructions */ - for (unsigned i = 0, offset = 0; i < instr->operands.size(); offset += instr->operands[i].size(), i++) { + for (unsigned i = 0, offset = 0; i < instr->operands.size(); offset += instr->operands[i].bytes(), i++) { // TODO: think about, if we can alias live operands on the same register if (!instr->operands[i].isTemp() || !instr->operands[i].isKillBeforeDef() || instr->operands[i].getTemp().type() != rc.type()) continue; - if (offset > instr->operands[i].physReg()) + if (offset > instr->operands[i].physReg().reg_b) continue; - unsigned reg_lo = instr->operands[i].physReg() - offset; + unsigned reg_lo = instr->operands[i].physReg().reg_b - offset; + if (reg_lo % 4) + continue; + reg_lo /= 4; unsigned reg_hi = reg_lo + size - 1; unsigned k = 0; @@ -915,9 +1317,9 @@ PhysReg get_reg_create_vector(ra_ctx& ctx, // TODO: this can be improved */ if (reg_lo < lb || reg_hi >= ub || reg_lo % stride != 0) continue; - if (reg_lo > lb && reg_file[reg_lo] != 0 && reg_file[reg_lo] == reg_file[reg_lo - 1]) + if (reg_lo > lb && reg_file[reg_lo] != 0 && reg_file.get_id(PhysReg(reg_lo)) == reg_file.get_id(PhysReg(reg_lo).advance(-1))) continue; - if (reg_hi < ub - 1 && reg_file[reg_hi] != 0 && reg_file[reg_hi] == reg_file[reg_hi + 1]) + if (reg_hi < ub - 1 && reg_file[reg_hi] != 0 && reg_file.get_id(PhysReg(reg_hi).advance(3)) == reg_file.get_id(PhysReg(reg_hi).advance(4))) continue; /* count variables to be moved and check war_hint */ @@ -925,10 +1327,18 @@ PhysReg get_reg_create_vector(ra_ctx& ctx, bool linear_vgpr = false; for (unsigned j = reg_lo; j <= reg_hi && !linear_vgpr; j++) { if (reg_file[j] != 0) { - k++; - /* we cannot split live ranges of linear vgprs */ - if (ctx.assignments[reg_file[j]].rc & (1 << 6)) - linear_vgpr = true; + if (reg_file[j] == 0xF0000000) { + PhysReg reg; + reg.reg_b = j * 4; + unsigned bytes_left = bytes - (j - reg_lo) * 4; + for (unsigned k = 0; k < MIN2(bytes_left, 4); k++, reg.reg_b++) + k += reg_file.test(reg, 1); + } else { + k += 4; + /* we cannot split live ranges of linear vgprs */ + if (ctx.assignments[reg_file[j]].rc & (1 << 6)) + linear_vgpr = true; + } } war_hint |= ctx.war_hint[j]; } @@ -936,13 +1346,13 @@ PhysReg get_reg_create_vector(ra_ctx& ctx, continue; /* count operands in wrong positions */ - for (unsigned j = 0, offset = 0; j < instr->operands.size(); offset += instr->operands[j].size(), j++) { + for (unsigned j = 0, offset = 0; j < instr->operands.size(); offset += instr->operands[j].bytes(), j++) { if (j == i || !instr->operands[j].isTemp() || instr->operands[j].getTemp().type() != rc.type()) continue; - if (instr->operands[j].physReg() != reg_lo + offset) - k += instr->operands[j].size(); + if (instr->operands[j].physReg().reg_b != reg_lo * 4 + offset) + k += instr->operands[j].bytes(); } bool aligned = rc == RegClass::v4 && reg_lo % 4 == 0; if (k > num_moves || (!aligned && k == num_moves)) @@ -953,69 +1363,54 @@ PhysReg get_reg_create_vector(ra_ctx& ctx, best_war_hint = war_hint; } - if (num_moves >= size) - return get_reg(ctx, reg_file, rc, parallelcopies, instr); + if (num_moves >= bytes) + return get_reg(ctx, reg_file, temp, parallelcopies, instr); + + /* re-enable killed operands which are in the wrong position */ + for (unsigned i = 0, offset = 0; i < instr->operands.size(); offset += instr->operands[i].bytes(), i++) { + if (instr->operands[i].isTemp() && + instr->operands[i].isFirstKillBeforeDef() && + instr->operands[i].physReg().reg_b != best_pos * 4 + offset) + reg_file.fill(instr->operands[i]); + } /* collect variables to be moved */ std::set> vars = collect_vars(ctx, reg_file, PhysReg{best_pos}, size); - /* move killed operands which aren't yet at the correct position */ - for (unsigned i = 0, offset = 0; i < instr->operands.size(); offset += instr->operands[i].size(), i++) { - if (instr->operands[i].isTemp() && - instr->operands[i].isFirstKillBeforeDef() && - instr->operands[i].getTemp().type() == rc.type() && - instr->operands[i].physReg() != best_pos + offset) + for (unsigned i = 0, offset = 0; i < instr->operands.size(); offset += instr->operands[i].bytes(), i++) { + if (!instr->operands[i].isTemp() || !instr->operands[i].isFirstKillBeforeDef() || + instr->operands[i].getTemp().type() != rc.type()) + continue; + bool correct_pos = instr->operands[i].physReg().reg_b == best_pos * 4 + offset; + /* GFX9+: move killed operands which aren't yet at the correct position + * Moving all killed operands generally leads to more register swaps. + * This is only done on GFX9+ because of the cheap v_swap instruction. + */ + if (ctx.program->chip_class >= GFX9 && !correct_pos) { vars.emplace(instr->operands[i].bytes(), instr->operands[i].tempId()); + reg_file.clear(instr->operands[i]); + /* fill operands which are in the correct position to avoid overwriting */ + } else if (correct_pos) { + reg_file.fill(instr->operands[i]); + } } - ASSERTED bool success = false; success = get_regs_for_copies(ctx, reg_file, parallelcopies, vars, lb, ub, instr, best_pos, best_pos + size - 1); assert(success); - update_renames(ctx, reg_file, parallelcopies, instr); + update_renames(ctx, reg_file, parallelcopies, instr, false); adjust_max_used_regs(ctx, rc, best_pos); - return PhysReg{best_pos}; -} - -bool get_reg_specified(ra_ctx& ctx, - RegisterFile& reg_file, - RegClass rc, - std::vector>& parallelcopies, - aco_ptr& instr, - PhysReg reg) -{ - uint32_t size = rc.size(); - uint32_t stride = 1; - uint32_t lb, ub; - if (rc.type() == RegType::vgpr) { - lb = 256; - ub = 256 + ctx.program->max_reg_demand.vgpr; - } else { - if (size == 2) - stride = 2; - else if (size >= 4) - stride = 4; - if (reg % stride != 0) - return false; - lb = 0; - ub = ctx.program->max_reg_demand.sgpr; + /* remove killed operands from reg_file once again */ + for (unsigned i = 0; i < instr->operands.size(); i++) { + if (!instr->operands[i].isTemp() || !instr->operands[i].isFixed()) + continue; + assert(!instr->operands[i].isUndefined()); + if (instr->operands[i].isFirstKillBeforeDef()) + reg_file.clear(instr->operands[i]); } - if (rc.is_subdword() && reg.byte() && !instr_can_access_subdword(instr)) - return false; - - uint32_t reg_lo = reg.reg(); - uint32_t reg_hi = reg + (size - 1); - - if (reg_lo < lb || reg_hi >= ub || reg_lo > reg_hi) - return false; - - if (reg_file.test(reg, rc.bytes())) - return false; - - adjust_max_used_regs(ctx, rc, reg_lo); - return true; + return PhysReg{best_pos}; } void handle_pseudo(ra_ctx& ctx, @@ -1047,13 +1442,18 @@ void handle_pseudo(ra_ctx& ctx, } /* if all operands are constant, no need to care either */ bool reads_sgpr = false; + bool reads_subdword = false; for (Operand& op : instr->operands) { if (op.isTemp() && op.getTemp().type() == RegType::sgpr) { reads_sgpr = true; break; } + if (op.isTemp() && op.regClass().is_subdword()) + reads_subdword = true; } - if (!(writes_sgpr && reads_sgpr)) + bool needs_scratch_reg = (writes_sgpr && reads_sgpr) || + (ctx.program->chip_class <= GFX7 && reads_subdword); + if (!needs_scratch_reg) return; Pseudo_instruction *pi = (Pseudo_instruction *)instr; @@ -1067,7 +1467,10 @@ void handle_pseudo(ra_ctx& ctx, reg = ctx.max_used_sgpr + 1; for (; reg < ctx.program->max_reg_demand.sgpr && reg_file[reg]; reg++) ; - assert(reg < ctx.program->max_reg_demand.sgpr); + if (reg == ctx.program->max_reg_demand.sgpr) { + assert(reads_subdword && reg_file[m0] == 0); + reg = m0; + } } adjust_max_used_regs(ctx, s1, reg); @@ -1077,10 +1480,16 @@ void handle_pseudo(ra_ctx& ctx, } } -bool operand_can_use_reg(aco_ptr& instr, unsigned idx, PhysReg reg) +bool operand_can_use_reg(chip_class chip, aco_ptr& instr, unsigned idx, PhysReg reg, RegClass rc) { - if (!instr_can_access_subdword(instr) && reg.byte()) - return false; + if (instr->operands[idx].isFixed()) + return instr->operands[idx].physReg() == reg; + + if (reg.byte()) { + unsigned stride = get_subdword_operand_stride(chip, instr, idx, rc); + if (reg.byte() % stride) + return false; + } switch (instr->format) { case Format::SMEM: @@ -1094,175 +1503,194 @@ bool operand_can_use_reg(aco_ptr& instr, unsigned idx, PhysReg reg) } } -} /* end namespace */ - - -void register_allocation(Program *program, std::vector& live_out_per_block) +void get_reg_for_operand(ra_ctx& ctx, RegisterFile& register_file, + std::vector>& parallelcopy, + aco_ptr& instr, Operand& operand, unsigned operand_index) { - ra_ctx ctx(program); - - std::vector> renames(program->blocks.size()); - - struct phi_info { - Instruction* phi; - unsigned block_idx; - std::set uses; - }; + /* check if the operand is fixed */ + PhysReg dst; + if (operand.isFixed()) { + assert(operand.physReg() != ctx.assignments[operand.tempId()].reg); + + /* check if target reg is blocked, and move away the blocking var */ + if (register_file[operand.physReg().reg()]) { + assert(register_file[operand.physReg()] != 0xF0000000); + uint32_t blocking_id = register_file[operand.physReg().reg()]; + RegClass rc = ctx.assignments[blocking_id].rc; + Operand pc_op = Operand(Temp{blocking_id, rc}); + pc_op.setFixed(operand.physReg()); + + /* find free reg */ + PhysReg reg = get_reg(ctx, register_file, pc_op.getTemp(), parallelcopy, ctx.pseudo_dummy); + Definition pc_def = Definition(PhysReg{reg}, pc_op.regClass()); + register_file.clear(pc_op); + parallelcopy.emplace_back(pc_op, pc_def); + } + dst = operand.physReg(); - bool filled[program->blocks.size()]; - bool sealed[program->blocks.size()]; - memset(filled, 0, sizeof filled); - memset(sealed, 0, sizeof sealed); - std::vector> incomplete_phis(program->blocks.size()); - std::unordered_map phi_map; - std::unordered_map affinities; - std::function read_variable; - std::function handle_live_in; - std::function try_remove_trivial_phi; - - read_variable = [&](Temp val, unsigned block_idx) -> Temp { - std::unordered_map::iterator it = renames[block_idx].find(val.id()); - if (it == renames[block_idx].end()) - return val; - else - return it->second; - }; + } else { + dst = get_reg(ctx, register_file, operand.getTemp(), parallelcopy, instr, operand_index); + } - handle_live_in = [&](Temp val, Block *block) -> Temp { - std::vector& preds = val.is_linear() ? block->linear_preds : block->logical_preds; - if (preds.size() == 0 || val.regClass() == val.regClass().as_linear()) - return val; + Operand pc_op = operand; + pc_op.setFixed(ctx.assignments[operand.tempId()].reg); + Definition pc_def = Definition(dst, pc_op.regClass()); + register_file.clear(pc_op); + parallelcopy.emplace_back(pc_op, pc_def); + update_renames(ctx, register_file, parallelcopy, instr, true); +} - assert(preds.size() > 0); +Temp read_variable(ra_ctx& ctx, Temp val, unsigned block_idx) +{ + std::unordered_map::iterator it = ctx.renames[block_idx].find(val.id()); + if (it == ctx.renames[block_idx].end()) + return val; + else + return it->second; +} - Temp new_val; - if (!sealed[block->index]) { - /* consider rename from already processed predecessor */ - Temp tmp = read_variable(val, preds[0]); +Temp handle_live_in(ra_ctx& ctx, Temp val, Block* block) +{ + std::vector& preds = val.is_linear() ? block->linear_preds : block->logical_preds; + if (preds.size() == 0 || val.regClass() == val.regClass().as_linear()) + return val; + + assert(preds.size() > 0); + + Temp new_val; + if (!ctx.sealed[block->index]) { + /* consider rename from already processed predecessor */ + Temp tmp = read_variable(ctx, val, preds[0]); + + /* if the block is not sealed yet, we create an incomplete phi (which might later get removed again) */ + new_val = Temp{ctx.program->allocateId(), val.regClass()}; + ctx.assignments.emplace_back(); + aco_opcode opcode = val.is_linear() ? aco_opcode::p_linear_phi : aco_opcode::p_phi; + aco_ptr phi{create_instruction(opcode, Format::PSEUDO, preds.size(), 1)}; + phi->definitions[0] = Definition(new_val); + for (unsigned i = 0; i < preds.size(); i++) + phi->operands[i] = Operand(val); + if (tmp.regClass() == new_val.regClass()) + ctx.affinities[new_val.id()] = tmp.id(); + + ctx.phi_map.emplace(new_val.id(), phi_info{phi.get(), block->index}); + ctx.incomplete_phis[block->index].emplace_back(phi.get()); + block->instructions.insert(block->instructions.begin(), std::move(phi)); + + } else if (preds.size() == 1) { + /* if the block has only one predecessor, just look there for the name */ + new_val = read_variable(ctx, val, preds[0]); + } else { + /* there are multiple predecessors and the block is sealed */ + Temp ops[preds.size()]; + + /* get the rename from each predecessor and check if they are the same */ + bool needs_phi = false; + for (unsigned i = 0; i < preds.size(); i++) { + ops[i] = read_variable(ctx, val, preds[i]); + if (i == 0) + new_val = ops[i]; + else + needs_phi |= !(new_val == ops[i]); + } - /* if the block is not sealed yet, we create an incomplete phi (which might later get removed again) */ - new_val = Temp{program->allocateId(), val.regClass()}; - ctx.assignments.emplace_back(); + if (needs_phi) { + /* the variable has been renamed differently in the predecessors: we need to insert a phi */ aco_opcode opcode = val.is_linear() ? aco_opcode::p_linear_phi : aco_opcode::p_phi; aco_ptr phi{create_instruction(opcode, Format::PSEUDO, preds.size(), 1)}; + new_val = Temp{ctx.program->allocateId(), val.regClass()}; phi->definitions[0] = Definition(new_val); - for (unsigned i = 0; i < preds.size(); i++) - phi->operands[i] = Operand(val); - if (tmp.regClass() == new_val.regClass()) - affinities[new_val.id()] = tmp.id(); - - phi_map.emplace(new_val.id(), phi_info{phi.get(), block->index}); - incomplete_phis[block->index].emplace_back(phi.get()); - block->instructions.insert(block->instructions.begin(), std::move(phi)); - - } else if (preds.size() == 1) { - /* if the block has only one predecessor, just look there for the name */ - new_val = read_variable(val, preds[0]); - } else { - /* there are multiple predecessors and the block is sealed */ - Temp ops[preds.size()]; - - /* get the rename from each predecessor and check if they are the same */ - bool needs_phi = false; for (unsigned i = 0; i < preds.size(); i++) { - ops[i] = read_variable(val, preds[i]); - if (i == 0) - new_val = ops[i]; - else - needs_phi |= !(new_val == ops[i]); - } - - if (needs_phi) { - /* the variable has been renamed differently in the predecessors: we need to insert a phi */ - aco_opcode opcode = val.is_linear() ? aco_opcode::p_linear_phi : aco_opcode::p_phi; - aco_ptr phi{create_instruction(opcode, Format::PSEUDO, preds.size(), 1)}; - new_val = Temp{program->allocateId(), val.regClass()}; - phi->definitions[0] = Definition(new_val); - for (unsigned i = 0; i < preds.size(); i++) { - phi->operands[i] = Operand(ops[i]); - phi->operands[i].setFixed(ctx.assignments[ops[i].id()].reg); - if (ops[i].regClass() == new_val.regClass()) - affinities[new_val.id()] = ops[i].id(); - } - ctx.assignments.emplace_back(); - assert(ctx.assignments.size() == ctx.program->peekAllocationId()); - phi_map.emplace(new_val.id(), phi_info{phi.get(), block->index}); - block->instructions.insert(block->instructions.begin(), std::move(phi)); + phi->operands[i] = Operand(ops[i]); + phi->operands[i].setFixed(ctx.assignments[ops[i].id()].reg); + if (ops[i].regClass() == new_val.regClass()) + ctx.affinities[new_val.id()] = ops[i].id(); } + ctx.assignments.emplace_back(); + assert(ctx.assignments.size() == ctx.program->peekAllocationId()); + ctx.phi_map.emplace(new_val.id(), phi_info{phi.get(), block->index}); + block->instructions.insert(block->instructions.begin(), std::move(phi)); } + } - if (new_val != val) { - renames[block->index][val.id()] = new_val; - ctx.orig_names[new_val.id()] = val; - } - return new_val; - }; + if (new_val != val) { + ctx.renames[block->index][val.id()] = new_val; + ctx.orig_names[new_val.id()] = val; + } + return new_val; +} - try_remove_trivial_phi = [&] (Temp temp) -> void { - std::unordered_map::iterator info = phi_map.find(temp.id()); +void try_remove_trivial_phi(ra_ctx& ctx, Temp temp) +{ + std::unordered_map::iterator info = ctx.phi_map.find(temp.id()); - if (info == phi_map.end() || !sealed[info->second.block_idx]) - return; + if (info == ctx.phi_map.end() || !ctx.sealed[info->second.block_idx]) + return; - assert(info->second.block_idx != 0); - Instruction* phi = info->second.phi; - Temp same = Temp(); - Definition def = phi->definitions[0]; + assert(info->second.block_idx != 0); + Instruction* phi = info->second.phi; + Temp same = Temp(); + Definition def = phi->definitions[0]; - /* a phi node is trivial if all operands are the same as the definition of the phi */ - for (const Operand& op : phi->operands) { - const Temp t = op.getTemp(); - if (t == same || t == def.getTemp()) { - assert(t == same || op.physReg() == def.physReg()); + /* a phi node is trivial if all operands are the same as the definition of the phi */ + for (const Operand& op : phi->operands) { + const Temp t = op.getTemp(); + if (t == same || t == def.getTemp()) { + assert(t == same || op.physReg() == def.physReg()); + continue; + } + if (same != Temp()) + return; + + same = t; + } + assert(same != Temp() || same == def.getTemp()); + + /* reroute all uses to same and remove phi */ + std::vector phi_users; + std::unordered_map::iterator same_phi_info = ctx.phi_map.find(same.id()); + for (Instruction* instr : info->second.uses) { + assert(phi != instr); + /* recursively try to remove trivial phis */ + if (is_phi(instr)) { + /* ignore if the phi was already flagged trivial */ + if (instr->definitions.empty()) continue; - } - if (same != Temp()) - return; - same = t; + if (instr->definitions[0].getTemp() != temp) + phi_users.emplace_back(instr->definitions[0].getTemp()); } - assert(same != Temp() || same == def.getTemp()); - - /* reroute all uses to same and remove phi */ - std::vector phi_users; - std::unordered_map::iterator same_phi_info = phi_map.find(same.id()); - for (Instruction* instr : info->second.uses) { - assert(phi != instr); - /* recursively try to remove trivial phis */ - if (is_phi(instr)) { - /* ignore if the phi was already flagged trivial */ - if (instr->definitions.empty()) - continue; - - if (instr->definitions[0].getTemp() != temp) - phi_users.emplace_back(instr->definitions[0].getTemp()); - } - for (Operand& op : instr->operands) { - if (op.isTemp() && op.tempId() == def.tempId()) { - op.setTemp(same); - if (same_phi_info != phi_map.end()) - same_phi_info->second.uses.emplace(instr); - } + for (Operand& op : instr->operands) { + if (op.isTemp() && op.tempId() == def.tempId()) { + op.setTemp(same); + if (same_phi_info != ctx.phi_map.end()) + same_phi_info->second.uses.emplace(instr); } } + } - auto it = ctx.orig_names.find(same.id()); - unsigned orig_var = it != ctx.orig_names.end() ? it->second.id() : same.id(); - for (unsigned i = 0; i < program->blocks.size(); i++) { - auto it = renames[i].find(orig_var); - if (it != renames[i].end() && it->second == def.getTemp()) - renames[i][orig_var] = same; - } + auto it = ctx.orig_names.find(same.id()); + unsigned orig_var = it != ctx.orig_names.end() ? it->second.id() : same.id(); + for (unsigned i = 0; i < ctx.program->blocks.size(); i++) { + auto it = ctx.renames[i].find(orig_var); + if (it != ctx.renames[i].end() && it->second == def.getTemp()) + ctx.renames[i][orig_var] = same; + } - phi->definitions.clear(); /* this indicates that the phi can be removed */ - phi_map.erase(info); - for (Temp t : phi_users) - try_remove_trivial_phi(t); + phi->definitions.clear(); /* this indicates that the phi can be removed */ + ctx.phi_map.erase(info); + for (Temp t : phi_users) + try_remove_trivial_phi(ctx, t); - return; - }; + return; +} - std::unordered_map vectors; +} /* end namespace */ + + +void register_allocation(Program *program, std::vector& live_out_per_block) +{ + ra_ctx ctx(program); std::vector> phi_ressources; std::unordered_map temp_to_phi_ressources; @@ -1276,9 +1704,10 @@ void register_allocation(Program *program, std::vector& live_out_per_bl for (rit = block.instructions.rbegin(); rit != block.instructions.rend(); ++rit) { aco_ptr& instr = *rit; if (is_phi(instr)) { - live.erase(instr->definitions[0].getTemp()); - if (instr->definitions[0].isKill() || instr->definitions[0].isFixed()) + if (instr->definitions[0].isKill() || instr->definitions[0].isFixed()) { + live.erase(instr->definitions[0].getTemp()); continue; + } /* collect information about affinity-related temporaries */ std::vector affinity_related; /* affinity_related[0] is the last seen affinity-related temp */ @@ -1291,23 +1720,25 @@ void register_allocation(Program *program, std::vector& live_out_per_bl } } phi_ressources.emplace_back(std::move(affinity_related)); - continue; - } + } else { + /* add vector affinities */ + if (instr->opcode == aco_opcode::p_create_vector) { + for (const Operand& op : instr->operands) { + if (op.isTemp() && op.isFirstKill() && op.getTemp().type() == instr->definitions[0].getTemp().type()) + ctx.vectors[op.tempId()] = instr.get(); + } + } - /* add vector affinities */ - if (instr->opcode == aco_opcode::p_create_vector) { + if (instr->opcode == aco_opcode::p_split_vector && instr->operands[0].isFirstKillBeforeDef()) + ctx.split_vectors[instr->operands[0].tempId()] = instr.get(); + + /* add operands to live variables */ for (const Operand& op : instr->operands) { - if (op.isTemp() && op.getTemp().type() == instr->definitions[0].getTemp().type()) - vectors[op.tempId()] = instr.get(); + if (op.isTemp()) + live.emplace(op.getTemp()); } } - /* add operands to live variables */ - for (const Operand& op : instr->operands) { - if (op.isTemp()) - live.emplace(op.getTemp()); - } - /* erase definitions from live */ for (unsigned i = 0; i < instr->definitions.size(); i++) { const Definition& def = instr->definitions[i]; @@ -1319,12 +1750,19 @@ void register_allocation(Program *program, std::vector& live_out_per_bl if (it != temp_to_phi_ressources.end() && def.regClass() == phi_ressources[it->second][0].regClass()) { phi_ressources[it->second][0] = def.getTemp(); /* try to coalesce phi affinities with parallelcopies */ - if (!def.isFixed() && instr->opcode == aco_opcode::p_parallelcopy) { - Operand op = instr->operands[i]; - if (op.isTemp() && op.isFirstKillBeforeDef() && def.regClass() == op.regClass()) { - phi_ressources[it->second].emplace_back(op.getTemp()); - temp_to_phi_ressources[op.tempId()] = it->second; - } + Operand op = Operand(); + if (!def.isFixed() && instr->opcode == aco_opcode::p_parallelcopy) + op = instr->operands[i]; + else if ((instr->opcode == aco_opcode::v_mad_f32 || + (instr->opcode == aco_opcode::v_fma_f32 && program->chip_class >= GFX10) || + instr->opcode == aco_opcode::v_mad_f16 || + instr->opcode == aco_opcode::v_mad_legacy_f16 || + (instr->opcode == aco_opcode::v_fma_f16 && program->chip_class >= GFX10)) && !instr->usesModifiers()) + op = instr->operands[2]; + + if (op.isTemp() && op.isFirstKillBeforeDef() && def.regClass() == op.regClass()) { + phi_ressources[it->second].emplace_back(op.getTemp()); + temp_to_phi_ressources[op.tempId()] = it->second; } } } @@ -1335,7 +1773,7 @@ void register_allocation(Program *program, std::vector& live_out_per_bl assert(vec.size() > 1); for (unsigned i = 1; i < vec.size(); i++) if (vec[i].id() != vec[0].id()) - affinities[vec[i].id()] = vec[0].id(); + ctx.affinities[vec[i].id()] = vec[0].id(); } /* state of register file after phis */ @@ -1349,9 +1787,11 @@ void register_allocation(Program *program, std::vector& live_out_per_bl ctx.war_hint.reset(); for (Temp t : live) { - Temp renamed = handle_live_in(t, &block); - if (ctx.assignments[renamed.id()].assigned) - register_file.fill(ctx.assignments[renamed.id()].reg, t.size(), renamed.id()); + Temp renamed = handle_live_in(ctx, t, &block); + assignment& var = ctx.assignments[renamed.id()]; + /* due to live-range splits, the live-in might be a phi, now */ + if (var.assigned) + register_file.fill(Definition(renamed.id(), var.reg, var.rc)); } std::vector> instructions; @@ -1399,10 +1839,10 @@ void register_allocation(Program *program, std::vector& live_out_per_bl if (definition.isKill() || definition.isFixed()) continue; - if (affinities.find(definition.tempId()) != affinities.end() && - ctx.assignments[affinities[definition.tempId()]].assigned) { - assert(ctx.assignments[affinities[definition.tempId()]].rc == definition.regClass()); - PhysReg reg = ctx.assignments[affinities[definition.tempId()]].reg; + if (ctx.affinities.find(definition.tempId()) != ctx.affinities.end() && + ctx.assignments[ctx.affinities[definition.tempId()]].assigned) { + assert(ctx.assignments[ctx.affinities[definition.tempId()]].rc == definition.regClass()); + PhysReg reg = ctx.assignments[ctx.affinities[definition.tempId()]].reg; bool try_use_special_reg = reg == scc || reg == exec; if (try_use_special_reg) { for (const Operand& op : phi->operands) { @@ -1450,7 +1890,7 @@ void register_allocation(Program *program, std::vector& live_out_per_bl } } if (!definition.isFixed()) - definition.setFixed(get_reg(ctx, register_file, definition.regClass(), parallelcopy, phi)); + definition.setFixed(get_reg(ctx, register_file, definition.getTemp(), parallelcopy, phi)); /* process parallelcopy */ for (std::pair pc : parallelcopy) { @@ -1470,9 +1910,10 @@ void register_allocation(Program *program, std::vector& live_out_per_bl } if (prev_phi) { /* if so, just update that phi's register */ + register_file.clear(prev_phi->definitions[0]); prev_phi->definitions[0].setFixed(pc.second.physReg()); ctx.assignments[prev_phi->definitions[0].tempId()] = {pc.second.physReg(), pc.second.regClass()}; - register_file.fill(pc.second.physReg(), pc.second.size(), prev_phi->definitions[0].tempId()); + register_file.fill(prev_phi->definitions[0]); continue; } @@ -1483,7 +1924,7 @@ void register_allocation(Program *program, std::vector& live_out_per_bl orig = orig_it->second; else ctx.orig_names[pc.second.tempId()] = orig; - renames[block.index][orig.id()] = pc.second.getTemp(); + ctx.renames[block.index][orig.id()] = pc.second.getTemp(); /* otherwise, this is a live-in and we need to create a new phi * to move it in this block's predecessors */ @@ -1504,7 +1945,7 @@ void register_allocation(Program *program, std::vector& live_out_per_bl /* update phi affinities */ for (const Operand& op : phi->operands) { if (op.isTemp() && op.regClass() == phi->definitions[0].regClass()) - affinities[op.tempId()] = definition.tempId(); + ctx.affinities[op.tempId()] = definition.tempId(); } instructions.emplace_back(std::move(*it)); @@ -1539,7 +1980,7 @@ void register_allocation(Program *program, std::vector& live_out_per_bl if (phi->operands[idx].isTemp() && phi->operands[idx].getTemp().type() == RegType::sgpr && phi->operands[idx].isFirstKillBeforeDef()) { - Temp phi_op = read_variable(phi->operands[idx].getTemp(), block.index); + Temp phi_op = read_variable(ctx, phi->operands[idx].getTemp(), block.index); PhysReg reg = ctx.assignments[phi_op.id()].reg; assert(register_file[reg] == phi_op.id()); register_file[reg] = 0; @@ -1563,86 +2004,27 @@ void register_allocation(Program *program, std::vector& live_out_per_bl continue; /* rename operands */ - operand.setTemp(read_variable(operand.getTemp(), block.index)); - - /* check if the operand is fixed */ - if (operand.isFixed()) { + operand.setTemp(read_variable(ctx, operand.getTemp(), block.index)); + assert(ctx.assignments[operand.tempId()].assigned); - if (operand.physReg() == ctx.assignments[operand.tempId()].reg) { - /* we are fine: the operand is already assigned the correct reg */ - - } else { - /* check if target reg is blocked, and move away the blocking var */ - if (register_file[operand.physReg().reg()]) { - uint32_t blocking_id = register_file[operand.physReg().reg()]; - RegClass rc = ctx.assignments[blocking_id].rc; - Operand pc_op = Operand(Temp{blocking_id, rc}); - pc_op.setFixed(operand.physReg()); - Definition pc_def = Definition(Temp{program->allocateId(), pc_op.regClass()}); - /* find free reg */ - PhysReg reg = get_reg(ctx, register_file, pc_op.regClass(), parallelcopy, instr); - pc_def.setFixed(reg); - ctx.assignments.emplace_back(reg, pc_def.regClass()); - assert(ctx.assignments.size() == ctx.program->peekAllocationId()); - register_file.clear(pc_op); - register_file.fill(pc_def); - parallelcopy.emplace_back(pc_op, pc_def); - - /* handle renames of previous operands */ - for (unsigned j = 0; j < i; j++) { - Operand& op = instr->operands[j]; - if (op.isTemp() && op.tempId() == blocking_id) { - op.setTemp(pc_def.getTemp()); - op.setFixed(reg); - } - } - } - /* move operand to fixed reg and create parallelcopy pair */ - Operand pc_op = operand; - Temp tmp = Temp{program->allocateId(), operand.regClass()}; - Definition pc_def = Definition(tmp); - pc_def.setFixed(operand.physReg()); - pc_op.setFixed(ctx.assignments[operand.tempId()].reg); - operand.setTemp(tmp); - ctx.assignments.emplace_back(pc_def.physReg(), pc_def.regClass()); - assert(ctx.assignments.size() == ctx.program->peekAllocationId()); - operand.setFixed(pc_def.physReg()); - register_file.clear(pc_op); - register_file.fill(pc_def); - parallelcopy.emplace_back(pc_op, pc_def); - } - } else { - assert(ctx.assignments[operand.tempId()].assigned); - PhysReg reg = ctx.assignments[operand.tempId()].reg; - - if (operand_can_use_reg(instr, i, reg)) { - operand.setFixed(ctx.assignments[operand.tempId()].reg); - } else { - Operand pc_op = operand; - pc_op.setFixed(reg); - PhysReg new_reg = get_reg(ctx, register_file, operand.regClass(), parallelcopy, instr); - Definition pc_def = Definition(program->allocateId(), new_reg, pc_op.regClass()); - ctx.assignments.emplace_back(new_reg, pc_def.regClass()); - assert(ctx.assignments.size() == ctx.program->peekAllocationId()); - register_file.clear(pc_op); - register_file.fill(pc_def); - parallelcopy.emplace_back(pc_op, pc_def); - operand.setTemp(pc_def.getTemp()); - operand.setFixed(new_reg); - } + PhysReg reg = ctx.assignments[operand.tempId()].reg; + if (operand_can_use_reg(program->chip_class, instr, i, reg, operand.regClass())) + operand.setFixed(reg); + else + get_reg_for_operand(ctx, register_file, parallelcopy, instr, operand, i); - if (instr->format == Format::EXP || - (instr->isVMEM() && i == 3 && program->chip_class == GFX6) || - (instr->format == Format::DS && static_cast(instr.get())->gds)) { - for (unsigned j = 0; j < operand.size(); j++) - ctx.war_hint.set(operand.physReg().reg() + j); - } + if (instr->format == Format::EXP || + (instr->isVMEM() && i == 3 && ctx.program->chip_class == GFX6) || + (instr->format == Format::DS && static_cast(instr.get())->gds)) { + for (unsigned j = 0; j < operand.size(); j++) + ctx.war_hint.set(operand.physReg().reg() + j); } - std::unordered_map::iterator phi = phi_map.find(operand.getTemp().id()); - if (phi != phi_map.end()) - phi->second.uses.emplace(instr.get()); + std::unordered_map::iterator phi = ctx.phi_map.find(operand.getTemp().id()); + if (phi != ctx.phi_map.end()) + phi->second.uses.emplace(instr.get()); } + /* remove dead vars from register file */ for (const Operand& op : instr->operands) { if (op.isTemp() && op.isFirstKillBeforeDef()) @@ -1650,25 +2032,52 @@ void register_allocation(Program *program, std::vector& live_out_per_bl } /* try to optimize v_mad_f32 -> v_mac_f32 */ - if (instr->opcode == aco_opcode::v_mad_f32 && + if ((instr->opcode == aco_opcode::v_mad_f32 || + (instr->opcode == aco_opcode::v_fma_f32 && program->chip_class >= GFX10) || + instr->opcode == aco_opcode::v_mad_f16 || + instr->opcode == aco_opcode::v_mad_legacy_f16 || + (instr->opcode == aco_opcode::v_fma_f16 && program->chip_class >= GFX10)) && instr->operands[2].isTemp() && instr->operands[2].isKillBeforeDef() && instr->operands[2].getTemp().type() == RegType::vgpr && instr->operands[1].isTemp() && - instr->operands[1].getTemp().type() == RegType::vgpr) { /* TODO: swap src0 and src1 in this case */ - VOP3A_instruction* vop3 = static_cast(instr.get()); - bool can_use_mac = !(vop3->abs[0] || vop3->abs[1] || vop3->abs[2] || - vop3->neg[0] || vop3->neg[1] || vop3->neg[2] || - vop3->clamp || vop3->omod || vop3->opsel); - if (can_use_mac) { + instr->operands[1].getTemp().type() == RegType::vgpr && + !instr->usesModifiers() && + instr->operands[0].physReg().byte() == 0 && + instr->operands[1].physReg().byte() == 0 && + instr->operands[2].physReg().byte() == 0) { + unsigned def_id = instr->definitions[0].tempId(); + auto it = ctx.affinities.find(def_id); + if (it == ctx.affinities.end() || !ctx.assignments[it->second].assigned || + instr->operands[2].physReg() == ctx.assignments[it->second].reg || + register_file.test(ctx.assignments[it->second].reg, instr->operands[2].bytes())) { instr->format = Format::VOP2; - instr->opcode = aco_opcode::v_mac_f32; + switch (instr->opcode) { + case aco_opcode::v_mad_f32: + instr->opcode = aco_opcode::v_mac_f32; + break; + case aco_opcode::v_fma_f32: + instr->opcode = aco_opcode::v_fmac_f32; + break; + case aco_opcode::v_mad_f16: + case aco_opcode::v_mad_legacy_f16: + instr->opcode = aco_opcode::v_mac_f16; + break; + case aco_opcode::v_fma_f16: + instr->opcode = aco_opcode::v_fmac_f16; + break; + default: + break; + } } } /* handle definitions which must have the same register as an operand */ if (instr->opcode == aco_opcode::v_interp_p2_f32 || instr->opcode == aco_opcode::v_mac_f32 || + instr->opcode == aco_opcode::v_fmac_f32 || + instr->opcode == aco_opcode::v_mac_f16 || + instr->opcode == aco_opcode::v_fmac_f16 || instr->opcode == aco_opcode::v_writelane_b32 || instr->opcode == aco_opcode::v_writelane_b32_e64) { instr->definitions[0].setFixed(instr->operands[2].physReg()); @@ -1695,23 +2104,26 @@ void register_allocation(Program *program, std::vector& live_out_per_bl adjust_max_used_regs(ctx, definition.regClass(), definition.physReg()); /* check if the target register is blocked */ - if (register_file[definition.physReg().reg()] != 0) { - /* create parallelcopy pair to move blocking var */ - Temp tmp = {register_file[definition.physReg()], ctx.assignments[register_file[definition.physReg()]].rc}; - Operand pc_op = Operand(tmp); - pc_op.setFixed(ctx.assignments[register_file[definition.physReg().reg()]].reg); - RegClass rc = pc_op.regClass(); - tmp = Temp{program->allocateId(), rc}; - Definition pc_def = Definition(tmp); - - /* re-enable the killed operands, so that we don't move the blocking var there */ + if (register_file.test(definition.physReg(), definition.bytes())) { + /* create parallelcopy pair to move blocking vars */ + std::set> vars = collect_vars(ctx, register_file, definition.physReg(), definition.size()); + + /* re-enable the killed operands, so that we don't move the blocking vars there */ for (const Operand& op : instr->operands) { if (op.isTemp() && op.isFirstKillBeforeDef()) register_file.fill(op); } - /* find a new register for the blocking variable */ - PhysReg reg = get_reg(ctx, register_file, rc, parallelcopy, instr); + ASSERTED bool success = false; + DefInfo info(ctx, instr, definition.regClass(), -1); + success = get_regs_for_copies(ctx, register_file, parallelcopy, + vars, info.lb, info.ub, instr, + definition.physReg(), + definition.physReg() + definition.size() - 1); + assert(success); + + update_renames(ctx, register_file, parallelcopy, instr, false); + /* once again, disable killed operands */ for (const Operand& op : instr->operands) { if (op.isTemp() && op.isFirstKillBeforeDef()) @@ -1721,16 +2133,6 @@ void register_allocation(Program *program, std::vector& live_out_per_bl if (instr->definitions[k].isTemp() && ctx.defs_done.test(k) && !instr->definitions[k].isKill()) register_file.fill(instr->definitions[k]); } - pc_def.setFixed(reg); - - /* finish assignment of parallelcopy */ - ctx.assignments.emplace_back(reg, pc_def.regClass()); - assert(ctx.assignments.size() == ctx.program->peekAllocationId()); - parallelcopy.emplace_back(pc_op, pc_def); - - /* add changes to reg_file */ - register_file.clear(pc_op); - register_file.fill(pc_def); } ctx.defs_done.set(i); @@ -1747,114 +2149,78 @@ void register_allocation(Program *program, std::vector& live_out_per_bl /* handle all other definitions */ for (unsigned i = 0; i < instr->definitions.size(); ++i) { - auto& definition = instr->definitions[i]; + Definition *definition = &instr->definitions[i]; - if (definition.isFixed() || !definition.isTemp()) + if (definition->isFixed() || !definition->isTemp()) continue; /* find free reg */ - if (definition.hasHint() && register_file[definition.physReg().reg()] == 0) - definition.setFixed(definition.physReg()); + if (definition->hasHint() && register_file[definition->physReg().reg()] == 0) + definition->setFixed(definition->physReg()); else if (instr->opcode == aco_opcode::p_split_vector) { PhysReg reg = instr->operands[0].physReg(); - reg.reg_b += i * definition.bytes(); - if (!get_reg_specified(ctx, register_file, definition.regClass(), parallelcopy, instr, reg)) - reg = get_reg(ctx, register_file, definition.regClass(), parallelcopy, instr); - definition.setFixed(reg); - } else if (instr->opcode == aco_opcode::p_wqm) { - PhysReg reg; - if (instr->operands[0].isKillBeforeDef() && instr->operands[0].getTemp().type() == definition.getTemp().type()) { - reg = instr->operands[0].physReg(); - assert(register_file[reg.reg()] == 0); - } else { - reg = get_reg(ctx, register_file, definition.regClass(), parallelcopy, instr); - } - definition.setFixed(reg); + for (unsigned j = 0; j < i; j++) + reg.reg_b += instr->definitions[j].bytes(); + if (get_reg_specified(ctx, register_file, definition->regClass(), parallelcopy, instr, reg)) + definition->setFixed(reg); + } else if (instr->opcode == aco_opcode::p_wqm || instr->opcode == aco_opcode::p_parallelcopy) { + PhysReg reg = instr->operands[i].physReg(); + if (instr->operands[i].isTemp() && + instr->operands[i].getTemp().type() == definition->getTemp().type() && + !register_file.test(reg, definition->bytes())) + definition->setFixed(reg); } else if (instr->opcode == aco_opcode::p_extract_vector) { PhysReg reg; if (instr->operands[0].isKillBeforeDef() && - instr->operands[0].getTemp().type() == definition.getTemp().type()) { + instr->operands[0].getTemp().type() == definition->getTemp().type()) { reg = instr->operands[0].physReg(); - reg.reg_b += definition.bytes() * instr->operands[1].constantValue(); - assert(!register_file.test(reg, definition.bytes())); - } else { - reg = get_reg(ctx, register_file, definition.regClass(), parallelcopy, instr); + reg.reg_b += definition->bytes() * instr->operands[1].constantValue(); + assert(!register_file.test(reg, definition->bytes())); + definition->setFixed(reg); } - definition.setFixed(reg); } else if (instr->opcode == aco_opcode::p_create_vector) { - PhysReg reg = get_reg_create_vector(ctx, register_file, definition.regClass(), + PhysReg reg = get_reg_create_vector(ctx, register_file, definition->getTemp(), parallelcopy, instr); - definition.setFixed(reg); - } else if (affinities.find(definition.tempId()) != affinities.end() && - ctx.assignments[affinities[definition.tempId()]].assigned) { - PhysReg reg = ctx.assignments[affinities[definition.tempId()]].reg; - if (get_reg_specified(ctx, register_file, definition.regClass(), parallelcopy, instr, reg)) - definition.setFixed(reg); - else - definition.setFixed(get_reg(ctx, register_file, definition.regClass(), parallelcopy, instr)); + definition->setFixed(reg); + } - } else if (vectors.find(definition.tempId()) != vectors.end()) { - Instruction* vec = vectors[definition.tempId()]; - unsigned byte_offset = 0; - for (const Operand& op : vec->operands) { - if (op.isTemp() && op.tempId() == definition.tempId()) - break; - else - byte_offset += op.bytes(); - } - unsigned k = 0; - for (const Operand& op : vec->operands) { - if (op.isTemp() && - op.tempId() != definition.tempId() && - op.getTemp().type() == definition.getTemp().type() && - ctx.assignments[op.tempId()].assigned) { - PhysReg reg = ctx.assignments[op.tempId()].reg; - reg.reg_b += (byte_offset - k); - if (get_reg_specified(ctx, register_file, definition.regClass(), parallelcopy, instr, reg)) { - definition.setFixed(reg); - break; - } - } - k += op.bytes(); - } - if (!definition.isFixed()) { - std::pair res = get_reg_vec(ctx, register_file, vec->definitions[0].regClass()); - PhysReg reg = res.first; - if (res.second) { - reg.reg_b += byte_offset; - /* make sure to only use byte offset if the instruction supports it */ - if (vec->definitions[0].regClass().is_subdword() && reg.byte() && !instr_can_access_subdword(instr)) - reg = get_reg(ctx, register_file, definition.regClass(), parallelcopy, instr); - } else { - reg = get_reg(ctx, register_file, definition.regClass(), parallelcopy, instr); - } - definition.setFixed(reg); + if (!definition->isFixed()) { + Temp tmp = definition->getTemp(); + if (definition->regClass().is_subdword() && definition->bytes() < 4) { + PhysReg reg = get_reg(ctx, register_file, tmp, parallelcopy, instr); + bool partial = !(tmp.bytes() <= 4 && reg.byte() == 0 && !register_file.test(reg, 4)); + add_subdword_definition(program, instr, i, reg, partial); + definition = &instr->definitions[i]; /* add_subdword_definition can invalidate the reference */ + } else { + definition->setFixed(get_reg(ctx, register_file, tmp, parallelcopy, instr)); } - } else - definition.setFixed(get_reg(ctx, register_file, definition.regClass(), parallelcopy, instr)); + } - assert(definition.isFixed() && ((definition.getTemp().type() == RegType::vgpr && definition.physReg() >= 256) || - (definition.getTemp().type() != RegType::vgpr && definition.physReg() < 256))); + assert(definition->isFixed() && ((definition->getTemp().type() == RegType::vgpr && definition->physReg() >= 256) || + (definition->getTemp().type() != RegType::vgpr && definition->physReg() < 256))); ctx.defs_done.set(i); /* set live if it has a kill point */ - if (!definition.isKill()) - live.emplace(definition.getTemp()); + if (!definition->isKill()) + live.emplace(definition->getTemp()); - ctx.assignments[definition.tempId()] = {definition.physReg(), definition.regClass()}; - register_file.fill(definition); + ctx.assignments[definition->tempId()] = {definition->physReg(), definition->regClass()}; + register_file.fill(*definition); } handle_pseudo(ctx, register_file, instr.get()); - /* kill definitions and late-kill operands */ + /* kill definitions and late-kill operands and ensure that sub-dword operands can actually be read */ for (const Definition& def : instr->definitions) { if (def.isTemp() && def.isKill()) register_file.clear(def); } - for (const Operand& op : instr->operands) { + for (unsigned i = 0; i < instr->operands.size(); i++) { + const Operand& op = instr->operands[i]; if (op.isTemp() && op.isFirstKill() && op.isLateKill()) register_file.clear(op); + if (op.isTemp() && op.physReg().byte() != 0) + add_subdword_operand(program->chip_class, instr, i, op.physReg().byte(), op.regClass()); } /* emit parallelcopy */ @@ -1886,10 +2252,10 @@ void register_allocation(Program *program, std::vector& live_out_per_bl std::unordered_map::iterator it = ctx.orig_names.find(pc->operands[i].tempId()); Temp orig = it != ctx.orig_names.end() ? it->second : pc->operands[i].getTemp(); ctx.orig_names[pc->definitions[i].tempId()] = orig; - renames[block.index][orig.id()] = pc->definitions[i].getTemp(); + ctx.renames[block.index][orig.id()] = pc->definitions[i].getTemp(); - std::unordered_map::iterator phi = phi_map.find(pc->operands[i].tempId()); - if (phi != phi_map.end()) + std::unordered_map::iterator phi = ctx.phi_map.find(pc->operands[i].tempId()); + if (phi != ctx.phi_map.end()) phi->second.uses.emplace(pc.get()); } @@ -1901,7 +2267,7 @@ void register_allocation(Program *program, std::vector& live_out_per_bl } for (const Operand& op : instr->operands) { if (op.isTemp() && op.isFirstKill()) - register_file.fill(op.physReg(), op.size(), 0xFFFF); + register_file.block(op.physReg(), op.regClass()); } handle_pseudo(ctx, register_file, pc.get()); @@ -1954,12 +2320,11 @@ void register_allocation(Program *program, std::vector& live_out_per_bl register_file.clear(def); for (const Operand& op : instr->operands) { if (op.isTemp() && op.isFirstKill()) - register_file.fill(op.physReg(), op.size(), 0xFFFF); + register_file.block(op.physReg(), op.regClass()); } - RegClass rc = can_sgpr ? s1 : v1; - PhysReg reg = get_reg(ctx, register_file, rc, parallelcopy, instr); - Temp tmp = {program->allocateId(), rc}; - ctx.assignments.emplace_back(reg, rc); + Temp tmp = {program->allocateId(), can_sgpr ? s1 : v1}; + ctx.assignments.emplace_back(); + PhysReg reg = get_reg(ctx, register_file, tmp, parallelcopy, instr); aco_ptr mov; if (can_sgpr) @@ -1993,8 +2358,8 @@ void register_allocation(Program *program, std::vector& live_out_per_bl instr->operands[i] = operand; /* keep phi_map up to date */ if (operand.isTemp()) { - std::unordered_map::iterator phi = phi_map.find(operand.tempId()); - if (phi != phi_map.end()) { + std::unordered_map::iterator phi = ctx.phi_map.find(operand.tempId()); + if (phi != ctx.phi_map.end()) { phi->second.uses.erase(tmp.get()); phi->second.uses.emplace(instr.get()); } @@ -2002,34 +2367,35 @@ void register_allocation(Program *program, std::vector& live_out_per_bl } std::copy(tmp->definitions.begin(), tmp->definitions.end(), instr->definitions.begin()); } + instructions.emplace_back(std::move(*it)); } /* end for Instr */ block.instructions = std::move(instructions); - filled[block.index] = true; + ctx.filled[block.index] = true; for (unsigned succ_idx : block.linear_succs) { Block& succ = program->blocks[succ_idx]; /* seal block if all predecessors are filled */ bool all_filled = true; for (unsigned pred_idx : succ.linear_preds) { - if (!filled[pred_idx]) { + if (!ctx.filled[pred_idx]) { all_filled = false; break; } } if (all_filled) { - sealed[succ_idx] = true; + ctx.sealed[succ_idx] = true; /* finish incomplete phis and check if they became trivial */ - for (Instruction* phi : incomplete_phis[succ_idx]) { + for (Instruction* phi : ctx.incomplete_phis[succ_idx]) { std::vector preds = phi->definitions[0].getTemp().is_linear() ? succ.linear_preds : succ.logical_preds; for (unsigned i = 0; i < phi->operands.size(); i++) { - phi->operands[i].setTemp(read_variable(phi->operands[i].getTemp(), preds[i])); + phi->operands[i].setTemp(read_variable(ctx, phi->operands[i].getTemp(), preds[i])); phi->operands[i].setFixed(ctx.assignments[phi->operands[i].tempId()].reg); } - try_remove_trivial_phi(phi->definitions[0].getTemp()); + try_remove_trivial_phi(ctx, phi->definitions[0].getTemp()); } /* complete the original phi nodes, but no need to check triviality */ for (aco_ptr& instr : succ.instructions) { @@ -2041,10 +2407,10 @@ void register_allocation(Program *program, std::vector& live_out_per_bl auto& operand = instr->operands[i]; if (!operand.isTemp()) continue; - operand.setTemp(read_variable(operand.getTemp(), preds[i])); + operand.setTemp(read_variable(ctx, operand.getTemp(), preds[i])); operand.setFixed(ctx.assignments[operand.tempId()].reg); - std::unordered_map::iterator phi = phi_map.find(operand.getTemp().id()); - if (phi != phi_map.end()) + std::unordered_map::iterator phi = ctx.phi_map.find(operand.getTemp().id()); + if (phi != ctx.phi_map.end()) phi->second.uses.emplace(instr.get()); } }