#include <array>
#include <map>
#include <unordered_map>
-#include <functional>
#include "aco_ir.h"
#include "sid.h"
namespace aco {
namespace {
+struct assignment {
+ PhysReg reg;
+ RegClass rc;
+ uint8_t assigned = 0;
+ assignment() = default;
+ assignment(PhysReg reg, RegClass rc) : reg(reg), rc(rc), assigned(-1) {}
+};
+
+struct phi_info {
+ Instruction* phi;
+ unsigned block_idx;
+ std::set<Instruction*> uses;
+};
+
struct ra_ctx {
std::bitset<512> war_hint;
Program* program;
- std::unordered_map<unsigned, std::pair<PhysReg, RegClass>> assignments;
- std::map<unsigned, Temp> orig_names;
+ std::vector<assignment> assignments;
+ std::vector<std::unordered_map<unsigned, Temp>> renames;
+ std::vector<std::vector<Instruction*>> incomplete_phis;
+ std::vector<bool> filled;
+ std::vector<bool> sealed;
+ std::unordered_map<unsigned, Temp> orig_names;
+ std::unordered_map<unsigned, phi_info> phi_map;
+ std::unordered_map<unsigned, unsigned> affinities;
unsigned max_used_sgpr = 0;
unsigned max_used_vgpr = 0;
std::bitset<64> defs_done; /* see MAX_ARGS in aco_instruction_selection_setup.cpp */
- ra_ctx(Program* program) : program(program) {}
+ ra_ctx(Program* program) : program(program),
+ assignments(program->peekAllocationId()),
+ renames(program->blocks.size()),
+ incomplete_phis(program->blocks.size()),
+ filled(program->blocks.size()),
+ sealed(program->blocks.size()) {}
};
class RegisterFile {
RegisterFile() {regs.fill(0);}
std::array<uint32_t, 512> regs;
+ std::map<uint32_t, std::array<uint32_t, 4>> subdword_regs;
const uint32_t& operator [] (unsigned index) const {
return regs[index];
return res;
}
- bool test(PhysReg start, unsigned size) {
- for (unsigned i = 0; i < size; i++) {
- if (regs[start + i])
+ bool test(PhysReg start, unsigned num_bytes) {
+ for (PhysReg i = start; i.reg_b < start.reg_b + num_bytes; i = PhysReg(i + 1)) {
+ if (regs[i] & 0x0FFFFFFF)
return true;
+ if (regs[i] == 0xF0000000) {
+ assert(subdword_regs.find(i) != subdword_regs.end());
+ for (unsigned j = i.byte(); i * 4 + j < start.reg_b + num_bytes && j < 4; j++) {
+ if (subdword_regs[i][j])
+ return true;
+ }
+ }
}
return false;
}
- void fill(PhysReg start, unsigned size, uint32_t val) {
- for (unsigned i = 0; i < size; i++)
- regs[start + i] = val;
+ void block(PhysReg start, unsigned num_bytes) {
+ if (start.byte() || num_bytes % 4)
+ fill_subdword(start, num_bytes, 0xFFFFFFFF);
+ else
+ fill(start, num_bytes / 4, 0xFFFFFFFF);
}
- void clear(PhysReg start, unsigned size) {
- fill(start, size, 0);
+ bool is_blocked(PhysReg start) {
+ if (regs[start] == 0xFFFFFFFF)
+ return true;
+ if (regs[start] == 0xF0000000) {
+ for (unsigned i = start.byte(); i < 4; i++)
+ if (subdword_regs[start][i] == 0xFFFFFFFF)
+ return true;
+ }
+ return false;
+ }
+
+ void clear(PhysReg start, RegClass rc) {
+ if (rc.is_subdword())
+ fill_subdword(start, rc.bytes(), 0);
+ else
+ fill(start, rc.size(), 0);
}
void fill(Operand op) {
- fill(op.physReg(), op.size(), op.tempId());
+ if (op.regClass().is_subdword())
+ fill_subdword(op.physReg(), op.bytes(), op.tempId());
+ else
+ fill(op.physReg(), op.size(), op.tempId());
}
void clear(Operand op) {
- fill(op.physReg(), op.size(), 0);
+ clear(op.physReg(), op.regClass());
}
void fill(Definition def) {
- fill(def.physReg(), def.size(), def.tempId());
+ if (def.regClass().is_subdword())
+ fill_subdword(def.physReg(), def.bytes(), def.tempId());
+ else
+ fill(def.physReg(), def.size(), def.tempId());
}
void clear(Definition def) {
- fill(def.physReg(), def.size(), 0);
+ clear(def.physReg(), def.regClass());
+ }
+
+private:
+ void fill(PhysReg start, unsigned size, uint32_t val) {
+ for (unsigned i = 0; i < size; i++)
+ regs[start + i] = val;
+ }
+
+ void fill_subdword(PhysReg start, unsigned num_bytes, uint32_t val) {
+ fill(start, DIV_ROUND_UP(num_bytes, 4), 0xF0000000);
+ for (PhysReg i = start; i.reg_b < start.reg_b + num_bytes; i = PhysReg(i + 1)) {
+ /* emplace or get */
+ std::array<uint32_t, 4>& sub = subdword_regs.emplace(i, std::array<uint32_t, 4>{0, 0, 0, 0}).first->second;
+ for (unsigned j = i.byte(); i * 4 + j < start.reg_b + num_bytes && j < 4; j++)
+ sub[j] = val;
+
+ if (sub == std::array<uint32_t, 4>{0, 0, 0, 0}) {
+ subdword_regs.erase(i);
+ regs[i] = 0;
+ }
+ }
}
};
}
// FIXME: if a definition got moved, change the target location and remove the parallelcopy
copy.second.setTemp(Temp(ctx.program->allocateId(), copy.second.regClass()));
- ctx.assignments[copy.second.tempId()] = {copy.second.physReg(), copy.second.regClass()};
- for (unsigned i = copy.second.physReg().reg(); i < copy.second.physReg() + copy.second.size(); i++)
- reg_file[i] = copy.second.tempId();
+ ctx.assignments.emplace_back(copy.second.physReg(), copy.second.regClass());
+ assert(ctx.assignments.size() == ctx.program->peekAllocationId());
+ reg_file.fill(copy.second);
+
/* check if we moved an operand */
for (Operand& op : instr->operands) {
if (!op.isTemp())
}
}
+bool instr_can_access_subdword(aco_ptr<Instruction>& instr)
+{
+ return instr->isSDWA() || instr->format == Format::PSEUDO;
+}
+
std::pair<PhysReg, bool> get_reg_simple(ra_ctx& ctx,
RegisterFile& reg_file,
uint32_t lb, uint32_t ub,
uint32_t size, uint32_t stride,
RegClass rc)
{
+ if (rc.is_subdword()) {
+ for (std::pair<uint32_t, std::array<uint32_t, 4>> entry : reg_file.subdword_regs) {
+ assert(reg_file[entry.first] == 0xF0000000);
+ if (lb > entry.first || entry.first >= ub)
+ continue;
+
+ for (unsigned i = 0; i < 4; i+= stride) {
+ if (entry.second[i] != 0)
+ continue;
+
+ bool reg_found = true;
+ for (unsigned j = 1; reg_found && i + j < 4 && j < rc.bytes(); j++)
+ reg_found &= entry.second[i + j] == 0;
+
+ /* check neighboring reg if needed */
+ reg_found &= (i <= 4 - rc.bytes() || reg_file[entry.first + 1] == 0);
+ if (reg_found) {
+ PhysReg res{entry.first};
+ res.reg_b += i;
+ return {res, true};
+ }
+ }
+ }
+
+ stride = 1; /* stride in full registers */
+ }
+
/* best fit algorithm: find the smallest gap to fit in the variable */
if (stride == 1) {
unsigned best_pos = 0xFFFF;
return {{}, false};
}
+/* collect variables from a register area and clear reg_file */
+std::set<std::pair<unsigned, unsigned>> collect_vars(ra_ctx& ctx, RegisterFile& reg_file,
+ PhysReg reg, unsigned size)
+{
+ std::set<std::pair<unsigned, unsigned>> vars;
+ for (unsigned j = reg; j < reg + size; j++) {
+ if (reg_file.is_blocked(PhysReg{j}))
+ continue;
+ if (reg_file[j] == 0xF0000000) {
+ for (unsigned k = 0; k < 4; k++) {
+ unsigned id = reg_file.subdword_regs[j][k];
+ if (id) {
+ assignment& var = ctx.assignments[id];
+ vars.emplace(var.rc.bytes(), id);
+ reg_file.clear(var.reg, var.rc);
+ if (!reg_file[j])
+ break;
+ }
+ }
+ } else if (reg_file[j] != 0) {
+ unsigned id = reg_file[j];
+ assignment& var = ctx.assignments[id];
+ vars.emplace(var.rc.bytes(), id);
+ reg_file.clear(var.reg, var.rc);
+ }
+ }
+ return vars;
+}
+
bool get_regs_for_copies(ra_ctx& ctx,
RegisterFile& reg_file,
std::vector<std::pair<Operand, Definition>>& parallelcopies,
/* NOTE: variables are also sorted by ID. this only affects a very small number of shaders slightly though. */
for (std::set<std::pair<unsigned, unsigned>>::const_reverse_iterator it = vars.rbegin(); it != vars.rend(); ++it) {
unsigned id = it->second;
- std::pair<PhysReg, RegClass> var = ctx.assignments[id];
- uint32_t size = it->first;
+ assignment& var = ctx.assignments[id];
+ uint32_t size = var.rc.size();
uint32_t stride = 1;
- if (var.second.type() == RegType::sgpr) {
+ if (var.rc.type() == RegType::sgpr) {
if (size == 2)
stride = 2;
if (size > 3)
std::pair<PhysReg, bool> res;
if (is_dead_operand) {
if (instr->opcode == aco_opcode::p_create_vector) {
- for (unsigned i = 0, offset = 0; i < instr->operands.size(); offset += instr->operands[i].size(), i++) {
+ for (unsigned i = 0, offset = 0; i < instr->operands.size(); offset += instr->operands[i].bytes(), i++) {
if (instr->operands[i].isTemp() && instr->operands[i].tempId() == id) {
- for (unsigned j = 0; j < size; j++)
- assert(reg_file[def_reg_lo + offset + j] == 0);
- res = {PhysReg{def_reg_lo + offset}, true};
+ PhysReg reg(def_reg_lo);
+ reg.reg_b += offset;
+ assert(!reg_file.test(reg, var.rc.bytes()));
+ res = {reg, true};
break;
}
}
} else {
- res = get_reg_simple(ctx, reg_file, def_reg_lo, def_reg_hi + 1, size, stride, var.second);
+ res = get_reg_simple(ctx, reg_file, def_reg_lo, def_reg_hi + 1, size, stride, var.rc);
}
} else {
- res = get_reg_simple(ctx, reg_file, lb, def_reg_lo, size, stride, var.second);
+ res = get_reg_simple(ctx, reg_file, lb, def_reg_lo, size, stride, var.rc);
if (!res.second) {
unsigned lb = (def_reg_hi + stride) & ~(stride - 1);
- res = get_reg_simple(ctx, reg_file, lb, ub, size, stride, var.second);
+ res = get_reg_simple(ctx, reg_file, lb, ub, size, stride, var.rc);
}
}
if (res.second) {
/* mark the area as blocked */
- reg_file.fill(res.first, size, 0xFFFFFFFF);
+ reg_file.block(res.first, var.rc.bytes());
+
/* create parallelcopy pair (without definition id) */
- Temp tmp = Temp(id, var.second);
+ Temp tmp = Temp(id, var.rc);
Operand pc_op = Operand(tmp);
- pc_op.setFixed(var.first);
+ pc_op.setFixed(var.reg);
Definition pc_def = Definition(res.first, pc_op.regClass());
parallelcopies.emplace_back(pc_op, pc_def);
continue;
if (reg_file[j] == 0 || reg_file[j] == last_var)
continue;
- /* 0xFFFF signals that this area is already blocked! */
- if (reg_file[j] == 0xFFFFFFFF || k > num_moves) {
+ if (reg_file.is_blocked(PhysReg{j}) || k > num_moves) {
found = false;
break;
}
+ if (reg_file[j] == 0xF0000000) {
+ k += 1;
+ n++;
+ continue;
+ }
/* we cannot split live ranges of linear vgprs */
- if (ctx.assignments[reg_file[j]].second & (1 << 6)) {
+ if (ctx.assignments[reg_file[j]].rc & (1 << 6)) {
found = false;
break;
}
break;
}
}
- if (!is_kill && ctx.assignments[reg_file[j]].second.size() >= size) {
+ if (!is_kill && ctx.assignments[reg_file[j]].rc.size() >= size) {
found = false;
break;
}
- k += ctx.assignments[reg_file[j]].second.size();
+ k += ctx.assignments[reg_file[j]].rc.size();
last_var = reg_file[j];
n++;
if (k > num_moves || (k == num_moves && n <= num_vars)) {
reg_hi = best_pos + size - 1;
/* collect variables and block reg file */
- std::set<std::pair<unsigned, unsigned>> new_vars;
- for (unsigned j = reg_lo; j <= reg_hi; j++) {
- if (reg_file[j] != 0) {
- unsigned size = ctx.assignments[reg_file[j]].second.size();
- unsigned id = reg_file[j];
- new_vars.emplace(size, id);
- reg_file.clear(ctx.assignments[id].first, size);
- }
- }
+ std::set<std::pair<unsigned, unsigned>> new_vars = collect_vars(ctx, reg_file, PhysReg{reg_lo}, size);
/* mark the area as blocked */
- reg_file.fill(PhysReg{reg_lo}, size, 0xFFFFFFFF);
+ reg_file.block(PhysReg{reg_lo}, size * 4);
if (!get_regs_for_copies(ctx, reg_file, parallelcopies, new_vars, lb, ub, instr, def_reg_lo, def_reg_hi))
return false;
- adjust_max_used_regs(ctx, var.second, reg_lo);
+ adjust_max_used_regs(ctx, var.rc, reg_lo);
/* create parallelcopy pair (without definition id) */
- Temp tmp = Temp(id, var.second);
+ Temp tmp = Temp(id, var.rc);
Operand pc_op = Operand(tmp);
- pc_op.setFixed(var.first);
+ pc_op.setFixed(var.reg);
Definition pc_def = Definition(PhysReg{reg_lo}, pc_op.regClass());
parallelcopies.emplace_back(pc_op, pc_def);
}
instr->operands[j].physReg() >= lb &&
instr->operands[j].physReg() < ub) {
assert(instr->operands[j].isFixed());
- assert(reg_file[instr->operands[j].physReg()] == 0);
- reg_file.fill(instr->operands[j].physReg(), instr->operands[j].size(), 0xFFFFFFFF);
+ assert(!reg_file.test(instr->operands[j].physReg(), instr->operands[j].bytes()));
+ reg_file.block(instr->operands[j].physReg(), instr->operands[j].bytes());
killed_ops += instr->operands[j].getTemp().size();
}
}
continue;
/* dead operands effectively reduce the number of estimated moves */
- if (remaining_op_moves && reg_file[j] == 0xFFFFFFFF) {
- k--;
- remaining_op_moves--;
+ if (reg_file.is_blocked(PhysReg{j})) {
+ if (remaining_op_moves) {
+ k--;
+ remaining_op_moves--;
+ }
continue;
}
- if (ctx.assignments[reg_file[j]].second.size() >= size) {
+ if (reg_file[j] == 0xF0000000) {
+ k += 1;
+ n++;
+ continue;
+ }
+
+ if (ctx.assignments[reg_file[j]].rc.size() >= size) {
found = false;
break;
}
-
/* we cannot split live ranges of linear vgprs */
- if (ctx.assignments[reg_file[j]].second & (1 << 6)) {
+ if (ctx.assignments[reg_file[j]].rc & (1 << 6)) {
found = false;
break;
}
- k += ctx.assignments[reg_file[j]].second.size();
+ k += ctx.assignments[reg_file[j]].rc.size();
n++;
last_var = reg_file[j];
}
RegisterFile register_file = reg_file;
/* now, we figured the placement for our definition */
- std::set<std::pair<unsigned, unsigned>> vars;
- for (unsigned j = best_pos; j < best_pos + size; j++) {
- if (reg_file[j] != 0xFFFFFFFF && reg_file[j] != 0)
- vars.emplace(ctx.assignments[reg_file[j]].second.size(), reg_file[j]);
- reg_file[j] = 0;
- }
+ std::set<std::pair<unsigned, unsigned>> vars = collect_vars(ctx, reg_file, PhysReg{best_pos}, size);
if (instr->opcode == aco_opcode::p_create_vector) {
/* move killed operands which aren't yet at the correct position */
instr->operands[i].getTemp().type() == rc.type()) {
if (instr->operands[i].physReg() != best_pos + offset) {
- vars.emplace(instr->operands[i].size(), instr->operands[i].tempId());
+ vars.emplace(instr->operands[i].bytes(), instr->operands[i].tempId());
reg_file.clear(instr->operands[i]);
} else {
reg_file.fill(instr->operands[i]);
parallelcopies.insert(parallelcopies.end(), pc.begin(), pc.end());
/* we set the definition regs == 0. the actual caller is responsible for correct setting */
- reg_file.clear(PhysReg{best_pos}, size);
+ reg_file.clear(PhysReg{best_pos}, rc);
update_renames(ctx, reg_file, parallelcopies, instr);
return {PhysReg{best_pos}, true};
}
+bool get_reg_specified(ra_ctx& ctx,
+ RegisterFile& reg_file,
+ RegClass rc,
+ std::vector<std::pair<Operand, Definition>>& parallelcopies,
+ aco_ptr<Instruction>& instr,
+ PhysReg reg)
+{
+ uint32_t size = rc.size();
+ uint32_t stride = 1;
+ uint32_t lb, ub;
+
+ if (rc.type() == RegType::vgpr) {
+ lb = 256;
+ ub = 256 + ctx.program->max_reg_demand.vgpr;
+ } else {
+ if (size == 2)
+ stride = 2;
+ else if (size >= 4)
+ stride = 4;
+ if (reg % stride != 0)
+ return false;
+ lb = 0;
+ ub = ctx.program->max_reg_demand.sgpr;
+ }
+
+ if (rc.is_subdword() && reg.byte() && !instr_can_access_subdword(instr))
+ return false;
+
+ uint32_t reg_lo = reg.reg();
+ uint32_t reg_hi = reg + (size - 1);
+
+ if (reg_lo < lb || reg_hi >= ub || reg_lo > reg_hi)
+ return false;
+
+ if (reg_file.test(reg, rc.bytes()))
+ return false;
+
+ adjust_max_used_regs(ctx, rc, reg_lo);
+ return true;
+}
+
+std::pair<PhysReg, bool> get_reg_vec(ra_ctx& ctx,
+ RegisterFile& reg_file,
+ RegClass rc)
+{
+ uint32_t size = rc.size();
+ uint32_t stride = 1;
+ uint32_t lb, ub;
+ if (rc.type() == RegType::vgpr) {
+ lb = 256;
+ ub = 256 + ctx.program->max_reg_demand.vgpr;
+ } else {
+ lb = 0;
+ ub = ctx.program->max_reg_demand.sgpr;
+ if (size == 2)
+ stride = 2;
+ else if (size >= 4)
+ stride = 4;
+ }
+ return get_reg_simple(ctx, reg_file, lb, ub, size, stride, rc);
+}
+
PhysReg get_reg(ra_ctx& ctx,
RegisterFile& reg_file,
- RegClass rc,
+ Temp temp,
std::vector<std::pair<Operand, Definition>>& parallelcopies,
aco_ptr<Instruction>& instr)
{
+ RegClass rc = temp.regClass();
uint32_t size = rc.size();
uint32_t stride = 1;
uint32_t lb, ub;
stride = 4;
}
+ if (rc.is_subdword()) {
+ /* stride in bytes */
+ if(!instr_can_access_subdword(instr))
+ stride = 4;
+ else if (rc.bytes() % 4 == 0)
+ stride = 4;
+ else if (rc.bytes() % 2 == 0)
+ stride = 2;
+ }
+
std::pair<PhysReg, bool> res = {{}, false};
/* try to find space without live-range splits */
if (rc.type() == RegType::vgpr && (size == 4 || size == 8))
uint16_t max_addressible_vgpr = ctx.program->vgpr_limit;
if (rc.type() == RegType::vgpr && ctx.program->max_reg_demand.vgpr < max_addressible_vgpr) {
update_vgpr_sgpr_demand(ctx.program, RegisterDemand(ctx.program->max_reg_demand.vgpr + 1, ctx.program->max_reg_demand.sgpr));
- return get_reg(ctx, reg_file, rc, parallelcopies, instr);
+ return get_reg(ctx, reg_file, temp, parallelcopies, instr);
} else if (rc.type() == RegType::sgpr && ctx.program->max_reg_demand.sgpr < max_addressible_sgpr) {
update_vgpr_sgpr_demand(ctx.program, RegisterDemand(ctx.program->max_reg_demand.vgpr, ctx.program->max_reg_demand.sgpr + 1));
- return get_reg(ctx, reg_file, rc, parallelcopies, instr);
+ return get_reg(ctx, reg_file, temp, parallelcopies, instr);
}
//FIXME: if nothing helps, shift-rotate the registers to make space
unreachable("did not find a register");
}
-
-std::pair<PhysReg, bool> get_reg_vec(ra_ctx& ctx,
- RegisterFile& reg_file,
- RegClass rc)
-{
- uint32_t size = rc.size();
- uint32_t stride = 1;
- uint32_t lb, ub;
- if (rc.type() == RegType::vgpr) {
- lb = 256;
- ub = 256 + ctx.program->max_reg_demand.vgpr;
- } else {
- lb = 0;
- ub = ctx.program->max_reg_demand.sgpr;
- if (size == 2)
- stride = 2;
- else if (size >= 4)
- stride = 4;
- }
- return get_reg_simple(ctx, reg_file, lb, ub, size, stride, rc);
-}
-
-
PhysReg get_reg_create_vector(ra_ctx& ctx,
RegisterFile& reg_file,
- RegClass rc,
+ Temp temp,
std::vector<std::pair<Operand, Definition>>& parallelcopies,
aco_ptr<Instruction>& instr)
{
+ RegClass rc = temp.regClass();
/* create_vector instructions have different costs w.r.t. register coalescing */
uint32_t size = rc.size();
+ uint32_t bytes = rc.bytes();
uint32_t stride = 1;
uint32_t lb, ub;
if (rc.type() == RegType::vgpr) {
stride = 4;
}
+ //TODO: improve p_create_vector for sub-dword vectors
+
unsigned best_pos = -1;
unsigned num_moves = 0xFF;
bool best_war_hint = true;
/* test for each operand which definition placement causes the least shuffle instructions */
- for (unsigned i = 0, offset = 0; i < instr->operands.size(); offset += instr->operands[i].size(), i++) {
+ for (unsigned i = 0, offset = 0; i < instr->operands.size(); offset += instr->operands[i].bytes(), i++) {
// TODO: think about, if we can alias live operands on the same register
if (!instr->operands[i].isTemp() || !instr->operands[i].isKillBeforeDef() || instr->operands[i].getTemp().type() != rc.type())
continue;
- if (offset > instr->operands[i].physReg())
+ if (offset > instr->operands[i].physReg().reg_b)
continue;
- unsigned reg_lo = instr->operands[i].physReg() - offset;
+ unsigned reg_lo = instr->operands[i].physReg().reg_b - offset;
+ if (reg_lo % 4)
+ continue;
+ reg_lo /= 4;
unsigned reg_hi = reg_lo + size - 1;
unsigned k = 0;
bool linear_vgpr = false;
for (unsigned j = reg_lo; j <= reg_hi && !linear_vgpr; j++) {
if (reg_file[j] != 0) {
- k++;
- /* we cannot split live ranges of linear vgprs */
- if (ctx.assignments[reg_file[j]].second & (1 << 6))
- linear_vgpr = true;
+ if (reg_file[j] == 0xF0000000) {
+ PhysReg reg;
+ reg.reg_b = j * 4;
+ unsigned bytes_left = bytes - (j - reg_lo) * 4;
+ for (unsigned k = 0; k < MIN2(bytes_left, 4); k++, reg.reg_b++)
+ k += reg_file.test(reg, 1);
+ } else {
+ k += 4;
+ /* we cannot split live ranges of linear vgprs */
+ if (ctx.assignments[reg_file[j]].rc & (1 << 6))
+ linear_vgpr = true;
+ }
}
war_hint |= ctx.war_hint[j];
}
continue;
/* count operands in wrong positions */
- for (unsigned j = 0, offset = 0; j < instr->operands.size(); offset += instr->operands[j].size(), j++) {
+ for (unsigned j = 0, offset = 0; j < instr->operands.size(); offset += instr->operands[j].bytes(), j++) {
if (j == i ||
!instr->operands[j].isTemp() ||
instr->operands[j].getTemp().type() != rc.type())
continue;
- if (instr->operands[j].physReg() != reg_lo + offset)
- k += instr->operands[j].size();
+ if (instr->operands[j].physReg().reg_b != reg_lo * 4 + offset)
+ k += instr->operands[j].bytes();
}
bool aligned = rc == RegClass::v4 && reg_lo % 4 == 0;
if (k > num_moves || (!aligned && k == num_moves))
best_war_hint = war_hint;
}
- if (num_moves >= size)
- return get_reg(ctx, reg_file, rc, parallelcopies, instr);
+ if (num_moves >= bytes)
+ return get_reg(ctx, reg_file, temp, parallelcopies, instr);
/* collect variables to be moved */
- std::set<std::pair<unsigned, unsigned>> vars;
- for (unsigned i = best_pos; i < best_pos + size; i++) {
- if (reg_file[i] != 0)
- vars.emplace(ctx.assignments[reg_file[i]].second.size(), reg_file[i]);
- reg_file[i] = 0;
- }
+ std::set<std::pair<unsigned, unsigned>> vars = collect_vars(ctx, reg_file, PhysReg{best_pos}, size);
/* move killed operands which aren't yet at the correct position */
- for (unsigned i = 0, offset = 0; i < instr->operands.size(); offset += instr->operands[i].size(), i++) {
- if (instr->operands[i].isTemp() && instr->operands[i].isFirstKillBeforeDef() && instr->operands[i].getTemp().type() == rc.type()) {
- if (instr->operands[i].physReg() != best_pos + offset)
- vars.emplace(instr->operands[i].size(), instr->operands[i].tempId());
- else
- reg_file.fill(instr->operands[i]);
+ uint64_t moved_operand_mask = 0;
+ for (unsigned i = 0, offset = 0; i < instr->operands.size(); offset += instr->operands[i].bytes(), i++) {
+ if (instr->operands[i].isTemp() &&
+ instr->operands[i].isFirstKillBeforeDef() &&
+ instr->operands[i].getTemp().type() == rc.type() &&
+ instr->operands[i].physReg().reg_b != best_pos * 4 + offset) {
+ vars.emplace(instr->operands[i].bytes(), instr->operands[i].tempId());
+ moved_operand_mask |= (uint64_t)1 << i;
}
}
update_renames(ctx, reg_file, parallelcopies, instr);
adjust_max_used_regs(ctx, rc, best_pos);
- return PhysReg{best_pos};
-}
-bool get_reg_specified(ra_ctx& ctx,
- RegisterFile& reg_file,
- RegClass rc,
- std::vector<std::pair<Operand, Definition>>& parallelcopies,
- aco_ptr<Instruction>& instr,
- PhysReg reg)
-{
- uint32_t size = rc.size();
- uint32_t stride = 1;
- uint32_t lb, ub;
-
- if (rc.type() == RegType::vgpr) {
- lb = 256;
- ub = 256 + ctx.program->max_reg_demand.vgpr;
- } else {
- if (size == 2)
- stride = 2;
- else if (size >= 4)
- stride = 4;
- if (reg % stride != 0)
- return false;
- lb = 0;
- ub = ctx.program->max_reg_demand.sgpr;
+ while (moved_operand_mask) {
+ unsigned i = u_bit_scan64(&moved_operand_mask);
+ assert(instr->operands[i].isFirstKillBeforeDef());
+ reg_file.clear(instr->operands[i]);
}
- uint32_t reg_lo = reg.reg();
- uint32_t reg_hi = reg + (size - 1);
-
- if (reg_lo < lb || reg_hi >= ub || reg_lo > reg_hi)
- return false;
-
- if (reg_file.test(reg, size))
- return false;
- adjust_max_used_regs(ctx, rc, reg_lo);
- return true;
+ return PhysReg{best_pos};
}
void handle_pseudo(ra_ctx& ctx,
bool operand_can_use_reg(aco_ptr<Instruction>& instr, unsigned idx, PhysReg reg)
{
+ if (instr->operands[idx].isFixed())
+ return instr->operands[idx].physReg() == reg;
+
+ if (!instr_can_access_subdword(instr) && reg.byte())
+ return false;
+
switch (instr->format) {
case Format::SMEM:
return reg != scc &&
}
}
-} /* end namespace */
+void get_reg_for_operand(ra_ctx& ctx, RegisterFile& register_file,
+ std::vector<std::pair<Operand, Definition>>& parallelcopy,
+ aco_ptr<Instruction>& instr, Operand& operand)
+{
+ /* check if the operand is fixed */
+ PhysReg dst;
+ if (operand.isFixed()) {
+ assert(operand.physReg() != ctx.assignments[operand.tempId()].reg);
+
+ /* check if target reg is blocked, and move away the blocking var */
+ if (register_file[operand.physReg().reg()]) {
+ assert(register_file[operand.physReg()] != 0xF0000000);
+ uint32_t blocking_id = register_file[operand.physReg().reg()];
+ RegClass rc = ctx.assignments[blocking_id].rc;
+ Operand pc_op = Operand(Temp{blocking_id, rc});
+ pc_op.setFixed(operand.physReg());
+
+ /* find free reg */
+ PhysReg reg = get_reg(ctx, register_file, pc_op.getTemp(), parallelcopy, instr);
+ Definition pc_def = Definition(PhysReg{reg}, pc_op.regClass());
+ register_file.clear(pc_op);
+ parallelcopy.emplace_back(pc_op, pc_def);
+ }
+ dst = operand.physReg();
+ } else {
+ dst = get_reg(ctx, register_file, operand.getTemp(), parallelcopy, instr);
+ }
-void register_allocation(Program *program, std::vector<std::set<Temp>> live_out_per_block)
-{
- ra_ctx ctx(program);
+ Operand pc_op = operand;
+ pc_op.setFixed(ctx.assignments[operand.tempId()].reg);
+ Definition pc_def = Definition(dst, pc_op.regClass());
+ register_file.clear(pc_op);
+ parallelcopy.emplace_back(pc_op, pc_def);
+ update_renames(ctx, register_file, parallelcopy, instr);
+}
- std::vector<std::unordered_map<unsigned, Temp>> renames(program->blocks.size());
-
- struct phi_info {
- Instruction* phi;
- unsigned block_idx;
- std::set<Instruction*> uses;
- };
-
- bool filled[program->blocks.size()];
- bool sealed[program->blocks.size()];
- memset(filled, 0, sizeof filled);
- memset(sealed, 0, sizeof sealed);
- std::vector<std::vector<Instruction*>> incomplete_phis(program->blocks.size());
- std::map<unsigned, phi_info> phi_map;
- std::map<unsigned, unsigned> affinities;
- std::function<Temp(Temp,unsigned)> read_variable;
- std::function<Temp(Temp,Block*)> handle_live_in;
- std::function<Temp(std::map<unsigned, phi_info>::iterator)> try_remove_trivial_phi;
-
- read_variable = [&](Temp val, unsigned block_idx) -> Temp {
- std::unordered_map<unsigned, Temp>::iterator it = renames[block_idx].find(val.id());
- assert(it != renames[block_idx].end());
+Temp read_variable(ra_ctx& ctx, Temp val, unsigned block_idx)
+{
+ std::unordered_map<unsigned, Temp>::iterator it = ctx.renames[block_idx].find(val.id());
+ if (it == ctx.renames[block_idx].end())
+ return val;
+ else
return it->second;
- };
+}
- handle_live_in = [&](Temp val, Block *block) -> Temp {
- std::vector<unsigned>& preds = val.is_linear() ? block->linear_preds : block->logical_preds;
- if (preds.size() == 0 || val.regClass() == val.regClass().as_linear()) {
- renames[block->index][val.id()] = val;
- return val;
+Temp handle_live_in(ra_ctx& ctx, Temp val, Block* block)
+{
+ std::vector<unsigned>& preds = val.is_linear() ? block->linear_preds : block->logical_preds;
+ if (preds.size() == 0 || val.regClass() == val.regClass().as_linear())
+ return val;
+
+ assert(preds.size() > 0);
+
+ Temp new_val;
+ if (!ctx.sealed[block->index]) {
+ /* consider rename from already processed predecessor */
+ Temp tmp = read_variable(ctx, val, preds[0]);
+
+ /* if the block is not sealed yet, we create an incomplete phi (which might later get removed again) */
+ new_val = Temp{ctx.program->allocateId(), val.regClass()};
+ ctx.assignments.emplace_back();
+ aco_opcode opcode = val.is_linear() ? aco_opcode::p_linear_phi : aco_opcode::p_phi;
+ aco_ptr<Instruction> phi{create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, preds.size(), 1)};
+ phi->definitions[0] = Definition(new_val);
+ for (unsigned i = 0; i < preds.size(); i++)
+ phi->operands[i] = Operand(val);
+ if (tmp.regClass() == new_val.regClass())
+ ctx.affinities[new_val.id()] = tmp.id();
+
+ ctx.phi_map.emplace(new_val.id(), phi_info{phi.get(), block->index});
+ ctx.incomplete_phis[block->index].emplace_back(phi.get());
+ block->instructions.insert(block->instructions.begin(), std::move(phi));
+
+ } else if (preds.size() == 1) {
+ /* if the block has only one predecessor, just look there for the name */
+ new_val = read_variable(ctx, val, preds[0]);
+ } else {
+ /* there are multiple predecessors and the block is sealed */
+ Temp ops[preds.size()];
+
+ /* get the rename from each predecessor and check if they are the same */
+ bool needs_phi = false;
+ for (unsigned i = 0; i < preds.size(); i++) {
+ ops[i] = read_variable(ctx, val, preds[i]);
+ if (i == 0)
+ new_val = ops[i];
+ else
+ needs_phi |= !(new_val == ops[i]);
}
- assert(preds.size() > 0);
-
- Temp new_val;
- if (!sealed[block->index]) {
- /* consider rename from already processed predecessor */
- Temp tmp = read_variable(val, preds[0]);
- /* if the block is not sealed yet, we create an incomplete phi (which might later get removed again) */
- new_val = Temp{program->allocateId(), val.regClass()};
+ if (needs_phi) {
+ /* the variable has been renamed differently in the predecessors: we need to insert a phi */
aco_opcode opcode = val.is_linear() ? aco_opcode::p_linear_phi : aco_opcode::p_phi;
aco_ptr<Instruction> phi{create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, preds.size(), 1)};
+ new_val = Temp{ctx.program->allocateId(), val.regClass()};
phi->definitions[0] = Definition(new_val);
- for (unsigned i = 0; i < preds.size(); i++)
- phi->operands[i] = Operand(val);
- if (tmp.regClass() == new_val.regClass())
- affinities[new_val.id()] = tmp.id();
-
- phi_map.emplace(new_val.id(), phi_info{phi.get(), block->index});
- incomplete_phis[block->index].emplace_back(phi.get());
+ for (unsigned i = 0; i < preds.size(); i++) {
+ phi->operands[i] = Operand(ops[i]);
+ phi->operands[i].setFixed(ctx.assignments[ops[i].id()].reg);
+ if (ops[i].regClass() == new_val.regClass())
+ ctx.affinities[new_val.id()] = ops[i].id();
+ }
+ ctx.assignments.emplace_back();
+ assert(ctx.assignments.size() == ctx.program->peekAllocationId());
+ ctx.phi_map.emplace(new_val.id(), phi_info{phi.get(), block->index});
block->instructions.insert(block->instructions.begin(), std::move(phi));
+ }
+ }
- } else if (preds.size() == 1) {
- /* if the block has only one predecessor, just look there for the name */
- new_val = read_variable(val, preds[0]);
- } else {
- /* there are multiple predecessors and the block is sealed */
- Temp ops[preds.size()];
+ if (new_val != val) {
+ ctx.renames[block->index][val.id()] = new_val;
+ ctx.orig_names[new_val.id()] = val;
+ }
+ return new_val;
+}
- /* we start assuming that the name is the same from all predecessors */
- renames[block->index][val.id()] = val;
- bool needs_phi = false;
+void try_remove_trivial_phi(ra_ctx& ctx, Temp temp)
+{
+ std::unordered_map<unsigned, phi_info>::iterator info = ctx.phi_map.find(temp.id());
- /* get the rename from each predecessor and check if they are the same */
- for (unsigned i = 0; i < preds.size(); i++) {
- ops[i] = read_variable(val, preds[i]);
- if (i == 0)
- new_val = ops[i];
- else
- needs_phi |= !(new_val == ops[i]);
- }
+ if (info == ctx.phi_map.end() || !ctx.sealed[info->second.block_idx])
+ return;
- if (needs_phi) {
- /* the variable has been renamed differently in the predecessors: we need to insert a phi */
- aco_opcode opcode = val.is_linear() ? aco_opcode::p_linear_phi : aco_opcode::p_phi;
- aco_ptr<Instruction> phi{create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, preds.size(), 1)};
- new_val = Temp{program->allocateId(), val.regClass()};
- phi->definitions[0] = Definition(new_val);
- for (unsigned i = 0; i < preds.size(); i++) {
- phi->operands[i] = Operand(ops[i]);
- phi->operands[i].setFixed(ctx.assignments[ops[i].id()].first);
- if (ops[i].regClass() == new_val.regClass())
- affinities[new_val.id()] = ops[i].id();
- }
- phi_map.emplace(new_val.id(), phi_info{phi.get(), block->index});
- block->instructions.insert(block->instructions.begin(), std::move(phi));
- }
+ assert(info->second.block_idx != 0);
+ Instruction* phi = info->second.phi;
+ Temp same = Temp();
+ Definition def = phi->definitions[0];
+
+ /* a phi node is trivial if all operands are the same as the definition of the phi */
+ for (const Operand& op : phi->operands) {
+ const Temp t = op.getTemp();
+ if (t == same || t == def.getTemp()) {
+ assert(t == same || op.physReg() == def.physReg());
+ continue;
}
+ if (same != Temp())
+ return;
- renames[block->index][val.id()] = new_val;
- renames[block->index][new_val.id()] = new_val;
- ctx.orig_names[new_val.id()] = val;
- return new_val;
- };
-
- try_remove_trivial_phi = [&] (std::map<unsigned, phi_info>::iterator info) -> Temp {
- assert(info->second.block_idx != 0);
- Instruction* phi = info->second.phi;
- Temp same = Temp();
-
- Definition def = phi->definitions[0];
- /* a phi node is trivial if all operands are the same as the definition of the phi */
- for (const Operand& op : phi->operands) {
- const Temp t = op.getTemp();
- if (t == same || t == def.getTemp())
+ same = t;
+ }
+ assert(same != Temp() || same == def.getTemp());
+
+ /* reroute all uses to same and remove phi */
+ std::vector<Temp> phi_users;
+ std::unordered_map<unsigned, phi_info>::iterator same_phi_info = ctx.phi_map.find(same.id());
+ for (Instruction* instr : info->second.uses) {
+ assert(phi != instr);
+ /* recursively try to remove trivial phis */
+ if (is_phi(instr)) {
+ /* ignore if the phi was already flagged trivial */
+ if (instr->definitions.empty())
continue;
- if (!(same == Temp()) || !(op.physReg() == def.physReg())) {
- /* phi is not trivial */
- return def.getTemp();
- }
- same = t;
- }
- assert(!(same == Temp() || same == def.getTemp()));
-
- /* reroute all uses to same and remove phi */
- std::vector<std::map<unsigned, phi_info>::iterator> phi_users;
- std::map<unsigned, phi_info>::iterator same_phi_info = phi_map.find(same.id());
- for (Instruction* instr : info->second.uses) {
- assert(phi != instr);
- /* recursively try to remove trivial phis */
- if (is_phi(instr)) {
- /* ignore if the phi was already flagged trivial */
- if (instr->definitions.empty())
- continue;
- std::map<unsigned, phi_info>::iterator it = phi_map.find(instr->definitions[0].tempId());
- if (it != phi_map.end() && it != info)
- phi_users.emplace_back(it);
- }
- for (Operand& op : instr->operands) {
- if (op.isTemp() && op.tempId() == def.tempId()) {
- op.setTemp(same);
- if (same_phi_info != phi_map.end())
- same_phi_info->second.uses.emplace(instr);
- }
+ if (instr->definitions[0].getTemp() != temp)
+ phi_users.emplace_back(instr->definitions[0].getTemp());
+ }
+ for (Operand& op : instr->operands) {
+ if (op.isTemp() && op.tempId() == def.tempId()) {
+ op.setTemp(same);
+ if (same_phi_info != ctx.phi_map.end())
+ same_phi_info->second.uses.emplace(instr);
}
}
+ }
- auto it = ctx.orig_names.find(same.id());
- unsigned orig_var = it != ctx.orig_names.end() ? it->second.id() : same.id();
- for (unsigned i = 0; i < program->blocks.size(); i++) {
- auto it = renames[i].find(orig_var);
- if (it != renames[i].end() && it->second == def.getTemp())
- renames[i][orig_var] = same;
- }
+ auto it = ctx.orig_names.find(same.id());
+ unsigned orig_var = it != ctx.orig_names.end() ? it->second.id() : same.id();
+ for (unsigned i = 0; i < ctx.program->blocks.size(); i++) {
+ auto it = ctx.renames[i].find(orig_var);
+ if (it != ctx.renames[i].end() && it->second == def.getTemp())
+ ctx.renames[i][orig_var] = same;
+ }
- unsigned block_idx = info->second.block_idx;
- phi->definitions.clear(); /* this indicates that the phi can be removed */
- phi_map.erase(info);
- for (auto it : phi_users) {
- if (sealed[it->second.block_idx])
- try_remove_trivial_phi(it);
- }
+ phi->definitions.clear(); /* this indicates that the phi can be removed */
+ ctx.phi_map.erase(info);
+ for (Temp t : phi_users)
+ try_remove_trivial_phi(ctx, t);
- /* due to the removal of other phis, the name might have changed once again! */
- return renames[block_idx][orig_var];
- };
+ return;
+}
- std::map<unsigned, Instruction*> vectors;
+} /* end namespace */
+
+
+void register_allocation(Program *program, std::vector<TempSet>& live_out_per_block)
+{
+ ra_ctx ctx(program);
+
+ std::unordered_map<unsigned, Instruction*> vectors;
std::vector<std::vector<Temp>> phi_ressources;
- std::map<unsigned, unsigned> temp_to_phi_ressources;
+ std::unordered_map<unsigned, unsigned> temp_to_phi_ressources;
for (std::vector<Block>::reverse_iterator it = program->blocks.rbegin(); it != program->blocks.rend(); it++) {
Block& block = *it;
/* first, compute the death points of all live vars within the block */
- std::set<Temp>& live = live_out_per_block[block.index];
+ TempSet& live = live_out_per_block[block.index];
std::vector<aco_ptr<Instruction>>::reverse_iterator rit;
for (rit = block.instructions.rbegin(); rit != block.instructions.rend(); ++rit) {
continue;
live.erase(def.getTemp());
/* mark last-seen phi operand */
- std::map<unsigned, unsigned>::iterator it = temp_to_phi_ressources.find(def.tempId());
+ std::unordered_map<unsigned, unsigned>::iterator it = temp_to_phi_ressources.find(def.tempId());
if (it != temp_to_phi_ressources.end() && def.regClass() == phi_ressources[it->second][0].regClass()) {
phi_ressources[it->second][0] = def.getTemp();
/* try to coalesce phi affinities with parallelcopies */
assert(vec.size() > 1);
for (unsigned i = 1; i < vec.size(); i++)
if (vec[i].id() != vec[0].id())
- affinities[vec[i].id()] = vec[0].id();
+ ctx.affinities[vec[i].id()] = vec[0].id();
}
/* state of register file after phis */
std::vector<std::bitset<128>> sgpr_live_in(program->blocks.size());
for (Block& block : program->blocks) {
- std::set<Temp>& live = live_out_per_block[block.index];
+ TempSet& live = live_out_per_block[block.index];
/* initialize register file */
assert(block.index != 0 || live.empty());
RegisterFile register_file;
ctx.war_hint.reset();
for (Temp t : live) {
- Temp renamed = handle_live_in(t, &block);
- if (ctx.assignments.find(renamed.id()) != ctx.assignments.end())
- register_file.fill(ctx.assignments[renamed.id()].first, t.size(), renamed.id());
+ Temp renamed = handle_live_in(ctx, t, &block);
+ assignment& var = ctx.assignments[renamed.id()];
+ /* due to live-range splits, the live-in might be a phi, now */
+ if (var.assigned)
+ register_file.fill(Definition(renamed.id(), var.reg, var.rc));
}
std::vector<aco_ptr<Instruction>> instructions;
if (definition.isKill()) {
for (Operand& op : phi->operands) {
assert(op.isTemp());
- if (ctx.assignments.find(op.tempId()) == ctx.assignments.end() ||
- ctx.assignments[op.tempId()].first != exec) {
+ if (!ctx.assignments[op.tempId()].assigned ||
+ ctx.assignments[op.tempId()].reg != exec) {
definition.setKill(false);
break;
}
continue;
assert(definition.physReg() == exec);
- assert(!register_file.test(definition.physReg(), definition.size()));
+ assert(!register_file.test(definition.physReg(), definition.bytes()));
register_file.fill(definition);
ctx.assignments[definition.tempId()] = {definition.physReg(), definition.regClass()};
}
if (definition.isKill() || definition.isFixed())
continue;
- if (affinities.find(definition.tempId()) != affinities.end() &&
- ctx.assignments.find(affinities[definition.tempId()]) != ctx.assignments.end()) {
- assert(ctx.assignments[affinities[definition.tempId()]].second == definition.regClass());
- PhysReg reg = ctx.assignments[affinities[definition.tempId()]].first;
+ if (ctx.affinities.find(definition.tempId()) != ctx.affinities.end() &&
+ ctx.assignments[ctx.affinities[definition.tempId()]].assigned) {
+ assert(ctx.assignments[ctx.affinities[definition.tempId()]].rc == definition.regClass());
+ PhysReg reg = ctx.assignments[ctx.affinities[definition.tempId()]].reg;
bool try_use_special_reg = reg == scc || reg == exec;
if (try_use_special_reg) {
for (const Operand& op : phi->operands) {
- if (!op.isTemp() ||
- ctx.assignments.find(op.tempId()) == ctx.assignments.end() ||
- !(ctx.assignments[op.tempId()].first == reg)) {
+ if (!(op.isTemp() && ctx.assignments[op.tempId()].assigned &&
+ ctx.assignments[op.tempId()].reg == reg)) {
try_use_special_reg = false;
break;
}
continue;
}
/* only assign if register is still free */
- if (!register_file.test(reg, definition.size())) {
+ if (!register_file.test(reg, definition.bytes())) {
definition.setFixed(reg);
register_file.fill(definition);
ctx.assignments[definition.tempId()] = {definition.physReg(), definition.regClass()};
if (definition.isKill())
continue;
- renames[block.index][definition.tempId()] = definition.getTemp();
-
if (!definition.isFixed()) {
std::vector<std::pair<Operand, Definition>> parallelcopy;
/* try to find a register that is used by at least one operand */
for (const Operand& op : phi->operands) {
- if (!op.isTemp() ||
- ctx.assignments.find(op.tempId()) == ctx.assignments.end())
+ if (!(op.isTemp() && ctx.assignments[op.tempId()].assigned))
continue;
- PhysReg reg = ctx.assignments[op.tempId()].first;
+ PhysReg reg = ctx.assignments[op.tempId()].reg;
/* we tried this already on the previous loop */
if (reg == scc || reg == exec)
continue;
}
}
if (!definition.isFixed())
- definition.setFixed(get_reg(ctx, register_file, definition.regClass(), parallelcopy, phi));
+ definition.setFixed(get_reg(ctx, register_file, definition.getTemp(), parallelcopy, phi));
/* process parallelcopy */
for (std::pair<Operand, Definition> pc : parallelcopy) {
}
if (prev_phi) {
/* if so, just update that phi's register */
+ register_file.clear(prev_phi->definitions[0]);
prev_phi->definitions[0].setFixed(pc.second.physReg());
ctx.assignments[prev_phi->definitions[0].tempId()] = {pc.second.physReg(), pc.second.regClass()};
- register_file.fill(pc.second.physReg(), pc.second.size(), prev_phi->definitions[0].tempId());
+ register_file.fill(prev_phi->definitions[0]);
continue;
}
/* rename */
- std::map<unsigned, Temp>::iterator orig_it = ctx.orig_names.find(pc.first.tempId());
+ std::unordered_map<unsigned, Temp>::iterator orig_it = ctx.orig_names.find(pc.first.tempId());
Temp orig = pc.first.getTemp();
if (orig_it != ctx.orig_names.end())
orig = orig_it->second;
else
ctx.orig_names[pc.second.tempId()] = orig;
- renames[block.index][orig.id()] = pc.second.getTemp();
- renames[block.index][pc.second.tempId()] = pc.second.getTemp();
+ ctx.renames[block.index][orig.id()] = pc.second.getTemp();
/* otherwise, this is a live-in and we need to create a new phi
* to move it in this block's predecessors */
/* update phi affinities */
for (const Operand& op : phi->operands) {
if (op.isTemp() && op.regClass() == phi->definitions[0].regClass())
- affinities[op.tempId()] = definition.tempId();
+ ctx.affinities[op.tempId()] = definition.tempId();
}
instructions.emplace_back(std::move(*it));
if (phi->operands[idx].isTemp() &&
phi->operands[idx].getTemp().type() == RegType::sgpr &&
phi->operands[idx].isFirstKillBeforeDef()) {
- Temp phi_op = read_variable(phi->operands[idx].getTemp(), block.index);
- PhysReg reg = ctx.assignments[phi_op.id()].first;
+ Temp phi_op = read_variable(ctx, phi->operands[idx].getTemp(), block.index);
+ PhysReg reg = ctx.assignments[phi_op.id()].reg;
assert(register_file[reg] == phi_op.id());
register_file[reg] = 0;
}
continue;
/* rename operands */
- operand.setTemp(read_variable(operand.getTemp(), block.index));
-
- /* check if the operand is fixed */
- if (operand.isFixed()) {
-
- if (operand.physReg() == ctx.assignments[operand.tempId()].first) {
- /* we are fine: the operand is already assigned the correct reg */
-
- } else {
- /* check if target reg is blocked, and move away the blocking var */
- if (register_file[operand.physReg().reg()]) {
- uint32_t blocking_id = register_file[operand.physReg().reg()];
- RegClass rc = ctx.assignments[blocking_id].second;
- Operand pc_op = Operand(Temp{blocking_id, rc});
- pc_op.setFixed(operand.physReg());
- Definition pc_def = Definition(Temp{program->allocateId(), pc_op.regClass()});
- /* find free reg */
- PhysReg reg = get_reg(ctx, register_file, pc_op.regClass(), parallelcopy, instr);
- pc_def.setFixed(reg);
- ctx.assignments[pc_def.tempId()] = {reg, pc_def.regClass()};
- register_file.clear(pc_op);
- register_file.fill(pc_def);
- parallelcopy.emplace_back(pc_op, pc_def);
-
- /* handle renames of previous operands */
- for (unsigned j = 0; j < i; j++) {
- Operand& op = instr->operands[j];
- if (op.isTemp() && op.tempId() == blocking_id) {
- op.setTemp(pc_def.getTemp());
- op.setFixed(reg);
- }
- }
- }
- /* move operand to fixed reg and create parallelcopy pair */
- Operand pc_op = operand;
- Temp tmp = Temp{program->allocateId(), operand.regClass()};
- Definition pc_def = Definition(tmp);
- pc_def.setFixed(operand.physReg());
- pc_op.setFixed(ctx.assignments[operand.tempId()].first);
- operand.setTemp(tmp);
- ctx.assignments[tmp.id()] = {pc_def.physReg(), pc_def.regClass()};
- operand.setFixed(pc_def.physReg());
- register_file.clear(pc_op);
- register_file.fill(pc_def);
- parallelcopy.emplace_back(pc_op, pc_def);
- }
- } else {
- assert(ctx.assignments.find(operand.tempId()) != ctx.assignments.end());
- PhysReg reg = ctx.assignments[operand.tempId()].first;
+ operand.setTemp(read_variable(ctx, operand.getTemp(), block.index));
+ assert(ctx.assignments[operand.tempId()].assigned);
- if (operand_can_use_reg(instr, i, reg)) {
- operand.setFixed(ctx.assignments[operand.tempId()].first);
- } else {
- Operand pc_op = operand;
- pc_op.setFixed(reg);
- PhysReg new_reg = get_reg(ctx, register_file, operand.regClass(), parallelcopy, instr);
- Definition pc_def = Definition(program->allocateId(), new_reg, pc_op.regClass());
- ctx.assignments[pc_def.tempId()] = {new_reg, pc_def.regClass()};
- register_file.clear(pc_op);
- register_file.fill(pc_def);
- parallelcopy.emplace_back(pc_op, pc_def);
- operand.setTemp(pc_def.getTemp());
- operand.setFixed(new_reg);
- }
+ PhysReg reg = ctx.assignments[operand.tempId()].reg;
+ if (operand_can_use_reg(instr, i, reg))
+ operand.setFixed(reg);
+ else
+ get_reg_for_operand(ctx, register_file, parallelcopy, instr, operand);
- if (instr->format == Format::EXP ||
- (instr->isVMEM() && i == 3 && program->chip_class == GFX6) ||
- (instr->format == Format::DS && static_cast<DS_instruction*>(instr.get())->gds)) {
- for (unsigned j = 0; j < operand.size(); j++)
- ctx.war_hint.set(operand.physReg().reg() + j);
- }
+ if (instr->format == Format::EXP ||
+ (instr->isVMEM() && i == 3 && ctx.program->chip_class == GFX6) ||
+ (instr->format == Format::DS && static_cast<DS_instruction*>(instr.get())->gds)) {
+ for (unsigned j = 0; j < operand.size(); j++)
+ ctx.war_hint.set(operand.physReg().reg() + j);
}
- std::map<unsigned, phi_info>::iterator phi = phi_map.find(operand.getTemp().id());
- if (phi != phi_map.end())
- phi->second.uses.emplace(instr.get());
+ std::unordered_map<unsigned, phi_info>::iterator phi = ctx.phi_map.find(operand.getTemp().id());
+ if (phi != ctx.phi_map.end())
+ phi->second.uses.emplace(instr.get());
}
+
/* remove dead vars from register file */
for (const Operand& op : instr->operands) {
if (op.isTemp() && op.isFirstKillBeforeDef())
/* check if the target register is blocked */
if (register_file[definition.physReg().reg()] != 0) {
/* create parallelcopy pair to move blocking var */
- Temp tmp = {register_file[definition.physReg()], ctx.assignments[register_file[definition.physReg()]].second};
+ Temp tmp = {register_file[definition.physReg()], ctx.assignments[register_file[definition.physReg()]].rc};
Operand pc_op = Operand(tmp);
- pc_op.setFixed(ctx.assignments[register_file[definition.physReg().reg()]].first);
+ pc_op.setFixed(ctx.assignments[register_file[definition.physReg().reg()]].reg);
RegClass rc = pc_op.regClass();
tmp = Temp{program->allocateId(), rc};
Definition pc_def = Definition(tmp);
/* re-enable the killed operands, so that we don't move the blocking var there */
for (const Operand& op : instr->operands) {
if (op.isTemp() && op.isFirstKillBeforeDef())
- register_file.fill(op.physReg(), op.size(), 0xFFFF);
+ register_file.fill(op);
}
/* find a new register for the blocking variable */
- PhysReg reg = get_reg(ctx, register_file, rc, parallelcopy, instr);
+ PhysReg reg = get_reg(ctx, register_file, pc_op.getTemp(), parallelcopy, instr);
/* once again, disable killed operands */
for (const Operand& op : instr->operands) {
if (op.isTemp() && op.isFirstKillBeforeDef())
pc_def.setFixed(reg);
/* finish assignment of parallelcopy */
- ctx.assignments[pc_def.tempId()] = {reg, pc_def.regClass()};
+ ctx.assignments.emplace_back(reg, pc_def.regClass());
+ assert(ctx.assignments.size() == ctx.program->peekAllocationId());
parallelcopy.emplace_back(pc_op, pc_def);
/* add changes to reg_file */
live.emplace(definition.getTemp());
ctx.assignments[definition.tempId()] = {definition.physReg(), definition.regClass()};
- renames[block.index][definition.tempId()] = definition.getTemp();
register_file.fill(definition);
}
if (definition.hasHint() && register_file[definition.physReg().reg()] == 0)
definition.setFixed(definition.physReg());
else if (instr->opcode == aco_opcode::p_split_vector) {
- PhysReg reg = PhysReg{instr->operands[0].physReg() + i * definition.size()};
+ PhysReg reg = instr->operands[0].physReg();
+ reg.reg_b += i * definition.bytes();
if (!get_reg_specified(ctx, register_file, definition.regClass(), parallelcopy, instr, reg))
- reg = get_reg(ctx, register_file, definition.regClass(), parallelcopy, instr);
+ reg = get_reg(ctx, register_file, definition.getTemp(), parallelcopy, instr);
definition.setFixed(reg);
} else if (instr->opcode == aco_opcode::p_wqm) {
PhysReg reg;
reg = instr->operands[0].physReg();
assert(register_file[reg.reg()] == 0);
} else {
- reg = get_reg(ctx, register_file, definition.regClass(), parallelcopy, instr);
+ reg = get_reg(ctx, register_file, definition.getTemp(), parallelcopy, instr);
}
definition.setFixed(reg);
} else if (instr->opcode == aco_opcode::p_extract_vector) {
if (instr->operands[0].isKillBeforeDef() &&
instr->operands[0].getTemp().type() == definition.getTemp().type()) {
reg = instr->operands[0].physReg();
- reg = PhysReg(reg.reg() + definition.size() * instr->operands[1].constantValue());
- assert(register_file[reg.reg()] == 0);
+ reg.reg_b += definition.bytes() * instr->operands[1].constantValue();
+ assert(!register_file.test(reg, definition.bytes()));
} else {
- reg = get_reg(ctx, register_file, definition.regClass(), parallelcopy, instr);
+ reg = get_reg(ctx, register_file, definition.getTemp(), parallelcopy, instr);
}
definition.setFixed(reg);
} else if (instr->opcode == aco_opcode::p_create_vector) {
- PhysReg reg = get_reg_create_vector(ctx, register_file, definition.regClass(),
+ PhysReg reg = get_reg_create_vector(ctx, register_file, definition.getTemp(),
parallelcopy, instr);
definition.setFixed(reg);
- } else if (affinities.find(definition.tempId()) != affinities.end() &&
- ctx.assignments.find(affinities[definition.tempId()]) != ctx.assignments.end()) {
- PhysReg reg = ctx.assignments[affinities[definition.tempId()]].first;
+ } else if (ctx.affinities.find(definition.tempId()) != ctx.affinities.end() &&
+ ctx.assignments[ctx.affinities[definition.tempId()]].assigned) {
+ PhysReg reg = ctx.assignments[ctx.affinities[definition.tempId()]].reg;
if (get_reg_specified(ctx, register_file, definition.regClass(), parallelcopy, instr, reg))
definition.setFixed(reg);
else
- definition.setFixed(get_reg(ctx, register_file, definition.regClass(), parallelcopy, instr));
+ definition.setFixed(get_reg(ctx, register_file, definition.getTemp(), parallelcopy, instr));
} else if (vectors.find(definition.tempId()) != vectors.end()) {
Instruction* vec = vectors[definition.tempId()];
- unsigned offset = 0;
+ unsigned byte_offset = 0;
for (const Operand& op : vec->operands) {
if (op.isTemp() && op.tempId() == definition.tempId())
break;
else
- offset += op.size();
+ byte_offset += op.bytes();
}
unsigned k = 0;
for (const Operand& op : vec->operands) {
if (op.isTemp() &&
op.tempId() != definition.tempId() &&
op.getTemp().type() == definition.getTemp().type() &&
- ctx.assignments.find(op.tempId()) != ctx.assignments.end()) {
- PhysReg reg = ctx.assignments[op.tempId()].first;
- reg = PhysReg(reg.reg() - k + offset);
+ ctx.assignments[op.tempId()].assigned) {
+ PhysReg reg = ctx.assignments[op.tempId()].reg;
+ reg.reg_b += (byte_offset - k);
if (get_reg_specified(ctx, register_file, definition.regClass(), parallelcopy, instr, reg)) {
definition.setFixed(reg);
break;
}
}
- k += op.size();
+ k += op.bytes();
}
if (!definition.isFixed()) {
std::pair<PhysReg, bool> res = get_reg_vec(ctx, register_file, vec->definitions[0].regClass());
PhysReg reg = res.first;
if (res.second) {
- reg = PhysReg(reg.reg() + offset);
+ reg.reg_b += byte_offset;
+ /* make sure to only use byte offset if the instruction supports it */
+ if (vec->definitions[0].regClass().is_subdword() && reg.byte() && !instr_can_access_subdword(instr))
+ reg = get_reg(ctx, register_file, definition.getTemp(), parallelcopy, instr);
} else {
- reg = get_reg(ctx, register_file, definition.regClass(), parallelcopy, instr);
+ reg = get_reg(ctx, register_file, definition.getTemp(), parallelcopy, instr);
}
definition.setFixed(reg);
}
} else
- definition.setFixed(get_reg(ctx, register_file, definition.regClass(), parallelcopy, instr));
+ definition.setFixed(get_reg(ctx, register_file, definition.getTemp(), parallelcopy, instr));
assert(definition.isFixed() && ((definition.getTemp().type() == RegType::vgpr && definition.physReg() >= 256) ||
(definition.getTemp().type() != RegType::vgpr && definition.physReg() < 256)));
live.emplace(definition.getTemp());
ctx.assignments[definition.tempId()] = {definition.physReg(), definition.regClass()};
- renames[block.index][definition.tempId()] = definition.getTemp();
register_file.fill(definition);
}
assert(pc->operands[i].size() == pc->definitions[i].size());
/* it might happen that the operand is already renamed. we have to restore the original name. */
- std::map<unsigned, Temp>::iterator it = ctx.orig_names.find(pc->operands[i].tempId());
+ std::unordered_map<unsigned, Temp>::iterator it = ctx.orig_names.find(pc->operands[i].tempId());
Temp orig = it != ctx.orig_names.end() ? it->second : pc->operands[i].getTemp();
ctx.orig_names[pc->definitions[i].tempId()] = orig;
- renames[block.index][orig.id()] = pc->definitions[i].getTemp();
- renames[block.index][pc->definitions[i].tempId()] = pc->definitions[i].getTemp();
+ ctx.renames[block.index][orig.id()] = pc->definitions[i].getTemp();
- std::map<unsigned, phi_info>::iterator phi = phi_map.find(pc->operands[i].tempId());
- if (phi != phi_map.end())
+ std::unordered_map<unsigned, phi_info>::iterator phi = ctx.phi_map.find(pc->operands[i].tempId());
+ if (phi != ctx.phi_map.end())
phi->second.uses.emplace(pc.get());
}
}
for (const Operand& op : instr->operands) {
if (op.isTemp() && op.isFirstKill())
- register_file.fill(op.physReg(), op.size(), 0xFFFF);
+ register_file.block(op.physReg(), op.bytes());
}
handle_pseudo(ctx, register_file, pc.get());
break;
}
}
+ /* disable definitions and re-enable operands */
+ for (const Definition& def : instr->definitions)
+ register_file.clear(def);
+ for (const Operand& op : instr->operands) {
+ if (op.isTemp() && op.isFirstKill())
+ register_file.block(op.physReg(), op.bytes());
+ }
+ Temp tmp = {program->allocateId(), can_sgpr ? s1 : v1};
+ ctx.assignments.emplace_back();
+ PhysReg reg = get_reg(ctx, register_file, tmp, parallelcopy, instr);
+
aco_ptr<Instruction> mov;
if (can_sgpr)
mov.reset(create_instruction<SOP1_instruction>(aco_opcode::s_mov_b32, Format::SOP1, 1, 1));
else
mov.reset(create_instruction<VOP1_instruction>(aco_opcode::v_mov_b32, Format::VOP1, 1, 1));
mov->operands[0] = instr->operands[0];
- Temp tmp = {program->allocateId(), can_sgpr ? s1 : v1};
mov->definitions[0] = Definition(tmp);
- /* disable definitions and re-enable operands */
- for (const Definition& def : instr->definitions)
- register_file.clear(def);
- for (const Operand& op : instr->operands) {
- if (op.isTemp() && op.isFirstKill())
- register_file.fill(op.physReg(), op.size(), 0xFFFF);
- }
- mov->definitions[0].setFixed(get_reg(ctx, register_file, tmp.regClass(), parallelcopy, mov));
+ mov->definitions[0].setFixed(reg);
+
instr->operands[0] = Operand(tmp);
- instr->operands[0].setFixed(mov->definitions[0].physReg());
+ instr->operands[0].setFixed(reg);
instructions.emplace_back(std::move(mov));
/* re-enable live vars */
for (const Operand& op : instr->operands) {
instr->operands[i] = operand;
/* keep phi_map up to date */
if (operand.isTemp()) {
- std::map<unsigned, phi_info>::iterator phi = phi_map.find(operand.tempId());
- if (phi != phi_map.end()) {
+ std::unordered_map<unsigned, phi_info>::iterator phi = ctx.phi_map.find(operand.tempId());
+ if (phi != ctx.phi_map.end()) {
phi->second.uses.erase(tmp.get());
phi->second.uses.emplace(instr.get());
}
block.instructions = std::move(instructions);
- filled[block.index] = true;
+ ctx.filled[block.index] = true;
for (unsigned succ_idx : block.linear_succs) {
Block& succ = program->blocks[succ_idx];
/* seal block if all predecessors are filled */
bool all_filled = true;
for (unsigned pred_idx : succ.linear_preds) {
- if (!filled[pred_idx]) {
+ if (!ctx.filled[pred_idx]) {
all_filled = false;
break;
}
}
if (all_filled) {
+ ctx.sealed[succ_idx] = true;
+
/* finish incomplete phis and check if they became trivial */
- for (Instruction* phi : incomplete_phis[succ_idx]) {
+ for (Instruction* phi : ctx.incomplete_phis[succ_idx]) {
std::vector<unsigned> preds = phi->definitions[0].getTemp().is_linear() ? succ.linear_preds : succ.logical_preds;
for (unsigned i = 0; i < phi->operands.size(); i++) {
- phi->operands[i].setTemp(read_variable(phi->operands[i].getTemp(), preds[i]));
- phi->operands[i].setFixed(ctx.assignments[phi->operands[i].tempId()].first);
+ phi->operands[i].setTemp(read_variable(ctx, phi->operands[i].getTemp(), preds[i]));
+ phi->operands[i].setFixed(ctx.assignments[phi->operands[i].tempId()].reg);
}
- try_remove_trivial_phi(phi_map.find(phi->definitions[0].tempId()));
+ try_remove_trivial_phi(ctx, phi->definitions[0].getTemp());
}
/* complete the original phi nodes, but no need to check triviality */
for (aco_ptr<Instruction>& instr : succ.instructions) {
auto& operand = instr->operands[i];
if (!operand.isTemp())
continue;
- operand.setTemp(read_variable(operand.getTemp(), preds[i]));
- operand.setFixed(ctx.assignments[operand.tempId()].first);
- std::map<unsigned, phi_info>::iterator phi = phi_map.find(operand.getTemp().id());
- if (phi != phi_map.end())
+ operand.setTemp(read_variable(ctx, operand.getTemp(), preds[i]));
+ operand.setFixed(ctx.assignments[operand.tempId()].reg);
+ std::unordered_map<unsigned, phi_info>::iterator phi = ctx.phi_map.find(operand.getTemp().id());
+ if (phi != ctx.phi_map.end())
phi->second.uses.emplace(instr.get());
}
}
- sealed[succ_idx] = true;
}
}
} /* end for BB */