#include "aco_ir.h"
#include "aco_builder.h"
+#include <algorithm>
namespace aco {
while (true) {
auto pos = state->latest.find(block_idx);
if (pos != state->latest.end())
- return Operand({pos->second, s2});
+ return Operand(Temp(pos->second, program->lane_mask));
Block& block = program->blocks[block_idx];
size_t pred = block.linear_preds.size();
if (pred == 0) {
- return Operand(s2);
+ return Operand(program->lane_mask);
} else if (pred == 1) {
block_idx = block.linear_preds[0];
continue;
state->phis[phi->operands[i].tempId()][(phi_use){&block, res}] |= (uint64_t)1 << i;
}
}
- phi->definitions[0] = Definition(Temp{res, s2});
+ phi->definitions[0] = Definition(Temp{res, program->lane_mask});
block.instructions.emplace(block.instructions.begin(), std::move(phi));
- return Operand({res, s2});
+ return Operand(Temp(res, program->lane_mask));
}
}
}
update_phi(program, state, phi.first.block, phi.first.phi_def, phi.second);
}
- return {id, s2};
-}
-
-void insert_before_branch(Block *block, aco_ptr<Instruction> instr)
-{
- int end = block->instructions.size() - 1;
- if (block->instructions[end]->format == Format::PSEUDO_BRANCH)
- block->instructions.emplace(std::prev(block->instructions.end()), std::move(instr));
- else
- block->instructions.emplace_back(std::move(instr));
+ return {id, program->lane_mask};
}
void insert_before_logical_end(Block *block, aco_ptr<Instruction> instr)
{
- for (int i = block->instructions.size() - 1; i >= 0; --i) {
- if (block->instructions[i]->opcode == aco_opcode::p_logical_end) {
- block->instructions.emplace(std::next(block->instructions.begin(), i), std::move(instr));
- return;
- }
+ auto IsLogicalEnd = [] (const aco_ptr<Instruction>& instr) -> bool {
+ return instr->opcode == aco_opcode::p_logical_end;
+ };
+ auto it = std::find_if(block->instructions.crbegin(), block->instructions.crend(), IsLogicalEnd);
+
+ if (it == block->instructions.crend()) {
+ assert(block->instructions.back()->format == Format::PSEUDO_BRANCH);
+ block->instructions.insert(std::prev(block->instructions.end()), std::move(instr));
}
- insert_before_branch(block, std::move(instr));
+ else
+ block->instructions.insert(std::prev(it.base()), std::move(instr));
}
-aco_ptr<Instruction> lower_divergent_bool_phi(Program *program, Block *block, aco_ptr<Instruction>& phi)
+void lower_divergent_bool_phi(Program *program, Block *block, aco_ptr<Instruction>& phi)
{
Builder bld(program);
ssa_state state;
+ state.latest[block->index] = phi->definitions[0].tempId();
for (unsigned i = 0; i < phi->operands.size(); i++) {
Block *pred = &program->blocks[block->logical_preds[i]];
assert(phi->operands[i].isTemp());
Temp phi_src = phi->operands[i].getTemp();
- if (phi_src.regClass() == s1) {
- Temp new_phi_src = bld.tmp(s2);
- insert_before_logical_end(pred,
- bld.sop2(aco_opcode::s_cselect_b64, Definition(new_phi_src),
- Operand((uint32_t)-1), Operand(0u), bld.scc(phi_src)).get_ptr());
- phi_src = new_phi_src;
- }
- assert(phi_src.regClass() == s2);
+ assert(phi_src.regClass() == bld.lm);
Operand cur = get_ssa(program, pred->index, &state);
+ assert(cur.regClass() == bld.lm);
Temp new_cur = write_ssa(program, pred, &state, cur.isTemp() ? cur.tempId() : 0);
+ assert(new_cur.regClass() == bld.lm);
if (cur.isUndefined()) {
insert_before_logical_end(pred, bld.sop1(aco_opcode::s_mov_b64, Definition(new_cur), phi_src).get_ptr());
} else {
- Temp tmp1 = bld.tmp(s2), tmp2 = bld.tmp(s2);
+ Temp tmp1 = bld.tmp(bld.lm), tmp2 = bld.tmp(bld.lm);
insert_before_logical_end(pred,
- bld.sop2(aco_opcode::s_andn2_b64, Definition(tmp1), bld.def(s1, scc),
- cur, Operand(exec, s2)).get_ptr());
+ bld.sop2(Builder::s_andn2, Definition(tmp1), bld.def(s1, scc),
+ cur, Operand(exec, bld.lm)).get_ptr());
insert_before_logical_end(pred,
- bld.sop2(aco_opcode::s_and_b64, Definition(tmp2), bld.def(s1, scc),
- phi_src, Operand(exec, s2)).get_ptr());
+ bld.sop2(Builder::s_and, Definition(tmp2), bld.def(s1, scc),
+ phi_src, Operand(exec, bld.lm)).get_ptr());
insert_before_logical_end(pred,
- bld.sop2(aco_opcode::s_or_b64, Definition(new_cur), bld.def(s1, scc),
+ bld.sop2(Builder::s_or, Definition(new_cur), bld.def(s1, scc),
tmp1, tmp2).get_ptr());
}
}
- return bld.sop1(aco_opcode::s_mov_b64, phi->definitions[0], get_ssa(program, block->index, &state)).get_ptr();
-}
-
-void lower_linear_bool_phi(Program *program, Block *block, aco_ptr<Instruction>& phi)
-{
- Builder bld(program);
+ unsigned num_preds = block->linear_preds.size();
+ if (phi->operands.size() != num_preds) {
+ Pseudo_instruction* new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, num_preds, 1)};
+ new_phi->definitions[0] = phi->definitions[0];
+ phi.reset(new_phi);
+ } else {
+ phi->opcode = aco_opcode::p_linear_phi;
+ }
+ assert(phi->operands.size() == num_preds);
- for (unsigned i = 0; i < phi->operands.size(); i++) {
- if (!phi->operands[i].isTemp())
- continue;
+ for (unsigned i = 0; i < num_preds; i++)
+ phi->operands[i] = get_ssa(program, block->linear_preds[i], &state);
- Temp phi_src = phi->operands[i].getTemp();
- if (phi_src.regClass() == s2) {
- Temp new_phi_src = bld.tmp(s1);
- insert_before_logical_end(&program->blocks[block->linear_preds[i]],
- bld.sopc(aco_opcode::s_cmp_lg_u64, bld.scc(Definition(new_phi_src)),
- Operand(0u), phi_src).get_ptr());
- phi->operands[i].setTemp(new_phi_src);
- }
- }
+ return;
}
void lower_bool_phis(Program* program)
{
for (Block& block : program->blocks) {
- std::vector<aco_ptr<Instruction>> instructions;
- std::vector<aco_ptr<Instruction>> non_phi;
- instructions.swap(block.instructions);
- block.instructions.reserve(instructions.size());
- unsigned i = 0;
- for (; i < instructions.size(); i++)
- {
- aco_ptr<Instruction>& phi = instructions[i];
- if (phi->opcode != aco_opcode::p_phi && phi->opcode != aco_opcode::p_linear_phi)
+ for (aco_ptr<Instruction>& phi : block.instructions) {
+ if (phi->opcode == aco_opcode::p_phi) {
+ assert(program->wave_size == 64 ? phi->definitions[0].regClass() != s1 : phi->definitions[0].regClass() != s2);
+ if (phi->definitions[0].regClass() == program->lane_mask)
+ lower_divergent_bool_phi(program, &block, phi);
+ } else if (!is_phi(phi)) {
break;
- if (phi->opcode == aco_opcode::p_phi && phi->definitions[0].regClass() == s2) {
- non_phi.emplace_back(std::move(lower_divergent_bool_phi(program, &block, phi)));
- } else if (phi->opcode == aco_opcode::p_linear_phi && phi->definitions[0].regClass() == s1) {
- /* if it's a valid non-boolean phi, this should be a no-op */
- lower_linear_bool_phi(program, &block, phi);
- block.instructions.emplace_back(std::move(phi));
- } else {
- block.instructions.emplace_back(std::move(phi));
}
}
- for (auto&& instr : non_phi) {
- assert(instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi);
- block.instructions.emplace_back(std::move(instr));
- }
- for (; i < instructions.size(); i++) {
- aco_ptr<Instruction> instr = std::move(instructions[i]);
- assert(instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi);
- block.instructions.emplace_back(std::move(instr));
- }
}
}
projects / mesa.git / blobdiff