From: Daniel Schürmann Date: Tue, 7 Apr 2020 11:20:52 +0000 (+0100) Subject: aco: rename aco_lower_bool_phis() -> aco_lower_phis() X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=331794495ed0e8bbd87cafedfa9ef334bb43b0b7;p=mesa.git aco: rename aco_lower_bool_phis() -> aco_lower_phis() We also lower subdword phis, now. Reviewed-by: Rhys Perry Part-of: --- diff --git a/src/amd/Makefile.sources b/src/amd/Makefile.sources index a30619641be..04d4d7dd848 100644 --- a/src/amd/Makefile.sources +++ b/src/amd/Makefile.sources @@ -86,7 +86,7 @@ ACO_FILES = \ compiler/aco_reduce_assign.cpp \ compiler/aco_register_allocation.cpp \ compiler/aco_live_var_analysis.cpp \ - compiler/aco_lower_bool_phis.cpp \ + compiler/aco_lower_phis.cpp \ compiler/aco_lower_to_cssa.cpp \ compiler/aco_lower_to_hw_instr.cpp \ compiler/aco_optimizer.cpp \ diff --git a/src/amd/compiler/aco_interface.cpp b/src/amd/compiler/aco_interface.cpp index 8991d616eed..3031bb43f7a 100644 --- a/src/amd/compiler/aco_interface.cpp +++ b/src/amd/compiler/aco_interface.cpp @@ -93,9 +93,9 @@ void aco_compile_shader(unsigned shader_count, } aco::validate(program.get(), stderr); - /* Boolean phi lowering */ - aco::lower_bool_phis(program.get()); - //std::cerr << "After Boolean Phi Lowering:\n"; + /* Phi lowering */ + aco::lower_phis(program.get()); + //std::cerr << "After Phi Lowering:\n"; //aco_print_program(program.get(), stderr); aco::dominator_tree(program.get()); diff --git a/src/amd/compiler/aco_ir.h b/src/amd/compiler/aco_ir.h index 8ff5a9e220a..fb0c9beb208 100644 --- a/src/amd/compiler/aco_ir.h +++ b/src/amd/compiler/aco_ir.h @@ -1468,7 +1468,7 @@ void select_gs_copy_shader(Program *program, struct nir_shader *gs_shader, void lower_wqm(Program* program, live& live_vars, const struct radv_nir_compiler_options *options); -void lower_bool_phis(Program* program); +void lower_phis(Program* program); void calc_min_waves(Program* program); void update_vgpr_sgpr_demand(Program* program, const RegisterDemand new_demand); live live_var_analysis(Program* program, const struct radv_nir_compiler_options *options); diff --git a/src/amd/compiler/aco_lower_bool_phis.cpp b/src/amd/compiler/aco_lower_bool_phis.cpp deleted file mode 100644 index 1613e4be94a..00000000000 --- a/src/amd/compiler/aco_lower_bool_phis.cpp +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Copyright © 2019 Valve Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - * Authors: - * Rhys Perry (pendingchaos02@gmail.com) - * - */ - -#include - -#include "aco_ir.h" -#include "aco_builder.h" -#include - - -namespace aco { - -struct phi_use { - Block *block; - unsigned phi_def; - - bool operator<(const phi_use& other) const { - return std::make_tuple(block, phi_def) < - std::make_tuple(other.block, other.phi_def); - } -}; - -struct ssa_state { - std::map latest; - std::map> phis; -}; - -Operand get_ssa(Program *program, unsigned block_idx, ssa_state *state) -{ - while (true) { - auto pos = state->latest.find(block_idx); - if (pos != state->latest.end()) - return Operand(Temp(pos->second, program->lane_mask)); - - Block& block = program->blocks[block_idx]; - size_t pred = block.linear_preds.size(); - if (pred == 0) { - return Operand(program->lane_mask); - } else if (pred == 1) { - block_idx = block.linear_preds[0]; - continue; - } else { - unsigned res = program->allocateId(); - state->latest[block_idx] = res; - - aco_ptr phi{create_instruction(aco_opcode::p_linear_phi, Format::PSEUDO, pred, 1)}; - for (unsigned i = 0; i < pred; i++) { - phi->operands[i] = get_ssa(program, block.linear_preds[i], state); - if (phi->operands[i].isTemp()) { - assert(i < 64); - state->phis[phi->operands[i].tempId()][(phi_use){&block, res}] |= (uint64_t)1 << i; - } - } - phi->definitions[0] = Definition(Temp{res, program->lane_mask}); - block.instructions.emplace(block.instructions.begin(), std::move(phi)); - - return Operand(Temp(res, program->lane_mask)); - } - } -} - -void update_phi(Program *program, ssa_state *state, Block *block, unsigned phi_def, uint64_t operand_mask) { - for (auto& phi : block->instructions) { - if (phi->opcode != aco_opcode::p_phi && phi->opcode != aco_opcode::p_linear_phi) - break; - if (phi->opcode != aco_opcode::p_linear_phi) - continue; - if (phi->definitions[0].tempId() != phi_def) - continue; - assert(ffsll(operand_mask) <= phi->operands.size()); - - uint64_t operands = operand_mask; - while (operands) { - unsigned operand = u_bit_scan64(&operands); - Operand new_operand = get_ssa(program, block->linear_preds[operand], state); - phi->operands[operand] = new_operand; - if (!new_operand.isUndefined()) - state->phis[new_operand.tempId()][(phi_use){block, phi_def}] |= (uint64_t)1 << operand; - } - return; - } - assert(false); -} - -Temp write_ssa(Program *program, Block *block, ssa_state *state, unsigned previous) { - unsigned id = program->allocateId(); - state->latest[block->index] = id; - - /* update phis */ - if (previous) { - std::map phis; - phis.swap(state->phis[previous]); - for (auto& phi : phis) - update_phi(program, state, phi.first.block, phi.first.phi_def, phi.second); - } - - return {id, program->lane_mask}; -} - -void insert_before_logical_end(Block *block, aco_ptr instr) -{ - auto IsLogicalEnd = [] (const aco_ptr& instr) -> bool { - return instr->opcode == aco_opcode::p_logical_end; - }; - auto it = std::find_if(block->instructions.crbegin(), block->instructions.crend(), IsLogicalEnd); - - if (it == block->instructions.crend()) { - assert(block->instructions.back()->format == Format::PSEUDO_BRANCH); - block->instructions.insert(std::prev(block->instructions.end()), std::move(instr)); - } - else - block->instructions.insert(std::prev(it.base()), std::move(instr)); -} - -void lower_divergent_bool_phi(Program *program, Block *block, aco_ptr& phi) -{ - Builder bld(program); - - ssa_state state; - state.latest[block->index] = phi->definitions[0].tempId(); - for (unsigned i = 0; i < phi->operands.size(); i++) { - Block *pred = &program->blocks[block->logical_preds[i]]; - - if (phi->operands[i].isUndefined()) - continue; - - assert(phi->operands[i].isTemp()); - Temp phi_src = phi->operands[i].getTemp(); - assert(phi_src.regClass() == bld.lm); - - Operand cur = get_ssa(program, pred->index, &state); - assert(cur.regClass() == bld.lm); - Temp new_cur = write_ssa(program, pred, &state, cur.isTemp() ? cur.tempId() : 0); - assert(new_cur.regClass() == bld.lm); - - if (cur.isUndefined()) { - insert_before_logical_end(pred, bld.sop1(aco_opcode::s_mov_b64, Definition(new_cur), phi_src).get_ptr()); - } else { - Temp tmp1 = bld.tmp(bld.lm), tmp2 = bld.tmp(bld.lm); - insert_before_logical_end(pred, - bld.sop2(Builder::s_andn2, Definition(tmp1), bld.def(s1, scc), - cur, Operand(exec, bld.lm)).get_ptr()); - insert_before_logical_end(pred, - bld.sop2(Builder::s_and, Definition(tmp2), bld.def(s1, scc), - phi_src, Operand(exec, bld.lm)).get_ptr()); - insert_before_logical_end(pred, - bld.sop2(Builder::s_or, Definition(new_cur), bld.def(s1, scc), - tmp1, tmp2).get_ptr()); - } - } - - unsigned num_preds = block->linear_preds.size(); - if (phi->operands.size() != num_preds) { - Pseudo_instruction* new_phi{create_instruction(aco_opcode::p_linear_phi, Format::PSEUDO, num_preds, 1)}; - new_phi->definitions[0] = phi->definitions[0]; - phi.reset(new_phi); - } else { - phi->opcode = aco_opcode::p_linear_phi; - } - assert(phi->operands.size() == num_preds); - - for (unsigned i = 0; i < num_preds; i++) - phi->operands[i] = get_ssa(program, block->linear_preds[i], &state); - - return; -} - -void lower_subdword_phis(Program *program, Block *block, aco_ptr& phi) -{ - Builder bld(program); - for (unsigned i = 0; i < phi->operands.size(); i++) { - if (phi->operands[i].isUndefined()) - continue; - if (phi->operands[i].regClass() == phi->definitions[0].regClass()) - continue; - - assert(phi->operands[i].isTemp()); - Block *pred = &program->blocks[block->logical_preds[i]]; - Temp phi_src = phi->operands[i].getTemp(); - - assert(phi_src.regClass().type() == RegType::sgpr); - Temp tmp = bld.tmp(RegClass(RegType::vgpr, phi_src.size())); - insert_before_logical_end(pred, bld.pseudo(aco_opcode::p_create_vector, Definition(tmp), phi_src).get_ptr()); - Temp new_phi_src = bld.tmp(phi->definitions[0].regClass()); - insert_before_logical_end(pred, bld.pseudo(aco_opcode::p_extract_vector, Definition(new_phi_src), tmp, Operand(0u)).get_ptr()); - - phi->operands[i].setTemp(new_phi_src); - } - return; -} - -void lower_bool_phis(Program* program) -{ - for (Block& block : program->blocks) { - for (aco_ptr& phi : block.instructions) { - if (phi->opcode == aco_opcode::p_phi) { - assert(program->wave_size == 64 ? phi->definitions[0].regClass() != s1 : phi->definitions[0].regClass() != s2); - if (phi->definitions[0].regClass() == program->lane_mask) - lower_divergent_bool_phi(program, &block, phi); - else if (phi->definitions[0].regClass().is_subdword()) - lower_subdword_phis(program, &block, phi); - } else if (!is_phi(phi)) { - break; - } - } - } -} - -} diff --git a/src/amd/compiler/aco_lower_phis.cpp b/src/amd/compiler/aco_lower_phis.cpp new file mode 100644 index 00000000000..c47f59e7cc6 --- /dev/null +++ b/src/amd/compiler/aco_lower_phis.cpp @@ -0,0 +1,233 @@ +/* + * Copyright © 2019 Valve Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + * Authors: + * Rhys Perry (pendingchaos02@gmail.com) + * + */ + +#include + +#include "aco_ir.h" +#include "aco_builder.h" +#include + + +namespace aco { + +struct phi_use { + Block *block; + unsigned phi_def; + + bool operator<(const phi_use& other) const { + return std::make_tuple(block, phi_def) < + std::make_tuple(other.block, other.phi_def); + } +}; + +struct ssa_state { + std::map latest; + std::map> phis; +}; + +Operand get_ssa(Program *program, unsigned block_idx, ssa_state *state) +{ + while (true) { + auto pos = state->latest.find(block_idx); + if (pos != state->latest.end()) + return Operand(Temp(pos->second, program->lane_mask)); + + Block& block = program->blocks[block_idx]; + size_t pred = block.linear_preds.size(); + if (pred == 0) { + return Operand(program->lane_mask); + } else if (pred == 1) { + block_idx = block.linear_preds[0]; + continue; + } else { + unsigned res = program->allocateId(); + state->latest[block_idx] = res; + + aco_ptr phi{create_instruction(aco_opcode::p_linear_phi, Format::PSEUDO, pred, 1)}; + for (unsigned i = 0; i < pred; i++) { + phi->operands[i] = get_ssa(program, block.linear_preds[i], state); + if (phi->operands[i].isTemp()) { + assert(i < 64); + state->phis[phi->operands[i].tempId()][(phi_use){&block, res}] |= (uint64_t)1 << i; + } + } + phi->definitions[0] = Definition(Temp{res, program->lane_mask}); + block.instructions.emplace(block.instructions.begin(), std::move(phi)); + + return Operand(Temp(res, program->lane_mask)); + } + } +} + +void update_phi(Program *program, ssa_state *state, Block *block, unsigned phi_def, uint64_t operand_mask) { + for (auto& phi : block->instructions) { + if (phi->opcode != aco_opcode::p_phi && phi->opcode != aco_opcode::p_linear_phi) + break; + if (phi->opcode != aco_opcode::p_linear_phi) + continue; + if (phi->definitions[0].tempId() != phi_def) + continue; + assert(ffsll(operand_mask) <= phi->operands.size()); + + uint64_t operands = operand_mask; + while (operands) { + unsigned operand = u_bit_scan64(&operands); + Operand new_operand = get_ssa(program, block->linear_preds[operand], state); + phi->operands[operand] = new_operand; + if (!new_operand.isUndefined()) + state->phis[new_operand.tempId()][(phi_use){block, phi_def}] |= (uint64_t)1 << operand; + } + return; + } + assert(false); +} + +Temp write_ssa(Program *program, Block *block, ssa_state *state, unsigned previous) { + unsigned id = program->allocateId(); + state->latest[block->index] = id; + + /* update phis */ + if (previous) { + std::map phis; + phis.swap(state->phis[previous]); + for (auto& phi : phis) + update_phi(program, state, phi.first.block, phi.first.phi_def, phi.second); + } + + return {id, program->lane_mask}; +} + +void insert_before_logical_end(Block *block, aco_ptr instr) +{ + auto IsLogicalEnd = [] (const aco_ptr& instr) -> bool { + return instr->opcode == aco_opcode::p_logical_end; + }; + auto it = std::find_if(block->instructions.crbegin(), block->instructions.crend(), IsLogicalEnd); + + if (it == block->instructions.crend()) { + assert(block->instructions.back()->format == Format::PSEUDO_BRANCH); + block->instructions.insert(std::prev(block->instructions.end()), std::move(instr)); + } + else + block->instructions.insert(std::prev(it.base()), std::move(instr)); +} + +void lower_divergent_bool_phi(Program *program, Block *block, aco_ptr& phi) +{ + Builder bld(program); + + ssa_state state; + state.latest[block->index] = phi->definitions[0].tempId(); + for (unsigned i = 0; i < phi->operands.size(); i++) { + Block *pred = &program->blocks[block->logical_preds[i]]; + + if (phi->operands[i].isUndefined()) + continue; + + assert(phi->operands[i].isTemp()); + Temp phi_src = phi->operands[i].getTemp(); + assert(phi_src.regClass() == bld.lm); + + Operand cur = get_ssa(program, pred->index, &state); + assert(cur.regClass() == bld.lm); + Temp new_cur = write_ssa(program, pred, &state, cur.isTemp() ? cur.tempId() : 0); + assert(new_cur.regClass() == bld.lm); + + if (cur.isUndefined()) { + insert_before_logical_end(pred, bld.sop1(aco_opcode::s_mov_b64, Definition(new_cur), phi_src).get_ptr()); + } else { + Temp tmp1 = bld.tmp(bld.lm), tmp2 = bld.tmp(bld.lm); + insert_before_logical_end(pred, + bld.sop2(Builder::s_andn2, Definition(tmp1), bld.def(s1, scc), + cur, Operand(exec, bld.lm)).get_ptr()); + insert_before_logical_end(pred, + bld.sop2(Builder::s_and, Definition(tmp2), bld.def(s1, scc), + phi_src, Operand(exec, bld.lm)).get_ptr()); + insert_before_logical_end(pred, + bld.sop2(Builder::s_or, Definition(new_cur), bld.def(s1, scc), + tmp1, tmp2).get_ptr()); + } + } + + unsigned num_preds = block->linear_preds.size(); + if (phi->operands.size() != num_preds) { + Pseudo_instruction* new_phi{create_instruction(aco_opcode::p_linear_phi, Format::PSEUDO, num_preds, 1)}; + new_phi->definitions[0] = phi->definitions[0]; + phi.reset(new_phi); + } else { + phi->opcode = aco_opcode::p_linear_phi; + } + assert(phi->operands.size() == num_preds); + + for (unsigned i = 0; i < num_preds; i++) + phi->operands[i] = get_ssa(program, block->linear_preds[i], &state); + + return; +} + +void lower_subdword_phis(Program *program, Block *block, aco_ptr& phi) +{ + Builder bld(program); + for (unsigned i = 0; i < phi->operands.size(); i++) { + if (phi->operands[i].isUndefined()) + continue; + if (phi->operands[i].regClass() == phi->definitions[0].regClass()) + continue; + + assert(phi->operands[i].isTemp()); + Block *pred = &program->blocks[block->logical_preds[i]]; + Temp phi_src = phi->operands[i].getTemp(); + + assert(phi_src.regClass().type() == RegType::sgpr); + Temp tmp = bld.tmp(RegClass(RegType::vgpr, phi_src.size())); + insert_before_logical_end(pred, bld.pseudo(aco_opcode::p_create_vector, Definition(tmp), phi_src).get_ptr()); + Temp new_phi_src = bld.tmp(phi->definitions[0].regClass()); + insert_before_logical_end(pred, bld.pseudo(aco_opcode::p_extract_vector, Definition(new_phi_src), tmp, Operand(0u)).get_ptr()); + + phi->operands[i].setTemp(new_phi_src); + } + return; +} + +void lower_phis(Program* program) +{ + for (Block& block : program->blocks) { + for (aco_ptr& phi : block.instructions) { + if (phi->opcode == aco_opcode::p_phi) { + assert(program->wave_size == 64 ? phi->definitions[0].regClass() != s1 : phi->definitions[0].regClass() != s2); + if (phi->definitions[0].regClass() == program->lane_mask) + lower_divergent_bool_phi(program, &block, phi); + else if (phi->definitions[0].regClass().is_subdword()) + lower_subdword_phis(program, &block, phi); + } else if (!is_phi(phi)) { + break; + } + } + } +} + +} diff --git a/src/amd/compiler/meson.build b/src/amd/compiler/meson.build index 44b56baab4c..778cf8ee3ee 100644 --- a/src/amd/compiler/meson.build +++ b/src/amd/compiler/meson.build @@ -68,7 +68,7 @@ libaco_files = files( 'aco_reduce_assign.cpp', 'aco_register_allocation.cpp', 'aco_live_var_analysis.cpp', - 'aco_lower_bool_phis.cpp', + 'aco_lower_phis.cpp', 'aco_lower_to_cssa.cpp', 'aco_lower_to_hw_instr.cpp', 'aco_optimizer.cpp',