*
*/
+#include <algorithm>
+
#include "aco_ir.h"
+#include <stack>
+#include <functional>
+
+#include <time.h>
namespace aco {
namespace {
-struct NOP_ctx {
- enum chip_class chip_class;
- unsigned vcc_physical;
+struct NOP_ctx_gfx6 {
+ void join(const NOP_ctx_gfx6 &other) {
+ set_vskip_mode_then_vector = MAX2(set_vskip_mode_then_vector, other.set_vskip_mode_then_vector);
+ valu_wr_vcc_then_vccz = MAX2(valu_wr_vcc_then_vccz, other.valu_wr_vcc_then_vccz);
+ valu_wr_exec_then_execz = MAX2(valu_wr_exec_then_execz, other.valu_wr_exec_then_execz);
+ valu_wr_vcc_then_div_fmas = MAX2(valu_wr_vcc_then_div_fmas, other.valu_wr_vcc_then_div_fmas);
+ salu_wr_m0_then_gds_msg_ttrace = MAX2(salu_wr_m0_then_gds_msg_ttrace, other.salu_wr_m0_then_gds_msg_ttrace);
+ valu_wr_exec_then_dpp = MAX2(valu_wr_exec_then_dpp, other.valu_wr_exec_then_dpp);
+ salu_wr_m0_then_lds = MAX2(salu_wr_m0_then_lds, other.salu_wr_m0_then_lds);
+ salu_wr_m0_then_moverel = MAX2(salu_wr_m0_then_moverel, other.salu_wr_m0_then_moverel);
+ setreg_then_getsetreg = MAX2(setreg_then_getsetreg, other.setreg_then_getsetreg);
+ vmem_store_then_wr_data |= other.vmem_store_then_wr_data;
+ smem_clause |= other.smem_clause;
+ smem_write |= other.smem_write;
+ for (unsigned i = 0; i < BITSET_WORDS(128); i++) {
+ smem_clause_read_write[i] |= other.smem_clause_read_write[i];
+ smem_clause_write[i] |= other.smem_clause_write[i];
+ }
+ }
+
+ bool operator==(const NOP_ctx_gfx6 &other)
+ {
+ return
+ set_vskip_mode_then_vector == other.set_vskip_mode_then_vector &&
+ valu_wr_vcc_then_vccz == other.valu_wr_vcc_then_vccz &&
+ valu_wr_exec_then_execz == other.valu_wr_exec_then_execz &&
+ valu_wr_vcc_then_div_fmas == other.valu_wr_vcc_then_div_fmas &&
+ vmem_store_then_wr_data == other.vmem_store_then_wr_data &&
+ salu_wr_m0_then_gds_msg_ttrace == other.salu_wr_m0_then_gds_msg_ttrace &&
+ valu_wr_exec_then_dpp == other.valu_wr_exec_then_dpp &&
+ salu_wr_m0_then_lds == other.salu_wr_m0_then_lds &&
+ salu_wr_m0_then_moverel == other.salu_wr_m0_then_moverel &&
+ setreg_then_getsetreg == other.setreg_then_getsetreg &&
+ smem_clause == other.smem_clause &&
+ smem_write == other.smem_write &&
+ BITSET_EQUAL(smem_clause_read_write, other.smem_clause_read_write) &&
+ BITSET_EQUAL(smem_clause_write, other.smem_clause_write);
+ }
+
+ void add_wait_states(unsigned amount)
+ {
+ if ((set_vskip_mode_then_vector -= amount) < 0)
+ set_vskip_mode_then_vector = 0;
+
+ if ((valu_wr_vcc_then_vccz -= amount) < 0)
+ valu_wr_vcc_then_vccz = 0;
+
+ if ((valu_wr_exec_then_execz -= amount) < 0)
+ valu_wr_exec_then_execz = 0;
+
+ if ((valu_wr_vcc_then_div_fmas -= amount) < 0)
+ valu_wr_vcc_then_div_fmas = 0;
+
+ if ((salu_wr_m0_then_gds_msg_ttrace -= amount) < 0)
+ salu_wr_m0_then_gds_msg_ttrace = 0;
+
+ if ((valu_wr_exec_then_dpp -= amount) < 0)
+ valu_wr_exec_then_dpp = 0;
+
+ if ((salu_wr_m0_then_lds -= amount) < 0)
+ salu_wr_m0_then_lds = 0;
+
+ if ((salu_wr_m0_then_moverel -= amount) < 0)
+ salu_wr_m0_then_moverel = 0;
+
+ if ((setreg_then_getsetreg -= amount) < 0)
+ setreg_then_getsetreg = 0;
+
+ vmem_store_then_wr_data.reset();
+ }
+
+ /* setting MODE.vskip and then any vector op requires 2 wait states */
+ int8_t set_vskip_mode_then_vector = 0;
+
+ /* VALU writing VCC/EXEC and then a VALU reading VCCZ/EXECZ requires 5 wait states */
+ int8_t valu_wr_vcc_then_vccz = 0;
+ int8_t valu_wr_exec_then_execz = 0;
+
+ /* VALU writing VCC followed by v_div_fmas require 4 wait states */
+ int8_t valu_wr_vcc_then_div_fmas = 0;
- /* pre-GFX10 */
- /* just initialize these with something less than max NOPs */
- int VALU_wrexec = -10;
- int VALU_wrvcc = -10;
- int VALU_wrsgpr = -10;
+ /* SALU writing M0 followed by GDS, s_sendmsg or s_ttrace_data requires 1 wait state */
+ int8_t salu_wr_m0_then_gds_msg_ttrace = 0;
- /* GFX10 */
- int last_VMEM_since_scalar_write = -1;
+ /* VALU writing EXEC followed by DPP requires 5 wait states */
+ int8_t valu_wr_exec_then_dpp = 0;
- NOP_ctx(Program* program) : chip_class(program->chip_class) {
- vcc_physical = program->config->num_sgprs - 2;
+ /* SALU writing M0 followed by some LDS instructions requires 1 wait state on GFX10 */
+ int8_t salu_wr_m0_then_lds = 0;
+
+ /* SALU writing M0 followed by s_moverel requires 1 wait state on GFX9 */
+ int8_t salu_wr_m0_then_moverel = 0;
+
+ /* s_setreg followed by a s_getreg/s_setreg of the same register needs 2 wait states
+ * currently we don't look at the actual register */
+ int8_t setreg_then_getsetreg = 0;
+
+ /* some memory instructions writing >64bit followed by a instructions
+ * writing the VGPRs holding the writedata requires 1 wait state */
+ std::bitset<256> vmem_store_then_wr_data;
+
+ /* we break up SMEM clauses that contain stores or overwrite an
+ * operand/definition of another instruction in the clause */
+ bool smem_clause = false;
+ bool smem_write = false;
+ BITSET_DECLARE(smem_clause_read_write, 128) = {0};
+ BITSET_DECLARE(smem_clause_write, 128) = {0};
+};
+
+struct NOP_ctx_gfx10 {
+ bool has_VOPC = false;
+ bool has_nonVALU_exec_read = false;
+ bool has_VMEM = false;
+ bool has_branch_after_VMEM = false;
+ bool has_DS = false;
+ bool has_branch_after_DS = false;
+ std::bitset<128> sgprs_read_by_VMEM;
+ std::bitset<128> sgprs_read_by_SMEM;
+
+ void join(const NOP_ctx_gfx10 &other) {
+ has_VOPC |= other.has_VOPC;
+ has_nonVALU_exec_read |= other.has_nonVALU_exec_read;
+ has_VMEM |= other.has_VMEM;
+ has_branch_after_VMEM |= other.has_branch_after_VMEM;
+ has_DS |= other.has_DS;
+ has_branch_after_DS |= other.has_branch_after_DS;
+ sgprs_read_by_VMEM |= other.sgprs_read_by_VMEM;
+ sgprs_read_by_SMEM |= other.sgprs_read_by_SMEM;
+ }
+
+ bool operator==(const NOP_ctx_gfx10 &other)
+ {
+ return
+ has_VOPC == other.has_VOPC &&
+ has_nonVALU_exec_read == other.has_nonVALU_exec_read &&
+ has_VMEM == other.has_VMEM &&
+ has_branch_after_VMEM == other.has_branch_after_VMEM &&
+ has_DS == other.has_DS &&
+ has_branch_after_DS == other.has_branch_after_DS &&
+ sgprs_read_by_VMEM == other.sgprs_read_by_VMEM &&
+ sgprs_read_by_SMEM == other.sgprs_read_by_SMEM;
}
};
-bool VALU_writes_sgpr(aco_ptr<Instruction>& instr)
+int get_wait_states(aco_ptr<Instruction>& instr)
{
- if ((uint32_t) instr->format & (uint32_t) Format::VOPC)
- return true;
- if (instr->isVOP3() && instr->definitions.size() == 2)
- return true;
- if (instr->opcode == aco_opcode::v_readfirstlane_b32 || instr->opcode == aco_opcode::v_readlane_b32)
- return true;
- return false;
+ if (instr->opcode == aco_opcode::s_nop)
+ return static_cast<SOPP_instruction*>(instr.get())->imm + 1;
+ else if (instr->opcode == aco_opcode::p_constaddr)
+ return 3; /* lowered to 3 instructions in the assembler */
+ else
+ return 1;
}
bool regs_intersect(PhysReg a_reg, unsigned a_size, PhysReg b_reg, unsigned b_size)
(b_reg - a_reg < a_size);
}
-unsigned handle_SMEM_clause(aco_ptr<Instruction>& instr, int new_idx,
- std::vector<aco_ptr<Instruction>>& new_instructions)
+template <bool Valu, bool Vintrp, bool Salu>
+int handle_raw_hazard_internal(Program *program, Block *block,
+ int nops_needed, PhysReg reg, uint32_t mask)
{
- //TODO: s_dcache_inv needs to be in it's own group on GFX10 (and previous versions?)
- const bool is_store = instr->definitions.empty();
- for (int pred_idx = new_idx - 1; pred_idx >= 0; pred_idx--) {
- aco_ptr<Instruction>& pred = new_instructions[pred_idx];
- if (pred->format != Format::SMEM)
- break;
-
- /* Don't allow clauses with store instructions since the clause's
- * instructions may use the same address. */
- if (is_store || pred->definitions.empty())
- return 1;
+ unsigned mask_size = util_last_bit(mask);
+ for (int pred_idx = block->instructions.size() - 1; pred_idx >= 0; pred_idx--) {
+ aco_ptr<Instruction>& pred = block->instructions[pred_idx];
+
+ uint32_t writemask = 0;
+ for (Definition& def : pred->definitions) {
+ if (regs_intersect(reg, mask_size, def.physReg(), def.size())) {
+ unsigned start = def.physReg() > reg ? def.physReg() - reg : 0;
+ unsigned end = MIN2(mask_size, start + def.size());
+ writemask |= u_bit_consecutive(start, end - start);
+ }
+ }
- Definition& instr_def = instr->definitions[0];
- Definition& pred_def = pred->definitions[0];
+ bool is_hazard = writemask != 0 &&
+ ((pred->isVALU() && Valu) ||
+ (pred->format == Format::VINTRP && Vintrp) ||
+ (pred->isSALU() && Salu));
+ if (is_hazard)
+ return nops_needed;
- /* ISA reference doesn't say anything about this, but best to be safe */
- if (regs_intersect(instr_def.physReg(), instr_def.size(), pred_def.physReg(), pred_def.size()))
- return 1;
+ mask &= ~writemask;
+ nops_needed -= get_wait_states(pred);
- for (const Operand& op : pred->operands) {
- if (op.isConstant() || !op.isFixed())
- continue;
- if (regs_intersect(instr_def.physReg(), instr_def.size(), op.physReg(), op.size()))
- return 1;
- }
- for (const Operand& op : instr->operands) {
- if (op.isConstant() || !op.isFixed())
- continue;
- if (regs_intersect(pred_def.physReg(), pred_def.size(), op.physReg(), op.size()))
- return 1;
- }
+ if (nops_needed <= 0 || mask == 0)
+ return 0;
}
- return 0;
+ int res = 0;
+
+ /* Loops require branch instructions, which count towards the wait
+ * states. So even with loops this should finish unless nops_needed is some
+ * huge value. */
+ for (unsigned lin_pred : block->linear_preds) {
+ res = std::max(res, handle_raw_hazard_internal<Valu, Vintrp, Salu>(
+ program, &program->blocks[lin_pred], nops_needed, reg, mask));
+ }
+ return res;
}
-int handle_instruction(NOP_ctx& ctx, aco_ptr<Instruction>& instr,
- std::vector<aco_ptr<Instruction>>& old_instructions,
- std::vector<aco_ptr<Instruction>>& new_instructions)
+template <bool Valu, bool Vintrp, bool Salu>
+void handle_raw_hazard(Program *program, Block *cur_block, int *NOPs, int min_states, Operand op)
{
- int new_idx = new_instructions.size();
+ if (*NOPs >= min_states)
+ return;
+ int res = handle_raw_hazard_internal<Valu, Vintrp, Salu>(program, cur_block, min_states, op.physReg(), u_bit_consecutive(0, op.size()));
+ *NOPs = MAX2(*NOPs, res);
+}
- // TODO: setreg / getreg / m0 writes
- // TODO: try to schedule the NOP-causing instruction up to reduce the number of stall cycles
+static auto handle_valu_then_read_hazard = handle_raw_hazard<true, true, false>;
+static auto handle_vintrp_then_read_hazard = handle_raw_hazard<false, true, false>;
+static auto handle_valu_salu_then_read_hazard = handle_raw_hazard<true, true, true>;
+
+void set_bitset_range(BITSET_WORD *words, unsigned start, unsigned size) {
+ unsigned end = start + size - 1;
+ unsigned start_mod = start % BITSET_WORDBITS;
+ if (start_mod + size <= BITSET_WORDBITS) {
+ BITSET_SET_RANGE(words, start, end);
+ } else {
+ unsigned first_size = BITSET_WORDBITS - start_mod;
+ set_bitset_range(words, start, BITSET_WORDBITS - start_mod);
+ set_bitset_range(words, start + first_size, size - first_size);
+ }
+}
- /* break off from prevous SMEM clause if needed */
- if (instr->format == Format::SMEM && ctx.chip_class >= GFX8) {
- return handle_SMEM_clause(instr, new_idx, new_instructions);
- } else if (instr->isVALU() || instr->format == Format::VINTRP) {
- int NOPs = 0;
+bool test_bitset_range(BITSET_WORD *words, unsigned start, unsigned size) {
+ unsigned end = start + size - 1;
+ unsigned start_mod = start % BITSET_WORDBITS;
+ if (start_mod + size <= BITSET_WORDBITS) {
+ return BITSET_TEST_RANGE(words, start, end);
+ } else {
+ unsigned first_size = BITSET_WORDBITS - start_mod;
+ return test_bitset_range(words, start, BITSET_WORDBITS - start_mod) ||
+ test_bitset_range(words, start + first_size, size - first_size);
+ }
+}
- if (instr->isDPP()) {
- /* VALU does not forward EXEC to DPP. */
- if (ctx.VALU_wrexec + 5 >= new_idx)
- NOPs = 5 + ctx.VALU_wrexec - new_idx + 1;
-
- /* VALU DPP reads VGPR written by VALU */
- for (int pred_idx = new_idx - 1; pred_idx >= 0 && pred_idx >= new_idx - 2; pred_idx--) {
- aco_ptr<Instruction>& pred = new_instructions[pred_idx];
- if ((pred->isVALU() || pred->format == Format::VINTRP) &&
- !pred->definitions.empty() &&
- pred->definitions[0].physReg() == instr->operands[0].physReg()) {
- NOPs = std::max(NOPs, 2 + pred_idx - new_idx + 1);
+/* A SMEM clause is any group of consecutive SMEM instructions. The
+ * instructions in this group may return out of order and/or may be replayed.
+ *
+ * To fix this potential hazard correctly, we have to make sure that when a
+ * clause has more than one instruction, no instruction in the clause writes
+ * to a register that is read by another instruction in the clause (including
+ * itself). In this case, we have to break the SMEM clause by inserting non
+ * SMEM instructions.
+ *
+ * SMEM clauses are only present on GFX8+, and only matter when XNACK is set.
+ */
+void handle_smem_clause_hazards(Program *program, NOP_ctx_gfx6 &ctx,
+ aco_ptr<Instruction>& instr, int *NOPs)
+{
+ /* break off from previous SMEM clause if needed */
+ if (!*NOPs & (ctx.smem_clause || ctx.smem_write)) {
+ /* Don't allow clauses with store instructions since the clause's
+ * instructions may use the same address. */
+ if (ctx.smem_write || instr->definitions.empty() || instr_info.is_atomic[(unsigned)instr->opcode]) {
+ *NOPs = 1;
+ } else if (program->xnack_enabled) {
+ for (Operand op : instr->operands) {
+ if (!op.isConstant() && test_bitset_range(ctx.smem_clause_write, op.physReg(), op.size())) {
+ *NOPs = 1;
break;
}
}
- }
- /* SALU writes M0 */
- if (instr->format == Format::VINTRP && new_idx > 0 && ctx.chip_class >= GFX9) {
- aco_ptr<Instruction>& pred = new_instructions.back();
- if (pred->isSALU() &&
- !pred->definitions.empty() &&
- pred->definitions[0].physReg() == m0)
- NOPs = std::max(NOPs, 1);
+ Definition def = instr->definitions[0];
+ if (!*NOPs && test_bitset_range(ctx.smem_clause_read_write, def.physReg(), def.size()))
+ *NOPs = 1;
}
+ }
+}
- for (const Operand& op : instr->operands) {
- /* VALU which uses VCCZ */
- if (op.physReg() == PhysReg{251} &&
- ctx.VALU_wrvcc + 5 >= new_idx)
- NOPs = std::max(NOPs, 5 + ctx.VALU_wrvcc - new_idx + 1);
-
- /* VALU which uses EXECZ */
- if (op.physReg() == PhysReg{252} &&
- ctx.VALU_wrexec + 5 >= new_idx)
- NOPs = std::max(NOPs, 5 + ctx.VALU_wrexec - new_idx + 1);
-
- /* VALU which reads VCC as a constant */
- if (ctx.VALU_wrvcc + 1 >= new_idx) {
- for (unsigned k = 0; k < op.size(); k++) {
- unsigned reg = op.physReg() + k;
- if (reg == ctx.vcc_physical || reg == ctx.vcc_physical + 1)
- NOPs = std::max(NOPs, 1);
- }
+/* TODO: we don't handle accessing VCC using the actual SGPR instead of using the alias */
+void handle_instruction_gfx6(Program *program, Block *cur_block, NOP_ctx_gfx6 &ctx,
+ aco_ptr<Instruction>& instr, std::vector<aco_ptr<Instruction>>& new_instructions)
+{
+ /* check hazards */
+ int NOPs = 0;
+
+ if (instr->format == Format::SMEM) {
+ if (program->chip_class == GFX6) {
+ /* A read of an SGPR by SMRD instruction requires 4 wait states
+ * when the SGPR was written by a VALU instruction. According to LLVM,
+ * there is also an undocumented hardware behavior when the buffer
+ * descriptor is written by a SALU instruction */
+ for (unsigned i = 0; i < instr->operands.size(); i++) {
+ Operand op = instr->operands[i];
+ if (op.isConstant())
+ continue;
+
+ bool is_buffer_desc = i == 0 && op.size() > 2;
+ if (is_buffer_desc)
+ handle_valu_salu_then_read_hazard(program, cur_block, &NOPs, 4, op);
+ else
+ handle_valu_then_read_hazard(program, cur_block, &NOPs, 4, op);
}
}
- switch (instr->opcode) {
- case aco_opcode::v_readlane_b32:
- case aco_opcode::v_writelane_b32: {
- if (ctx.VALU_wrsgpr + 4 < new_idx)
- break;
- PhysReg reg = instr->operands[1].physReg();
- for (int pred_idx = new_idx - 1; pred_idx >= 0 && pred_idx >= new_idx - 4; pred_idx--) {
- aco_ptr<Instruction>& pred = new_instructions[pred_idx];
- if (!pred->isVALU() || !VALU_writes_sgpr(pred))
- continue;
- for (const Definition& def : pred->definitions) {
- if (def.physReg() == reg)
- NOPs = std::max(NOPs, 4 + pred_idx - new_idx + 1);
- }
- }
- break;
- }
- case aco_opcode::v_div_fmas_f32:
- case aco_opcode::v_div_fmas_f64: {
- if (ctx.VALU_wrvcc + 4 >= new_idx)
- NOPs = std::max(NOPs, 4 + ctx.VALU_wrvcc - new_idx + 1);
- break;
- }
- default:
- break;
+ handle_smem_clause_hazards(program, ctx, instr, &NOPs);
+ } else if (instr->isSALU()) {
+ if (instr->opcode == aco_opcode::s_setreg_b32 || instr->opcode == aco_opcode::s_setreg_imm32_b32 ||
+ instr->opcode == aco_opcode::s_getreg_b32) {
+ NOPs = MAX2(NOPs, ctx.setreg_then_getsetreg);
}
- /* Write VGPRs holding writedata > 64 bit from MIMG/MUBUF instructions */
- // FIXME: handle case if the last instruction of a block without branch is such store
- // TODO: confirm that DS instructions cannot cause WAR hazards here
- if (new_idx > 0) {
- aco_ptr<Instruction>& pred = new_instructions.back();
- if (pred->isVMEM() &&
- pred->operands.size() == 4 &&
- pred->operands[3].size() > 2 &&
- pred->operands[1].size() != 8 &&
- (pred->format != Format::MUBUF || pred->operands[2].physReg() >= 102)) {
- /* Ops that use a 256-bit T# do not need a wait state.
- * BUFFER_STORE_* operations that use an SGPR for "offset"
- * do not require any wait states. */
- PhysReg wrdata = pred->operands[3].physReg();
- unsigned size = pred->operands[3].size();
- assert(wrdata >= 256);
- for (const Definition& def : instr->definitions) {
- if (regs_intersect(def.physReg(), def.size(), wrdata, size))
- NOPs = std::max(NOPs, 1);
- }
+ if (program->chip_class == GFX9) {
+ if (instr->opcode == aco_opcode::s_movrels_b32 || instr->opcode == aco_opcode::s_movrels_b64 ||
+ instr->opcode == aco_opcode::s_movreld_b32 || instr->opcode == aco_opcode::s_movreld_b64) {
+ NOPs = MAX2(NOPs, ctx.salu_wr_m0_then_moverel);
}
}
- if (VALU_writes_sgpr(instr)) {
- for (const Definition& def : instr->definitions) {
- if (def.physReg() == vcc)
- ctx.VALU_wrvcc = NOPs ? new_idx : new_idx + 1;
- else if (def.physReg() == exec)
- ctx.VALU_wrexec = NOPs ? new_idx : new_idx + 1;
- else if (def.physReg() <= 102)
- ctx.VALU_wrsgpr = NOPs ? new_idx : new_idx + 1;
+ if (instr->opcode == aco_opcode::s_sendmsg || instr->opcode == aco_opcode::s_ttracedata)
+ NOPs = MAX2(NOPs, ctx.salu_wr_m0_then_gds_msg_ttrace);
+ } else if (instr->format == Format::DS && static_cast<DS_instruction *>(instr.get())->gds) {
+ NOPs = MAX2(NOPs, ctx.salu_wr_m0_then_gds_msg_ttrace);
+ } else if (instr->isVALU() || instr->format == Format::VINTRP) {
+ for (Operand op : instr->operands) {
+ if (op.physReg() == vccz)
+ NOPs = MAX2(NOPs, ctx.valu_wr_vcc_then_vccz);
+ if (op.physReg() == execz)
+ NOPs = MAX2(NOPs, ctx.valu_wr_exec_then_execz);
+ }
+
+ if (instr->isDPP()) {
+ NOPs = MAX2(NOPs, ctx.valu_wr_exec_then_dpp);
+ handle_valu_then_read_hazard(program, cur_block, &NOPs, 2, instr->operands[0]);
+ }
+
+ for (Definition def : instr->definitions) {
+ if (def.regClass().type() != RegType::sgpr) {
+ for (unsigned i = 0; i < def.size(); i++)
+ NOPs = MAX2(NOPs, ctx.vmem_store_then_wr_data[(def.physReg() & 0xff) + i]);
}
}
- return NOPs;
- } else if (instr->isVMEM() && ctx.VALU_wrsgpr + 5 >= new_idx) {
+
+ if ((instr->opcode == aco_opcode::v_readlane_b32 ||
+ instr->opcode == aco_opcode::v_readlane_b32_e64 ||
+ instr->opcode == aco_opcode::v_writelane_b32 ||
+ instr->opcode == aco_opcode::v_writelane_b32_e64) &&
+ !instr->operands[1].isConstant()) {
+ handle_valu_then_read_hazard(program, cur_block, &NOPs, 4, instr->operands[1]);
+ }
+
+ /* It's required to insert 1 wait state if the dst VGPR of any v_interp_*
+ * is followed by a read with v_readfirstlane or v_readlane to fix GPU
+ * hangs on GFX6. Note that v_writelane_* is apparently not affected.
+ * This hazard isn't documented anywhere but AMD confirmed that hazard.
+ */
+ if (program->chip_class == GFX6 &&
+ (instr->opcode == aco_opcode::v_readlane_b32 || /* GFX6 doesn't have v_readlane_b32_e64 */
+ instr->opcode == aco_opcode::v_readfirstlane_b32)) {
+ handle_vintrp_then_read_hazard(program, cur_block, &NOPs, 1, instr->operands[0]);
+ }
+
+ if (instr->opcode == aco_opcode::v_div_fmas_f32 || instr->opcode == aco_opcode::v_div_fmas_f64)
+ NOPs = MAX2(NOPs, ctx.valu_wr_vcc_then_div_fmas);
+ } else if (instr->isVMEM() || instr->isFlatOrGlobal() || instr->format == Format::SCRATCH) {
/* If the VALU writes the SGPR that is used by a VMEM, the user must add five wait states. */
- for (int pred_idx = new_idx - 1; pred_idx >= 0 && pred_idx >= new_idx - 5; pred_idx--) {
- aco_ptr<Instruction>& pred = new_instructions[pred_idx];
- if (!(pred->isVALU() && VALU_writes_sgpr(pred)))
- continue;
+ for (Operand op : instr->operands) {
+ if (!op.isConstant() && !op.isUndefined() && op.regClass().type() == RegType::sgpr)
+ handle_valu_then_read_hazard(program, cur_block, &NOPs, 5, op);
+ }
+ }
- for (const Definition& def : pred->definitions) {
- if (def.physReg() > 102)
- continue;
+ if (!instr->isSALU() && instr->format != Format::SMEM)
+ NOPs = MAX2(NOPs, ctx.set_vskip_mode_then_vector);
+
+ if (program->chip_class == GFX9) {
+ bool lds_scratch_global = (instr->format == Format::SCRATCH || instr->format == Format::GLOBAL) &&
+ static_cast<FLAT_instruction *>(instr.get())->lds;
+ if (instr->format == Format::VINTRP ||
+ instr->opcode == aco_opcode::ds_read_addtid_b32 ||
+ instr->opcode == aco_opcode::ds_write_addtid_b32 ||
+ instr->opcode == aco_opcode::buffer_store_lds_dword ||
+ lds_scratch_global) {
+ NOPs = MAX2(NOPs, ctx.salu_wr_m0_then_lds);
+ }
+ }
+
+ ctx.add_wait_states(NOPs + get_wait_states(instr));
+
+ // TODO: try to schedule the NOP-causing instruction up to reduce the number of stall cycles
+ if (NOPs) {
+ /* create NOP */
+ aco_ptr<SOPP_instruction> nop{create_instruction<SOPP_instruction>(aco_opcode::s_nop, Format::SOPP, 0, 0)};
+ nop->imm = NOPs - 1;
+ nop->block = -1;
+ new_instructions.emplace_back(std::move(nop));
+ }
+
+ /* update information to check for later hazards */
+ if ((ctx.smem_clause || ctx.smem_write) && (NOPs || instr->format != Format::SMEM)) {
+ ctx.smem_clause = false;
+ ctx.smem_write = false;
+
+ if (program->xnack_enabled) {
+ BITSET_ZERO(ctx.smem_clause_read_write);
+ BITSET_ZERO(ctx.smem_clause_write);
+ }
+ }
+
+ if (instr->format == Format::SMEM) {
+ if (instr->definitions.empty() || instr_info.is_atomic[(unsigned)instr->opcode]) {
+ ctx.smem_write = true;
+ } else {
+ ctx.smem_clause = true;
- if (instr->operands.size() > 1 &&
- regs_intersect(instr->operands[1].physReg(), instr->operands[1].size(),
- def.physReg(), def.size())) {
- return 5 + pred_idx - new_idx + 1;
+ if (program->xnack_enabled) {
+ for (Operand op : instr->operands) {
+ if (!op.isConstant()) {
+ set_bitset_range(ctx.smem_clause_read_write, op.physReg(), op.size());
+ }
}
- if (instr->operands.size() > 2 &&
- regs_intersect(instr->operands[2].physReg(), instr->operands[2].size(),
- def.physReg(), def.size())) {
- return 5 + pred_idx - new_idx + 1;
+ Definition def = instr->definitions[0];
+ set_bitset_range(ctx.smem_clause_read_write, def.physReg(), def.size());
+ set_bitset_range(ctx.smem_clause_write, def.physReg(), def.size());
+ }
+ }
+ } else if (instr->isVALU()) {
+ for (Definition def : instr->definitions) {
+ if (def.regClass().type() == RegType::sgpr) {
+ if (def.physReg() == vcc || def.physReg() == vcc_hi) {
+ ctx.valu_wr_vcc_then_vccz = 5;
+ ctx.valu_wr_vcc_then_div_fmas = 4;
+ }
+ if (def.physReg() == exec || def.physReg() == exec_hi) {
+ ctx.valu_wr_exec_then_execz = 5;
+ ctx.valu_wr_exec_then_dpp = 5;
}
}
}
+ } else if (instr->isSALU() && !instr->definitions.empty()) {
+ if (!instr->definitions.empty()) {
+ /* all other definitions should be SCC */
+ Definition def = instr->definitions[0];
+ if (def.physReg() == m0) {
+ ctx.salu_wr_m0_then_gds_msg_ttrace = 1;
+ ctx.salu_wr_m0_then_lds = 1;
+ ctx.salu_wr_m0_then_moverel = 1;
+ }
+ } else if (instr->opcode == aco_opcode::s_setreg_b32 || instr->opcode == aco_opcode::s_setreg_imm32_b32) {
+ SOPK_instruction *sopk = static_cast<SOPK_instruction *>(instr.get());
+ unsigned offset = (sopk->imm >> 6) & 0x1f;
+ unsigned size = ((sopk->imm >> 11) & 0x1f) + 1;
+ unsigned reg = sopk->imm & 0x3f;
+ ctx.setreg_then_getsetreg = 2;
+
+ if (reg == 1 && offset >= 28 && size > (28 - offset))
+ ctx.set_vskip_mode_then_vector = 2;
+ }
+ } else if (instr->isVMEM() || instr->isFlatOrGlobal() || instr->format == Format::SCRATCH) {
+ /* >64-bit MUBUF/MTBUF store with a constant in SOFFSET */
+ bool consider_buf = (instr->format == Format::MUBUF || instr->format == Format::MTBUF) &&
+ instr->operands.size() == 4 &&
+ instr->operands[3].size() > 2 &&
+ instr->operands[2].physReg() >= 128;
+ /* MIMG store with a 128-bit T# with more than two bits set in dmask (making it a >64-bit store) */
+ bool consider_mimg = instr->format == Format::MIMG &&
+ instr->operands[1].regClass().type() == RegType::vgpr &&
+ instr->operands[1].size() > 2 &&
+ instr->operands[0].size() == 4;
+ /* FLAT/GLOBAL/SCRATCH store with >64-bit data */
+ bool consider_flat = (instr->isFlatOrGlobal() || instr->format == Format::SCRATCH) &&
+ instr->operands.size() == 3 &&
+ instr->operands[2].size() > 2;
+ if (consider_buf || consider_mimg || consider_flat) {
+ PhysReg wrdata = instr->operands[consider_flat ? 2 : 3].physReg();
+ unsigned size = instr->operands[consider_flat ? 2 : 3].size();
+ for (unsigned i = 0; i < size; i++)
+ ctx.vmem_store_then_wr_data[(wrdata & 0xff) + i] = 1;
+ }
}
+}
- return 0;
+template <std::size_t N>
+bool check_written_regs(const aco_ptr<Instruction> &instr, const std::bitset<N> &check_regs)
+{
+ return std::any_of(instr->definitions.begin(), instr->definitions.end(), [&check_regs](const Definition &def) -> bool {
+ bool writes_any = false;
+ for (unsigned i = 0; i < def.size(); i++) {
+ unsigned def_reg = def.physReg() + i;
+ writes_any |= def_reg < check_regs.size() && check_regs[def_reg];
+ }
+ return writes_any;
+ });
}
-std::pair<int, int> handle_instruction_gfx10(NOP_ctx& ctx, aco_ptr<Instruction>& instr,
- std::vector<aco_ptr<Instruction>>& old_instructions,
- std::vector<aco_ptr<Instruction>>& new_instructions)
+template <std::size_t N>
+void mark_read_regs(const aco_ptr<Instruction> &instr, std::bitset<N> ®_reads)
{
- int new_idx = new_instructions.size();
- unsigned vNOPs = 0;
- unsigned sNOPs = 0;
-
- /* break off from prevous SMEM group ("clause" seems to mean something different in RDNA) if needed */
- if (instr->format == Format::SMEM)
- sNOPs = std::max(sNOPs, handle_SMEM_clause(instr, new_idx, new_instructions));
-
- /* handle EXEC/M0/SGPR write following a VMEM instruction without a VALU or "waitcnt vmcnt(0)" in-between */
- if (instr->isSALU() || instr->format == Format::SMEM) {
- if (!instr->definitions.empty() && ctx.last_VMEM_since_scalar_write != -1) {
- ctx.last_VMEM_since_scalar_write = -1;
- vNOPs = 1;
+ for (const Operand &op : instr->operands) {
+ for (unsigned i = 0; i < op.size(); i++) {
+ unsigned reg = op.physReg() + i;
+ if (reg < reg_reads.size())
+ reg_reads.set(reg);
}
- } else if (instr->isVMEM() || instr->isFlatOrGlobal()) {
- ctx.last_VMEM_since_scalar_write = new_idx;
- } else if (instr->opcode == aco_opcode::s_waitcnt) {
- uint16_t imm = static_cast<SOPP_instruction*>(instr.get())->imm;
- unsigned vmcnt = (imm & 0xF) | ((imm & (0x3 << 14)) >> 10);
- if (vmcnt == 0)
- ctx.last_VMEM_since_scalar_write = -1;
- } else if (instr->isVALU()) {
- ctx.last_VMEM_since_scalar_write = -1;
}
-
- return std::make_pair(sNOPs, vNOPs);
}
+bool VALU_writes_sgpr(aco_ptr<Instruction>& instr)
+{
+ if ((uint32_t) instr->format & (uint32_t) Format::VOPC)
+ return true;
+ if (instr->isVOP3() && instr->definitions.size() == 2)
+ return true;
+ if (instr->opcode == aco_opcode::v_readfirstlane_b32 ||
+ instr->opcode == aco_opcode::v_readlane_b32 ||
+ instr->opcode == aco_opcode::v_readlane_b32_e64)
+ return true;
+ return false;
+}
-void handle_block(NOP_ctx& ctx, Block& block)
+bool instr_writes_exec(const aco_ptr<Instruction>& instr)
{
- std::vector<aco_ptr<Instruction>> instructions;
- instructions.reserve(block.instructions.size());
- for (unsigned i = 0; i < block.instructions.size(); i++) {
- aco_ptr<Instruction>& instr = block.instructions[i];
- unsigned NOPs = handle_instruction(ctx, instr, block.instructions, instructions);
- if (NOPs) {
- // TODO: try to move the instruction down
- /* create NOP */
- aco_ptr<SOPP_instruction> nop{create_instruction<SOPP_instruction>(aco_opcode::s_nop, Format::SOPP, 0, 0)};
- nop->imm = NOPs - 1;
- nop->block = -1;
- instructions.emplace_back(std::move(nop));
- }
+ return std::any_of(instr->definitions.begin(), instr->definitions.end(), [](const Definition &def) -> bool {
+ return def.physReg() == exec_lo || def.physReg() == exec_hi;
+ });
+}
- instructions.emplace_back(std::move(instr));
- }
+bool instr_writes_sgpr(const aco_ptr<Instruction>& instr)
+{
+ return std::any_of(instr->definitions.begin(), instr->definitions.end(), [](const Definition &def) -> bool {
+ return def.getTemp().type() == RegType::sgpr;
+ });
+}
- ctx.VALU_wrvcc -= instructions.size();
- ctx.VALU_wrexec -= instructions.size();
- ctx.VALU_wrsgpr -= instructions.size();
- block.instructions = std::move(instructions);
+inline bool instr_is_branch(const aco_ptr<Instruction>& instr)
+{
+ return instr->opcode == aco_opcode::s_branch ||
+ instr->opcode == aco_opcode::s_cbranch_scc0 ||
+ instr->opcode == aco_opcode::s_cbranch_scc1 ||
+ instr->opcode == aco_opcode::s_cbranch_vccz ||
+ instr->opcode == aco_opcode::s_cbranch_vccnz ||
+ instr->opcode == aco_opcode::s_cbranch_execz ||
+ instr->opcode == aco_opcode::s_cbranch_execnz ||
+ instr->opcode == aco_opcode::s_cbranch_cdbgsys ||
+ instr->opcode == aco_opcode::s_cbranch_cdbguser ||
+ instr->opcode == aco_opcode::s_cbranch_cdbgsys_or_user ||
+ instr->opcode == aco_opcode::s_cbranch_cdbgsys_and_user ||
+ instr->opcode == aco_opcode::s_subvector_loop_begin ||
+ instr->opcode == aco_opcode::s_subvector_loop_end ||
+ instr->opcode == aco_opcode::s_setpc_b64 ||
+ instr->opcode == aco_opcode::s_swappc_b64 ||
+ instr->opcode == aco_opcode::s_getpc_b64 ||
+ instr->opcode == aco_opcode::s_call_b64;
}
-void handle_block_gfx10(NOP_ctx& ctx, Block& block)
+void handle_instruction_gfx10(Program *program, Block *cur_block, NOP_ctx_gfx10 &ctx,
+ aco_ptr<Instruction>& instr, std::vector<aco_ptr<Instruction>>& new_instructions)
{
- std::vector<aco_ptr<Instruction>> instructions;
- instructions.reserve(block.instructions.size());
- for (unsigned i = 0; i < block.instructions.size(); i++) {
- aco_ptr<Instruction>& instr = block.instructions[i];
- std::pair<int, int> NOPs = handle_instruction_gfx10(ctx, instr, block.instructions, instructions);
- for (int i = 0; i < NOPs.second; i++) {
- // TODO: try to move the instruction down
- /* create NOP */
- aco_ptr<VOP1_instruction> nop{create_instruction<VOP1_instruction>(aco_opcode::v_nop, Format::VOP1, 0, 0)};
- instructions.emplace_back(std::move(nop));
+ //TODO: s_dcache_inv needs to be in it's own group on GFX10
+
+ /* VMEMtoScalarWriteHazard
+ * Handle EXEC/M0/SGPR write following a VMEM instruction without a VALU or "waitcnt vmcnt(0)" in-between.
+ */
+ if (instr->isVMEM() || instr->format == Format::FLAT || instr->format == Format::GLOBAL ||
+ instr->format == Format::SCRATCH || instr->format == Format::DS) {
+ /* Remember all SGPRs that are read by the VMEM instruction */
+ mark_read_regs(instr, ctx.sgprs_read_by_VMEM);
+ ctx.sgprs_read_by_VMEM.set(exec);
+ if (program->wave_size == 64)
+ ctx.sgprs_read_by_VMEM.set(exec_hi);
+ } else if (instr->isSALU() || instr->format == Format::SMEM) {
+ if (instr->opcode == aco_opcode::s_waitcnt) {
+ /* Hazard is mitigated by "s_waitcnt vmcnt(0)" */
+ uint16_t imm = static_cast<SOPP_instruction*>(instr.get())->imm;
+ unsigned vmcnt = (imm & 0xF) | ((imm & (0x3 << 14)) >> 10);
+ if (vmcnt == 0)
+ ctx.sgprs_read_by_VMEM.reset();
+ } else if (instr->opcode == aco_opcode::s_waitcnt_depctr) {
+ /* Hazard is mitigated by a s_waitcnt_depctr with a magic imm */
+ const SOPP_instruction *sopp = static_cast<const SOPP_instruction *>(instr.get());
+ if (sopp->imm == 0xffe3)
+ ctx.sgprs_read_by_VMEM.reset();
}
- if (NOPs.first) {
- // TODO: try to move the instruction down
- /* create NOP */
- aco_ptr<SOPP_instruction> nop{create_instruction<SOPP_instruction>(aco_opcode::s_nop, Format::SOPP, 0, 0)};
- nop->imm = NOPs.first - 1;
- nop->block = -1;
- instructions.emplace_back(std::move(nop));
+
+ /* Check if SALU writes an SGPR that was previously read by the VALU */
+ if (check_written_regs(instr, ctx.sgprs_read_by_VMEM)) {
+ ctx.sgprs_read_by_VMEM.reset();
+
+ /* Insert s_waitcnt_depctr instruction with magic imm to mitigate the problem */
+ aco_ptr<SOPP_instruction> depctr{create_instruction<SOPP_instruction>(aco_opcode::s_waitcnt_depctr, Format::SOPP, 0, 0)};
+ depctr->imm = 0xffe3;
+ depctr->block = -1;
+ new_instructions.emplace_back(std::move(depctr));
+ }
+ } else if (instr->isVALU()) {
+ /* Hazard is mitigated by any VALU instruction */
+ ctx.sgprs_read_by_VMEM.reset();
+ }
+
+ /* VcmpxPermlaneHazard
+ * Handle any permlane following a VOPC instruction, insert v_mov between them.
+ */
+ if (instr->format == Format::VOPC) {
+ ctx.has_VOPC = true;
+ } else if (ctx.has_VOPC &&
+ (instr->opcode == aco_opcode::v_permlane16_b32 ||
+ instr->opcode == aco_opcode::v_permlanex16_b32)) {
+ ctx.has_VOPC = false;
+
+ /* v_nop would be discarded by SQ, so use v_mov with the first operand of the permlane */
+ aco_ptr<VOP1_instruction> v_mov{create_instruction<VOP1_instruction>(aco_opcode::v_mov_b32, Format::VOP1, 1, 1)};
+ v_mov->definitions[0] = Definition(instr->operands[0].physReg(), v1);
+ v_mov->operands[0] = Operand(instr->operands[0].physReg(), v1);
+ new_instructions.emplace_back(std::move(v_mov));
+ } else if (instr->isVALU() && instr->opcode != aco_opcode::v_nop) {
+ ctx.has_VOPC = false;
+ }
+
+ /* VcmpxExecWARHazard
+ * Handle any VALU instruction writing the exec mask after it was read by a non-VALU instruction.
+ */
+ if (!instr->isVALU() && instr->reads_exec()) {
+ ctx.has_nonVALU_exec_read = true;
+ } else if (instr->isVALU()) {
+ if (instr_writes_exec(instr)) {
+ ctx.has_nonVALU_exec_read = false;
+
+ /* Insert s_waitcnt_depctr instruction with magic imm to mitigate the problem */
+ aco_ptr<SOPP_instruction> depctr{create_instruction<SOPP_instruction>(aco_opcode::s_waitcnt_depctr, Format::SOPP, 0, 0)};
+ depctr->imm = 0xfffe;
+ depctr->block = -1;
+ new_instructions.emplace_back(std::move(depctr));
+ } else if (instr_writes_sgpr(instr)) {
+ /* Any VALU instruction that writes an SGPR mitigates the problem */
+ ctx.has_nonVALU_exec_read = false;
}
+ } else if (instr->opcode == aco_opcode::s_waitcnt_depctr) {
+ /* s_waitcnt_depctr can mitigate the problem if it has a magic imm */
+ const SOPP_instruction *sopp = static_cast<const SOPP_instruction *>(instr.get());
+ if ((sopp->imm & 0xfffe) == 0xfffe)
+ ctx.has_nonVALU_exec_read = false;
+ }
- instructions.emplace_back(std::move(instr));
+ /* SMEMtoVectorWriteHazard
+ * Handle any VALU instruction writing an SGPR after an SMEM reads it.
+ */
+ if (instr->format == Format::SMEM) {
+ /* Remember all SGPRs that are read by the SMEM instruction */
+ mark_read_regs(instr, ctx.sgprs_read_by_SMEM);
+ } else if (VALU_writes_sgpr(instr)) {
+ /* Check if VALU writes an SGPR that was previously read by SMEM */
+ if (check_written_regs(instr, ctx.sgprs_read_by_SMEM)) {
+ ctx.sgprs_read_by_SMEM.reset();
+
+ /* Insert s_mov to mitigate the problem */
+ aco_ptr<SOP1_instruction> s_mov{create_instruction<SOP1_instruction>(aco_opcode::s_mov_b32, Format::SOP1, 1, 1)};
+ s_mov->definitions[0] = Definition(sgpr_null, s1);
+ s_mov->operands[0] = Operand(0u);
+ new_instructions.emplace_back(std::move(s_mov));
+ }
+ } else if (instr->isSALU()) {
+ if (instr->format != Format::SOPP) {
+ /* SALU can mitigate the hazard */
+ ctx.sgprs_read_by_SMEM.reset();
+ } else {
+ /* Reducing lgkmcnt count to 0 always mitigates the hazard. */
+ const SOPP_instruction *sopp = static_cast<const SOPP_instruction *>(instr.get());
+ if (sopp->opcode == aco_opcode::s_waitcnt_lgkmcnt) {
+ if (sopp->imm == 0 && sopp->definitions[0].physReg() == sgpr_null)
+ ctx.sgprs_read_by_SMEM.reset();
+ } else if (sopp->opcode == aco_opcode::s_waitcnt) {
+ unsigned lgkm = (sopp->imm >> 8) & 0x3f;
+ if (lgkm == 0)
+ ctx.sgprs_read_by_SMEM.reset();
+ }
+ }
}
- block.instructions = std::move(instructions);
+ /* LdsBranchVmemWARHazard
+ * Handle VMEM/GLOBAL/SCRATCH->branch->DS and DS->branch->VMEM/GLOBAL/SCRATCH patterns.
+ */
+ if (instr->isVMEM() || instr->format == Format::GLOBAL || instr->format == Format::SCRATCH) {
+ ctx.has_VMEM = true;
+ ctx.has_branch_after_VMEM = false;
+ /* Mitigation for DS is needed only if there was already a branch after */
+ ctx.has_DS = ctx.has_branch_after_DS;
+ } else if (instr->format == Format::DS) {
+ ctx.has_DS = true;
+ ctx.has_branch_after_DS = false;
+ /* Mitigation for VMEM is needed only if there was already a branch after */
+ ctx.has_VMEM = ctx.has_branch_after_VMEM;
+ } else if (instr_is_branch(instr)) {
+ ctx.has_branch_after_VMEM = ctx.has_VMEM;
+ ctx.has_branch_after_DS = ctx.has_DS;
+ } else if (instr->opcode == aco_opcode::s_waitcnt_vscnt) {
+ /* Only s_waitcnt_vscnt can mitigate the hazard */
+ const SOPK_instruction *sopk = static_cast<const SOPK_instruction *>(instr.get());
+ if (sopk->definitions[0].physReg() == sgpr_null && sopk->imm == 0)
+ ctx.has_VMEM = ctx.has_branch_after_VMEM = ctx.has_DS = ctx.has_branch_after_DS = false;
+ }
+ if ((ctx.has_VMEM && ctx.has_branch_after_DS) || (ctx.has_DS && ctx.has_branch_after_VMEM)) {
+ ctx.has_VMEM = ctx.has_branch_after_VMEM = ctx.has_DS = ctx.has_branch_after_DS = false;
+
+ /* Insert s_waitcnt_vscnt to mitigate the problem */
+ aco_ptr<SOPK_instruction> wait{create_instruction<SOPK_instruction>(aco_opcode::s_waitcnt_vscnt, Format::SOPK, 0, 1)};
+ wait->definitions[0] = Definition(sgpr_null, s1);
+ wait->imm = 0;
+ new_instructions.emplace_back(std::move(wait));
+ }
}
-} /* end namespace */
+template <typename Ctx>
+using HandleInstr = void (*)(Program *, Block *block, Ctx&, aco_ptr<Instruction>&,
+ std::vector<aco_ptr<Instruction>>&);
+template <typename Ctx, HandleInstr<Ctx> Handle>
+void handle_block(Program *program, Ctx& ctx, Block& block)
+{
+ if (block.instructions.empty())
+ return;
-void insert_NOPs(Program* program)
+ std::vector<aco_ptr<Instruction>> old_instructions = std::move(block.instructions);
+
+ block.instructions.reserve(block.instructions.size());
+
+ for (aco_ptr<Instruction>& instr : old_instructions) {
+ Handle(program, &block, ctx, instr, block.instructions);
+ block.instructions.emplace_back(std::move(instr));
+ }
+}
+
+template <typename Ctx, HandleInstr<Ctx> Handle>
+void mitigate_hazards(Program *program)
{
- NOP_ctx ctx(program);
+ std::vector<Ctx> all_ctx(program->blocks.size());
+ std::stack<unsigned> loop_header_indices;
+
+ for (unsigned i = 0; i < program->blocks.size(); i++) {
+ Block& block = program->blocks[i];
+ Ctx &ctx = all_ctx[i];
+
+ if (block.kind & block_kind_loop_header) {
+ loop_header_indices.push(i);
+ } else if (block.kind & block_kind_loop_exit) {
+ /* Go through the whole loop again */
+ for (unsigned idx = loop_header_indices.top(); idx < i; idx++) {
+ Ctx loop_block_ctx;
+ for (unsigned b : program->blocks[idx].linear_preds)
+ loop_block_ctx.join(all_ctx[b]);
+
+ handle_block<Ctx, Handle>(program, loop_block_ctx, program->blocks[idx]);
+
+ /* We only need to continue if the loop header context changed */
+ if (idx == loop_header_indices.top() && loop_block_ctx == all_ctx[idx])
+ break;
- for (Block& block : program->blocks) {
- if (block.instructions.empty())
- continue;
+ all_ctx[idx] = loop_block_ctx;
+ }
- if (ctx.chip_class >= GFX10)
- handle_block_gfx10(ctx, block);
- else
- handle_block(ctx, block);
+ loop_header_indices.pop();
+ }
+
+ for (unsigned b : block.linear_preds)
+ ctx.join(all_ctx[b]);
+
+ handle_block<Ctx, Handle>(program, ctx, block);
}
}
+} /* end namespace */
+
+void insert_NOPs(Program* program)
+{
+ if (program->chip_class >= GFX10)
+ mitigate_hazards<NOP_ctx_gfx10, handle_instruction_gfx10>(program);
+ else
+ mitigate_hazards<NOP_ctx_gfx6, handle_instruction_gfx6>(program);
+}
+
}
projects / mesa.git / blobdiff