X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Famd%2Fcompiler%2Faco_optimizer.cpp;h=c1f0bc20f9e41a38acd841cda76952346c1d235c;hb=3af2b9e3de2ba9878c03b192d3b1574172d1923d;hp=211f347c0ab4222d2a3c697bb5e3e083a1cc07cd;hpb=1210e0bd6205c5f5365a29c91425dea3e49d89a7;p=mesa.git diff --git a/src/amd/compiler/aco_optimizer.cpp b/src/amd/compiler/aco_optimizer.cpp index 211f347c0ab..c1f0bc20f9e 100644 --- a/src/amd/compiler/aco_optimizer.cpp +++ b/src/amd/compiler/aco_optimizer.cpp @@ -52,7 +52,7 @@ namespace aco { struct mad_info { aco_ptr add_instr; uint32_t mul_temp_id; - uint32_t literal_idx; + uint16_t literal_idx; bool check_literal; mad_info(aco_ptr instr, uint32_t id) @@ -61,7 +61,7 @@ struct mad_info { enum Label { label_vec = 1 << 0, - label_constant = 1 << 1, + label_constant_32bit = 1 << 1, /* label_{abs,neg,mul,omod2,omod4,omod5,clamp} are used for both 16 and * 32-bit operations but this shouldn't cause any issues because we don't * look through any conversions */ @@ -83,7 +83,7 @@ enum Label { label_add_sub = 1 << 17, label_bitwise = 1 << 18, label_minmax = 1 << 19, - label_fcmp = 1 << 20, + label_vopc = 1 << 20, label_uniform_bool = 1 << 21, label_constant_64bit = 1 << 22, label_uniform_bitwise = 1 << 23, @@ -91,21 +91,22 @@ enum Label { label_vcc_hint = 1 << 25, label_scc_needed = 1 << 26, label_b2i = 1 << 27, + label_constant_16bit = 1 << 29, }; -static constexpr uint32_t instr_labels = label_vec | label_mul | label_mad | label_omod_success | label_clamp_success | - label_add_sub | label_bitwise | label_uniform_bitwise | label_minmax | label_fcmp; -static constexpr uint32_t temp_labels = label_abs | label_neg | label_temp | label_vcc | label_b2f | label_uniform_bool | +static constexpr uint64_t instr_labels = label_vec | label_mul | label_mad | label_omod_success | label_clamp_success | + label_add_sub | label_bitwise | label_uniform_bitwise | label_minmax | label_vopc; +static constexpr uint64_t temp_labels = label_abs | label_neg | label_temp | label_vcc | label_b2f | label_uniform_bool | label_omod2 | label_omod4 | label_omod5 | label_clamp | label_scc_invert | label_b2i; -static constexpr uint32_t val_labels = label_constant | label_constant_64bit | label_literal | label_mad; +static constexpr uint32_t val_labels = label_constant_32bit | label_constant_64bit | label_constant_16bit | label_literal; struct ssa_info { - uint32_t val; + uint64_t label; union { + uint32_t val; Temp temp; Instruction* instr; }; - uint32_t label; ssa_info() : label(0) {} @@ -115,15 +116,21 @@ struct ssa_info { * (indicating the defining instruction), there is no need to clear * any other instr labels. */ if (new_label & instr_labels) - label &= ~temp_labels; /* instr and temp alias */ + label &= ~(temp_labels | val_labels); /* instr, temp and val alias */ if (new_label & temp_labels) { label &= ~temp_labels; - label &= ~instr_labels; /* instr and temp alias */ + label &= ~(instr_labels | val_labels); /* instr, temp and val alias */ } - if (new_label & val_labels) + uint32_t const_labels = label_literal | label_constant_32bit | label_constant_64bit | label_constant_16bit; + if (new_label & const_labels) { + label &= ~val_labels | const_labels; + label &= ~(instr_labels | temp_labels); /* instr, temp and val alias */ + } else if (new_label & val_labels) { label &= ~val_labels; + label &= ~(instr_labels | temp_labels); /* instr, temp and val alias */ + } label |= new_label; } @@ -139,26 +146,85 @@ struct ssa_info { return label & label_vec; } - void set_constant(uint32_t constant) + void set_constant(chip_class chip, uint64_t constant) { - add_label(label_constant); + Operand op16((uint16_t)constant); + Operand op32((uint32_t)constant); + add_label(label_literal); val = constant; - } - bool is_constant() + if (chip >= GFX8 && !op16.isLiteral()) + add_label(label_constant_16bit); + + if (!op32.isLiteral() || ((uint32_t)constant == 0x3e22f983 && chip >= GFX8)) + add_label(label_constant_32bit); + + if (constant <= 64) { + add_label(label_constant_64bit); + } else if (constant >= 0xFFFFFFFFFFFFFFF0) { /* [-16 .. -1] */ + add_label(label_constant_64bit); + } else if (constant == 0x3FE0000000000000) { /* 0.5 */ + add_label(label_constant_64bit); + } else if (constant == 0xBFE0000000000000) { /* -0.5 */ + add_label(label_constant_64bit); + } else if (constant == 0x3FF0000000000000) { /* 1.0 */ + add_label(label_constant_64bit); + } else if (constant == 0xBFF0000000000000) { /* -1.0 */ + add_label(label_constant_64bit); + } else if (constant == 0x4000000000000000) { /* 2.0 */ + add_label(label_constant_64bit); + } else if (constant == 0xC000000000000000) { /* -2.0 */ + add_label(label_constant_64bit); + } else if (constant == 0x4010000000000000) { /* 4.0 */ + add_label(label_constant_64bit); + } else if (constant == 0xC010000000000000) { /* -4.0 */ + add_label(label_constant_64bit); + } + + if (label & label_constant_64bit) { + val = Operand(constant).constantValue(); + if (val != constant) + label &= ~(label_literal | label_constant_16bit | label_constant_32bit); + } + } + + bool is_constant(unsigned bits) { - return label & label_constant; + switch (bits) { + case 8: + return label & label_literal; + case 16: + return label & label_constant_16bit; + case 32: + return label & label_constant_32bit; + case 64: + return label & label_constant_64bit; + } + return false; } - void set_constant_64bit(uint32_t constant) + bool is_literal(unsigned bits) { - add_label(label_constant_64bit); - val = constant; + bool is_lit = label & label_literal; + switch (bits) { + case 8: + return false; + case 16: + return is_lit && ~(label & label_constant_16bit); + case 32: + return is_lit && ~(label & label_constant_32bit); + case 64: + return false; + } + return false; } - bool is_constant_64bit() + bool is_constant_or_literal(unsigned bits) { - return label & label_constant_64bit; + if (bits == 64) + return label & label_constant_64bit; + else + return label & label_literal; } void set_abs(Temp abs_temp) @@ -211,21 +277,10 @@ struct ssa_info { return label & label_temp; } - void set_literal(uint32_t lit) - { - add_label(label_literal); - val = lit; - } - - bool is_literal() - { - return label & label_literal; - } - void set_mad(Instruction* mad, uint32_t mad_info_idx) { add_label(label_mad); - val = mad_info_idx; + mad->pass_flags = mad_info_idx; instr = mad; } @@ -321,11 +376,6 @@ struct ssa_info { return label & label_vcc; } - bool is_constant_or_literal() - { - return is_constant() || is_literal(); - } - void set_b2f(Temp val) { add_label(label_b2f); @@ -380,15 +430,15 @@ struct ssa_info { return label & label_minmax; } - void set_fcmp(Instruction *fcmp_instr) + void set_vopc(Instruction *vopc_instr) { - add_label(label_fcmp); - instr = fcmp_instr; + add_label(label_vopc); + instr = vopc_instr; } - bool is_fcmp() + bool is_vopc() { - return label & label_fcmp; + return label & label_vopc; } void set_scc_needed() @@ -455,6 +505,18 @@ struct opt_ctx { std::vector uses; }; +struct CmpInfo { + aco_opcode ordered; + aco_opcode unordered; + aco_opcode ordered_swapped; + aco_opcode unordered_swapped; + aco_opcode inverse; + aco_opcode f32; + unsigned size; +}; + +ALWAYS_INLINE bool get_cmp_info(aco_opcode op, CmpInfo *info); + bool can_swap_operands(aco_ptr& instr) { if (instr->operands[0].isConstant() || @@ -462,35 +524,63 @@ bool can_swap_operands(aco_ptr& instr) return false; switch (instr->opcode) { + case aco_opcode::v_add_u32: + case aco_opcode::v_add_co_u32: + case aco_opcode::v_add_co_u32_e64: + case aco_opcode::v_add_i32: + case aco_opcode::v_add_f16: case aco_opcode::v_add_f32: + case aco_opcode::v_mul_f16: case aco_opcode::v_mul_f32: case aco_opcode::v_or_b32: case aco_opcode::v_and_b32: case aco_opcode::v_xor_b32: + case aco_opcode::v_max_f16: case aco_opcode::v_max_f32: + case aco_opcode::v_min_f16: case aco_opcode::v_min_f32: case aco_opcode::v_max_i32: case aco_opcode::v_min_i32: case aco_opcode::v_max_u32: case aco_opcode::v_min_u32: - case aco_opcode::v_cmp_eq_f32: - case aco_opcode::v_cmp_lg_f32: + case aco_opcode::v_max_i16: + case aco_opcode::v_min_i16: + case aco_opcode::v_max_u16: + case aco_opcode::v_min_u16: + case aco_opcode::v_max_i16_e64: + case aco_opcode::v_min_i16_e64: + case aco_opcode::v_max_u16_e64: + case aco_opcode::v_min_u16_e64: + return true; + case aco_opcode::v_sub_f16: + instr->opcode = aco_opcode::v_subrev_f16; return true; case aco_opcode::v_sub_f32: instr->opcode = aco_opcode::v_subrev_f32; return true; - case aco_opcode::v_cmp_lt_f32: - instr->opcode = aco_opcode::v_cmp_gt_f32; + case aco_opcode::v_sub_co_u32: + instr->opcode = aco_opcode::v_subrev_co_u32; return true; - case aco_opcode::v_cmp_ge_f32: - instr->opcode = aco_opcode::v_cmp_le_f32; + case aco_opcode::v_sub_u16: + instr->opcode = aco_opcode::v_subrev_u16; return true; - case aco_opcode::v_cmp_lt_i32: - instr->opcode = aco_opcode::v_cmp_gt_i32; + case aco_opcode::v_sub_u32: + instr->opcode = aco_opcode::v_subrev_u32; return true; - default: + default: { + CmpInfo info; + get_cmp_info(instr->opcode, &info); + if (info.ordered == instr->opcode) { + instr->opcode = info.ordered_swapped; + return true; + } + if (info.unordered == instr->opcode) { + instr->opcode = info.unordered_swapped; + return true; + } return false; } + } } bool can_use_VOP3(opt_ctx& ctx, const aco_ptr& instr) @@ -625,7 +715,7 @@ bool check_vop3_operands(opt_ctx& ctx, unsigned num_operands, Operand *operands) return true; } -bool parse_base_offset(opt_ctx &ctx, Instruction* instr, unsigned op_index, Temp *base, uint32_t *offset) +bool parse_base_offset(opt_ctx &ctx, Instruction* instr, unsigned op_index, Temp *base, uint32_t *offset, bool prevent_overflow) { Operand op = instr->operands[op_index]; @@ -647,6 +737,8 @@ bool parse_base_offset(opt_ctx &ctx, Instruction* instr, unsigned op_index, Temp default: return false; } + if (prevent_overflow && !add_instr->definitions[0].isNUW()) + return false; if (add_instr->usesModifiers()) return false; @@ -655,7 +747,7 @@ bool parse_base_offset(opt_ctx &ctx, Instruction* instr, unsigned op_index, Temp if (add_instr->operands[i].isConstant()) { *offset = add_instr->operands[i].constantValue(); } else if (add_instr->operands[i].isTemp() && - ctx.info[add_instr->operands[i].tempId()].is_constant_or_literal()) { + ctx.info[add_instr->operands[i].tempId()].is_constant_or_literal(32)) { *offset = ctx.info[add_instr->operands[i].tempId()].val; } else { continue; @@ -664,7 +756,7 @@ bool parse_base_offset(opt_ctx &ctx, Instruction* instr, unsigned op_index, Temp continue; uint32_t offset2 = 0; - if (parse_base_offset(ctx, add_instr, !i, base, &offset2)) { + if (parse_base_offset(ctx, add_instr, !i, base, &offset2, prevent_overflow)) { *offset += offset2; } else { *base = add_instr->operands[!i].getTemp(); @@ -687,11 +779,15 @@ unsigned get_operand_size(aco_ptr& instr, unsigned index) return 0; } -Operand get_constant_op(opt_ctx &ctx, uint32_t val, bool is64bit = false) +Operand get_constant_op(opt_ctx &ctx, ssa_info info, uint32_t bits) { + if (bits == 8) + return Operand((uint8_t)info.val); + if (bits == 16) + return Operand((uint16_t)info.val); // TODO: this functions shouldn't be needed if we store Operand instead of value. - Operand op(val, is64bit); - if (val == 0x3e22f983 && ctx.program->chip_class >= GFX8) + Operand op(info.val, bits == 64); + if (info.is_literal(32) && info.val == 0x3e22f983 && ctx.program->chip_class >= GFX8) op.setFixed(PhysReg{248}); /* 1/2 PI can be an inline constant on GFX8+ */ return op; } @@ -706,7 +802,7 @@ void label_instruction(opt_ctx &ctx, Block& block, aco_ptr& instr) if (instr->isSALU() || instr->isVALU() || instr->format == Format::PSEUDO) { ASSERTED bool all_const = false; for (Operand& op : instr->operands) - all_const = all_const && (!op.isTemp() || ctx.info[op.tempId()].is_constant_or_literal()); + all_const = all_const && (!op.isTemp() || ctx.info[op.tempId()].is_constant_or_literal(32)); perfwarn(all_const, "All instruction operands are constant", instr.get()); } @@ -727,11 +823,16 @@ void label_instruction(opt_ctx &ctx, Block& block, aco_ptr& instr) /* SALU / PSEUDO: propagate inline constants */ if (instr->isSALU() || instr->format == Format::PSEUDO) { - const bool is_subdword = std::any_of(instr->definitions.begin(), instr->definitions.end(), - [] (const Definition& def) { return def.regClass().is_subdword();}); - // TODO: optimize SGPR and constant propagation for subdword pseudo instructions on gfx9+ - if (is_subdword) - continue; + bool is_subdword = false; + // TODO: optimize SGPR propagation for subdword pseudo instructions on gfx9+ + if (instr->format == Format::PSEUDO) { + is_subdword = std::any_of(instr->definitions.begin(), instr->definitions.end(), + [] (const Definition& def) { return def.regClass().is_subdword();}); + is_subdword = is_subdword || std::any_of(instr->operands.begin(), instr->operands.end(), + [] (const Operand& op) { return op.hasRegClass() && op.regClass().is_subdword();}); + if (is_subdword && ctx.program->chip_class < GFX9) + continue; + } if (info.is_temp() && info.temp.type() == RegType::sgpr) { instr->operands[i].setTemp(info.temp); @@ -755,9 +856,10 @@ void label_instruction(opt_ctx &ctx, Block& block, aco_ptr& instr) break; } } - if ((info.is_constant() || info.is_constant_64bit() || (info.is_literal() && instr->format == Format::PSEUDO)) && + unsigned bits = get_operand_size(instr, i); + if ((info.is_constant(bits) || (!is_subdword && info.is_literal(bits) && instr->format == Format::PSEUDO)) && !instr->operands[i].isFixed() && alu_can_accept_constant(instr->opcode, i)) { - instr->operands[i] = get_constant_op(ctx, info.val, info.is_constant_64bit()); + instr->operands[i] = get_constant_op(ctx, info, bits); continue; } } @@ -800,8 +902,9 @@ void label_instruction(opt_ctx &ctx, Block& block, aco_ptr& instr) static_cast(instr.get())->neg[i] = true; continue; } - if ((info.is_constant() || info.is_constant_64bit()) && alu_can_accept_constant(instr->opcode, i)) { - Operand op = get_constant_op(ctx, info.val, info.is_constant_64bit()); + unsigned bits = get_operand_size(instr, i); + if (info.is_constant(bits) && alu_can_accept_constant(instr->opcode, i)) { + Operand op = get_constant_op(ctx, info, bits); perfwarn(instr->opcode == aco_opcode::v_cndmask_b32 && i == 2, "v_cndmask_b32 with a constant selector", instr.get()); if (i == 0 || instr->opcode == aco_opcode::v_readlane_b32 || instr->opcode == aco_opcode::v_writelane_b32) { instr->operands[i] = op; @@ -826,22 +929,33 @@ void label_instruction(opt_ctx &ctx, Block& block, aco_ptr& instr) while (info.is_temp()) info = ctx.info[info.temp.id()]; - if (mubuf->offen && i == 1 && info.is_constant_or_literal() && mubuf->offset + info.val < 4096) { + /* According to AMDGPUDAGToDAGISel::SelectMUBUFScratchOffen(), vaddr + * overflow for scratch accesses works only on GFX9+ and saddr overflow + * never works. Since swizzling is the only thing that separates + * scratch accesses and other accesses and swizzling changing how + * addressing works significantly, this probably applies to swizzled + * MUBUF accesses. */ + bool vaddr_prevent_overflow = mubuf->swizzled && ctx.program->chip_class < GFX9; + bool saddr_prevent_overflow = mubuf->swizzled; + + if (mubuf->offen && i == 1 && info.is_constant_or_literal(32) && mubuf->offset + info.val < 4096) { assert(!mubuf->idxen); instr->operands[1] = Operand(v1); mubuf->offset += info.val; mubuf->offen = false; continue; - } else if (i == 2 && info.is_constant_or_literal() && mubuf->offset + info.val < 4096) { + } else if (i == 2 && info.is_constant_or_literal(32) && mubuf->offset + info.val < 4096) { instr->operands[2] = Operand((uint32_t) 0); mubuf->offset += info.val; continue; - } else if (mubuf->offen && i == 1 && parse_base_offset(ctx, instr.get(), i, &base, &offset) && base.regClass() == v1 && mubuf->offset + offset < 4096) { + } else if (mubuf->offen && i == 1 && parse_base_offset(ctx, instr.get(), i, &base, &offset, vaddr_prevent_overflow) && + base.regClass() == v1 && mubuf->offset + offset < 4096) { assert(!mubuf->idxen); instr->operands[1].setTemp(base); mubuf->offset += offset; continue; - } else if (i == 2 && parse_base_offset(ctx, instr.get(), i, &base, &offset) && base.regClass() == s1 && mubuf->offset + offset < 4096) { + } else if (i == 2 && parse_base_offset(ctx, instr.get(), i, &base, &offset, saddr_prevent_overflow) && + base.regClass() == s1 && mubuf->offset + offset < 4096) { instr->operands[i].setTemp(base); mubuf->offset += offset; continue; @@ -856,7 +970,7 @@ void label_instruction(opt_ctx &ctx, Block& block, aco_ptr& instr) uint32_t offset; bool has_usable_ds_offset = ctx.program->chip_class >= GFX7; if (has_usable_ds_offset && - i == 0 && parse_base_offset(ctx, instr.get(), i, &base, &offset) && + i == 0 && parse_base_offset(ctx, instr.get(), i, &base, &offset, false) && base.regClass() == instr->operands[i].regClass() && instr->opcode != aco_opcode::ds_swizzle_b32) { if (instr->opcode == aco_opcode::ds_write2_b32 || instr->opcode == aco_opcode::ds_read2_b32 || @@ -886,16 +1000,17 @@ void label_instruction(opt_ctx &ctx, Block& block, aco_ptr& instr) SMEM_instruction *smem = static_cast(instr.get()); Temp base; uint32_t offset; - if (i == 1 && info.is_constant_or_literal() && + bool prevent_overflow = smem->operands[0].size() > 2 || smem->prevent_overflow; + if (i == 1 && info.is_constant_or_literal(32) && ((ctx.program->chip_class == GFX6 && info.val <= 0x3FF) || (ctx.program->chip_class == GFX7 && info.val <= 0xFFFFFFFF) || (ctx.program->chip_class >= GFX8 && info.val <= 0xFFFFF))) { instr->operands[i] = Operand(info.val); continue; - } else if (i == 1 && parse_base_offset(ctx, instr.get(), i, &base, &offset) && base.regClass() == s1 && offset <= 0xFFFFF && ctx.program->chip_class >= GFX9) { + } else if (i == 1 && parse_base_offset(ctx, instr.get(), i, &base, &offset, prevent_overflow) && base.regClass() == s1 && offset <= 0xFFFFF && ctx.program->chip_class >= GFX9) { bool soe = smem->operands.size() >= (!smem->definitions.empty() ? 3 : 4); if (soe && - (!ctx.info[smem->operands.back().tempId()].is_constant_or_literal() || + (!ctx.info[smem->operands.back().tempId()].is_constant_or_literal(32) || ctx.info[smem->operands.back().tempId()].val != 0)) { continue; } @@ -911,8 +1026,7 @@ void label_instruction(opt_ctx &ctx, Block& block, aco_ptr& instr) new_instr->operands.back() = Operand(base); if (!smem->definitions.empty()) new_instr->definitions[0] = smem->definitions[0]; - new_instr->can_reorder = smem->can_reorder; - new_instr->barrier = smem->barrier; + new_instr->sync = smem->sync; new_instr->glc = smem->glc; new_instr->dlc = smem->dlc; new_instr->nv = smem->nv; @@ -937,6 +1051,11 @@ void label_instruction(opt_ctx &ctx, Block& block, aco_ptr& instr) if (instr->definitions.empty()) return; + if ((uint16_t) instr->format & (uint16_t) Format::VOPC) { + ctx.info[instr->definitions[0].tempId()].set_vopc(instr.get()); + return; + } + switch (instr->opcode) { case aco_opcode::p_create_vector: { bool copy_prop = instr->operands.size() == 1 && instr->operands[0].isTemp() && @@ -976,8 +1095,20 @@ void label_instruction(opt_ctx &ctx, Block& block, aco_ptr& instr) break; } case aco_opcode::p_split_vector: { - if (!ctx.info[instr->operands[0].tempId()].is_vec()) + ssa_info& info = ctx.info[instr->operands[0].tempId()]; + + if (info.is_constant_or_literal(32)) { + uint32_t val = info.val; + for (Definition def : instr->definitions) { + uint32_t mask = u_bit_consecutive(0, def.bytes() * 8u); + ctx.info[def.tempId()].set_constant(ctx.program->chip_class, val & mask); + val >>= def.bytes() * 8u; + } break; + } else if (!info.is_vec()) { + break; + } + Instruction* vec = ctx.info[instr->operands[0].tempId()].instr; unsigned split_offset = 0; unsigned vec_offset = 0; @@ -991,12 +1122,7 @@ void label_instruction(opt_ctx &ctx, Block& block, aco_ptr& instr) Operand vec_op = vec->operands[vec_index]; if (vec_op.isConstant()) { - if (vec_op.isLiteral()) - ctx.info[instr->definitions[i].tempId()].set_literal(vec_op.constantValue()); - else if (vec_op.size() == 1) - ctx.info[instr->definitions[i].tempId()].set_constant(vec_op.constantValue()); - else if (vec_op.size() == 2) - ctx.info[instr->definitions[i].tempId()].set_constant_64bit(vec_op.constantValue()); + ctx.info[instr->definitions[i].tempId()].set_constant(ctx.program->chip_class, vec_op.constantValue64()); } else if (vec_op.isUndefined()) { ctx.info[instr->definitions[i].tempId()].set_undefined(); } else { @@ -1007,13 +1133,20 @@ void label_instruction(opt_ctx &ctx, Block& block, aco_ptr& instr) break; } case aco_opcode::p_extract_vector: { /* mov */ - if (!ctx.info[instr->operands[0].tempId()].is_vec()) + ssa_info& info = ctx.info[instr->operands[0].tempId()]; + const unsigned index = instr->operands[1].constantValue(); + const unsigned dst_offset = index * instr->definitions[0].bytes(); + + if (info.is_constant_or_literal(32)) { + uint32_t mask = u_bit_consecutive(0, instr->definitions[0].bytes() * 8u); + ctx.info[instr->definitions[0].tempId()].set_constant(ctx.program->chip_class, (info.val >> (dst_offset * 8u)) & mask); + break; + } else if (!info.is_vec()) { break; + } /* check if we index directly into a vector element */ - Instruction* vec = ctx.info[instr->operands[0].tempId()].instr; - const unsigned index = instr->operands[1].constantValue(); - const unsigned dst_offset = index * instr->definitions[0].bytes(); + Instruction* vec = info.instr; unsigned offset = 0; for (const Operand& op : vec->operands) { @@ -1030,12 +1163,7 @@ void label_instruction(opt_ctx &ctx, Block& block, aco_ptr& instr) instr->operands[0] = op; if (op.isConstant()) { - if (op.isLiteral()) - ctx.info[instr->definitions[0].tempId()].set_literal(op.constantValue()); - else if (op.size() == 1) - ctx.info[instr->definitions[0].tempId()].set_constant(op.constantValue()); - else if (op.size() == 2) - ctx.info[instr->definitions[0].tempId()].set_constant_64bit(op.constantValue()); + ctx.info[instr->definitions[0].tempId()].set_constant(ctx.program->chip_class, op.constantValue64()); } else if (op.isUndefined()) { ctx.info[instr->definitions[0].tempId()].set_undefined(); } else { @@ -1055,12 +1183,7 @@ void label_instruction(opt_ctx &ctx, Block& block, aco_ptr& instr) } else if (instr->usesModifiers()) { // TODO } else if (instr->operands[0].isConstant()) { - if (instr->operands[0].isLiteral()) - ctx.info[instr->definitions[0].tempId()].set_literal(instr->operands[0].constantValue()); - else if (instr->operands[0].size() == 1) - ctx.info[instr->definitions[0].tempId()].set_constant(instr->operands[0].constantValue()); - else if (instr->operands[0].size() == 2) - ctx.info[instr->definitions[0].tempId()].set_constant_64bit(instr->operands[0].constantValue()); + ctx.info[instr->definitions[0].tempId()].set_constant(ctx.program->chip_class, instr->operands[0].constantValue64()); } else if (instr->operands[0].isTemp()) { ctx.info[instr->definitions[0].tempId()].set_temp(instr->operands[0].getTemp()); } else { @@ -1069,25 +1192,19 @@ void label_instruction(opt_ctx &ctx, Block& block, aco_ptr& instr) break; case aco_opcode::p_is_helper: if (!ctx.program->needs_wqm) - ctx.info[instr->definitions[0].tempId()].set_constant(0u); + ctx.info[instr->definitions[0].tempId()].set_constant(ctx.program->chip_class, 0u); break; case aco_opcode::s_movk_i32: { uint32_t v = static_cast(instr.get())->imm; v = v & 0x8000 ? (v | 0xffff0000) : v; - if (v <= 64 || v >= 0xfffffff0) - ctx.info[instr->definitions[0].tempId()].set_constant(v); - else - ctx.info[instr->definitions[0].tempId()].set_literal(v); + ctx.info[instr->definitions[0].tempId()].set_constant(ctx.program->chip_class, v); break; } case aco_opcode::v_bfrev_b32: case aco_opcode::s_brev_b32: { if (instr->operands[0].isConstant()) { uint32_t v = util_bitreverse(instr->operands[0].constantValue()); - if (v <= 64 || v >= 0xfffffff0) - ctx.info[instr->definitions[0].tempId()].set_constant(v); - else - ctx.info[instr->definitions[0].tempId()].set_literal(v); + ctx.info[instr->definitions[0].tempId()].set_constant(ctx.program->chip_class, v); } break; } @@ -1096,10 +1213,7 @@ void label_instruction(opt_ctx &ctx, Block& block, aco_ptr& instr) unsigned size = instr->operands[0].constantValue() & 0x1f; unsigned start = instr->operands[1].constantValue() & 0x1f; uint32_t v = ((1u << size) - 1u) << start; - if (v <= 64 || v >= 0xfffffff0) - ctx.info[instr->definitions[0].tempId()].set_constant(v); - else - ctx.info[instr->definitions[0].tempId()].set_literal(v); + ctx.info[instr->definitions[0].tempId()].set_constant(ctx.program->chip_class, v); } break; } @@ -1259,6 +1373,14 @@ void label_instruction(opt_ctx &ctx, Block& block, aco_ptr& instr) ctx.info[instr->definitions[1].tempId()].set_temp(ctx.info[instr->operands[0].tempId()].instr->definitions[1].getTemp()); ctx.info[instr->definitions[0].tempId()].set_uniform_bool(ctx.info[instr->operands[0].tempId()].instr->definitions[1].getTemp()); break; + } else if (ctx.info[instr->operands[0].tempId()].is_vopc()) { + Instruction* vopc_instr = ctx.info[instr->operands[0].tempId()].instr; + /* Remove superfluous s_and when the VOPC instruction uses the same exec and thus already produces the same result */ + if (vopc_instr->pass_flags == instr->pass_flags) { + assert(instr->pass_flags > 0); + ctx.info[instr->definitions[0].tempId()].set_temp(vopc_instr->definitions[0].getTemp()); + break; + } } } /* fallthrough */ @@ -1291,22 +1413,6 @@ void label_instruction(opt_ctx &ctx, Block& block, aco_ptr& instr) case aco_opcode::v_max_i16: ctx.info[instr->definitions[0].tempId()].set_minmax(instr.get()); break; - case aco_opcode::v_cmp_lt_f32: - case aco_opcode::v_cmp_eq_f32: - case aco_opcode::v_cmp_le_f32: - case aco_opcode::v_cmp_gt_f32: - case aco_opcode::v_cmp_lg_f32: - case aco_opcode::v_cmp_ge_f32: - case aco_opcode::v_cmp_o_f32: - case aco_opcode::v_cmp_u_f32: - case aco_opcode::v_cmp_nge_f32: - case aco_opcode::v_cmp_nlg_f32: - case aco_opcode::v_cmp_ngt_f32: - case aco_opcode::v_cmp_nle_f32: - case aco_opcode::v_cmp_neq_f32: - case aco_opcode::v_cmp_nlt_f32: - ctx.info[instr->definitions[0].tempId()].set_fcmp(instr.get()); - break; case aco_opcode::s_cselect_b64: case aco_opcode::s_cselect_b32: if (instr->operands[0].constantEquals((unsigned) -1) && @@ -1331,24 +1437,51 @@ void label_instruction(opt_ctx &ctx, Block& block, aco_ptr& instr) } } -ALWAYS_INLINE bool get_cmp_info(aco_opcode op, aco_opcode *ordered, aco_opcode *unordered, aco_opcode *inverse) +ALWAYS_INLINE bool get_cmp_info(aco_opcode op, CmpInfo *info) { - *ordered = *unordered = op; + info->ordered = aco_opcode::num_opcodes; + info->unordered = aco_opcode::num_opcodes; + info->ordered_swapped = aco_opcode::num_opcodes; + info->unordered_swapped = aco_opcode::num_opcodes; switch (op) { - #define CMP(ord, unord) \ - case aco_opcode::v_cmp_##ord##_f32:\ - case aco_opcode::v_cmp_n##unord##_f32:\ - *ordered = aco_opcode::v_cmp_##ord##_f32;\ - *unordered = aco_opcode::v_cmp_n##unord##_f32;\ - *inverse = op == aco_opcode::v_cmp_n##unord##_f32 ? aco_opcode::v_cmp_##unord##_f32 : aco_opcode::v_cmp_n##ord##_f32;\ + #define CMP2(ord, unord, ord_swap, unord_swap, sz) \ + case aco_opcode::v_cmp_##ord##_f##sz:\ + case aco_opcode::v_cmp_n##unord##_f##sz:\ + info->ordered = aco_opcode::v_cmp_##ord##_f##sz;\ + info->unordered = aco_opcode::v_cmp_n##unord##_f##sz;\ + info->ordered_swapped = aco_opcode::v_cmp_##ord_swap##_f##sz;\ + info->unordered_swapped = aco_opcode::v_cmp_n##unord_swap##_f##sz;\ + info->inverse = op == aco_opcode::v_cmp_n##unord##_f##sz ? aco_opcode::v_cmp_##unord##_f##sz : aco_opcode::v_cmp_n##ord##_f##sz;\ + info->f32 = op == aco_opcode::v_cmp_##ord##_f##sz ? aco_opcode::v_cmp_##ord##_f32 : aco_opcode::v_cmp_n##unord##_f32;\ + info->size = sz;\ return true; - CMP(lt, /*n*/ge) - CMP(eq, /*n*/lg) - CMP(le, /*n*/gt) - CMP(gt, /*n*/le) - CMP(lg, /*n*/eq) - CMP(ge, /*n*/lt) + #define CMP(ord, unord, ord_swap, unord_swap) \ + CMP2(ord, unord, ord_swap, unord_swap, 16)\ + CMP2(ord, unord, ord_swap, unord_swap, 32)\ + CMP2(ord, unord, ord_swap, unord_swap, 64) + CMP(lt, /*n*/ge, gt, /*n*/le) + CMP(eq, /*n*/lg, eq, /*n*/lg) + CMP(le, /*n*/gt, ge, /*n*/lt) + CMP(gt, /*n*/le, lt, /*n*/le) + CMP(lg, /*n*/eq, lg, /*n*/eq) + CMP(ge, /*n*/lt, le, /*n*/gt) #undef CMP + #undef CMP2 + #define ORD_TEST(sz) \ + case aco_opcode::v_cmp_u_f##sz:\ + info->f32 = aco_opcode::v_cmp_u_f32;\ + info->inverse = aco_opcode::v_cmp_o_f##sz;\ + info->size = sz;\ + return true;\ + case aco_opcode::v_cmp_o_f##sz:\ + info->f32 = aco_opcode::v_cmp_o_f32;\ + info->inverse = aco_opcode::v_cmp_u_f##sz;\ + info->size = sz;\ + return true; + ORD_TEST(16) + ORD_TEST(32) + ORD_TEST(64) + #undef ORD_TEST default: return false; } @@ -1356,26 +1489,38 @@ ALWAYS_INLINE bool get_cmp_info(aco_opcode op, aco_opcode *ordered, aco_opcode * aco_opcode get_ordered(aco_opcode op) { - aco_opcode ordered, unordered, inverse; - return get_cmp_info(op, &ordered, &unordered, &inverse) ? ordered : aco_opcode::num_opcodes; + CmpInfo info; + return get_cmp_info(op, &info) ? info.ordered : aco_opcode::num_opcodes; } aco_opcode get_unordered(aco_opcode op) { - aco_opcode ordered, unordered, inverse; - return get_cmp_info(op, &ordered, &unordered, &inverse) ? unordered : aco_opcode::num_opcodes; + CmpInfo info; + return get_cmp_info(op, &info) ? info.unordered : aco_opcode::num_opcodes; } aco_opcode get_inverse(aco_opcode op) { - aco_opcode ordered, unordered, inverse; - return get_cmp_info(op, &ordered, &unordered, &inverse) ? inverse : aco_opcode::num_opcodes; + CmpInfo info; + return get_cmp_info(op, &info) ? info.inverse : aco_opcode::num_opcodes; +} + +aco_opcode get_f32_cmp(aco_opcode op) +{ + CmpInfo info; + return get_cmp_info(op, &info) ? info.f32 : aco_opcode::num_opcodes; +} + +unsigned get_cmp_bitsize(aco_opcode op) +{ + CmpInfo info; + return get_cmp_info(op, &info) ? info.size : 0; } bool is_cmp(aco_opcode op) { - aco_opcode ordered, unordered, inverse; - return get_cmp_info(op, &ordered, &unordered, &inverse); + CmpInfo info; + return get_cmp_info(op, &info) && info.ordered != aco_opcode::num_opcodes; } unsigned original_temp_id(opt_ctx &ctx, Temp tmp) @@ -1431,14 +1576,18 @@ bool combine_ordering_test(opt_ctx &ctx, aco_ptr& instr) Instruction *op_instr[2]; Temp op[2]; + unsigned bitsize = 0; for (unsigned i = 0; i < 2; i++) { op_instr[i] = follow_operand(ctx, instr->operands[i], true); if (!op_instr[i]) return false; aco_opcode expected_cmp = is_or ? aco_opcode::v_cmp_neq_f32 : aco_opcode::v_cmp_eq_f32; + unsigned op_bitsize = get_cmp_bitsize(op_instr[i]->opcode); - if (op_instr[i]->opcode != expected_cmp) + if (get_f32_cmp(op_instr[i]->opcode) != expected_cmp) + return false; + if (bitsize && op_bitsize != bitsize) return false; if (!op_instr[i]->operands[0].isTemp() || !op_instr[i]->operands[1].isTemp()) return false; @@ -1458,6 +1607,7 @@ bool combine_ordering_test(opt_ctx &ctx, aco_ptr& instr) return false; op[i] = op1; + bitsize = op_bitsize; } if (op[1].type() == RegType::sgpr) @@ -1471,7 +1621,18 @@ bool combine_ordering_test(opt_ctx &ctx, aco_ptr& instr) decrease_uses(ctx, op_instr[0]); decrease_uses(ctx, op_instr[1]); - aco_opcode new_op = is_or ? aco_opcode::v_cmp_u_f32 : aco_opcode::v_cmp_o_f32; + aco_opcode new_op = aco_opcode::num_opcodes; + switch (bitsize) { + case 16: + new_op = is_or ? aco_opcode::v_cmp_u_f16 : aco_opcode::v_cmp_o_f16; + break; + case 32: + new_op = is_or ? aco_opcode::v_cmp_u_f32 : aco_opcode::v_cmp_o_f32; + break; + case 64: + new_op = is_or ? aco_opcode::v_cmp_u_f64 : aco_opcode::v_cmp_o_f64; + break; + } Instruction *new_instr; if (neg[0] || neg[1] || abs[0] || abs[1] || opsel || num_sgprs > 1) { VOP3A_instruction *vop3 = create_instruction(new_op, asVOP3(Format::VOPC), 2, 1); @@ -1489,7 +1650,7 @@ bool combine_ordering_test(opt_ctx &ctx, aco_ptr& instr) new_instr->definitions[0] = instr->definitions[0]; ctx.info[instr->definitions[0].tempId()].label = 0; - ctx.info[instr->definitions[0].tempId()].set_fcmp(new_instr); + ctx.info[instr->definitions[0].tempId()].set_vopc(new_instr); instr.reset(new_instr); @@ -1513,12 +1674,12 @@ bool combine_comparison_ordering(opt_ctx &ctx, aco_ptr& instr) if (!nan_test || !cmp) return false; - if (cmp->opcode == expected_nan_test) + if (get_f32_cmp(cmp->opcode) == expected_nan_test) std::swap(nan_test, cmp); - else if (nan_test->opcode != expected_nan_test) + else if (get_f32_cmp(nan_test->opcode) != expected_nan_test) return false; - if (!is_cmp(cmp->opcode)) + if (!is_cmp(cmp->opcode) || get_cmp_bitsize(cmp->opcode) != get_cmp_bitsize(nan_test->opcode)) return false; if (!nan_test->operands[0].isTemp() || !nan_test->operands[1].isTemp()) @@ -1559,7 +1720,7 @@ bool combine_comparison_ordering(opt_ctx &ctx, aco_ptr& instr) new_instr->definitions[0] = instr->definitions[0]; ctx.info[instr->definitions[0].tempId()].label = 0; - ctx.info[instr->definitions[0].tempId()].set_fcmp(new_instr); + ctx.info[instr->definitions[0].tempId()].set_vopc(new_instr); instr.reset(new_instr); @@ -1584,12 +1745,12 @@ bool combine_constant_comparison_ordering(opt_ctx &ctx, aco_ptr& in return false; aco_opcode expected_nan_test = is_or ? aco_opcode::v_cmp_neq_f32 : aco_opcode::v_cmp_eq_f32; - if (cmp->opcode == expected_nan_test) + if (get_f32_cmp(cmp->opcode) == expected_nan_test) std::swap(nan_test, cmp); - else if (nan_test->opcode != expected_nan_test) + else if (get_f32_cmp(nan_test->opcode) != expected_nan_test) return false; - if (!is_cmp(cmp->opcode)) + if (!is_cmp(cmp->opcode) || get_cmp_bitsize(cmp->opcode) != get_cmp_bitsize(nan_test->opcode)) return false; if (!nan_test->operands[0].isTemp() || !nan_test->operands[1].isTemp()) @@ -1624,7 +1785,7 @@ bool combine_constant_comparison_ordering(opt_ctx &ctx, aco_ptr& in } else if (cmp->operands[constant_operand].isTemp()) { Temp tmp = cmp->operands[constant_operand].getTemp(); unsigned id = original_temp_id(ctx, tmp); - if (!ctx.info[id].is_constant() && !ctx.info[id].is_literal()) + if (!ctx.info[id].is_constant_or_literal(32)) return false; constant = ctx.info[id].val; } else { @@ -1662,7 +1823,7 @@ bool combine_constant_comparison_ordering(opt_ctx &ctx, aco_ptr& in new_instr->definitions[0] = instr->definitions[0]; ctx.info[instr->definitions[0].tempId()].label = 0; - ctx.info[instr->definitions[0].tempId()].set_fcmp(new_instr); + ctx.info[instr->definitions[0].tempId()].set_vopc(new_instr); instr.reset(new_instr); @@ -1711,7 +1872,7 @@ bool combine_inverse_comparison(opt_ctx &ctx, aco_ptr& instr) new_instr->definitions[0] = instr->definitions[0]; ctx.info[instr->definitions[0].tempId()].label = 0; - ctx.info[instr->definitions[0].tempId()].set_fcmp(new_instr); + ctx.info[instr->definitions[0].tempId()].set_vopc(new_instr); instr.reset(new_instr); @@ -1809,7 +1970,7 @@ void create_vop3_for_op3(opt_ctx& ctx, aco_opcode opcode, aco_ptr& bool combine_three_valu_op(opt_ctx& ctx, aco_ptr& instr, aco_opcode op2, aco_opcode new_op, const char *shuffle, uint8_t ops) { - uint32_t omod_clamp = ctx.info[instr->definitions[0].tempId()].label & + uint64_t omod_clamp = ctx.info[instr->definitions[0].tempId()].label & (label_omod_success | label_clamp_success); for (unsigned swap = 0; swap < 2; swap++) { @@ -1840,7 +2001,7 @@ bool combine_minmax(opt_ctx& ctx, aco_ptr& instr, aco_opcode opposi if (combine_three_valu_op(ctx, instr, instr->opcode, minmax3, "012", 1 | 2)) return true; - uint32_t omod_clamp = ctx.info[instr->definitions[0].tempId()].label & + uint64_t omod_clamp = ctx.info[instr->definitions[0].tempId()].label & (label_omod_success | label_clamp_success); /* min(-max(a, b), c) -> min3(-a, -b, c) * @@ -2094,7 +2255,7 @@ bool combine_clamp(opt_ctx& ctx, aco_ptr& instr, else return false; - uint32_t omod_clamp = ctx.info[instr->definitions[0].tempId()].label & + uint64_t omod_clamp = ctx.info[instr->definitions[0].tempId()].label & (label_omod_success | label_clamp_success); for (unsigned swap = 0; swap < 2; swap++) { @@ -2110,7 +2271,7 @@ bool combine_clamp(opt_ctx& ctx, aco_ptr& instr, uint32_t val; if (operands[i].isConstant()) { val = operands[i].constantValue(); - } else if (operands[i].isTemp() && ctx.info[operands[i].tempId()].is_constant_or_literal()) { + } else if (operands[i].isTemp() && ctx.info[operands[i].tempId()].is_constant_or_literal(32)) { val = ctx.info[operands[i].tempId()].val; } else { continue; @@ -2289,7 +2450,7 @@ bool apply_omod_clamp(opt_ctx &ctx, Block& block, aco_ptr& instr) /* omod was successfully applied */ /* if the omod instruction is v_mad, we also have to change the original add */ if (ctx.info[instr->operands[idx].tempId()].is_mad()) { - Instruction* add_instr = ctx.mad_infos[ctx.info[instr->operands[idx].tempId()].val].add_instr.get(); + Instruction* add_instr = ctx.mad_infos[ctx.info[instr->operands[idx].tempId()].instr->pass_flags].add_instr.get(); if (ctx.info[instr->definitions[0].tempId()].is_clamp()) static_cast(add_instr)->clamp = true; add_instr->definitions[0] = instr->definitions[0]; @@ -2335,7 +2496,7 @@ bool apply_omod_clamp(opt_ctx &ctx, Block& block, aco_ptr& instr) /* clamp was successfully applied */ /* if the clamp instruction is v_mad, we also have to change the original add */ if (ctx.info[instr->operands[idx].tempId()].is_mad()) { - Instruction* add_instr = ctx.mad_infos[ctx.info[instr->operands[idx].tempId()].val].add_instr.get(); + Instruction* add_instr = ctx.mad_infos[ctx.info[instr->operands[idx].tempId()].instr->pass_flags].add_instr.get(); add_instr->definitions[0] = instr->definitions[0]; } Instruction* clamp_instr = ctx.info[instr->operands[idx].tempId()].instr; @@ -2757,7 +2918,7 @@ void select_instruction(opt_ctx &ctx, aco_ptr& instr) mad_info* mad_info = NULL; if (!instr->definitions.empty() && ctx.info[instr->definitions[0].tempId()].is_mad()) { - mad_info = &ctx.mad_infos[ctx.info[instr->definitions[0].tempId()].val]; + mad_info = &ctx.mad_infos[ctx.info[instr->definitions[0].tempId()].instr->pass_flags]; /* re-check mad instructions */ if (ctx.uses[mad_info->mul_temp_id]) { ctx.uses[mad_info->mul_temp_id]++; @@ -2786,9 +2947,10 @@ void select_instruction(opt_ctx &ctx, aco_ptr& instr) } if (!instr->operands[i].isTemp()) continue; + unsigned bits = get_operand_size(instr, i); /* if one of the operands is sgpr, we cannot add a literal somewhere else on pre-GFX10 or operands other than the 1st */ if (instr->operands[i].getTemp().type() == RegType::sgpr && (i > 0 || ctx.program->chip_class < GFX10)) { - if (!sgpr_used && ctx.info[instr->operands[i].tempId()].is_literal()) { + if (!sgpr_used && ctx.info[instr->operands[i].tempId()].is_literal(bits)) { literal_uses = ctx.uses[instr->operands[i].tempId()]; literal_idx = i; } else { @@ -2797,7 +2959,7 @@ void select_instruction(opt_ctx &ctx, aco_ptr& instr) sgpr_used = true; /* don't break because we still need to check constants */ } else if (!sgpr_used && - ctx.info[instr->operands[i].tempId()].is_literal() && + ctx.info[instr->operands[i].tempId()].is_literal(bits) && ctx.uses[instr->operands[i].tempId()] < literal_uses) { literal_uses = ctx.uses[instr->operands[i].tempId()]; literal_idx = i; @@ -2876,6 +3038,7 @@ void select_instruction(opt_ctx &ctx, aco_ptr& instr) /* choose a literal to apply */ for (unsigned i = 0; i < num_operands; i++) { Operand op = instr->operands[i]; + unsigned bits = get_operand_size(instr, i); if (instr->isVALU() && op.isTemp() && op.getTemp().type() == RegType::sgpr && op.tempId() != sgpr_ids[0]) @@ -2884,7 +3047,7 @@ void select_instruction(opt_ctx &ctx, aco_ptr& instr) if (op.isLiteral()) { current_literal = op; continue; - } else if (!op.isTemp() || !ctx.info[op.tempId()].is_literal()) { + } else if (!op.isTemp() || !ctx.info[op.tempId()].is_literal(bits)) { continue; } @@ -2937,7 +3100,7 @@ void apply_literals(opt_ctx &ctx, aco_ptr& instr) /* apply literals on MAD */ if (!instr->definitions.empty() && ctx.info[instr->definitions[0].tempId()].is_mad()) { - mad_info* info = &ctx.mad_infos[ctx.info[instr->definitions[0].tempId()].val]; + mad_info* info = &ctx.mad_infos[ctx.info[instr->definitions[0].tempId()].instr->pass_flags]; if (info->check_literal && (ctx.uses[instr->operands[info->literal_idx].tempId()] == 0 || info->literal_idx == 2)) { aco_ptr new_mad; @@ -2969,7 +3132,8 @@ void apply_literals(opt_ctx &ctx, aco_ptr& instr) if (instr->isSALU() || instr->isVALU()) { for (unsigned i = 0; i < instr->operands.size(); i++) { Operand op = instr->operands[i]; - if (op.isTemp() && ctx.info[op.tempId()].is_literal() && ctx.uses[op.tempId()] == 0) { + unsigned bits = get_operand_size(instr, i); + if (op.isTemp() && ctx.info[op.tempId()].is_literal(bits) && ctx.uses[op.tempId()] == 0) { Operand literal(ctx.info[op.tempId()].val); if (instr->isVALU() && i > 0) to_VOP3(ctx, instr);