X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Famd%2Fcompiler%2Faco_validate.cpp;h=97967aac9c190a1b8cdd2c09b2d0f3ef7d38c143;hb=1b3be07b5faf867f698668080b060a270c5f795e;hp=3e0f4584cd3fa7beffbee6b41540fab7af37466f;hpb=e18711cda3bf173e03d9773001f641fd85ffd18d;p=mesa.git diff --git a/src/amd/compiler/aco_validate.cpp b/src/amd/compiler/aco_validate.cpp index 3e0f4584cd3..97967aac9c1 100644 --- a/src/amd/compiler/aco_validate.cpp +++ b/src/amd/compiler/aco_validate.cpp @@ -46,11 +46,8 @@ void perfwarn(bool cond, const char *msg, Instruction *instr) } #endif -void validate(Program* program, FILE * output) +bool validate(Program* program, FILE *output) { - if (!(debug_flags & DEBUG_VALIDATE)) - return; - bool is_valid = true; auto check = [&output, &is_valid](bool check, const char * msg, aco::Instruction * instr) -> void { if (!check) { @@ -80,8 +77,19 @@ void validate(Program* program, FILE * output) base_format = Format::VOP2; else if ((uint32_t)base_format & (uint32_t)Format::VOPC) base_format = Format::VOPC; - else if ((uint32_t)base_format & (uint32_t)Format::VINTRP) - base_format = Format::VINTRP; + else if ((uint32_t)base_format & (uint32_t)Format::VINTRP) { + if (instr->opcode == aco_opcode::v_interp_p1ll_f16 || + instr->opcode == aco_opcode::v_interp_p1lv_f16 || + instr->opcode == aco_opcode::v_interp_p2_legacy_f16 || + instr->opcode == aco_opcode::v_interp_p2_f16) { + /* v_interp_*_fp16 are considered VINTRP by the compiler but + * they are emitted as VOP3. + */ + base_format = Format::VOP3; + } else { + base_format = Format::VINTRP; + } + } check(base_format == instr_info.format[(int)instr->opcode], "Wrong base format for instruction", instr.get()); /* check VOP3 modifiers */ @@ -135,13 +143,27 @@ void validate(Program* program, FILE * output) instr->opcode != aco_opcode::v_fmac_f16, "SDWA can't be used with this opcode", instr.get()); } + + for (unsigned i = 0; i < MIN2(instr->operands.size(), 2); i++) { + if (instr->operands[i].hasRegClass() && instr->operands[i].regClass().is_subdword()) + check((sdwa->sel[i] & sdwa_asuint) == (sdwa_isra | instr->operands[i].bytes()), "Unexpected SDWA sel for sub-dword operand", instr.get()); + } + if (instr->definitions[0].regClass().is_subdword()) + check((sdwa->dst_sel & sdwa_asuint) == (sdwa_isra | instr->definitions[0].bytes()), "Unexpected SDWA sel for sub-dword definition", instr.get()); } /* check opsel */ if (instr->isVOP3()) { VOP3A_instruction *vop3 = static_cast(instr.get()); check(vop3->opsel == 0 || program->chip_class >= GFX9, "Opsel is only supported on GFX9+", instr.get()); - check((vop3->opsel & ~(0x10 | ((1 << instr->operands.size()) - 1))) == 0, "Unused bits in opsel must be zeroed out", instr.get()); + + for (unsigned i = 0; i < 3; i++) { + if (i >= instr->operands.size() || + (instr->operands[i].hasRegClass() && instr->operands[i].regClass().is_subdword() && !instr->operands[i].isFixed())) + check((vop3->opsel & (1 << i)) == 0, "Unexpected opsel for operand", instr.get()); + } + if (instr->definitions[0].regClass().is_subdword() && !instr->definitions[0].isFixed()) + check((vop3->opsel & (1 << 3)) == 0, "Unexpected opsel for sub-dword definition", instr.get()); } /* check for undefs */ @@ -162,7 +184,7 @@ void validate(Program* program, FILE * output) /* check subdword definitions */ for (unsigned i = 0; i < instr->definitions.size(); i++) { if (instr->definitions[i].regClass().is_subdword()) - check(instr->isSDWA() || instr->format == Format::PSEUDO, "Only SDWA and Pseudo instructions can write subdword registers", instr.get()); + check(instr->format == Format::PSEUDO || instr->definitions[i].bytes() <= 4, "Only Pseudo instructions can write subdword registers larger than 4 bytes", instr.get()); } if (instr->isSALU() || instr->isVALU()) { @@ -201,12 +223,17 @@ void validate(Program* program, FILE * output) if (instr->isSDWA()) scalar_mask = program->chip_class >= GFX9 ? 0x7 : 0x4; - check(instr->definitions[0].getTemp().type() == RegType::vgpr || - (int) instr->format & (int) Format::VOPC || - instr->opcode == aco_opcode::v_readfirstlane_b32 || - instr->opcode == aco_opcode::v_readlane_b32 || - instr->opcode == aco_opcode::v_readlane_b32_e64, - "Wrong Definition type for VALU instruction", instr.get()); + if ((int) instr->format & (int) Format::VOPC || + instr->opcode == aco_opcode::v_readfirstlane_b32 || + instr->opcode == aco_opcode::v_readlane_b32 || + instr->opcode == aco_opcode::v_readlane_b32_e64) { + check(instr->definitions[0].getTemp().type() == RegType::sgpr, + "Wrong Definition type for VALU instruction", instr.get()); + } else { + check(instr->definitions[0].getTemp().type() == RegType::vgpr, + "Wrong Definition type for VALU instruction", instr.get()); + } + unsigned num_sgprs = 0; unsigned sgpr[] = {0, 0}; for (unsigned i = 0; i < instr->operands.size(); i++) @@ -214,11 +241,26 @@ void validate(Program* program, FILE * output) Operand op = instr->operands[i]; if (instr->opcode == aco_opcode::v_readfirstlane_b32 || instr->opcode == aco_opcode::v_readlane_b32 || - instr->opcode == aco_opcode::v_readlane_b32_e64 || - instr->opcode == aco_opcode::v_writelane_b32 || + instr->opcode == aco_opcode::v_readlane_b32_e64) { + check(i != 1 || + (op.isTemp() && op.regClass().type() == RegType::sgpr) || + op.isConstant(), + "Must be a SGPR or a constant", instr.get()); + check(i == 1 || + (op.isTemp() && op.regClass().type() == RegType::vgpr && op.bytes() <= 4), + "Wrong Operand type for VALU instruction", instr.get()); + continue; + } + + if (instr->opcode == aco_opcode::v_writelane_b32 || instr->opcode == aco_opcode::v_writelane_b32_e64) { - check(!op.isLiteral(), "No literal allowed on VALU instruction", instr.get()); - check(i == 1 || (op.isTemp() && op.regClass() == v1), "Wrong Operand type for VALU instruction", instr.get()); + check(i != 2 || + (op.isTemp() && op.regClass().type() == RegType::vgpr && op.bytes() <= 4), + "Wrong Operand type for VALU instruction", instr.get()); + check(i == 2 || + (op.isTemp() && op.regClass().type() == RegType::sgpr) || + op.isConstant(), + "Must be a SGPR or a constant", instr.get()); continue; } if (op.isTemp() && instr->operands[i].regClass().type() == RegType::sgpr) { @@ -247,6 +289,24 @@ void validate(Program* program, FILE * output) switch (instr->format) { case Format::PSEUDO: { + bool is_subdword = false; + bool has_const_sgpr = false; + bool has_literal = false; + for (Definition def : instr->definitions) + is_subdword |= def.regClass().is_subdword(); + for (unsigned i = 0; i < instr->operands.size(); i++) { + if (instr->opcode == aco_opcode::p_extract_vector && i == 1) + continue; + Operand op = instr->operands[i]; + is_subdword |= op.hasRegClass() && op.regClass().is_subdword(); + has_const_sgpr |= op.isConstant() || (op.hasRegClass() && op.regClass().type() == RegType::sgpr); + has_literal |= op.isLiteral(); + } + + check(!is_subdword || !has_const_sgpr || program->chip_class >= GFX9, + "Sub-dword pseudo instructions can only take constants or SGPRs on GFX9+", instr.get()); + check(!is_subdword || !has_literal, "Sub-dword pseudo instructions cannot take literals", instr.get()); + if (instr->opcode == aco_opcode::p_create_vector) { unsigned size = 0; for (const Operand& op : instr->operands) { @@ -274,7 +334,7 @@ void validate(Program* program, FILE * output) } } else if (instr->opcode == aco_opcode::p_phi) { check(instr->operands.size() == block.logical_preds.size(), "Number of Operands does not match number of predecessors", instr.get()); - check(instr->definitions[0].getTemp().type() == RegType::vgpr || instr->definitions[0].getTemp().regClass() == program->lane_mask, "Logical Phi Definition must be vgpr or divergent boolean", instr.get()); + check(instr->definitions[0].getTemp().type() == RegType::vgpr, "Logical Phi Definition must be vgpr", instr.get()); } else if (instr->opcode == aco_opcode::p_linear_phi) { for (const Operand& op : instr->operands) check(!op.isTemp() || op.getTemp().is_linear(), "Wrong Operand type", instr.get()); @@ -376,7 +436,7 @@ void validate(Program* program, FILE * output) } } - assert(is_valid); + return is_valid; } /* RA validation */ @@ -418,6 +478,136 @@ bool ra_fail(FILE *output, Location loc, Location loc2, const char *fmt, ...) { return true; } +bool validate_subdword_operand(chip_class chip, const aco_ptr& instr, unsigned index) +{ + Operand op = instr->operands[index]; + unsigned byte = op.physReg().byte(); + + if (instr->opcode == aco_opcode::p_as_uniform) + return byte == 0; + if (instr->format == Format::PSEUDO && chip >= GFX8) + return true; + if (instr->isSDWA() && (static_cast(instr.get())->sel[index] & sdwa_asuint) == (sdwa_isra | op.bytes())) + return true; + if (byte == 2 && can_use_opsel(chip, instr->opcode, index, 1)) + return true; + + switch (instr->opcode) { + case aco_opcode::v_cvt_f32_ubyte1: + if (byte == 1) + return true; + break; + case aco_opcode::v_cvt_f32_ubyte2: + if (byte == 2) + return true; + break; + case aco_opcode::v_cvt_f32_ubyte3: + if (byte == 3) + return true; + break; + case aco_opcode::ds_write_b8_d16_hi: + case aco_opcode::ds_write_b16_d16_hi: + if (byte == 2 && index == 1) + return true; + break; + case aco_opcode::buffer_store_byte_d16_hi: + case aco_opcode::buffer_store_short_d16_hi: + if (byte == 2 && index == 3) + return true; + break; + case aco_opcode::flat_store_byte_d16_hi: + case aco_opcode::flat_store_short_d16_hi: + case aco_opcode::scratch_store_byte_d16_hi: + case aco_opcode::scratch_store_short_d16_hi: + case aco_opcode::global_store_byte_d16_hi: + case aco_opcode::global_store_short_d16_hi: + if (byte == 2 && index == 2) + return true; + default: + break; + } + + return byte == 0; +} + +bool validate_subdword_definition(chip_class chip, const aco_ptr& instr) +{ + Definition def = instr->definitions[0]; + unsigned byte = def.physReg().byte(); + + if (instr->format == Format::PSEUDO && chip >= GFX8) + return true; + if (instr->isSDWA() && static_cast(instr.get())->dst_sel == (sdwa_isra | def.bytes())) + return true; + if (byte == 2 && can_use_opsel(chip, instr->opcode, -1, 1)) + return true; + + switch (instr->opcode) { + case aco_opcode::buffer_load_ubyte_d16_hi: + case aco_opcode::buffer_load_short_d16_hi: + case aco_opcode::flat_load_ubyte_d16_hi: + case aco_opcode::flat_load_short_d16_hi: + case aco_opcode::scratch_load_ubyte_d16_hi: + case aco_opcode::scratch_load_short_d16_hi: + case aco_opcode::global_load_ubyte_d16_hi: + case aco_opcode::global_load_short_d16_hi: + case aco_opcode::ds_read_u8_d16_hi: + case aco_opcode::ds_read_u16_d16_hi: + return byte == 2; + default: + break; + } + + return byte == 0; +} + +unsigned get_subdword_bytes_written(Program *program, const aco_ptr& instr, unsigned index) +{ + chip_class chip = program->chip_class; + Definition def = instr->definitions[index]; + + if (instr->format == Format::PSEUDO) + return chip >= GFX8 ? def.bytes() : def.size() * 4u; + if (instr->isSDWA() && static_cast(instr.get())->dst_sel == (sdwa_isra | def.bytes())) + return def.bytes(); + + switch (instr->opcode) { + case aco_opcode::buffer_load_ubyte_d16: + case aco_opcode::buffer_load_short_d16: + case aco_opcode::flat_load_ubyte_d16: + case aco_opcode::flat_load_short_d16: + case aco_opcode::scratch_load_ubyte_d16: + case aco_opcode::scratch_load_short_d16: + case aco_opcode::global_load_ubyte_d16: + case aco_opcode::global_load_short_d16: + case aco_opcode::ds_read_u8_d16: + case aco_opcode::ds_read_u16_d16: + case aco_opcode::buffer_load_ubyte_d16_hi: + case aco_opcode::buffer_load_short_d16_hi: + case aco_opcode::flat_load_ubyte_d16_hi: + case aco_opcode::flat_load_short_d16_hi: + case aco_opcode::scratch_load_ubyte_d16_hi: + case aco_opcode::scratch_load_short_d16_hi: + case aco_opcode::global_load_ubyte_d16_hi: + case aco_opcode::global_load_short_d16_hi: + case aco_opcode::ds_read_u8_d16_hi: + case aco_opcode::ds_read_u16_d16_hi: + return program->sram_ecc_enabled ? 4 : 2; + case aco_opcode::v_mad_f16: + case aco_opcode::v_mad_u16: + case aco_opcode::v_mad_i16: + case aco_opcode::v_fma_f16: + case aco_opcode::v_div_fixup_f16: + case aco_opcode::v_interp_p2_f16: + if (chip >= GFX9) + return 2; + default: + break; + } + + return MAX2(chip >= GFX10 ? def.bytes() : 4, instr_info.definition_size[(int)instr->opcode] / 8u); +} + } /* end namespace */ bool validate_ra(Program *program, const struct radv_nir_compiler_options *options, FILE *output) { @@ -451,13 +641,13 @@ bool validate_ra(Program *program, const struct radv_nir_compiler_options *optio err |= ra_fail(output, loc, Location(), "Operand %d is not assigned a register", i); if (assignments.count(op.tempId()) && assignments[op.tempId()].reg != op.physReg()) err |= ra_fail(output, loc, assignments.at(op.tempId()).firstloc, "Operand %d has an inconsistent register assignment with instruction", i); - if ((op.getTemp().type() == RegType::vgpr && op.physReg() + op.size() > 256 + program->config->num_vgprs) || + if ((op.getTemp().type() == RegType::vgpr && op.physReg().reg_b + op.bytes() > (256 + program->config->num_vgprs) * 4) || (op.getTemp().type() == RegType::sgpr && op.physReg() + op.size() > program->config->num_sgprs && op.physReg() < program->sgpr_limit)) err |= ra_fail(output, loc, assignments.at(op.tempId()).firstloc, "Operand %d has an out-of-bounds register assignment", i); if (op.physReg() == vcc && !program->needs_vcc) err |= ra_fail(output, loc, Location(), "Operand %d fixed to vcc but needs_vcc=false", i); - if (!(instr->isSDWA() || instr->format == Format::PSEUDO) && op.regClass().is_subdword() && op.physReg().byte()) - err |= ra_fail(output, loc, assignments.at(op.tempId()).firstloc, "Operand %d must be aligned to a full register", i); + if (op.regClass().is_subdword() && !validate_subdword_operand(program->chip_class, instr, i)) + err |= ra_fail(output, loc, Location(), "Operand %d not aligned correctly", i); if (!assignments[op.tempId()].firstloc.block) assignments[op.tempId()].firstloc = loc; if (!assignments[op.tempId()].defloc.block) @@ -472,11 +662,13 @@ bool validate_ra(Program *program, const struct radv_nir_compiler_options *optio err |= ra_fail(output, loc, Location(), "Definition %d is not assigned a register", i); if (assignments[def.tempId()].defloc.block) err |= ra_fail(output, loc, assignments.at(def.tempId()).defloc, "Temporary %%%d also defined by instruction", def.tempId()); - if ((def.getTemp().type() == RegType::vgpr && def.physReg() + def.size() > 256 + program->config->num_vgprs) || + if ((def.getTemp().type() == RegType::vgpr && def.physReg().reg_b + def.bytes() > (256 + program->config->num_vgprs) * 4) || (def.getTemp().type() == RegType::sgpr && def.physReg() + def.size() > program->config->num_sgprs && def.physReg() < program->sgpr_limit)) err |= ra_fail(output, loc, assignments.at(def.tempId()).firstloc, "Definition %d has an out-of-bounds register assignment", i); if (def.physReg() == vcc && !program->needs_vcc) err |= ra_fail(output, loc, Location(), "Definition %d fixed to vcc but needs_vcc=false", i); + if (def.regClass().is_subdword() && !validate_subdword_definition(program->chip_class, instr)) + err |= ra_fail(output, loc, Location(), "Definition %d not aligned correctly", i); if (!assignments[def.tempId()].firstloc.block) assignments[def.tempId()].firstloc = loc; assignments[def.tempId()].defloc = loc; @@ -579,9 +771,18 @@ bool validate_ra(Program *program, const struct radv_nir_compiler_options *optio PhysReg reg = assignments.at(tmp.id()).reg; for (unsigned j = 0; j < tmp.bytes(); j++) { if (regs[reg.reg_b + j]) - err |= ra_fail(output, loc, assignments.at(regs[reg.reg_b + i]).defloc, "Assignment of element %d of %%%d already taken by %%%d from instruction", i, tmp.id(), regs[reg.reg_b + j]); + err |= ra_fail(output, loc, assignments.at(regs[reg.reg_b + j]).defloc, "Assignment of element %d of %%%d already taken by %%%d from instruction", i, tmp.id(), regs[reg.reg_b + j]); regs[reg.reg_b + j] = tmp.id(); } + if (def.regClass().is_subdword() && def.bytes() < 4) { + unsigned written = get_subdword_bytes_written(program, instr, i); + /* If written=4, the instruction still might write the upper half. In that case, it's the lower half that isn't preserved */ + for (unsigned j = reg.byte() & ~(written - 1); j < written; j++) { + unsigned written_reg = reg.reg() * 4u + j; + if (regs[written_reg] && regs[written_reg] != def.tempId()) + err |= ra_fail(output, loc, assignments.at(regs[written_reg]).defloc, "Assignment of element %d of %%%d overwrites the full register taken by %%%d from instruction", i, tmp.id(), regs[written_reg]); + } + } } for (const Definition& def : instr->definitions) {