-#include <map>
+#include <vector>
+#include <algorithm>
#include "aco_ir.h"
#include "common/sid.h"
+#include "ac_shader_util.h"
+#include "util/u_math.h"
namespace aco {
struct asm_context {
Program *program;
enum chip_class chip_class;
- std::map<int, SOPP_instruction*> branches;
+ std::vector<std::pair<int, SOPP_instruction*>> branches;
std::vector<unsigned> constaddrs;
const int16_t* opcode;
// TODO: keep track of branch instructions referring blocks
// and, when emitting the block, correct the offset in instr
asm_context(Program* program) : program(program), chip_class(program->chip_class) {
- if (chip_class <= GFX9)
+ if (chip_class <= GFX7)
+ opcode = &instr_info.opcode_gfx7[0];
+ else if (chip_class <= GFX9)
opcode = &instr_info.opcode_gfx9[0];
else if (chip_class == GFX10)
opcode = &instr_info.opcode_gfx10[0];
}
+
+ int subvector_begin_pos = -1;
};
void emit_instruction(asm_context& ctx, std::vector<uint32_t>& out, Instruction* instr)
break;
}
case Format::SOPK: {
+ SOPK_instruction *sopk = static_cast<SOPK_instruction*>(instr);
+
+ if (instr->opcode == aco_opcode::s_subvector_loop_begin) {
+ assert(ctx.chip_class >= GFX10);
+ assert(ctx.subvector_begin_pos == -1);
+ ctx.subvector_begin_pos = out.size();
+ } else if (instr->opcode == aco_opcode::s_subvector_loop_end) {
+ assert(ctx.chip_class >= GFX10);
+ assert(ctx.subvector_begin_pos != -1);
+ /* Adjust s_subvector_loop_begin instruction to the address after the end */
+ out[ctx.subvector_begin_pos] |= (out.size() - ctx.subvector_begin_pos);
+ /* Adjust s_subvector_loop_end instruction to the address after the beginning */
+ sopk->imm = (uint16_t)(ctx.subvector_begin_pos - (int)out.size());
+ ctx.subvector_begin_pos = -1;
+ }
+
uint32_t encoding = (0b1011 << 28);
encoding |= opcode << 23;
encoding |=
!instr->definitions.empty() && !(instr->definitions[0].physReg() == scc) ?
instr->definitions[0].physReg() << 16 :
- !instr->operands.empty() && !(instr->operands[0].physReg() == scc) ?
+ !instr->operands.empty() && instr->operands[0].physReg() <= 127 ?
instr->operands[0].physReg() << 16 : 0;
- encoding |= static_cast<SOPK_instruction*>(instr)->imm;
+ encoding |= sopk->imm;
out.push_back(encoding);
break;
}
encoding |= opcode << 16;
encoding |= (uint16_t) sopp->imm;
if (sopp->block != -1)
- ctx.branches.insert({out.size(), sopp});
+ ctx.branches.emplace_back(out.size(), sopp);
out.push_back(encoding);
break;
}
SMEM_instruction* smem = static_cast<SMEM_instruction*>(instr);
bool soe = instr->operands.size() >= (!instr->definitions.empty() ? 3 : 4);
bool is_load = !instr->definitions.empty();
-
uint32_t encoding = 0;
+ if (ctx.chip_class <= GFX7) {
+ encoding = (0b11000 << 27);
+ encoding |= opcode << 22;
+ encoding |= instr->definitions.size() ? instr->definitions[0].physReg() << 15 : 0;
+ encoding |= instr->operands.size() ? (instr->operands[0].physReg() >> 1) << 9 : 0;
+ if (!instr->operands[1].isConstant() || instr->operands[1].constantValue() >= 1024) {
+ encoding |= instr->operands[1].physReg().reg;
+ } else {
+ encoding |= instr->operands[1].constantValue() >> 2;
+ encoding |= 1 << 8;
+ }
+ out.push_back(encoding);
+ /* SMRD instructions can take a literal on GFX6 & GFX7 */
+ if (instr->operands[1].isConstant() && instr->operands[1].constantValue() >= 1024)
+ out.push_back(instr->operands[1].constantValue() >> 2);
+ return;
+ }
+
if (ctx.chip_class <= GFX9) {
encoding = (0b110000 << 26);
assert(!smem->dlc); /* Device-level coherent is not supported on GFX9 and lower */
}
if (is_load || instr->operands.size() >= 3) { /* SDATA */
- encoding |= (is_load ? instr->definitions[0].physReg().reg : instr->operands[2].physReg().reg) << 6;
+ encoding |= (is_load ? instr->definitions[0].physReg() : instr->operands[2].physReg()) << 6;
}
if (instr->operands.size() >= 1) { /* SBASE */
- encoding |= instr->operands[0].physReg().reg >> 1;
+ encoding |= instr->operands[0].physReg() >> 1;
}
out.push_back(encoding);
case Format::VOP2: {
uint32_t encoding = 0;
encoding |= opcode << 25;
- encoding |= (0xFF & instr->definitions[0].physReg().reg) << 17;
- encoding |= (0xFF & instr->operands[1].physReg().reg) << 9;
- encoding |= instr->operands[0].physReg().reg;
+ encoding |= (0xFF & instr->definitions[0].physReg()) << 17;
+ encoding |= (0xFF & instr->operands[1].physReg()) << 9;
+ encoding |= instr->operands[0].physReg();
out.push_back(encoding);
break;
}
case Format::VOP1: {
uint32_t encoding = (0b0111111 << 25);
- encoding |= (0xFF & instr->definitions[0].physReg().reg) << 17;
+ if (!instr->definitions.empty())
+ encoding |= (0xFF & instr->definitions[0].physReg()) << 17;
encoding |= opcode << 9;
- encoding |= instr->operands[0].physReg().reg;
+ if (!instr->operands.empty())
+ encoding |= instr->operands[0].physReg();
out.push_back(encoding);
break;
}
case Format::VOPC: {
uint32_t encoding = (0b0111110 << 25);
encoding |= opcode << 17;
- encoding |= (0xFF & instr->operands[1].physReg().reg) << 9;
- encoding |= instr->operands[0].physReg().reg;
+ encoding |= (0xFF & instr->operands[1].physReg()) << 9;
+ encoding |= instr->operands[0].physReg();
out.push_back(encoding);
break;
}
}
assert(encoding);
- encoding |= (0xFF & instr->definitions[0].physReg().reg) << 18;
+ encoding |= (0xFF & instr->definitions[0].physReg()) << 18;
encoding |= opcode << 16;
encoding |= interp->attribute << 10;
encoding |= interp->component << 8;
if (instr->opcode == aco_opcode::v_interp_mov_f32)
encoding |= (0x3 & instr->operands[0].constantValue());
else
- encoding |= (0xFF & instr->operands[0].physReg().reg);
+ encoding |= (0xFF & instr->operands[0].physReg());
out.push_back(encoding);
break;
}
encoding |= (0xFF & reg) << 16;
reg = instr->operands.size() >= 2 && !(instr->operands[1].physReg() == m0) ? instr->operands[1].physReg() : 0;
encoding |= (0xFF & reg) << 8;
- encoding |= (0xFF & instr->operands[0].physReg().reg);
+ encoding |= (0xFF & instr->operands[0].physReg());
out.push_back(encoding);
break;
}
encoding |= (mubuf->glc ? 1 : 0) << 14;
encoding |= (mubuf->idxen ? 1 : 0) << 13;
encoding |= (mubuf->offen ? 1 : 0) << 12;
- if (ctx.chip_class <= GFX9) {
+ if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
assert(!mubuf->dlc); /* Device-level coherent is not supported on GFX9 and lower */
encoding |= (mubuf->slc ? 1 : 0) << 17;
} else if (ctx.chip_class >= GFX10) {
encoding |= instr->operands[2].physReg() << 24;
encoding |= (mubuf->tfe ? 1 : 0) << 23;
encoding |= (instr->operands[1].physReg() >> 2) << 16;
- unsigned reg = instr->operands.size() > 3 ? instr->operands[3].physReg() : instr->definitions[0].physReg().reg;
+ unsigned reg = instr->operands.size() > 3 ? instr->operands[3].physReg() : instr->definitions[0].physReg();
encoding |= (0xFF & reg) << 8;
- encoding |= (0xFF & instr->operands[0].physReg().reg);
+ encoding |= (0xFF & instr->operands[0].physReg());
out.push_back(encoding);
break;
}
case Format::MTBUF: {
MTBUF_instruction* mtbuf = static_cast<MTBUF_instruction*>(instr);
+
+ uint32_t img_format = ac_get_tbuffer_format(ctx.chip_class, mtbuf->dfmt, mtbuf->nfmt);
uint32_t encoding = (0b111010 << 26);
- encoding |= opcode << 15;
+ assert(img_format <= 0x7F);
+ assert(!mtbuf->dlc || ctx.chip_class >= GFX10);
+ encoding |= (mtbuf->dlc ? 1 : 0) << 15; /* DLC bit replaces one bit of the OPCODE on GFX10 */
encoding |= (mtbuf->glc ? 1 : 0) << 14;
encoding |= (mtbuf->idxen ? 1 : 0) << 13;
encoding |= (mtbuf->offen ? 1 : 0) << 12;
encoding |= 0x0FFF & mtbuf->offset;
- encoding |= (0xF & mtbuf->dfmt) << 19;
- encoding |= (0x7 & mtbuf->nfmt) << 23;
+ encoding |= (img_format << 19); /* Handles both the GFX10 FORMAT and the old NFMT+DFMT */
+
+ if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
+ encoding |= opcode << 15;
+ } else {
+ encoding |= (opcode & 0x07) << 16; /* 3 LSBs of 4-bit OPCODE */
+ }
+
out.push_back(encoding);
encoding = 0;
- encoding |= instr->operands[2].physReg().reg << 24;
+
+ encoding |= instr->operands[2].physReg() << 24;
encoding |= (mtbuf->tfe ? 1 : 0) << 23;
encoding |= (mtbuf->slc ? 1 : 0) << 22;
- encoding |= (instr->operands[1].physReg().reg >> 2) << 16;
- unsigned reg = instr->operands.size() > 3 ? instr->operands[3].physReg().reg : instr->definitions[0].physReg().reg;
+ encoding |= (instr->operands[1].physReg() >> 2) << 16;
+ unsigned reg = instr->operands.size() > 3 ? instr->operands[3].physReg() : instr->definitions[0].physReg();
encoding |= (0xFF & reg) << 8;
- encoding |= (0xFF & instr->operands[0].physReg().reg);
+ encoding |= (0xFF & instr->operands[0].physReg());
+
+ if (ctx.chip_class >= GFX10) {
+ encoding |= (((opcode & 0x08) >> 4) << 21); /* MSB of 4-bit OPCODE */
+ }
+
out.push_back(encoding);
break;
}
encoding |= opcode << 18;
encoding |= mimg->lwe ? 1 << 17 : 0;
encoding |= mimg->tfe ? 1 << 16 : 0;
- encoding |= mimg->r128 ? 1 << 15 : 0;
- encoding |= mimg->da ? 1 << 14 : 0;
encoding |= mimg->glc ? 1 << 13 : 0;
encoding |= mimg->unrm ? 1 << 12 : 0;
+ if (ctx.chip_class <= GFX9) {
+ assert(!mimg->dlc); /* Device-level coherent is not supported on GFX9 and lower */
+ assert(!mimg->r128);
+ encoding |= mimg->a16 ? 1 << 15 : 0;
+ encoding |= mimg->da ? 1 << 14 : 0;
+ } else {
+ encoding |= mimg->r128 ? 1 << 15 : 0; /* GFX10: A16 moved to 2nd word, R128 replaces it in 1st word */
+ encoding |= mimg->dim << 3; /* GFX10: dimensionality instead of declare array */
+ encoding |= mimg->dlc ? 1 << 7 : 0;
+ }
encoding |= (0xF & mimg->dmask) << 8;
out.push_back(encoding);
- encoding = (0xFF & instr->operands[0].physReg().reg); /* VADDR */
+ encoding = (0xFF & instr->operands[0].physReg()); /* VADDR */
if (!instr->definitions.empty()) {
- encoding |= (0xFF & instr->definitions[0].physReg().reg) << 8; /* VDATA */
+ encoding |= (0xFF & instr->definitions[0].physReg()) << 8; /* VDATA */
} else if (instr->operands.size() == 4) {
- encoding |= (0xFF & instr->operands[3].physReg().reg) << 8; /* VDATA */
+ encoding |= (0xFF & instr->operands[3].physReg()) << 8; /* VDATA */
}
encoding |= (0x1F & (instr->operands[1].physReg() >> 2)) << 16; /* T# (resource) */
if (instr->operands.size() > 2)
encoding |= (0x1F & (instr->operands[2].physReg() >> 2)) << 21; /* sampler */
- // TODO VEGA: D16
+
+ assert(!mimg->d16 || ctx.chip_class >= GFX9);
+ encoding |= mimg->d16 ? 1 << 15 : 0;
+ if (ctx.chip_class >= GFX10) {
+ encoding |= mimg->a16 ? 1 << 14 : 0; /* GFX10: A16 still exists, but is in a different place */
+ }
+
out.push_back(encoding);
break;
}
FLAT_instruction *flat = static_cast<FLAT_instruction*>(instr);
uint32_t encoding = (0b110111 << 26);
encoding |= opcode << 18;
- encoding |= flat->offset & 0x1fff;
+ if (ctx.chip_class <= GFX9) {
+ assert(flat->offset <= 0x1fff);
+ encoding |= flat->offset & 0x1fff;
+ } else if (instr->format == Format::FLAT) {
+ /* GFX10 has a 12-bit immediate OFFSET field,
+ * but it has a hw bug: it ignores the offset, called FlatSegmentOffsetBug
+ */
+ assert(flat->offset == 0);
+ } else {
+ assert(flat->offset <= 0xfff);
+ encoding |= flat->offset & 0xfff;
+ }
if (instr->format == Format::SCRATCH)
encoding |= 1 << 14;
else if (instr->format == Format::GLOBAL)
encoding |= 2 << 14;
encoding |= flat->lds ? 1 << 13 : 0;
- encoding |= flat->glc ? 1 << 13 : 0;
- encoding |= flat->slc ? 1 << 13 : 0;
+ encoding |= flat->glc ? 1 << 16 : 0;
+ encoding |= flat->slc ? 1 << 17 : 0;
+ if (ctx.chip_class >= GFX10) {
+ assert(!flat->nv);
+ encoding |= flat->dlc ? 1 << 12 : 0;
+ } else {
+ assert(!flat->dlc);
+ }
out.push_back(encoding);
- encoding = (0xFF & instr->operands[0].physReg().reg);
+ encoding = (0xFF & instr->operands[0].physReg());
if (!instr->definitions.empty())
- encoding |= (0xFF & instr->definitions[0].physReg().reg) << 24;
- else
- encoding |= (0xFF & instr->operands[2].physReg().reg) << 8;
+ encoding |= (0xFF & instr->definitions[0].physReg()) << 24;
+ if (instr->operands.size() >= 3)
+ encoding |= (0xFF & instr->operands[2].physReg()) << 8;
if (!instr->operands[1].isUndefined()) {
- assert(instr->operands[1].physReg() != 0x7f);
+ assert(ctx.chip_class >= GFX10 || instr->operands[1].physReg() != 0x7F);
assert(instr->format != Format::FLAT);
encoding |= instr->operands[1].physReg() << 16;
- } else if (instr->format != Format::FLAT) {
- encoding |= 0x7F << 16;
+ } else if (instr->format != Format::FLAT || ctx.chip_class >= GFX10) { /* SADDR is actually used with FLAT on GFX10 */
+ if (ctx.chip_class <= GFX9)
+ encoding |= 0x7F << 16;
+ else
+ encoding |= sgpr_null << 16;
}
encoding |= flat->nv ? 1 << 23 : 0;
out.push_back(encoding);
}
case Format::EXP: {
Export_instruction* exp = static_cast<Export_instruction*>(instr);
- uint32_t encoding = (0b110001 << 26);
+ uint32_t encoding;
+ if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
+ encoding = (0b110001 << 26);
+ } else {
+ encoding = (0b111110 << 26);
+ }
+
encoding |= exp->valid_mask ? 0b1 << 12 : 0;
encoding |= exp->done ? 0b1 << 11 : 0;
encoding |= exp->compressed ? 0b1 << 10 : 0;
encoding |= exp->dest << 4;
encoding |= exp->enabled_mask;
out.push_back(encoding);
- encoding = 0xFF & exp->operands[0].physReg().reg;
- encoding |= (0xFF & exp->operands[1].physReg().reg) << 8;
- encoding |= (0xFF & exp->operands[2].physReg().reg) << 16;
- encoding |= (0xFF & exp->operands[3].physReg().reg) << 24;
+ encoding = 0xFF & exp->operands[0].physReg();
+ encoding |= (0xFF & exp->operands[1].physReg()) << 8;
+ encoding |= (0xFF & exp->operands[2].physReg()) << 16;
+ encoding |= (0xFF & exp->operands[3].physReg()) << 24;
out.push_back(encoding);
break;
}
if ((uint16_t) instr->format & (uint16_t) Format::VOP3A) {
VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(instr);
- if ((uint16_t) instr->format & (uint16_t) Format::VOP2)
+ if ((uint16_t) instr->format & (uint16_t) Format::VOP2) {
opcode = opcode + 0x100;
- else if ((uint16_t) instr->format & (uint16_t) Format::VOP1)
- opcode = opcode + 0x140;
- else if ((uint16_t) instr->format & (uint16_t) Format::VOPC)
+ } else if ((uint16_t) instr->format & (uint16_t) Format::VOP1) {
+ if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9)
+ opcode = opcode + 0x140;
+ else
+ opcode = opcode + 0x180;
+ } else if ((uint16_t) instr->format & (uint16_t) Format::VOPC) {
opcode = opcode + 0x0;
- else if ((uint16_t) instr->format & (uint16_t) Format::VINTRP)
+ } else if ((uint16_t) instr->format & (uint16_t) Format::VINTRP) {
opcode = opcode + 0x270;
+ }
+
+ uint32_t encoding;
+ if (ctx.chip_class <= GFX9) {
+ encoding = (0b110100 << 26);
+ } else if (ctx.chip_class == GFX10) {
+ encoding = (0b110101 << 26);
+ }
- // TODO: op_sel
- uint32_t encoding = (0b110100 << 26);
- encoding |= opcode << 16;
- encoding |= (vop3->clamp ? 1 : 0) << 15;
+ if (ctx.chip_class <= GFX7) {
+ encoding |= opcode << 17;
+ encoding |= (vop3->clamp ? 1 : 0) << 11;
+ } else {
+ encoding |= opcode << 16;
+ encoding |= (vop3->clamp ? 1 : 0) << 15;
+ }
+ encoding |= vop3->opsel << 11;
for (unsigned i = 0; i < 3; i++)
encoding |= vop3->abs[i] << (8+i);
if (instr->definitions.size() == 2)
encoding |= instr->definitions[1].physReg() << 8;
- encoding |= (0xFF & instr->definitions[0].physReg().reg);
+ encoding |= (0xFF & instr->definitions[0].physReg());
out.push_back(encoding);
encoding = 0;
if (instr->opcode == aco_opcode::v_interp_mov_f32) {
for (unsigned i = 0; i < 3; i++)
encoding |= vop3->neg[i] << (29+i);
out.push_back(encoding);
- return;
} else if (instr->isDPP()){
+ assert(ctx.chip_class >= GFX8);
/* first emit the instruction without the DPP operand */
Operand dpp_op = instr->operands[0];
instr->operands[0] = Operand(PhysReg{250}, v1);
encoding |= dpp->neg[0] << 20;
encoding |= dpp->bound_ctrl << 19;
encoding |= dpp->dpp_ctrl << 8;
- encoding |= (0xFF) & dpp_op.physReg().reg;
+ encoding |= (0xFF) & dpp_op.physReg();
out.push_back(encoding);
return;
} else {
unreachable("unimplemented instruction format");
}
+ break;
}
/* append literal dword */
}
}
+static void fix_branches_gfx10(asm_context& ctx, std::vector<uint32_t>& out)
+{
+ /* Branches with an offset of 0x3f are buggy on GFX10, we workaround by inserting NOPs if needed. */
+ bool gfx10_3f_bug = false;
+
+ do {
+ auto buggy_branch_it = std::find_if(ctx.branches.begin(), ctx.branches.end(), [&ctx](const auto &branch) -> bool {
+ return ((int)ctx.program->blocks[branch.second->block].offset - branch.first - 1) == 0x3f;
+ });
+
+ gfx10_3f_bug = buggy_branch_it != ctx.branches.end();
+
+ if (gfx10_3f_bug) {
+ /* Insert an s_nop after the branch */
+ constexpr uint32_t s_nop_0 = 0xbf800000u;
+ int s_nop_pos = buggy_branch_it->first + 1;
+ auto out_pos = std::next(out.begin(), s_nop_pos);
+ out.insert(out_pos, s_nop_0);
+
+ /* Update the offset of each affected block */
+ for (Block& block : ctx.program->blocks) {
+ if (block.offset > (unsigned)buggy_branch_it->first)
+ block.offset++;
+ }
+
+ /* Update the branches following the current one */
+ for (auto branch_it = std::next(buggy_branch_it); branch_it != ctx.branches.end(); ++branch_it)
+ branch_it->first++;
+
+ /* Find first constant address after the inserted instruction */
+ auto caddr_it = std::find_if(ctx.constaddrs.begin(), ctx.constaddrs.end(), [s_nop_pos](const int &caddr_pos) -> bool {
+ return caddr_pos >= s_nop_pos;
+ });
+
+ /* Update the locations of constant addresses */
+ for (; caddr_it != ctx.constaddrs.end(); ++caddr_it)
+ (*caddr_it)++;
+
+ }
+ } while (gfx10_3f_bug);
+}
+
void fix_branches(asm_context& ctx, std::vector<uint32_t>& out)
{
- for (std::pair<int, SOPP_instruction*> branch : ctx.branches)
- {
+ if (ctx.chip_class >= GFX10)
+ fix_branches_gfx10(ctx, out);
+
+ for (std::pair<int, SOPP_instruction*> &branch : ctx.branches) {
int offset = (int)ctx.program->blocks[branch.second->block].offset - branch.first - 1;
out[branch.first] |= (uint16_t) offset;
}
}
fix_branches(ctx, code);
+
+ unsigned exec_size = code.size() * sizeof(uint32_t);
+
+ if (program->chip_class >= GFX10) {
+ /* Pad output with s_code_end so instruction prefetching doesn't cause
+ * page faults */
+ unsigned final_size = align(code.size() + 3 * 16, 16);
+ while (code.size() < final_size)
+ code.push_back(0xbf9f0000u);
+ }
+
fix_constaddrs(ctx, code);
- unsigned constant_data_offset = code.size() * sizeof(uint32_t);
while (program->constant_data.size() % 4u)
program->constant_data.push_back(0);
/* Copy constant data */
code.insert(code.end(), (uint32_t*)program->constant_data.data(),
(uint32_t*)(program->constant_data.data() + program->constant_data.size()));
- return constant_data_offset;
+ return exec_size;
}
}
projects / mesa.git / blobdiff