virtual bool emitInstruction(Instruction *);
virtual uint32_t getMinEncodingSize(const Instruction *) const;
+ virtual void prepareEmission(Function *);
inline void setProgramType(Program::Type pType) { progType = pType; }
private:
- const TargetNVC0 *targ;
+ const TargetNVC0 *targNVC0;
Program::Type progType;
+ const bool writeIssueDelays;
+
private:
void emitForm_A(const Instruction *, uint64_t);
void emitForm_B(const Instruction *, uint64_t);
void setAddress16(const ValueRef&);
void setImmediate(const Instruction *, const int s); // needs op already set
void setImmediateS8(const ValueRef&);
+ void setSUConst16(const Instruction *, const int s);
+ void setSUPred(const Instruction *, const int s);
void emitCondCode(CondCode cc, int pos);
void emitInterpMode(const Instruction *);
void emitLoadStoreType(DataType ty);
+ void emitSUGType(DataType);
void emitCachingMode(CacheMode c);
void emitShortSrc2(const ValueRef&);
void emitUMUL(const Instruction *);
void emitFMUL(const Instruction *);
void emitIMAD(const Instruction *);
+ void emitISAD(const Instruction *);
void emitFMAD(const Instruction *);
+ void emitMADSP(const Instruction *);
void emitNOT(Instruction *);
void emitLogicOp(const Instruction *, uint8_t subOp);
void emitSLCT(const CmpInstruction *);
void emitSELP(const Instruction *);
+ void emitTEXBAR(const Instruction *);
void emitTEX(const TexInstruction *);
void emitTEXCSAA(const TexInstruction *);
void emitTXQ(const TexInstruction *);
- void emitPIXLD(const TexInstruction *);
void emitQUADOP(const Instruction *, uint8_t qOp, uint8_t laneMask);
void emitFlow(const Instruction *);
+ void emitSUCLAMPMode(uint16_t);
+ void emitSUCalc(Instruction *);
+ void emitSULDGB(const TexInstruction *);
+ void emitSUSTGx(const TexInstruction *);
+
+ void emitVSHL(const Instruction *);
+ void emitVectorSubOp(const Instruction *);
+
inline void defId(const ValueDef&, const int pos);
inline void srcId(const ValueRef&, const int pos);
+ inline void srcId(const ValueRef *, const int pos);
+ inline void srcId(const Instruction *, int s, const int pos);
inline void srcAddr32(const ValueRef&, const int pos); // address / 4
- inline void srcId(const ValueRef *, const int pos);
-
inline bool isLIMM(const ValueRef&, DataType ty);
};
code[pos / 32] |= (src ? SDATA(*src).id : 63) << (pos % 32);
}
+void CodeEmitterNVC0::srcId(const Instruction *insn, int s, int pos)
+{
+ int r = insn->srcExists(s) ? SDATA(insn->src(s)).id : 63;
+ code[pos / 32] |= r << (pos % 32);
+}
+
void CodeEmitterNVC0::srcAddr32(const ValueRef& src, const int pos)
{
code[pos / 32] |= (SDATA(src).offset >> 2) << (pos % 32);
void
CodeEmitterNVC0::emitNegAbs12(const Instruction *i)
{
- if (i->src[1].mod.abs()) code[0] |= 1 << 6;
- if (i->src[0].mod.abs()) code[0] |= 1 << 7;
- if (i->src[1].mod.neg()) code[0] |= 1 << 8;
- if (i->src[0].mod.neg()) code[0] |= 1 << 9;
+ if (i->src(1).mod.abs()) code[0] |= 1 << 6;
+ if (i->src(0).mod.abs()) code[0] |= 1 << 7;
+ if (i->src(1).mod.neg()) code[0] |= 1 << 8;
+ if (i->src(0).mod.neg()) code[0] |= 1 << 9;
}
void CodeEmitterNVC0::emitCondCode(CondCode cc, int pos)
{
if (i->predSrc >= 0) {
assert(i->getPredicate()->reg.file == FILE_PREDICATE);
- srcId(i->src[i->predSrc], 10);
+ srcId(i->src(i->predSrc), 10);
if (i->cc == CC_NOT_P)
code[0] |= 0x2000; // negate
} else {
void
CodeEmitterNVC0::setImmediate(const Instruction *i, const int s)
{
- const ImmediateValue *imm = i->src[s].get()->asImm();
+ const ImmediateValue *imm = i->src(s).get()->asImm();
uint32_t u32;
assert(imm);
emitPredicate(i);
- defId(i->def[0], 14);
+ defId(i->def(0), 14);
int s1 = 26;
if (i->srcExists(2) && i->getSrc(2)->reg.file == FILE_MEMORY_CONST)
assert(!(code[1] & 0xc000));
code[1] |= (s == 2) ? 0x8000 : 0x4000;
code[1] |= i->getSrc(s)->reg.fileIndex << 10;
- setAddress16(i->src[s]);
+ setAddress16(i->src(s));
break;
case FILE_IMMEDIATE:
assert(s == 1 ||
case FILE_GPR:
if ((s == 2) && ((code[0] & 0x7) == 2)) // LIMM: 3rd src == dst
break;
- srcId(i->src[s], s ? ((s == 2) ? 49 : s1) : 20);
+ srcId(i->src(s), s ? ((s == 2) ? 49 : s1) : 20);
break;
default:
// ignore here, can be predicate or flags, but must not be address
emitPredicate(i);
- defId(i->def[0], 14);
+ defId(i->def(0), 14);
- switch (i->src[0].getFile()) {
+ switch (i->src(0).getFile()) {
case FILE_MEMORY_CONST:
assert(!(code[1] & 0xc000));
- code[1] |= 0x4000 | (i->src[0].get()->reg.fileIndex << 10);
- setAddress16(i->src[0]);
+ code[1] |= 0x4000 | (i->src(0).get()->reg.fileIndex << 10);
+ setAddress16(i->src(0));
break;
case FILE_IMMEDIATE:
assert(!(code[1] & 0xc000));
setImmediate(i, 0);
break;
case FILE_GPR:
- srcId(i->src[0], 26);
+ srcId(i->src(0), 26);
break;
default:
// ignore here, can be predicate or flags, but must not be address
if (opc == 0x0d || opc == 0x0e)
ss2a = 2;
- defId(i->def[0], 14);
- srcId(i->src[0], 20);
+ defId(i->def(0), 14);
+ srcId(i->src(0), 20);
assert(pred || (i->predSrc < 0));
if (pred)
emitPredicate(i);
for (int s = 1; s < 3 && i->srcExists(s); ++s) {
- if (i->src[s].get()->reg.file == FILE_MEMORY_CONST) {
+ if (i->src(s).get()->reg.file == FILE_MEMORY_CONST) {
assert(!(code[0] & (0x300 >> ss2a)));
- switch (i->src[s].get()->reg.fileIndex) {
+ switch (i->src(s).get()->reg.fileIndex) {
case 0: code[0] |= 0x100 >> ss2a; break;
case 1: code[0] |= 0x200 >> ss2a; break;
case 16: code[0] |= 0x300 >> ss2a; break;
else
code[0] |= i->getSrc(s)->reg.data.offset << 6;
} else
- if (i->src[s].getFile() == FILE_IMMEDIATE) {
+ if (i->src(s).getFile() == FILE_IMMEDIATE) {
assert(s == 1);
- setImmediateS8(i->src[s]);
+ setImmediateS8(i->src(s));
} else
- if (i->src[s].getFile() == FILE_GPR) {
- srcId(i->src[s], (s == 1) ? 26 : 8);
+ if (i->src(s).getFile() == FILE_GPR) {
+ srcId(i->src(s), (s == 1) ? 26 : 8);
}
}
}
void
CodeEmitterNVC0::emitFMAD(const Instruction *i)
{
- bool neg1 = (i->src[0].mod ^ i->src[1].mod).neg();
+ bool neg1 = (i->src(0).mod ^ i->src(1).mod).neg();
if (i->encSize == 8) {
- if (isLIMM(i->src[1], TYPE_F32)) {
+ if (isLIMM(i->src(1), TYPE_F32)) {
emitForm_A(i, HEX64(20000000, 00000002));
} else {
emitForm_A(i, HEX64(30000000, 00000000));
- if (i->src[2].mod.neg())
+ if (i->src(2).mod.neg())
code[0] |= 1 << 8;
}
roundMode_A(i);
if (i->ftz)
code[0] |= 1 << 6;
} else {
- assert(!i->saturate && !i->src[2].mod.neg());
- emitForm_S(i, (i->src[2].getFile() == FILE_MEMORY_CONST) ? 0x2e : 0x0e,
+ assert(!i->saturate && !i->src(2).mod.neg());
+ emitForm_S(i, (i->src(2).getFile() == FILE_MEMORY_CONST) ? 0x2e : 0x0e,
false);
if (neg1)
code[0] |= 1 << 4;
void
CodeEmitterNVC0::emitFMUL(const Instruction *i)
{
- bool neg = (i->src[0].mod ^ i->src[1].mod).neg();
+ bool neg = (i->src(0).mod ^ i->src(1).mod).neg();
assert(i->postFactor >= -3 && i->postFactor <= 3);
if (i->encSize == 8) {
- if (isLIMM(i->src[1], TYPE_F32)) {
+ if (isLIMM(i->src(1), TYPE_F32)) {
assert(i->postFactor == 0); // constant folded, hopefully
emitForm_A(i, HEX64(30000000, 00000002));
} else {
CodeEmitterNVC0::emitUMUL(const Instruction *i)
{
if (i->encSize == 8) {
- if (i->src[1].getFile() == FILE_IMMEDIATE) {
+ if (i->src(1).getFile() == FILE_IMMEDIATE) {
emitForm_A(i, HEX64(10000000, 00000002));
} else {
emitForm_A(i, HEX64(50000000, 00000003));
if (i->dType == TYPE_S32)
code[0] |= 1 << 7;
} else {
- emitForm_S(i, i->src[1].getFile() == FILE_IMMEDIATE ? 0xaa : 0x2a, true);
+ emitForm_S(i, i->src(1).getFile() == FILE_IMMEDIATE ? 0xaa : 0x2a, true);
if (i->sType == TYPE_S32)
code[0] |= 1 << 6;
CodeEmitterNVC0::emitFADD(const Instruction *i)
{
if (i->encSize == 8) {
- if (isLIMM(i->src[1], TYPE_F32)) {
+ if (isLIMM(i->src(1), TYPE_F32)) {
+ assert(!i->saturate);
emitForm_A(i, HEX64(28000000, 00000002));
- assert(!i->src[1].mod.neg() && !i->src[1].mod.abs() && !i->saturate);
+ code[0] |= i->src(0).mod.abs() << 7;
+ code[0] |= i->src(0).mod.neg() << 9;
+
+ if (i->src(1).mod.abs())
+ code[1] &= 0xfdffffff;
+ if ((i->op == OP_SUB) != static_cast<bool>(i->src(1).mod.neg()))
+ code[1] ^= 0x02000000;
} else {
emitForm_A(i, HEX64(50000000, 00000000));
roundMode_A(i);
if (i->saturate)
code[1] |= 1 << 17;
- }
- emitNegAbs12(i);
-
- if (i->op == OP_SUB) code[0] ^= 1 << 8;
+ emitNegAbs12(i);
+ if (i->op == OP_SUB) code[0] ^= 1 << 8;
+ }
if (i->ftz)
code[0] |= 1 << 5;
} else {
assert(!i->saturate && i->op != OP_SUB &&
- !i->src[0].mod.abs() &&
- !i->src[1].mod.neg() && !i->src[1].mod.abs());
+ !i->src(0).mod.abs() &&
+ !i->src(1).mod.neg() && !i->src(1).mod.abs());
emitForm_S(i, 0x49, true);
- if (i->src[0].mod.neg())
+ if (i->src(0).mod.neg())
code[0] |= 1 << 7;
}
}
{
uint32_t addOp = 0;
- assert(!i->src[0].mod.abs() && !i->src[1].mod.abs());
- assert(!i->src[0].mod.neg() || !i->src[1].mod.neg());
+ assert(!i->src(0).mod.abs() && !i->src(1).mod.abs());
+ assert(!i->src(0).mod.neg() || !i->src(1).mod.neg());
- if (i->src[0].mod.neg())
+ if (i->src(0).mod.neg())
addOp |= 0x200;
- if (i->src[1].mod.neg())
+ if (i->src(1).mod.neg())
addOp |= 0x100;
if (i->op == OP_SUB) {
addOp ^= 0x100;
}
if (i->encSize == 8) {
- if (isLIMM(i->src[1], TYPE_U32)) {
+ if (isLIMM(i->src(1), TYPE_U32)) {
emitForm_A(i, HEX64(08000000, 00000002));
- if (i->def[1].exists())
+ if (i->defExists(1))
code[1] |= 1 << 26; // write carry
} else {
emitForm_A(i, HEX64(48000000, 00000003));
- if (i->def[1].exists())
+ if (i->defExists(1))
code[1] |= 1 << 16; // write carry
}
code[0] |= addOp;
} else {
assert(!(addOp & 0x100));
emitForm_S(i, (addOp >> 3) |
- ((i->src[1].getFile() == FILE_IMMEDIATE) ? 0xac : 0x2c), true);
+ ((i->src(1).getFile() == FILE_IMMEDIATE) ? 0xac : 0x2c), true);
}
}
if (i->flagsDef >= 0) code[1] |= 1 << 16;
if (i->flagsSrc >= 0) code[1] |= 1 << 23;
- if (i->src[2].mod.neg()) code[0] |= 0x10;
- if (i->src[1].mod.neg() ^
- i->src[0].mod.neg()) code[0] |= 0x20;
+ if (i->src(2).mod.neg()) code[0] |= 0x10;
+ if (i->src(1).mod.neg() ^
+ i->src(0).mod.neg()) code[0] |= 0x20;
if (i->subOp == NV50_IR_SUBOP_MUL_HIGH)
code[0] |= 1 << 6;
}
+void
+CodeEmitterNVC0::emitMADSP(const Instruction *i)
+{
+ assert(targ->getChipset() >= NVISA_GK104_CHIPSET);
+
+ emitForm_A(i, HEX64(00000000, 00000003));
+
+ if (i->subOp == NV50_IR_SUBOP_MADSP_SD) {
+ code[1] |= 0x01800000;
+ } else {
+ code[0] |= (i->subOp & 0x00f) << 7;
+ code[0] |= (i->subOp & 0x0f0) << 1;
+ code[0] |= (i->subOp & 0x100) >> 3;
+ code[0] |= (i->subOp & 0x200) >> 2;
+ code[1] |= (i->subOp & 0xc00) << 13;
+ }
+
+ if (i->flagsDef >= 0)
+ code[1] |= 1 << 16;
+}
+
+void
+CodeEmitterNVC0::emitISAD(const Instruction *i)
+{
+ assert(i->dType == TYPE_S32 || i->dType == TYPE_U32);
+ assert(i->encSize == 8);
+
+ emitForm_A(i, HEX64(38000000, 00000003));
+
+ if (i->dType == TYPE_S32)
+ code[0] |= 1 << 5;
+}
+
void
CodeEmitterNVC0::emitNOT(Instruction *i)
{
assert(i->encSize == 8);
- i->src[1].set(i->src[0]);
+ i->setSrc(1, i->src(0));
emitForm_A(i, HEX64(68000000, 000001c3));
}
void
CodeEmitterNVC0::emitLogicOp(const Instruction *i, uint8_t subOp)
{
+ if (i->def(0).getFile() == FILE_PREDICATE) {
+ code[0] = 0x00000004 | (subOp << 30);
+ code[1] = 0x0c000000;
+
+ emitPredicate(i);
+
+ defId(i->def(0), 17);
+ srcId(i->src(0), 20);
+ if (i->src(0).mod == Modifier(NV50_IR_MOD_NOT)) code[0] |= 1 << 23;
+ srcId(i->src(1), 26);
+ if (i->src(1).mod == Modifier(NV50_IR_MOD_NOT)) code[0] |= 1 << 29;
+
+ if (i->defExists(1)) {
+ defId(i->def(1), 14);
+ } else {
+ code[0] |= 7 << 14;
+ }
+ // (a OP b) OP c
+ if (i->predSrc != 2 && i->srcExists(2)) {
+ code[1] |= subOp << 21;
+ srcId(i->src(2), 17);
+ if (i->src(2).mod == Modifier(NV50_IR_MOD_NOT)) code[0] |= 1 << 20;
+ } else {
+ code[1] |= 0x000e0000;
+ }
+ } else
if (i->encSize == 8) {
- if (isLIMM(i->src[1], TYPE_U32)) {
+ if (isLIMM(i->src(1), TYPE_U32)) {
emitForm_A(i, HEX64(38000000, 00000002));
- if (i->src[2].exists())
+ if (i->flagsDef >= 0)
code[1] |= 1 << 26;
} else {
emitForm_A(i, HEX64(68000000, 00000003));
- if (i->src[2].exists())
+ if (i->flagsDef >= 0)
code[1] |= 1 << 16;
}
code[0] |= subOp << 6;
- if (i->src[2].exists()) // carry
+ if (i->flagsSrc >= 0) // carry
code[0] |= 1 << 5;
- if (i->src[0].mod & Modifier(NV50_IR_MOD_NOT)) code[0] |= 1 << 9;
- if (i->src[1].mod & Modifier(NV50_IR_MOD_NOT)) code[0] |= 1 << 8;
+ if (i->src(0).mod & Modifier(NV50_IR_MOD_NOT)) code[0] |= 1 << 9;
+ if (i->src(1).mod & Modifier(NV50_IR_MOD_NOT)) code[0] |= 1 << 8;
} else {
emitForm_S(i, (subOp << 5) |
- ((i->src[1].getFile() == FILE_IMMEDIATE) ? 0x1d : 0x8d), true);
+ ((i->src(1).getFile() == FILE_IMMEDIATE) ? 0x1d : 0x8d), true);
}
}
{
emitForm_A(i, HEX64(54000000, 00000004));
- if (i->src[0].mod & Modifier(NV50_IR_MOD_NOT)) code[0] |= 1 << 9;
- if (i->src[1].mod & Modifier(NV50_IR_MOD_NOT)) code[0] |= 1 << 8;
+ if (i->src(0).mod & Modifier(NV50_IR_MOD_NOT)) code[0] |= 1 << 9;
+ if (i->src(1).mod & Modifier(NV50_IR_MOD_NOT)) code[0] |= 1 << 8;
}
void
if (i->op == OP_PREEX2)
code[0] |= 0x20;
- if (i->src[0].mod.abs()) code[0] |= 1 << 6;
- if (i->src[0].mod.neg()) code[0] |= 1 << 8;
+ if (i->src(0).mod.abs()) code[0] |= 1 << 6;
+ if (i->src(0).mod.neg()) code[0] |= 1 << 8;
} else {
emitForm_S(i, i->op == OP_PREEX2 ? 0x74000008 : 0x70000008, true);
}
emitPredicate(i);
- defId(i->def[0], 14);
- srcId(i->src[0], 20);
+ defId(i->def(0), 14);
+ srcId(i->src(0), 20);
- assert(i->src[0].getFile() == FILE_GPR);
+ assert(i->src(0).getFile() == FILE_GPR);
if (i->saturate) code[0] |= 1 << 5;
- if (i->src[0].mod.abs()) code[0] |= 1 << 7;
- if (i->src[0].mod.neg()) code[0] |= 1 << 9;
+ if (i->src(0).mod.abs()) code[0] |= 1 << 7;
+ if (i->src(0).mod.neg()) code[0] |= 1 << 9;
} else {
emitForm_S(i, 0x80000008 | (subOp << 26), true);
- assert(!i->src[0].mod.neg());
- if (i->src[0].mod.abs()) code[0] |= 1 << 30;
+ assert(!i->src(0).mod.neg());
+ if (i->src(0).mod.abs()) code[0] |= 1 << 30;
}
}
}
const bool sat = (i->op == OP_SAT) || i->saturate;
- const bool abs = (i->op == OP_ABS) || i->src[0].mod.abs();
- const bool neg = (i->op == OP_NEG) || i->src[0].mod.neg();
+ const bool abs = (i->op == OP_ABS) || i->src(0).mod.abs();
+ const bool neg = (i->op == OP_NEG) || i->src(0).mod.neg();
if (i->encSize == 8) {
emitForm_B(i, HEX64(10000000, 00000004));
}
emitForm_A(i, (static_cast<uint64_t>(hi) << 32) | lo);
- if (i->def[0].getFile() == FILE_PREDICATE) {
+ if (i->op != OP_SET)
+ srcId(i->src(2), 32 + 17);
+
+ if (i->def(0).getFile() == FILE_PREDICATE) {
if (i->sType == TYPE_F32)
code[1] += 0x10000000;
else
code[1] += 0x08000000;
code[0] &= ~0xfc000;
- defId(i->def[0], 17);
+ defId(i->def(0), 17);
if (i->defExists(1))
- defId(i->def[1], 14);
+ defId(i->def(1), 14);
else
code[0] |= 0x1c000;
}
CondCode cc = i->setCond;
- if (i->src[2].mod.neg())
+ if (i->src(2).mod.neg())
cc = reverseCondCode(cc);
emitCondCode(cc, 32 + 23);
{
emitForm_A(i, HEX64(20000000, 00000004));
- if (i->cc == CC_NOT_P || i->src[2].mod & Modifier(NV50_IR_MOD_NOT))
+ if (i->cc == CC_NOT_P || i->src(2).mod & Modifier(NV50_IR_MOD_NOT))
code[1] |= 1 << 20;
}
+void CodeEmitterNVC0::emitTEXBAR(const Instruction *i)
+{
+ code[0] = 0x00000006 | (i->subOp << 26);
+ code[1] = 0xf0000000;
+ emitPredicate(i);
+ emitCondCode(i->flagsSrc >= 0 ? i->cc : CC_ALWAYS, 5);
+}
+
void CodeEmitterNVC0::emitTEXCSAA(const TexInstruction *i)
{
code[0] = 0x00000086;
if (i->tex.liveOnly)
code[0] |= 1 << 9;
- defId(i->def[0], 14);
- srcId(i->src[0], 20);
+ defId(i->def(0), 14);
+ srcId(i->src(0), 20);
+}
+
+static inline bool
+isNextIndependentTex(const TexInstruction *i)
+{
+ if (!i->next || !isTextureOp(i->next->op))
+ return false;
+ if (i->getDef(0)->interfers(i->next->getSrc(0)))
+ return false;
+ return !i->next->srcExists(1) || !i->getDef(0)->interfers(i->next->getSrc(1));
}
void
{
code[0] = 0x00000006;
- if (1)
- code[0] |= 0x80; // normal/t/p mode = t, XXX: what is this ?
+ if (isNextIndependentTex(i))
+ code[0] |= 0x080; // t mode
+ else
+ code[0] |= 0x100; // p mode
if (i->tex.liveOnly)
code[0] |= 1 << 9;
code[1] |= 0x02000000;
}
- defId(i->def[0], 14);
- srcId(i->src[0], 20);
+ if (i->op != OP_TXD && i->tex.derivAll)
+ code[1] |= 1 << 13;
+
+ defId(i->def(0), 14);
+ srcId(i->src(0), 20);
emitPredicate(i);
if (i->tex.target.isShadow())
code[1] |= 1 << 24;
- int src1 = i->tex.target.getArgCount();
+ const int src1 = (i->predSrc == 1) ? 2 : 1; // if predSrc == 1, !srcExists(2)
- if (i->src[src1].getFile() == FILE_IMMEDIATE) { // lzero
+ if (i->srcExists(src1) && i->src(src1).getFile() == FILE_IMMEDIATE) {
+ // lzero
if (i->op == OP_TXL)
code[1] &= ~(1 << 26);
else
if (i->tex.useOffsets) // in vecSrc0.w
code[1] |= 1 << 22;
- srcId(i->src[src1], 26);
+ srcId(i, src1, 26);
}
void
if (i->tex.sIndirectSrc >= 0 || i->tex.rIndirectSrc >= 0)
code[1] |= 1 << 18;
- defId(i->def[0], 14);
- srcId(i->src[0], 20);
- srcId(i->src[1], 26);
+ const int src1 = (i->predSrc == 1) ? 2 : 1; // if predSrc == 1, !srcExists(2)
+
+ defId(i->def(0), 14);
+ srcId(i->src(0), 20);
+ srcId(i, src1, 26);
emitPredicate(i);
}
code[0] = 0x00000000 | (laneMask << 6);
code[1] = 0x48000000 | qOp;
- defId(i->def[0], 14);
- srcId(i->src[0], 20);
- srcId(i->srcExists(1) ? i->src[1] : i->src[0], 26);
+ defId(i->def(0), 14);
+ srcId(i->src(0), 20);
+ srcId(i->srcExists(1) ? i->src(1) : i->src(0), 26);
+
+ if (i->op == OP_QUADOP && progType != Program::TYPE_FRAGMENT)
+ code[0] |= 1 << 9; // dall
emitPredicate(i);
}
switch (i->op) {
case OP_BRA:
code[1] = f->absolute ? 0x00000000 : 0x40000000;
- if (i->src[0].getFile() == FILE_MEMORY_CONST ||
- i->src[1].getFile() == FILE_MEMORY_CONST)
- code[1] |= 0x4000;
+ if (i->srcExists(0) && i->src(0).getFile() == FILE_MEMORY_CONST)
+ code[0] |= 0x4000;
mask = 3;
break;
case OP_CALL:
code[1] = f->absolute ? 0x10000000 : 0x50000000;
- if (i->src[0].getFile() == FILE_MEMORY_CONST)
- code[1] |= 0x4000;
+ if (f->indirect)
+ code[0] |= 0x4000; // indirect calls always use c[] source
mask = 2;
break;
if (f->limit)
code[0] |= 1 << 16;
+ if (f->indirect) {
+ if (code[0] & 0x4000) {
+ assert(i->srcExists(0) && i->src(0).getFile() == FILE_MEMORY_CONST);
+ setAddress16(i->src(0));
+ code[1] |= i->getSrc(0)->reg.fileIndex << 10;
+ if (f->op == OP_BRA)
+ srcId(f->src(0).getIndirect(0), 20);
+ } else {
+ srcId(f, 0, 20);
+ }
+ }
+
if (f->op == OP_CALL) {
+ if (f->indirect) {
+ // nothing
+ } else
if (f->builtin) {
assert(f->absolute);
- uint32_t pcAbs = targ->getBuiltinOffset(f->target.builtin);
+ uint32_t pcAbs = targNVC0->getBuiltinOffset(f->target.builtin);
addReloc(RelocEntry::TYPE_BUILTIN, 0, pcAbs, 0xfc000000, 26);
addReloc(RelocEntry::TYPE_BUILTIN, 1, pcAbs, 0x03ffffff, -6);
} else {
void
CodeEmitterNVC0::emitPFETCH(const Instruction *i)
{
- uint32_t prim = i->src[0].get()->reg.data.u32;
+ uint32_t prim = i->src(0).get()->reg.data.u32;
code[0] = 0x00000006 | ((prim & 0x3f) << 26);
code[1] = 0x00000000 | (prim >> 6);
emitPredicate(i);
- defId(i->def[0], 14);
- srcId(i->src[1], 20);
+ defId(i->def(0), 14);
+ srcId(i->src(1), 20);
}
void
CodeEmitterNVC0::emitVFETCH(const Instruction *i)
{
code[0] = 0x00000006;
- code[1] = 0x06000000 | i->src[0].get()->reg.data.offset;
+ code[1] = 0x06000000 | i->src(0).get()->reg.data.offset;
if (i->perPatch)
code[0] |= 0x100;
emitPredicate(i);
- code[0] |= (i->defCount(0xf) - 1) << 5;
+ code[0] |= ((i->getDef(0)->reg.size / 4) - 1) << 5;
- defId(i->def[0], 14);
- srcId(i->src[0].getIndirect(0), 20);
- srcId(i->src[0].getIndirect(1), 26); // vertex address
+ defId(i->def(0), 14);
+ srcId(i->src(0).getIndirect(0), 20);
+ srcId(i->src(0).getIndirect(1), 26); // vertex address
}
void
unsigned int size = typeSizeof(i->dType);
code[0] = 0x00000006 | ((size / 4 - 1) << 5);
- code[1] = 0x0a000000 | i->src[0].get()->reg.data.offset;
+ code[1] = 0x0a000000 | i->src(0).get()->reg.data.offset;
- assert(size != 12 && !(code[1] & (size - 1)));
+ assert(!(code[1] & ((size == 12) ? 15 : (size - 1))));
if (i->perPatch)
code[0] |= 0x100;
emitPredicate(i);
- assert(i->src[1].getFile() == FILE_GPR);
+ assert(i->src(1).getFile() == FILE_GPR);
- srcId(i->src[0].getIndirect(0), 20);
- srcId(i->src[0].getIndirect(1), 32 + 17); // vertex base address
- srcId(i->src[1], 26);
+ srcId(i->src(0).getIndirect(0), 20);
+ srcId(i->src(0).getIndirect(1), 32 + 17); // vertex base address
+ srcId(i->src(1), 26);
}
void
emitPredicate(i);
- defId(i->def[0], 14); // new secret address
- srcId(i->src[0], 20); // old secret address, should be 0 initially
+ defId(i->def(0), 14); // new secret address
+ srcId(i->src(0), 20); // old secret address, should be 0 initially
- assert(i->src[0].getFile() == FILE_GPR);
+ assert(i->src(0).getFile() == FILE_GPR);
if (i->op == OP_EMIT)
code[0] |= 1 << 5;
code[0] |= 1 << 6;
// vertex stream
- if (i->src[1].getFile() == FILE_IMMEDIATE) {
+ if (i->src(1).getFile() == FILE_IMMEDIATE) {
code[1] |= 0xc000;
- code[0] |= SDATA(i->src[1]).u32 << 26;
+ code[0] |= SDATA(i->src(1)).u32 << 26;
} else {
- srcId(i->src[1], 26);
+ srcId(i->src(1), 26);
}
}
code[0] |= 1 << 5;
if (i->op == OP_PINTERP)
- srcId(i->src[1], 26);
+ srcId(i->src(1), 26);
else
code[0] |= 0x3f << 26;
- srcId(i->src[0].getIndirect(0), 20);
+ srcId(i->src(0).getIndirect(0), 20);
} else {
assert(i->op == OP_PINTERP);
code[0] = 0x00000009 | ((base & 0xc) << 6) | ((base >> 4) << 26);
- srcId(i->src[1], 20);
+ srcId(i->src(1), 20);
}
emitInterpMode(i);
emitPredicate(i);
- defId(i->def[0], 14);
+ defId(i->def(0), 14);
if (i->getSampleMode() == NV50_IR_INTERP_OFFSET)
- srcId(i->src[i->op == OP_PINTERP ? 2 : 1], 17);
+ srcId(i->src(i->op == OP_PINTERP ? 2 : 1), 17);
else
code[1] |= 0x3f << 17;
}
code[0] |= val;
}
+static inline bool
+uses64bitAddress(const Instruction *ldst)
+{
+ return ldst->src(0).getFile() == FILE_MEMORY_GLOBAL &&
+ ldst->src(0).isIndirect(0) &&
+ ldst->getIndirect(0, 0)->reg.size == 8;
+}
+
void
CodeEmitterNVC0::emitSTORE(const Instruction *i)
{
uint32_t opc;
- switch (i->src[0].getFile()) {
+ switch (i->src(0).getFile()) {
case FILE_MEMORY_GLOBAL: opc = 0x90000000; break;
case FILE_MEMORY_LOCAL: opc = 0xc8000000; break;
case FILE_MEMORY_SHARED: opc = 0xc9000000; break;
code[0] = 0x00000005;
code[1] = opc;
- setAddress16(i->src[0]);
- srcId(i->src[1], 14);
- srcId(i->src[0].getIndirect(0), 20);
+ setAddress16(i->src(0));
+ srcId(i->src(1), 14);
+ srcId(i->src(0).getIndirect(0), 20);
+ if (uses64bitAddress(i))
+ code[1] |= 1 << 26;
emitPredicate(i);
code[0] = 0x00000005;
- switch (i->src[0].getFile()) {
+ switch (i->src(0).getFile()) {
case FILE_MEMORY_GLOBAL: opc = 0x80000000; break;
case FILE_MEMORY_LOCAL: opc = 0xc0000000; break;
case FILE_MEMORY_SHARED: opc = 0xc1000000; break;
case FILE_MEMORY_CONST:
- if (!i->src[0].isIndirect(0) && typeSizeof(i->dType) == 4) {
+ if (!i->src(0).isIndirect(0) && typeSizeof(i->dType) == 4) {
emitMOV(i); // not sure if this is any better
return;
}
- opc = 0x14000000 | (i->src[0].get()->reg.fileIndex << 10);
+ opc = 0x14000000 | (i->src(0).get()->reg.fileIndex << 10);
code[0] = 0x00000006 | (i->subOp << 8);
break;
default:
}
code[1] = opc;
- defId(i->def[0], 14);
+ defId(i->def(0), 14);
- setAddress16(i->src[0]);
- srcId(i->src[0].getIndirect(0), 20);
+ setAddress16(i->src(0));
+ srcId(i->src(0).getIndirect(0), 20);
+ if (uses64bitAddress(i))
+ code[1] |= 1 << 26;
emitPredicate(i);
void
CodeEmitterNVC0::emitMOV(const Instruction *i)
{
- if (i->src[0].getFile() == FILE_SYSTEM_VALUE) {
- uint8_t sr = getSRegEncoding(i->src[0]);
+ if (i->def(0).getFile() == FILE_PREDICATE) {
+ if (i->src(0).getFile() == FILE_GPR) {
+ code[0] = 0xfc01c003;
+ code[1] = 0x1a8e0000;
+ srcId(i->src(0), 20);
+ } else {
+ code[0] = 0x0001c004;
+ code[1] = 0x0c0e0000;
+ if (i->src(0).getFile() == FILE_IMMEDIATE) {
+ code[0] |= 7 << 20;
+ if (!i->getSrc(0)->reg.data.u32)
+ code[0] |= 1 << 23;
+ } else {
+ srcId(i->src(0), 20);
+ }
+ }
+ defId(i->def(0), 17);
+ emitPredicate(i);
+ } else
+ if (i->src(0).getFile() == FILE_SYSTEM_VALUE) {
+ uint8_t sr = getSRegEncoding(i->src(0));
if (i->encSize == 8) {
code[0] = 0x00000004 | (sr << 26);
} else {
code[0] = 0x40000008 | (sr << 20);
}
- defId(i->def[0], 14);
+ defId(i->def(0), 14);
emitPredicate(i);
} else
if (i->encSize == 8) {
uint64_t opc;
- if (i->src[0].getFile() == FILE_IMMEDIATE)
+ if (i->src(0).getFile() == FILE_IMMEDIATE)
opc = HEX64(18000000, 000001e2);
else
- if (i->src[0].getFile() == FILE_PREDICATE)
+ if (i->src(0).getFile() == FILE_PREDICATE)
opc = HEX64(080e0000, 1c000004);
else
opc = HEX64(28000000, 00000004);
} else {
uint32_t imm;
- if (i->src[0].getFile() == FILE_IMMEDIATE) {
- imm = SDATA(i->src[0]).u32;
+ if (i->src(0).getFile() == FILE_IMMEDIATE) {
+ imm = SDATA(i->src(0)).u32;
if (imm & 0xfff00000) {
assert(!(imm & 0x000fffff));
code[0] = 0x00000318 | imm;
}
} else {
code[0] = 0x0028;
- emitShortSrc2(i->src[0]);
+ emitShortSrc2(i->src(0));
}
- defId(i->def[0], 14);
+ defId(i->def(0), 14);
emitPredicate(i);
}
}
+void
+CodeEmitterNVC0::emitSUCLAMPMode(uint16_t subOp)
+{
+ uint8_t m;
+ switch (subOp & ~NV50_IR_SUBOP_SUCLAMP_2D) {
+ case NV50_IR_SUBOP_SUCLAMP_SD(0, 1): m = 0; break;
+ case NV50_IR_SUBOP_SUCLAMP_SD(1, 1): m = 1; break;
+ case NV50_IR_SUBOP_SUCLAMP_SD(2, 1): m = 2; break;
+ case NV50_IR_SUBOP_SUCLAMP_SD(3, 1): m = 3; break;
+ case NV50_IR_SUBOP_SUCLAMP_SD(4, 1): m = 4; break;
+ case NV50_IR_SUBOP_SUCLAMP_PL(0, 1): m = 5; break;
+ case NV50_IR_SUBOP_SUCLAMP_PL(1, 1): m = 6; break;
+ case NV50_IR_SUBOP_SUCLAMP_PL(2, 1): m = 7; break;
+ case NV50_IR_SUBOP_SUCLAMP_PL(3, 1): m = 8; break;
+ case NV50_IR_SUBOP_SUCLAMP_PL(4, 1): m = 9; break;
+ case NV50_IR_SUBOP_SUCLAMP_BL(0, 1): m = 10; break;
+ case NV50_IR_SUBOP_SUCLAMP_BL(1, 1): m = 11; break;
+ case NV50_IR_SUBOP_SUCLAMP_BL(2, 1): m = 12; break;
+ case NV50_IR_SUBOP_SUCLAMP_BL(3, 1): m = 13; break;
+ case NV50_IR_SUBOP_SUCLAMP_BL(4, 1): m = 14; break;
+ default:
+ return;
+ }
+ code[0] |= m << 5;
+ if (subOp & NV50_IR_SUBOP_SUCLAMP_2D)
+ code[1] |= 1 << 16;
+}
+
+void
+CodeEmitterNVC0::emitSUCalc(Instruction *i)
+{
+ ImmediateValue *imm = NULL;
+ uint64_t opc;
+
+ if (i->srcExists(2)) {
+ imm = i->getSrc(2)->asImm();
+ if (imm)
+ i->setSrc(2, NULL); // special case, make emitForm_A not assert
+ }
+
+ switch (i->op) {
+ case OP_SUCLAMP: opc = HEX64(58000000, 00000004); break;
+ case OP_SUBFM: opc = HEX64(5c000000, 00000004); break;
+ case OP_SUEAU: opc = HEX64(60000000, 00000004); break;
+ default:
+ assert(0);
+ return;
+ }
+ emitForm_A(i, opc);
+
+ if (i->op == OP_SUCLAMP) {
+ if (i->dType == TYPE_S32)
+ code[0] |= 1 << 9;
+ emitSUCLAMPMode(i->subOp);
+ }
+
+ if (i->op == OP_SUBFM && i->subOp == NV50_IR_SUBOP_SUBFM_3D)
+ code[1] |= 1 << 16;
+
+ if (i->op != OP_SUEAU) {
+ if (i->def(0).getFile() == FILE_PREDICATE) { // p, #
+ code[0] |= 63 << 14;
+ code[1] |= i->getDef(0)->reg.data.id << 23;
+ } else
+ if (i->defExists(1)) { // r, p
+ assert(i->def(1).getFile() == FILE_PREDICATE);
+ code[1] |= i->getDef(1)->reg.data.id << 23;
+ } else { // r, #
+ code[1] |= 7 << 23;
+ }
+ }
+ if (imm) {
+ assert(i->op == OP_SUCLAMP);
+ i->setSrc(2, imm);
+ code[1] |= (imm->reg.data.u32 & 0x3f) << 17; // sint6
+ }
+}
+
+void
+CodeEmitterNVC0::emitSUGType(DataType ty)
+{
+ switch (ty) {
+ case TYPE_S32: code[1] |= 1 << 13; break;
+ case TYPE_U8: code[1] |= 2 << 13; break;
+ case TYPE_S8: code[1] |= 3 << 13; break;
+ default:
+ assert(ty == TYPE_U32);
+ break;
+ }
+}
+
+void
+CodeEmitterNVC0::setSUConst16(const Instruction *i, const int s)
+{
+ const uint32_t offset = i->getSrc(s)->reg.data.offset;
+
+ assert(i->src(s).getFile() == FILE_MEMORY_CONST);
+ assert(offset == (offset & 0xfffc));
+
+ code[1] |= 1 << 21;
+ code[0] |= offset << 24;
+ code[1] |= offset >> 8;
+ code[1] |= i->getSrc(s)->reg.fileIndex << 8;
+}
+
+void
+CodeEmitterNVC0::setSUPred(const Instruction *i, const int s)
+{
+ if (!i->srcExists(s) || (i->predSrc == s)) {
+ code[1] |= 0x7 << 17;
+ } else {
+ if (i->src(s).mod == Modifier(NV50_IR_MOD_NOT))
+ code[1] |= 1 << 20;
+ srcId(i->src(s), 32 + 17);
+ }
+}
+
+void
+CodeEmitterNVC0::emitSULDGB(const TexInstruction *i)
+{
+ code[0] = 0x5;
+ code[1] = 0xd4000000 | (i->subOp << 15);
+
+ emitLoadStoreType(i->dType);
+ emitSUGType(i->sType);
+ emitCachingMode(i->cache);
+
+ emitPredicate(i);
+ defId(i->def(0), 14); // destination
+ srcId(i->src(0), 20); // address
+ // format
+ if (i->src(1).getFile() == FILE_GPR)
+ srcId(i->src(1), 26);
+ else
+ setSUConst16(i, 1);
+ setSUPred(i, 2);
+}
+
+void
+CodeEmitterNVC0::emitSUSTGx(const TexInstruction *i)
+{
+ code[0] = 0x5;
+ code[1] = 0xdc000000 | (i->subOp << 15);
+
+ if (i->op == OP_SUSTP)
+ code[1] |= i->tex.mask << 22;
+ else
+ emitLoadStoreType(i->dType);
+ emitSUGType(i->sType);
+ emitCachingMode(i->cache);
+
+ emitPredicate(i);
+ srcId(i->src(0), 20); // address
+ // format
+ if (i->src(1).getFile() == FILE_GPR)
+ srcId(i->src(1), 26);
+ else
+ setSUConst16(i, 1);
+ srcId(i->src(3), 14); // values
+ setSUPred(i, 2);
+}
+
+void
+CodeEmitterNVC0::emitVectorSubOp(const Instruction *i)
+{
+ switch (NV50_IR_SUBOP_Vn(i->subOp)) {
+ case 0:
+ code[1] |= (i->subOp & 0x000f) << 12; // vsrc1
+ code[1] |= (i->subOp & 0x00e0) >> 5; // vsrc2
+ code[1] |= (i->subOp & 0x0100) << 7; // vsrc2
+ code[1] |= (i->subOp & 0x3c00) << 13; // vdst
+ break;
+ case 1:
+ code[1] |= (i->subOp & 0x000f) << 8; // v2src1
+ code[1] |= (i->subOp & 0x0010) << 11; // v2src1
+ code[1] |= (i->subOp & 0x01e0) >> 1; // v2src2
+ code[1] |= (i->subOp & 0x0200) << 6; // v2src2
+ code[1] |= (i->subOp & 0x3c00) << 2; // v4dst
+ code[1] |= (i->mask & 0x3) << 2;
+ break;
+ case 2:
+ code[1] |= (i->subOp & 0x000f) << 8; // v4src1
+ code[1] |= (i->subOp & 0x01e0) >> 1; // v4src2
+ code[1] |= (i->subOp & 0x3c00) << 2; // v4dst
+ code[1] |= (i->mask & 0x3) << 2;
+ code[1] |= (i->mask & 0xc) << 21;
+ break;
+ default:
+ assert(0);
+ break;
+ }
+}
+
+void
+CodeEmitterNVC0::emitVSHL(const Instruction *i)
+{
+ uint64_t opc = 0x4;
+
+ switch (NV50_IR_SUBOP_Vn(i->subOp)) {
+ case 0: opc |= 0xe8ULL << 56; break;
+ case 1: opc |= 0xb4ULL << 56; break;
+ case 2: opc |= 0x94ULL << 56; break;
+ default:
+ assert(0);
+ break;
+ }
+ if (NV50_IR_SUBOP_Vn(i->subOp) == 1) {
+ if (isSignedType(i->dType)) opc |= 1ULL << 0x2a;
+ if (isSignedType(i->sType)) opc |= (1 << 6) | (1 << 5);
+ } else {
+ if (isSignedType(i->dType)) opc |= 1ULL << 0x39;
+ if (isSignedType(i->sType)) opc |= 1 << 6;
+ }
+ emitForm_A(i, opc);
+ emitVectorSubOp(i);
+
+ if (i->saturate)
+ code[0] |= 1 << 9;
+ if (i->flagsDef >= 0)
+ code[1] |= 1 << 16;
+}
+
bool
CodeEmitterNVC0::emitInstruction(Instruction *insn)
{
+ unsigned int size = insn->encSize;
+
+ if (writeIssueDelays && !(codeSize & 0x3f))
+ size += 8;
+
if (!insn->encSize) {
ERROR("skipping unencodable instruction: "); insn->print();
return false;
} else
- if (codeSize + insn->encSize > codeSizeLimit) {
+ if (codeSize + size > codeSizeLimit) {
ERROR("code emitter output buffer too small\n");
return false;
}
+ if (writeIssueDelays) {
+ if (!(codeSize & 0x3f)) {
+ code[0] = 0x00000007; // cf issue delay "instruction"
+ code[1] = 0x20000000;
+ code += 2;
+ codeSize += 8;
+ }
+ const unsigned int id = (codeSize & 0x3f) / 8 - 1;
+ uint32_t *data = code - (id * 2 + 2);
+ if (id <= 2) {
+ data[0] |= insn->sched << (id * 8 + 4);
+ } else
+ if (id == 3) {
+ data[0] |= insn->sched << 28;
+ data[1] |= insn->sched >> 4;
+ } else {
+ data[1] |= insn->sched << ((id - 4) * 8 + 4);
+ }
+ }
+
// assert that instructions with multiple defs don't corrupt registers
for (int d = 0; insn->defExists(d); ++d)
- assert(insn->asTex() || insn->def[d].rep()->reg.data.id >= 0);
+ assert(insn->asTex() || insn->def(d).rep()->reg.data.id >= 0);
switch (insn->op) {
case OP_MOV:
else
emitIMAD(insn);
break;
+ case OP_SAD:
+ emitISAD(insn);
+ break;
case OP_NOT:
emitNOT(insn);
break;
case OP_TXQ:
emitTXQ(insn->asTex());
break;
+ case OP_TEXBAR:
+ emitTEXBAR(insn);
+ break;
+ case OP_SUBFM:
+ case OP_SUCLAMP:
+ case OP_SUEAU:
+ emitSUCalc(insn);
+ break;
+ case OP_MADSP:
+ emitMADSP(insn);
+ break;
+ case OP_SULDB:
+ if (targ->getChipset() >= NVISA_GK104_CHIPSET)
+ emitSULDGB(insn->asTex());
+ else
+ ERROR("SULDB not yet supported on < nve4\n");
+ break;
+ case OP_SUSTB:
+ case OP_SUSTP:
+ if (targ->getChipset() >= NVISA_GK104_CHIPSET)
+ emitSUSTGx(insn->asTex());
+ else
+ ERROR("SUSTx not yet supported on < nve4\n");
+ break;
case OP_BRA:
case OP_CALL:
case OP_PRERET:
emitQUADOP(insn, insn->subOp, insn->lanes);
break;
case OP_DFDX:
- emitQUADOP(insn, insn->src[0].mod.neg() ? 0x66 : 0x99, 0x4);
+ emitQUADOP(insn, insn->src(0).mod.neg() ? 0x66 : 0x99, 0x4);
break;
case OP_DFDY:
- emitQUADOP(insn, insn->src[0].mod.neg() ? 0x5a : 0xa5, 0x5);
+ emitQUADOP(insn, insn->src(0).mod.neg() ? 0x5a : 0xa5, 0x5);
break;
case OP_POPCNT:
emitPOPC(insn);
emitNOP(insn);
insn->join = 1;
break;
+ case OP_VSHL:
+ emitVSHL(insn);
+ break;
case OP_PHI:
case OP_UNION:
case OP_CONSTRAINT:
{
const Target::OpInfo &info = targ->getOpInfo(i);
- if (info.minEncSize == 8 || 1)
+ if (writeIssueDelays || info.minEncSize == 8 || 1)
return 8;
if (i->ftz || i->saturate || i->join)
}
for (int s = 0; i->srcExists(s); ++s) {
- if (i->src[s].isIndirect(0))
+ if (i->src(s).isIndirect(0))
return 8;
- if (i->src[s].getFile() == FILE_MEMORY_CONST) {
- if (SDATA(i->src[s]).offset >= 0x100)
+ if (i->src(s).getFile() == FILE_MEMORY_CONST) {
+ if (SDATA(i->src(s)).offset >= 0x100)
return 8;
if (i->getSrc(s)->reg.fileIndex > 1 &&
i->getSrc(s)->reg.fileIndex != 16)
return 8;
} else
- if (i->src[s].getFile() == FILE_IMMEDIATE) {
+ if (i->src(s).getFile() == FILE_IMMEDIATE) {
if (i->dType == TYPE_F32) {
- if (SDATA(i->src[s]).u32 >= 0x100)
+ if (SDATA(i->src(s)).u32 >= 0x100)
return 8;
} else {
- if (SDATA(i->src[s]).u32 > 0xff)
+ if (SDATA(i->src(s)).u32 > 0xff)
return 8;
}
}
if (i->op == OP_CVT)
continue;
- if (i->src[s].mod != Modifier(0)) {
- if (i->src[s].mod == Modifier(NV50_IR_MOD_ABS))
+ if (i->src(s).mod != Modifier(0)) {
+ if (i->src(s).mod == Modifier(NV50_IR_MOD_ABS))
if (i->op != OP_RSQ)
return 8;
- if (i->src[s].mod == Modifier(NV50_IR_MOD_NEG))
+ if (i->src(s).mod == Modifier(NV50_IR_MOD_NEG))
if (i->op != OP_ADD || s != 0)
return 8;
}
return 4;
}
-CodeEmitterNVC0::CodeEmitterNVC0(const TargetNVC0 *target) : targ(target)
+// Simplified, erring on safe side.
+class SchedDataCalculator : public Pass
+{
+public:
+ SchedDataCalculator(const Target *targ) : targ(targ) { }
+
+private:
+ struct RegScores
+ {
+ struct Resource {
+ int st[DATA_FILE_COUNT]; // LD to LD delay 3
+ int ld[DATA_FILE_COUNT]; // ST to ST delay 3
+ int tex; // TEX to non-TEX delay 17 (0x11)
+ int sfu; // SFU to SFU delay 3 (except PRE-ops)
+ int imul; // integer MUL to MUL delay 3
+ } res;
+ struct ScoreData {
+ int r[64];
+ int p[8];
+ int c;
+ } rd, wr;
+ int base;
+
+ void rebase(const int base)
+ {
+ const int delta = this->base - base;
+ if (!delta)
+ return;
+ this->base = 0;
+
+ for (int i = 0; i < 64; ++i) {
+ rd.r[i] += delta;
+ wr.r[i] += delta;
+ }
+ for (int i = 0; i < 8; ++i) {
+ rd.p[i] += delta;
+ wr.p[i] += delta;
+ }
+ rd.c += delta;
+ wr.c += delta;
+
+ for (unsigned int f = 0; f < DATA_FILE_COUNT; ++f) {
+ res.ld[f] += delta;
+ res.st[f] += delta;
+ }
+ res.sfu += delta;
+ res.imul += delta;
+ res.tex += delta;
+ }
+ void wipe()
+ {
+ memset(&rd, 0, sizeof(rd));
+ memset(&wr, 0, sizeof(wr));
+ memset(&res, 0, sizeof(res));
+ }
+ int getLatest(const ScoreData& d) const
+ {
+ int max = 0;
+ for (int i = 0; i < 64; ++i)
+ if (d.r[i] > max)
+ max = d.r[i];
+ for (int i = 0; i < 8; ++i)
+ if (d.p[i] > max)
+ max = d.p[i];
+ if (d.c > max)
+ max = d.c;
+ return max;
+ }
+ inline int getLatestRd() const
+ {
+ return getLatest(rd);
+ }
+ inline int getLatestWr() const
+ {
+ return getLatest(wr);
+ }
+ inline int getLatest() const
+ {
+ const int a = getLatestRd();
+ const int b = getLatestWr();
+
+ int max = MAX2(a, b);
+ for (unsigned int f = 0; f < DATA_FILE_COUNT; ++f) {
+ max = MAX2(res.ld[f], max);
+ max = MAX2(res.st[f], max);
+ }
+ max = MAX2(res.sfu, max);
+ max = MAX2(res.imul, max);
+ max = MAX2(res.tex, max);
+ return max;
+ }
+ void setMax(const RegScores *that)
+ {
+ for (int i = 0; i < 64; ++i) {
+ rd.r[i] = MAX2(rd.r[i], that->rd.r[i]);
+ wr.r[i] = MAX2(wr.r[i], that->wr.r[i]);
+ }
+ for (int i = 0; i < 8; ++i) {
+ rd.p[i] = MAX2(rd.p[i], that->rd.p[i]);
+ wr.p[i] = MAX2(wr.p[i], that->wr.p[i]);
+ }
+ rd.c = MAX2(rd.c, that->rd.c);
+ wr.c = MAX2(wr.c, that->wr.c);
+
+ for (unsigned int f = 0; f < DATA_FILE_COUNT; ++f) {
+ res.ld[f] = MAX2(res.ld[f], that->res.ld[f]);
+ res.st[f] = MAX2(res.st[f], that->res.st[f]);
+ }
+ res.sfu = MAX2(res.sfu, that->res.sfu);
+ res.imul = MAX2(res.imul, that->res.imul);
+ res.tex = MAX2(res.tex, that->res.tex);
+ }
+ void print(int cycle)
+ {
+ for (int i = 0; i < 64; ++i) {
+ if (rd.r[i] > cycle)
+ INFO("rd $r%i @ %i\n", i, rd.r[i]);
+ if (wr.r[i] > cycle)
+ INFO("wr $r%i @ %i\n", i, wr.r[i]);
+ }
+ for (int i = 0; i < 8; ++i) {
+ if (rd.p[i] > cycle)
+ INFO("rd $p%i @ %i\n", i, rd.p[i]);
+ if (wr.p[i] > cycle)
+ INFO("wr $p%i @ %i\n", i, wr.p[i]);
+ }
+ if (rd.c > cycle)
+ INFO("rd $c @ %i\n", rd.c);
+ if (wr.c > cycle)
+ INFO("wr $c @ %i\n", wr.c);
+ if (res.sfu > cycle)
+ INFO("sfu @ %i\n", res.sfu);
+ if (res.imul > cycle)
+ INFO("imul @ %i\n", res.imul);
+ if (res.tex > cycle)
+ INFO("tex @ %i\n", res.tex);
+ }
+ };
+
+ RegScores *score; // for current BB
+ std::vector<RegScores> scoreBoards;
+ int cycle;
+ int prevData;
+ operation prevOp;
+
+ const Target *targ;
+
+ bool visit(Function *);
+ bool visit(BasicBlock *);
+
+ void commitInsn(const Instruction *, int cycle);
+ int calcDelay(const Instruction *, int cycle) const;
+ void setDelay(Instruction *, int delay, Instruction *next);
+
+ void recordRd(const Value *, const int ready);
+ void recordWr(const Value *, const int ready);
+ void checkRd(const Value *, int cycle, int& delay) const;
+ void checkWr(const Value *, int cycle, int& delay) const;
+
+ int getCycles(const Instruction *, int origDelay) const;
+};
+
+void
+SchedDataCalculator::setDelay(Instruction *insn, int delay, Instruction *next)
+{
+ if (insn->op == OP_EXIT)
+ delay = MAX2(delay, 14);
+
+ if (insn->op == OP_TEXBAR) {
+ // TODO: except if results not used before EXIT
+ insn->sched = 0xc2;
+ } else
+ if (insn->op == OP_JOIN || insn->join) {
+ insn->sched = 0x00;
+ } else
+ if (delay >= 0 || prevData == 0x04 ||
+ !next || !targ->canDualIssue(insn, next)) {
+ insn->sched = static_cast<uint8_t>(MAX2(delay, 0));
+ if (prevOp == OP_EXPORT)
+ insn->sched |= 0x40;
+ else
+ insn->sched |= 0x20;
+ } else {
+ insn->sched = 0x04; // dual-issue
+ }
+
+ if (prevData != 0x04 || prevOp != OP_EXPORT)
+ if (insn->sched != 0x04 || insn->op == OP_EXPORT)
+ prevOp = insn->op;
+
+ prevData = insn->sched;
+}
+
+int
+SchedDataCalculator::getCycles(const Instruction *insn, int origDelay) const
+{
+ if (insn->sched & 0x80) {
+ int c = (insn->sched & 0x0f) * 2 + 1;
+ if (insn->op == OP_TEXBAR && origDelay > 0)
+ c += origDelay;
+ return c;
+ }
+ if (insn->sched & 0x60)
+ return (insn->sched & 0x1f) + 1;
+ return (insn->sched == 0x04) ? 0 : 32;
+}
+
+bool
+SchedDataCalculator::visit(Function *func)
+{
+ scoreBoards.resize(func->cfg.getSize());
+ for (size_t i = 0; i < scoreBoards.size(); ++i)
+ scoreBoards[i].wipe();
+ return true;
+}
+
+bool
+SchedDataCalculator::visit(BasicBlock *bb)
+{
+ Instruction *insn;
+ Instruction *next = NULL;
+
+ int cycle = 0;
+
+ prevData = 0x00;
+ prevOp = OP_NOP;
+ score = &scoreBoards.at(bb->getId());
+
+ for (Graph::EdgeIterator ei = bb->cfg.incident(); !ei.end(); ei.next()) {
+ BasicBlock *in = BasicBlock::get(ei.getNode());
+ if (in->getExit()) {
+ if (prevData != 0x04)
+ prevData = in->getExit()->sched;
+ prevOp = in->getExit()->op;
+ }
+ if (ei.getType() != Graph::Edge::BACK)
+ score->setMax(&scoreBoards.at(in->getId()));
+ // back branches will wait until all target dependencies are satisfied
+ }
+ if (bb->cfg.incidentCount() > 1)
+ prevOp = OP_NOP;
+
+#ifdef NVC0_DEBUG_SCHED_DATA
+ INFO("=== BB:%i initial scores\n", bb->getId());
+ score->print(cycle);
+#endif
+
+ for (insn = bb->getEntry(); insn && insn->next; insn = insn->next) {
+ next = insn->next;
+
+ commitInsn(insn, cycle);
+ int delay = calcDelay(next, cycle);
+ setDelay(insn, delay, next);
+ cycle += getCycles(insn, delay);
+
+#ifdef NVC0_DEBUG_SCHED_DATA
+ INFO("cycle %i, sched %02x\n", cycle, insn->sched);
+ insn->print();
+ next->print();
+#endif
+ }
+ if (!insn)
+ return true;
+ commitInsn(insn, cycle);
+
+ int bbDelay = -1;
+
+ for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) {
+ BasicBlock *out = BasicBlock::get(ei.getNode());
+
+ if (ei.getType() != Graph::Edge::BACK) {
+ // only test the first instruction of the outgoing block
+ next = out->getEntry();
+ if (next)
+ bbDelay = MAX2(bbDelay, calcDelay(next, cycle));
+ } else {
+ // wait until all dependencies are satisfied
+ const int regsFree = score->getLatest();
+ next = out->getFirst();
+ for (int c = cycle; next && c < regsFree; next = next->next) {
+ bbDelay = MAX2(bbDelay, calcDelay(next, c));
+ c += getCycles(next, bbDelay);
+ }
+ next = NULL;
+ }
+ }
+ if (bb->cfg.outgoingCount() != 1)
+ next = NULL;
+ setDelay(insn, bbDelay, next);
+ cycle += getCycles(insn, bbDelay);
+
+ score->rebase(cycle); // common base for initializing out blocks' scores
+ return true;
+}
+
+#define NVE4_MAX_ISSUE_DELAY 0x1f
+int
+SchedDataCalculator::calcDelay(const Instruction *insn, int cycle) const
+{
+ int delay = 0, ready = cycle;
+
+ for (int s = 0; insn->srcExists(s); ++s)
+ checkRd(insn->getSrc(s), cycle, delay);
+ // WAR & WAW don't seem to matter
+ // for (int s = 0; insn->srcExists(s); ++s)
+ // recordRd(insn->getSrc(s), cycle);
+
+ switch (Target::getOpClass(insn->op)) {
+ case OPCLASS_SFU:
+ ready = score->res.sfu;
+ break;
+ case OPCLASS_ARITH:
+ if (insn->op == OP_MUL && !isFloatType(insn->dType))
+ ready = score->res.imul;
+ break;
+ case OPCLASS_TEXTURE:
+ ready = score->res.tex;
+ break;
+ case OPCLASS_LOAD:
+ ready = score->res.ld[insn->src(0).getFile()];
+ break;
+ case OPCLASS_STORE:
+ ready = score->res.st[insn->src(0).getFile()];
+ break;
+ default:
+ break;
+ }
+ if (Target::getOpClass(insn->op) != OPCLASS_TEXTURE)
+ ready = MAX2(ready, score->res.tex);
+
+ delay = MAX2(delay, ready - cycle);
+
+ // if can issue next cycle, delay is 0, not 1
+ return MIN2(delay - 1, NVE4_MAX_ISSUE_DELAY);
+}
+
+void
+SchedDataCalculator::commitInsn(const Instruction *insn, int cycle)
+{
+ const int ready = cycle + targ->getLatency(insn);
+
+ for (int d = 0; insn->defExists(d); ++d)
+ recordWr(insn->getDef(d), ready);
+ // WAR & WAW don't seem to matter
+ // for (int s = 0; insn->srcExists(s); ++s)
+ // recordRd(insn->getSrc(s), cycle);
+
+ switch (Target::getOpClass(insn->op)) {
+ case OPCLASS_SFU:
+ score->res.sfu = cycle + 4;
+ break;
+ case OPCLASS_ARITH:
+ if (insn->op == OP_MUL && !isFloatType(insn->dType))
+ score->res.imul = cycle + 4;
+ break;
+ case OPCLASS_TEXTURE:
+ score->res.tex = cycle + 18;
+ break;
+ case OPCLASS_LOAD:
+ if (insn->src(0).getFile() == FILE_MEMORY_CONST)
+ break;
+ score->res.ld[insn->src(0).getFile()] = cycle + 4;
+ score->res.st[insn->src(0).getFile()] = ready;
+ break;
+ case OPCLASS_STORE:
+ score->res.st[insn->src(0).getFile()] = cycle + 4;
+ score->res.ld[insn->src(0).getFile()] = ready;
+ break;
+ case OPCLASS_OTHER:
+ if (insn->op == OP_TEXBAR)
+ score->res.tex = cycle;
+ break;
+ default:
+ break;
+ }
+
+#ifdef NVC0_DEBUG_SCHED_DATA
+ score->print(cycle);
+#endif
+}
+
+void
+SchedDataCalculator::checkRd(const Value *v, int cycle, int& delay) const
+{
+ int ready = cycle;
+ int a, b;
+
+ switch (v->reg.file) {
+ case FILE_GPR:
+ a = v->reg.data.id;
+ b = a + v->reg.size / 4;
+ for (int r = a; r < b; ++r)
+ ready = MAX2(ready, score->rd.r[r]);
+ break;
+ case FILE_PREDICATE:
+ ready = MAX2(ready, score->rd.p[v->reg.data.id]);
+ break;
+ case FILE_FLAGS:
+ ready = MAX2(ready, score->rd.c);
+ break;
+ case FILE_SHADER_INPUT:
+ case FILE_SHADER_OUTPUT: // yes, TCPs can read outputs
+ case FILE_MEMORY_LOCAL:
+ case FILE_MEMORY_CONST:
+ case FILE_MEMORY_SHARED:
+ case FILE_MEMORY_GLOBAL:
+ case FILE_SYSTEM_VALUE:
+ // TODO: any restrictions here ?
+ break;
+ case FILE_IMMEDIATE:
+ break;
+ default:
+ assert(0);
+ break;
+ }
+ if (cycle < ready)
+ delay = MAX2(delay, ready - cycle);
+}
+
+void
+SchedDataCalculator::checkWr(const Value *v, int cycle, int& delay) const
+{
+ int ready = cycle;
+ int a, b;
+
+ switch (v->reg.file) {
+ case FILE_GPR:
+ a = v->reg.data.id;
+ b = a + v->reg.size / 4;
+ for (int r = a; r < b; ++r)
+ ready = MAX2(ready, score->wr.r[r]);
+ break;
+ case FILE_PREDICATE:
+ ready = MAX2(ready, score->wr.p[v->reg.data.id]);
+ break;
+ default:
+ assert(v->reg.file == FILE_FLAGS);
+ ready = MAX2(ready, score->wr.c);
+ break;
+ }
+ if (cycle < ready)
+ delay = MAX2(delay, ready - cycle);
+}
+
+void
+SchedDataCalculator::recordWr(const Value *v, const int ready)
+{
+ int a = v->reg.data.id;
+
+ if (v->reg.file == FILE_GPR) {
+ int b = a + v->reg.size / 4;
+ for (int r = a; r < b; ++r)
+ score->rd.r[r] = ready;
+ } else
+ // $c, $pX: shorter issue-to-read delay (at least as exec pred and carry)
+ if (v->reg.file == FILE_PREDICATE) {
+ score->rd.p[a] = ready + 4;
+ } else {
+ assert(v->reg.file == FILE_FLAGS);
+ score->rd.c = ready + 4;
+ }
+}
+
+void
+SchedDataCalculator::recordRd(const Value *v, const int ready)
+{
+ int a = v->reg.data.id;
+
+ if (v->reg.file == FILE_GPR) {
+ int b = a + v->reg.size / 4;
+ for (int r = a; r < b; ++r)
+ score->wr.r[r] = ready;
+ } else
+ if (v->reg.file == FILE_PREDICATE) {
+ score->wr.p[a] = ready;
+ } else
+ if (v->reg.file == FILE_FLAGS) {
+ score->wr.c = ready;
+ }
+}
+
+bool
+calculateSchedDataNVC0(const Target *targ, Function *func)
+{
+ SchedDataCalculator sched(targ);
+ return sched.run(func, true, true);
+}
+
+void
+CodeEmitterNVC0::prepareEmission(Function *func)
+{
+ CodeEmitter::prepareEmission(func);
+
+ if (targ->hasSWSched)
+ calculateSchedDataNVC0(targ, func);
+}
+
+CodeEmitterNVC0::CodeEmitterNVC0(const TargetNVC0 *target)
+ : CodeEmitter(target),
+ targNVC0(target),
+ writeIssueDelays(target->hasSWSched)
{
code = NULL;
codeSize = codeSizeLimit = 0;
}
CodeEmitter *
-TargetNVC0::getCodeEmitter(Program::Type type)
+TargetNVC0::createCodeEmitterNVC0(Program::Type type)
{
CodeEmitterNVC0 *emit = new CodeEmitterNVC0(this);
emit->setProgramType(type);
return emit;
}
+CodeEmitter *
+TargetNVC0::getCodeEmitter(Program::Type type)
+{
+ if (chipset >= NVISA_GK110_CHIPSET)
+ return createCodeEmitterGK110(type);
+ return createCodeEmitterNVC0(type);
+}
+
} // namespace nv50_ir
projects / mesa.git / blobdiff