void emitFlow(const Instruction *);
+ void emitVOTE(const Instruction *);
+
inline void defId(const ValueDef&, const int pos);
inline void srcId(const ValueRef&, const int pos);
inline void srcId(const ValueRef *, const int pos);
assert(!i->src(0).mod.neg() && !i->src(1).mod.neg());
assert(!i->src(0).mod.abs() && !i->src(1).mod.abs());
- if (isLIMM(i->src(1), TYPE_S32)) {
+ if (i->src(1).getFile() == FILE_IMMEDIATE) {
emitForm_L(i, 0x280, 2, Modifier(0));
if (i->subOp == NV50_IR_SUBOP_MUL_HIGH)
void
CodeEmitterGK110::emitLogicOp(const Instruction *i, uint8_t subOp)
{
+ if (i->def(0).getFile() == FILE_PREDICATE) {
+ code[0] = 0x00000002 | (subOp << 27);
+ code[1] = 0x84800000;
+
+ emitPredicate(i);
+
+ defId(i->def(0), 5);
+ srcId(i->src(0), 14);
+ if (i->src(0).mod == Modifier(NV50_IR_MOD_NOT)) code[0] |= 1 << 17;
+ srcId(i->src(1), 32);
+ if (i->src(1).mod == Modifier(NV50_IR_MOD_NOT)) code[1] |= 1 << 3;
+
+ if (i->defExists(1)) {
+ defId(i->def(1), 2);
+ } else {
+ code[0] |= 7 << 2;
+ }
+ // (a OP b) OP c
+ if (i->predSrc != 2 && i->srcExists(2)) {
+ code[1] |= subOp << 16;
+ srcId(i->src(2), 42);
+ if (i->src(2).mod == Modifier(NV50_IR_MOD_NOT)) code[1] |= 1 << 13;
+ } else {
+ code[1] |= 7 << 10;
+ }
+ } else
if (isLIMM(i->src(1), TYPE_S32)) {
emitForm_L(i, 0x200, 0, i->src(1).mod);
code[1] |= subOp << 24;
case NV50_IR_SUBOP_BAR_RED_OR: code[1] |= 0x90; break;
case NV50_IR_SUBOP_BAR_RED_POPC: code[1] |= 0x10; break;
default:
- code[1] |= 0x20;
assert(i->subOp == NV50_IR_SUBOP_BAR_SYNC);
break;
}
emitPredicate(i);
- srcId(i->src(0), 10);
- srcId(i->src(1), 23);
+ // barrier id
+ if (i->src(0).getFile() == FILE_GPR) {
+ srcId(i->src(0), 10);
+ } else {
+ ImmediateValue *imm = i->getSrc(0)->asImm();
+ assert(imm);
+ code[0] |= imm->reg.data.u32 << 10;
+ code[1] |= 0x8000;
+ }
+
+ // thread count
+ if (i->src(1).getFile() == FILE_GPR) {
+ srcId(i->src(1), 23);
+ } else {
+ ImmediateValue *imm = i->getSrc(0)->asImm();
+ assert(imm);
+ assert(imm->reg.data.u32 <= 0xfff);
+ code[0] |= imm->reg.data.u32 << 23;
+ code[1] |= imm->reg.data.u32 >> 9;
+ code[1] |= 0x4000;
+ }
+
+ if (i->srcExists(2) && (i->predSrc != 2)) {
+ srcId(i->src(2), 32 + 10);
+ if (i->src(2).mod == Modifier(NV50_IR_MOD_NOT))
+ code[1] |= 1 << 13;
+ } else {
+ code[1] |= 7 << 10;
+ }
}
void CodeEmitterGK110::emitMEMBAR(const Instruction *i)
}
}
+void
+CodeEmitterGK110::emitVOTE(const Instruction *i)
+{
+ assert(i->src(0).getFile() == FILE_PREDICATE &&
+ i->def(1).getFile() == FILE_PREDICATE);
+
+ code[0] = 0x00000002;
+ code[1] = 0x86c00000 | (i->subOp << 19);
+
+ emitPredicate(i);
+
+ defId(i->def(0), 2);
+ defId(i->def(1), 48);
+ if (i->src(0).mod == Modifier(NV50_IR_MOD_NOT))
+ code[1] |= 1 << 13;
+ srcId(i->src(0), 42);
+}
+
void
CodeEmitterGK110::emitAFETCH(const Instruction *i)
{
switch (i->src(0).getFile()) {
case FILE_MEMORY_GLOBAL: code[1] = 0xe0000000; code[0] = 0x00000000; break;
case FILE_MEMORY_LOCAL: code[1] = 0x7a800000; code[0] = 0x00000002; break;
- case FILE_MEMORY_SHARED: code[1] = 0x7ac00000; code[0] = 0x00000002; break;
+ case FILE_MEMORY_SHARED:
+ code[0] = 0x00000002;
+ if (i->subOp == NV50_IR_SUBOP_STORE_UNLOCKED)
+ code[1] = 0x78400000;
+ else
+ code[1] = 0x7ac00000;
+ break;
default:
assert(!"invalid memory file");
break;
}
- if (i->src(0).getFile() != FILE_MEMORY_GLOBAL)
- offset &= 0xffffff;
-
if (code[0] & 0x2) {
+ offset &= 0xffffff;
emitLoadStoreType(i->dType, 0x33);
if (i->src(0).getFile() == FILE_MEMORY_LOCAL)
emitCachingMode(i->cache, 0x2f);
code[0] |= offset << 23;
code[1] |= offset >> 9;
+ // Unlocked store on shared memory can fail.
+ if (i->src(0).getFile() == FILE_MEMORY_SHARED &&
+ i->subOp == NV50_IR_SUBOP_STORE_UNLOCKED) {
+ assert(i->defExists(0));
+ defId(i->def(0), 32 + 16);
+ }
+
emitPredicate(i);
srcId(i->src(1), 2);
switch (i->src(0).getFile()) {
case FILE_MEMORY_GLOBAL: code[1] = 0xc0000000; code[0] = 0x00000000; break;
case FILE_MEMORY_LOCAL: code[1] = 0x7a000000; code[0] = 0x00000002; break;
- case FILE_MEMORY_SHARED: code[1] = 0x7a400000; code[0] = 0x00000002; break;
+ case FILE_MEMORY_SHARED:
+ code[0] = 0x00000002;
+ if (i->subOp == NV50_IR_SUBOP_LOAD_LOCKED)
+ code[1] = 0x77400000;
+ else
+ code[1] = 0x7a400000;
+ break;
case FILE_MEMORY_CONST:
if (!i->src(0).isIndirect(0) && typeSizeof(i->dType) == 4) {
emitMOV(i);
code[0] |= offset << 23;
code[1] |= offset >> 9;
+ // Locked store on shared memory can fail.
+ if (i->src(0).getFile() == FILE_MEMORY_SHARED &&
+ i->subOp == NV50_IR_SUBOP_LOAD_LOCKED) {
+ assert(i->defExists(1));
+ defId(i->def(1), 32 + 16);
+ }
+
emitPredicate(i);
defId(i->def(0), 2);
void
CodeEmitterGK110::emitATOM(const Instruction *i)
{
+ const bool hasDst = i->defExists(0);
+ const bool exch = i->subOp == NV50_IR_SUBOP_ATOM_EXCH;
+
code[0] = 0x00000002;
if (i->subOp == NV50_IR_SUBOP_ATOM_CAS)
code[1] = 0x77800000;
/* TODO: cas: flip bits if $r255 is used */
srcId(i->src(1), 23);
- if (i->defExists(0))
+ if (hasDst) {
defId(i->def(0), 2);
- else
+ } else
+ if (!exch) {
code[0] |= 255 << 2;
+ }
- const int32_t offset = SDATA(i->src(0)).offset;
- assert(offset < 0x80000 && offset >= -0x80000);
- code[0] |= (offset & 1) << 31;
- code[1] |= (offset & 0xffffe) >> 1;
+ if (hasDst || !exch) {
+ const int32_t offset = SDATA(i->src(0)).offset;
+ assert(offset < 0x80000 && offset >= -0x80000);
+ code[0] |= (offset & 1) << 31;
+ code[1] |= (offset & 0xffffe) >> 1;
+ } else {
+ srcAddr32(i->src(0), 31);
+ }
if (i->getIndirect(0, 0)) {
srcId(i->getIndirect(0, 0), 10);
case OP_CCTL:
emitCCTL(insn);
break;
+ case OP_VOTE:
+ emitVOTE(insn);
+ break;
case OP_PHI:
case OP_UNION:
case OP_CONSTRAINT: