src/gallium/drivers/nvc0/codegen/nv50_ir_emit_nvc0.cpp

   1 /*
   2  * Copyright 2011 Christoph Bumiller
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  18  * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  19  * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  20  * SOFTWARE.
  21  */
  22
  23 #include "nv50_ir_target_nvc0.h"
  24
  25 namespace nv50_ir {
  26
  27 // Argh, all these assertions ...
  28
  29 class CodeEmitterNVC0 : public CodeEmitter
  30 {
  31 public:
  32    CodeEmitterNVC0(const TargetNVC0 *);
  33
  34    virtual bool emitInstruction(Instruction *);
  35    virtual uint32_t getMinEncodingSize(const Instruction *) const;
  36    virtual void prepareEmission(Function *);
  37
  38    inline void setProgramType(Program::Type pType) { progType = pType; }
  39
  40 private:
  41    const TargetNVC0 *targ;
  42
  43    Program::Type progType;
  44
  45    const bool writeIssueDelays;
  46
  47 private:
  48    void emitForm_A(const Instruction *, uint64_t);
  49    void emitForm_B(const Instruction *, uint64_t);
  50    void emitForm_S(const Instruction *, uint32_t, bool pred);
  51
  52    void emitPredicate(const Instruction *);
  53
  54    void setAddress16(const ValueRef&);
  55    void setImmediate(const Instruction *, const int s); // needs op already set
  56    void setImmediateS8(const ValueRef&);
  57
  58    void emitCondCode(CondCode cc, int pos);
  59    void emitInterpMode(const Instruction *);
  60    void emitLoadStoreType(DataType ty);
  61    void emitCachingMode(CacheMode c);
  62
  63    void emitShortSrc2(const ValueRef&);
  64
  65    inline uint8_t getSRegEncoding(const ValueRef&);
  66
  67    void roundMode_A(const Instruction *);
  68    void roundMode_C(const Instruction *);
  69    void roundMode_CS(const Instruction *);
  70
  71    void emitNegAbs12(const Instruction *);
  72
  73    void emitNOP(const Instruction *);
  74
  75    void emitLOAD(const Instruction *);
  76    void emitSTORE(const Instruction *);
  77    void emitMOV(const Instruction *);
  78
  79    void emitINTERP(const Instruction *);
  80    void emitPFETCH(const Instruction *);
  81    void emitVFETCH(const Instruction *);
  82    void emitEXPORT(const Instruction *);
  83    void emitOUT(const Instruction *);
  84
  85    void emitUADD(const Instruction *);
  86    void emitFADD(const Instruction *);
  87    void emitUMUL(const Instruction *);
  88    void emitFMUL(const Instruction *);
  89    void emitIMAD(const Instruction *);
  90    void emitFMAD(const Instruction *);
  91
  92    void emitNOT(Instruction *);
  93    void emitLogicOp(const Instruction *, uint8_t subOp);
  94    void emitPOPC(const Instruction *);
  95    void emitINSBF(const Instruction *);
  96    void emitShift(const Instruction *);
  97
  98    void emitSFnOp(const Instruction *, uint8_t subOp);
  99
 100    void emitCVT(Instruction *);
 101    void emitMINMAX(const Instruction *);
 102    void emitPreOp(const Instruction *);
 103
 104    void emitSET(const CmpInstruction *);
 105    void emitSLCT(const CmpInstruction *);
 106    void emitSELP(const Instruction *);
 107
 108    void emitTEXBAR(const Instruction *);
 109    void emitTEX(const TexInstruction *);
 110    void emitTEXCSAA(const TexInstruction *);
 111    void emitTXQ(const TexInstruction *);
 112    void emitPIXLD(const TexInstruction *);
 113
 114    void emitQUADOP(const Instruction *, uint8_t qOp, uint8_t laneMask);
 115
 116    void emitFlow(const Instruction *);
 117
 118    inline void defId(const ValueDef&, const int pos);
 119    inline void srcId(const ValueRef&, const int pos);
 120    inline void srcId(const ValueRef *, const int pos);
 121    inline void srcId(const Instruction *, int s, const int pos);
 122
 123    inline void srcAddr32(const ValueRef&, const int pos); // address / 4
 124
 125    inline bool isLIMM(const ValueRef&, DataType ty);
 126 };
 127
 128 // for better visibility
 129 #define HEX64(h, l) 0x##h##l##ULL
 130
 131 #define SDATA(a) ((a).rep()->reg.data)
 132 #define DDATA(a) ((a).rep()->reg.data)
 133
 134 void CodeEmitterNVC0::srcId(const ValueRef& src, const int pos)
 135 {
 136    code[pos / 32] |= (src.get() ? SDATA(src).id : 63) << (pos % 32);
 137 }
 138
 139 void CodeEmitterNVC0::srcId(const ValueRef *src, const int pos)
 140 {
 141    code[pos / 32] |= (src ? SDATA(*src).id : 63) << (pos % 32);
 142 }
 143
 144 void CodeEmitterNVC0::srcId(const Instruction *insn, int s, int pos)
 145 {
 146    int r = insn->srcExists(s) ? SDATA(insn->src(s)).id : 63;
 147    code[pos / 32] |= r << (pos % 32);
 148 }
 149
 150 void CodeEmitterNVC0::srcAddr32(const ValueRef& src, const int pos)
 151 {
 152    code[pos / 32] |= (SDATA(src).offset >> 2) << (pos % 32);
 153 }
 154
 155 void CodeEmitterNVC0::defId(const ValueDef& def, const int pos)
 156 {
 157    code[pos / 32] |= (def.get() ? DDATA(def).id : 63) << (pos % 32);
 158 }
 159
 160 bool CodeEmitterNVC0::isLIMM(const ValueRef& ref, DataType ty)
 161 {
 162    const ImmediateValue *imm = ref.get()->asImm();
 163
 164    return imm && (imm->reg.data.u32 & ((ty == TYPE_F32) ? 0xfff : 0xfff00000));
 165 }
 166
 167 void
 168 CodeEmitterNVC0::roundMode_A(const Instruction *insn)
 169 {
 170    switch (insn->rnd) {
 171    case ROUND_M: code[1] |= 1 << 23; break;
 172    case ROUND_P: code[1] |= 2 << 23; break;
 173    case ROUND_Z: code[1] |= 3 << 23; break;
 174    default:
 175       assert(insn->rnd == ROUND_N);
 176       break;
 177    }
 178 }
 179
 180 void
 181 CodeEmitterNVC0::emitNegAbs12(const Instruction *i)
 182 {
 183    if (i->src(1).mod.abs()) code[0] |= 1 << 6;
 184    if (i->src(0).mod.abs()) code[0] |= 1 << 7;
 185    if (i->src(1).mod.neg()) code[0] |= 1 << 8;
 186    if (i->src(0).mod.neg()) code[0] |= 1 << 9;
 187 }
 188
 189 void CodeEmitterNVC0::emitCondCode(CondCode cc, int pos)
 190 {
 191    uint8_t val;
 192
 193    switch (cc) {
 194    case CC_LT:  val = 0x1; break;
 195    case CC_LTU: val = 0x9; break;
 196    case CC_EQ:  val = 0x2; break;
 197    case CC_EQU: val = 0xa; break;
 198    case CC_LE:  val = 0x3; break;
 199    case CC_LEU: val = 0xb; break;
 200    case CC_GT:  val = 0x4; break;
 201    case CC_GTU: val = 0xc; break;
 202    case CC_NE:  val = 0x5; break;
 203    case CC_NEU: val = 0xd; break;
 204    case CC_GE:  val = 0x6; break;
 205    case CC_GEU: val = 0xe; break;
 206    case CC_TR:  val = 0xf; break;
 207    case CC_FL:  val = 0x0; break;
 208
 209    case CC_A:  val = 0x14; break;
 210    case CC_NA: val = 0x13; break;
 211    case CC_S:  val = 0x15; break;
 212    case CC_NS: val = 0x12; break;
 213    case CC_C:  val = 0x16; break;
 214    case CC_NC: val = 0x11; break;
 215    case CC_O:  val = 0x17; break;
 216    case CC_NO: val = 0x10; break;
 217
 218    default:
 219       val = 0;
 220       assert(!"invalid condition code");
 221       break;
 222    }
 223    code[pos / 32] |= val << (pos % 32);
 224 }
 225
 226 void
 227 CodeEmitterNVC0::emitPredicate(const Instruction *i)
 228 {
 229    if (i->predSrc >= 0) {
 230       assert(i->getPredicate()->reg.file == FILE_PREDICATE);
 231       srcId(i->src(i->predSrc), 10);
 232       if (i->cc == CC_NOT_P)
 233          code[0] |= 0x2000; // negate
 234    } else {
 235       code[0] |= 0x1c00;
 236    }
 237 }
 238
 239 void
 240 CodeEmitterNVC0::setAddress16(const ValueRef& src)
 241 {
 242    Symbol *sym = src.get()->asSym();
 243
 244    assert(sym);
 245
 246    code[0] |= (sym->reg.data.offset & 0x003f) << 26;
 247    code[1] |= (sym->reg.data.offset & 0xffc0) >> 6;
 248 }
 249
 250 void
 251 CodeEmitterNVC0::setImmediate(const Instruction *i, const int s)
 252 {
 253    const ImmediateValue *imm = i->src(s).get()->asImm();
 254    uint32_t u32;
 255
 256    assert(imm);
 257    u32 = imm->reg.data.u32;
 258
 259    if ((code[0] & 0xf) == 0x2) {
 260       // LIMM
 261       code[0] |= (u32 & 0x3f) << 26;
 262       code[1] |= u32 >> 6;
 263    } else
 264    if ((code[0] & 0xf) == 0x3 || (code[0] & 0xf) == 4) {
 265       // integer immediate
 266       assert((u32 & 0xfff00000) == 0 || (u32 & 0xfff00000) == 0xfff00000);
 267       assert(!(code[1] & 0xc000));
 268       u32 &= 0xfffff;
 269       code[0] |= (u32 & 0x3f) << 26;
 270       code[1] |= 0xc000 | (u32 >> 6);
 271    } else {
 272       // float immediate
 273       assert(!(u32 & 0x00000fff));
 274       assert(!(code[1] & 0xc000));
 275       code[0] |= ((u32 >> 12) & 0x3f) << 26;
 276       code[1] |= 0xc000 | (u32 >> 18);
 277    }
 278 }
 279
 280 void CodeEmitterNVC0::setImmediateS8(const ValueRef &ref)
 281 {
 282    const ImmediateValue *imm = ref.get()->asImm();
 283
 284    int8_t s8 = static_cast<int8_t>(imm->reg.data.s32);
 285
 286    assert(s8 == imm->reg.data.s32);
 287
 288    code[0] |= (s8 & 0x3f) << 26;
 289    code[0] |= (s8 >> 6) << 8;
 290 }
 291
 292 void
 293 CodeEmitterNVC0::emitForm_A(const Instruction *i, uint64_t opc)
 294 {
 295    code[0] = opc;
 296    code[1] = opc >> 32;
 297
 298    emitPredicate(i);
 299
 300    defId(i->def(0), 14);
 301
 302    int s1 = 26;
 303    if (i->srcExists(2) && i->getSrc(2)->reg.file == FILE_MEMORY_CONST)
 304       s1 = 49;
 305
 306    for (int s = 0; s < 3 && i->srcExists(s); ++s) {
 307       switch (i->getSrc(s)->reg.file) {
 308       case FILE_MEMORY_CONST:
 309          assert(!(code[1] & 0xc000));
 310          code[1] |= (s == 2) ? 0x8000 : 0x4000;
 311          code[1] |= i->getSrc(s)->reg.fileIndex << 10;
 312          setAddress16(i->src(s));
 313          break;
 314       case FILE_IMMEDIATE:
 315          assert(s == 1 ||
 316                 i->op == OP_MOV || i->op == OP_PRESIN || i->op == OP_PREEX2);
 317          assert(!(code[1] & 0xc000));
 318          setImmediate(i, s);
 319          break;
 320       case FILE_GPR:
 321          if ((s == 2) && ((code[0] & 0x7) == 2)) // LIMM: 3rd src == dst
 322             break;
 323          srcId(i->src(s), s ? ((s == 2) ? 49 : s1) : 20);
 324          break;
 325       default:
 326          // ignore here, can be predicate or flags, but must not be address
 327          break;
 328       }
 329    }
 330 }
 331
 332 void
 333 CodeEmitterNVC0::emitForm_B(const Instruction *i, uint64_t opc)
 334 {
 335    code[0] = opc;
 336    code[1] = opc >> 32;
 337
 338    emitPredicate(i);
 339
 340    defId(i->def(0), 14);
 341
 342    switch (i->src(0).getFile()) {
 343    case FILE_MEMORY_CONST:
 344       assert(!(code[1] & 0xc000));
 345       code[1] |= 0x4000 | (i->src(0).get()->reg.fileIndex << 10);
 346       setAddress16(i->src(0));
 347       break;
 348    case FILE_IMMEDIATE:
 349       assert(!(code[1] & 0xc000));
 350       setImmediate(i, 0);
 351       break;
 352    case FILE_GPR:
 353       srcId(i->src(0), 26);
 354       break;
 355    default:
 356       // ignore here, can be predicate or flags, but must not be address
 357       break;
 358    }
 359 }
 360
 361 void
 362 CodeEmitterNVC0::emitForm_S(const Instruction *i, uint32_t opc, bool pred)
 363 {
 364    code[0] = opc;
 365
 366    int ss2a = 0;
 367    if (opc == 0x0d || opc == 0x0e)
 368       ss2a = 2;
 369
 370    defId(i->def(0), 14);
 371    srcId(i->src(0), 20);
 372
 373    assert(pred || (i->predSrc < 0));
 374    if (pred)
 375       emitPredicate(i);
 376
 377    for (int s = 1; s < 3 && i->srcExists(s); ++s) {
 378       if (i->src(s).get()->reg.file == FILE_MEMORY_CONST) {
 379          assert(!(code[0] & (0x300 >> ss2a)));
 380          switch (i->src(s).get()->reg.fileIndex) {
 381          case 0:  code[0] |= 0x100 >> ss2a; break;
 382          case 1:  code[0] |= 0x200 >> ss2a; break;
 383          case 16: code[0] |= 0x300 >> ss2a; break;
 384          default:
 385             ERROR("invalid c[] space for short form\n");
 386             break;
 387          }
 388          if (s == 1)
 389             code[0] |= i->getSrc(s)->reg.data.offset << 24;
 390          else
 391             code[0] |= i->getSrc(s)->reg.data.offset << 6;
 392       } else
 393       if (i->src(s).getFile() == FILE_IMMEDIATE) {
 394          assert(s == 1);
 395          setImmediateS8(i->src(s));
 396       } else
 397       if (i->src(s).getFile() == FILE_GPR) {
 398          srcId(i->src(s), (s == 1) ? 26 : 8);
 399       }
 400    }
 401 }
 402
 403 void
 404 CodeEmitterNVC0::emitShortSrc2(const ValueRef &src)
 405 {
 406    if (src.getFile() == FILE_MEMORY_CONST) {
 407       switch (src.get()->reg.fileIndex) {
 408       case 0:  code[0] |= 0x100; break;
 409       case 1:  code[0] |= 0x200; break;
 410       case 16: code[0] |= 0x300; break;
 411       default:
 412          assert(!"unsupported file index for short op");
 413          break;
 414       }
 415       srcAddr32(src, 20);
 416    } else {
 417       srcId(src, 20);
 418       assert(src.getFile() == FILE_GPR);
 419    }
 420 }
 421
 422 void
 423 CodeEmitterNVC0::emitNOP(const Instruction *i)
 424 {
 425    code[0] = 0x000001e4;
 426    code[1] = 0x40000000;
 427    emitPredicate(i);
 428 }
 429
 430 void
 431 CodeEmitterNVC0::emitFMAD(const Instruction *i)
 432 {
 433    bool neg1 = (i->src(0).mod ^ i->src(1).mod).neg();
 434
 435    if (i->encSize == 8) {
 436       if (isLIMM(i->src(1), TYPE_F32)) {
 437          emitForm_A(i, HEX64(20000000, 00000002));
 438       } else {
 439          emitForm_A(i, HEX64(30000000, 00000000));
 440
 441          if (i->src(2).mod.neg())
 442             code[0] |= 1 << 8;
 443       }
 444       roundMode_A(i);
 445
 446       if (neg1)
 447          code[0] |= 1 << 9;
 448
 449       if (i->saturate)
 450          code[0] |= 1 << 5;
 451       if (i->ftz)
 452          code[0] |= 1 << 6;
 453    } else {
 454       assert(!i->saturate && !i->src(2).mod.neg());
 455       emitForm_S(i, (i->src(2).getFile() == FILE_MEMORY_CONST) ? 0x2e : 0x0e,
 456                  false);
 457       if (neg1)
 458          code[0] |= 1 << 4;
 459    }
 460 }
 461
 462 void
 463 CodeEmitterNVC0::emitFMUL(const Instruction *i)
 464 {
 465    bool neg = (i->src(0).mod ^ i->src(1).mod).neg();
 466
 467    assert(i->postFactor >= -3 && i->postFactor <= 3);
 468
 469    if (i->encSize == 8) {
 470       if (isLIMM(i->src(1), TYPE_F32)) {
 471          assert(i->postFactor == 0); // constant folded, hopefully
 472          emitForm_A(i, HEX64(30000000, 00000002));
 473       } else {
 474          emitForm_A(i, HEX64(58000000, 00000000));
 475          roundMode_A(i);
 476          code[1] |= ((i->postFactor > 0) ?
 477                      (7 - i->postFactor) : (0 - i->postFactor)) << 17;
 478       }
 479       if (neg)
 480          code[1] ^= 1 << 25; // aliases with LIMM sign bit
 481
 482       if (i->saturate)
 483          code[0] |= 1 << 5;
 484
 485       if (i->dnz)
 486          code[0] |= 1 << 7;
 487       else
 488       if (i->ftz)
 489          code[0] |= 1 << 6;
 490    } else {
 491       assert(!neg && !i->saturate && !i->ftz && !i->postFactor);
 492       emitForm_S(i, 0xa8, true);
 493    }
 494 }
 495
 496 void
 497 CodeEmitterNVC0::emitUMUL(const Instruction *i)
 498 {
 499    if (i->encSize == 8) {
 500       if (i->src(1).getFile() == FILE_IMMEDIATE) {
 501          emitForm_A(i, HEX64(10000000, 00000002));
 502       } else {
 503          emitForm_A(i, HEX64(50000000, 00000003));
 504       }
 505       if (i->subOp == NV50_IR_SUBOP_MUL_HIGH)
 506          code[0] |= 1 << 6;
 507       if (i->sType == TYPE_S32)
 508          code[0] |= 1 << 5;
 509       if (i->dType == TYPE_S32)
 510          code[0] |= 1 << 7;
 511    } else {
 512       emitForm_S(i, i->src(1).getFile() == FILE_IMMEDIATE ? 0xaa : 0x2a, true);
 513
 514       if (i->sType == TYPE_S32)
 515          code[0] |= 1 << 6;
 516    }
 517 }
 518
 519 void
 520 CodeEmitterNVC0::emitFADD(const Instruction *i)
 521 {
 522    if (i->encSize == 8) {
 523       if (isLIMM(i->src(1), TYPE_F32)) {
 524          assert(!i->saturate);
 525          emitForm_A(i, HEX64(28000000, 00000002));
 526
 527          code[0] |= i->src(0).mod.abs() << 7;
 528          code[0] |= i->src(0).mod.neg() << 9;
 529
 530          if (i->src(1).mod.abs())
 531             code[1] &= 0xfdffffff;
 532          if ((i->op == OP_SUB) != static_cast<bool>(i->src(1).mod.neg()))
 533             code[1] ^= 0x02000000;
 534       } else {
 535          emitForm_A(i, HEX64(50000000, 00000000));
 536
 537          roundMode_A(i);
 538          if (i->saturate)
 539             code[1] |= 1 << 17;
 540
 541          emitNegAbs12(i);
 542          if (i->op == OP_SUB) code[0] ^= 1 << 8;
 543       }
 544       if (i->ftz)
 545          code[0] |= 1 << 5;
 546    } else {
 547       assert(!i->saturate && i->op != OP_SUB &&
 548              !i->src(0).mod.abs() &&
 549              !i->src(1).mod.neg() && !i->src(1).mod.abs());
 550
 551       emitForm_S(i, 0x49, true);
 552
 553       if (i->src(0).mod.neg())
 554          code[0] |= 1 << 7;
 555    }
 556 }
 557
 558 void
 559 CodeEmitterNVC0::emitUADD(const Instruction *i)
 560 {
 561    uint32_t addOp = 0;
 562
 563    assert(!i->src(0).mod.abs() && !i->src(1).mod.abs());
 564    assert(!i->src(0).mod.neg() || !i->src(1).mod.neg());
 565
 566    if (i->src(0).mod.neg())
 567       addOp |= 0x200;
 568    if (i->src(1).mod.neg())
 569       addOp |= 0x100;
 570    if (i->op == OP_SUB) {
 571       addOp ^= 0x100;
 572       assert(addOp != 0x300); // would be add-plus-one
 573    }
 574
 575    if (i->encSize == 8) {
 576       if (isLIMM(i->src(1), TYPE_U32)) {
 577          emitForm_A(i, HEX64(08000000, 00000002));
 578          if (i->defExists(1))
 579             code[1] |= 1 << 26; // write carry
 580       } else {
 581          emitForm_A(i, HEX64(48000000, 00000003));
 582          if (i->defExists(1))
 583             code[1] |= 1 << 16; // write carry
 584       }
 585       code[0] |= addOp;
 586
 587       if (i->saturate)
 588          code[0] |= 1 << 5;
 589       if (i->flagsSrc >= 0) // add carry
 590          code[0] |= 1 << 6;
 591    } else {
 592       assert(!(addOp & 0x100));
 593       emitForm_S(i, (addOp >> 3) |
 594                  ((i->src(1).getFile() == FILE_IMMEDIATE) ? 0xac : 0x2c), true);
 595    }
 596 }
 597
 598 // TODO: shl-add
 599 void
 600 CodeEmitterNVC0::emitIMAD(const Instruction *i)
 601 {
 602    assert(i->encSize == 8);
 603    emitForm_A(i, HEX64(20000000, 00000003));
 604
 605    if (isSignedType(i->dType))
 606       code[0] |= 1 << 7;
 607    if (isSignedType(i->sType))
 608       code[0] |= 1 << 5;
 609
 610    code[1] |= i->saturate << 24;
 611
 612    if (i->flagsDef >= 0) code[1] |= 1 << 16;
 613    if (i->flagsSrc >= 0) code[1] |= 1 << 23;
 614
 615    if (i->src(2).mod.neg()) code[0] |= 0x10;
 616    if (i->src(1).mod.neg() ^
 617        i->src(0).mod.neg()) code[0] |= 0x20;
 618
 619    if (i->subOp == NV50_IR_SUBOP_MUL_HIGH)
 620       code[0] |= 1 << 6;
 621 }
 622
 623 void
 624 CodeEmitterNVC0::emitNOT(Instruction *i)
 625 {
 626    assert(i->encSize == 8);
 627    i->setSrc(1, i->src(0));
 628    emitForm_A(i, HEX64(68000000, 000001c3));
 629 }
 630
 631 void
 632 CodeEmitterNVC0::emitLogicOp(const Instruction *i, uint8_t subOp)
 633 {
 634    if (i->encSize == 8) {
 635       if (isLIMM(i->src(1), TYPE_U32)) {
 636          emitForm_A(i, HEX64(38000000, 00000002));
 637
 638          if (i->srcExists(2))
 639             code[1] |= 1 << 26;
 640       } else {
 641          emitForm_A(i, HEX64(68000000, 00000003));
 642
 643          if (i->srcExists(2))
 644             code[1] |= 1 << 16;
 645       }
 646       code[0] |= subOp << 6;
 647
 648       if (i->srcExists(2)) // carry
 649          code[0] |= 1 << 5;
 650
 651       if (i->src(0).mod & Modifier(NV50_IR_MOD_NOT)) code[0] |= 1 << 9;
 652       if (i->src(1).mod & Modifier(NV50_IR_MOD_NOT)) code[0] |= 1 << 8;
 653    } else {
 654       emitForm_S(i, (subOp << 5) |
 655                  ((i->src(1).getFile() == FILE_IMMEDIATE) ? 0x1d : 0x8d), true);
 656    }
 657 }
 658
 659 void
 660 CodeEmitterNVC0::emitPOPC(const Instruction *i)
 661 {
 662    emitForm_A(i, HEX64(54000000, 00000004));
 663
 664    if (i->src(0).mod & Modifier(NV50_IR_MOD_NOT)) code[0] |= 1 << 9;
 665    if (i->src(1).mod & Modifier(NV50_IR_MOD_NOT)) code[0] |= 1 << 8;
 666 }
 667
 668 void
 669 CodeEmitterNVC0::emitINSBF(const Instruction *i)
 670 {
 671    emitForm_A(i, HEX64(28000000, 30000000));
 672 }
 673
 674 void
 675 CodeEmitterNVC0::emitShift(const Instruction *i)
 676 {
 677    if (i->op == OP_SHR) {
 678       emitForm_A(i, HEX64(58000000, 00000003)
 679                  | (isSignedType(i->dType) ? 0x20 : 0x00));
 680    } else {
 681       emitForm_A(i, HEX64(60000000, 00000003));
 682    }
 683
 684    if (i->subOp == NV50_IR_SUBOP_SHIFT_WRAP)
 685       code[0] |= 1 << 9;
 686 }
 687
 688 void
 689 CodeEmitterNVC0::emitPreOp(const Instruction *i)
 690 {
 691    if (i->encSize == 8) {
 692       emitForm_B(i, HEX64(60000000, 00000000));
 693
 694       if (i->op == OP_PREEX2)
 695          code[0] |= 0x20;
 696
 697       if (i->src(0).mod.abs()) code[0] |= 1 << 6;
 698       if (i->src(0).mod.neg()) code[0] |= 1 << 8;
 699    } else {
 700       emitForm_S(i, i->op == OP_PREEX2 ? 0x74000008 : 0x70000008, true);
 701    }
 702 }
 703
 704 void
 705 CodeEmitterNVC0::emitSFnOp(const Instruction *i, uint8_t subOp)
 706 {
 707    if (i->encSize == 8) {
 708       code[0] = 0x00000000 | (subOp << 26);
 709       code[1] = 0xc8000000;
 710
 711       emitPredicate(i);
 712
 713       defId(i->def(0), 14);
 714       srcId(i->src(0), 20);
 715
 716       assert(i->src(0).getFile() == FILE_GPR);
 717
 718       if (i->saturate) code[0] |= 1 << 5;
 719
 720       if (i->src(0).mod.abs()) code[0] |= 1 << 7;
 721       if (i->src(0).mod.neg()) code[0] |= 1 << 9;
 722    } else {
 723       emitForm_S(i, 0x80000008 | (subOp << 26), true);
 724
 725       assert(!i->src(0).mod.neg());
 726       if (i->src(0).mod.abs()) code[0] |= 1 << 30;
 727    }
 728 }
 729
 730 void
 731 CodeEmitterNVC0::emitMINMAX(const Instruction *i)
 732 {
 733    uint64_t op;
 734
 735    assert(i->encSize == 8);
 736
 737    op = (i->op == OP_MIN) ? 0x080e000000000000ULL : 0x081e000000000000ULL;
 738
 739    if (i->ftz)
 740       op |= 1 << 5;
 741    else
 742    if (!isFloatType(i->dType))
 743       op |= isSignedType(i->dType) ? 0x23 : 0x03;
 744
 745    emitForm_A(i, op);
 746    emitNegAbs12(i);
 747 }
 748
 749 void
 750 CodeEmitterNVC0::roundMode_C(const Instruction *i)
 751 {
 752    switch (i->rnd) {
 753    case ROUND_M:  code[1] |= 1 << 17; break;
 754    case ROUND_P:  code[1] |= 2 << 17; break;
 755    case ROUND_Z:  code[1] |= 3 << 17; break;
 756    case ROUND_NI: code[0] |= 1 << 7; break;
 757    case ROUND_MI: code[0] |= 1 << 7; code[1] |= 1 << 17; break;
 758    case ROUND_PI: code[0] |= 1 << 7; code[1] |= 2 << 17; break;
 759    case ROUND_ZI: code[0] |= 1 << 7; code[1] |= 3 << 17; break;
 760    case ROUND_N: break;
 761    default:
 762       assert(!"invalid round mode");
 763       break;
 764    }
 765 }
 766
 767 void
 768 CodeEmitterNVC0::roundMode_CS(const Instruction *i)
 769 {
 770    switch (i->rnd) {
 771    case ROUND_M:
 772    case ROUND_MI: code[0] |= 1 << 16; break;
 773    case ROUND_P:
 774    case ROUND_PI: code[0] |= 2 << 16; break;
 775    case ROUND_Z:
 776    case ROUND_ZI: code[0] |= 3 << 16; break;
 777    default:
 778       break;
 779    }
 780 }
 781
 782 void
 783 CodeEmitterNVC0::emitCVT(Instruction *i)
 784 {
 785    const bool f2f = isFloatType(i->dType) && isFloatType(i->sType);
 786
 787    switch (i->op) {
 788    case OP_CEIL:  i->rnd = f2f ? ROUND_PI : ROUND_P; break;
 789    case OP_FLOOR: i->rnd = f2f ? ROUND_MI : ROUND_M; break;
 790    case OP_TRUNC: i->rnd = f2f ? ROUND_ZI : ROUND_Z; break;
 791    default:
 792       break;
 793    }
 794
 795    const bool sat = (i->op == OP_SAT) || i->saturate;
 796    const bool abs = (i->op == OP_ABS) || i->src(0).mod.abs();
 797    const bool neg = (i->op == OP_NEG) || i->src(0).mod.neg();
 798
 799    if (i->encSize == 8) {
 800       emitForm_B(i, HEX64(10000000, 00000004));
 801
 802       roundMode_C(i);
 803
 804       // cvt u16 f32 sets high bits to 0, so we don't have to use Value::Size()
 805       code[0] |= util_logbase2(typeSizeof(i->dType)) << 20;
 806       code[0] |= util_logbase2(typeSizeof(i->sType)) << 23;
 807
 808       if (sat)
 809          code[0] |= 0x20;
 810       if (abs)
 811          code[0] |= 1 << 6;
 812       if (neg && i->op != OP_ABS)
 813          code[0] |= 1 << 8;
 814
 815       if (i->ftz)
 816          code[1] |= 1 << 23;
 817
 818       if (isSignedIntType(i->dType))
 819          code[0] |= 0x080;
 820       if (isSignedIntType(i->sType))
 821          code[0] |= 0x200;
 822
 823       if (isFloatType(i->dType)) {
 824          if (!isFloatType(i->sType))
 825             code[1] |= 0x08000000;
 826       } else {
 827          if (isFloatType(i->sType))
 828             code[1] |= 0x04000000;
 829          else
 830             code[1] |= 0x0c000000;
 831       }
 832    } else {
 833       if (i->op == OP_CEIL || i->op == OP_FLOOR || i->op == OP_TRUNC) {
 834          code[0] = 0x298;
 835       } else
 836       if (isFloatType(i->dType)) {
 837          if (isFloatType(i->sType))
 838             code[0] = 0x098;
 839          else
 840             code[0] = 0x088 | (isSignedType(i->sType) ? (1 << 8) : 0);
 841       } else {
 842          assert(isFloatType(i->sType));
 843
 844          code[0] = 0x288 | (isSignedType(i->sType) ? (1 << 8) : 0);
 845       }
 846
 847       if (neg) code[0] |= 1 << 16;
 848       if (sat) code[0] |= 1 << 18;
 849       if (abs) code[0] |= 1 << 19;
 850
 851       roundMode_CS(i);
 852    }
 853 }
 854
 855 void
 856 CodeEmitterNVC0::emitSET(const CmpInstruction *i)
 857 {
 858    uint32_t hi;
 859    uint32_t lo = 0;
 860
 861    if (i->sType == TYPE_F64)
 862       lo = 0x1;
 863    else
 864    if (!isFloatType(i->sType))
 865       lo = 0x3;
 866
 867    if (isFloatType(i->dType) || isSignedIntType(i->sType))
 868       lo |= 0x20;
 869
 870    switch (i->op) {
 871    case OP_SET_AND: hi = 0x10000000; break;
 872    case OP_SET_OR:  hi = 0x10200000; break;
 873    case OP_SET_XOR: hi = 0x10400000; break;
 874    default:
 875       hi = 0x100e0000;
 876       break;
 877    }
 878    emitForm_A(i, (static_cast<uint64_t>(hi) << 32) | lo);
 879
 880    if (i->op != OP_SET)
 881       srcId(i->src(2), 32 + 17);
 882
 883    if (i->def(0).getFile() == FILE_PREDICATE) {
 884       if (i->sType == TYPE_F32)
 885          code[1] += 0x10000000;
 886       else
 887          code[1] += 0x08000000;
 888
 889       code[0] &= ~0xfc000;
 890       defId(i->def(0), 17);
 891       if (i->defExists(1))
 892          defId(i->def(1), 14);
 893       else
 894          code[0] |= 0x1c000;
 895    }
 896
 897    if (i->ftz)
 898       code[1] |= 1 << 27;
 899
 900    emitCondCode(i->setCond, 32 + 23);
 901    emitNegAbs12(i);
 902 }
 903
 904 void
 905 CodeEmitterNVC0::emitSLCT(const CmpInstruction *i)
 906 {
 907    uint64_t op;
 908
 909    switch (i->dType) {
 910    case TYPE_S32:
 911       op = HEX64(30000000, 00000023);
 912       break;
 913    case TYPE_U32:
 914       op = HEX64(30000000, 00000003);
 915       break;
 916    case TYPE_F32:
 917       op = HEX64(38000000, 00000000);
 918       break;
 919    default:
 920       assert(!"invalid type for SLCT");
 921       op = 0;
 922       break;
 923    }
 924    emitForm_A(i, op);
 925
 926    CondCode cc = i->setCond;
 927
 928    if (i->src(2).mod.neg())
 929       cc = reverseCondCode(cc);
 930
 931    emitCondCode(cc, 32 + 23);
 932
 933    if (i->ftz)
 934       code[0] |= 1 << 5;
 935 }
 936
 937 void CodeEmitterNVC0::emitSELP(const Instruction *i)
 938 {
 939    emitForm_A(i, HEX64(20000000, 00000004));
 940
 941    if (i->cc == CC_NOT_P || i->src(2).mod & Modifier(NV50_IR_MOD_NOT))
 942       code[1] |= 1 << 20;
 943 }
 944
 945 void CodeEmitterNVC0::emitTEXBAR(const Instruction *i)
 946 {
 947    code[0] = 0x00000006 | (i->subOp << 26);
 948    code[1] = 0xf0000000;
 949    emitPredicate(i);
 950    emitCondCode(i->predSrc >= 0 ? i->cc : CC_ALWAYS, 5);
 951 }
 952
 953 void CodeEmitterNVC0::emitTEXCSAA(const TexInstruction *i)
 954 {
 955    code[0] = 0x00000086;
 956    code[1] = 0xd0000000;
 957
 958    code[1] |= i->tex.r;
 959    code[1] |= i->tex.s << 8;
 960
 961    if (i->tex.liveOnly)
 962       code[0] |= 1 << 9;
 963
 964    defId(i->def(0), 14);
 965    srcId(i->src(0), 20);
 966 }
 967
 968 static inline bool
 969 isNextIndependentTex(const TexInstruction *i)
 970 {
 971    if (!i->next || !isTextureOp(i->next->op))
 972       return false;
 973    if (i->getDef(0)->interfers(i->next->getSrc(0)))
 974       return false;
 975    return !i->next->srcExists(1) || !i->getDef(0)->interfers(i->next->getSrc(1));
 976 }
 977
 978 void
 979 CodeEmitterNVC0::emitTEX(const TexInstruction *i)
 980 {
 981    code[0] = 0x00000006;
 982
 983    if (isNextIndependentTex(i))
 984       code[0] |= 0x080; // t mode
 985    else
 986       code[0] |= 0x100; // p mode
 987
 988    if (i->tex.liveOnly)
 989       code[0] |= 1 << 9;
 990
 991    switch (i->op) {
 992    case OP_TEX: code[1] = 0x80000000; break;
 993    case OP_TXB: code[1] = 0x84000000; break;
 994    case OP_TXL: code[1] = 0x86000000; break;
 995    case OP_TXF: code[1] = 0x90000000; break;
 996    case OP_TXG: code[1] = 0xa0000000; break;
 997    case OP_TXD: code[1] = 0xe0000000; break;
 998    default:
 999       assert(!"invalid texture op");
1000       break;
1001    }
1002    if (i->op == OP_TXF) {
1003       if (!i->tex.levelZero)
1004          code[1] |= 0x02000000;
1005    } else
1006    if (i->tex.levelZero) {
1007       code[1] |= 0x02000000;
1008    }
1009
1010    if (i->tex.derivAll)
1011       code[1] |= 1 << 13;
1012
1013    defId(i->def(0), 14);
1014    srcId(i->src(0), 20);
1015
1016    emitPredicate(i);
1017
1018    if (i->op == OP_TXG) code[0] |= i->tex.gatherComp << 5;
1019
1020    code[1] |= i->tex.mask << 14;
1021
1022    code[1] |= i->tex.r;
1023    code[1] |= i->tex.s << 8;
1024    if (i->tex.rIndirectSrc >= 0 || i->tex.sIndirectSrc >= 0)
1025       code[1] |= 1 << 18; // in 1st source (with array index)
1026
1027    // texture target:
1028    code[1] |= (i->tex.target.getDim() - 1) << 20;
1029    if (i->tex.target.isCube())
1030       code[1] += 2 << 20;
1031    if (i->tex.target.isArray())
1032       code[1] |= 1 << 19;
1033    if (i->tex.target.isShadow())
1034       code[1] |= 1 << 24;
1035
1036    const int src1 = (i->predSrc == 1) ? 2 : 1; // if predSrc == 1, !srcExists(2)
1037
1038    if (i->srcExists(src1) && i->src(src1).getFile() == FILE_IMMEDIATE) {
1039       // lzero
1040       if (i->op == OP_TXL)
1041          code[1] &= ~(1 << 26);
1042       else
1043       if (i->op == OP_TXF)
1044          code[1] &= ~(1 << 25);
1045    }
1046    if (i->tex.target == TEX_TARGET_2D_MS ||
1047        i->tex.target == TEX_TARGET_2D_MS_ARRAY)
1048       code[1] |= 1 << 23;
1049
1050    if (i->tex.useOffsets) // in vecSrc0.w
1051       code[1] |= 1 << 22;
1052
1053    srcId(i, src1, 26);
1054 }
1055
1056 void
1057 CodeEmitterNVC0::emitTXQ(const TexInstruction *i)
1058 {
1059    code[0] = 0x00000086;
1060    code[1] = 0xc0000000;
1061
1062    switch (i->tex.query) {
1063    case TXQ_DIMS:            code[1] |= 0 << 22; break;
1064    case TXQ_TYPE:            code[1] |= 1 << 22; break;
1065    case TXQ_SAMPLE_POSITION: code[1] |= 2 << 22; break;
1066    case TXQ_FILTER:          code[1] |= 3 << 22; break;
1067    case TXQ_LOD:             code[1] |= 4 << 22; break;
1068    case TXQ_BORDER_COLOUR:   code[1] |= 5 << 22; break;
1069    default:
1070       assert(!"invalid texture query");
1071       break;
1072    }
1073
1074    code[1] |= i->tex.mask << 14;
1075
1076    code[1] |= i->tex.r;
1077    code[1] |= i->tex.s << 8;
1078    if (i->tex.sIndirectSrc >= 0 || i->tex.rIndirectSrc >= 0)
1079       code[1] |= 1 << 18;
1080
1081    const int src1 = (i->predSrc == 1) ? 2 : 1; // if predSrc == 1, !srcExists(2)
1082
1083    defId(i->def(0), 14);
1084    srcId(i->src(0), 20);
1085    srcId(i, src1, 26);
1086
1087    emitPredicate(i);
1088 }
1089
1090 void
1091 CodeEmitterNVC0::emitQUADOP(const Instruction *i, uint8_t qOp, uint8_t laneMask)
1092 {
1093    code[0] = 0x00000000 | (laneMask << 6);
1094    code[1] = 0x48000000 | qOp;
1095
1096    defId(i->def(0), 14);
1097    srcId(i->src(0), 20);
1098    srcId(i->srcExists(1) ? i->src(1) : i->src(0), 26);
1099
1100    if (i->op == OP_QUADOP && progType != Program::TYPE_FRAGMENT)
1101       code[0] |= 1 << 9; // dall
1102
1103    emitPredicate(i);
1104 }
1105
1106 void
1107 CodeEmitterNVC0::emitFlow(const Instruction *i)
1108 {
1109    const FlowInstruction *f = i->asFlow();
1110
1111    unsigned mask; // bit 0: predicate, bit 1: target
1112
1113    code[0] = 0x00000007;
1114
1115    switch (i->op) {
1116    case OP_BRA:
1117       code[1] = f->absolute ? 0x00000000 : 0x40000000;
1118       if (i->srcExists(0) && i->src(0).getFile() == FILE_MEMORY_CONST)
1119          code[0] |= 0x4000;
1120       mask = 3;
1121       break;
1122    case OP_CALL:
1123       code[1] = f->absolute ? 0x10000000 : 0x50000000;
1124       if (i->srcExists(0) && i->src(0).getFile() == FILE_MEMORY_CONST)
1125          code[0] |= 0x4000;
1126       mask = 2;
1127       break;
1128
1129    case OP_EXIT:    code[1] = 0x80000000; mask = 1; break;
1130    case OP_RET:     code[1] = 0x90000000; mask = 1; break;
1131    case OP_DISCARD: code[1] = 0x98000000; mask = 1; break;
1132    case OP_BREAK:   code[1] = 0xa8000000; mask = 1; break;
1133    case OP_CONT:    code[1] = 0xb0000000; mask = 1; break;
1134
1135    case OP_JOINAT:   code[1] = 0x60000000; mask = 2; break;
1136    case OP_PREBREAK: code[1] = 0x68000000; mask = 2; break;
1137    case OP_PRECONT:  code[1] = 0x70000000; mask = 2; break;
1138    case OP_PRERET:   code[1] = 0x78000000; mask = 2; break;
1139
1140    case OP_QUADON:  code[1] = 0xc0000000; mask = 0; break;
1141    case OP_QUADPOP: code[1] = 0xc8000000; mask = 0; break;
1142    case OP_BRKPT:   code[1] = 0xd0000000; mask = 0; break;
1143    default:
1144       assert(!"invalid flow operation");
1145       return;
1146    }
1147
1148    if (mask & 1) {
1149       emitPredicate(i);
1150       if (i->flagsSrc < 0)
1151          code[0] |= 0x1e0;
1152    }
1153
1154    if (!f)
1155       return;
1156
1157    if (f->allWarp)
1158       code[0] |= 1 << 15;
1159    if (f->limit)
1160       code[0] |= 1 << 16;
1161
1162    if (f->op == OP_CALL) {
1163       if (f->builtin) {
1164          assert(f->absolute);
1165          uint32_t pcAbs = targ->getBuiltinOffset(f->target.builtin);
1166          addReloc(RelocEntry::TYPE_BUILTIN, 0, pcAbs, 0xfc000000, 26);
1167          addReloc(RelocEntry::TYPE_BUILTIN, 1, pcAbs, 0x03ffffff, -6);
1168       } else {
1169          assert(!f->absolute);
1170          int32_t pcRel = f->target.fn->binPos - (codeSize + 8);
1171          code[0] |= (pcRel & 0x3f) << 26;
1172          code[1] |= (pcRel >> 6) & 0x3ffff;
1173       }
1174    } else
1175    if (mask & 2) {
1176       int32_t pcRel = f->target.bb->binPos - (codeSize + 8);
1177       // currently we don't want absolute branches
1178       assert(!f->absolute);
1179       code[0] |= (pcRel & 0x3f) << 26;
1180       code[1] |= (pcRel >> 6) & 0x3ffff;
1181    }
1182 }
1183
1184 void
1185 CodeEmitterNVC0::emitPFETCH(const Instruction *i)
1186 {
1187    uint32_t prim = i->src(0).get()->reg.data.u32;
1188
1189    code[0] = 0x00000006 | ((prim & 0x3f) << 26);
1190    code[1] = 0x00000000 | (prim >> 6);
1191
1192    emitPredicate(i);
1193
1194    defId(i->def(0), 14);
1195    srcId(i->src(1), 20);
1196 }
1197
1198 void
1199 CodeEmitterNVC0::emitVFETCH(const Instruction *i)
1200 {
1201    code[0] = 0x00000006;
1202    code[1] = 0x06000000 | i->src(0).get()->reg.data.offset;
1203
1204    if (i->perPatch)
1205       code[0] |= 0x100;
1206    if (i->getSrc(0)->reg.file == FILE_SHADER_OUTPUT)
1207       code[0] |= 0x200; // yes, TCPs can read from *outputs* of other threads
1208
1209    emitPredicate(i);
1210
1211    code[0] |= ((i->getDef(0)->reg.size / 4) - 1) << 5;
1212
1213    defId(i->def(0), 14);
1214    srcId(i->src(0).getIndirect(0), 20);
1215    srcId(i->src(0).getIndirect(1), 26); // vertex address
1216 }
1217
1218 void
1219 CodeEmitterNVC0::emitEXPORT(const Instruction *i)
1220 {
1221    unsigned int size = typeSizeof(i->dType);
1222
1223    code[0] = 0x00000006 | ((size / 4 - 1) << 5);
1224    code[1] = 0x0a000000 | i->src(0).get()->reg.data.offset;
1225
1226    assert(!(code[1] & ((size == 12) ? 15 : (size - 1))));
1227
1228    if (i->perPatch)
1229       code[0] |= 0x100;
1230
1231    emitPredicate(i);
1232
1233    assert(i->src(1).getFile() == FILE_GPR);
1234
1235    srcId(i->src(0).getIndirect(0), 20);
1236    srcId(i->src(0).getIndirect(1), 32 + 17); // vertex base address
1237    srcId(i->src(1), 26);
1238 }
1239
1240 void
1241 CodeEmitterNVC0::emitOUT(const Instruction *i)
1242 {
1243    code[0] = 0x00000006;
1244    code[1] = 0x1c000000;
1245
1246    emitPredicate(i);
1247
1248    defId(i->def(0), 14); // new secret address
1249    srcId(i->src(0), 20); // old secret address, should be 0 initially
1250
1251    assert(i->src(0).getFile() == FILE_GPR);
1252
1253    if (i->op == OP_EMIT)
1254       code[0] |= 1 << 5;
1255    if (i->op == OP_RESTART || i->subOp == NV50_IR_SUBOP_EMIT_RESTART)
1256       code[0] |= 1 << 6;
1257
1258    // vertex stream
1259    if (i->src(1).getFile() == FILE_IMMEDIATE) {
1260       code[1] |= 0xc000;
1261       code[0] |= SDATA(i->src(1)).u32 << 26;
1262    } else {
1263       srcId(i->src(1), 26);
1264    }
1265 }
1266
1267 void
1268 CodeEmitterNVC0::emitInterpMode(const Instruction *i)
1269 {
1270    if (i->encSize == 8) {
1271       code[0] |= i->ipa << 6; // TODO: INTERP_SAMPLEID
1272    } else {
1273       if (i->getInterpMode() == NV50_IR_INTERP_SC)
1274          code[0] |= 0x80;
1275       assert(i->op == OP_PINTERP && i->getSampleMode() == 0);
1276    }
1277 }
1278
1279 void
1280 CodeEmitterNVC0::emitINTERP(const Instruction *i)
1281 {
1282    const uint32_t base = i->getSrc(0)->reg.data.offset;
1283
1284    if (i->encSize == 8) {
1285       code[0] = 0x00000000;
1286       code[1] = 0xc0000000 | (base & 0xffff);
1287
1288       if (i->saturate)
1289          code[0] |= 1 << 5;
1290
1291       if (i->op == OP_PINTERP)
1292          srcId(i->src(1), 26);
1293       else
1294          code[0] |= 0x3f << 26;
1295
1296       srcId(i->src(0).getIndirect(0), 20);
1297    } else {
1298       assert(i->op == OP_PINTERP);
1299       code[0] = 0x00000009 | ((base & 0xc) << 6) | ((base >> 4) << 26);
1300       srcId(i->src(1), 20);
1301    }
1302    emitInterpMode(i);
1303
1304    emitPredicate(i);
1305    defId(i->def(0), 14);
1306
1307    if (i->getSampleMode() == NV50_IR_INTERP_OFFSET)
1308       srcId(i->src(i->op == OP_PINTERP ? 2 : 1), 17);
1309    else
1310       code[1] |= 0x3f << 17;
1311 }
1312
1313 void
1314 CodeEmitterNVC0::emitLoadStoreType(DataType ty)
1315 {
1316    uint8_t val;
1317
1318    switch (ty) {
1319    case TYPE_U8:
1320       val = 0x00;
1321       break;
1322    case TYPE_S8:
1323       val = 0x20;
1324       break;
1325    case TYPE_F16:
1326    case TYPE_U16:
1327       val = 0x40;
1328       break;
1329    case TYPE_S16:
1330       val = 0x60;
1331       break;
1332    case TYPE_F32:
1333    case TYPE_U32:
1334    case TYPE_S32:
1335       val = 0x80;
1336       break;
1337    case TYPE_F64:
1338    case TYPE_U64:
1339    case TYPE_S64:
1340       val = 0xa0;
1341       break;
1342    case TYPE_B128:
1343       val = 0xc0;
1344       break;
1345    default:
1346       val = 0x80;
1347       assert(!"invalid type");
1348       break;
1349    }
1350    code[0] |= val;
1351 }
1352
1353 void
1354 CodeEmitterNVC0::emitCachingMode(CacheMode c)
1355 {
1356    uint32_t val;
1357
1358    switch (c) {
1359    case CACHE_CA:
1360 // case CACHE_WB:
1361       val = 0x000;
1362       break;
1363    case CACHE_CG:
1364       val = 0x100;
1365       break;
1366    case CACHE_CS:
1367       val = 0x200;
1368       break;
1369    case CACHE_CV:
1370 // case CACHE_WT:
1371       val = 0x300;
1372       break;
1373    default:
1374       val = 0;
1375       assert(!"invalid caching mode");
1376       break;
1377    }
1378    code[0] |= val;
1379 }
1380
1381 void
1382 CodeEmitterNVC0::emitSTORE(const Instruction *i)
1383 {
1384    uint32_t opc;
1385
1386    switch (i->src(0).getFile()) {
1387    case FILE_MEMORY_GLOBAL: opc = 0x90000000; break;
1388    case FILE_MEMORY_LOCAL:  opc = 0xc8000000; break;
1389    case FILE_MEMORY_SHARED: opc = 0xc9000000; break;
1390    default:
1391       assert(!"invalid memory file");
1392       opc = 0;
1393       break;
1394    }
1395    code[0] = 0x00000005;
1396    code[1] = opc;
1397
1398    setAddress16(i->src(0));
1399    srcId(i->src(1), 14);
1400    srcId(i->src(0).getIndirect(0), 20);
1401
1402    emitPredicate(i);
1403
1404    emitLoadStoreType(i->dType);
1405    emitCachingMode(i->cache);
1406 }
1407
1408 void
1409 CodeEmitterNVC0::emitLOAD(const Instruction *i)
1410 {
1411    uint32_t opc;
1412
1413    code[0] = 0x00000005;
1414
1415    switch (i->src(0).getFile()) {
1416    case FILE_MEMORY_GLOBAL: opc = 0x80000000; break;
1417    case FILE_MEMORY_LOCAL:  opc = 0xc0000000; break;
1418    case FILE_MEMORY_SHARED: opc = 0xc1000000; break;
1419    case FILE_MEMORY_CONST:
1420       if (!i->src(0).isIndirect(0) && typeSizeof(i->dType) == 4) {
1421          emitMOV(i); // not sure if this is any better
1422          return;
1423       }
1424       opc = 0x14000000 | (i->src(0).get()->reg.fileIndex << 10);
1425       code[0] = 0x00000006 | (i->subOp << 8);
1426       break;
1427    default:
1428       assert(!"invalid memory file");
1429       opc = 0;
1430       break;
1431    }
1432    code[1] = opc;
1433
1434    defId(i->def(0), 14);
1435
1436    setAddress16(i->src(0));
1437    srcId(i->src(0).getIndirect(0), 20);
1438
1439    emitPredicate(i);
1440
1441    emitLoadStoreType(i->dType);
1442    emitCachingMode(i->cache);
1443 }
1444
1445 uint8_t
1446 CodeEmitterNVC0::getSRegEncoding(const ValueRef& ref)
1447 {
1448    switch (SDATA(ref).sv.sv) {
1449    case SV_LANEID:        return 0x00;
1450    case SV_PHYSID:        return 0x03;
1451    case SV_VERTEX_COUNT:  return 0x10;
1452    case SV_INVOCATION_ID: return 0x11;
1453    case SV_YDIR:          return 0x12;
1454    case SV_TID:           return 0x21 + SDATA(ref).sv.index;
1455    case SV_CTAID:         return 0x25 + SDATA(ref).sv.index;
1456    case SV_NTID:          return 0x29 + SDATA(ref).sv.index;
1457    case SV_GRIDID:        return 0x2c;
1458    case SV_NCTAID:        return 0x2d + SDATA(ref).sv.index;
1459    case SV_LBASE:         return 0x34;
1460    case SV_SBASE:         return 0x30;
1461    case SV_CLOCK:         return 0x50 + SDATA(ref).sv.index;
1462    default:
1463       assert(!"no sreg for system value");
1464       return 0;
1465    }
1466 }
1467
1468 void
1469 CodeEmitterNVC0::emitMOV(const Instruction *i)
1470 {
1471    if (i->src(0).getFile() == FILE_SYSTEM_VALUE) {
1472       uint8_t sr = getSRegEncoding(i->src(0));
1473
1474       if (i->encSize == 8) {
1475          code[0] = 0x00000004 | (sr << 26);
1476          code[1] = 0x2c000000;
1477       } else {
1478          code[0] = 0x40000008 | (sr << 20);
1479       }
1480       defId(i->def(0), 14);
1481
1482       emitPredicate(i);
1483    } else
1484    if (i->encSize == 8) {
1485       uint64_t opc;
1486
1487       if (i->src(0).getFile() == FILE_IMMEDIATE)
1488          opc = HEX64(18000000, 000001e2);
1489       else
1490       if (i->src(0).getFile() == FILE_PREDICATE)
1491          opc = HEX64(080e0000, 1c000004);
1492       else
1493          opc = HEX64(28000000, 00000004);
1494
1495       opc |= i->lanes << 5;
1496
1497       emitForm_B(i, opc);
1498    } else {
1499       uint32_t imm;
1500
1501       if (i->src(0).getFile() == FILE_IMMEDIATE) {
1502          imm = SDATA(i->src(0)).u32;
1503          if (imm & 0xfff00000) {
1504             assert(!(imm & 0x000fffff));
1505             code[0] = 0x00000318 | imm;
1506          } else {
1507             assert(imm < 0x800 || ((int32_t)imm >= -0x800));
1508             code[0] = 0x00000118 | (imm << 20);
1509          }
1510       } else {
1511          code[0] = 0x0028;
1512          emitShortSrc2(i->src(0));
1513       }
1514       defId(i->def(0), 14);
1515
1516       emitPredicate(i);
1517    }
1518 }
1519
1520 bool
1521 CodeEmitterNVC0::emitInstruction(Instruction *insn)
1522 {
1523    unsigned int size = insn->encSize;
1524
1525    if (writeIssueDelays && !(codeSize & 0x3f))
1526       size += 8;
1527
1528    if (!insn->encSize) {
1529       ERROR("skipping unencodable instruction: "); insn->print();
1530       return false;
1531    } else
1532    if (codeSize + size > codeSizeLimit) {
1533       ERROR("code emitter output buffer too small\n");
1534       return false;
1535    }
1536
1537    if (writeIssueDelays) {
1538       if (!(codeSize & 0x3f)) {
1539          code[0] = 0x00000007; // cf issue delay "instruction"
1540          code[1] = 0x20000000;
1541          code += 2;
1542          codeSize += 8;
1543       }
1544       const unsigned int id = (codeSize & 0x3f) / 8 - 1;
1545       uint32_t *data = code - (id * 2 + 2);
1546       if (id <= 2) {
1547          data[0] |= insn->sched << (id * 8 + 4);
1548       } else
1549       if (id == 3) {
1550          data[0] |= insn->sched << 28;
1551          data[1] |= insn->sched >> 4;
1552       } else {
1553          data[1] |= insn->sched << ((id - 4) * 8 + 4);
1554       }
1555    }
1556
1557    // assert that instructions with multiple defs don't corrupt registers
1558    for (int d = 0; insn->defExists(d); ++d)
1559       assert(insn->asTex() || insn->def(d).rep()->reg.data.id >= 0);
1560
1561    switch (insn->op) {
1562    case OP_MOV:
1563    case OP_RDSV:
1564       emitMOV(insn);
1565       break;
1566    case OP_NOP:
1567       break;
1568    case OP_LOAD:
1569       emitLOAD(insn);
1570       break;
1571    case OP_STORE:
1572       emitSTORE(insn);
1573       break;
1574    case OP_LINTERP:
1575    case OP_PINTERP:
1576       emitINTERP(insn);
1577       break;
1578    case OP_VFETCH:
1579       emitVFETCH(insn);
1580       break;
1581    case OP_EXPORT:
1582       emitEXPORT(insn);
1583       break;
1584    case OP_PFETCH:
1585       emitPFETCH(insn);
1586       break;
1587    case OP_EMIT:
1588    case OP_RESTART:
1589       emitOUT(insn);
1590       break;
1591    case OP_ADD:
1592    case OP_SUB:
1593       if (isFloatType(insn->dType))
1594          emitFADD(insn);
1595       else
1596          emitUADD(insn);
1597       break;
1598    case OP_MUL:
1599       if (isFloatType(insn->dType))
1600          emitFMUL(insn);
1601       else
1602          emitUMUL(insn);
1603       break;
1604    case OP_MAD:
1605    case OP_FMA:
1606       if (isFloatType(insn->dType))
1607          emitFMAD(insn);
1608       else
1609          emitIMAD(insn);
1610       break;
1611    case OP_NOT:
1612       emitNOT(insn);
1613       break;
1614    case OP_AND:
1615       emitLogicOp(insn, 0);
1616       break;
1617    case OP_OR:
1618       emitLogicOp(insn, 1);
1619       break;
1620    case OP_XOR:
1621       emitLogicOp(insn, 2);
1622       break;
1623    case OP_SHL:
1624    case OP_SHR:
1625       emitShift(insn);
1626       break;
1627    case OP_SET:
1628    case OP_SET_AND:
1629    case OP_SET_OR:
1630    case OP_SET_XOR:
1631       emitSET(insn->asCmp());
1632       break;
1633    case OP_SELP:
1634       emitSELP(insn);
1635       break;
1636    case OP_SLCT:
1637       emitSLCT(insn->asCmp());
1638       break;
1639    case OP_MIN:
1640    case OP_MAX:
1641       emitMINMAX(insn);
1642       break;
1643    case OP_ABS:
1644    case OP_NEG:
1645    case OP_CEIL:
1646    case OP_FLOOR:
1647    case OP_TRUNC:
1648    case OP_CVT:
1649    case OP_SAT:
1650       emitCVT(insn);
1651       break;
1652    case OP_RSQ:
1653       emitSFnOp(insn, 5);
1654       break;
1655    case OP_RCP:
1656       emitSFnOp(insn, 4);
1657       break;
1658    case OP_LG2:
1659       emitSFnOp(insn, 3);
1660       break;
1661    case OP_EX2:
1662       emitSFnOp(insn, 2);
1663       break;
1664    case OP_SIN:
1665       emitSFnOp(insn, 1);
1666       break;
1667    case OP_COS:
1668       emitSFnOp(insn, 0);
1669       break;
1670    case OP_PRESIN:
1671    case OP_PREEX2:
1672       emitPreOp(insn);
1673       break;
1674    case OP_TEX:
1675    case OP_TXB:
1676    case OP_TXL:
1677    case OP_TXD:
1678    case OP_TXF:
1679       emitTEX(insn->asTex());
1680       break;
1681    case OP_TXQ:
1682       emitTXQ(insn->asTex());
1683       break;
1684    case OP_TEXBAR:
1685       emitTEXBAR(insn);
1686       break;
1687    case OP_BRA:
1688    case OP_CALL:
1689    case OP_PRERET:
1690    case OP_RET:
1691    case OP_DISCARD:
1692    case OP_EXIT:
1693    case OP_PRECONT:
1694    case OP_CONT:
1695    case OP_PREBREAK:
1696    case OP_BREAK:
1697    case OP_JOINAT:
1698    case OP_BRKPT:
1699    case OP_QUADON:
1700    case OP_QUADPOP:
1701       emitFlow(insn);
1702       break;
1703    case OP_QUADOP:
1704       emitQUADOP(insn, insn->subOp, insn->lanes);
1705       break;
1706    case OP_DFDX:
1707       emitQUADOP(insn, insn->src(0).mod.neg() ? 0x66 : 0x99, 0x4);
1708       break;
1709    case OP_DFDY:
1710       emitQUADOP(insn, insn->src(0).mod.neg() ? 0x5a : 0xa5, 0x5);
1711       break;
1712    case OP_POPCNT:
1713       emitPOPC(insn);
1714       break;
1715    case OP_JOIN:
1716       emitNOP(insn);
1717       insn->join = 1;
1718       break;
1719    case OP_PHI:
1720    case OP_UNION:
1721    case OP_CONSTRAINT:
1722       ERROR("operation should have been eliminated");
1723       return false;
1724    case OP_EXP:
1725    case OP_LOG:
1726    case OP_SQRT:
1727    case OP_POW:
1728       ERROR("operation should have been lowered\n");
1729       return false;
1730    default:
1731       ERROR("unknow op\n");
1732       return false;
1733    }
1734
1735    if (insn->join) {
1736       code[0] |= 0x10;
1737       assert(insn->encSize == 8);
1738    }
1739
1740    code += insn->encSize / 4;
1741    codeSize += insn->encSize;
1742    return true;
1743 }
1744
1745 uint32_t
1746 CodeEmitterNVC0::getMinEncodingSize(const Instruction *i) const
1747 {
1748    const Target::OpInfo &info = targ->getOpInfo(i);
1749
1750    if (writeIssueDelays || info.minEncSize == 8 || 1)
1751       return 8;
1752
1753    if (i->ftz || i->saturate || i->join)
1754       return 8;
1755    if (i->rnd != ROUND_N)
1756       return 8;
1757    if (i->predSrc >= 0 && i->op == OP_MAD)
1758       return 8;
1759
1760    if (i->op == OP_PINTERP) {
1761       if (i->getSampleMode() || 1) // XXX: grr, short op doesn't work
1762          return 8;
1763    } else
1764    if (i->op == OP_MOV && i->lanes != 0xf) {
1765       return 8;
1766    }
1767
1768    for (int s = 0; i->srcExists(s); ++s) {
1769       if (i->src(s).isIndirect(0))
1770          return 8;
1771
1772       if (i->src(s).getFile() == FILE_MEMORY_CONST) {
1773          if (SDATA(i->src(s)).offset >= 0x100)
1774             return 8;
1775          if (i->getSrc(s)->reg.fileIndex > 1 &&
1776              i->getSrc(s)->reg.fileIndex != 16)
1777              return 8;
1778       } else
1779       if (i->src(s).getFile() == FILE_IMMEDIATE) {
1780          if (i->dType == TYPE_F32) {
1781             if (SDATA(i->src(s)).u32 >= 0x100)
1782                return 8;
1783          } else {
1784             if (SDATA(i->src(s)).u32 > 0xff)
1785                return 8;
1786          }
1787       }
1788
1789       if (i->op == OP_CVT)
1790          continue;
1791       if (i->src(s).mod != Modifier(0)) {
1792          if (i->src(s).mod == Modifier(NV50_IR_MOD_ABS))
1793             if (i->op != OP_RSQ)
1794                return 8;
1795          if (i->src(s).mod == Modifier(NV50_IR_MOD_NEG))
1796             if (i->op != OP_ADD || s != 0)
1797                return 8;
1798       }
1799    }
1800
1801    return 4;
1802 }
1803
1804 // Simplified, erring on safe side.
1805 class SchedDataCalculator : public Pass
1806 {
1807 public:
1808    SchedDataCalculator(const Target *targ) : targ(targ) { }
1809
1810 private:
1811    struct RegScores
1812    {
1813       struct Resource {
1814          int st[DATA_FILE_COUNT]; // LD to LD delay 3
1815          int ld[DATA_FILE_COUNT]; // ST to ST delay 3
1816          int tex; // TEX to non-TEX delay 17 (0x11)
1817          int sfu; // SFU to SFU delay 3 (except PRE-ops)
1818          int imul; // integer MUL to MUL delay 3
1819       } res;
1820       struct ScoreData {
1821          int r[64];
1822          int p[8];
1823          int c;
1824       } rd, wr;
1825       int base;
1826
1827       void rebase(const int base)
1828       {
1829          const int delta = this->base - base;
1830          if (!delta)
1831             return;
1832          this->base = 0;
1833
1834          for (int i = 0; i < 64; ++i) {
1835             rd.r[i] += delta;
1836             wr.r[i] += delta;
1837          }
1838          for (int i = 0; i < 8; ++i) {
1839             rd.p[i] += delta;
1840             wr.p[i] += delta;
1841          }
1842          rd.c += delta;
1843          wr.c += delta;
1844
1845          for (unsigned int f = 0; f < DATA_FILE_COUNT; ++f) {
1846             res.ld[f] += delta;
1847             res.st[f] += delta;
1848          }
1849          res.sfu += delta;
1850          res.imul += delta;
1851          res.tex += delta;
1852       }
1853       void wipe()
1854       {
1855          memset(&rd, 0, sizeof(rd));
1856          memset(&wr, 0, sizeof(wr));
1857          memset(&res, 0, sizeof(res));
1858       }
1859       int getLatest(const ScoreData& d) const
1860       {
1861          int max = 0;
1862          for (int i = 0; i < 64; ++i)
1863             if (d.r[i] > max)
1864                max = d.r[i];
1865          for (int i = 0; i < 8; ++i)
1866             if (d.p[i] > max)
1867                max = d.p[i];
1868          if (d.c > max)
1869             max = d.c;
1870          return max;
1871       }
1872       inline int getLatestRd() const
1873       {
1874          return getLatest(rd);
1875       }
1876       inline int getLatestWr() const
1877       {
1878          return getLatest(wr);
1879       }
1880       inline int getLatest() const
1881       {
1882          const int a = getLatestRd();
1883          const int b = getLatestWr();
1884
1885          int max = MAX2(a, b);
1886          for (unsigned int f = 0; f < DATA_FILE_COUNT; ++f) {
1887             max = MAX2(res.ld[f], max);
1888             max = MAX2(res.st[f], max);
1889          }
1890          max = MAX2(res.sfu, max);
1891          max = MAX2(res.imul, max);
1892          max = MAX2(res.tex, max);
1893          return max;
1894       }
1895       void setMax(const RegScores *that)
1896       {
1897          for (int i = 0; i < 64; ++i) {
1898             rd.r[i] = MAX2(rd.r[i], that->rd.r[i]);
1899             wr.r[i] = MAX2(wr.r[i], that->wr.r[i]);
1900          }
1901          for (int i = 0; i < 8; ++i) {
1902             rd.p[i] = MAX2(rd.p[i], that->rd.p[i]);
1903             wr.p[i] = MAX2(wr.p[i], that->wr.p[i]);
1904          }
1905          rd.c = MAX2(rd.c, that->rd.c);
1906          wr.c = MAX2(wr.c, that->wr.c);
1907
1908          for (unsigned int f = 0; f < DATA_FILE_COUNT; ++f) {
1909             res.ld[f] = MAX2(res.ld[f], that->res.ld[f]);
1910             res.st[f] = MAX2(res.st[f], that->res.st[f]);
1911          }
1912          res.sfu = MAX2(res.sfu, that->res.sfu);
1913          res.imul = MAX2(res.imul, that->res.imul);
1914          res.tex = MAX2(res.tex, that->res.tex);
1915       }
1916       void print(int cycle)
1917       {
1918          for (int i = 0; i < 64; ++i) {
1919             if (rd.r[i] > cycle)
1920                INFO("rd $r%i @ %i\n", i, rd.r[i]);
1921             if (wr.r[i] > cycle)
1922                INFO("wr $r%i @ %i\n", i, wr.r[i]);
1923          }
1924          for (int i = 0; i < 8; ++i) {
1925             if (rd.p[i] > cycle)
1926                INFO("rd $p%i @ %i\n", i, rd.p[i]);
1927             if (wr.p[i] > cycle)
1928                INFO("wr $p%i @ %i\n", i, wr.p[i]);
1929          }
1930          if (rd.c > cycle)
1931             INFO("rd $c @ %i\n", rd.c);
1932          if (wr.c > cycle)
1933             INFO("wr $c @ %i\n", wr.c);
1934          if (res.sfu > cycle)
1935             INFO("sfu @ %i\n", res.sfu);
1936          if (res.imul > cycle)
1937             INFO("imul @ %i\n", res.imul);
1938          if (res.tex > cycle)
1939             INFO("tex @ %i\n", res.tex);
1940       }
1941    };
1942
1943    RegScores *score; // for current BB
1944    std::vector<RegScores> scoreBoards;
1945    int cycle;
1946    int prevData;
1947    operation prevOp;
1948
1949    const Target *targ;
1950
1951    bool visit(Function *);
1952    bool visit(BasicBlock *);
1953
1954    void commitInsn(const Instruction *, int cycle);
1955    int calcDelay(const Instruction *, int cycle) const;
1956    void setDelay(Instruction *, int delay, Instruction *next);
1957
1958    void recordRd(const Value *, const int ready);
1959    void recordWr(const Value *, const int ready);
1960    void checkRd(const Value *, int cycle, int& delay) const;
1961    void checkWr(const Value *, int cycle, int& delay) const;
1962
1963    int getCycles(const Instruction *, int origDelay) const;
1964 };
1965
1966 void
1967 SchedDataCalculator::setDelay(Instruction *insn, int delay, Instruction *next)
1968 {
1969    if (insn->op == OP_EXIT)
1970       delay = MAX2(delay, 14);
1971
1972    if (insn->op == OP_TEXBAR) {
1973       // TODO: except if results not used before EXIT
1974       insn->sched = 0xc2;
1975    } else
1976    if (insn->op == OP_JOIN || insn->join) {
1977       insn->sched = 0x00;
1978    } else
1979    if (delay >= 0 || prevData == 0x04 ||
1980        !next || !targ->canDualIssue(insn, next)) {
1981       insn->sched = static_cast<uint8_t>(MAX2(delay, 0));
1982       if (prevOp == OP_EXPORT)
1983          insn->sched |= 0x40;
1984       else
1985          insn->sched |= 0x20;
1986    } else {
1987       insn->sched = 0x04; // dual-issue
1988    }
1989
1990    if (prevData != 0x04 || prevOp != OP_EXPORT)
1991       if (insn->sched != 0x04 || insn->op == OP_EXPORT)
1992          prevOp = insn->op;
1993
1994    prevData = insn->sched;
1995 }
1996
1997 int
1998 SchedDataCalculator::getCycles(const Instruction *insn, int origDelay) const
1999 {
2000    if (insn->sched & 0x80) {
2001       int c = (insn->sched & 0x0f) * 2 + 1;
2002       if (insn->op == OP_TEXBAR && origDelay > 0)
2003          c += origDelay;
2004       return c;
2005    }
2006    if (insn->sched & 0x60)
2007       return (insn->sched & 0x1f) + 1;
2008    return (insn->sched == 0x04) ? 0 : 32;
2009 }
2010
2011 bool
2012 SchedDataCalculator::visit(Function *func)
2013 {
2014    scoreBoards.resize(func->cfg.getSize());
2015    for (size_t i = 0; i < scoreBoards.size(); ++i)
2016       scoreBoards[i].wipe();
2017    return true;
2018 }
2019
2020 bool
2021 SchedDataCalculator::visit(BasicBlock *bb)
2022 {
2023    Instruction *insn;
2024    Instruction *next = NULL;
2025
2026    int cycle = 0;
2027
2028    prevData = 0x00;
2029    prevOp = OP_NOP;
2030    score = &scoreBoards.at(bb->getId());
2031
2032    for (Graph::EdgeIterator ei = bb->cfg.incident(); !ei.end(); ei.next()) {
2033       BasicBlock *in = BasicBlock::get(ei.getNode());
2034       if (in->getExit()) {
2035          if (prevData != 0x04)
2036             prevData = in->getExit()->sched;
2037          prevOp = in->getExit()->op;
2038       }
2039       if (ei.getType() != Graph::Edge::BACK)
2040          score->setMax(&scoreBoards.at(in->getId()));
2041       // back branches will wait until all target dependencies are satisfied
2042    }
2043    if (bb->cfg.incidentCount() > 1)
2044       prevOp = OP_NOP;
2045
2046 #ifdef NVC0_DEBUG_SCHED_DATA
2047    INFO("=== BB:%i initial scores\n", bb->getId());
2048    score->print(cycle);
2049 #endif
2050
2051    for (insn = bb->getEntry(); insn && insn->next; insn = insn->next) {
2052       next = insn->next;
2053
2054       commitInsn(insn, cycle);
2055       int delay = calcDelay(next, cycle);
2056       setDelay(insn, delay, next);
2057       cycle += getCycles(insn, delay);
2058
2059 #ifdef NVC0_DEBUG_SCHED_DATA
2060       INFO("cycle %i, sched %02x\n", cycle, insn->sched);
2061       insn->print();
2062       next->print();
2063 #endif
2064    }
2065    if (!insn)
2066       return true;
2067    commitInsn(insn, cycle);
2068
2069    int bbDelay = -1;
2070
2071    for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) {
2072       BasicBlock *out = BasicBlock::get(ei.getNode());
2073
2074       if (ei.getType() != Graph::Edge::BACK) {
2075          // only test the first instruction of the outgoing block
2076          next = out->getEntry();
2077          if (next)
2078             bbDelay = MAX2(bbDelay, calcDelay(next, cycle));
2079       } else {
2080          // wait until all dependencies are satisfied
2081          const int regsFree = score->getLatest();
2082          next = out->getFirst();
2083          for (int c = cycle; next && c < regsFree; next = next->next) {
2084             bbDelay = MAX2(bbDelay, calcDelay(next, c));
2085             c += getCycles(next, bbDelay);
2086          }
2087          next = NULL;
2088       }
2089    }
2090    if (bb->cfg.outgoingCount() != 1)
2091       next = NULL;
2092    setDelay(insn, bbDelay, next);
2093    cycle += getCycles(insn, bbDelay);
2094
2095    score->rebase(cycle); // common base for initializing out blocks' scores
2096    return true;
2097 }
2098
2099 #define NVE4_MAX_ISSUE_DELAY 0x1f
2100 int
2101 SchedDataCalculator::calcDelay(const Instruction *insn, int cycle) const
2102 {
2103    int delay = 0, ready = cycle;
2104
2105    for (int s = 0; insn->srcExists(s); ++s)
2106       checkRd(insn->getSrc(s), cycle, delay);
2107    // WAR & WAW don't seem to matter
2108    // for (int s = 0; insn->srcExists(s); ++s)
2109    //   recordRd(insn->getSrc(s), cycle);
2110
2111    switch (Target::getOpClass(insn->op)) {
2112    case OPCLASS_SFU:
2113       ready = score->res.sfu;
2114       break;
2115    case OPCLASS_ARITH:
2116       if (insn->op == OP_MUL && !isFloatType(insn->dType))
2117          ready = score->res.imul;
2118       break;
2119    case OPCLASS_TEXTURE:
2120       ready = score->res.tex;
2121       break;
2122    case OPCLASS_LOAD:
2123       ready = score->res.ld[insn->src(0).getFile()];
2124       break;
2125    case OPCLASS_STORE:
2126       ready = score->res.st[insn->src(0).getFile()];
2127       break;
2128    default:
2129       break;
2130    }
2131    if (Target::getOpClass(insn->op) != OPCLASS_TEXTURE)
2132       ready = MAX2(ready, score->res.tex);
2133
2134    delay = MAX2(delay, ready - cycle);
2135
2136    // if can issue next cycle, delay is 0, not 1
2137    return MIN2(delay - 1, NVE4_MAX_ISSUE_DELAY);
2138 }
2139
2140 void
2141 SchedDataCalculator::commitInsn(const Instruction *insn, int cycle)
2142 {
2143    const int ready = cycle + targ->getLatency(insn);
2144
2145    for (int d = 0; insn->defExists(d); ++d)
2146       recordWr(insn->getDef(d), ready);
2147    // WAR & WAW don't seem to matter
2148    // for (int s = 0; insn->srcExists(s); ++s)
2149    //   recordRd(insn->getSrc(s), cycle);
2150
2151    switch (Target::getOpClass(insn->op)) {
2152    case OPCLASS_SFU:
2153       score->res.sfu = cycle + 4;
2154       break;
2155    case OPCLASS_ARITH:
2156       if (insn->op == OP_MUL && !isFloatType(insn->dType))
2157          score->res.imul = cycle + 4;
2158       break;
2159    case OPCLASS_TEXTURE:
2160       score->res.tex = cycle + 18;
2161       break;
2162    case OPCLASS_LOAD:
2163       if (insn->src(0).getFile() == FILE_MEMORY_CONST)
2164          break;
2165       score->res.ld[insn->src(0).getFile()] = cycle + 4;
2166       score->res.st[insn->src(0).getFile()] = ready;
2167       break;
2168    case OPCLASS_STORE:
2169       score->res.st[insn->src(0).getFile()] = cycle + 4;
2170       score->res.ld[insn->src(0).getFile()] = ready;
2171       break;
2172    case OPCLASS_OTHER:
2173       if (insn->op == OP_TEXBAR)
2174          score->res.tex = cycle;
2175       break;
2176    default:
2177       break;
2178    }
2179
2180 #ifdef NVC0_DEBUG_SCHED_DATA
2181    score->print(cycle);
2182 #endif
2183 }
2184
2185 void
2186 SchedDataCalculator::checkRd(const Value *v, int cycle, int& delay) const
2187 {
2188    int ready = cycle;
2189    int a, b;
2190
2191    switch (v->reg.file) {
2192    case FILE_GPR:
2193       a = v->reg.data.id;
2194       b = a + v->reg.size / 4;
2195       for (int r = a; r < b; ++r)
2196          ready = MAX2(ready, score->rd.r[r]);
2197       break;
2198    case FILE_PREDICATE:
2199       ready = MAX2(ready, score->rd.p[v->reg.data.id]);
2200       break;
2201    case FILE_FLAGS:
2202       ready = MAX2(ready, score->rd.c);
2203       break;
2204    case FILE_SHADER_INPUT:
2205    case FILE_SHADER_OUTPUT: // yes, TCPs can read outputs
2206    case FILE_MEMORY_LOCAL:
2207    case FILE_MEMORY_CONST:
2208    case FILE_MEMORY_SHARED:
2209    case FILE_MEMORY_GLOBAL:
2210    case FILE_SYSTEM_VALUE:
2211       // TODO: any restrictions here ?
2212       break;
2213    case FILE_IMMEDIATE:
2214       break;
2215    default:
2216       assert(0);
2217       break;
2218    }
2219    if (cycle < ready)
2220       delay = MAX2(delay, ready - cycle);
2221 }
2222
2223 void
2224 SchedDataCalculator::checkWr(const Value *v, int cycle, int& delay) const
2225 {
2226    int ready = cycle;
2227    int a, b;
2228
2229    switch (v->reg.file) {
2230    case FILE_GPR:
2231       a = v->reg.data.id;
2232       b = a + v->reg.size / 4;
2233       for (int r = a; r < b; ++r)
2234          ready = MAX2(ready, score->wr.r[r]);
2235       break;
2236    case FILE_PREDICATE:
2237       ready = MAX2(ready, score->wr.p[v->reg.data.id]);
2238       break;
2239    default:
2240       assert(v->reg.file == FILE_FLAGS);
2241       ready = MAX2(ready, score->wr.c);
2242       break;
2243    }
2244    if (cycle < ready)
2245       delay = MAX2(delay, ready - cycle);
2246 }
2247
2248 void
2249 SchedDataCalculator::recordWr(const Value *v, const int ready)
2250 {
2251    int a = v->reg.data.id;
2252
2253    if (v->reg.file == FILE_GPR) {
2254       int b = a + v->reg.size / 4;
2255       for (int r = a; r < b; ++r)
2256          score->rd.r[r] = ready;
2257    } else
2258    // $c, $pX: shorter issue-to-read delay (at least as exec pred and carry)
2259    if (v->reg.file == FILE_PREDICATE) {
2260       score->rd.p[a] = ready + 4;
2261    } else {
2262       assert(v->reg.file == FILE_FLAGS);
2263       score->rd.c = ready + 4;
2264    }
2265 }
2266
2267 void
2268 SchedDataCalculator::recordRd(const Value *v, const int ready)
2269 {
2270    int a = v->reg.data.id;
2271
2272    if (v->reg.file == FILE_GPR) {
2273       int b = a + v->reg.size / 4;
2274       for (int r = a; r < b; ++r)
2275          score->wr.r[r] = ready;
2276    } else
2277    if (v->reg.file == FILE_PREDICATE) {
2278       score->wr.p[a] = ready;
2279    } else
2280    if (v->reg.file == FILE_FLAGS) {
2281       score->wr.c = ready;
2282    }
2283 }
2284
2285 void
2286 CodeEmitterNVC0::prepareEmission(Function *func)
2287 {
2288    const Target *targ = func->getProgram()->getTarget();
2289
2290    CodeEmitter::prepareEmission(func);
2291
2292    if (targ->hasSWSched) {
2293       SchedDataCalculator sched(targ);
2294       sched.run(func, true, true);
2295    }
2296 }
2297
2298 CodeEmitterNVC0::CodeEmitterNVC0(const TargetNVC0 *target)
2299    : CodeEmitter(target),
2300      writeIssueDelays(target->hasSWSched)
2301 {
2302    code = NULL;
2303    codeSize = codeSizeLimit = 0;
2304    relocInfo = NULL;
2305 }
2306
2307 CodeEmitter *
2308 TargetNVC0::getCodeEmitter(Program::Type type)
2309 {
2310    CodeEmitterNVC0 *emit = new CodeEmitterNVC0(this);
2311    emit->setProgramType(type);
2312    return emit;
2313 }
2314
2315 } // namespace nv50_ir