src/gallium/drivers/nv50/codegen/nv50_ir_emit_nv50.cpp

   1
   2 #include "nv50_ir.h"
   3 #include "nv50_ir_target.h"
   4
   5 namespace nv50_ir {
   6
   7 class CodeEmitterNV50 : public CodeEmitter
   8 {
   9 public:
  10    CodeEmitterNV50(const Target *);
  11
  12    virtual bool emitInstruction(Instruction *);
  13
  14    virtual uint32_t getMinEncodingSize(const Instruction *) const;
  15
  16    inline void setProgramType(Program::Type pType) { progType = pType; }
  17
  18 private:
  19    const Target *targ;
  20
  21    Program::Type progType;
  22
  23 private:
  24    inline void defId(const ValueDef&, const int pos);
  25    inline void srcId(const ValueRef&, const int pos);
  26    inline void srcId(const ValueRef *, const int pos);
  27
  28    inline void srcAddr16(const ValueRef&, const int pos);
  29    inline void srcAddr8(const ValueRef&, const int pos);
  30
  31    void emitFlagsRd(const Instruction *);
  32    void emitFlagsWr(const Instruction *);
  33
  34    void emitCondCode(CondCode cc, int pos);
  35
  36    inline void setARegBits(unsigned int);
  37
  38    void setAReg16(const Instruction *, int s);
  39    void setImmediate(const Instruction *, int s);
  40
  41    void setDst(const Value *);
  42    void setDst(const Instruction *, int d);
  43    void emitSrc0(const ValueRef&);
  44    void emitSrc1(const ValueRef&);
  45    void emitSrc2(const ValueRef&);
  46
  47    void emitForm_MAD(const Instruction *);
  48    void emitForm_ADD(const Instruction *);
  49    void emitForm_MUL(const Instruction *);
  50    void emitForm_IMM(const Instruction *);
  51
  52    void emitLoadStoreSize(DataType ty, int pos);
  53
  54    void roundMode_MAD(const Instruction *);
  55    void roundMode_CVT(RoundMode);
  56
  57    void emitMNeg12(const Instruction *);
  58
  59    void emitLOAD(const Instruction *);
  60    void emitSTORE(const Instruction *);
  61    void emitMOV(const Instruction *);
  62    void emitNOP();
  63    void emitINTERP(const Instruction *);
  64    void emitPFETCH(const Instruction *);
  65    void emitOUT(const Instruction *);
  66
  67    void emitUADD(const Instruction *);
  68    void emitAADD(const Instruction *);
  69    void emitFADD(const Instruction *);
  70    void emitUMUL(const Instruction *);
  71    void emitFMUL(const Instruction *);
  72    void emitFMAD(const Instruction *);
  73
  74    void emitMINMAX(const Instruction *);
  75
  76    void emitPreOp(const Instruction *);
  77    void emitSFnOp(const Instruction *, uint8_t subOp);
  78
  79    void emitShift(const Instruction *);
  80    void emitARL(const Instruction *);
  81    void emitLogicOp(const Instruction *);
  82
  83    void emitCVT(const Instruction *);
  84    void emitSET(const Instruction *);
  85
  86    void emitTEX(const TexInstruction *);
  87
  88    void emitQUADOP(const Instruction *, uint8_t lane, uint8_t quOp);
  89
  90    void emitFlow(const Instruction *, uint8_t flowOp);
  91 };
  92
  93 #define SDATA(a) ((a).rep()->reg.data)
  94 #define DDATA(a) ((a).rep()->reg.data)
  95
  96 void CodeEmitterNV50::srcId(const ValueRef& src, const int pos)
  97 {
  98    assert(src.get());
  99    code[pos / 32] |= SDATA(src).id << (pos % 32);
 100 }
 101
 102 void CodeEmitterNV50::srcId(const ValueRef *src, const int pos)
 103 {
 104    assert(src->get());
 105    code[pos / 32] |= SDATA(*src).id << (pos % 32);
 106 }
 107
 108 void CodeEmitterNV50::srcAddr16(const ValueRef& src, const int pos)
 109 {
 110    assert(src.get());
 111
 112    uint32_t offset = SDATA(src).offset;
 113
 114    assert(offset <= 0xffff && (pos % 32) <= 16);
 115
 116    code[pos / 32] |= offset << (pos % 32);
 117 }
 118
 119 void CodeEmitterNV50::srcAddr8(const ValueRef& src, const int pos)
 120 {
 121    assert(src.get());
 122
 123    uint32_t offset = SDATA(src).offset;
 124
 125    assert(offset <= 0x1fc && !(offset & 0x3));
 126
 127    code[pos / 32] |= (offset >> 2) << (pos % 32);
 128 }
 129
 130 void CodeEmitterNV50::defId(const ValueDef& def, const int pos)
 131 {
 132    assert(def.get());
 133    code[pos / 32] |= DDATA(def).id << (pos % 32);
 134 }
 135
 136 void
 137 CodeEmitterNV50::roundMode_MAD(const Instruction *insn)
 138 {
 139    switch (insn->rnd) {
 140    case ROUND_M: code[1] |= 1 << 22; break;
 141    case ROUND_P: code[1] |= 2 << 22; break;
 142    case ROUND_Z: code[1] |= 3 << 22; break;
 143    default:
 144       assert(insn->rnd == ROUND_N);
 145       break;
 146    }
 147 }
 148
 149 void
 150 CodeEmitterNV50::emitMNeg12(const Instruction *i)
 151 {
 152    code[1] |= i->src[0].mod.neg() << 26;
 153    code[1] |= i->src[1].mod.neg() << 27;
 154 }
 155
 156 void CodeEmitterNV50::emitCondCode(CondCode cc, int pos)
 157 {
 158    uint8_t enc;
 159
 160    assert(pos >= 32 || pos <= 27);
 161
 162    switch (cc) {
 163    case CC_LT:  enc = 0x1; break;
 164    case CC_LTU: enc = 0x9; break;
 165    case CC_EQ:  enc = 0x2; break;
 166    case CC_EQU: enc = 0xa; break;
 167    case CC_LE:  enc = 0x3; break;
 168    case CC_LEU: enc = 0xb; break;
 169    case CC_GT:  enc = 0x4; break;
 170    case CC_GTU: enc = 0xc; break;
 171    case CC_NE:  enc = 0x5; break;
 172    case CC_NEU: enc = 0xd; break;
 173    case CC_GE:  enc = 0x6; break;
 174    case CC_GEU: enc = 0xe; break;
 175    case CC_TR:  enc = 0xf; break;
 176    case CC_FL:  enc = 0x0; break;
 177
 178    case CC_O:  enc = 0x10; break;
 179    case CC_C:  enc = 0x11; break;
 180    case CC_A:  enc = 0x12; break;
 181    case CC_S:  enc = 0x13; break;
 182    case CC_NS: enc = 0x1c; break;
 183    case CC_NA: enc = 0x1d; break;
 184    case CC_NC: enc = 0x1e; break;
 185    case CC_NO: enc = 0x1f; break;
 186
 187    default:
 188       enc = 0;
 189       assert(!"invalid condition code");
 190       break;
 191    }
 192    code[pos / 32] |= enc << (pos % 32);
 193 }
 194
 195 void
 196 CodeEmitterNV50::emitFlagsRd(const Instruction *i)
 197 {
 198    int s = (i->flagsSrc >= 0) ? i->flagsSrc : i->predSrc;
 199
 200    assert(!(code[1] & 0x00003f80));
 201
 202    if (s >= 0) {
 203       assert(i->getSrc(s)->reg.file == FILE_FLAGS);
 204       emitCondCode(i->cc, 32 + 7);
 205       srcId(i->src[s], 32 + 12);
 206    } else {
 207       code[1] |= 0x0780;
 208    }
 209 }
 210
 211 void
 212 CodeEmitterNV50::emitFlagsWr(const Instruction *i)
 213 {
 214    assert(!(code[1] & 0x70));
 215
 216    if (i->flagsDef >= 0)
 217       code[1] |= (DDATA(i->def[i->flagsDef]).id << 4) | 0x40;
 218 }
 219
 220 void
 221 CodeEmitterNV50::setARegBits(unsigned int u)
 222 {
 223    code[0] |= (u & 3) << 26;
 224    code[1] |= (u & 4);
 225 }
 226
 227 void
 228 CodeEmitterNV50::setAReg16(const Instruction *i, int s)
 229 {
 230    s = i->src[s].indirect[0];
 231    if (s >= 0)
 232       setARegBits(SDATA(i->src[s]).id + 1);
 233 }
 234
 235 void
 236 CodeEmitterNV50::setImmediate(const Instruction *i, int s)
 237 {
 238    const ImmediateValue *imm = i->src[s].get()->asImm();
 239    assert(imm);
 240
 241    code[1] |= 3;
 242    code[0] |= (imm->reg.data.u32 & 0x3f) << 16;
 243    code[1] |= (imm->reg.data.u32 >> 6) << 2;
 244 }
 245
 246 void
 247 CodeEmitterNV50::setDst(const Value *dst)
 248 {
 249    const Storage *reg = &dst->join->reg;
 250
 251    assert(reg->file != FILE_ADDRESS);
 252
 253    if (reg->data.id < 0) {
 254       code[0] |= (127 << 2) | 1;
 255       code[1] |= 8;
 256    } else {
 257       if (reg->file == FILE_SHADER_OUTPUT)
 258          code[1] |= 8;
 259       code[0] |= reg->data.id << 2;
 260    }
 261 }
 262
 263 void
 264 CodeEmitterNV50::setDst(const Instruction *i, int d)
 265 {
 266    if (i->defExists(d)) {
 267       setDst(i->getDef(d));
 268    } else
 269    if (!d) {
 270       code[0] |= 0x01fc; // bit bucket
 271       code[1] |= 0x0008;
 272    }
 273 }
 274
 275 void
 276 CodeEmitterNV50::emitSrc0(const ValueRef& ref)
 277 {
 278    const Storage *reg = &ref.rep()->reg;
 279
 280    if (reg->file == FILE_SHADER_INPUT)
 281       code[1] |= 0x00200000;
 282    else
 283    if (reg->file != FILE_GPR)
 284       ERROR("invalid src0 register file: %d\n", reg->file);
 285
 286    assert(reg->data.id < 128);
 287    code[0] |= reg->data.id << 9;
 288 }
 289
 290 void
 291 CodeEmitterNV50::emitSrc1(const ValueRef& ref)
 292 {
 293    const Storage *reg = &ref.rep()->reg;
 294
 295    if (reg->file == FILE_MEMORY_CONST) {
 296       assert(!(code[1] & 0x01800000));
 297       code[0] |= 1 << 23;
 298       code[1] |= reg->fileIndex << 22;
 299    } else
 300    if (reg->file != FILE_GPR) {
 301       ERROR("invalid src1 register file: %d\n", reg->file);
 302    }
 303
 304    assert(reg->data.id < 128);
 305    code[0] |= reg->data.id << 16;
 306 }
 307
 308 void
 309 CodeEmitterNV50::emitSrc2(const ValueRef& ref)
 310 {
 311    const Storage *reg = &ref.rep()->reg;
 312
 313    if (reg->file == FILE_MEMORY_CONST) {
 314       assert(!(code[1] & 0x01800000));
 315       code[0] |= 1 << 24;
 316       code[1] |= reg->fileIndex << 22;
 317    } else
 318    if (reg->file != FILE_GPR) {
 319       ERROR("invalid src1 register file: %d\n", reg->file);
 320    }
 321
 322    assert(reg->data.id < 128);
 323    code[1] |= reg->data.id << 14;
 324 }
 325
 326 // the default form:
 327 //  - long instruction
 328 //  - 1 to 3 sources in slots 0, 1, 2
 329 //  - address & flags
 330 void
 331 CodeEmitterNV50::emitForm_MAD(const Instruction *i)
 332 {
 333    assert(i->encSize == 8);
 334    code[0] |= 1;
 335
 336    emitFlagsRd(i);
 337    emitFlagsWr(i);
 338
 339    setDst(i, 0);
 340
 341    if (i->srcExists(0))
 342       emitSrc0(i->src[0]);
 343
 344    if (i->srcExists(1))
 345       emitSrc1(i->src[1]);
 346
 347    if (i->srcExists(2))
 348       emitSrc2(i->src[2]);
 349
 350    setAReg16(i, 1);
 351 }
 352
 353 // like default form, but 2nd source in slot 2, and no 3rd source
 354 void
 355 CodeEmitterNV50::emitForm_ADD(const Instruction *i)
 356 {
 357    assert(i->encSize == 8);
 358    code[0] |= 1;
 359
 360    emitFlagsRd(i);
 361    emitFlagsWr(i);
 362
 363    setDst(i, 0);
 364
 365    if (i->srcExists(0))
 366       emitSrc0(i->src[0]);
 367
 368    if (i->srcExists(1))
 369       emitSrc2(i->src[1]);
 370
 371    setAReg16(i, 1);
 372 }
 373
 374 // default short form
 375 void
 376 CodeEmitterNV50::emitForm_MUL(const Instruction *i)
 377 {
 378    assert(i->encSize == 4 && !(code[0] & 1));
 379    assert(i->defExists(0));
 380    assert(!i->getPredicate());
 381
 382    setDst(i, 0);
 383
 384    if (i->srcExists(0))
 385       emitSrc0(i->src[0]);
 386
 387    if (i->srcExists(1))
 388       emitSrc1(i->src[1]);
 389 }
 390
 391 // usual immediate form
 392 // - 1 to 3 sources where last is immediate
 393 // - no address or predicate possible
 394 void
 395 CodeEmitterNV50::emitForm_IMM(const Instruction *i)
 396 {
 397    assert(i->encSize == 8);
 398    code[0] |= 1;
 399
 400    assert(i->defExists(0) && i->srcExists(0));
 401
 402    setDst(i, 0);
 403
 404    if (i->srcExists(2)) {
 405       emitSrc0(i->src[0]);
 406       emitSrc1(i->src[1]);
 407       setImmediate(i, 2);
 408    } else
 409    if (i->srcExists(1)) {
 410       emitSrc0(i->src[0]);
 411       setImmediate(i, 1);
 412    } else {
 413       setImmediate(i, 0);
 414    }
 415 }
 416
 417 void
 418 CodeEmitterNV50::emitLoadStoreSize(DataType ty, int pos)
 419 {
 420    uint8_t enc;
 421
 422    switch (ty) {
 423    case TYPE_F32: // fall through
 424    case TYPE_S32: // fall through
 425    case TYPE_U32:  enc = 0x6; break;
 426    case TYPE_B128: enc = 0x5; break;
 427    case TYPE_F64:  enc = 0x4; break;
 428    case TYPE_S16:  enc = 0x3; break;
 429    case TYPE_U16:  enc = 0x2; break;
 430    case TYPE_S8:   enc = 0x1; break;
 431    case TYPE_U8:   enc = 0x0; break;
 432    default:
 433       enc = 0;
 434       assert(!"invalid load/store type");
 435       break;
 436    }
 437    code[pos / 32] |= enc << (pos % 32);
 438 }
 439
 440 void
 441 CodeEmitterNV50::emitLOAD(const Instruction *i)
 442 {
 443    DataFile sf = i->src[0].getFile();
 444
 445    switch (sf) {
 446    case FILE_SHADER_INPUT:
 447       code[0] = 0x10000001;
 448       code[1] = 0x04200000 | (i->lanes << 14);
 449       break;
 450    case FILE_MEMORY_CONST:
 451       code[0] = 0x10000001;
 452       code[1] = 0x24000000 | (i->getSrc(0)->reg.fileIndex << 22);
 453       break;
 454    case FILE_MEMORY_LOCAL:
 455       code[0] = 0xd0000001;
 456       code[1] = 0x40000000;
 457       break;
 458    case FILE_MEMORY_GLOBAL:
 459       code[0] = 0xd0000001 | (i->getSrc(0)->reg.fileIndex << 16);
 460       code[1] = 0x80000000;
 461       break;
 462    default:
 463       assert(!"invalid load source file");
 464       break;
 465    }
 466    if (sf == FILE_MEMORY_LOCAL ||
 467        sf == FILE_MEMORY_GLOBAL)
 468       emitLoadStoreSize(i->sType, 21 + 32);
 469
 470    setDst(i, 0);
 471
 472    emitFlagsRd(i);
 473    emitFlagsWr(i);
 474
 475    if (i->src[0].getFile() == FILE_MEMORY_GLOBAL) {
 476       srcId(*i->src[0].getIndirect(0), 9);
 477    } else {
 478       setAReg16(i, 0);
 479       srcAddr16(i->src[0], 9);
 480    }
 481 }
 482
 483 void
 484 CodeEmitterNV50::emitSTORE(const Instruction *i)
 485 {
 486    DataFile f = i->getSrc(0)->reg.file;
 487    int32_t offset = i->getSrc(0)->reg.data.offset;
 488
 489    switch (f) {
 490    case FILE_SHADER_OUTPUT:
 491       code[0] = 0x00000001 | ((offset >> 2) << 2);
 492       code[1] = 0x80c00000;
 493       srcId(i->src[1], 32 + 15);
 494       break;
 495    case FILE_MEMORY_GLOBAL:
 496       code[0] = 0xd0000000;
 497       code[1] = 0xa0000000;
 498       emitLoadStoreSize(i->dType, 21 + 32);
 499       break;
 500    case FILE_MEMORY_LOCAL:
 501       code[0] = 0xd0000001;
 502       code[1] = 0x60000000;
 503       emitLoadStoreSize(i->dType, 21 + 32);
 504       break;
 505    case FILE_MEMORY_SHARED:
 506       code[0] = 0x00000001;
 507       code[1] = 0xe0000000;
 508       switch (typeSizeof(i->dType)) {
 509       case 1:
 510          code[0] |= offset << 9;
 511          code[1] |= 0x00400000;
 512          break;
 513       case 2:
 514          code[0] |= (offset >> 1) << 9;
 515          break;
 516       case 4:
 517          code[0] |= (offset >> 2) << 9;
 518          code[1] |= 0x04000000;
 519          break;
 520       default:
 521          assert(0);
 522          break;
 523       }
 524       break;
 525    default:
 526       assert(!"invalid store destination file");
 527       break;
 528    }
 529
 530    if (f != FILE_SHADER_OUTPUT) {
 531       srcId(i->src[1], 2);
 532       if (f == FILE_MEMORY_GLOBAL)
 533          srcId(*i->src[0].getIndirect(0), 9);
 534       if (f == FILE_MEMORY_LOCAL)
 535          srcAddr16(i->src[0], 9);
 536    }
 537    if (f != FILE_MEMORY_GLOBAL)
 538       setAReg16(i, 0);
 539
 540    emitFlagsRd(i);
 541 }
 542
 543 void
 544 CodeEmitterNV50::emitMOV(const Instruction *i)
 545 {
 546    DataFile sf = i->getSrc(0)->reg.file;
 547    DataFile df = i->getDef(0)->reg.file;
 548
 549    assert(sf == FILE_GPR || df == FILE_GPR);
 550
 551    if (sf == FILE_FLAGS) {
 552       code[0] = 0x00000001;
 553       code[1] = 0x20000000;
 554       defId(i->def[0], 2);
 555       srcId(i->src[0], 12);
 556       emitFlagsRd(i);
 557    } else
 558    if (sf == FILE_ADDRESS) {
 559       code[0] = 0x00000001;
 560       code[1] = 0x40000000;
 561       defId(i->def[0], 2);
 562       setARegBits(SDATA(i->src[0]).id + 1);
 563    } else
 564    if (df == FILE_FLAGS) {
 565       code[0] = 0x00000001;
 566       code[1] = 0xa0000000;
 567       defId(i->def[0], 4);
 568       srcId(i->src[0], 9);
 569       emitFlagsRd(i);
 570    } else
 571    if (sf == FILE_IMMEDIATE) {
 572       code[0] = 0x10008001;
 573       code[1] = 0x00000003;
 574       emitForm_IMM(i);
 575    } else {
 576       if (i->encSize == 4) {
 577          code[0] = 0x10008000;
 578       } else {
 579          code[0] = 0x10000001;
 580          code[1] = 0x04000000 | (i->lanes << 14);
 581       }
 582       defId(i->def[0], 2);
 583       srcId(i->src[0], 9);
 584    }
 585    if (df == FILE_SHADER_OUTPUT) {
 586       assert(i->encSize == 8);
 587       code[1] |= 0x8;
 588    }
 589 }
 590
 591 void
 592 CodeEmitterNV50::emitNOP()
 593 {
 594    code[0] = 0xf0000001;
 595    code[1] = 0xe0000000;
 596 }
 597
 598 void
 599 CodeEmitterNV50::emitQUADOP(const Instruction *i, uint8_t lane, uint8_t quOp)
 600 {
 601    code[0] = 0xc0000000 | (lane << 16);
 602    code[1] = 0x80000000;
 603
 604    code[0] |= (quOp & 0x03) << 20;
 605    code[1] |= (quOp & 0xfc) << 20;
 606
 607    emitForm_ADD(i);
 608
 609    if (!i->srcExists(1))
 610       srcId(i->src[0], 32 + 14);
 611 }
 612
 613 void
 614 CodeEmitterNV50::emitPFETCH(const Instruction *i)
 615 {
 616    code[0] = 0x11800001;
 617    code[1] = 0x04200000 | (0xf << 14);
 618
 619    defId(i->def[0], 2);
 620    srcAddr8(i->src[0], 9);
 621    setAReg16(i, 0);
 622 }
 623
 624 void
 625 CodeEmitterNV50::emitINTERP(const Instruction *i)
 626 {
 627    code[0] = 0x80000000;
 628
 629    defId(i->def[0], 2);
 630    srcAddr8(i->src[0], 16);
 631
 632    if (i->getInterpMode() == NV50_IR_INTERP_FLAT) {
 633       code[0] |= 1 << 8;
 634    } else {
 635       if (i->op == OP_PINTERP) {
 636          code[0] |= 1 << 25;
 637          srcId(i->src[1], 9);
 638       }
 639       if (i->getSampleMode() == NV50_IR_INTERP_CENTROID)
 640          code[0] |= 1 << 24;
 641    }
 642
 643    if (i->encSize == 8) {
 644       emitFlagsRd(i);
 645       code[1] |=
 646          (code[0] & (3 << 24)) >> (24 - 16) |
 647          (code[0] & (1 <<  8)) >> (18 -  8);
 648       code[0] &= ~0x03000100;
 649       code[0] |= 1;
 650    }
 651 }
 652
 653 void
 654 CodeEmitterNV50::emitMINMAX(const Instruction *i)
 655 {
 656    if (i->dType == TYPE_F64) {
 657       code[0] = 0xe0000000;
 658       code[1] = (i->op == OP_MIN) ? 0xa0000000 : 0xc0000000;
 659    } else {
 660       code[0] = 0x30000000;
 661       code[1] = 0x80000000;
 662       if (i->op == OP_MIN)
 663          code[1] |= 0x20000000;
 664
 665       switch (i->dType) {
 666       case TYPE_F32: code[0] |= 0x80000000; break;
 667       case TYPE_S32: code[1] |= 0x8c000000; break;
 668       case TYPE_U32: code[1] |= 0x84000000; break;
 669       case TYPE_S16: code[1] |= 0x80000000; break;
 670       case TYPE_U16: break;
 671       default:
 672          assert(0);
 673          break;
 674       }
 675       code[1] |= i->src[0].mod.abs() << 20;
 676       code[1] |= i->src[1].mod.abs() << 19;
 677    }
 678    emitForm_MAD(i);
 679 }
 680
 681 void
 682 CodeEmitterNV50::emitFMAD(const Instruction *i)
 683 {
 684    const int neg_mul = i->src[0].mod.neg() ^ i->src[1].mod.neg();
 685    const int neg_add = i->src[2].mod.neg();
 686
 687    code[0] = 0xe0000000;
 688
 689    if (i->encSize == 4) {
 690       emitForm_MUL(i);
 691       assert(!neg_mul && !neg_add);
 692    } else {
 693       emitForm_MAD(i);
 694       code[1] |= neg_mul << 26;
 695       code[1] |= neg_add << 27;
 696       if (i->saturate)
 697          code[1] |= 1 << 29;
 698    }
 699 }
 700
 701 void
 702 CodeEmitterNV50::emitFADD(const Instruction *i)
 703 {
 704    const int neg0 = i->src[0].mod.neg();
 705    const int neg1 = i->src[1].mod.neg() ^ ((i->op == OP_SUB) ? 1 : 0);
 706
 707    code[0] = 0xb0000000;
 708
 709    assert(!(i->src[0].mod | i->src[1].mod).abs());
 710
 711    if (i->src[1].getFile() == FILE_IMMEDIATE) {
 712       emitForm_IMM(i);
 713       code[0] |= neg0 << 15;
 714       code[0] |= neg1 << 22;
 715    } else
 716    if (i->encSize == 8) {
 717       emitForm_ADD(i);
 718       code[1] |= neg0 << 26;
 719       code[1] |= neg1 << 27;
 720       if (i->saturate)
 721          code[1] |= 1 << 29;
 722    } else {
 723       emitForm_MUL(i);
 724       code[0] |= neg0 << 15;
 725       code[0] |= neg1 << 22;
 726    }
 727 }
 728
 729 void
 730 CodeEmitterNV50::emitUADD(const Instruction *i)
 731 {
 732    code[0] = 0x20008000;
 733
 734    if (i->src[0].getFile() == FILE_IMMEDIATE) {
 735       emitForm_IMM(i);
 736    } else
 737    if (i->encSize == 8) {
 738       code[0] = 0x20000000;
 739       code[1] = 0x04000000;
 740       emitForm_ADD(i);
 741    } else {
 742       emitForm_MUL(i);
 743    }
 744    assert(!(i->src[0].mod.neg() && i->src[1].mod.neg()));
 745    code[0] |= i->src[0].mod.neg() << 28;
 746    code[0] |= i->src[1].mod.neg() << 22;
 747 }
 748
 749 void
 750 CodeEmitterNV50::emitAADD(const Instruction *i)
 751 {
 752    const int s = (i->op == OP_MOV) ? 0 : 1;
 753
 754    code[0] = 0xd0000001 | (i->getSrc(s)->reg.data.u16 << 9);
 755    code[1] = 0x20000000;
 756
 757    code[0] |= (DDATA(i->def[0]).id + 1) << 2;
 758
 759    emitFlagsRd(i);
 760
 761    if (s && i->srcExists(0))
 762       setARegBits(SDATA(i->src[0]).id + 1);
 763 }
 764
 765 void
 766 CodeEmitterNV50::emitFMUL(const Instruction *i)
 767 {
 768    const int neg = (i->src[0].mod ^ i->src[1].mod).neg();
 769
 770    code[0] = 0xc0000000;
 771
 772    if (i->src[0].getFile() == FILE_IMMEDIATE) {
 773       emitForm_IMM(i);
 774       if (neg)
 775          code[0] |= 0x8000;
 776    } else
 777    if (i->encSize == 8) {
 778       emitForm_MAD(i);
 779       if (neg)
 780          code[1] |= 0x08000000;
 781    } else {
 782       emitForm_MUL(i);
 783       if (neg)
 784          code[0] |= 0x8000;
 785    }
 786 }
 787
 788 void
 789 CodeEmitterNV50::emitSET(const Instruction *i)
 790 {
 791    code[0] = 0x30000000;
 792    code[1] = 0x60000000;
 793
 794    emitCondCode(i->asCmp()->setCond, 32 + 14);
 795
 796    switch (i->sType) {
 797    case TYPE_F32: code[0] |= 0x80000000; break;
 798    case TYPE_S32: code[1] |= 0x0c000000; break;
 799    case TYPE_U32: code[1] |= 0x04000000; break;
 800    case TYPE_S16: code[1] |= 0x08000000; break;
 801    case TYPE_U16: break;
 802    default:
 803       assert(0);
 804       break;
 805    }
 806    emitForm_MAD(i);
 807 }
 808
 809 void
 810 CodeEmitterNV50::roundMode_CVT(RoundMode rnd)
 811 {
 812    switch (rnd) {
 813    case ROUND_NI: code[1] |= 0x08000000; break;
 814    case ROUND_M:  code[1] |= 0x00020000; break;
 815    case ROUND_MI: code[1] |= 0x08020000; break;
 816    case ROUND_P:  code[1] |= 0x00040000; break;
 817    case ROUND_PI: code[1] |= 0x08040000; break;
 818    case ROUND_Z:  code[1] |= 0x00060000; break;
 819    case ROUND_ZI: code[1] |= 0x08060000; break;
 820    default:
 821       assert(rnd == ROUND_N);
 822       break;
 823    }
 824 }
 825
 826 void
 827 CodeEmitterNV50::emitCVT(const Instruction *i)
 828 {
 829    const bool f2f = isFloatType(i->dType) && isFloatType(i->sType);
 830    RoundMode rnd;
 831
 832    switch (i->op) {
 833    case OP_CEIL:  rnd = f2f ? ROUND_PI : ROUND_P; break;
 834    case OP_FLOOR: rnd = f2f ? ROUND_MI : ROUND_M; break;
 835    case OP_TRUNC: rnd = f2f ? ROUND_ZI : ROUND_Z; break;
 836    default:
 837       rnd = i->rnd;
 838       break;
 839    }
 840
 841    code[0] = 0xa0000000;
 842
 843    switch (i->dType) {
 844    case TYPE_F64:
 845       switch (i->sType) {
 846       case TYPE_F64: code[1] = 0xc4404000; break;
 847       case TYPE_S64: code[1] = 0x44414000; break;
 848       case TYPE_U64: code[1] = 0x44404000; break;
 849       case TYPE_F32: code[1] = 0xc4400000; break;
 850       case TYPE_S32: code[1] = 0x44410000; break;
 851       case TYPE_U32: code[1] = 0x44400000; break;
 852       default:
 853          assert(0);
 854          break;
 855       }
 856       break;
 857    case TYPE_S64:
 858       switch (i->sType) {
 859       case TYPE_F64: code[1] = 0x8c404000; break;
 860       case TYPE_F32: code[1] = 0x8c400000; break;
 861       default:
 862          assert(0);
 863          break;
 864       }
 865       break;
 866    case TYPE_U64:
 867       switch (i->sType) {
 868       case TYPE_F64: code[1] = 0x84404000; break;
 869       case TYPE_F32: code[1] = 0x84400000; break;
 870       default:
 871          assert(0);
 872          break;
 873       }
 874       break;
 875    case TYPE_F32:
 876       switch (i->sType) {
 877       case TYPE_F64: code[1] = 0xc0404000; break;
 878       case TYPE_S64: code[1] = 0x40414000; break;
 879       case TYPE_U64: code[1] = 0x40404000; break;
 880       case TYPE_F32: code[1] = 0xc4004000; break;
 881       case TYPE_S32: code[1] = 0x44014000; break;
 882       case TYPE_U32: code[1] = 0x44004000; break;
 883       case TYPE_F16: code[1] = 0xc4000000; break;
 884       default:
 885          assert(0);
 886          break;
 887       }
 888       break;
 889    case TYPE_S32:
 890       switch (i->sType) {
 891       case TYPE_F64: code[1] = 0x88404000; break;
 892       case TYPE_F32: code[1] = 0x8c004000; break;
 893       case TYPE_S32: code[1] = 0x0c014000; break;
 894       case TYPE_U32: code[1] = 0x0c004000; break;
 895       case TYPE_F16: code[1] = 0x8c000000; break;
 896       case TYPE_S16: code[1] = 0x0c010000; break;
 897       case TYPE_U16: code[1] = 0x0c000000; break;
 898       case TYPE_S8:  code[1] = 0x0c018000; break;
 899       case TYPE_U8:  code[1] = 0x0c008000; break;
 900       default:
 901          assert(0);
 902          break;
 903       }
 904       break;
 905    case TYPE_U32:
 906       switch (i->sType) {
 907       case TYPE_F64: code[1] = 0x80404000; break;
 908       case TYPE_F32: code[1] = 0x84004000; break;
 909       case TYPE_S32: code[1] = 0x04014000; break;
 910       case TYPE_U32: code[1] = 0x04004000; break;
 911       case TYPE_F16: code[1] = 0x84000000; break;
 912       case TYPE_S16: code[1] = 0x04010000; break;
 913       case TYPE_U16: code[1] = 0x04000000; break;
 914       case TYPE_S8:  code[1] = 0x04018000; break;
 915       case TYPE_U8:  code[1] = 0x04008000; break;
 916       default:
 917          assert(0);
 918          break;
 919       }
 920    case TYPE_S16:
 921    case TYPE_U16:
 922    case TYPE_S8:
 923    case TYPE_U8:
 924    default:
 925       assert(0);
 926       break;
 927    }
 928    if (typeSizeof(i->sType) == 1 && i->getSrc(0)->reg.size == 4)
 929       code[1] |= 0x00004000;
 930
 931    roundMode_CVT(rnd);
 932
 933    switch (i->op) {
 934    case OP_ABS: code[1] |= 1 << 20; break;
 935    case OP_SAT: code[1] |= 1 << 19; break;
 936    case OP_NEG: code[1] |= 1 << 29; break;
 937    default:
 938       break;
 939    }
 940    code[1] ^= i->src[0].mod.neg() << 29;
 941    code[1] |= i->src[0].mod.abs() << 20;
 942    if (i->saturate)
 943       code[1] |= 1 << 19;
 944
 945    assert(i->op != OP_ABS || !i->src[0].mod.neg());
 946
 947    emitForm_MAD(i);
 948 }
 949
 950 void
 951 CodeEmitterNV50::emitPreOp(const Instruction *i)
 952 {
 953    code[0] = 0xb0000000;
 954    code[1] = (i->op == OP_PREEX2) ? 0xc0004000 : 0xc0000000;
 955
 956    code[1] |= i->src[0].mod.abs() << 20;
 957    code[1] |= i->src[0].mod.neg() << 26;
 958
 959    emitForm_MAD(i);
 960 }
 961
 962 void
 963 CodeEmitterNV50::emitSFnOp(const Instruction *i, uint8_t subOp)
 964 {
 965    code[0] = 0x90000000;
 966
 967    if (i->encSize == 4) {
 968       assert(i->op == OP_RCP);
 969       emitForm_MUL(i);
 970    } else {
 971       code[1] = subOp << 29;
 972       code[1] |= i->src[0].mod.abs() << 20;
 973       code[1] |= i->src[0].mod.neg() << 26;
 974       emitForm_MAD(i);
 975    }
 976 }
 977
 978 void
 979 CodeEmitterNV50::emitLogicOp(const Instruction *i)
 980 {
 981    code[0] = 0xd0000000;
 982
 983    if (i->src[1].getFile() == FILE_IMMEDIATE) {
 984       switch (i->op) {
 985       case OP_OR:  code[0] |= 0x0100; break;
 986       case OP_XOR: code[0] |= 0x8000; break;
 987       default:
 988          assert(i->op == OP_AND);
 989          break;
 990       }
 991       emitForm_IMM(i);
 992    } else {
 993       switch (i->op) {
 994       case OP_AND: code[1] = 0x04000000; break;
 995       case OP_OR:  code[1] = 0x04004000; break;
 996       case OP_XOR: code[1] = 0x04008000; break;
 997       default:
 998          assert(0);
 999          break;
1000       }
1001       emitForm_MAD(i);
1002    }
1003 }
1004
1005 void
1006 CodeEmitterNV50::emitARL(const Instruction *i)
1007 {
1008    assert(i->src[1].getFile() == FILE_IMMEDIATE);
1009
1010    code[0] = 0x00000001 | (i->getSrc(1)->reg.data.u32 & 0x3f) << 16;
1011    code[1] = 0xc0000000;
1012
1013    code[0] |= (DDATA(i->def[0]).id + 1) << 2;
1014    emitSrc0(i->src[0]);
1015    emitFlagsRd(i);
1016 }
1017
1018 void
1019 CodeEmitterNV50::emitShift(const Instruction *i)
1020 {
1021    if (i->def[0].getFile() == FILE_ADDRESS) {
1022       emitARL(i);
1023    } else {
1024       code[0] = 0x30000001;
1025       code[1] = (i->op == OP_SHR) ? 0xe4000000 : 0xc4000000;
1026       if (isSignedType(i->sType))
1027           code[1] |= 1 << 27;
1028
1029       if (i->src[1].getFile() == FILE_IMMEDIATE) {
1030          code[1] |= 1 << 20;
1031          code[0] |= (i->getSrc(1)->reg.data.u32 & 0x7f) << 16;
1032          emitFlagsRd(i);
1033       } else {
1034          emitForm_MAD(i);
1035       }
1036    }
1037 }
1038
1039 void
1040 CodeEmitterNV50::emitOUT(const Instruction *i)
1041 {
1042    code[0] = (i->op == OP_EMIT) ? 0xf0000200 : 0xf0000400;
1043    code[1] = 0xc0000001;
1044
1045    emitFlagsRd(i);
1046 }
1047
1048 void
1049 CodeEmitterNV50::emitTEX(const TexInstruction *i)
1050 {
1051    code[0] = 0xf0000001;
1052    code[1] = 0x00000000;
1053
1054    switch (i->op) {
1055    case OP_TXB:
1056       code[1] = 0x20000000;
1057       break;
1058    case OP_TXL:
1059       code[1] = 0x40000000;
1060       break;
1061    case OP_TXF:
1062       code[0] = 0x01000000;
1063       break;
1064    case OP_TXG:
1065       code[0] = 0x01000000;
1066       code[1] = 0x80000000;
1067       break;
1068    default:
1069       assert(i->op == OP_TEX);
1070       break;
1071    }
1072
1073    code[0] |= i->tex.r << 9;
1074    code[0] |= i->tex.s << 17;
1075
1076    int argc = i->tex.target.getArgCount();
1077
1078    if (i->op == OP_TXB || i->op == OP_TXL)
1079       argc += 1;
1080    if (i->tex.target.isShadow())
1081       argc += 1;
1082    assert(argc <= 4);
1083
1084    code[0] |= (argc - 1) << 22;
1085
1086    if (i->tex.target.isCube()) {
1087       code[0] |= 0x08000000;
1088    } else
1089    if (i->tex.useOffsets) {
1090       code[1] |= (i->tex.offset[0][0] & 0xf) << 16;
1091       code[1] |= (i->tex.offset[0][1] & 0xf) << 20;
1092       code[1] |= (i->tex.offset[0][2] & 0xf) << 24;
1093    }
1094
1095    code[0] |= (i->tex.mask & 0x3) << 25;
1096    code[1] |= (i->tex.mask & 0xc) << 12;
1097
1098    if (i->tex.liveOnly)
1099       code[1] |= 4;
1100
1101    defId(i->def[0], 2);
1102
1103    emitFlagsRd(i);
1104 }
1105
1106 void
1107 CodeEmitterNV50::emitFlow(const Instruction *i, uint8_t flowOp)
1108 {
1109    const FlowInstruction *f = i->asFlow();
1110
1111    code[0] = 0x00000003 | (flowOp << 28);
1112    code[1] = 0x00000000;
1113
1114    emitFlagsRd(i);
1115
1116    if (f && f->target.bb) {
1117       uint32_t pos;
1118
1119       if (f->op == OP_CALL) {
1120          if (f->builtin) {
1121             pos = 0; // XXX: TODO
1122          } else {
1123             pos = f->target.fn->binPos;
1124          }
1125       } else {
1126          pos = f->target.bb->binPos;
1127       }
1128
1129       code[0] |= ((pos >>  2) & 0xffff) << 11;
1130       code[1] |= ((pos >> 18) & 0x003f) << 14;
1131    }
1132 }
1133
1134 bool
1135 CodeEmitterNV50::emitInstruction(Instruction *insn)
1136 {
1137    if (!insn->encSize) {
1138       ERROR("skipping unencodable instruction: "); insn->print();
1139       return false;
1140    } else
1141    if (codeSize + insn->encSize > codeSizeLimit) {
1142       ERROR("code emitter output buffer too small\n");
1143       return false;
1144    }
1145
1146    switch (insn->op) {
1147    case OP_MOV:
1148       emitMOV(insn);
1149       break;
1150    case OP_NOP:
1151    case OP_JOIN:
1152       emitNOP();
1153       break;
1154    case OP_VFETCH:
1155    case OP_LOAD:
1156       emitLOAD(insn);
1157       break;
1158    case OP_EXPORT:
1159    case OP_STORE:
1160       emitSTORE(insn);
1161       break;
1162    case OP_PFETCH:
1163       emitPFETCH(insn);
1164       break;
1165    case OP_LINTERP:
1166    case OP_PINTERP:
1167       emitINTERP(insn);
1168       break;
1169    case OP_ADD:
1170    case OP_SUB:
1171       if (isFloatType(insn->dType))
1172          emitFADD(insn);
1173       else
1174          emitUADD(insn);
1175       break;
1176    case OP_MUL:
1177       if (isFloatType(insn->dType))
1178          emitFMUL(insn);
1179       else
1180          emitUMUL(insn);
1181       break;
1182    case OP_MAD:
1183    case OP_FMA:
1184       emitFMAD(insn);
1185       break;
1186       break;
1187    case OP_AND:
1188    case OP_OR:
1189    case OP_XOR:
1190       emitLogicOp(insn);
1191       break;
1192    case OP_MIN:
1193    case OP_MAX:
1194       emitMINMAX(insn);
1195       break;
1196    case OP_CEIL:
1197    case OP_FLOOR:
1198    case OP_TRUNC:
1199    case OP_CVT:
1200       emitCVT(insn);
1201       break;
1202    case OP_RCP:
1203       emitSFnOp(insn, 0);
1204       break;
1205    case OP_RSQ:
1206       emitSFnOp(insn, 2);
1207       break;
1208    case OP_LG2:
1209       emitSFnOp(insn, 3);
1210       break;
1211    case OP_SIN:
1212       emitSFnOp(insn, 4);
1213       break;
1214    case OP_COS:
1215       emitSFnOp(insn, 5);
1216       break;
1217    case OP_EX2:
1218       emitSFnOp(insn, 6);
1219       break;
1220    case OP_PRESIN:
1221    case OP_PREEX2:
1222       emitPreOp(insn);
1223       break;
1224    case OP_TEX:
1225    case OP_TXB:
1226    case OP_TXL:
1227       emitTEX(insn->asTex());
1228       break;
1229    case OP_EMIT:
1230    case OP_RESTART:
1231       emitOUT(insn);
1232       break;
1233    case OP_DISCARD:
1234       emitFlow(insn, 0x0);
1235       break;
1236    case OP_BRA:
1237       emitFlow(insn, 0x1);
1238       break;
1239    case OP_CALL:
1240       emitFlow(insn, 0x2);
1241       break;
1242    case OP_RET:
1243       emitFlow(insn, 0x3);
1244       break;
1245    case OP_PREBREAK:
1246       emitFlow(insn, 0x4);
1247       break;
1248    case OP_BREAK:
1249       emitFlow(insn, 0x5);
1250       break;
1251    case OP_QUADON:
1252       emitFlow(insn, 0x6);
1253       break;
1254    case OP_QUADPOP:
1255       emitFlow(insn, 0x7);
1256       break;
1257    case OP_JOINAT:
1258       emitFlow(insn, 0xa);
1259       break;
1260    case OP_PRERET:
1261       emitFlow(insn, 0xd);
1262       break;
1263    case OP_QUADOP:
1264       emitQUADOP(insn, insn->lanes, insn->subOp);
1265       break;
1266    case OP_DFDX:
1267       emitQUADOP(insn, 4, insn->src[0].mod.neg() ? 0x66 : 0x99);
1268       break;
1269    case OP_DFDY:
1270       emitQUADOP(insn, 5, insn->src[0].mod.neg() ? 0x5a : 0xa5);
1271       break;
1272    case OP_PHI:
1273    case OP_UNION:
1274    case OP_CONSTRAINT:
1275       ERROR("operation should have been eliminated");
1276       return false;
1277    case OP_EXP:
1278    case OP_LOG:
1279    case OP_SQRT:
1280    case OP_POW:
1281    case OP_SELP:
1282    case OP_SLCT:
1283    case OP_TXD:
1284    case OP_PRECONT:
1285    case OP_CONT:
1286    case OP_POPCNT:
1287    case OP_INSBF:
1288    case OP_EXTBF:
1289       ERROR("operation should have been lowered\n");
1290       return false;
1291    default:
1292       ERROR("unknow op\n");
1293       return false;
1294    }
1295    if (insn->join)
1296       code[1] |= 0x2;
1297    else
1298    if (insn->exit)
1299       code[1] |= 0x1;
1300
1301    assert((insn->encSize == 8) == (code[1] & 1));
1302
1303    code += insn->encSize / 4;
1304    codeSize += insn->encSize;
1305    return true;
1306 }
1307
1308 uint32_t
1309 CodeEmitterNV50::getMinEncodingSize(const Instruction *i) const
1310 {
1311    const Target::OpInfo &info = targ->getOpInfo(i);
1312
1313    if (info.minEncSize == 8)
1314       return 8;
1315
1316    return 4;
1317 }
1318
1319 CodeEmitterNV50::CodeEmitterNV50(const Target *target) : targ(target)
1320 {
1321    code = NULL;
1322    codeSize = codeSizeLimit = 0;
1323 }
1324
1325 CodeEmitter *
1326 Target::getCodeEmitter(Program::Type type)
1327 {
1328    CodeEmitterNV50 *emit = new CodeEmitterNV50(this);
1329    emit->setProgramType(type);
1330    return emit;
1331 }
1332
1333 } // namespace nv50_ir