src/gallium/drivers/nv50/codegen/nv50_ir_emit_nv50.cpp

   1 /*
   2  * Copyright 2011 Christoph Bumiller
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  18  * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  19  * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  20  * SOFTWARE.
  21  */
  22
  23 #include "nv50_ir.h"
  24 #include "nv50_ir_target.h"
  25
  26 namespace nv50_ir {
  27
  28 class CodeEmitterNV50 : public CodeEmitter
  29 {
  30 public:
  31    CodeEmitterNV50(const Target *);
  32
  33    virtual bool emitInstruction(Instruction *);
  34
  35    virtual uint32_t getMinEncodingSize(const Instruction *) const;
  36
  37    inline void setProgramType(Program::Type pType) { progType = pType; }
  38
  39 private:
  40    const Target *targ;
  41
  42    Program::Type progType;
  43
  44 private:
  45    inline void defId(const ValueDef&, const int pos);
  46    inline void srcId(const ValueRef&, const int pos);
  47    inline void srcId(const ValueRef *, const int pos);
  48
  49    inline void srcAddr16(const ValueRef&, const int pos);
  50    inline void srcAddr8(const ValueRef&, const int pos);
  51
  52    void emitFlagsRd(const Instruction *);
  53    void emitFlagsWr(const Instruction *);
  54
  55    void emitCondCode(CondCode cc, int pos);
  56
  57    inline void setARegBits(unsigned int);
  58
  59    void setAReg16(const Instruction *, int s);
  60    void setImmediate(const Instruction *, int s);
  61
  62    void setDst(const Value *);
  63    void setDst(const Instruction *, int d);
  64    void emitSrc0(const ValueRef&);
  65    void emitSrc1(const ValueRef&);
  66    void emitSrc2(const ValueRef&);
  67
  68    void emitForm_MAD(const Instruction *);
  69    void emitForm_ADD(const Instruction *);
  70    void emitForm_MUL(const Instruction *);
  71    void emitForm_IMM(const Instruction *);
  72
  73    void emitLoadStoreSize(DataType ty, int pos);
  74
  75    void roundMode_MAD(const Instruction *);
  76    void roundMode_CVT(RoundMode);
  77
  78    void emitMNeg12(const Instruction *);
  79
  80    void emitLOAD(const Instruction *);
  81    void emitSTORE(const Instruction *);
  82    void emitMOV(const Instruction *);
  83    void emitNOP();
  84    void emitINTERP(const Instruction *);
  85    void emitPFETCH(const Instruction *);
  86    void emitOUT(const Instruction *);
  87
  88    void emitUADD(const Instruction *);
  89    void emitAADD(const Instruction *);
  90    void emitFADD(const Instruction *);
  91    void emitUMUL(const Instruction *);
  92    void emitFMUL(const Instruction *);
  93    void emitFMAD(const Instruction *);
  94
  95    void emitMINMAX(const Instruction *);
  96
  97    void emitPreOp(const Instruction *);
  98    void emitSFnOp(const Instruction *, uint8_t subOp);
  99
 100    void emitShift(const Instruction *);
 101    void emitARL(const Instruction *);
 102    void emitLogicOp(const Instruction *);
 103
 104    void emitCVT(const Instruction *);
 105    void emitSET(const Instruction *);
 106
 107    void emitTEX(const TexInstruction *);
 108
 109    void emitQUADOP(const Instruction *, uint8_t lane, uint8_t quOp);
 110
 111    void emitFlow(const Instruction *, uint8_t flowOp);
 112 };
 113
 114 #define SDATA(a) ((a).rep()->reg.data)
 115 #define DDATA(a) ((a).rep()->reg.data)
 116
 117 void CodeEmitterNV50::srcId(const ValueRef& src, const int pos)
 118 {
 119    assert(src.get());
 120    code[pos / 32] |= SDATA(src).id << (pos % 32);
 121 }
 122
 123 void CodeEmitterNV50::srcId(const ValueRef *src, const int pos)
 124 {
 125    assert(src->get());
 126    code[pos / 32] |= SDATA(*src).id << (pos % 32);
 127 }
 128
 129 void CodeEmitterNV50::srcAddr16(const ValueRef& src, const int pos)
 130 {
 131    assert(src.get());
 132
 133    uint32_t offset = SDATA(src).offset;
 134
 135    assert(offset <= 0xffff && (pos % 32) <= 16);
 136
 137    code[pos / 32] |= offset << (pos % 32);
 138 }
 139
 140 void CodeEmitterNV50::srcAddr8(const ValueRef& src, const int pos)
 141 {
 142    assert(src.get());
 143
 144    uint32_t offset = SDATA(src).offset;
 145
 146    assert(offset <= 0x1fc && !(offset & 0x3));
 147
 148    code[pos / 32] |= (offset >> 2) << (pos % 32);
 149 }
 150
 151 void CodeEmitterNV50::defId(const ValueDef& def, const int pos)
 152 {
 153    assert(def.get());
 154    code[pos / 32] |= DDATA(def).id << (pos % 32);
 155 }
 156
 157 void
 158 CodeEmitterNV50::roundMode_MAD(const Instruction *insn)
 159 {
 160    switch (insn->rnd) {
 161    case ROUND_M: code[1] |= 1 << 22; break;
 162    case ROUND_P: code[1] |= 2 << 22; break;
 163    case ROUND_Z: code[1] |= 3 << 22; break;
 164    default:
 165       assert(insn->rnd == ROUND_N);
 166       break;
 167    }
 168 }
 169
 170 void
 171 CodeEmitterNV50::emitMNeg12(const Instruction *i)
 172 {
 173    code[1] |= i->src[0].mod.neg() << 26;
 174    code[1] |= i->src[1].mod.neg() << 27;
 175 }
 176
 177 void CodeEmitterNV50::emitCondCode(CondCode cc, int pos)
 178 {
 179    uint8_t enc;
 180
 181    assert(pos >= 32 || pos <= 27);
 182
 183    switch (cc) {
 184    case CC_LT:  enc = 0x1; break;
 185    case CC_LTU: enc = 0x9; break;
 186    case CC_EQ:  enc = 0x2; break;
 187    case CC_EQU: enc = 0xa; break;
 188    case CC_LE:  enc = 0x3; break;
 189    case CC_LEU: enc = 0xb; break;
 190    case CC_GT:  enc = 0x4; break;
 191    case CC_GTU: enc = 0xc; break;
 192    case CC_NE:  enc = 0x5; break;
 193    case CC_NEU: enc = 0xd; break;
 194    case CC_GE:  enc = 0x6; break;
 195    case CC_GEU: enc = 0xe; break;
 196    case CC_TR:  enc = 0xf; break;
 197    case CC_FL:  enc = 0x0; break;
 198
 199    case CC_O:  enc = 0x10; break;
 200    case CC_C:  enc = 0x11; break;
 201    case CC_A:  enc = 0x12; break;
 202    case CC_S:  enc = 0x13; break;
 203    case CC_NS: enc = 0x1c; break;
 204    case CC_NA: enc = 0x1d; break;
 205    case CC_NC: enc = 0x1e; break;
 206    case CC_NO: enc = 0x1f; break;
 207
 208    default:
 209       enc = 0;
 210       assert(!"invalid condition code");
 211       break;
 212    }
 213    code[pos / 32] |= enc << (pos % 32);
 214 }
 215
 216 void
 217 CodeEmitterNV50::emitFlagsRd(const Instruction *i)
 218 {
 219    int s = (i->flagsSrc >= 0) ? i->flagsSrc : i->predSrc;
 220
 221    assert(!(code[1] & 0x00003f80));
 222
 223    if (s >= 0) {
 224       assert(i->getSrc(s)->reg.file == FILE_FLAGS);
 225       emitCondCode(i->cc, 32 + 7);
 226       srcId(i->src[s], 32 + 12);
 227    } else {
 228       code[1] |= 0x0780;
 229    }
 230 }
 231
 232 void
 233 CodeEmitterNV50::emitFlagsWr(const Instruction *i)
 234 {
 235    assert(!(code[1] & 0x70));
 236
 237    if (i->flagsDef >= 0)
 238       code[1] |= (DDATA(i->def[i->flagsDef]).id << 4) | 0x40;
 239 }
 240
 241 void
 242 CodeEmitterNV50::setARegBits(unsigned int u)
 243 {
 244    code[0] |= (u & 3) << 26;
 245    code[1] |= (u & 4);
 246 }
 247
 248 void
 249 CodeEmitterNV50::setAReg16(const Instruction *i, int s)
 250 {
 251    s = i->src[s].indirect[0];
 252    if (s >= 0)
 253       setARegBits(SDATA(i->src[s]).id + 1);
 254 }
 255
 256 void
 257 CodeEmitterNV50::setImmediate(const Instruction *i, int s)
 258 {
 259    const ImmediateValue *imm = i->src[s].get()->asImm();
 260    assert(imm);
 261
 262    code[1] |= 3;
 263    code[0] |= (imm->reg.data.u32 & 0x3f) << 16;
 264    code[1] |= (imm->reg.data.u32 >> 6) << 2;
 265 }
 266
 267 void
 268 CodeEmitterNV50::setDst(const Value *dst)
 269 {
 270    const Storage *reg = &dst->join->reg;
 271
 272    assert(reg->file != FILE_ADDRESS);
 273
 274    if (reg->data.id < 0) {
 275       code[0] |= (127 << 2) | 1;
 276       code[1] |= 8;
 277    } else {
 278       if (reg->file == FILE_SHADER_OUTPUT)
 279          code[1] |= 8;
 280       code[0] |= reg->data.id << 2;
 281    }
 282 }
 283
 284 void
 285 CodeEmitterNV50::setDst(const Instruction *i, int d)
 286 {
 287    if (i->defExists(d)) {
 288       setDst(i->getDef(d));
 289    } else
 290    if (!d) {
 291       code[0] |= 0x01fc; // bit bucket
 292       code[1] |= 0x0008;
 293    }
 294 }
 295
 296 void
 297 CodeEmitterNV50::emitSrc0(const ValueRef& ref)
 298 {
 299    const Storage *reg = &ref.rep()->reg;
 300
 301    if (reg->file == FILE_SHADER_INPUT)
 302       code[1] |= 0x00200000;
 303    else
 304    if (reg->file != FILE_GPR)
 305       ERROR("invalid src0 register file: %d\n", reg->file);
 306
 307    assert(reg->data.id < 128);
 308    code[0] |= reg->data.id << 9;
 309 }
 310
 311 void
 312 CodeEmitterNV50::emitSrc1(const ValueRef& ref)
 313 {
 314    const Storage *reg = &ref.rep()->reg;
 315
 316    if (reg->file == FILE_MEMORY_CONST) {
 317       assert(!(code[1] & 0x01800000));
 318       code[0] |= 1 << 23;
 319       code[1] |= reg->fileIndex << 22;
 320    } else
 321    if (reg->file != FILE_GPR) {
 322       ERROR("invalid src1 register file: %d\n", reg->file);
 323    }
 324
 325    assert(reg->data.id < 128);
 326    code[0] |= reg->data.id << 16;
 327 }
 328
 329 void
 330 CodeEmitterNV50::emitSrc2(const ValueRef& ref)
 331 {
 332    const Storage *reg = &ref.rep()->reg;
 333
 334    if (reg->file == FILE_MEMORY_CONST) {
 335       assert(!(code[1] & 0x01800000));
 336       code[0] |= 1 << 24;
 337       code[1] |= reg->fileIndex << 22;
 338    } else
 339    if (reg->file != FILE_GPR) {
 340       ERROR("invalid src1 register file: %d\n", reg->file);
 341    }
 342
 343    assert(reg->data.id < 128);
 344    code[1] |= reg->data.id << 14;
 345 }
 346
 347 // the default form:
 348 //  - long instruction
 349 //  - 1 to 3 sources in slots 0, 1, 2
 350 //  - address & flags
 351 void
 352 CodeEmitterNV50::emitForm_MAD(const Instruction *i)
 353 {
 354    assert(i->encSize == 8);
 355    code[0] |= 1;
 356
 357    emitFlagsRd(i);
 358    emitFlagsWr(i);
 359
 360    setDst(i, 0);
 361
 362    if (i->srcExists(0))
 363       emitSrc0(i->src[0]);
 364
 365    if (i->srcExists(1))
 366       emitSrc1(i->src[1]);
 367
 368    if (i->srcExists(2))
 369       emitSrc2(i->src[2]);
 370
 371    setAReg16(i, 1);
 372 }
 373
 374 // like default form, but 2nd source in slot 2, and no 3rd source
 375 void
 376 CodeEmitterNV50::emitForm_ADD(const Instruction *i)
 377 {
 378    assert(i->encSize == 8);
 379    code[0] |= 1;
 380
 381    emitFlagsRd(i);
 382    emitFlagsWr(i);
 383
 384    setDst(i, 0);
 385
 386    if (i->srcExists(0))
 387       emitSrc0(i->src[0]);
 388
 389    if (i->srcExists(1))
 390       emitSrc2(i->src[1]);
 391
 392    setAReg16(i, 1);
 393 }
 394
 395 // default short form
 396 void
 397 CodeEmitterNV50::emitForm_MUL(const Instruction *i)
 398 {
 399    assert(i->encSize == 4 && !(code[0] & 1));
 400    assert(i->defExists(0));
 401    assert(!i->getPredicate());
 402
 403    setDst(i, 0);
 404
 405    if (i->srcExists(0))
 406       emitSrc0(i->src[0]);
 407
 408    if (i->srcExists(1))
 409       emitSrc1(i->src[1]);
 410 }
 411
 412 // usual immediate form
 413 // - 1 to 3 sources where last is immediate
 414 // - no address or predicate possible
 415 void
 416 CodeEmitterNV50::emitForm_IMM(const Instruction *i)
 417 {
 418    assert(i->encSize == 8);
 419    code[0] |= 1;
 420
 421    assert(i->defExists(0) && i->srcExists(0));
 422
 423    setDst(i, 0);
 424
 425    if (i->srcExists(2)) {
 426       emitSrc0(i->src[0]);
 427       emitSrc1(i->src[1]);
 428       setImmediate(i, 2);
 429    } else
 430    if (i->srcExists(1)) {
 431       emitSrc0(i->src[0]);
 432       setImmediate(i, 1);
 433    } else {
 434       setImmediate(i, 0);
 435    }
 436 }
 437
 438 void
 439 CodeEmitterNV50::emitLoadStoreSize(DataType ty, int pos)
 440 {
 441    uint8_t enc;
 442
 443    switch (ty) {
 444    case TYPE_F32: // fall through
 445    case TYPE_S32: // fall through
 446    case TYPE_U32:  enc = 0x6; break;
 447    case TYPE_B128: enc = 0x5; break;
 448    case TYPE_F64:  enc = 0x4; break;
 449    case TYPE_S16:  enc = 0x3; break;
 450    case TYPE_U16:  enc = 0x2; break;
 451    case TYPE_S8:   enc = 0x1; break;
 452    case TYPE_U8:   enc = 0x0; break;
 453    default:
 454       enc = 0;
 455       assert(!"invalid load/store type");
 456       break;
 457    }
 458    code[pos / 32] |= enc << (pos % 32);
 459 }
 460
 461 void
 462 CodeEmitterNV50::emitLOAD(const Instruction *i)
 463 {
 464    DataFile sf = i->src[0].getFile();
 465
 466    switch (sf) {
 467    case FILE_SHADER_INPUT:
 468       code[0] = 0x10000001;
 469       code[1] = 0x04200000 | (i->lanes << 14);
 470       break;
 471    case FILE_MEMORY_CONST:
 472       code[0] = 0x10000001;
 473       code[1] = 0x24000000 | (i->getSrc(0)->reg.fileIndex << 22);
 474       break;
 475    case FILE_MEMORY_LOCAL:
 476       code[0] = 0xd0000001;
 477       code[1] = 0x40000000;
 478       break;
 479    case FILE_MEMORY_GLOBAL:
 480       code[0] = 0xd0000001 | (i->getSrc(0)->reg.fileIndex << 16);
 481       code[1] = 0x80000000;
 482       break;
 483    default:
 484       assert(!"invalid load source file");
 485       break;
 486    }
 487    if (sf == FILE_MEMORY_LOCAL ||
 488        sf == FILE_MEMORY_GLOBAL)
 489       emitLoadStoreSize(i->sType, 21 + 32);
 490
 491    setDst(i, 0);
 492
 493    emitFlagsRd(i);
 494    emitFlagsWr(i);
 495
 496    if (i->src[0].getFile() == FILE_MEMORY_GLOBAL) {
 497       srcId(*i->src[0].getIndirect(0), 9);
 498    } else {
 499       setAReg16(i, 0);
 500       srcAddr16(i->src[0], 9);
 501    }
 502 }
 503
 504 void
 505 CodeEmitterNV50::emitSTORE(const Instruction *i)
 506 {
 507    DataFile f = i->getSrc(0)->reg.file;
 508    int32_t offset = i->getSrc(0)->reg.data.offset;
 509
 510    switch (f) {
 511    case FILE_SHADER_OUTPUT:
 512       code[0] = 0x00000001 | ((offset >> 2) << 2);
 513       code[1] = 0x80c00000;
 514       srcId(i->src[1], 32 + 15);
 515       break;
 516    case FILE_MEMORY_GLOBAL:
 517       code[0] = 0xd0000000;
 518       code[1] = 0xa0000000;
 519       emitLoadStoreSize(i->dType, 21 + 32);
 520       break;
 521    case FILE_MEMORY_LOCAL:
 522       code[0] = 0xd0000001;
 523       code[1] = 0x60000000;
 524       emitLoadStoreSize(i->dType, 21 + 32);
 525       break;
 526    case FILE_MEMORY_SHARED:
 527       code[0] = 0x00000001;
 528       code[1] = 0xe0000000;
 529       switch (typeSizeof(i->dType)) {
 530       case 1:
 531          code[0] |= offset << 9;
 532          code[1] |= 0x00400000;
 533          break;
 534       case 2:
 535          code[0] |= (offset >> 1) << 9;
 536          break;
 537       case 4:
 538          code[0] |= (offset >> 2) << 9;
 539          code[1] |= 0x04000000;
 540          break;
 541       default:
 542          assert(0);
 543          break;
 544       }
 545       break;
 546    default:
 547       assert(!"invalid store destination file");
 548       break;
 549    }
 550
 551    if (f != FILE_SHADER_OUTPUT) {
 552       srcId(i->src[1], 2);
 553       if (f == FILE_MEMORY_GLOBAL)
 554          srcId(*i->src[0].getIndirect(0), 9);
 555       if (f == FILE_MEMORY_LOCAL)
 556          srcAddr16(i->src[0], 9);
 557    }
 558    if (f != FILE_MEMORY_GLOBAL)
 559       setAReg16(i, 0);
 560
 561    emitFlagsRd(i);
 562 }
 563
 564 void
 565 CodeEmitterNV50::emitMOV(const Instruction *i)
 566 {
 567    DataFile sf = i->getSrc(0)->reg.file;
 568    DataFile df = i->getDef(0)->reg.file;
 569
 570    assert(sf == FILE_GPR || df == FILE_GPR);
 571
 572    if (sf == FILE_FLAGS) {
 573       code[0] = 0x00000001;
 574       code[1] = 0x20000000;
 575       defId(i->def[0], 2);
 576       srcId(i->src[0], 12);
 577       emitFlagsRd(i);
 578    } else
 579    if (sf == FILE_ADDRESS) {
 580       code[0] = 0x00000001;
 581       code[1] = 0x40000000;
 582       defId(i->def[0], 2);
 583       setARegBits(SDATA(i->src[0]).id + 1);
 584    } else
 585    if (df == FILE_FLAGS) {
 586       code[0] = 0x00000001;
 587       code[1] = 0xa0000000;
 588       defId(i->def[0], 4);
 589       srcId(i->src[0], 9);
 590       emitFlagsRd(i);
 591    } else
 592    if (sf == FILE_IMMEDIATE) {
 593       code[0] = 0x10008001;
 594       code[1] = 0x00000003;
 595       emitForm_IMM(i);
 596    } else {
 597       if (i->encSize == 4) {
 598          code[0] = 0x10008000;
 599       } else {
 600          code[0] = 0x10000001;
 601          code[1] = 0x04000000 | (i->lanes << 14);
 602       }
 603       defId(i->def[0], 2);
 604       srcId(i->src[0], 9);
 605    }
 606    if (df == FILE_SHADER_OUTPUT) {
 607       assert(i->encSize == 8);
 608       code[1] |= 0x8;
 609    }
 610 }
 611
 612 void
 613 CodeEmitterNV50::emitNOP()
 614 {
 615    code[0] = 0xf0000001;
 616    code[1] = 0xe0000000;
 617 }
 618
 619 void
 620 CodeEmitterNV50::emitQUADOP(const Instruction *i, uint8_t lane, uint8_t quOp)
 621 {
 622    code[0] = 0xc0000000 | (lane << 16);
 623    code[1] = 0x80000000;
 624
 625    code[0] |= (quOp & 0x03) << 20;
 626    code[1] |= (quOp & 0xfc) << 20;
 627
 628    emitForm_ADD(i);
 629
 630    if (!i->srcExists(1))
 631       srcId(i->src[0], 32 + 14);
 632 }
 633
 634 void
 635 CodeEmitterNV50::emitPFETCH(const Instruction *i)
 636 {
 637    code[0] = 0x11800001;
 638    code[1] = 0x04200000 | (0xf << 14);
 639
 640    defId(i->def[0], 2);
 641    srcAddr8(i->src[0], 9);
 642    setAReg16(i, 0);
 643 }
 644
 645 void
 646 CodeEmitterNV50::emitINTERP(const Instruction *i)
 647 {
 648    code[0] = 0x80000000;
 649
 650    defId(i->def[0], 2);
 651    srcAddr8(i->src[0], 16);
 652
 653    if (i->getInterpMode() == NV50_IR_INTERP_FLAT) {
 654       code[0] |= 1 << 8;
 655    } else {
 656       if (i->op == OP_PINTERP) {
 657          code[0] |= 1 << 25;
 658          srcId(i->src[1], 9);
 659       }
 660       if (i->getSampleMode() == NV50_IR_INTERP_CENTROID)
 661          code[0] |= 1 << 24;
 662    }
 663
 664    if (i->encSize == 8) {
 665       emitFlagsRd(i);
 666       code[1] |=
 667          (code[0] & (3 << 24)) >> (24 - 16) |
 668          (code[0] & (1 <<  8)) >> (18 -  8);
 669       code[0] &= ~0x03000100;
 670       code[0] |= 1;
 671    }
 672 }
 673
 674 void
 675 CodeEmitterNV50::emitMINMAX(const Instruction *i)
 676 {
 677    if (i->dType == TYPE_F64) {
 678       code[0] = 0xe0000000;
 679       code[1] = (i->op == OP_MIN) ? 0xa0000000 : 0xc0000000;
 680    } else {
 681       code[0] = 0x30000000;
 682       code[1] = 0x80000000;
 683       if (i->op == OP_MIN)
 684          code[1] |= 0x20000000;
 685
 686       switch (i->dType) {
 687       case TYPE_F32: code[0] |= 0x80000000; break;
 688       case TYPE_S32: code[1] |= 0x8c000000; break;
 689       case TYPE_U32: code[1] |= 0x84000000; break;
 690       case TYPE_S16: code[1] |= 0x80000000; break;
 691       case TYPE_U16: break;
 692       default:
 693          assert(0);
 694          break;
 695       }
 696       code[1] |= i->src[0].mod.abs() << 20;
 697       code[1] |= i->src[1].mod.abs() << 19;
 698    }
 699    emitForm_MAD(i);
 700 }
 701
 702 void
 703 CodeEmitterNV50::emitFMAD(const Instruction *i)
 704 {
 705    const int neg_mul = i->src[0].mod.neg() ^ i->src[1].mod.neg();
 706    const int neg_add = i->src[2].mod.neg();
 707
 708    code[0] = 0xe0000000;
 709
 710    if (i->encSize == 4) {
 711       emitForm_MUL(i);
 712       assert(!neg_mul && !neg_add);
 713    } else {
 714       emitForm_MAD(i);
 715       code[1] |= neg_mul << 26;
 716       code[1] |= neg_add << 27;
 717       if (i->saturate)
 718          code[1] |= 1 << 29;
 719    }
 720 }
 721
 722 void
 723 CodeEmitterNV50::emitFADD(const Instruction *i)
 724 {
 725    const int neg0 = i->src[0].mod.neg();
 726    const int neg1 = i->src[1].mod.neg() ^ ((i->op == OP_SUB) ? 1 : 0);
 727
 728    code[0] = 0xb0000000;
 729
 730    assert(!(i->src[0].mod | i->src[1].mod).abs());
 731
 732    if (i->src[1].getFile() == FILE_IMMEDIATE) {
 733       emitForm_IMM(i);
 734       code[0] |= neg0 << 15;
 735       code[0] |= neg1 << 22;
 736    } else
 737    if (i->encSize == 8) {
 738       emitForm_ADD(i);
 739       code[1] |= neg0 << 26;
 740       code[1] |= neg1 << 27;
 741       if (i->saturate)
 742          code[1] |= 1 << 29;
 743    } else {
 744       emitForm_MUL(i);
 745       code[0] |= neg0 << 15;
 746       code[0] |= neg1 << 22;
 747    }
 748 }
 749
 750 void
 751 CodeEmitterNV50::emitUADD(const Instruction *i)
 752 {
 753    code[0] = 0x20008000;
 754
 755    if (i->src[0].getFile() == FILE_IMMEDIATE) {
 756       emitForm_IMM(i);
 757    } else
 758    if (i->encSize == 8) {
 759       code[0] = 0x20000000;
 760       code[1] = 0x04000000;
 761       emitForm_ADD(i);
 762    } else {
 763       emitForm_MUL(i);
 764    }
 765    assert(!(i->src[0].mod.neg() && i->src[1].mod.neg()));
 766    code[0] |= i->src[0].mod.neg() << 28;
 767    code[0] |= i->src[1].mod.neg() << 22;
 768 }
 769
 770 void
 771 CodeEmitterNV50::emitAADD(const Instruction *i)
 772 {
 773    const int s = (i->op == OP_MOV) ? 0 : 1;
 774
 775    code[0] = 0xd0000001 | (i->getSrc(s)->reg.data.u16 << 9);
 776    code[1] = 0x20000000;
 777
 778    code[0] |= (DDATA(i->def[0]).id + 1) << 2;
 779
 780    emitFlagsRd(i);
 781
 782    if (s && i->srcExists(0))
 783       setARegBits(SDATA(i->src[0]).id + 1);
 784 }
 785
 786 void
 787 CodeEmitterNV50::emitFMUL(const Instruction *i)
 788 {
 789    const int neg = (i->src[0].mod ^ i->src[1].mod).neg();
 790
 791    code[0] = 0xc0000000;
 792
 793    if (i->src[0].getFile() == FILE_IMMEDIATE) {
 794       emitForm_IMM(i);
 795       if (neg)
 796          code[0] |= 0x8000;
 797    } else
 798    if (i->encSize == 8) {
 799       emitForm_MAD(i);
 800       if (neg)
 801          code[1] |= 0x08000000;
 802    } else {
 803       emitForm_MUL(i);
 804       if (neg)
 805          code[0] |= 0x8000;
 806    }
 807 }
 808
 809 void
 810 CodeEmitterNV50::emitSET(const Instruction *i)
 811 {
 812    code[0] = 0x30000000;
 813    code[1] = 0x60000000;
 814
 815    emitCondCode(i->asCmp()->setCond, 32 + 14);
 816
 817    switch (i->sType) {
 818    case TYPE_F32: code[0] |= 0x80000000; break;
 819    case TYPE_S32: code[1] |= 0x0c000000; break;
 820    case TYPE_U32: code[1] |= 0x04000000; break;
 821    case TYPE_S16: code[1] |= 0x08000000; break;
 822    case TYPE_U16: break;
 823    default:
 824       assert(0);
 825       break;
 826    }
 827    emitForm_MAD(i);
 828 }
 829
 830 void
 831 CodeEmitterNV50::roundMode_CVT(RoundMode rnd)
 832 {
 833    switch (rnd) {
 834    case ROUND_NI: code[1] |= 0x08000000; break;
 835    case ROUND_M:  code[1] |= 0x00020000; break;
 836    case ROUND_MI: code[1] |= 0x08020000; break;
 837    case ROUND_P:  code[1] |= 0x00040000; break;
 838    case ROUND_PI: code[1] |= 0x08040000; break;
 839    case ROUND_Z:  code[1] |= 0x00060000; break;
 840    case ROUND_ZI: code[1] |= 0x08060000; break;
 841    default:
 842       assert(rnd == ROUND_N);
 843       break;
 844    }
 845 }
 846
 847 void
 848 CodeEmitterNV50::emitCVT(const Instruction *i)
 849 {
 850    const bool f2f = isFloatType(i->dType) && isFloatType(i->sType);
 851    RoundMode rnd;
 852
 853    switch (i->op) {
 854    case OP_CEIL:  rnd = f2f ? ROUND_PI : ROUND_P; break;
 855    case OP_FLOOR: rnd = f2f ? ROUND_MI : ROUND_M; break;
 856    case OP_TRUNC: rnd = f2f ? ROUND_ZI : ROUND_Z; break;
 857    default:
 858       rnd = i->rnd;
 859       break;
 860    }
 861
 862    code[0] = 0xa0000000;
 863
 864    switch (i->dType) {
 865    case TYPE_F64:
 866       switch (i->sType) {
 867       case TYPE_F64: code[1] = 0xc4404000; break;
 868       case TYPE_S64: code[1] = 0x44414000; break;
 869       case TYPE_U64: code[1] = 0x44404000; break;
 870       case TYPE_F32: code[1] = 0xc4400000; break;
 871       case TYPE_S32: code[1] = 0x44410000; break;
 872       case TYPE_U32: code[1] = 0x44400000; break;
 873       default:
 874          assert(0);
 875          break;
 876       }
 877       break;
 878    case TYPE_S64:
 879       switch (i->sType) {
 880       case TYPE_F64: code[1] = 0x8c404000; break;
 881       case TYPE_F32: code[1] = 0x8c400000; break;
 882       default:
 883          assert(0);
 884          break;
 885       }
 886       break;
 887    case TYPE_U64:
 888       switch (i->sType) {
 889       case TYPE_F64: code[1] = 0x84404000; break;
 890       case TYPE_F32: code[1] = 0x84400000; break;
 891       default:
 892          assert(0);
 893          break;
 894       }
 895       break;
 896    case TYPE_F32:
 897       switch (i->sType) {
 898       case TYPE_F64: code[1] = 0xc0404000; break;
 899       case TYPE_S64: code[1] = 0x40414000; break;
 900       case TYPE_U64: code[1] = 0x40404000; break;
 901       case TYPE_F32: code[1] = 0xc4004000; break;
 902       case TYPE_S32: code[1] = 0x44014000; break;
 903       case TYPE_U32: code[1] = 0x44004000; break;
 904       case TYPE_F16: code[1] = 0xc4000000; break;
 905       default:
 906          assert(0);
 907          break;
 908       }
 909       break;
 910    case TYPE_S32:
 911       switch (i->sType) {
 912       case TYPE_F64: code[1] = 0x88404000; break;
 913       case TYPE_F32: code[1] = 0x8c004000; break;
 914       case TYPE_S32: code[1] = 0x0c014000; break;
 915       case TYPE_U32: code[1] = 0x0c004000; break;
 916       case TYPE_F16: code[1] = 0x8c000000; break;
 917       case TYPE_S16: code[1] = 0x0c010000; break;
 918       case TYPE_U16: code[1] = 0x0c000000; break;
 919       case TYPE_S8:  code[1] = 0x0c018000; break;
 920       case TYPE_U8:  code[1] = 0x0c008000; break;
 921       default:
 922          assert(0);
 923          break;
 924       }
 925       break;
 926    case TYPE_U32:
 927       switch (i->sType) {
 928       case TYPE_F64: code[1] = 0x80404000; break;
 929       case TYPE_F32: code[1] = 0x84004000; break;
 930       case TYPE_S32: code[1] = 0x04014000; break;
 931       case TYPE_U32: code[1] = 0x04004000; break;
 932       case TYPE_F16: code[1] = 0x84000000; break;
 933       case TYPE_S16: code[1] = 0x04010000; break;
 934       case TYPE_U16: code[1] = 0x04000000; break;
 935       case TYPE_S8:  code[1] = 0x04018000; break;
 936       case TYPE_U8:  code[1] = 0x04008000; break;
 937       default:
 938          assert(0);
 939          break;
 940       }
 941    case TYPE_S16:
 942    case TYPE_U16:
 943    case TYPE_S8:
 944    case TYPE_U8:
 945    default:
 946       assert(0);
 947       break;
 948    }
 949    if (typeSizeof(i->sType) == 1 && i->getSrc(0)->reg.size == 4)
 950       code[1] |= 0x00004000;
 951
 952    roundMode_CVT(rnd);
 953
 954    switch (i->op) {
 955    case OP_ABS: code[1] |= 1 << 20; break;
 956    case OP_SAT: code[1] |= 1 << 19; break;
 957    case OP_NEG: code[1] |= 1 << 29; break;
 958    default:
 959       break;
 960    }
 961    code[1] ^= i->src[0].mod.neg() << 29;
 962    code[1] |= i->src[0].mod.abs() << 20;
 963    if (i->saturate)
 964       code[1] |= 1 << 19;
 965
 966    assert(i->op != OP_ABS || !i->src[0].mod.neg());
 967
 968    emitForm_MAD(i);
 969 }
 970
 971 void
 972 CodeEmitterNV50::emitPreOp(const Instruction *i)
 973 {
 974    code[0] = 0xb0000000;
 975    code[1] = (i->op == OP_PREEX2) ? 0xc0004000 : 0xc0000000;
 976
 977    code[1] |= i->src[0].mod.abs() << 20;
 978    code[1] |= i->src[0].mod.neg() << 26;
 979
 980    emitForm_MAD(i);
 981 }
 982
 983 void
 984 CodeEmitterNV50::emitSFnOp(const Instruction *i, uint8_t subOp)
 985 {
 986    code[0] = 0x90000000;
 987
 988    if (i->encSize == 4) {
 989       assert(i->op == OP_RCP);
 990       emitForm_MUL(i);
 991    } else {
 992       code[1] = subOp << 29;
 993       code[1] |= i->src[0].mod.abs() << 20;
 994       code[1] |= i->src[0].mod.neg() << 26;
 995       emitForm_MAD(i);
 996    }
 997 }
 998
 999 void
1000 CodeEmitterNV50::emitLogicOp(const Instruction *i)
1001 {
1002    code[0] = 0xd0000000;
1003
1004    if (i->src[1].getFile() == FILE_IMMEDIATE) {
1005       switch (i->op) {
1006       case OP_OR:  code[0] |= 0x0100; break;
1007       case OP_XOR: code[0] |= 0x8000; break;
1008       default:
1009          assert(i->op == OP_AND);
1010          break;
1011       }
1012       emitForm_IMM(i);
1013    } else {
1014       switch (i->op) {
1015       case OP_AND: code[1] = 0x04000000; break;
1016       case OP_OR:  code[1] = 0x04004000; break;
1017       case OP_XOR: code[1] = 0x04008000; break;
1018       default:
1019          assert(0);
1020          break;
1021       }
1022       emitForm_MAD(i);
1023    }
1024 }
1025
1026 void
1027 CodeEmitterNV50::emitARL(const Instruction *i)
1028 {
1029    assert(i->src[1].getFile() == FILE_IMMEDIATE);
1030
1031    code[0] = 0x00000001 | (i->getSrc(1)->reg.data.u32 & 0x3f) << 16;
1032    code[1] = 0xc0000000;
1033
1034    code[0] |= (DDATA(i->def[0]).id + 1) << 2;
1035    emitSrc0(i->src[0]);
1036    emitFlagsRd(i);
1037 }
1038
1039 void
1040 CodeEmitterNV50::emitShift(const Instruction *i)
1041 {
1042    if (i->def[0].getFile() == FILE_ADDRESS) {
1043       emitARL(i);
1044    } else {
1045       code[0] = 0x30000001;
1046       code[1] = (i->op == OP_SHR) ? 0xe4000000 : 0xc4000000;
1047       if (isSignedType(i->sType))
1048           code[1] |= 1 << 27;
1049
1050       if (i->src[1].getFile() == FILE_IMMEDIATE) {
1051          code[1] |= 1 << 20;
1052          code[0] |= (i->getSrc(1)->reg.data.u32 & 0x7f) << 16;
1053          emitFlagsRd(i);
1054       } else {
1055          emitForm_MAD(i);
1056       }
1057    }
1058 }
1059
1060 void
1061 CodeEmitterNV50::emitOUT(const Instruction *i)
1062 {
1063    code[0] = (i->op == OP_EMIT) ? 0xf0000200 : 0xf0000400;
1064    code[1] = 0xc0000001;
1065
1066    emitFlagsRd(i);
1067 }
1068
1069 void
1070 CodeEmitterNV50::emitTEX(const TexInstruction *i)
1071 {
1072    code[0] = 0xf0000001;
1073    code[1] = 0x00000000;
1074
1075    switch (i->op) {
1076    case OP_TXB:
1077       code[1] = 0x20000000;
1078       break;
1079    case OP_TXL:
1080       code[1] = 0x40000000;
1081       break;
1082    case OP_TXF:
1083       code[0] = 0x01000000;
1084       break;
1085    case OP_TXG:
1086       code[0] = 0x01000000;
1087       code[1] = 0x80000000;
1088       break;
1089    default:
1090       assert(i->op == OP_TEX);
1091       break;
1092    }
1093
1094    code[0] |= i->tex.r << 9;
1095    code[0] |= i->tex.s << 17;
1096
1097    int argc = i->tex.target.getArgCount();
1098
1099    if (i->op == OP_TXB || i->op == OP_TXL)
1100       argc += 1;
1101    if (i->tex.target.isShadow())
1102       argc += 1;
1103    assert(argc <= 4);
1104
1105    code[0] |= (argc - 1) << 22;
1106
1107    if (i->tex.target.isCube()) {
1108       code[0] |= 0x08000000;
1109    } else
1110    if (i->tex.useOffsets) {
1111       code[1] |= (i->tex.offset[0][0] & 0xf) << 16;
1112       code[1] |= (i->tex.offset[0][1] & 0xf) << 20;
1113       code[1] |= (i->tex.offset[0][2] & 0xf) << 24;
1114    }
1115
1116    code[0] |= (i->tex.mask & 0x3) << 25;
1117    code[1] |= (i->tex.mask & 0xc) << 12;
1118
1119    if (i->tex.liveOnly)
1120       code[1] |= 4;
1121
1122    defId(i->def[0], 2);
1123
1124    emitFlagsRd(i);
1125 }
1126
1127 void
1128 CodeEmitterNV50::emitFlow(const Instruction *i, uint8_t flowOp)
1129 {
1130    const FlowInstruction *f = i->asFlow();
1131
1132    code[0] = 0x00000003 | (flowOp << 28);
1133    code[1] = 0x00000000;
1134
1135    emitFlagsRd(i);
1136
1137    if (f && f->target.bb) {
1138       uint32_t pos;
1139
1140       if (f->op == OP_CALL) {
1141          if (f->builtin) {
1142             pos = 0; // XXX: TODO
1143          } else {
1144             pos = f->target.fn->binPos;
1145          }
1146       } else {
1147          pos = f->target.bb->binPos;
1148       }
1149
1150       code[0] |= ((pos >>  2) & 0xffff) << 11;
1151       code[1] |= ((pos >> 18) & 0x003f) << 14;
1152    }
1153 }
1154
1155 bool
1156 CodeEmitterNV50::emitInstruction(Instruction *insn)
1157 {
1158    if (!insn->encSize) {
1159       ERROR("skipping unencodable instruction: "); insn->print();
1160       return false;
1161    } else
1162    if (codeSize + insn->encSize > codeSizeLimit) {
1163       ERROR("code emitter output buffer too small\n");
1164       return false;
1165    }
1166
1167    switch (insn->op) {
1168    case OP_MOV:
1169       emitMOV(insn);
1170       break;
1171    case OP_NOP:
1172    case OP_JOIN:
1173       emitNOP();
1174       break;
1175    case OP_VFETCH:
1176    case OP_LOAD:
1177       emitLOAD(insn);
1178       break;
1179    case OP_EXPORT:
1180    case OP_STORE:
1181       emitSTORE(insn);
1182       break;
1183    case OP_PFETCH:
1184       emitPFETCH(insn);
1185       break;
1186    case OP_LINTERP:
1187    case OP_PINTERP:
1188       emitINTERP(insn);
1189       break;
1190    case OP_ADD:
1191    case OP_SUB:
1192       if (isFloatType(insn->dType))
1193          emitFADD(insn);
1194       else
1195          emitUADD(insn);
1196       break;
1197    case OP_MUL:
1198       if (isFloatType(insn->dType))
1199          emitFMUL(insn);
1200       else
1201          emitUMUL(insn);
1202       break;
1203    case OP_MAD:
1204    case OP_FMA:
1205       emitFMAD(insn);
1206       break;
1207       break;
1208    case OP_AND:
1209    case OP_OR:
1210    case OP_XOR:
1211       emitLogicOp(insn);
1212       break;
1213    case OP_MIN:
1214    case OP_MAX:
1215       emitMINMAX(insn);
1216       break;
1217    case OP_CEIL:
1218    case OP_FLOOR:
1219    case OP_TRUNC:
1220    case OP_CVT:
1221       emitCVT(insn);
1222       break;
1223    case OP_RCP:
1224       emitSFnOp(insn, 0);
1225       break;
1226    case OP_RSQ:
1227       emitSFnOp(insn, 2);
1228       break;
1229    case OP_LG2:
1230       emitSFnOp(insn, 3);
1231       break;
1232    case OP_SIN:
1233       emitSFnOp(insn, 4);
1234       break;
1235    case OP_COS:
1236       emitSFnOp(insn, 5);
1237       break;
1238    case OP_EX2:
1239       emitSFnOp(insn, 6);
1240       break;
1241    case OP_PRESIN:
1242    case OP_PREEX2:
1243       emitPreOp(insn);
1244       break;
1245    case OP_TEX:
1246    case OP_TXB:
1247    case OP_TXL:
1248       emitTEX(insn->asTex());
1249       break;
1250    case OP_EMIT:
1251    case OP_RESTART:
1252       emitOUT(insn);
1253       break;
1254    case OP_DISCARD:
1255       emitFlow(insn, 0x0);
1256       break;
1257    case OP_BRA:
1258       emitFlow(insn, 0x1);
1259       break;
1260    case OP_CALL:
1261       emitFlow(insn, 0x2);
1262       break;
1263    case OP_RET:
1264       emitFlow(insn, 0x3);
1265       break;
1266    case OP_PREBREAK:
1267       emitFlow(insn, 0x4);
1268       break;
1269    case OP_BREAK:
1270       emitFlow(insn, 0x5);
1271       break;
1272    case OP_QUADON:
1273       emitFlow(insn, 0x6);
1274       break;
1275    case OP_QUADPOP:
1276       emitFlow(insn, 0x7);
1277       break;
1278    case OP_JOINAT:
1279       emitFlow(insn, 0xa);
1280       break;
1281    case OP_PRERET:
1282       emitFlow(insn, 0xd);
1283       break;
1284    case OP_QUADOP:
1285       emitQUADOP(insn, insn->lanes, insn->subOp);
1286       break;
1287    case OP_DFDX:
1288       emitQUADOP(insn, 4, insn->src[0].mod.neg() ? 0x66 : 0x99);
1289       break;
1290    case OP_DFDY:
1291       emitQUADOP(insn, 5, insn->src[0].mod.neg() ? 0x5a : 0xa5);
1292       break;
1293    case OP_PHI:
1294    case OP_UNION:
1295    case OP_CONSTRAINT:
1296       ERROR("operation should have been eliminated");
1297       return false;
1298    case OP_EXP:
1299    case OP_LOG:
1300    case OP_SQRT:
1301    case OP_POW:
1302    case OP_SELP:
1303    case OP_SLCT:
1304    case OP_TXD:
1305    case OP_PRECONT:
1306    case OP_CONT:
1307    case OP_POPCNT:
1308    case OP_INSBF:
1309    case OP_EXTBF:
1310       ERROR("operation should have been lowered\n");
1311       return false;
1312    default:
1313       ERROR("unknow op\n");
1314       return false;
1315    }
1316    if (insn->join)
1317       code[1] |= 0x2;
1318    else
1319    if (insn->exit)
1320       code[1] |= 0x1;
1321
1322    assert((insn->encSize == 8) == (code[1] & 1));
1323
1324    code += insn->encSize / 4;
1325    codeSize += insn->encSize;
1326    return true;
1327 }
1328
1329 uint32_t
1330 CodeEmitterNV50::getMinEncodingSize(const Instruction *i) const
1331 {
1332    const Target::OpInfo &info = targ->getOpInfo(i);
1333
1334    if (info.minEncSize == 8)
1335       return 8;
1336
1337    return 4;
1338 }
1339
1340 CodeEmitterNV50::CodeEmitterNV50(const Target *target) : targ(target)
1341 {
1342    code = NULL;
1343    codeSize = codeSizeLimit = 0;
1344 }
1345
1346 CodeEmitter *
1347 Target::getCodeEmitter(Program::Type type)
1348 {
1349    CodeEmitterNV50 *emit = new CodeEmitterNV50(this);
1350    emit->setProgramType(type);
1351    return emit;
1352 }
1353
1354 } // namespace nv50_ir