src/gallium/drivers/nouveau/codegen/nv50_ir_from_tgsi.cpp

   1 /*
   2  * Copyright 2011 Christoph Bumiller
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 extern "C" {
  24 #include "tgsi/tgsi_dump.h"
  25 #include "tgsi/tgsi_scan.h"
  26 #include "tgsi/tgsi_util.h"
  27 }
  28
  29 #include <set>
  30
  31 #include "codegen/nv50_ir.h"
  32 #include "codegen/nv50_ir_util.h"
  33 #include "codegen/nv50_ir_build_util.h"
  34
  35 namespace tgsi {
  36
  37 class Source;
  38
  39 static nv50_ir::operation translateOpcode(uint opcode);
  40 static nv50_ir::DataFile translateFile(uint file);
  41 static nv50_ir::TexTarget translateTexture(uint texTarg);
  42 static nv50_ir::SVSemantic translateSysVal(uint sysval);
  43
  44 class Instruction
  45 {
  46 public:
  47    Instruction(const struct tgsi_full_instruction *inst) : insn(inst) { }
  48
  49    class SrcRegister
  50    {
  51    public:
  52       SrcRegister(const struct tgsi_full_src_register *src)
  53          : reg(src->Register),
  54            fsr(src)
  55       { }
  56
  57       SrcRegister(const struct tgsi_src_register& src) : reg(src), fsr(NULL) { }
  58
  59       SrcRegister(const struct tgsi_ind_register& ind)
  60          : reg(tgsi_util_get_src_from_ind(&ind)),
  61            fsr(NULL)
  62       { }
  63
  64       struct tgsi_src_register offsetToSrc(struct tgsi_texture_offset off)
  65       {
  66          struct tgsi_src_register reg;
  67          memset(&reg, 0, sizeof(reg));
  68          reg.Index = off.Index;
  69          reg.File = off.File;
  70          reg.SwizzleX = off.SwizzleX;
  71          reg.SwizzleY = off.SwizzleY;
  72          reg.SwizzleZ = off.SwizzleZ;
  73          return reg;
  74       }
  75
  76       SrcRegister(const struct tgsi_texture_offset& off) :
  77          reg(offsetToSrc(off)),
  78          fsr(NULL)
  79       { }
  80
  81       uint getFile() const { return reg.File; }
  82
  83       bool is2D() const { return reg.Dimension; }
  84
  85       bool isIndirect(int dim) const
  86       {
  87          return (dim && fsr) ? fsr->Dimension.Indirect : reg.Indirect;
  88       }
  89
  90       int getIndex(int dim) const
  91       {
  92          return (dim && fsr) ? fsr->Dimension.Index : reg.Index;
  93       }
  94
  95       int getSwizzle(int chan) const
  96       {
  97          return tgsi_util_get_src_register_swizzle(&reg, chan);
  98       }
  99
 100       nv50_ir::Modifier getMod(int chan) const;
 101
 102       SrcRegister getIndirect(int dim) const
 103       {
 104          assert(fsr && isIndirect(dim));
 105          if (dim)
 106             return SrcRegister(fsr->DimIndirect);
 107          return SrcRegister(fsr->Indirect);
 108       }
 109
 110       uint32_t getValueU32(int c, const struct nv50_ir_prog_info *info) const
 111       {
 112          assert(reg.File == TGSI_FILE_IMMEDIATE);
 113          assert(!reg.Absolute);
 114          assert(!reg.Negate);
 115          return info->immd.data[reg.Index * 4 + getSwizzle(c)];
 116       }
 117
 118    private:
 119       const struct tgsi_src_register reg;
 120       const struct tgsi_full_src_register *fsr;
 121    };
 122
 123    class DstRegister
 124    {
 125    public:
 126       DstRegister(const struct tgsi_full_dst_register *dst)
 127          : reg(dst->Register),
 128            fdr(dst)
 129       { }
 130
 131       DstRegister(const struct tgsi_dst_register& dst) : reg(dst), fdr(NULL) { }
 132
 133       uint getFile() const { return reg.File; }
 134
 135       bool is2D() const { return reg.Dimension; }
 136
 137       bool isIndirect(int dim) const
 138       {
 139          return (dim && fdr) ? fdr->Dimension.Indirect : reg.Indirect;
 140       }
 141
 142       int getIndex(int dim) const
 143       {
 144          return (dim && fdr) ? fdr->Dimension.Dimension : reg.Index;
 145       }
 146
 147       unsigned int getMask() const { return reg.WriteMask; }
 148
 149       bool isMasked(int chan) const { return !(getMask() & (1 << chan)); }
 150
 151       SrcRegister getIndirect(int dim) const
 152       {
 153          assert(fdr && isIndirect(dim));
 154          if (dim)
 155             return SrcRegister(fdr->DimIndirect);
 156          return SrcRegister(fdr->Indirect);
 157       }
 158
 159    private:
 160       const struct tgsi_dst_register reg;
 161       const struct tgsi_full_dst_register *fdr;
 162    };
 163
 164    inline uint getOpcode() const { return insn->Instruction.Opcode; }
 165
 166    unsigned int srcCount() const { return insn->Instruction.NumSrcRegs; }
 167    unsigned int dstCount() const { return insn->Instruction.NumDstRegs; }
 168
 169    // mask of used components of source s
 170    unsigned int srcMask(unsigned int s) const;
 171
 172    SrcRegister getSrc(unsigned int s) const
 173    {
 174       assert(s < srcCount());
 175       return SrcRegister(&insn->Src[s]);
 176    }
 177
 178    DstRegister getDst(unsigned int d) const
 179    {
 180       assert(d < dstCount());
 181       return DstRegister(&insn->Dst[d]);
 182    }
 183
 184    SrcRegister getTexOffset(unsigned int i) const
 185    {
 186       assert(i < TGSI_FULL_MAX_TEX_OFFSETS);
 187       return SrcRegister(insn->TexOffsets[i]);
 188    }
 189
 190    unsigned int getNumTexOffsets() const { return insn->Texture.NumOffsets; }
 191
 192    bool checkDstSrcAliasing() const;
 193
 194    inline nv50_ir::operation getOP() const {
 195       return translateOpcode(getOpcode()); }
 196
 197    nv50_ir::DataType inferSrcType() const;
 198    nv50_ir::DataType inferDstType() const;
 199
 200    nv50_ir::CondCode getSetCond() const;
 201
 202    nv50_ir::TexInstruction::Target getTexture(const Source *, int s) const;
 203
 204    inline uint getLabel() { return insn->Label.Label; }
 205
 206    unsigned getSaturate() const { return insn->Instruction.Saturate; }
 207
 208    void print() const
 209    {
 210       tgsi_dump_instruction(insn, 1);
 211    }
 212
 213 private:
 214    const struct tgsi_full_instruction *insn;
 215 };
 216
 217 unsigned int Instruction::srcMask(unsigned int s) const
 218 {
 219    unsigned int mask = insn->Dst[0].Register.WriteMask;
 220
 221    switch (insn->Instruction.Opcode) {
 222    case TGSI_OPCODE_COS:
 223    case TGSI_OPCODE_SIN:
 224       return (mask & 0x8) | ((mask & 0x7) ? 0x1 : 0x0);
 225    case TGSI_OPCODE_DP2:
 226       return 0x3;
 227    case TGSI_OPCODE_DP3:
 228       return 0x7;
 229    case TGSI_OPCODE_DP4:
 230    case TGSI_OPCODE_DPH:
 231    case TGSI_OPCODE_KILL_IF: /* WriteMask ignored */
 232       return 0xf;
 233    case TGSI_OPCODE_DST:
 234       return mask & (s ? 0xa : 0x6);
 235    case TGSI_OPCODE_EX2:
 236    case TGSI_OPCODE_EXP:
 237    case TGSI_OPCODE_LG2:
 238    case TGSI_OPCODE_LOG:
 239    case TGSI_OPCODE_POW:
 240    case TGSI_OPCODE_RCP:
 241    case TGSI_OPCODE_RSQ:
 242    case TGSI_OPCODE_SCS:
 243       return 0x1;
 244    case TGSI_OPCODE_IF:
 245    case TGSI_OPCODE_UIF:
 246       return 0x1;
 247    case TGSI_OPCODE_LIT:
 248       return 0xb;
 249    case TGSI_OPCODE_TEX2:
 250    case TGSI_OPCODE_TXB2:
 251    case TGSI_OPCODE_TXL2:
 252       return (s == 0) ? 0xf : 0x3;
 253    case TGSI_OPCODE_TEX:
 254    case TGSI_OPCODE_TXB:
 255    case TGSI_OPCODE_TXD:
 256    case TGSI_OPCODE_TXL:
 257    case TGSI_OPCODE_TXP:
 258    case TGSI_OPCODE_LODQ:
 259    {
 260       const struct tgsi_instruction_texture *tex = &insn->Texture;
 261
 262       assert(insn->Instruction.Texture);
 263
 264       mask = 0x7;
 265       if (insn->Instruction.Opcode != TGSI_OPCODE_TEX &&
 266           insn->Instruction.Opcode != TGSI_OPCODE_TXD)
 267          mask |= 0x8; /* bias, lod or proj */
 268
 269       switch (tex->Texture) {
 270       case TGSI_TEXTURE_1D:
 271          mask &= 0x9;
 272          break;
 273       case TGSI_TEXTURE_SHADOW1D:
 274          mask &= 0xd;
 275          break;
 276       case TGSI_TEXTURE_1D_ARRAY:
 277       case TGSI_TEXTURE_2D:
 278       case TGSI_TEXTURE_RECT:
 279          mask &= 0xb;
 280          break;
 281       case TGSI_TEXTURE_CUBE_ARRAY:
 282       case TGSI_TEXTURE_SHADOW2D_ARRAY:
 283       case TGSI_TEXTURE_SHADOWCUBE:
 284       case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
 285          mask |= 0x8;
 286          break;
 287       default:
 288          break;
 289       }
 290    }
 291       return mask;
 292    case TGSI_OPCODE_XPD:
 293    {
 294       unsigned int x = 0;
 295       if (mask & 1) x |= 0x6;
 296       if (mask & 2) x |= 0x5;
 297       if (mask & 4) x |= 0x3;
 298       return x;
 299    }
 300    default:
 301       break;
 302    }
 303
 304    return mask;
 305 }
 306
 307 nv50_ir::Modifier Instruction::SrcRegister::getMod(int chan) const
 308 {
 309    nv50_ir::Modifier m(0);
 310
 311    if (reg.Absolute)
 312       m = m | nv50_ir::Modifier(NV50_IR_MOD_ABS);
 313    if (reg.Negate)
 314       m = m | nv50_ir::Modifier(NV50_IR_MOD_NEG);
 315    return m;
 316 }
 317
 318 static nv50_ir::DataFile translateFile(uint file)
 319 {
 320    switch (file) {
 321    case TGSI_FILE_CONSTANT:        return nv50_ir::FILE_MEMORY_CONST;
 322    case TGSI_FILE_INPUT:           return nv50_ir::FILE_SHADER_INPUT;
 323    case TGSI_FILE_OUTPUT:          return nv50_ir::FILE_SHADER_OUTPUT;
 324    case TGSI_FILE_TEMPORARY:       return nv50_ir::FILE_GPR;
 325    case TGSI_FILE_ADDRESS:         return nv50_ir::FILE_ADDRESS;
 326    case TGSI_FILE_PREDICATE:       return nv50_ir::FILE_PREDICATE;
 327    case TGSI_FILE_IMMEDIATE:       return nv50_ir::FILE_IMMEDIATE;
 328    case TGSI_FILE_SYSTEM_VALUE:    return nv50_ir::FILE_SYSTEM_VALUE;
 329    case TGSI_FILE_RESOURCE:        return nv50_ir::FILE_MEMORY_GLOBAL;
 330    case TGSI_FILE_SAMPLER:
 331    case TGSI_FILE_NULL:
 332    default:
 333       return nv50_ir::FILE_NULL;
 334    }
 335 }
 336
 337 static nv50_ir::SVSemantic translateSysVal(uint sysval)
 338 {
 339    switch (sysval) {
 340    case TGSI_SEMANTIC_FACE:       return nv50_ir::SV_FACE;
 341    case TGSI_SEMANTIC_PSIZE:      return nv50_ir::SV_POINT_SIZE;
 342    case TGSI_SEMANTIC_PRIMID:     return nv50_ir::SV_PRIMITIVE_ID;
 343    case TGSI_SEMANTIC_INSTANCEID: return nv50_ir::SV_INSTANCE_ID;
 344    case TGSI_SEMANTIC_VERTEXID:   return nv50_ir::SV_VERTEX_ID;
 345    case TGSI_SEMANTIC_GRID_SIZE:  return nv50_ir::SV_NCTAID;
 346    case TGSI_SEMANTIC_BLOCK_ID:   return nv50_ir::SV_CTAID;
 347    case TGSI_SEMANTIC_BLOCK_SIZE: return nv50_ir::SV_NTID;
 348    case TGSI_SEMANTIC_THREAD_ID:  return nv50_ir::SV_TID;
 349    case TGSI_SEMANTIC_SAMPLEID:   return nv50_ir::SV_SAMPLE_INDEX;
 350    case TGSI_SEMANTIC_SAMPLEPOS:  return nv50_ir::SV_SAMPLE_POS;
 351    case TGSI_SEMANTIC_SAMPLEMASK: return nv50_ir::SV_SAMPLE_MASK;
 352    case TGSI_SEMANTIC_INVOCATIONID: return nv50_ir::SV_INVOCATION_ID;
 353    default:
 354       assert(0);
 355       return nv50_ir::SV_CLOCK;
 356    }
 357 }
 358
 359 #define NV50_IR_TEX_TARG_CASE(a, b) \
 360    case TGSI_TEXTURE_##a: return nv50_ir::TEX_TARGET_##b;
 361
 362 static nv50_ir::TexTarget translateTexture(uint tex)
 363 {
 364    switch (tex) {
 365    NV50_IR_TEX_TARG_CASE(1D, 1D);
 366    NV50_IR_TEX_TARG_CASE(2D, 2D);
 367    NV50_IR_TEX_TARG_CASE(2D_MSAA, 2D_MS);
 368    NV50_IR_TEX_TARG_CASE(3D, 3D);
 369    NV50_IR_TEX_TARG_CASE(CUBE, CUBE);
 370    NV50_IR_TEX_TARG_CASE(RECT, RECT);
 371    NV50_IR_TEX_TARG_CASE(1D_ARRAY, 1D_ARRAY);
 372    NV50_IR_TEX_TARG_CASE(2D_ARRAY, 2D_ARRAY);
 373    NV50_IR_TEX_TARG_CASE(2D_ARRAY_MSAA, 2D_MS_ARRAY);
 374    NV50_IR_TEX_TARG_CASE(CUBE_ARRAY, CUBE_ARRAY);
 375    NV50_IR_TEX_TARG_CASE(SHADOW1D, 1D_SHADOW);
 376    NV50_IR_TEX_TARG_CASE(SHADOW2D, 2D_SHADOW);
 377    NV50_IR_TEX_TARG_CASE(SHADOWCUBE, CUBE_SHADOW);
 378    NV50_IR_TEX_TARG_CASE(SHADOWRECT, RECT_SHADOW);
 379    NV50_IR_TEX_TARG_CASE(SHADOW1D_ARRAY, 1D_ARRAY_SHADOW);
 380    NV50_IR_TEX_TARG_CASE(SHADOW2D_ARRAY, 2D_ARRAY_SHADOW);
 381    NV50_IR_TEX_TARG_CASE(SHADOWCUBE_ARRAY, CUBE_ARRAY_SHADOW);
 382    NV50_IR_TEX_TARG_CASE(BUFFER, BUFFER);
 383
 384    case TGSI_TEXTURE_UNKNOWN:
 385    default:
 386       assert(!"invalid texture target");
 387       return nv50_ir::TEX_TARGET_2D;
 388    }
 389 }
 390
 391 nv50_ir::DataType Instruction::inferSrcType() const
 392 {
 393    switch (getOpcode()) {
 394    case TGSI_OPCODE_UIF:
 395    case TGSI_OPCODE_AND:
 396    case TGSI_OPCODE_OR:
 397    case TGSI_OPCODE_XOR:
 398    case TGSI_OPCODE_NOT:
 399    case TGSI_OPCODE_U2F:
 400    case TGSI_OPCODE_UADD:
 401    case TGSI_OPCODE_UDIV:
 402    case TGSI_OPCODE_UMOD:
 403    case TGSI_OPCODE_UMAD:
 404    case TGSI_OPCODE_UMUL:
 405    case TGSI_OPCODE_UMUL_HI:
 406    case TGSI_OPCODE_UMAX:
 407    case TGSI_OPCODE_UMIN:
 408    case TGSI_OPCODE_USEQ:
 409    case TGSI_OPCODE_USGE:
 410    case TGSI_OPCODE_USLT:
 411    case TGSI_OPCODE_USNE:
 412    case TGSI_OPCODE_USHR:
 413    case TGSI_OPCODE_UCMP:
 414    case TGSI_OPCODE_ATOMUADD:
 415    case TGSI_OPCODE_ATOMXCHG:
 416    case TGSI_OPCODE_ATOMCAS:
 417    case TGSI_OPCODE_ATOMAND:
 418    case TGSI_OPCODE_ATOMOR:
 419    case TGSI_OPCODE_ATOMXOR:
 420    case TGSI_OPCODE_ATOMUMIN:
 421    case TGSI_OPCODE_ATOMUMAX:
 422    case TGSI_OPCODE_UBFE:
 423    case TGSI_OPCODE_UMSB:
 424       return nv50_ir::TYPE_U32;
 425    case TGSI_OPCODE_I2F:
 426    case TGSI_OPCODE_IDIV:
 427    case TGSI_OPCODE_IMUL_HI:
 428    case TGSI_OPCODE_IMAX:
 429    case TGSI_OPCODE_IMIN:
 430    case TGSI_OPCODE_IABS:
 431    case TGSI_OPCODE_INEG:
 432    case TGSI_OPCODE_ISGE:
 433    case TGSI_OPCODE_ISHR:
 434    case TGSI_OPCODE_ISLT:
 435    case TGSI_OPCODE_ISSG:
 436    case TGSI_OPCODE_SAD: // not sure about SAD, but no one has a float version
 437    case TGSI_OPCODE_MOD:
 438    case TGSI_OPCODE_UARL:
 439    case TGSI_OPCODE_ATOMIMIN:
 440    case TGSI_OPCODE_ATOMIMAX:
 441    case TGSI_OPCODE_IBFE:
 442    case TGSI_OPCODE_IMSB:
 443       return nv50_ir::TYPE_S32;
 444    default:
 445       return nv50_ir::TYPE_F32;
 446    }
 447 }
 448
 449 nv50_ir::DataType Instruction::inferDstType() const
 450 {
 451    switch (getOpcode()) {
 452    case TGSI_OPCODE_F2U: return nv50_ir::TYPE_U32;
 453    case TGSI_OPCODE_F2I: return nv50_ir::TYPE_S32;
 454    case TGSI_OPCODE_FSEQ:
 455    case TGSI_OPCODE_FSGE:
 456    case TGSI_OPCODE_FSLT:
 457    case TGSI_OPCODE_FSNE:
 458       return nv50_ir::TYPE_U32;
 459    case TGSI_OPCODE_I2F:
 460    case TGSI_OPCODE_U2F:
 461       return nv50_ir::TYPE_F32;
 462    default:
 463       return inferSrcType();
 464    }
 465 }
 466
 467 nv50_ir::CondCode Instruction::getSetCond() const
 468 {
 469    using namespace nv50_ir;
 470
 471    switch (getOpcode()) {
 472    case TGSI_OPCODE_SLT:
 473    case TGSI_OPCODE_ISLT:
 474    case TGSI_OPCODE_USLT:
 475    case TGSI_OPCODE_FSLT:
 476       return CC_LT;
 477    case TGSI_OPCODE_SLE:
 478       return CC_LE;
 479    case TGSI_OPCODE_SGE:
 480    case TGSI_OPCODE_ISGE:
 481    case TGSI_OPCODE_USGE:
 482    case TGSI_OPCODE_FSGE:
 483       return CC_GE;
 484    case TGSI_OPCODE_SGT:
 485       return CC_GT;
 486    case TGSI_OPCODE_SEQ:
 487    case TGSI_OPCODE_USEQ:
 488    case TGSI_OPCODE_FSEQ:
 489       return CC_EQ;
 490    case TGSI_OPCODE_SNE:
 491    case TGSI_OPCODE_FSNE:
 492       return CC_NEU;
 493    case TGSI_OPCODE_USNE:
 494       return CC_NE;
 495    case TGSI_OPCODE_SFL:
 496       return CC_NEVER;
 497    case TGSI_OPCODE_STR:
 498    default:
 499       return CC_ALWAYS;
 500    }
 501 }
 502
 503 #define NV50_IR_OPCODE_CASE(a, b) case TGSI_OPCODE_##a: return nv50_ir::OP_##b
 504
 505 static nv50_ir::operation translateOpcode(uint opcode)
 506 {
 507    switch (opcode) {
 508    NV50_IR_OPCODE_CASE(ARL, SHL);
 509    NV50_IR_OPCODE_CASE(MOV, MOV);
 510
 511    NV50_IR_OPCODE_CASE(RCP, RCP);
 512    NV50_IR_OPCODE_CASE(RSQ, RSQ);
 513
 514    NV50_IR_OPCODE_CASE(MUL, MUL);
 515    NV50_IR_OPCODE_CASE(ADD, ADD);
 516
 517    NV50_IR_OPCODE_CASE(MIN, MIN);
 518    NV50_IR_OPCODE_CASE(MAX, MAX);
 519    NV50_IR_OPCODE_CASE(SLT, SET);
 520    NV50_IR_OPCODE_CASE(SGE, SET);
 521    NV50_IR_OPCODE_CASE(MAD, MAD);
 522    NV50_IR_OPCODE_CASE(SUB, SUB);
 523
 524    NV50_IR_OPCODE_CASE(FLR, FLOOR);
 525    NV50_IR_OPCODE_CASE(ROUND, CVT);
 526    NV50_IR_OPCODE_CASE(EX2, EX2);
 527    NV50_IR_OPCODE_CASE(LG2, LG2);
 528    NV50_IR_OPCODE_CASE(POW, POW);
 529
 530    NV50_IR_OPCODE_CASE(ABS, ABS);
 531
 532    NV50_IR_OPCODE_CASE(COS, COS);
 533    NV50_IR_OPCODE_CASE(DDX, DFDX);
 534    NV50_IR_OPCODE_CASE(DDX_FINE, DFDX);
 535    NV50_IR_OPCODE_CASE(DDY, DFDY);
 536    NV50_IR_OPCODE_CASE(DDY_FINE, DFDY);
 537    NV50_IR_OPCODE_CASE(KILL, DISCARD);
 538
 539    NV50_IR_OPCODE_CASE(SEQ, SET);
 540    NV50_IR_OPCODE_CASE(SFL, SET);
 541    NV50_IR_OPCODE_CASE(SGT, SET);
 542    NV50_IR_OPCODE_CASE(SIN, SIN);
 543    NV50_IR_OPCODE_CASE(SLE, SET);
 544    NV50_IR_OPCODE_CASE(SNE, SET);
 545    NV50_IR_OPCODE_CASE(STR, SET);
 546    NV50_IR_OPCODE_CASE(TEX, TEX);
 547    NV50_IR_OPCODE_CASE(TXD, TXD);
 548    NV50_IR_OPCODE_CASE(TXP, TEX);
 549
 550    NV50_IR_OPCODE_CASE(BRA, BRA);
 551    NV50_IR_OPCODE_CASE(CAL, CALL);
 552    NV50_IR_OPCODE_CASE(RET, RET);
 553    NV50_IR_OPCODE_CASE(CMP, SLCT);
 554
 555    NV50_IR_OPCODE_CASE(TXB, TXB);
 556
 557    NV50_IR_OPCODE_CASE(DIV, DIV);
 558
 559    NV50_IR_OPCODE_CASE(TXL, TXL);
 560
 561    NV50_IR_OPCODE_CASE(CEIL, CEIL);
 562    NV50_IR_OPCODE_CASE(I2F, CVT);
 563    NV50_IR_OPCODE_CASE(NOT, NOT);
 564    NV50_IR_OPCODE_CASE(TRUNC, TRUNC);
 565    NV50_IR_OPCODE_CASE(SHL, SHL);
 566
 567    NV50_IR_OPCODE_CASE(AND, AND);
 568    NV50_IR_OPCODE_CASE(OR, OR);
 569    NV50_IR_OPCODE_CASE(MOD, MOD);
 570    NV50_IR_OPCODE_CASE(XOR, XOR);
 571    NV50_IR_OPCODE_CASE(SAD, SAD);
 572    NV50_IR_OPCODE_CASE(TXF, TXF);
 573    NV50_IR_OPCODE_CASE(TXQ, TXQ);
 574    NV50_IR_OPCODE_CASE(TG4, TXG);
 575    NV50_IR_OPCODE_CASE(LODQ, TXLQ);
 576
 577    NV50_IR_OPCODE_CASE(EMIT, EMIT);
 578    NV50_IR_OPCODE_CASE(ENDPRIM, RESTART);
 579
 580    NV50_IR_OPCODE_CASE(KILL_IF, DISCARD);
 581
 582    NV50_IR_OPCODE_CASE(F2I, CVT);
 583    NV50_IR_OPCODE_CASE(FSEQ, SET);
 584    NV50_IR_OPCODE_CASE(FSGE, SET);
 585    NV50_IR_OPCODE_CASE(FSLT, SET);
 586    NV50_IR_OPCODE_CASE(FSNE, SET);
 587    NV50_IR_OPCODE_CASE(IDIV, DIV);
 588    NV50_IR_OPCODE_CASE(IMAX, MAX);
 589    NV50_IR_OPCODE_CASE(IMIN, MIN);
 590    NV50_IR_OPCODE_CASE(IABS, ABS);
 591    NV50_IR_OPCODE_CASE(INEG, NEG);
 592    NV50_IR_OPCODE_CASE(ISGE, SET);
 593    NV50_IR_OPCODE_CASE(ISHR, SHR);
 594    NV50_IR_OPCODE_CASE(ISLT, SET);
 595    NV50_IR_OPCODE_CASE(F2U, CVT);
 596    NV50_IR_OPCODE_CASE(U2F, CVT);
 597    NV50_IR_OPCODE_CASE(UADD, ADD);
 598    NV50_IR_OPCODE_CASE(UDIV, DIV);
 599    NV50_IR_OPCODE_CASE(UMAD, MAD);
 600    NV50_IR_OPCODE_CASE(UMAX, MAX);
 601    NV50_IR_OPCODE_CASE(UMIN, MIN);
 602    NV50_IR_OPCODE_CASE(UMOD, MOD);
 603    NV50_IR_OPCODE_CASE(UMUL, MUL);
 604    NV50_IR_OPCODE_CASE(USEQ, SET);
 605    NV50_IR_OPCODE_CASE(USGE, SET);
 606    NV50_IR_OPCODE_CASE(USHR, SHR);
 607    NV50_IR_OPCODE_CASE(USLT, SET);
 608    NV50_IR_OPCODE_CASE(USNE, SET);
 609
 610    NV50_IR_OPCODE_CASE(IMUL_HI, MUL);
 611    NV50_IR_OPCODE_CASE(UMUL_HI, MUL);
 612
 613    NV50_IR_OPCODE_CASE(SAMPLE, TEX);
 614    NV50_IR_OPCODE_CASE(SAMPLE_B, TXB);
 615    NV50_IR_OPCODE_CASE(SAMPLE_C, TEX);
 616    NV50_IR_OPCODE_CASE(SAMPLE_C_LZ, TEX);
 617    NV50_IR_OPCODE_CASE(SAMPLE_D, TXD);
 618    NV50_IR_OPCODE_CASE(SAMPLE_L, TXL);
 619    NV50_IR_OPCODE_CASE(SAMPLE_I, TXF);
 620    NV50_IR_OPCODE_CASE(SAMPLE_I_MS, TXF);
 621    NV50_IR_OPCODE_CASE(GATHER4, TXG);
 622    NV50_IR_OPCODE_CASE(SVIEWINFO, TXQ);
 623
 624    NV50_IR_OPCODE_CASE(ATOMUADD, ATOM);
 625    NV50_IR_OPCODE_CASE(ATOMXCHG, ATOM);
 626    NV50_IR_OPCODE_CASE(ATOMCAS, ATOM);
 627    NV50_IR_OPCODE_CASE(ATOMAND, ATOM);
 628    NV50_IR_OPCODE_CASE(ATOMOR, ATOM);
 629    NV50_IR_OPCODE_CASE(ATOMXOR, ATOM);
 630    NV50_IR_OPCODE_CASE(ATOMUMIN, ATOM);
 631    NV50_IR_OPCODE_CASE(ATOMUMAX, ATOM);
 632    NV50_IR_OPCODE_CASE(ATOMIMIN, ATOM);
 633    NV50_IR_OPCODE_CASE(ATOMIMAX, ATOM);
 634
 635    NV50_IR_OPCODE_CASE(TEX2, TEX);
 636    NV50_IR_OPCODE_CASE(TXB2, TXB);
 637    NV50_IR_OPCODE_CASE(TXL2, TXL);
 638
 639    NV50_IR_OPCODE_CASE(IBFE, EXTBF);
 640    NV50_IR_OPCODE_CASE(UBFE, EXTBF);
 641    NV50_IR_OPCODE_CASE(BFI, INSBF);
 642    NV50_IR_OPCODE_CASE(BREV, EXTBF);
 643    NV50_IR_OPCODE_CASE(POPC, POPCNT);
 644    NV50_IR_OPCODE_CASE(LSB, BFIND);
 645    NV50_IR_OPCODE_CASE(IMSB, BFIND);
 646    NV50_IR_OPCODE_CASE(UMSB, BFIND);
 647
 648    NV50_IR_OPCODE_CASE(END, EXIT);
 649
 650    default:
 651       return nv50_ir::OP_NOP;
 652    }
 653 }
 654
 655 static uint16_t opcodeToSubOp(uint opcode)
 656 {
 657    switch (opcode) {
 658    case TGSI_OPCODE_LFENCE:   return NV50_IR_SUBOP_MEMBAR(L, GL);
 659    case TGSI_OPCODE_SFENCE:   return NV50_IR_SUBOP_MEMBAR(S, GL);
 660    case TGSI_OPCODE_MFENCE:   return NV50_IR_SUBOP_MEMBAR(M, GL);
 661    case TGSI_OPCODE_ATOMUADD: return NV50_IR_SUBOP_ATOM_ADD;
 662    case TGSI_OPCODE_ATOMXCHG: return NV50_IR_SUBOP_ATOM_EXCH;
 663    case TGSI_OPCODE_ATOMCAS:  return NV50_IR_SUBOP_ATOM_CAS;
 664    case TGSI_OPCODE_ATOMAND:  return NV50_IR_SUBOP_ATOM_AND;
 665    case TGSI_OPCODE_ATOMOR:   return NV50_IR_SUBOP_ATOM_OR;
 666    case TGSI_OPCODE_ATOMXOR:  return NV50_IR_SUBOP_ATOM_XOR;
 667    case TGSI_OPCODE_ATOMUMIN: return NV50_IR_SUBOP_ATOM_MIN;
 668    case TGSI_OPCODE_ATOMIMIN: return NV50_IR_SUBOP_ATOM_MIN;
 669    case TGSI_OPCODE_ATOMUMAX: return NV50_IR_SUBOP_ATOM_MAX;
 670    case TGSI_OPCODE_ATOMIMAX: return NV50_IR_SUBOP_ATOM_MAX;
 671    case TGSI_OPCODE_IMUL_HI:
 672    case TGSI_OPCODE_UMUL_HI:
 673       return NV50_IR_SUBOP_MUL_HIGH;
 674    default:
 675       return 0;
 676    }
 677 }
 678
 679 bool Instruction::checkDstSrcAliasing() const
 680 {
 681    if (insn->Dst[0].Register.Indirect) // no danger if indirect, using memory
 682       return false;
 683
 684    for (int s = 0; s < TGSI_FULL_MAX_SRC_REGISTERS; ++s) {
 685       if (insn->Src[s].Register.File == TGSI_FILE_NULL)
 686          break;
 687       if (insn->Src[s].Register.File == insn->Dst[0].Register.File &&
 688           insn->Src[s].Register.Index == insn->Dst[0].Register.Index)
 689          return true;
 690    }
 691    return false;
 692 }
 693
 694 class Source
 695 {
 696 public:
 697    Source(struct nv50_ir_prog_info *);
 698    ~Source();
 699
 700 public:
 701    bool scanSource();
 702    unsigned fileSize(unsigned file) const { return scan.file_max[file] + 1; }
 703
 704 public:
 705    struct tgsi_shader_info scan;
 706    struct tgsi_full_instruction *insns;
 707    const struct tgsi_token *tokens;
 708    struct nv50_ir_prog_info *info;
 709
 710    nv50_ir::DynArray tempArrays;
 711    nv50_ir::DynArray immdArrays;
 712
 713    typedef nv50_ir::BuildUtil::Location Location;
 714    // these registers are per-subroutine, cannot be used for parameter passing
 715    std::set<Location> locals;
 716
 717    bool mainTempsInLMem;
 718
 719    int clipVertexOutput;
 720
 721    struct TextureView {
 722       uint8_t target; // TGSI_TEXTURE_*
 723    };
 724    std::vector<TextureView> textureViews;
 725
 726    struct Resource {
 727       uint8_t target; // TGSI_TEXTURE_*
 728       bool raw;
 729       uint8_t slot; // $surface index
 730    };
 731    std::vector<Resource> resources;
 732
 733 private:
 734    int inferSysValDirection(unsigned sn) const;
 735    bool scanDeclaration(const struct tgsi_full_declaration *);
 736    bool scanInstruction(const struct tgsi_full_instruction *);
 737    void scanProperty(const struct tgsi_full_property *);
 738    void scanImmediate(const struct tgsi_full_immediate *);
 739
 740    inline bool isEdgeFlagPassthrough(const Instruction&) const;
 741 };
 742
 743 Source::Source(struct nv50_ir_prog_info *prog) : info(prog)
 744 {
 745    tokens = (const struct tgsi_token *)info->bin.source;
 746
 747    if (prog->dbgFlags & NV50_IR_DEBUG_BASIC)
 748       tgsi_dump(tokens, 0);
 749
 750    mainTempsInLMem = FALSE;
 751 }
 752
 753 Source::~Source()
 754 {
 755    if (insns)
 756       FREE(insns);
 757
 758    if (info->immd.data)
 759       FREE(info->immd.data);
 760    if (info->immd.type)
 761       FREE(info->immd.type);
 762 }
 763
 764 bool Source::scanSource()
 765 {
 766    unsigned insnCount = 0;
 767    struct tgsi_parse_context parse;
 768
 769    tgsi_scan_shader(tokens, &scan);
 770
 771    insns = (struct tgsi_full_instruction *)MALLOC(scan.num_instructions *
 772                                                   sizeof(insns[0]));
 773    if (!insns)
 774       return false;
 775
 776    clipVertexOutput = -1;
 777
 778    textureViews.resize(scan.file_max[TGSI_FILE_SAMPLER_VIEW] + 1);
 779    resources.resize(scan.file_max[TGSI_FILE_RESOURCE] + 1);
 780
 781    info->immd.bufSize = 0;
 782
 783    info->numInputs = scan.file_max[TGSI_FILE_INPUT] + 1;
 784    info->numOutputs = scan.file_max[TGSI_FILE_OUTPUT] + 1;
 785    info->numSysVals = scan.file_max[TGSI_FILE_SYSTEM_VALUE] + 1;
 786
 787    if (info->type == PIPE_SHADER_FRAGMENT) {
 788       info->prop.fp.writesDepth = scan.writes_z;
 789       info->prop.fp.usesDiscard = scan.uses_kill;
 790    } else
 791    if (info->type == PIPE_SHADER_GEOMETRY) {
 792       info->prop.gp.instanceCount = 1; // default value
 793    }
 794
 795    info->io.viewportId = -1;
 796
 797    info->immd.data = (uint32_t *)MALLOC(scan.immediate_count * 16);
 798    info->immd.type = (ubyte *)MALLOC(scan.immediate_count * sizeof(ubyte));
 799
 800    tgsi_parse_init(&parse, tokens);
 801    while (!tgsi_parse_end_of_tokens(&parse)) {
 802       tgsi_parse_token(&parse);
 803
 804       switch (parse.FullToken.Token.Type) {
 805       case TGSI_TOKEN_TYPE_IMMEDIATE:
 806          scanImmediate(&parse.FullToken.FullImmediate);
 807          break;
 808       case TGSI_TOKEN_TYPE_DECLARATION:
 809          scanDeclaration(&parse.FullToken.FullDeclaration);
 810          break;
 811       case TGSI_TOKEN_TYPE_INSTRUCTION:
 812          insns[insnCount++] = parse.FullToken.FullInstruction;
 813          scanInstruction(&parse.FullToken.FullInstruction);
 814          break;
 815       case TGSI_TOKEN_TYPE_PROPERTY:
 816          scanProperty(&parse.FullToken.FullProperty);
 817          break;
 818       default:
 819          INFO("unknown TGSI token type: %d\n", parse.FullToken.Token.Type);
 820          break;
 821       }
 822    }
 823    tgsi_parse_free(&parse);
 824
 825    if (mainTempsInLMem)
 826       info->bin.tlsSpace += (scan.file_max[TGSI_FILE_TEMPORARY] + 1) * 16;
 827
 828    if (info->io.genUserClip > 0) {
 829       info->io.clipDistanceMask = (1 << info->io.genUserClip) - 1;
 830
 831       const unsigned int nOut = (info->io.genUserClip + 3) / 4;
 832
 833       for (unsigned int n = 0; n < nOut; ++n) {
 834          unsigned int i = info->numOutputs++;
 835          info->out[i].id = i;
 836          info->out[i].sn = TGSI_SEMANTIC_CLIPDIST;
 837          info->out[i].si = n;
 838          info->out[i].mask = info->io.clipDistanceMask >> (n * 4);
 839       }
 840    }
 841
 842    return info->assignSlots(info) == 0;
 843 }
 844
 845 void Source::scanProperty(const struct tgsi_full_property *prop)
 846 {
 847    switch (prop->Property.PropertyName) {
 848    case TGSI_PROPERTY_GS_OUTPUT_PRIM:
 849       info->prop.gp.outputPrim = prop->u[0].Data;
 850       break;
 851    case TGSI_PROPERTY_GS_INPUT_PRIM:
 852       info->prop.gp.inputPrim = prop->u[0].Data;
 853       break;
 854    case TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES:
 855       info->prop.gp.maxVertices = prop->u[0].Data;
 856       break;
 857    case TGSI_PROPERTY_GS_INVOCATIONS:
 858       info->prop.gp.instanceCount = prop->u[0].Data;
 859       break;
 860    case TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS:
 861       info->prop.fp.separateFragData = TRUE;
 862       break;
 863    case TGSI_PROPERTY_FS_COORD_ORIGIN:
 864    case TGSI_PROPERTY_FS_COORD_PIXEL_CENTER:
 865       // we don't care
 866       break;
 867    case TGSI_PROPERTY_VS_PROHIBIT_UCPS:
 868       info->io.genUserClip = -1;
 869       break;
 870    default:
 871       INFO("unhandled TGSI property %d\n", prop->Property.PropertyName);
 872       break;
 873    }
 874 }
 875
 876 void Source::scanImmediate(const struct tgsi_full_immediate *imm)
 877 {
 878    const unsigned n = info->immd.count++;
 879
 880    assert(n < scan.immediate_count);
 881
 882    for (int c = 0; c < 4; ++c)
 883       info->immd.data[n * 4 + c] = imm->u[c].Uint;
 884
 885    info->immd.type[n] = imm->Immediate.DataType;
 886 }
 887
 888 int Source::inferSysValDirection(unsigned sn) const
 889 {
 890    switch (sn) {
 891    case TGSI_SEMANTIC_INSTANCEID:
 892    case TGSI_SEMANTIC_VERTEXID:
 893       return 1;
 894    case TGSI_SEMANTIC_LAYER:
 895 #if 0
 896    case TGSI_SEMANTIC_VIEWPORTINDEX:
 897       return 0;
 898 #endif
 899    case TGSI_SEMANTIC_PRIMID:
 900       return (info->type == PIPE_SHADER_FRAGMENT) ? 1 : 0;
 901    default:
 902       return 0;
 903    }
 904 }
 905
 906 bool Source::scanDeclaration(const struct tgsi_full_declaration *decl)
 907 {
 908    unsigned i, c;
 909    unsigned sn = TGSI_SEMANTIC_GENERIC;
 910    unsigned si = 0;
 911    const unsigned first = decl->Range.First, last = decl->Range.Last;
 912
 913    if (decl->Declaration.Semantic) {
 914       sn = decl->Semantic.Name;
 915       si = decl->Semantic.Index;
 916    }
 917
 918    if (decl->Declaration.Local) {
 919       for (i = first; i <= last; ++i) {
 920          for (c = 0; c < 4; ++c) {
 921             locals.insert(
 922                Location(decl->Declaration.File, decl->Dim.Index2D, i, c));
 923          }
 924       }
 925    }
 926
 927    switch (decl->Declaration.File) {
 928    case TGSI_FILE_INPUT:
 929       if (info->type == PIPE_SHADER_VERTEX) {
 930          // all vertex attributes are equal
 931          for (i = first; i <= last; ++i) {
 932             info->in[i].sn = TGSI_SEMANTIC_GENERIC;
 933             info->in[i].si = i;
 934          }
 935       } else {
 936          for (i = first; i <= last; ++i, ++si) {
 937             info->in[i].id = i;
 938             info->in[i].sn = sn;
 939             info->in[i].si = si;
 940             if (info->type == PIPE_SHADER_FRAGMENT) {
 941                // translate interpolation mode
 942                switch (decl->Interp.Interpolate) {
 943                case TGSI_INTERPOLATE_CONSTANT:
 944                   info->in[i].flat = 1;
 945                   break;
 946                case TGSI_INTERPOLATE_COLOR:
 947                   info->in[i].sc = 1;
 948                   break;
 949                case TGSI_INTERPOLATE_LINEAR:
 950                   info->in[i].linear = 1;
 951                   break;
 952                default:
 953                   break;
 954                }
 955                if (decl->Interp.Location || info->io.sampleInterp)
 956                   info->in[i].centroid = 1;
 957             }
 958          }
 959       }
 960       break;
 961    case TGSI_FILE_OUTPUT:
 962       for (i = first; i <= last; ++i, ++si) {
 963          switch (sn) {
 964          case TGSI_SEMANTIC_POSITION:
 965             if (info->type == PIPE_SHADER_FRAGMENT)
 966                info->io.fragDepth = i;
 967             else
 968             if (clipVertexOutput < 0)
 969                clipVertexOutput = i;
 970             break;
 971          case TGSI_SEMANTIC_COLOR:
 972             if (info->type == PIPE_SHADER_FRAGMENT)
 973                info->prop.fp.numColourResults++;
 974             break;
 975          case TGSI_SEMANTIC_EDGEFLAG:
 976             info->io.edgeFlagOut = i;
 977             break;
 978          case TGSI_SEMANTIC_CLIPVERTEX:
 979             clipVertexOutput = i;
 980             break;
 981          case TGSI_SEMANTIC_CLIPDIST:
 982             info->io.clipDistanceMask |=
 983                decl->Declaration.UsageMask << (si * 4);
 984             info->io.genUserClip = -1;
 985             break;
 986          case TGSI_SEMANTIC_SAMPLEMASK:
 987             info->io.sampleMask = i;
 988             break;
 989          case TGSI_SEMANTIC_VIEWPORT_INDEX:
 990             info->io.viewportId = i;
 991             break;
 992          default:
 993             break;
 994          }
 995          info->out[i].id = i;
 996          info->out[i].sn = sn;
 997          info->out[i].si = si;
 998       }
 999       break;
1000    case TGSI_FILE_SYSTEM_VALUE:
1001       switch (sn) {
1002       case TGSI_SEMANTIC_INSTANCEID:
1003          info->io.instanceId = first;
1004          break;
1005       case TGSI_SEMANTIC_VERTEXID:
1006          info->io.vertexId = first;
1007          break;
1008       default:
1009          break;
1010       }
1011       for (i = first; i <= last; ++i, ++si) {
1012          info->sv[i].sn = sn;
1013          info->sv[i].si = si;
1014          info->sv[i].input = inferSysValDirection(sn);
1015       }
1016       break;
1017    case TGSI_FILE_RESOURCE:
1018       for (i = first; i <= last; ++i) {
1019          resources[i].target = decl->Resource.Resource;
1020          resources[i].raw = decl->Resource.Raw;
1021          resources[i].slot = i;
1022       }
1023       break;
1024    case TGSI_FILE_SAMPLER_VIEW:
1025       for (i = first; i <= last; ++i)
1026          textureViews[i].target = decl->SamplerView.Resource;
1027       break;
1028    case TGSI_FILE_NULL:
1029    case TGSI_FILE_TEMPORARY:
1030    case TGSI_FILE_ADDRESS:
1031    case TGSI_FILE_CONSTANT:
1032    case TGSI_FILE_IMMEDIATE:
1033    case TGSI_FILE_PREDICATE:
1034    case TGSI_FILE_SAMPLER:
1035       break;
1036    default:
1037       ERROR("unhandled TGSI_FILE %d\n", decl->Declaration.File);
1038       return false;
1039    }
1040    return true;
1041 }
1042
1043 inline bool Source::isEdgeFlagPassthrough(const Instruction& insn) const
1044 {
1045    return insn.getOpcode() == TGSI_OPCODE_MOV &&
1046       insn.getDst(0).getIndex(0) == info->io.edgeFlagOut &&
1047       insn.getSrc(0).getFile() == TGSI_FILE_INPUT;
1048 }
1049
1050 bool Source::scanInstruction(const struct tgsi_full_instruction *inst)
1051 {
1052    Instruction insn(inst);
1053
1054    if (insn.getOpcode() == TGSI_OPCODE_BARRIER)
1055       info->numBarriers = 1;
1056
1057    if (insn.dstCount()) {
1058       if (insn.getDst(0).getFile() == TGSI_FILE_OUTPUT) {
1059          Instruction::DstRegister dst = insn.getDst(0);
1060
1061          if (dst.isIndirect(0))
1062             for (unsigned i = 0; i < info->numOutputs; ++i)
1063                info->out[i].mask = 0xf;
1064          else
1065             info->out[dst.getIndex(0)].mask |= dst.getMask();
1066
1067          if (info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PSIZE ||
1068              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PRIMID ||
1069              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_LAYER ||
1070              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_VIEWPORT_INDEX ||
1071              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_FOG)
1072             info->out[dst.getIndex(0)].mask &= 1;
1073
1074          if (isEdgeFlagPassthrough(insn))
1075             info->io.edgeFlagIn = insn.getSrc(0).getIndex(0);
1076       } else
1077       if (insn.getDst(0).getFile() == TGSI_FILE_TEMPORARY) {
1078          if (insn.getDst(0).isIndirect(0))
1079             mainTempsInLMem = TRUE;
1080       }
1081    }
1082
1083    for (unsigned s = 0; s < insn.srcCount(); ++s) {
1084       Instruction::SrcRegister src = insn.getSrc(s);
1085       if (src.getFile() == TGSI_FILE_TEMPORARY) {
1086          if (src.isIndirect(0))
1087             mainTempsInLMem = TRUE;
1088       } else
1089       if (src.getFile() == TGSI_FILE_RESOURCE) {
1090          if (src.getIndex(0) == TGSI_RESOURCE_GLOBAL)
1091             info->io.globalAccess |= (insn.getOpcode() == TGSI_OPCODE_LOAD) ?
1092                0x1 : 0x2;
1093       }
1094       if (src.getFile() != TGSI_FILE_INPUT)
1095          continue;
1096       unsigned mask = insn.srcMask(s);
1097
1098       if (src.isIndirect(0)) {
1099          for (unsigned i = 0; i < info->numInputs; ++i)
1100             info->in[i].mask = 0xf;
1101       } else {
1102          const int i = src.getIndex(0);
1103          for (unsigned c = 0; c < 4; ++c) {
1104             if (!(mask & (1 << c)))
1105                continue;
1106             int k = src.getSwizzle(c);
1107             if (k <= TGSI_SWIZZLE_W)
1108                info->in[i].mask |= 1 << k;
1109          }
1110          switch (info->in[i].sn) {
1111          case TGSI_SEMANTIC_PSIZE:
1112          case TGSI_SEMANTIC_PRIMID:
1113          case TGSI_SEMANTIC_FOG:
1114             info->in[i].mask &= 0x1;
1115             break;
1116          case TGSI_SEMANTIC_PCOORD:
1117             info->in[i].mask &= 0x3;
1118             break;
1119          default:
1120             break;
1121          }
1122       }
1123    }
1124    return true;
1125 }
1126
1127 nv50_ir::TexInstruction::Target
1128 Instruction::getTexture(const tgsi::Source *code, int s) const
1129 {
1130    // XXX: indirect access
1131    unsigned int r;
1132
1133    switch (getSrc(s).getFile()) {
1134    case TGSI_FILE_RESOURCE:
1135       r = getSrc(s).getIndex(0);
1136       return translateTexture(code->resources.at(r).target);
1137    case TGSI_FILE_SAMPLER_VIEW:
1138       r = getSrc(s).getIndex(0);
1139       return translateTexture(code->textureViews.at(r).target);
1140    default:
1141       return translateTexture(insn->Texture.Texture);
1142    }
1143 }
1144
1145 } // namespace tgsi
1146
1147 namespace {
1148
1149 using namespace nv50_ir;
1150
1151 class Converter : public BuildUtil
1152 {
1153 public:
1154    Converter(Program *, const tgsi::Source *);
1155    ~Converter();
1156
1157    bool run();
1158
1159 private:
1160    struct Subroutine
1161    {
1162       Subroutine(Function *f) : f(f) { }
1163       Function *f;
1164       ValueMap values;
1165    };
1166
1167    Value *shiftAddress(Value *);
1168    Value *getVertexBase(int s);
1169    DataArray *getArrayForFile(unsigned file, int idx);
1170    Value *fetchSrc(int s, int c);
1171    Value *acquireDst(int d, int c);
1172    void storeDst(int d, int c, Value *);
1173
1174    Value *fetchSrc(const tgsi::Instruction::SrcRegister src, int c, Value *ptr);
1175    void storeDst(const tgsi::Instruction::DstRegister dst, int c,
1176                  Value *val, Value *ptr);
1177
1178    Value *applySrcMod(Value *, int s, int c);
1179
1180    Symbol *makeSym(uint file, int fileIndex, int idx, int c, uint32_t addr);
1181    Symbol *srcToSym(tgsi::Instruction::SrcRegister, int c);
1182    Symbol *dstToSym(tgsi::Instruction::DstRegister, int c);
1183
1184    bool handleInstruction(const struct tgsi_full_instruction *);
1185    void exportOutputs();
1186    inline Subroutine *getSubroutine(unsigned ip);
1187    inline Subroutine *getSubroutine(Function *);
1188    inline bool isEndOfSubroutine(uint ip);
1189
1190    void loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask);
1191
1192    // R,S,L,C,Dx,Dy encode TGSI sources for respective values (0xSf for auto)
1193    void setTexRS(TexInstruction *, unsigned int& s, int R, int S);
1194    void handleTEX(Value *dst0[4], int R, int S, int L, int C, int Dx, int Dy);
1195    void handleTXF(Value *dst0[4], int R, int L_M);
1196    void handleTXQ(Value *dst0[4], enum TexQuery);
1197    void handleLIT(Value *dst0[4]);
1198    void handleUserClipPlanes();
1199
1200    Symbol *getResourceBase(int r);
1201    void getResourceCoords(std::vector<Value *>&, int r, int s);
1202
1203    void handleLOAD(Value *dst0[4]);
1204    void handleSTORE();
1205    void handleATOM(Value *dst0[4], DataType, uint16_t subOp);
1206
1207    void handleINTERP(Value *dst0[4]);
1208
1209    Value *interpolate(tgsi::Instruction::SrcRegister, int c, Value *ptr);
1210
1211    void insertConvergenceOps(BasicBlock *conv, BasicBlock *fork);
1212
1213    Value *buildDot(int dim);
1214
1215    class BindArgumentsPass : public Pass {
1216    public:
1217       BindArgumentsPass(Converter &conv) : conv(conv) { }
1218
1219    private:
1220       Converter &conv;
1221       Subroutine *sub;
1222
1223       inline const Location *getValueLocation(Subroutine *, Value *);
1224
1225       template<typename T> inline void
1226       updateCallArgs(Instruction *i, void (Instruction::*setArg)(int, Value *),
1227                      T (Function::*proto));
1228
1229       template<typename T> inline void
1230       updatePrototype(BitSet *set, void (Function::*updateSet)(),
1231                       T (Function::*proto));
1232
1233    protected:
1234       bool visit(Function *);
1235       bool visit(BasicBlock *bb) { return false; }
1236    };
1237
1238 private:
1239    const struct tgsi::Source *code;
1240    const struct nv50_ir_prog_info *info;
1241
1242    struct {
1243       std::map<unsigned, Subroutine> map;
1244       Subroutine *cur;
1245    } sub;
1246
1247    uint ip; // instruction pointer
1248
1249    tgsi::Instruction tgsi;
1250
1251    DataType dstTy;
1252    DataType srcTy;
1253
1254    DataArray tData; // TGSI_FILE_TEMPORARY
1255    DataArray aData; // TGSI_FILE_ADDRESS
1256    DataArray pData; // TGSI_FILE_PREDICATE
1257    DataArray oData; // TGSI_FILE_OUTPUT (if outputs in registers)
1258
1259    Value *zero;
1260    Value *fragCoord[4];
1261    Value *clipVtx[4];
1262
1263    Value *vtxBase[5]; // base address of vertex in primitive (for TP/GP)
1264    uint8_t vtxBaseValid;
1265
1266    Stack condBBs;  // fork BB, then else clause BB
1267    Stack joinBBs;  // fork BB, for inserting join ops on ENDIF
1268    Stack loopBBs;  // loop headers
1269    Stack breakBBs; // end of / after loop
1270
1271    Value *viewport;
1272 };
1273
1274 Symbol *
1275 Converter::srcToSym(tgsi::Instruction::SrcRegister src, int c)
1276 {
1277    const int swz = src.getSwizzle(c);
1278
1279    return makeSym(src.getFile(),
1280                   src.is2D() ? src.getIndex(1) : 0,
1281                   src.isIndirect(0) ? -1 : src.getIndex(0), swz,
1282                   src.getIndex(0) * 16 + swz * 4);
1283 }
1284
1285 Symbol *
1286 Converter::dstToSym(tgsi::Instruction::DstRegister dst, int c)
1287 {
1288    return makeSym(dst.getFile(),
1289                   dst.is2D() ? dst.getIndex(1) : 0,
1290                   dst.isIndirect(0) ? -1 : dst.getIndex(0), c,
1291                   dst.getIndex(0) * 16 + c * 4);
1292 }
1293
1294 Symbol *
1295 Converter::makeSym(uint tgsiFile, int fileIdx, int idx, int c, uint32_t address)
1296 {
1297    Symbol *sym = new_Symbol(prog, tgsi::translateFile(tgsiFile));
1298
1299    sym->reg.fileIndex = fileIdx;
1300
1301    if (idx >= 0) {
1302       if (sym->reg.file == FILE_SHADER_INPUT)
1303          sym->setOffset(info->in[idx].slot[c] * 4);
1304       else
1305       if (sym->reg.file == FILE_SHADER_OUTPUT)
1306          sym->setOffset(info->out[idx].slot[c] * 4);
1307       else
1308       if (sym->reg.file == FILE_SYSTEM_VALUE)
1309          sym->setSV(tgsi::translateSysVal(info->sv[idx].sn), c);
1310       else
1311          sym->setOffset(address);
1312    } else {
1313       sym->setOffset(address);
1314    }
1315    return sym;
1316 }
1317
1318 static inline uint8_t
1319 translateInterpMode(const struct nv50_ir_varying *var, operation& op)
1320 {
1321    uint8_t mode = NV50_IR_INTERP_PERSPECTIVE;
1322
1323    if (var->flat)
1324       mode = NV50_IR_INTERP_FLAT;
1325    else
1326    if (var->linear)
1327       mode = NV50_IR_INTERP_LINEAR;
1328    else
1329    if (var->sc)
1330       mode = NV50_IR_INTERP_SC;
1331
1332    op = (mode == NV50_IR_INTERP_PERSPECTIVE || mode == NV50_IR_INTERP_SC)
1333       ? OP_PINTERP : OP_LINTERP;
1334
1335    if (var->centroid)
1336       mode |= NV50_IR_INTERP_CENTROID;
1337
1338    return mode;
1339 }
1340
1341 Value *
1342 Converter::interpolate(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
1343 {
1344    operation op;
1345
1346    // XXX: no way to know interpolation mode if we don't know what's accessed
1347    const uint8_t mode = translateInterpMode(&info->in[ptr ? 0 :
1348                                                       src.getIndex(0)], op);
1349
1350    Instruction *insn = new_Instruction(func, op, TYPE_F32);
1351
1352    insn->setDef(0, getScratch());
1353    insn->setSrc(0, srcToSym(src, c));
1354    if (op == OP_PINTERP)
1355       insn->setSrc(1, fragCoord[3]);
1356    if (ptr)
1357       insn->setIndirect(0, 0, ptr);
1358
1359    insn->setInterpolate(mode);
1360
1361    bb->insertTail(insn);
1362    return insn->getDef(0);
1363 }
1364
1365 Value *
1366 Converter::applySrcMod(Value *val, int s, int c)
1367 {
1368    Modifier m = tgsi.getSrc(s).getMod(c);
1369    DataType ty = tgsi.inferSrcType();
1370
1371    if (m & Modifier(NV50_IR_MOD_ABS))
1372       val = mkOp1v(OP_ABS, ty, getScratch(), val);
1373
1374    if (m & Modifier(NV50_IR_MOD_NEG))
1375       val = mkOp1v(OP_NEG, ty, getScratch(), val);
1376
1377    return val;
1378 }
1379
1380 Value *
1381 Converter::getVertexBase(int s)
1382 {
1383    assert(s < 5);
1384    if (!(vtxBaseValid & (1 << s))) {
1385       const int index = tgsi.getSrc(s).getIndex(1);
1386       Value *rel = NULL;
1387       if (tgsi.getSrc(s).isIndirect(1))
1388          rel = fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL);
1389       vtxBaseValid |= 1 << s;
1390       vtxBase[s] = mkOp2v(OP_PFETCH, TYPE_U32, getSSA(4, FILE_ADDRESS),
1391                           mkImm(index), rel);
1392    }
1393    return vtxBase[s];
1394 }
1395
1396 Value *
1397 Converter::fetchSrc(int s, int c)
1398 {
1399    Value *res;
1400    Value *ptr = NULL, *dimRel = NULL;
1401
1402    tgsi::Instruction::SrcRegister src = tgsi.getSrc(s);
1403
1404    if (src.isIndirect(0))
1405       ptr = fetchSrc(src.getIndirect(0), 0, NULL);
1406
1407    if (src.is2D()) {
1408       switch (src.getFile()) {
1409       case TGSI_FILE_INPUT:
1410          dimRel = getVertexBase(s);
1411          break;
1412       case TGSI_FILE_CONSTANT:
1413          // on NVC0, this is valid and c{I+J}[k] == cI[(J << 16) + k]
1414          if (src.isIndirect(1))
1415             dimRel = fetchSrc(src.getIndirect(1), 0, 0);
1416          break;
1417       default:
1418          break;
1419       }
1420    }
1421
1422    res = fetchSrc(src, c, ptr);
1423
1424    if (dimRel)
1425       res->getInsn()->setIndirect(0, 1, dimRel);
1426
1427    return applySrcMod(res, s, c);
1428 }
1429
1430 Converter::DataArray *
1431 Converter::getArrayForFile(unsigned file, int idx)
1432 {
1433    switch (file) {
1434    case TGSI_FILE_TEMPORARY:
1435       return &tData;
1436    case TGSI_FILE_PREDICATE:
1437       return &pData;
1438    case TGSI_FILE_ADDRESS:
1439       return &aData;
1440    case TGSI_FILE_OUTPUT:
1441       assert(prog->getType() == Program::TYPE_FRAGMENT);
1442       return &oData;
1443    default:
1444       assert(!"invalid/unhandled TGSI source file");
1445       return NULL;
1446    }
1447 }
1448
1449 Value *
1450 Converter::shiftAddress(Value *index)
1451 {
1452    if (!index)
1453       return NULL;
1454    return mkOp2v(OP_SHL, TYPE_U32, getSSA(4, FILE_ADDRESS), index, mkImm(4));
1455 }
1456
1457 Value *
1458 Converter::fetchSrc(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
1459 {
1460    const int idx2d = src.is2D() ? src.getIndex(1) : 0;
1461    const int idx = src.getIndex(0);
1462    const int swz = src.getSwizzle(c);
1463
1464    switch (src.getFile()) {
1465    case TGSI_FILE_IMMEDIATE:
1466       assert(!ptr);
1467       return loadImm(NULL, info->immd.data[idx * 4 + swz]);
1468    case TGSI_FILE_CONSTANT:
1469       return mkLoadv(TYPE_U32, srcToSym(src, c), shiftAddress(ptr));
1470    case TGSI_FILE_INPUT:
1471       if (prog->getType() == Program::TYPE_FRAGMENT) {
1472          // don't load masked inputs, won't be assigned a slot
1473          if (!ptr && !(info->in[idx].mask & (1 << swz)))
1474             return loadImm(NULL, swz == TGSI_SWIZZLE_W ? 1.0f : 0.0f);
1475          if (!ptr && info->in[idx].sn == TGSI_SEMANTIC_FACE)
1476             return mkOp1v(OP_RDSV, TYPE_F32, getSSA(), mkSysVal(SV_FACE, 0));
1477          return interpolate(src, c, shiftAddress(ptr));
1478       } else
1479       if (prog->getType() == Program::TYPE_GEOMETRY) {
1480          if (!ptr && info->in[idx].sn == TGSI_SEMANTIC_PRIMID)
1481             return mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_PRIMITIVE_ID, 0));
1482          // XXX: This is going to be a problem with scalar arrays, i.e. when
1483          // we cannot assume that the address is given in units of vec4.
1484          //
1485          // nv50 and nvc0 need different things here, so let the lowering
1486          // passes decide what to do with the address
1487          if (ptr)
1488             return mkLoadv(TYPE_U32, srcToSym(src, c), ptr);
1489       }
1490       return mkLoadv(TYPE_U32, srcToSym(src, c), shiftAddress(ptr));
1491    case TGSI_FILE_OUTPUT:
1492       assert(!"load from output file");
1493       return NULL;
1494    case TGSI_FILE_SYSTEM_VALUE:
1495       assert(!ptr);
1496       return mkOp1v(OP_RDSV, TYPE_U32, getSSA(), srcToSym(src, c));
1497    default:
1498       return getArrayForFile(src.getFile(), idx2d)->load(
1499          sub.cur->values, idx, swz, shiftAddress(ptr));
1500    }
1501 }
1502
1503 Value *
1504 Converter::acquireDst(int d, int c)
1505 {
1506    const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
1507    const unsigned f = dst.getFile();
1508    const int idx = dst.getIndex(0);
1509    const int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
1510
1511    if (dst.isMasked(c) || f == TGSI_FILE_RESOURCE)
1512       return NULL;
1513
1514    if (dst.isIndirect(0) ||
1515        f == TGSI_FILE_SYSTEM_VALUE ||
1516        (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT))
1517       return getScratch();
1518
1519    return getArrayForFile(f, idx2d)-> acquire(sub.cur->values, idx, c);
1520 }
1521
1522 void
1523 Converter::storeDst(int d, int c, Value *val)
1524 {
1525    const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
1526
1527    switch (tgsi.getSaturate()) {
1528    case TGSI_SAT_NONE:
1529       break;
1530    case TGSI_SAT_ZERO_ONE:
1531       mkOp1(OP_SAT, dstTy, val, val);
1532       break;
1533    case TGSI_SAT_MINUS_PLUS_ONE:
1534       mkOp2(OP_MAX, dstTy, val, val, mkImm(-1.0f));
1535       mkOp2(OP_MIN, dstTy, val, val, mkImm(+1.0f));
1536       break;
1537    default:
1538       assert(!"invalid saturation mode");
1539       break;
1540    }
1541
1542    Value *ptr = NULL;
1543    if (dst.isIndirect(0))
1544       ptr = shiftAddress(fetchSrc(dst.getIndirect(0), 0, NULL));
1545
1546    if (info->io.genUserClip > 0 &&
1547        dst.getFile() == TGSI_FILE_OUTPUT &&
1548        !dst.isIndirect(0) && dst.getIndex(0) == code->clipVertexOutput) {
1549       mkMov(clipVtx[c], val);
1550       val = clipVtx[c];
1551    }
1552
1553    storeDst(dst, c, val, ptr);
1554 }
1555
1556 void
1557 Converter::storeDst(const tgsi::Instruction::DstRegister dst, int c,
1558                     Value *val, Value *ptr)
1559 {
1560    const unsigned f = dst.getFile();
1561    const int idx = dst.getIndex(0);
1562    const int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
1563
1564    if (f == TGSI_FILE_SYSTEM_VALUE) {
1565       assert(!ptr);
1566       mkOp2(OP_WRSV, TYPE_U32, NULL, dstToSym(dst, c), val);
1567    } else
1568    if (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT) {
1569
1570       if (ptr || (info->out[idx].mask & (1 << c))) {
1571          /* Save the viewport index into a scratch register so that it can be
1572             exported at EMIT time */
1573          if (info->out[idx].sn == TGSI_SEMANTIC_VIEWPORT_INDEX &&
1574              viewport != NULL)
1575             mkOp1(OP_MOV, TYPE_U32, viewport, val);
1576          else
1577             mkStore(OP_EXPORT, TYPE_U32, dstToSym(dst, c), ptr, val);
1578       }
1579    } else
1580    if (f == TGSI_FILE_TEMPORARY ||
1581        f == TGSI_FILE_PREDICATE ||
1582        f == TGSI_FILE_ADDRESS ||
1583        f == TGSI_FILE_OUTPUT) {
1584       getArrayForFile(f, idx2d)->store(sub.cur->values, idx, c, ptr, val);
1585    } else {
1586       assert(!"invalid dst file");
1587    }
1588 }
1589
1590 #define FOR_EACH_DST_ENABLED_CHANNEL(d, chan, inst) \
1591    for (chan = 0; chan < 4; ++chan)                 \
1592       if (!inst.getDst(d).isMasked(chan))
1593
1594 Value *
1595 Converter::buildDot(int dim)
1596 {
1597    assert(dim > 0);
1598
1599    Value *src0 = fetchSrc(0, 0), *src1 = fetchSrc(1, 0);
1600    Value *dotp = getScratch();
1601
1602    mkOp2(OP_MUL, TYPE_F32, dotp, src0, src1);
1603
1604    for (int c = 1; c < dim; ++c) {
1605       src0 = fetchSrc(0, c);
1606       src1 = fetchSrc(1, c);
1607       mkOp3(OP_MAD, TYPE_F32, dotp, src0, src1, dotp);
1608    }
1609    return dotp;
1610 }
1611
1612 void
1613 Converter::insertConvergenceOps(BasicBlock *conv, BasicBlock *fork)
1614 {
1615    FlowInstruction *join = new_FlowInstruction(func, OP_JOIN, NULL);
1616    join->fixed = 1;
1617    conv->insertHead(join);
1618
1619    fork->joinAt = new_FlowInstruction(func, OP_JOINAT, conv);
1620    fork->insertBefore(fork->getExit(), fork->joinAt);
1621 }
1622
1623 void
1624 Converter::setTexRS(TexInstruction *tex, unsigned int& s, int R, int S)
1625 {
1626    unsigned rIdx = 0, sIdx = 0;
1627
1628    if (R >= 0)
1629       rIdx = tgsi.getSrc(R).getIndex(0);
1630    if (S >= 0)
1631       sIdx = tgsi.getSrc(S).getIndex(0);
1632
1633    tex->setTexture(tgsi.getTexture(code, R), rIdx, sIdx);
1634
1635    if (tgsi.getSrc(R).isIndirect(0)) {
1636       tex->tex.rIndirectSrc = s;
1637       tex->setSrc(s++, fetchSrc(tgsi.getSrc(R).getIndirect(0), 0, NULL));
1638    }
1639    if (S >= 0 && tgsi.getSrc(S).isIndirect(0)) {
1640       tex->tex.sIndirectSrc = s;
1641       tex->setSrc(s++, fetchSrc(tgsi.getSrc(S).getIndirect(0), 0, NULL));
1642    }
1643 }
1644
1645 void
1646 Converter::handleTXQ(Value *dst0[4], enum TexQuery query)
1647 {
1648    TexInstruction *tex = new_TexInstruction(func, OP_TXQ);
1649    tex->tex.query = query;
1650    unsigned int c, d;
1651
1652    for (d = 0, c = 0; c < 4; ++c) {
1653       if (!dst0[c])
1654          continue;
1655       tex->tex.mask |= 1 << c;
1656       tex->setDef(d++, dst0[c]);
1657    }
1658    tex->setSrc((c = 0), fetchSrc(0, 0)); // mip level
1659
1660    setTexRS(tex, c, 1, -1);
1661
1662    bb->insertTail(tex);
1663 }
1664
1665 void
1666 Converter::loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask)
1667 {
1668    Value *proj = fetchSrc(0, 3);
1669    Instruction *insn = proj->getUniqueInsn();
1670    int c;
1671
1672    if (insn->op == OP_PINTERP) {
1673       bb->insertTail(insn = cloneForward(func, insn));
1674       insn->op = OP_LINTERP;
1675       insn->setInterpolate(NV50_IR_INTERP_LINEAR | insn->getSampleMode());
1676       insn->setSrc(1, NULL);
1677       proj = insn->getDef(0);
1678    }
1679    proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), proj);
1680
1681    for (c = 0; c < 4; ++c) {
1682       if (!(mask & (1 << c)))
1683          continue;
1684       if ((insn = src[c]->getUniqueInsn())->op != OP_PINTERP)
1685          continue;
1686       mask &= ~(1 << c);
1687
1688       bb->insertTail(insn = cloneForward(func, insn));
1689       insn->setInterpolate(NV50_IR_INTERP_PERSPECTIVE | insn->getSampleMode());
1690       insn->setSrc(1, proj);
1691       dst[c] = insn->getDef(0);
1692    }
1693    if (!mask)
1694       return;
1695
1696    proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), fetchSrc(0, 3));
1697
1698    for (c = 0; c < 4; ++c)
1699       if (mask & (1 << c))
1700          dst[c] = mkOp2v(OP_MUL, TYPE_F32, getSSA(), src[c], proj);
1701 }
1702
1703 // order of nv50 ir sources: x y z layer lod/bias shadow
1704 // order of TGSI TEX sources: x y z layer shadow lod/bias
1705 //  lowering will finally set the hw specific order (like array first on nvc0)
1706 void
1707 Converter::handleTEX(Value *dst[4], int R, int S, int L, int C, int Dx, int Dy)
1708 {
1709    Value *val;
1710    Value *arg[4], *src[8];
1711    Value *lod = NULL, *shd = NULL;
1712    unsigned int s, c, d;
1713    TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
1714
1715    TexInstruction::Target tgt = tgsi.getTexture(code, R);
1716
1717    for (s = 0; s < tgt.getArgCount(); ++s)
1718       arg[s] = src[s] = fetchSrc(0, s);
1719
1720    if (texi->op == OP_TXL || texi->op == OP_TXB)
1721       lod = fetchSrc(L >> 4, L & 3);
1722
1723    if (C == 0x0f)
1724       C = 0x00 | MAX2(tgt.getArgCount(), 2); // guess DC src
1725
1726    if (tgsi.getOpcode() == TGSI_OPCODE_TG4 &&
1727        tgt == TEX_TARGET_CUBE_ARRAY_SHADOW)
1728       shd = fetchSrc(1, 0);
1729    else if (tgt.isShadow())
1730       shd = fetchSrc(C >> 4, C & 3);
1731
1732    if (texi->op == OP_TXD) {
1733       for (c = 0; c < tgt.getDim(); ++c) {
1734          texi->dPdx[c].set(fetchSrc(Dx >> 4, (Dx & 3) + c));
1735          texi->dPdy[c].set(fetchSrc(Dy >> 4, (Dy & 3) + c));
1736       }
1737    }
1738
1739    // cube textures don't care about projection value, it's divided out
1740    if (tgsi.getOpcode() == TGSI_OPCODE_TXP && !tgt.isCube() && !tgt.isArray()) {
1741       unsigned int n = tgt.getDim();
1742       if (shd) {
1743          arg[n] = shd;
1744          ++n;
1745          assert(tgt.getDim() == tgt.getArgCount());
1746       }
1747       loadProjTexCoords(src, arg, (1 << n) - 1);
1748       if (shd)
1749          shd = src[n - 1];
1750    }
1751
1752    if (tgt.isCube()) {
1753       for (c = 0; c < 3; ++c)
1754          src[c] = mkOp1v(OP_ABS, TYPE_F32, getSSA(), arg[c]);
1755       val = getScratch();
1756       mkOp2(OP_MAX, TYPE_F32, val, src[0], src[1]);
1757       mkOp2(OP_MAX, TYPE_F32, val, src[2], val);
1758       mkOp1(OP_RCP, TYPE_F32, val, val);
1759       for (c = 0; c < 3; ++c)
1760          src[c] = mkOp2v(OP_MUL, TYPE_F32, getSSA(), arg[c], val);
1761    }
1762
1763    for (c = 0, d = 0; c < 4; ++c) {
1764       if (dst[c]) {
1765          texi->setDef(d++, dst[c]);
1766          texi->tex.mask |= 1 << c;
1767       } else {
1768          // NOTE: maybe hook up def too, for CSE
1769       }
1770    }
1771    for (s = 0; s < tgt.getArgCount(); ++s)
1772       texi->setSrc(s, src[s]);
1773    if (lod)
1774       texi->setSrc(s++, lod);
1775    if (shd)
1776       texi->setSrc(s++, shd);
1777
1778    setTexRS(texi, s, R, S);
1779
1780    if (tgsi.getOpcode() == TGSI_OPCODE_SAMPLE_C_LZ)
1781       texi->tex.levelZero = true;
1782    if (tgsi.getOpcode() == TGSI_OPCODE_TG4 && !tgt.isShadow())
1783       texi->tex.gatherComp = tgsi.getSrc(1).getValueU32(0, info);
1784
1785    texi->tex.useOffsets = tgsi.getNumTexOffsets();
1786    for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
1787       for (c = 0; c < 3; ++c) {
1788          texi->offset[s][c].set(fetchSrc(tgsi.getTexOffset(s), c, NULL));
1789          texi->offset[s][c].setInsn(texi);
1790       }
1791    }
1792
1793    bb->insertTail(texi);
1794 }
1795
1796 // 1st source: xyz = coordinates, w = lod/sample
1797 // 2nd source: offset
1798 void
1799 Converter::handleTXF(Value *dst[4], int R, int L_M)
1800 {
1801    TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
1802    int ms;
1803    unsigned int c, d, s;
1804
1805    texi->tex.target = tgsi.getTexture(code, R);
1806
1807    ms = texi->tex.target.isMS() ? 1 : 0;
1808    texi->tex.levelZero = ms; /* MS textures don't have mip-maps */
1809
1810    for (c = 0, d = 0; c < 4; ++c) {
1811       if (dst[c]) {
1812          texi->setDef(d++, dst[c]);
1813          texi->tex.mask |= 1 << c;
1814       }
1815    }
1816    for (c = 0; c < (texi->tex.target.getArgCount() - ms); ++c)
1817       texi->setSrc(c, fetchSrc(0, c));
1818    texi->setSrc(c++, fetchSrc(L_M >> 4, L_M & 3)); // lod or ms
1819
1820    setTexRS(texi, c, R, -1);
1821
1822    texi->tex.useOffsets = tgsi.getNumTexOffsets();
1823    for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
1824       for (c = 0; c < 3; ++c) {
1825          texi->offset[s][c].set(fetchSrc(tgsi.getTexOffset(s), c, NULL));
1826          texi->offset[s][c].setInsn(texi);
1827       }
1828    }
1829
1830    bb->insertTail(texi);
1831 }
1832
1833 void
1834 Converter::handleLIT(Value *dst0[4])
1835 {
1836    Value *val0 = NULL;
1837    unsigned int mask = tgsi.getDst(0).getMask();
1838
1839    if (mask & (1 << 0))
1840       loadImm(dst0[0], 1.0f);
1841
1842    if (mask & (1 << 3))
1843       loadImm(dst0[3], 1.0f);
1844
1845    if (mask & (3 << 1)) {
1846       val0 = getScratch();
1847       mkOp2(OP_MAX, TYPE_F32, val0, fetchSrc(0, 0), zero);
1848       if (mask & (1 << 1))
1849          mkMov(dst0[1], val0);
1850    }
1851
1852    if (mask & (1 << 2)) {
1853       Value *src1 = fetchSrc(0, 1), *src3 = fetchSrc(0, 3);
1854       Value *val1 = getScratch(), *val3 = getScratch();
1855
1856       Value *pos128 = loadImm(NULL, +127.999999f);
1857       Value *neg128 = loadImm(NULL, -127.999999f);
1858
1859       mkOp2(OP_MAX, TYPE_F32, val1, src1, zero);
1860       mkOp2(OP_MAX, TYPE_F32, val3, src3, neg128);
1861       mkOp2(OP_MIN, TYPE_F32, val3, val3, pos128);
1862       mkOp2(OP_POW, TYPE_F32, val3, val1, val3);
1863
1864       mkCmp(OP_SLCT, CC_GT, TYPE_F32, dst0[2], TYPE_F32, val3, zero, val0);
1865    }
1866 }
1867
1868 static inline bool
1869 isResourceSpecial(const int r)
1870 {
1871    return (r == TGSI_RESOURCE_GLOBAL ||
1872            r == TGSI_RESOURCE_LOCAL ||
1873            r == TGSI_RESOURCE_PRIVATE ||
1874            r == TGSI_RESOURCE_INPUT);
1875 }
1876
1877 static inline bool
1878 isResourceRaw(const struct tgsi::Source *code, const int r)
1879 {
1880    return isResourceSpecial(r) || code->resources[r].raw;
1881 }
1882
1883 static inline nv50_ir::TexTarget
1884 getResourceTarget(const struct tgsi::Source *code, int r)
1885 {
1886    if (isResourceSpecial(r))
1887       return nv50_ir::TEX_TARGET_BUFFER;
1888    return tgsi::translateTexture(code->resources.at(r).target);
1889 }
1890
1891 Symbol *
1892 Converter::getResourceBase(const int r)
1893 {
1894    Symbol *sym = NULL;
1895
1896    switch (r) {
1897    case TGSI_RESOURCE_GLOBAL:
1898       sym = new_Symbol(prog, nv50_ir::FILE_MEMORY_GLOBAL, 15);
1899       break;
1900    case TGSI_RESOURCE_LOCAL:
1901       assert(prog->getType() == Program::TYPE_COMPUTE);
1902       sym = mkSymbol(nv50_ir::FILE_MEMORY_SHARED, 0, TYPE_U32,
1903                      info->prop.cp.sharedOffset);
1904       break;
1905    case TGSI_RESOURCE_PRIVATE:
1906       sym = mkSymbol(nv50_ir::FILE_MEMORY_LOCAL, 0, TYPE_U32,
1907                      info->bin.tlsSpace);
1908       break;
1909    case TGSI_RESOURCE_INPUT:
1910       assert(prog->getType() == Program::TYPE_COMPUTE);
1911       sym = mkSymbol(nv50_ir::FILE_SHADER_INPUT, 0, TYPE_U32,
1912                      info->prop.cp.inputOffset);
1913       break;
1914    default:
1915       sym = new_Symbol(prog,
1916                        nv50_ir::FILE_MEMORY_GLOBAL, code->resources.at(r).slot);
1917       break;
1918    }
1919    return sym;
1920 }
1921
1922 void
1923 Converter::getResourceCoords(std::vector<Value *> &coords, int r, int s)
1924 {
1925    const int arg =
1926       TexInstruction::Target(getResourceTarget(code, r)).getArgCount();
1927
1928    for (int c = 0; c < arg; ++c)
1929       coords.push_back(fetchSrc(s, c));
1930
1931    // NOTE: TGSI_RESOURCE_GLOBAL needs FILE_GPR; this is an nv50 quirk
1932    if (r == TGSI_RESOURCE_LOCAL ||
1933        r == TGSI_RESOURCE_PRIVATE ||
1934        r == TGSI_RESOURCE_INPUT)
1935       coords[0] = mkOp1v(OP_MOV, TYPE_U32, getScratch(4, FILE_ADDRESS),
1936                          coords[0]);
1937 }
1938
1939 static inline int
1940 partitionLoadStore(uint8_t comp[2], uint8_t size[2], uint8_t mask)
1941 {
1942    int n = 0;
1943
1944    while (mask) {
1945       if (mask & 1) {
1946          size[n]++;
1947       } else {
1948          if (size[n])
1949             comp[n = 1] = size[0] + 1;
1950          else
1951             comp[n]++;
1952       }
1953       mask >>= 1;
1954    }
1955    if (size[0] == 3) {
1956       n = 1;
1957       size[0] = (comp[0] == 1) ? 1 : 2;
1958       size[1] = 3 - size[0];
1959       comp[1] = comp[0] + size[0];
1960    }
1961    return n + 1;
1962 }
1963
1964 // For raw loads, granularity is 4 byte.
1965 // Usage of the texture read mask on OP_SULDP is not allowed.
1966 void
1967 Converter::handleLOAD(Value *dst0[4])
1968 {
1969    const int r = tgsi.getSrc(0).getIndex(0);
1970    int c;
1971    std::vector<Value *> off, src, ldv, def;
1972
1973    getResourceCoords(off, r, 1);
1974
1975    if (isResourceRaw(code, r)) {
1976       uint8_t mask = 0;
1977       uint8_t comp[2] = { 0, 0 };
1978       uint8_t size[2] = { 0, 0 };
1979
1980       Symbol *base = getResourceBase(r);
1981
1982       // determine the base and size of the at most 2 load ops
1983       for (c = 0; c < 4; ++c)
1984          if (!tgsi.getDst(0).isMasked(c))
1985             mask |= 1 << (tgsi.getSrc(0).getSwizzle(c) - TGSI_SWIZZLE_X);
1986
1987       int n = partitionLoadStore(comp, size, mask);
1988
1989       src = off;
1990
1991       def.resize(4); // index by component, the ones we need will be non-NULL
1992       for (c = 0; c < 4; ++c) {
1993          if (dst0[c] && tgsi.getSrc(0).getSwizzle(c) == (TGSI_SWIZZLE_X + c))
1994             def[c] = dst0[c];
1995          else
1996          if (mask & (1 << c))
1997             def[c] = getScratch();
1998       }
1999
2000       const bool useLd = isResourceSpecial(r) ||
2001          (info->io.nv50styleSurfaces &&
2002           code->resources[r].target == TGSI_TEXTURE_BUFFER);
2003
2004       for (int i = 0; i < n; ++i) {
2005          ldv.assign(def.begin() + comp[i], def.begin() + comp[i] + size[i]);
2006
2007          if (comp[i]) // adjust x component of source address if necessary
2008             src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
2009                             off[0], mkImm(comp[i] * 4));
2010          else
2011             src[0] = off[0];
2012
2013          if (useLd) {
2014             Instruction *ld =
2015                mkLoad(typeOfSize(size[i] * 4), ldv[0], base, src[0]);
2016             for (size_t c = 1; c < ldv.size(); ++c)
2017                ld->setDef(c, ldv[c]);
2018          } else {
2019             mkTex(OP_SULDB, getResourceTarget(code, r), code->resources[r].slot,
2020                   0, ldv, src)->dType = typeOfSize(size[i] * 4);
2021          }
2022       }
2023    } else {
2024       def.resize(4);
2025       for (c = 0; c < 4; ++c) {
2026          if (!dst0[c] || tgsi.getSrc(0).getSwizzle(c) != (TGSI_SWIZZLE_X + c))
2027             def[c] = getScratch();
2028          else
2029             def[c] = dst0[c];
2030       }
2031
2032       mkTex(OP_SULDP, getResourceTarget(code, r), code->resources[r].slot, 0,
2033             def, off);
2034    }
2035    FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2036       if (dst0[c] != def[c])
2037          mkMov(dst0[c], def[tgsi.getSrc(0).getSwizzle(c)]);
2038 }
2039
2040 // For formatted stores, the write mask on OP_SUSTP can be used.
2041 // Raw stores have to be split.
2042 void
2043 Converter::handleSTORE()
2044 {
2045    const int r = tgsi.getDst(0).getIndex(0);
2046    int c;
2047    std::vector<Value *> off, src, dummy;
2048
2049    getResourceCoords(off, r, 0);
2050    src = off;
2051    const int s = src.size();
2052
2053    if (isResourceRaw(code, r)) {
2054       uint8_t comp[2] = { 0, 0 };
2055       uint8_t size[2] = { 0, 0 };
2056
2057       int n = partitionLoadStore(comp, size, tgsi.getDst(0).getMask());
2058
2059       Symbol *base = getResourceBase(r);
2060
2061       const bool useSt = isResourceSpecial(r) ||
2062          (info->io.nv50styleSurfaces &&
2063           code->resources[r].target == TGSI_TEXTURE_BUFFER);
2064
2065       for (int i = 0; i < n; ++i) {
2066          if (comp[i]) // adjust x component of source address if necessary
2067             src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
2068                             off[0], mkImm(comp[i] * 4));
2069          else
2070             src[0] = off[0];
2071
2072          const DataType stTy = typeOfSize(size[i] * 4);
2073
2074          if (useSt) {
2075             Instruction *st =
2076                mkStore(OP_STORE, stTy, base, NULL, fetchSrc(1, comp[i]));
2077             for (c = 1; c < size[i]; ++c)
2078                st->setSrc(1 + c, fetchSrc(1, comp[i] + c));
2079             st->setIndirect(0, 0, src[0]);
2080          } else {
2081             // attach values to be stored
2082             src.resize(s + size[i]);
2083             for (c = 0; c < size[i]; ++c)
2084                src[s + c] = fetchSrc(1, comp[i] + c);
2085             mkTex(OP_SUSTB, getResourceTarget(code, r), code->resources[r].slot,
2086                   0, dummy, src)->setType(stTy);
2087          }
2088       }
2089    } else {
2090       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2091          src.push_back(fetchSrc(1, c));
2092
2093       mkTex(OP_SUSTP, getResourceTarget(code, r), code->resources[r].slot, 0,
2094             dummy, src)->tex.mask = tgsi.getDst(0).getMask();
2095    }
2096 }
2097
2098 // XXX: These only work on resources with the single-component u32/s32 formats.
2099 // Therefore the result is replicated. This might not be intended by TGSI, but
2100 // operating on more than 1 component would produce undefined results because
2101 // they do not exist.
2102 void
2103 Converter::handleATOM(Value *dst0[4], DataType ty, uint16_t subOp)
2104 {
2105    const int r = tgsi.getSrc(0).getIndex(0);
2106    std::vector<Value *> srcv;
2107    std::vector<Value *> defv;
2108    LValue *dst = getScratch();
2109
2110    getResourceCoords(srcv, r, 1);
2111
2112    if (isResourceSpecial(r)) {
2113       assert(r != TGSI_RESOURCE_INPUT);
2114       Instruction *insn;
2115       insn = mkOp2(OP_ATOM, ty, dst, getResourceBase(r), fetchSrc(2, 0));
2116       insn->subOp = subOp;
2117       if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2118          insn->setSrc(2, fetchSrc(3, 0));
2119       insn->setIndirect(0, 0, srcv.at(0));
2120    } else {
2121       operation op = isResourceRaw(code, r) ? OP_SUREDB : OP_SUREDP;
2122       TexTarget targ = getResourceTarget(code, r);
2123       int idx = code->resources[r].slot;
2124       defv.push_back(dst);
2125       srcv.push_back(fetchSrc(2, 0));
2126       if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2127          srcv.push_back(fetchSrc(3, 0));
2128       TexInstruction *tex = mkTex(op, targ, idx, 0, defv, srcv);
2129       tex->subOp = subOp;
2130       tex->tex.mask = 1;
2131       tex->setType(ty);
2132    }
2133
2134    for (int c = 0; c < 4; ++c)
2135       if (dst0[c])
2136          dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov
2137 }
2138
2139 void
2140 Converter::handleINTERP(Value *dst[4])
2141 {
2142    // Check whether the input is linear. All other attributes ignored.
2143    Instruction *insn;
2144    Value *offset = NULL, *ptr = NULL, *w = NULL;
2145    bool linear;
2146    operation op;
2147    int c, mode;
2148
2149    tgsi::Instruction::SrcRegister src = tgsi.getSrc(0);
2150    assert(src.getFile() == TGSI_FILE_INPUT);
2151
2152    if (src.isIndirect(0))
2153       ptr = fetchSrc(src.getIndirect(0), 0, NULL);
2154
2155    // XXX: no way to know interp mode if we don't know the index
2156    linear = info->in[ptr ? 0 : src.getIndex(0)].linear;
2157    if (linear) {
2158       op = OP_LINTERP;
2159       mode = NV50_IR_INTERP_LINEAR;
2160    } else {
2161       op = OP_PINTERP;
2162       mode = NV50_IR_INTERP_PERSPECTIVE;
2163    }
2164
2165    switch (tgsi.getOpcode()) {
2166    case TGSI_OPCODE_INTERP_CENTROID:
2167       mode |= NV50_IR_INTERP_CENTROID;
2168       break;
2169    case TGSI_OPCODE_INTERP_SAMPLE:
2170       insn = mkOp1(OP_PIXLD, TYPE_U32, (offset = getScratch()), fetchSrc(1, 0));
2171       insn->subOp = NV50_IR_SUBOP_PIXLD_OFFSET;
2172       mode |= NV50_IR_INTERP_OFFSET;
2173       break;
2174    case TGSI_OPCODE_INTERP_OFFSET: {
2175       // The input in src1.xy is float, but we need a single 32-bit value
2176       // where the upper and lower 16 bits are encoded in S0.12 format. We need
2177       // to clamp the input coordinates to (-0.5, 0.4375), multiply by 4096,
2178       // and then convert to s32.
2179       Value *offs[2];
2180       for (c = 0; c < 2; c++) {
2181          offs[c] = fetchSrc(1, c);
2182          mkOp2(OP_MIN, TYPE_F32, offs[c], offs[c], loadImm(NULL, 0.4375f));
2183          mkOp2(OP_MAX, TYPE_F32, offs[c], offs[c], loadImm(NULL, -0.5f));
2184          mkOp2(OP_MUL, TYPE_F32, offs[c], offs[c], loadImm(NULL, 4096.0f));
2185          mkCvt(OP_CVT, TYPE_S32, offs[c], TYPE_F32, offs[c]);
2186       }
2187       offset = mkOp3v(OP_INSBF, TYPE_U32, getScratch(),
2188                       offs[1], mkImm(0x1010), offs[0]);
2189       mode |= NV50_IR_INTERP_OFFSET;
2190       break;
2191    }
2192    }
2193
2194    if (op == OP_PINTERP) {
2195       if (offset) {
2196          w = mkOp2v(OP_RDSV, TYPE_F32, getSSA(), mkSysVal(SV_POSITION, 3), offset);
2197          mkOp1(OP_RCP, TYPE_F32, w, w);
2198       } else {
2199          w = fragCoord[3];
2200       }
2201    }
2202
2203
2204    FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2205       insn = mkOp1(op, TYPE_F32, dst[c], srcToSym(src, c));
2206       if (op == OP_PINTERP)
2207          insn->setSrc(1, w);
2208       if (ptr)
2209          insn->setIndirect(0, 0, ptr);
2210       if (offset)
2211          insn->setSrc(op == OP_PINTERP ? 2 : 1, offset);
2212
2213       insn->setInterpolate(mode);
2214    }
2215 }
2216
2217 Converter::Subroutine *
2218 Converter::getSubroutine(unsigned ip)
2219 {
2220    std::map<unsigned, Subroutine>::iterator it = sub.map.find(ip);
2221
2222    if (it == sub.map.end())
2223       it = sub.map.insert(std::make_pair(
2224               ip, Subroutine(new Function(prog, "SUB", ip)))).first;
2225
2226    return &it->second;
2227 }
2228
2229 Converter::Subroutine *
2230 Converter::getSubroutine(Function *f)
2231 {
2232    unsigned ip = f->getLabel();
2233    std::map<unsigned, Subroutine>::iterator it = sub.map.find(ip);
2234
2235    if (it == sub.map.end())
2236       it = sub.map.insert(std::make_pair(ip, Subroutine(f))).first;
2237
2238    return &it->second;
2239 }
2240
2241 bool
2242 Converter::isEndOfSubroutine(uint ip)
2243 {
2244    assert(ip < code->scan.num_instructions);
2245    tgsi::Instruction insn(&code->insns[ip]);
2246    return (insn.getOpcode() == TGSI_OPCODE_END ||
2247            insn.getOpcode() == TGSI_OPCODE_ENDSUB ||
2248            // does END occur at end of main or the very end ?
2249            insn.getOpcode() == TGSI_OPCODE_BGNSUB);
2250 }
2251
2252 bool
2253 Converter::handleInstruction(const struct tgsi_full_instruction *insn)
2254 {
2255    Instruction *geni;
2256
2257    Value *dst0[4], *rDst0[4];
2258    Value *src0, *src1, *src2, *src3;
2259    Value *val0, *val1;
2260    int c;
2261
2262    tgsi = tgsi::Instruction(insn);
2263
2264    bool useScratchDst = tgsi.checkDstSrcAliasing();
2265
2266    operation op = tgsi.getOP();
2267    dstTy = tgsi.inferDstType();
2268    srcTy = tgsi.inferSrcType();
2269
2270    unsigned int mask = tgsi.dstCount() ? tgsi.getDst(0).getMask() : 0;
2271
2272    if (tgsi.dstCount()) {
2273       for (c = 0; c < 4; ++c) {
2274          rDst0[c] = acquireDst(0, c);
2275          dst0[c] = (useScratchDst && rDst0[c]) ? getScratch() : rDst0[c];
2276       }
2277    }
2278
2279    switch (tgsi.getOpcode()) {
2280    case TGSI_OPCODE_ADD:
2281    case TGSI_OPCODE_UADD:
2282    case TGSI_OPCODE_AND:
2283    case TGSI_OPCODE_DIV:
2284    case TGSI_OPCODE_IDIV:
2285    case TGSI_OPCODE_UDIV:
2286    case TGSI_OPCODE_MAX:
2287    case TGSI_OPCODE_MIN:
2288    case TGSI_OPCODE_IMAX:
2289    case TGSI_OPCODE_IMIN:
2290    case TGSI_OPCODE_UMAX:
2291    case TGSI_OPCODE_UMIN:
2292    case TGSI_OPCODE_MOD:
2293    case TGSI_OPCODE_UMOD:
2294    case TGSI_OPCODE_MUL:
2295    case TGSI_OPCODE_UMUL:
2296    case TGSI_OPCODE_IMUL_HI:
2297    case TGSI_OPCODE_UMUL_HI:
2298    case TGSI_OPCODE_OR:
2299    case TGSI_OPCODE_SHL:
2300    case TGSI_OPCODE_ISHR:
2301    case TGSI_OPCODE_USHR:
2302    case TGSI_OPCODE_SUB:
2303    case TGSI_OPCODE_XOR:
2304       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2305          src0 = fetchSrc(0, c);
2306          src1 = fetchSrc(1, c);
2307          geni = mkOp2(op, dstTy, dst0[c], src0, src1);
2308          geni->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode());
2309       }
2310       break;
2311    case TGSI_OPCODE_MAD:
2312    case TGSI_OPCODE_UMAD:
2313    case TGSI_OPCODE_SAD:
2314       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2315          src0 = fetchSrc(0, c);
2316          src1 = fetchSrc(1, c);
2317          src2 = fetchSrc(2, c);
2318          mkOp3(op, dstTy, dst0[c], src0, src1, src2);
2319       }
2320       break;
2321    case TGSI_OPCODE_MOV:
2322    case TGSI_OPCODE_ABS:
2323    case TGSI_OPCODE_CEIL:
2324    case TGSI_OPCODE_FLR:
2325    case TGSI_OPCODE_TRUNC:
2326    case TGSI_OPCODE_RCP:
2327    case TGSI_OPCODE_IABS:
2328    case TGSI_OPCODE_INEG:
2329    case TGSI_OPCODE_NOT:
2330    case TGSI_OPCODE_DDX:
2331    case TGSI_OPCODE_DDY:
2332    case TGSI_OPCODE_DDX_FINE:
2333    case TGSI_OPCODE_DDY_FINE:
2334       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2335          mkOp1(op, dstTy, dst0[c], fetchSrc(0, c));
2336       break;
2337    case TGSI_OPCODE_RSQ:
2338       src0 = fetchSrc(0, 0);
2339       val0 = getScratch();
2340       mkOp1(OP_ABS, TYPE_F32, val0, src0);
2341       mkOp1(OP_RSQ, TYPE_F32, val0, val0);
2342       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2343          mkMov(dst0[c], val0);
2344       break;
2345    case TGSI_OPCODE_ARL:
2346       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2347          src0 = fetchSrc(0, c);
2348          mkCvt(OP_CVT, TYPE_S32, dst0[c], TYPE_F32, src0)->rnd = ROUND_M;
2349       }
2350       break;
2351    case TGSI_OPCODE_UARL:
2352       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2353          mkOp1(OP_MOV, TYPE_U32, dst0[c], fetchSrc(0, c));
2354       break;
2355    case TGSI_OPCODE_POW:
2356       val0 = mkOp2v(op, TYPE_F32, getScratch(), fetchSrc(0, 0), fetchSrc(1, 0));
2357       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2358          mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
2359       break;
2360    case TGSI_OPCODE_EX2:
2361    case TGSI_OPCODE_LG2:
2362       val0 = mkOp1(op, TYPE_F32, getScratch(), fetchSrc(0, 0))->getDef(0);
2363       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2364          mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
2365       break;
2366    case TGSI_OPCODE_COS:
2367    case TGSI_OPCODE_SIN:
2368       val0 = getScratch();
2369       if (mask & 7) {
2370          mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 0));
2371          mkOp1(op, TYPE_F32, val0, val0);
2372          for (c = 0; c < 3; ++c)
2373             if (dst0[c])
2374                mkMov(dst0[c], val0);
2375       }
2376       if (dst0[3]) {
2377          mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 3));
2378          mkOp1(op, TYPE_F32, dst0[3], val0);
2379       }
2380       break;
2381    case TGSI_OPCODE_SCS:
2382       if (mask & 3) {
2383          val0 = mkOp1v(OP_PRESIN, TYPE_F32, getSSA(), fetchSrc(0, 0));
2384          if (dst0[0])
2385             mkOp1(OP_COS, TYPE_F32, dst0[0], val0);
2386          if (dst0[1])
2387             mkOp1(OP_SIN, TYPE_F32, dst0[1], val0);
2388       }
2389       if (dst0[2])
2390          loadImm(dst0[2], 0.0f);
2391       if (dst0[3])
2392          loadImm(dst0[3], 1.0f);
2393       break;
2394    case TGSI_OPCODE_EXP:
2395       src0 = fetchSrc(0, 0);
2396       val0 = mkOp1v(OP_FLOOR, TYPE_F32, getSSA(), src0);
2397       if (dst0[1])
2398          mkOp2(OP_SUB, TYPE_F32, dst0[1], src0, val0);
2399       if (dst0[0])
2400          mkOp1(OP_EX2, TYPE_F32, dst0[0], val0);
2401       if (dst0[2])
2402          mkOp1(OP_EX2, TYPE_F32, dst0[2], src0);
2403       if (dst0[3])
2404          loadImm(dst0[3], 1.0f);
2405       break;
2406    case TGSI_OPCODE_LOG:
2407       src0 = mkOp1v(OP_ABS, TYPE_F32, getSSA(), fetchSrc(0, 0));
2408       val0 = mkOp1v(OP_LG2, TYPE_F32, dst0[2] ? dst0[2] : getSSA(), src0);
2409       if (dst0[0] || dst0[1])
2410          val1 = mkOp1v(OP_FLOOR, TYPE_F32, dst0[0] ? dst0[0] : getSSA(), val0);
2411       if (dst0[1]) {
2412          mkOp1(OP_EX2, TYPE_F32, dst0[1], val1);
2413          mkOp1(OP_RCP, TYPE_F32, dst0[1], dst0[1]);
2414          mkOp2(OP_MUL, TYPE_F32, dst0[1], dst0[1], src0);
2415       }
2416       if (dst0[3])
2417          loadImm(dst0[3], 1.0f);
2418       break;
2419    case TGSI_OPCODE_DP2:
2420       val0 = buildDot(2);
2421       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2422          mkMov(dst0[c], val0);
2423       break;
2424    case TGSI_OPCODE_DP3:
2425       val0 = buildDot(3);
2426       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2427          mkMov(dst0[c], val0);
2428       break;
2429    case TGSI_OPCODE_DP4:
2430       val0 = buildDot(4);
2431       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2432          mkMov(dst0[c], val0);
2433       break;
2434    case TGSI_OPCODE_DPH:
2435       val0 = buildDot(3);
2436       src1 = fetchSrc(1, 3);
2437       mkOp2(OP_ADD, TYPE_F32, val0, val0, src1);
2438       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2439          mkMov(dst0[c], val0);
2440       break;
2441    case TGSI_OPCODE_DST:
2442       if (dst0[0])
2443          loadImm(dst0[0], 1.0f);
2444       if (dst0[1]) {
2445          src0 = fetchSrc(0, 1);
2446          src1 = fetchSrc(1, 1);
2447          mkOp2(OP_MUL, TYPE_F32, dst0[1], src0, src1);
2448       }
2449       if (dst0[2])
2450          mkMov(dst0[2], fetchSrc(0, 2));
2451       if (dst0[3])
2452          mkMov(dst0[3], fetchSrc(1, 3));
2453       break;
2454    case TGSI_OPCODE_LRP:
2455       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2456          src0 = fetchSrc(0, c);
2457          src1 = fetchSrc(1, c);
2458          src2 = fetchSrc(2, c);
2459          mkOp3(OP_MAD, TYPE_F32, dst0[c],
2460                mkOp2v(OP_SUB, TYPE_F32, getSSA(), src1, src2), src0, src2);
2461       }
2462       break;
2463    case TGSI_OPCODE_LIT:
2464       handleLIT(dst0);
2465       break;
2466    case TGSI_OPCODE_XPD:
2467       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2468          if (c < 3) {
2469             val0 = getSSA();
2470             src0 = fetchSrc(1, (c + 1) % 3);
2471             src1 = fetchSrc(0, (c + 2) % 3);
2472             mkOp2(OP_MUL, TYPE_F32, val0, src0, src1);
2473             mkOp1(OP_NEG, TYPE_F32, val0, val0);
2474
2475             src0 = fetchSrc(0, (c + 1) % 3);
2476             src1 = fetchSrc(1, (c + 2) % 3);
2477             mkOp3(OP_MAD, TYPE_F32, dst0[c], src0, src1, val0);
2478          } else {
2479             loadImm(dst0[c], 1.0f);
2480          }
2481       }
2482       break;
2483    case TGSI_OPCODE_ISSG:
2484    case TGSI_OPCODE_SSG:
2485       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2486          src0 = fetchSrc(0, c);
2487          val0 = getScratch();
2488          val1 = getScratch();
2489          mkCmp(OP_SET, CC_GT, srcTy, val0, srcTy, src0, zero);
2490          mkCmp(OP_SET, CC_LT, srcTy, val1, srcTy, src0, zero);
2491          if (srcTy == TYPE_F32)
2492             mkOp2(OP_SUB, TYPE_F32, dst0[c], val0, val1);
2493          else
2494             mkOp2(OP_SUB, TYPE_S32, dst0[c], val1, val0);
2495       }
2496       break;
2497    case TGSI_OPCODE_UCMP:
2498    case TGSI_OPCODE_CMP:
2499       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2500          src0 = fetchSrc(0, c);
2501          src1 = fetchSrc(1, c);
2502          src2 = fetchSrc(2, c);
2503          if (src1 == src2)
2504             mkMov(dst0[c], src1);
2505          else
2506             mkCmp(OP_SLCT, (srcTy == TYPE_F32) ? CC_LT : CC_NE,
2507                   srcTy, dst0[c], srcTy, src1, src2, src0);
2508       }
2509       break;
2510    case TGSI_OPCODE_FRC:
2511       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2512          src0 = fetchSrc(0, c);
2513          val0 = getScratch();
2514          mkOp1(OP_FLOOR, TYPE_F32, val0, src0);
2515          mkOp2(OP_SUB, TYPE_F32, dst0[c], src0, val0);
2516       }
2517       break;
2518    case TGSI_OPCODE_ROUND:
2519       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2520          mkCvt(OP_CVT, TYPE_F32, dst0[c], TYPE_F32, fetchSrc(0, c))
2521          ->rnd = ROUND_NI;
2522       break;
2523    case TGSI_OPCODE_CLAMP:
2524       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2525          src0 = fetchSrc(0, c);
2526          src1 = fetchSrc(1, c);
2527          src2 = fetchSrc(2, c);
2528          val0 = getScratch();
2529          mkOp2(OP_MIN, TYPE_F32, val0, src0, src1);
2530          mkOp2(OP_MAX, TYPE_F32, dst0[c], val0, src2);
2531       }
2532       break;
2533    case TGSI_OPCODE_SLT:
2534    case TGSI_OPCODE_SGE:
2535    case TGSI_OPCODE_SEQ:
2536    case TGSI_OPCODE_SFL:
2537    case TGSI_OPCODE_SGT:
2538    case TGSI_OPCODE_SLE:
2539    case TGSI_OPCODE_SNE:
2540    case TGSI_OPCODE_STR:
2541    case TGSI_OPCODE_FSEQ:
2542    case TGSI_OPCODE_FSGE:
2543    case TGSI_OPCODE_FSLT:
2544    case TGSI_OPCODE_FSNE:
2545    case TGSI_OPCODE_ISGE:
2546    case TGSI_OPCODE_ISLT:
2547    case TGSI_OPCODE_USEQ:
2548    case TGSI_OPCODE_USGE:
2549    case TGSI_OPCODE_USLT:
2550    case TGSI_OPCODE_USNE:
2551       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2552          src0 = fetchSrc(0, c);
2553          src1 = fetchSrc(1, c);
2554          mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], srcTy, src0, src1);
2555       }
2556       break;
2557    case TGSI_OPCODE_KILL_IF:
2558       val0 = new_LValue(func, FILE_PREDICATE);
2559       mask = 0;
2560       for (c = 0; c < 4; ++c) {
2561          const int s = tgsi.getSrc(0).getSwizzle(c);
2562          if (mask & (1 << s))
2563             continue;
2564          mask |= 1 << s;
2565          mkCmp(OP_SET, CC_LT, TYPE_F32, val0, TYPE_F32, fetchSrc(0, c), zero);
2566          mkOp(OP_DISCARD, TYPE_NONE, NULL)->setPredicate(CC_P, val0);
2567       }
2568       break;
2569    case TGSI_OPCODE_KILL:
2570       mkOp(OP_DISCARD, TYPE_NONE, NULL);
2571       break;
2572    case TGSI_OPCODE_TEX:
2573    case TGSI_OPCODE_TXB:
2574    case TGSI_OPCODE_TXL:
2575    case TGSI_OPCODE_TXP:
2576    case TGSI_OPCODE_LODQ:
2577       //              R  S     L     C    Dx    Dy
2578       handleTEX(dst0, 1, 1, 0x03, 0x0f, 0x00, 0x00);
2579       break;
2580    case TGSI_OPCODE_TXD:
2581       handleTEX(dst0, 3, 3, 0x03, 0x0f, 0x10, 0x20);
2582       break;
2583    case TGSI_OPCODE_TG4:
2584       handleTEX(dst0, 2, 2, 0x03, 0x0f, 0x00, 0x00);
2585       break;
2586    case TGSI_OPCODE_TEX2:
2587       handleTEX(dst0, 2, 2, 0x03, 0x10, 0x00, 0x00);
2588       break;
2589    case TGSI_OPCODE_TXB2:
2590    case TGSI_OPCODE_TXL2:
2591       handleTEX(dst0, 2, 2, 0x10, 0x0f, 0x00, 0x00);
2592       break;
2593    case TGSI_OPCODE_SAMPLE:
2594    case TGSI_OPCODE_SAMPLE_B:
2595    case TGSI_OPCODE_SAMPLE_D:
2596    case TGSI_OPCODE_SAMPLE_L:
2597    case TGSI_OPCODE_SAMPLE_C:
2598    case TGSI_OPCODE_SAMPLE_C_LZ:
2599       handleTEX(dst0, 1, 2, 0x30, 0x30, 0x30, 0x40);
2600       break;
2601    case TGSI_OPCODE_TXF:
2602       handleTXF(dst0, 1, 0x03);
2603       break;
2604    case TGSI_OPCODE_SAMPLE_I:
2605       handleTXF(dst0, 1, 0x03);
2606       break;
2607    case TGSI_OPCODE_SAMPLE_I_MS:
2608       handleTXF(dst0, 1, 0x20);
2609       break;
2610    case TGSI_OPCODE_TXQ:
2611    case TGSI_OPCODE_SVIEWINFO:
2612       handleTXQ(dst0, TXQ_DIMS);
2613       break;
2614    case TGSI_OPCODE_F2I:
2615    case TGSI_OPCODE_F2U:
2616       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2617          mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c))->rnd = ROUND_Z;
2618       break;
2619    case TGSI_OPCODE_I2F:
2620    case TGSI_OPCODE_U2F:
2621       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2622          mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c));
2623       break;
2624    case TGSI_OPCODE_EMIT:
2625       /* export the saved viewport index */
2626       if (viewport != NULL) {
2627          Symbol *vpSym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_U32,
2628                                   info->out[info->io.viewportId].slot[0] * 4);
2629          mkStore(OP_EXPORT, TYPE_U32, vpSym, NULL, viewport);
2630       }
2631       /* fallthrough */
2632    case TGSI_OPCODE_ENDPRIM:
2633    {
2634       // get vertex stream (must be immediate)
2635       unsigned int stream = tgsi.getSrc(0).getValueU32(0, info);
2636       if (stream && op == OP_RESTART)
2637          break;
2638       src0 = mkImm(stream);
2639       mkOp1(op, TYPE_U32, NULL, src0)->fixed = 1;
2640       break;
2641    }
2642    case TGSI_OPCODE_IF:
2643    case TGSI_OPCODE_UIF:
2644    {
2645       BasicBlock *ifBB = new BasicBlock(func);
2646
2647       bb->cfg.attach(&ifBB->cfg, Graph::Edge::TREE);
2648       condBBs.push(bb);
2649       joinBBs.push(bb);
2650
2651       mkFlow(OP_BRA, NULL, CC_NOT_P, fetchSrc(0, 0))->setType(srcTy);
2652
2653       setPosition(ifBB, true);
2654    }
2655       break;
2656    case TGSI_OPCODE_ELSE:
2657    {
2658       BasicBlock *elseBB = new BasicBlock(func);
2659       BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
2660
2661       forkBB->cfg.attach(&elseBB->cfg, Graph::Edge::TREE);
2662       condBBs.push(bb);
2663
2664       forkBB->getExit()->asFlow()->target.bb = elseBB;
2665       if (!bb->isTerminated())
2666          mkFlow(OP_BRA, NULL, CC_ALWAYS, NULL);
2667
2668       setPosition(elseBB, true);
2669    }
2670       break;
2671    case TGSI_OPCODE_ENDIF:
2672    {
2673       BasicBlock *convBB = new BasicBlock(func);
2674       BasicBlock *prevBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
2675       BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(joinBBs.pop().u.p);
2676
2677       if (!bb->isTerminated()) {
2678          // we only want join if none of the clauses ended with CONT/BREAK/RET
2679          if (prevBB->getExit()->op == OP_BRA && joinBBs.getSize() < 6)
2680             insertConvergenceOps(convBB, forkBB);
2681          mkFlow(OP_BRA, convBB, CC_ALWAYS, NULL);
2682          bb->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
2683       }
2684
2685       if (prevBB->getExit()->op == OP_BRA) {
2686          prevBB->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
2687          prevBB->getExit()->asFlow()->target.bb = convBB;
2688       }
2689       setPosition(convBB, true);
2690    }
2691       break;
2692    case TGSI_OPCODE_BGNLOOP:
2693    {
2694       BasicBlock *lbgnBB = new BasicBlock(func);
2695       BasicBlock *lbrkBB = new BasicBlock(func);
2696
2697       loopBBs.push(lbgnBB);
2698       breakBBs.push(lbrkBB);
2699       if (loopBBs.getSize() > func->loopNestingBound)
2700          func->loopNestingBound++;
2701
2702       mkFlow(OP_PREBREAK, lbrkBB, CC_ALWAYS, NULL);
2703
2704       bb->cfg.attach(&lbgnBB->cfg, Graph::Edge::TREE);
2705       setPosition(lbgnBB, true);
2706       mkFlow(OP_PRECONT, lbgnBB, CC_ALWAYS, NULL);
2707    }
2708       break;
2709    case TGSI_OPCODE_ENDLOOP:
2710    {
2711       BasicBlock *loopBB = reinterpret_cast<BasicBlock *>(loopBBs.pop().u.p);
2712
2713       if (!bb->isTerminated()) {
2714          mkFlow(OP_CONT, loopBB, CC_ALWAYS, NULL);
2715          bb->cfg.attach(&loopBB->cfg, Graph::Edge::BACK);
2716       }
2717       setPosition(reinterpret_cast<BasicBlock *>(breakBBs.pop().u.p), true);
2718    }
2719       break;
2720    case TGSI_OPCODE_BRK:
2721    {
2722       if (bb->isTerminated())
2723          break;
2724       BasicBlock *brkBB = reinterpret_cast<BasicBlock *>(breakBBs.peek().u.p);
2725       mkFlow(OP_BREAK, brkBB, CC_ALWAYS, NULL);
2726       bb->cfg.attach(&brkBB->cfg, Graph::Edge::CROSS);
2727    }
2728       break;
2729    case TGSI_OPCODE_CONT:
2730    {
2731       if (bb->isTerminated())
2732          break;
2733       BasicBlock *contBB = reinterpret_cast<BasicBlock *>(loopBBs.peek().u.p);
2734       mkFlow(OP_CONT, contBB, CC_ALWAYS, NULL);
2735       contBB->explicitCont = true;
2736       bb->cfg.attach(&contBB->cfg, Graph::Edge::BACK);
2737    }
2738       break;
2739    case TGSI_OPCODE_BGNSUB:
2740    {
2741       Subroutine *s = getSubroutine(ip);
2742       BasicBlock *entry = new BasicBlock(s->f);
2743       BasicBlock *leave = new BasicBlock(s->f);
2744
2745       // multiple entrypoints possible, keep the graph connected
2746       if (prog->getType() == Program::TYPE_COMPUTE)
2747          prog->main->call.attach(&s->f->call, Graph::Edge::TREE);
2748
2749       sub.cur = s;
2750       s->f->setEntry(entry);
2751       s->f->setExit(leave);
2752       setPosition(entry, true);
2753       return true;
2754    }
2755    case TGSI_OPCODE_ENDSUB:
2756    {
2757       sub.cur = getSubroutine(prog->main);
2758       setPosition(BasicBlock::get(sub.cur->f->cfg.getRoot()), true);
2759       return true;
2760    }
2761    case TGSI_OPCODE_CAL:
2762    {
2763       Subroutine *s = getSubroutine(tgsi.getLabel());
2764       mkFlow(OP_CALL, s->f, CC_ALWAYS, NULL);
2765       func->call.attach(&s->f->call, Graph::Edge::TREE);
2766       return true;
2767    }
2768    case TGSI_OPCODE_RET:
2769    {
2770       if (bb->isTerminated())
2771          return true;
2772       BasicBlock *leave = BasicBlock::get(func->cfgExit);
2773
2774       if (!isEndOfSubroutine(ip + 1)) {
2775          // insert a PRERET at the entry if this is an early return
2776          // (only needed for sharing code in the epilogue)
2777          BasicBlock *pos = getBB();
2778          setPosition(BasicBlock::get(func->cfg.getRoot()), false);
2779          mkFlow(OP_PRERET, leave, CC_ALWAYS, NULL)->fixed = 1;
2780          setPosition(pos, true);
2781       }
2782       mkFlow(OP_RET, NULL, CC_ALWAYS, NULL)->fixed = 1;
2783       bb->cfg.attach(&leave->cfg, Graph::Edge::CROSS);
2784    }
2785       break;
2786    case TGSI_OPCODE_END:
2787    {
2788       // attach and generate epilogue code
2789       BasicBlock *epilogue = BasicBlock::get(func->cfgExit);
2790       bb->cfg.attach(&epilogue->cfg, Graph::Edge::TREE);
2791       setPosition(epilogue, true);
2792       if (prog->getType() == Program::TYPE_FRAGMENT)
2793          exportOutputs();
2794       if (info->io.genUserClip > 0)
2795          handleUserClipPlanes();
2796       mkOp(OP_EXIT, TYPE_NONE, NULL)->terminator = 1;
2797    }
2798       break;
2799    case TGSI_OPCODE_SWITCH:
2800    case TGSI_OPCODE_CASE:
2801       ERROR("switch/case opcode encountered, should have been lowered\n");
2802       abort();
2803       break;
2804    case TGSI_OPCODE_LOAD:
2805       handleLOAD(dst0);
2806       break;
2807    case TGSI_OPCODE_STORE:
2808       handleSTORE();
2809       break;
2810    case TGSI_OPCODE_BARRIER:
2811       geni = mkOp2(OP_BAR, TYPE_U32, NULL, mkImm(0), mkImm(0));
2812       geni->fixed = 1;
2813       geni->subOp = NV50_IR_SUBOP_BAR_SYNC;
2814       break;
2815    case TGSI_OPCODE_MFENCE:
2816    case TGSI_OPCODE_LFENCE:
2817    case TGSI_OPCODE_SFENCE:
2818       geni = mkOp(OP_MEMBAR, TYPE_NONE, NULL);
2819       geni->fixed = 1;
2820       geni->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode());
2821       break;
2822    case TGSI_OPCODE_ATOMUADD:
2823    case TGSI_OPCODE_ATOMXCHG:
2824    case TGSI_OPCODE_ATOMCAS:
2825    case TGSI_OPCODE_ATOMAND:
2826    case TGSI_OPCODE_ATOMOR:
2827    case TGSI_OPCODE_ATOMXOR:
2828    case TGSI_OPCODE_ATOMUMIN:
2829    case TGSI_OPCODE_ATOMIMIN:
2830    case TGSI_OPCODE_ATOMUMAX:
2831    case TGSI_OPCODE_ATOMIMAX:
2832       handleATOM(dst0, dstTy, tgsi::opcodeToSubOp(tgsi.getOpcode()));
2833       break;
2834    case TGSI_OPCODE_IBFE:
2835    case TGSI_OPCODE_UBFE:
2836       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2837          src0 = fetchSrc(0, c);
2838          src1 = fetchSrc(1, c);
2839          src2 = fetchSrc(2, c);
2840          mkOp3(OP_INSBF, TYPE_U32, src1, src2, mkImm(0x808), src1);
2841          mkOp2(OP_EXTBF, dstTy, dst0[c], src0, src1);
2842       }
2843       break;
2844    case TGSI_OPCODE_BFI:
2845       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2846          src0 = fetchSrc(0, c);
2847          src1 = fetchSrc(1, c);
2848          src2 = fetchSrc(2, c);
2849          src3 = fetchSrc(3, c);
2850          mkOp3(OP_INSBF, TYPE_U32, src2, src3, mkImm(0x808), src2);
2851          mkOp3(OP_INSBF, TYPE_U32, dst0[c], src1, src2, src0);
2852       }
2853       break;
2854    case TGSI_OPCODE_LSB:
2855       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2856          src0 = fetchSrc(0, c);
2857          geni = mkOp2(OP_EXTBF, TYPE_U32, src0, src0, mkImm(0x2000));
2858          geni->subOp = NV50_IR_SUBOP_EXTBF_REV;
2859          geni = mkOp1(OP_BFIND, TYPE_U32, dst0[c], src0);
2860          geni->subOp = NV50_IR_SUBOP_BFIND_SAMT;
2861       }
2862       break;
2863    case TGSI_OPCODE_IMSB:
2864    case TGSI_OPCODE_UMSB:
2865       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2866          src0 = fetchSrc(0, c);
2867          mkOp1(OP_BFIND, srcTy, dst0[c], src0);
2868       }
2869       break;
2870    case TGSI_OPCODE_BREV:
2871       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2872          src0 = fetchSrc(0, c);
2873          geni = mkOp2(OP_EXTBF, TYPE_U32, dst0[c], src0, mkImm(0x2000));
2874          geni->subOp = NV50_IR_SUBOP_EXTBF_REV;
2875       }
2876       break;
2877    case TGSI_OPCODE_POPC:
2878       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2879          src0 = fetchSrc(0, c);
2880          mkOp2(OP_POPCNT, TYPE_U32, dst0[c], src0, src0);
2881       }
2882       break;
2883    case TGSI_OPCODE_INTERP_CENTROID:
2884    case TGSI_OPCODE_INTERP_SAMPLE:
2885    case TGSI_OPCODE_INTERP_OFFSET:
2886       handleINTERP(dst0);
2887       break;
2888    default:
2889       ERROR("unhandled TGSI opcode: %u\n", tgsi.getOpcode());
2890       assert(0);
2891       break;
2892    }
2893
2894    if (tgsi.dstCount()) {
2895       for (c = 0; c < 4; ++c) {
2896          if (!dst0[c])
2897             continue;
2898          if (dst0[c] != rDst0[c])
2899             mkMov(rDst0[c], dst0[c]);
2900          storeDst(0, c, rDst0[c]);
2901       }
2902    }
2903    vtxBaseValid = 0;
2904
2905    return true;
2906 }
2907
2908 void
2909 Converter::handleUserClipPlanes()
2910 {
2911    Value *res[8];
2912    int n, i, c;
2913
2914    for (c = 0; c < 4; ++c) {
2915       for (i = 0; i < info->io.genUserClip; ++i) {
2916          Symbol *sym = mkSymbol(FILE_MEMORY_CONST, info->io.ucpCBSlot,
2917                                 TYPE_F32, info->io.ucpBase + i * 16 + c * 4);
2918          Value *ucp = mkLoadv(TYPE_F32, sym, NULL);
2919          if (c == 0)
2920             res[i] = mkOp2v(OP_MUL, TYPE_F32, getScratch(), clipVtx[c], ucp);
2921          else
2922             mkOp3(OP_MAD, TYPE_F32, res[i], clipVtx[c], ucp, res[i]);
2923       }
2924    }
2925
2926    const int first = info->numOutputs - (info->io.genUserClip + 3) / 4;
2927
2928    for (i = 0; i < info->io.genUserClip; ++i) {
2929       n = i / 4 + first;
2930       c = i % 4;
2931       Symbol *sym =
2932          mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32, info->out[n].slot[c] * 4);
2933       mkStore(OP_EXPORT, TYPE_F32, sym, NULL, res[i]);
2934    }
2935 }
2936
2937 void
2938 Converter::exportOutputs()
2939 {
2940    for (unsigned int i = 0; i < info->numOutputs; ++i) {
2941       for (unsigned int c = 0; c < 4; ++c) {
2942          if (!oData.exists(sub.cur->values, i, c))
2943             continue;
2944          Symbol *sym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32,
2945                                 info->out[i].slot[c] * 4);
2946          Value *val = oData.load(sub.cur->values, i, c, NULL);
2947          if (val)
2948             mkStore(OP_EXPORT, TYPE_F32, sym, NULL, val);
2949       }
2950    }
2951 }
2952
2953 Converter::Converter(Program *ir, const tgsi::Source *code) : BuildUtil(ir),
2954      code(code),
2955      tgsi(NULL),
2956      tData(this), aData(this), pData(this), oData(this)
2957 {
2958    info = code->info;
2959
2960    const DataFile tFile = code->mainTempsInLMem ? FILE_MEMORY_LOCAL : FILE_GPR;
2961
2962    const unsigned tSize = code->fileSize(TGSI_FILE_TEMPORARY);
2963    const unsigned pSize = code->fileSize(TGSI_FILE_PREDICATE);
2964    const unsigned aSize = code->fileSize(TGSI_FILE_ADDRESS);
2965    const unsigned oSize = code->fileSize(TGSI_FILE_OUTPUT);
2966
2967    tData.setup(TGSI_FILE_TEMPORARY, 0, 0, tSize, 4, 4, tFile, 0);
2968    pData.setup(TGSI_FILE_PREDICATE, 0, 0, pSize, 4, 4, FILE_PREDICATE, 0);
2969    aData.setup(TGSI_FILE_ADDRESS, 0, 0, aSize, 4, 4, FILE_GPR, 0);
2970    oData.setup(TGSI_FILE_OUTPUT, 0, 0, oSize, 4, 4, FILE_GPR, 0);
2971
2972    zero = mkImm((uint32_t)0);
2973
2974    vtxBaseValid = 0;
2975 }
2976
2977 Converter::~Converter()
2978 {
2979 }
2980
2981 inline const Converter::Location *
2982 Converter::BindArgumentsPass::getValueLocation(Subroutine *s, Value *v)
2983 {
2984    ValueMap::l_iterator it = s->values.l.find(v);
2985    return it == s->values.l.end() ? NULL : &it->second;
2986 }
2987
2988 template<typename T> inline void
2989 Converter::BindArgumentsPass::updateCallArgs(
2990    Instruction *i, void (Instruction::*setArg)(int, Value *),
2991    T (Function::*proto))
2992 {
2993    Function *g = i->asFlow()->target.fn;
2994    Subroutine *subg = conv.getSubroutine(g);
2995
2996    for (unsigned a = 0; a < (g->*proto).size(); ++a) {
2997       Value *v = (g->*proto)[a].get();
2998       const Converter::Location &l = *getValueLocation(subg, v);
2999       Converter::DataArray *array = conv.getArrayForFile(l.array, l.arrayIdx);
3000
3001       (i->*setArg)(a, array->acquire(sub->values, l.i, l.c));
3002    }
3003 }
3004
3005 template<typename T> inline void
3006 Converter::BindArgumentsPass::updatePrototype(
3007    BitSet *set, void (Function::*updateSet)(), T (Function::*proto))
3008 {
3009    (func->*updateSet)();
3010
3011    for (unsigned i = 0; i < set->getSize(); ++i) {
3012       Value *v = func->getLValue(i);
3013       const Converter::Location *l = getValueLocation(sub, v);
3014
3015       // only include values with a matching TGSI register
3016       if (set->test(i) && l && !conv.code->locals.count(*l))
3017          (func->*proto).push_back(v);
3018    }
3019 }
3020
3021 bool
3022 Converter::BindArgumentsPass::visit(Function *f)
3023 {
3024    sub = conv.getSubroutine(f);
3025
3026    for (ArrayList::Iterator bi = f->allBBlocks.iterator();
3027         !bi.end(); bi.next()) {
3028       for (Instruction *i = BasicBlock::get(bi)->getFirst();
3029            i; i = i->next) {
3030          if (i->op == OP_CALL && !i->asFlow()->builtin) {
3031             updateCallArgs(i, &Instruction::setSrc, &Function::ins);
3032             updateCallArgs(i, &Instruction::setDef, &Function::outs);
3033          }
3034       }
3035    }
3036
3037    if (func == prog->main && prog->getType() != Program::TYPE_COMPUTE)
3038       return true;
3039    updatePrototype(&BasicBlock::get(f->cfg.getRoot())->liveSet,
3040                    &Function::buildLiveSets, &Function::ins);
3041    updatePrototype(&BasicBlock::get(f->cfgExit)->defSet,
3042                    &Function::buildDefSets, &Function::outs);
3043
3044    return true;
3045 }
3046
3047 bool
3048 Converter::run()
3049 {
3050    BasicBlock *entry = new BasicBlock(prog->main);
3051    BasicBlock *leave = new BasicBlock(prog->main);
3052
3053    prog->main->setEntry(entry);
3054    prog->main->setExit(leave);
3055
3056    setPosition(entry, true);
3057    sub.cur = getSubroutine(prog->main);
3058
3059    if (info->io.genUserClip > 0) {
3060       for (int c = 0; c < 4; ++c)
3061          clipVtx[c] = getScratch();
3062    }
3063
3064    if (prog->getType() == Program::TYPE_FRAGMENT) {
3065       Symbol *sv = mkSysVal(SV_POSITION, 3);
3066       fragCoord[3] = mkOp1v(OP_RDSV, TYPE_F32, getSSA(), sv);
3067       mkOp1(OP_RCP, TYPE_F32, fragCoord[3], fragCoord[3]);
3068    }
3069
3070    if (info->io.viewportId >= 0)
3071       viewport = getScratch();
3072    else
3073       viewport = NULL;
3074
3075    for (ip = 0; ip < code->scan.num_instructions; ++ip) {
3076       if (!handleInstruction(&code->insns[ip]))
3077          return false;
3078    }
3079
3080    if (!BindArgumentsPass(*this).run(prog))
3081       return false;
3082
3083    return true;
3084 }
3085
3086 } // unnamed namespace
3087
3088 namespace nv50_ir {
3089
3090 bool
3091 Program::makeFromTGSI(struct nv50_ir_prog_info *info)
3092 {
3093    tgsi::Source src(info);
3094    if (!src.scanSource())
3095       return false;
3096    tlsSize = info->bin.tlsSpace;
3097
3098    Converter builder(this, &src);
3099    return builder.run();
3100 }
3101
3102 } // namespace nv50_ir