src/gallium/drivers/nouveau/codegen/nv50_ir_from_tgsi.cpp

   1 /*
   2  * Copyright 2011 Christoph Bumiller
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 #include "tgsi/tgsi_dump.h"
  24 #include "tgsi/tgsi_scan.h"
  25 #include "tgsi/tgsi_util.h"
  26
  27 #include <set>
  28
  29 #include "codegen/nv50_ir.h"
  30 #include "codegen/nv50_ir_util.h"
  31 #include "codegen/nv50_ir_build_util.h"
  32
  33 namespace tgsi {
  34
  35 class Source;
  36
  37 static nv50_ir::operation translateOpcode(uint opcode);
  38 static nv50_ir::DataFile translateFile(uint file);
  39 static nv50_ir::TexTarget translateTexture(uint texTarg);
  40 static nv50_ir::SVSemantic translateSysVal(uint sysval);
  41
  42 class Instruction
  43 {
  44 public:
  45    Instruction(const struct tgsi_full_instruction *inst) : insn(inst) { }
  46
  47    class SrcRegister
  48    {
  49    public:
  50       SrcRegister(const struct tgsi_full_src_register *src)
  51          : reg(src->Register),
  52            fsr(src)
  53       { }
  54
  55       SrcRegister(const struct tgsi_src_register& src) : reg(src), fsr(NULL) { }
  56
  57       SrcRegister(const struct tgsi_ind_register& ind)
  58          : reg(tgsi_util_get_src_from_ind(&ind)),
  59            fsr(NULL)
  60       { }
  61
  62       struct tgsi_src_register offsetToSrc(struct tgsi_texture_offset off)
  63       {
  64          struct tgsi_src_register reg;
  65          memset(&reg, 0, sizeof(reg));
  66          reg.Index = off.Index;
  67          reg.File = off.File;
  68          reg.SwizzleX = off.SwizzleX;
  69          reg.SwizzleY = off.SwizzleY;
  70          reg.SwizzleZ = off.SwizzleZ;
  71          return reg;
  72       }
  73
  74       SrcRegister(const struct tgsi_texture_offset& off) :
  75          reg(offsetToSrc(off)),
  76          fsr(NULL)
  77       { }
  78
  79       uint getFile() const { return reg.File; }
  80
  81       bool is2D() const { return reg.Dimension; }
  82
  83       bool isIndirect(int dim) const
  84       {
  85          return (dim && fsr) ? fsr->Dimension.Indirect : reg.Indirect;
  86       }
  87
  88       int getIndex(int dim) const
  89       {
  90          return (dim && fsr) ? fsr->Dimension.Index : reg.Index;
  91       }
  92
  93       int getSwizzle(int chan) const
  94       {
  95          return tgsi_util_get_src_register_swizzle(&reg, chan);
  96       }
  97
  98       nv50_ir::Modifier getMod(int chan) const;
  99
 100       SrcRegister getIndirect(int dim) const
 101       {
 102          assert(fsr && isIndirect(dim));
 103          if (dim)
 104             return SrcRegister(fsr->DimIndirect);
 105          return SrcRegister(fsr->Indirect);
 106       }
 107
 108       uint32_t getValueU32(int c, const struct nv50_ir_prog_info *info) const
 109       {
 110          assert(reg.File == TGSI_FILE_IMMEDIATE);
 111          assert(!reg.Absolute);
 112          assert(!reg.Negate);
 113          return info->immd.data[reg.Index * 4 + getSwizzle(c)];
 114       }
 115
 116    private:
 117       const struct tgsi_src_register reg;
 118       const struct tgsi_full_src_register *fsr;
 119    };
 120
 121    class DstRegister
 122    {
 123    public:
 124       DstRegister(const struct tgsi_full_dst_register *dst)
 125          : reg(dst->Register),
 126            fdr(dst)
 127       { }
 128
 129       DstRegister(const struct tgsi_dst_register& dst) : reg(dst), fdr(NULL) { }
 130
 131       uint getFile() const { return reg.File; }
 132
 133       bool is2D() const { return reg.Dimension; }
 134
 135       bool isIndirect(int dim) const
 136       {
 137          return (dim && fdr) ? fdr->Dimension.Indirect : reg.Indirect;
 138       }
 139
 140       int getIndex(int dim) const
 141       {
 142          return (dim && fdr) ? fdr->Dimension.Dimension : reg.Index;
 143       }
 144
 145       unsigned int getMask() const { return reg.WriteMask; }
 146
 147       bool isMasked(int chan) const { return !(getMask() & (1 << chan)); }
 148
 149       SrcRegister getIndirect(int dim) const
 150       {
 151          assert(fdr && isIndirect(dim));
 152          if (dim)
 153             return SrcRegister(fdr->DimIndirect);
 154          return SrcRegister(fdr->Indirect);
 155       }
 156
 157    private:
 158       const struct tgsi_dst_register reg;
 159       const struct tgsi_full_dst_register *fdr;
 160    };
 161
 162    inline uint getOpcode() const { return insn->Instruction.Opcode; }
 163
 164    unsigned int srcCount() const { return insn->Instruction.NumSrcRegs; }
 165    unsigned int dstCount() const { return insn->Instruction.NumDstRegs; }
 166
 167    // mask of used components of source s
 168    unsigned int srcMask(unsigned int s) const;
 169
 170    SrcRegister getSrc(unsigned int s) const
 171    {
 172       assert(s < srcCount());
 173       return SrcRegister(&insn->Src[s]);
 174    }
 175
 176    DstRegister getDst(unsigned int d) const
 177    {
 178       assert(d < dstCount());
 179       return DstRegister(&insn->Dst[d]);
 180    }
 181
 182    SrcRegister getTexOffset(unsigned int i) const
 183    {
 184       assert(i < TGSI_FULL_MAX_TEX_OFFSETS);
 185       return SrcRegister(insn->TexOffsets[i]);
 186    }
 187
 188    unsigned int getNumTexOffsets() const { return insn->Texture.NumOffsets; }
 189
 190    bool checkDstSrcAliasing() const;
 191
 192    inline nv50_ir::operation getOP() const {
 193       return translateOpcode(getOpcode()); }
 194
 195    nv50_ir::DataType inferSrcType() const;
 196    nv50_ir::DataType inferDstType() const;
 197
 198    nv50_ir::CondCode getSetCond() const;
 199
 200    nv50_ir::TexInstruction::Target getTexture(const Source *, int s) const;
 201
 202    inline uint getLabel() { return insn->Label.Label; }
 203
 204    unsigned getSaturate() const { return insn->Instruction.Saturate; }
 205
 206    void print() const
 207    {
 208       tgsi_dump_instruction(insn, 1);
 209    }
 210
 211 private:
 212    const struct tgsi_full_instruction *insn;
 213 };
 214
 215 unsigned int Instruction::srcMask(unsigned int s) const
 216 {
 217    unsigned int mask = insn->Dst[0].Register.WriteMask;
 218
 219    switch (insn->Instruction.Opcode) {
 220    case TGSI_OPCODE_COS:
 221    case TGSI_OPCODE_SIN:
 222       return (mask & 0x8) | ((mask & 0x7) ? 0x1 : 0x0);
 223    case TGSI_OPCODE_DP2:
 224       return 0x3;
 225    case TGSI_OPCODE_DP3:
 226       return 0x7;
 227    case TGSI_OPCODE_DP4:
 228    case TGSI_OPCODE_DPH:
 229    case TGSI_OPCODE_KILL_IF: /* WriteMask ignored */
 230       return 0xf;
 231    case TGSI_OPCODE_DST:
 232       return mask & (s ? 0xa : 0x6);
 233    case TGSI_OPCODE_EX2:
 234    case TGSI_OPCODE_EXP:
 235    case TGSI_OPCODE_LG2:
 236    case TGSI_OPCODE_LOG:
 237    case TGSI_OPCODE_POW:
 238    case TGSI_OPCODE_RCP:
 239    case TGSI_OPCODE_RSQ:
 240    case TGSI_OPCODE_SCS:
 241       return 0x1;
 242    case TGSI_OPCODE_IF:
 243    case TGSI_OPCODE_UIF:
 244       return 0x1;
 245    case TGSI_OPCODE_LIT:
 246       return 0xb;
 247    case TGSI_OPCODE_TEX2:
 248    case TGSI_OPCODE_TXB2:
 249    case TGSI_OPCODE_TXL2:
 250       return (s == 0) ? 0xf : 0x3;
 251    case TGSI_OPCODE_TEX:
 252    case TGSI_OPCODE_TXB:
 253    case TGSI_OPCODE_TXD:
 254    case TGSI_OPCODE_TXL:
 255    case TGSI_OPCODE_TXP:
 256    case TGSI_OPCODE_LODQ:
 257    {
 258       const struct tgsi_instruction_texture *tex = &insn->Texture;
 259
 260       assert(insn->Instruction.Texture);
 261
 262       mask = 0x7;
 263       if (insn->Instruction.Opcode != TGSI_OPCODE_TEX &&
 264           insn->Instruction.Opcode != TGSI_OPCODE_TXD)
 265          mask |= 0x8; /* bias, lod or proj */
 266
 267       switch (tex->Texture) {
 268       case TGSI_TEXTURE_1D:
 269          mask &= 0x9;
 270          break;
 271       case TGSI_TEXTURE_SHADOW1D:
 272          mask &= 0xd;
 273          break;
 274       case TGSI_TEXTURE_1D_ARRAY:
 275       case TGSI_TEXTURE_2D:
 276       case TGSI_TEXTURE_RECT:
 277          mask &= 0xb;
 278          break;
 279       case TGSI_TEXTURE_CUBE_ARRAY:
 280       case TGSI_TEXTURE_SHADOW2D_ARRAY:
 281       case TGSI_TEXTURE_SHADOWCUBE:
 282       case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
 283          mask |= 0x8;
 284          break;
 285       default:
 286          break;
 287       }
 288    }
 289       return mask;
 290    case TGSI_OPCODE_XPD:
 291    {
 292       unsigned int x = 0;
 293       if (mask & 1) x |= 0x6;
 294       if (mask & 2) x |= 0x5;
 295       if (mask & 4) x |= 0x3;
 296       return x;
 297    }
 298    case TGSI_OPCODE_D2I:
 299    case TGSI_OPCODE_D2U:
 300    case TGSI_OPCODE_D2F:
 301    case TGSI_OPCODE_DSLT:
 302    case TGSI_OPCODE_DSGE:
 303    case TGSI_OPCODE_DSEQ:
 304    case TGSI_OPCODE_DSNE:
 305       switch (util_bitcount(mask)) {
 306       case 1: return 0x3;
 307       case 2: return 0xf;
 308       default:
 309          assert(!"unexpected mask");
 310          return 0xf;
 311       }
 312    case TGSI_OPCODE_I2D:
 313    case TGSI_OPCODE_U2D:
 314    case TGSI_OPCODE_F2D: {
 315       unsigned int x = 0;
 316       if ((mask & 0x3) == 0x3)
 317          x |= 1;
 318       if ((mask & 0xc) == 0xc)
 319          x |= 2;
 320       return x;
 321    }
 322    default:
 323       break;
 324    }
 325
 326    return mask;
 327 }
 328
 329 nv50_ir::Modifier Instruction::SrcRegister::getMod(int chan) const
 330 {
 331    nv50_ir::Modifier m(0);
 332
 333    if (reg.Absolute)
 334       m = m | nv50_ir::Modifier(NV50_IR_MOD_ABS);
 335    if (reg.Negate)
 336       m = m | nv50_ir::Modifier(NV50_IR_MOD_NEG);
 337    return m;
 338 }
 339
 340 static nv50_ir::DataFile translateFile(uint file)
 341 {
 342    switch (file) {
 343    case TGSI_FILE_CONSTANT:        return nv50_ir::FILE_MEMORY_CONST;
 344    case TGSI_FILE_INPUT:           return nv50_ir::FILE_SHADER_INPUT;
 345    case TGSI_FILE_OUTPUT:          return nv50_ir::FILE_SHADER_OUTPUT;
 346    case TGSI_FILE_TEMPORARY:       return nv50_ir::FILE_GPR;
 347    case TGSI_FILE_ADDRESS:         return nv50_ir::FILE_ADDRESS;
 348    case TGSI_FILE_PREDICATE:       return nv50_ir::FILE_PREDICATE;
 349    case TGSI_FILE_IMMEDIATE:       return nv50_ir::FILE_IMMEDIATE;
 350    case TGSI_FILE_SYSTEM_VALUE:    return nv50_ir::FILE_SYSTEM_VALUE;
 351    case TGSI_FILE_RESOURCE:        return nv50_ir::FILE_MEMORY_GLOBAL;
 352    case TGSI_FILE_SAMPLER:
 353    case TGSI_FILE_NULL:
 354    default:
 355       return nv50_ir::FILE_NULL;
 356    }
 357 }
 358
 359 static nv50_ir::SVSemantic translateSysVal(uint sysval)
 360 {
 361    switch (sysval) {
 362    case TGSI_SEMANTIC_FACE:       return nv50_ir::SV_FACE;
 363    case TGSI_SEMANTIC_PSIZE:      return nv50_ir::SV_POINT_SIZE;
 364    case TGSI_SEMANTIC_PRIMID:     return nv50_ir::SV_PRIMITIVE_ID;
 365    case TGSI_SEMANTIC_INSTANCEID: return nv50_ir::SV_INSTANCE_ID;
 366    case TGSI_SEMANTIC_VERTEXID:   return nv50_ir::SV_VERTEX_ID;
 367    case TGSI_SEMANTIC_GRID_SIZE:  return nv50_ir::SV_NCTAID;
 368    case TGSI_SEMANTIC_BLOCK_ID:   return nv50_ir::SV_CTAID;
 369    case TGSI_SEMANTIC_BLOCK_SIZE: return nv50_ir::SV_NTID;
 370    case TGSI_SEMANTIC_THREAD_ID:  return nv50_ir::SV_TID;
 371    case TGSI_SEMANTIC_SAMPLEID:   return nv50_ir::SV_SAMPLE_INDEX;
 372    case TGSI_SEMANTIC_SAMPLEPOS:  return nv50_ir::SV_SAMPLE_POS;
 373    case TGSI_SEMANTIC_SAMPLEMASK: return nv50_ir::SV_SAMPLE_MASK;
 374    case TGSI_SEMANTIC_INVOCATIONID: return nv50_ir::SV_INVOCATION_ID;
 375    case TGSI_SEMANTIC_TESSCOORD:  return nv50_ir::SV_TESS_COORD;
 376    case TGSI_SEMANTIC_TESSOUTER:  return nv50_ir::SV_TESS_OUTER;
 377    case TGSI_SEMANTIC_TESSINNER:  return nv50_ir::SV_TESS_INNER;
 378    case TGSI_SEMANTIC_VERTICESIN: return nv50_ir::SV_VERTEX_COUNT;
 379    default:
 380       assert(0);
 381       return nv50_ir::SV_CLOCK;
 382    }
 383 }
 384
 385 #define NV50_IR_TEX_TARG_CASE(a, b) \
 386    case TGSI_TEXTURE_##a: return nv50_ir::TEX_TARGET_##b;
 387
 388 static nv50_ir::TexTarget translateTexture(uint tex)
 389 {
 390    switch (tex) {
 391    NV50_IR_TEX_TARG_CASE(1D, 1D);
 392    NV50_IR_TEX_TARG_CASE(2D, 2D);
 393    NV50_IR_TEX_TARG_CASE(2D_MSAA, 2D_MS);
 394    NV50_IR_TEX_TARG_CASE(3D, 3D);
 395    NV50_IR_TEX_TARG_CASE(CUBE, CUBE);
 396    NV50_IR_TEX_TARG_CASE(RECT, RECT);
 397    NV50_IR_TEX_TARG_CASE(1D_ARRAY, 1D_ARRAY);
 398    NV50_IR_TEX_TARG_CASE(2D_ARRAY, 2D_ARRAY);
 399    NV50_IR_TEX_TARG_CASE(2D_ARRAY_MSAA, 2D_MS_ARRAY);
 400    NV50_IR_TEX_TARG_CASE(CUBE_ARRAY, CUBE_ARRAY);
 401    NV50_IR_TEX_TARG_CASE(SHADOW1D, 1D_SHADOW);
 402    NV50_IR_TEX_TARG_CASE(SHADOW2D, 2D_SHADOW);
 403    NV50_IR_TEX_TARG_CASE(SHADOWCUBE, CUBE_SHADOW);
 404    NV50_IR_TEX_TARG_CASE(SHADOWRECT, RECT_SHADOW);
 405    NV50_IR_TEX_TARG_CASE(SHADOW1D_ARRAY, 1D_ARRAY_SHADOW);
 406    NV50_IR_TEX_TARG_CASE(SHADOW2D_ARRAY, 2D_ARRAY_SHADOW);
 407    NV50_IR_TEX_TARG_CASE(SHADOWCUBE_ARRAY, CUBE_ARRAY_SHADOW);
 408    NV50_IR_TEX_TARG_CASE(BUFFER, BUFFER);
 409
 410    case TGSI_TEXTURE_UNKNOWN:
 411    default:
 412       assert(!"invalid texture target");
 413       return nv50_ir::TEX_TARGET_2D;
 414    }
 415 }
 416
 417 nv50_ir::DataType Instruction::inferSrcType() const
 418 {
 419    switch (getOpcode()) {
 420    case TGSI_OPCODE_UIF:
 421    case TGSI_OPCODE_AND:
 422    case TGSI_OPCODE_OR:
 423    case TGSI_OPCODE_XOR:
 424    case TGSI_OPCODE_NOT:
 425    case TGSI_OPCODE_SHL:
 426    case TGSI_OPCODE_U2F:
 427    case TGSI_OPCODE_U2D:
 428    case TGSI_OPCODE_UADD:
 429    case TGSI_OPCODE_UDIV:
 430    case TGSI_OPCODE_UMOD:
 431    case TGSI_OPCODE_UMAD:
 432    case TGSI_OPCODE_UMUL:
 433    case TGSI_OPCODE_UMUL_HI:
 434    case TGSI_OPCODE_UMAX:
 435    case TGSI_OPCODE_UMIN:
 436    case TGSI_OPCODE_USEQ:
 437    case TGSI_OPCODE_USGE:
 438    case TGSI_OPCODE_USLT:
 439    case TGSI_OPCODE_USNE:
 440    case TGSI_OPCODE_USHR:
 441    case TGSI_OPCODE_ATOMUADD:
 442    case TGSI_OPCODE_ATOMXCHG:
 443    case TGSI_OPCODE_ATOMCAS:
 444    case TGSI_OPCODE_ATOMAND:
 445    case TGSI_OPCODE_ATOMOR:
 446    case TGSI_OPCODE_ATOMXOR:
 447    case TGSI_OPCODE_ATOMUMIN:
 448    case TGSI_OPCODE_ATOMUMAX:
 449    case TGSI_OPCODE_UBFE:
 450    case TGSI_OPCODE_UMSB:
 451       return nv50_ir::TYPE_U32;
 452    case TGSI_OPCODE_I2F:
 453    case TGSI_OPCODE_I2D:
 454    case TGSI_OPCODE_IDIV:
 455    case TGSI_OPCODE_IMUL_HI:
 456    case TGSI_OPCODE_IMAX:
 457    case TGSI_OPCODE_IMIN:
 458    case TGSI_OPCODE_IABS:
 459    case TGSI_OPCODE_INEG:
 460    case TGSI_OPCODE_ISGE:
 461    case TGSI_OPCODE_ISHR:
 462    case TGSI_OPCODE_ISLT:
 463    case TGSI_OPCODE_ISSG:
 464    case TGSI_OPCODE_SAD: // not sure about SAD, but no one has a float version
 465    case TGSI_OPCODE_MOD:
 466    case TGSI_OPCODE_UARL:
 467    case TGSI_OPCODE_ATOMIMIN:
 468    case TGSI_OPCODE_ATOMIMAX:
 469    case TGSI_OPCODE_IBFE:
 470    case TGSI_OPCODE_IMSB:
 471       return nv50_ir::TYPE_S32;
 472    case TGSI_OPCODE_D2F:
 473    case TGSI_OPCODE_D2I:
 474    case TGSI_OPCODE_D2U:
 475    case TGSI_OPCODE_DABS:
 476    case TGSI_OPCODE_DNEG:
 477    case TGSI_OPCODE_DADD:
 478    case TGSI_OPCODE_DMUL:
 479    case TGSI_OPCODE_DMAX:
 480    case TGSI_OPCODE_DMIN:
 481    case TGSI_OPCODE_DSLT:
 482    case TGSI_OPCODE_DSGE:
 483    case TGSI_OPCODE_DSEQ:
 484    case TGSI_OPCODE_DSNE:
 485    case TGSI_OPCODE_DRCP:
 486    case TGSI_OPCODE_DSQRT:
 487    case TGSI_OPCODE_DMAD:
 488    case TGSI_OPCODE_DFRAC:
 489    case TGSI_OPCODE_DRSQ:
 490    case TGSI_OPCODE_DTRUNC:
 491    case TGSI_OPCODE_DCEIL:
 492    case TGSI_OPCODE_DFLR:
 493    case TGSI_OPCODE_DROUND:
 494       return nv50_ir::TYPE_F64;
 495    default:
 496       return nv50_ir::TYPE_F32;
 497    }
 498 }
 499
 500 nv50_ir::DataType Instruction::inferDstType() const
 501 {
 502    switch (getOpcode()) {
 503    case TGSI_OPCODE_D2U:
 504    case TGSI_OPCODE_F2U: return nv50_ir::TYPE_U32;
 505    case TGSI_OPCODE_D2I:
 506    case TGSI_OPCODE_F2I: return nv50_ir::TYPE_S32;
 507    case TGSI_OPCODE_FSEQ:
 508    case TGSI_OPCODE_FSGE:
 509    case TGSI_OPCODE_FSLT:
 510    case TGSI_OPCODE_FSNE:
 511    case TGSI_OPCODE_DSEQ:
 512    case TGSI_OPCODE_DSGE:
 513    case TGSI_OPCODE_DSLT:
 514    case TGSI_OPCODE_DSNE:
 515       return nv50_ir::TYPE_U32;
 516    case TGSI_OPCODE_I2F:
 517    case TGSI_OPCODE_U2F:
 518    case TGSI_OPCODE_D2F:
 519       return nv50_ir::TYPE_F32;
 520    case TGSI_OPCODE_I2D:
 521    case TGSI_OPCODE_U2D:
 522    case TGSI_OPCODE_F2D:
 523       return nv50_ir::TYPE_F64;
 524    default:
 525       return inferSrcType();
 526    }
 527 }
 528
 529 nv50_ir::CondCode Instruction::getSetCond() const
 530 {
 531    using namespace nv50_ir;
 532
 533    switch (getOpcode()) {
 534    case TGSI_OPCODE_SLT:
 535    case TGSI_OPCODE_ISLT:
 536    case TGSI_OPCODE_USLT:
 537    case TGSI_OPCODE_FSLT:
 538    case TGSI_OPCODE_DSLT:
 539       return CC_LT;
 540    case TGSI_OPCODE_SLE:
 541       return CC_LE;
 542    case TGSI_OPCODE_SGE:
 543    case TGSI_OPCODE_ISGE:
 544    case TGSI_OPCODE_USGE:
 545    case TGSI_OPCODE_FSGE:
 546    case TGSI_OPCODE_DSGE:
 547       return CC_GE;
 548    case TGSI_OPCODE_SGT:
 549       return CC_GT;
 550    case TGSI_OPCODE_SEQ:
 551    case TGSI_OPCODE_USEQ:
 552    case TGSI_OPCODE_FSEQ:
 553    case TGSI_OPCODE_DSEQ:
 554       return CC_EQ;
 555    case TGSI_OPCODE_SNE:
 556    case TGSI_OPCODE_FSNE:
 557    case TGSI_OPCODE_DSNE:
 558       return CC_NEU;
 559    case TGSI_OPCODE_USNE:
 560       return CC_NE;
 561    default:
 562       return CC_ALWAYS;
 563    }
 564 }
 565
 566 #define NV50_IR_OPCODE_CASE(a, b) case TGSI_OPCODE_##a: return nv50_ir::OP_##b
 567
 568 static nv50_ir::operation translateOpcode(uint opcode)
 569 {
 570    switch (opcode) {
 571    NV50_IR_OPCODE_CASE(ARL, SHL);
 572    NV50_IR_OPCODE_CASE(MOV, MOV);
 573
 574    NV50_IR_OPCODE_CASE(RCP, RCP);
 575    NV50_IR_OPCODE_CASE(RSQ, RSQ);
 576
 577    NV50_IR_OPCODE_CASE(MUL, MUL);
 578    NV50_IR_OPCODE_CASE(ADD, ADD);
 579
 580    NV50_IR_OPCODE_CASE(MIN, MIN);
 581    NV50_IR_OPCODE_CASE(MAX, MAX);
 582    NV50_IR_OPCODE_CASE(SLT, SET);
 583    NV50_IR_OPCODE_CASE(SGE, SET);
 584    NV50_IR_OPCODE_CASE(MAD, MAD);
 585    NV50_IR_OPCODE_CASE(SUB, SUB);
 586
 587    NV50_IR_OPCODE_CASE(FLR, FLOOR);
 588    NV50_IR_OPCODE_CASE(ROUND, CVT);
 589    NV50_IR_OPCODE_CASE(EX2, EX2);
 590    NV50_IR_OPCODE_CASE(LG2, LG2);
 591    NV50_IR_OPCODE_CASE(POW, POW);
 592
 593    NV50_IR_OPCODE_CASE(ABS, ABS);
 594
 595    NV50_IR_OPCODE_CASE(COS, COS);
 596    NV50_IR_OPCODE_CASE(DDX, DFDX);
 597    NV50_IR_OPCODE_CASE(DDX_FINE, DFDX);
 598    NV50_IR_OPCODE_CASE(DDY, DFDY);
 599    NV50_IR_OPCODE_CASE(DDY_FINE, DFDY);
 600    NV50_IR_OPCODE_CASE(KILL, DISCARD);
 601
 602    NV50_IR_OPCODE_CASE(SEQ, SET);
 603    NV50_IR_OPCODE_CASE(SGT, SET);
 604    NV50_IR_OPCODE_CASE(SIN, SIN);
 605    NV50_IR_OPCODE_CASE(SLE, SET);
 606    NV50_IR_OPCODE_CASE(SNE, SET);
 607    NV50_IR_OPCODE_CASE(TEX, TEX);
 608    NV50_IR_OPCODE_CASE(TXD, TXD);
 609    NV50_IR_OPCODE_CASE(TXP, TEX);
 610
 611    NV50_IR_OPCODE_CASE(CAL, CALL);
 612    NV50_IR_OPCODE_CASE(RET, RET);
 613    NV50_IR_OPCODE_CASE(CMP, SLCT);
 614
 615    NV50_IR_OPCODE_CASE(TXB, TXB);
 616
 617    NV50_IR_OPCODE_CASE(DIV, DIV);
 618
 619    NV50_IR_OPCODE_CASE(TXL, TXL);
 620
 621    NV50_IR_OPCODE_CASE(CEIL, CEIL);
 622    NV50_IR_OPCODE_CASE(I2F, CVT);
 623    NV50_IR_OPCODE_CASE(NOT, NOT);
 624    NV50_IR_OPCODE_CASE(TRUNC, TRUNC);
 625    NV50_IR_OPCODE_CASE(SHL, SHL);
 626
 627    NV50_IR_OPCODE_CASE(AND, AND);
 628    NV50_IR_OPCODE_CASE(OR, OR);
 629    NV50_IR_OPCODE_CASE(MOD, MOD);
 630    NV50_IR_OPCODE_CASE(XOR, XOR);
 631    NV50_IR_OPCODE_CASE(SAD, SAD);
 632    NV50_IR_OPCODE_CASE(TXF, TXF);
 633    NV50_IR_OPCODE_CASE(TXQ, TXQ);
 634    NV50_IR_OPCODE_CASE(TXQS, TXQ);
 635    NV50_IR_OPCODE_CASE(TG4, TXG);
 636    NV50_IR_OPCODE_CASE(LODQ, TXLQ);
 637
 638    NV50_IR_OPCODE_CASE(EMIT, EMIT);
 639    NV50_IR_OPCODE_CASE(ENDPRIM, RESTART);
 640
 641    NV50_IR_OPCODE_CASE(KILL_IF, DISCARD);
 642
 643    NV50_IR_OPCODE_CASE(F2I, CVT);
 644    NV50_IR_OPCODE_CASE(FSEQ, SET);
 645    NV50_IR_OPCODE_CASE(FSGE, SET);
 646    NV50_IR_OPCODE_CASE(FSLT, SET);
 647    NV50_IR_OPCODE_CASE(FSNE, SET);
 648    NV50_IR_OPCODE_CASE(IDIV, DIV);
 649    NV50_IR_OPCODE_CASE(IMAX, MAX);
 650    NV50_IR_OPCODE_CASE(IMIN, MIN);
 651    NV50_IR_OPCODE_CASE(IABS, ABS);
 652    NV50_IR_OPCODE_CASE(INEG, NEG);
 653    NV50_IR_OPCODE_CASE(ISGE, SET);
 654    NV50_IR_OPCODE_CASE(ISHR, SHR);
 655    NV50_IR_OPCODE_CASE(ISLT, SET);
 656    NV50_IR_OPCODE_CASE(F2U, CVT);
 657    NV50_IR_OPCODE_CASE(U2F, CVT);
 658    NV50_IR_OPCODE_CASE(UADD, ADD);
 659    NV50_IR_OPCODE_CASE(UDIV, DIV);
 660    NV50_IR_OPCODE_CASE(UMAD, MAD);
 661    NV50_IR_OPCODE_CASE(UMAX, MAX);
 662    NV50_IR_OPCODE_CASE(UMIN, MIN);
 663    NV50_IR_OPCODE_CASE(UMOD, MOD);
 664    NV50_IR_OPCODE_CASE(UMUL, MUL);
 665    NV50_IR_OPCODE_CASE(USEQ, SET);
 666    NV50_IR_OPCODE_CASE(USGE, SET);
 667    NV50_IR_OPCODE_CASE(USHR, SHR);
 668    NV50_IR_OPCODE_CASE(USLT, SET);
 669    NV50_IR_OPCODE_CASE(USNE, SET);
 670
 671    NV50_IR_OPCODE_CASE(DABS, ABS);
 672    NV50_IR_OPCODE_CASE(DNEG, NEG);
 673    NV50_IR_OPCODE_CASE(DADD, ADD);
 674    NV50_IR_OPCODE_CASE(DMUL, MUL);
 675    NV50_IR_OPCODE_CASE(DMAX, MAX);
 676    NV50_IR_OPCODE_CASE(DMIN, MIN);
 677    NV50_IR_OPCODE_CASE(DSLT, SET);
 678    NV50_IR_OPCODE_CASE(DSGE, SET);
 679    NV50_IR_OPCODE_CASE(DSEQ, SET);
 680    NV50_IR_OPCODE_CASE(DSNE, SET);
 681    NV50_IR_OPCODE_CASE(DRCP, RCP);
 682    NV50_IR_OPCODE_CASE(DSQRT, SQRT);
 683    NV50_IR_OPCODE_CASE(DMAD, MAD);
 684    NV50_IR_OPCODE_CASE(D2I, CVT);
 685    NV50_IR_OPCODE_CASE(D2U, CVT);
 686    NV50_IR_OPCODE_CASE(I2D, CVT);
 687    NV50_IR_OPCODE_CASE(U2D, CVT);
 688    NV50_IR_OPCODE_CASE(DRSQ, RSQ);
 689    NV50_IR_OPCODE_CASE(DTRUNC, TRUNC);
 690    NV50_IR_OPCODE_CASE(DCEIL, CEIL);
 691    NV50_IR_OPCODE_CASE(DFLR, FLOOR);
 692    NV50_IR_OPCODE_CASE(DROUND, CVT);
 693
 694    NV50_IR_OPCODE_CASE(IMUL_HI, MUL);
 695    NV50_IR_OPCODE_CASE(UMUL_HI, MUL);
 696
 697    NV50_IR_OPCODE_CASE(SAMPLE, TEX);
 698    NV50_IR_OPCODE_CASE(SAMPLE_B, TXB);
 699    NV50_IR_OPCODE_CASE(SAMPLE_C, TEX);
 700    NV50_IR_OPCODE_CASE(SAMPLE_C_LZ, TEX);
 701    NV50_IR_OPCODE_CASE(SAMPLE_D, TXD);
 702    NV50_IR_OPCODE_CASE(SAMPLE_L, TXL);
 703    NV50_IR_OPCODE_CASE(SAMPLE_I, TXF);
 704    NV50_IR_OPCODE_CASE(SAMPLE_I_MS, TXF);
 705    NV50_IR_OPCODE_CASE(GATHER4, TXG);
 706    NV50_IR_OPCODE_CASE(SVIEWINFO, TXQ);
 707
 708    NV50_IR_OPCODE_CASE(ATOMUADD, ATOM);
 709    NV50_IR_OPCODE_CASE(ATOMXCHG, ATOM);
 710    NV50_IR_OPCODE_CASE(ATOMCAS, ATOM);
 711    NV50_IR_OPCODE_CASE(ATOMAND, ATOM);
 712    NV50_IR_OPCODE_CASE(ATOMOR, ATOM);
 713    NV50_IR_OPCODE_CASE(ATOMXOR, ATOM);
 714    NV50_IR_OPCODE_CASE(ATOMUMIN, ATOM);
 715    NV50_IR_OPCODE_CASE(ATOMUMAX, ATOM);
 716    NV50_IR_OPCODE_CASE(ATOMIMIN, ATOM);
 717    NV50_IR_OPCODE_CASE(ATOMIMAX, ATOM);
 718
 719    NV50_IR_OPCODE_CASE(TEX2, TEX);
 720    NV50_IR_OPCODE_CASE(TXB2, TXB);
 721    NV50_IR_OPCODE_CASE(TXL2, TXL);
 722
 723    NV50_IR_OPCODE_CASE(IBFE, EXTBF);
 724    NV50_IR_OPCODE_CASE(UBFE, EXTBF);
 725    NV50_IR_OPCODE_CASE(BFI, INSBF);
 726    NV50_IR_OPCODE_CASE(BREV, EXTBF);
 727    NV50_IR_OPCODE_CASE(POPC, POPCNT);
 728    NV50_IR_OPCODE_CASE(LSB, BFIND);
 729    NV50_IR_OPCODE_CASE(IMSB, BFIND);
 730    NV50_IR_OPCODE_CASE(UMSB, BFIND);
 731
 732    NV50_IR_OPCODE_CASE(END, EXIT);
 733
 734    default:
 735       return nv50_ir::OP_NOP;
 736    }
 737 }
 738
 739 static uint16_t opcodeToSubOp(uint opcode)
 740 {
 741    switch (opcode) {
 742    case TGSI_OPCODE_LFENCE:   return NV50_IR_SUBOP_MEMBAR(L, GL);
 743    case TGSI_OPCODE_SFENCE:   return NV50_IR_SUBOP_MEMBAR(S, GL);
 744    case TGSI_OPCODE_MFENCE:   return NV50_IR_SUBOP_MEMBAR(M, GL);
 745    case TGSI_OPCODE_ATOMUADD: return NV50_IR_SUBOP_ATOM_ADD;
 746    case TGSI_OPCODE_ATOMXCHG: return NV50_IR_SUBOP_ATOM_EXCH;
 747    case TGSI_OPCODE_ATOMCAS:  return NV50_IR_SUBOP_ATOM_CAS;
 748    case TGSI_OPCODE_ATOMAND:  return NV50_IR_SUBOP_ATOM_AND;
 749    case TGSI_OPCODE_ATOMOR:   return NV50_IR_SUBOP_ATOM_OR;
 750    case TGSI_OPCODE_ATOMXOR:  return NV50_IR_SUBOP_ATOM_XOR;
 751    case TGSI_OPCODE_ATOMUMIN: return NV50_IR_SUBOP_ATOM_MIN;
 752    case TGSI_OPCODE_ATOMIMIN: return NV50_IR_SUBOP_ATOM_MIN;
 753    case TGSI_OPCODE_ATOMUMAX: return NV50_IR_SUBOP_ATOM_MAX;
 754    case TGSI_OPCODE_ATOMIMAX: return NV50_IR_SUBOP_ATOM_MAX;
 755    case TGSI_OPCODE_IMUL_HI:
 756    case TGSI_OPCODE_UMUL_HI:
 757       return NV50_IR_SUBOP_MUL_HIGH;
 758    default:
 759       return 0;
 760    }
 761 }
 762
 763 bool Instruction::checkDstSrcAliasing() const
 764 {
 765    if (insn->Dst[0].Register.Indirect) // no danger if indirect, using memory
 766       return false;
 767
 768    for (int s = 0; s < TGSI_FULL_MAX_SRC_REGISTERS; ++s) {
 769       if (insn->Src[s].Register.File == TGSI_FILE_NULL)
 770          break;
 771       if (insn->Src[s].Register.File == insn->Dst[0].Register.File &&
 772           insn->Src[s].Register.Index == insn->Dst[0].Register.Index)
 773          return true;
 774    }
 775    return false;
 776 }
 777
 778 class Source
 779 {
 780 public:
 781    Source(struct nv50_ir_prog_info *);
 782    ~Source();
 783
 784 public:
 785    bool scanSource();
 786    unsigned fileSize(unsigned file) const { return scan.file_max[file] + 1; }
 787
 788 public:
 789    struct tgsi_shader_info scan;
 790    struct tgsi_full_instruction *insns;
 791    const struct tgsi_token *tokens;
 792    struct nv50_ir_prog_info *info;
 793
 794    nv50_ir::DynArray tempArrays;
 795    nv50_ir::DynArray immdArrays;
 796
 797    typedef nv50_ir::BuildUtil::Location Location;
 798    // these registers are per-subroutine, cannot be used for parameter passing
 799    std::set<Location> locals;
 800
 801    bool mainTempsInLMem;
 802
 803    int clipVertexOutput;
 804
 805    struct TextureView {
 806       uint8_t target; // TGSI_TEXTURE_*
 807    };
 808    std::vector<TextureView> textureViews;
 809
 810    struct Resource {
 811       uint8_t target; // TGSI_TEXTURE_*
 812       bool raw;
 813       uint8_t slot; // $surface index
 814    };
 815    std::vector<Resource> resources;
 816
 817 private:
 818    int inferSysValDirection(unsigned sn) const;
 819    bool scanDeclaration(const struct tgsi_full_declaration *);
 820    bool scanInstruction(const struct tgsi_full_instruction *);
 821    void scanProperty(const struct tgsi_full_property *);
 822    void scanImmediate(const struct tgsi_full_immediate *);
 823
 824    inline bool isEdgeFlagPassthrough(const Instruction&) const;
 825 };
 826
 827 Source::Source(struct nv50_ir_prog_info *prog) : info(prog)
 828 {
 829    tokens = (const struct tgsi_token *)info->bin.source;
 830
 831    if (prog->dbgFlags & NV50_IR_DEBUG_BASIC)
 832       tgsi_dump(tokens, 0);
 833
 834    mainTempsInLMem = false;
 835 }
 836
 837 Source::~Source()
 838 {
 839    if (insns)
 840       FREE(insns);
 841
 842    if (info->immd.data)
 843       FREE(info->immd.data);
 844    if (info->immd.type)
 845       FREE(info->immd.type);
 846 }
 847
 848 bool Source::scanSource()
 849 {
 850    unsigned insnCount = 0;
 851    struct tgsi_parse_context parse;
 852
 853    tgsi_scan_shader(tokens, &scan);
 854
 855    insns = (struct tgsi_full_instruction *)MALLOC(scan.num_instructions *
 856                                                   sizeof(insns[0]));
 857    if (!insns)
 858       return false;
 859
 860    clipVertexOutput = -1;
 861
 862    textureViews.resize(scan.file_max[TGSI_FILE_SAMPLER_VIEW] + 1);
 863    resources.resize(scan.file_max[TGSI_FILE_RESOURCE] + 1);
 864
 865    info->immd.bufSize = 0;
 866
 867    info->numInputs = scan.file_max[TGSI_FILE_INPUT] + 1;
 868    info->numOutputs = scan.file_max[TGSI_FILE_OUTPUT] + 1;
 869    info->numSysVals = scan.file_max[TGSI_FILE_SYSTEM_VALUE] + 1;
 870
 871    if (info->type == PIPE_SHADER_FRAGMENT) {
 872       info->prop.fp.writesDepth = scan.writes_z;
 873       info->prop.fp.usesDiscard = scan.uses_kill;
 874    } else
 875    if (info->type == PIPE_SHADER_GEOMETRY) {
 876       info->prop.gp.instanceCount = 1; // default value
 877    }
 878
 879    info->io.viewportId = -1;
 880
 881    info->immd.data = (uint32_t *)MALLOC(scan.immediate_count * 16);
 882    info->immd.type = (ubyte *)MALLOC(scan.immediate_count * sizeof(ubyte));
 883
 884    tgsi_parse_init(&parse, tokens);
 885    while (!tgsi_parse_end_of_tokens(&parse)) {
 886       tgsi_parse_token(&parse);
 887
 888       switch (parse.FullToken.Token.Type) {
 889       case TGSI_TOKEN_TYPE_IMMEDIATE:
 890          scanImmediate(&parse.FullToken.FullImmediate);
 891          break;
 892       case TGSI_TOKEN_TYPE_DECLARATION:
 893          scanDeclaration(&parse.FullToken.FullDeclaration);
 894          break;
 895       case TGSI_TOKEN_TYPE_INSTRUCTION:
 896          insns[insnCount++] = parse.FullToken.FullInstruction;
 897          scanInstruction(&parse.FullToken.FullInstruction);
 898          break;
 899       case TGSI_TOKEN_TYPE_PROPERTY:
 900          scanProperty(&parse.FullToken.FullProperty);
 901          break;
 902       default:
 903          INFO("unknown TGSI token type: %d\n", parse.FullToken.Token.Type);
 904          break;
 905       }
 906    }
 907    tgsi_parse_free(&parse);
 908
 909    if (mainTempsInLMem)
 910       info->bin.tlsSpace += (scan.file_max[TGSI_FILE_TEMPORARY] + 1) * 16;
 911
 912    if (info->io.genUserClip > 0) {
 913       info->io.clipDistanceMask = (1 << info->io.genUserClip) - 1;
 914
 915       const unsigned int nOut = (info->io.genUserClip + 3) / 4;
 916
 917       for (unsigned int n = 0; n < nOut; ++n) {
 918          unsigned int i = info->numOutputs++;
 919          info->out[i].id = i;
 920          info->out[i].sn = TGSI_SEMANTIC_CLIPDIST;
 921          info->out[i].si = n;
 922          info->out[i].mask = info->io.clipDistanceMask >> (n * 4);
 923       }
 924    }
 925
 926    return info->assignSlots(info) == 0;
 927 }
 928
 929 void Source::scanProperty(const struct tgsi_full_property *prop)
 930 {
 931    switch (prop->Property.PropertyName) {
 932    case TGSI_PROPERTY_GS_OUTPUT_PRIM:
 933       info->prop.gp.outputPrim = prop->u[0].Data;
 934       break;
 935    case TGSI_PROPERTY_GS_INPUT_PRIM:
 936       info->prop.gp.inputPrim = prop->u[0].Data;
 937       break;
 938    case TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES:
 939       info->prop.gp.maxVertices = prop->u[0].Data;
 940       break;
 941    case TGSI_PROPERTY_GS_INVOCATIONS:
 942       info->prop.gp.instanceCount = prop->u[0].Data;
 943       break;
 944    case TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS:
 945       info->prop.fp.separateFragData = true;
 946       break;
 947    case TGSI_PROPERTY_FS_COORD_ORIGIN:
 948    case TGSI_PROPERTY_FS_COORD_PIXEL_CENTER:
 949       // we don't care
 950       break;
 951    case TGSI_PROPERTY_VS_PROHIBIT_UCPS:
 952       info->io.genUserClip = -1;
 953       break;
 954    case TGSI_PROPERTY_TCS_VERTICES_OUT:
 955       info->prop.tp.outputPatchSize = prop->u[0].Data;
 956       break;
 957    case TGSI_PROPERTY_TES_PRIM_MODE:
 958       info->prop.tp.domain = prop->u[0].Data;
 959       break;
 960    case TGSI_PROPERTY_TES_SPACING:
 961       info->prop.tp.partitioning = prop->u[0].Data;
 962       break;
 963    case TGSI_PROPERTY_TES_VERTEX_ORDER_CW:
 964       info->prop.tp.winding = prop->u[0].Data;
 965       break;
 966    case TGSI_PROPERTY_TES_POINT_MODE:
 967       if (prop->u[0].Data)
 968          info->prop.tp.outputPrim = PIPE_PRIM_POINTS;
 969       else
 970          info->prop.tp.outputPrim = PIPE_PRIM_TRIANGLES; /* anything but points */
 971       break;
 972    default:
 973       INFO("unhandled TGSI property %d\n", prop->Property.PropertyName);
 974       break;
 975    }
 976 }
 977
 978 void Source::scanImmediate(const struct tgsi_full_immediate *imm)
 979 {
 980    const unsigned n = info->immd.count++;
 981
 982    assert(n < scan.immediate_count);
 983
 984    for (int c = 0; c < 4; ++c)
 985       info->immd.data[n * 4 + c] = imm->u[c].Uint;
 986
 987    info->immd.type[n] = imm->Immediate.DataType;
 988 }
 989
 990 int Source::inferSysValDirection(unsigned sn) const
 991 {
 992    switch (sn) {
 993    case TGSI_SEMANTIC_INSTANCEID:
 994    case TGSI_SEMANTIC_VERTEXID:
 995       return 1;
 996    case TGSI_SEMANTIC_LAYER:
 997 #if 0
 998    case TGSI_SEMANTIC_VIEWPORTINDEX:
 999       return 0;
1000 #endif
1001    case TGSI_SEMANTIC_PRIMID:
1002       return (info->type == PIPE_SHADER_FRAGMENT) ? 1 : 0;
1003    default:
1004       return 0;
1005    }
1006 }
1007
1008 bool Source::scanDeclaration(const struct tgsi_full_declaration *decl)
1009 {
1010    unsigned i, c;
1011    unsigned sn = TGSI_SEMANTIC_GENERIC;
1012    unsigned si = 0;
1013    const unsigned first = decl->Range.First, last = decl->Range.Last;
1014
1015    if (decl->Declaration.Semantic) {
1016       sn = decl->Semantic.Name;
1017       si = decl->Semantic.Index;
1018    }
1019
1020    if (decl->Declaration.Local) {
1021       for (i = first; i <= last; ++i) {
1022          for (c = 0; c < 4; ++c) {
1023             locals.insert(
1024                Location(decl->Declaration.File, decl->Dim.Index2D, i, c));
1025          }
1026       }
1027    }
1028
1029    switch (decl->Declaration.File) {
1030    case TGSI_FILE_INPUT:
1031       if (info->type == PIPE_SHADER_VERTEX) {
1032          // all vertex attributes are equal
1033          for (i = first; i <= last; ++i) {
1034             info->in[i].sn = TGSI_SEMANTIC_GENERIC;
1035             info->in[i].si = i;
1036          }
1037       } else {
1038          for (i = first; i <= last; ++i, ++si) {
1039             info->in[i].id = i;
1040             info->in[i].sn = sn;
1041             info->in[i].si = si;
1042             if (info->type == PIPE_SHADER_FRAGMENT) {
1043                // translate interpolation mode
1044                switch (decl->Interp.Interpolate) {
1045                case TGSI_INTERPOLATE_CONSTANT:
1046                   info->in[i].flat = 1;
1047                   break;
1048                case TGSI_INTERPOLATE_COLOR:
1049                   info->in[i].sc = 1;
1050                   break;
1051                case TGSI_INTERPOLATE_LINEAR:
1052                   info->in[i].linear = 1;
1053                   break;
1054                default:
1055                   break;
1056                }
1057                if (decl->Interp.Location || info->io.sampleInterp)
1058                   info->in[i].centroid = 1;
1059             }
1060
1061             if (sn == TGSI_SEMANTIC_PATCH)
1062                info->in[i].patch = 1;
1063             if (sn == TGSI_SEMANTIC_PATCH)
1064                info->numPatchConstants = MAX2(info->numPatchConstants, si + 1);
1065          }
1066       }
1067       break;
1068    case TGSI_FILE_OUTPUT:
1069       for (i = first; i <= last; ++i, ++si) {
1070          switch (sn) {
1071          case TGSI_SEMANTIC_POSITION:
1072             if (info->type == PIPE_SHADER_FRAGMENT)
1073                info->io.fragDepth = i;
1074             else
1075             if (clipVertexOutput < 0)
1076                clipVertexOutput = i;
1077             break;
1078          case TGSI_SEMANTIC_COLOR:
1079             if (info->type == PIPE_SHADER_FRAGMENT)
1080                info->prop.fp.numColourResults++;
1081             break;
1082          case TGSI_SEMANTIC_EDGEFLAG:
1083             info->io.edgeFlagOut = i;
1084             break;
1085          case TGSI_SEMANTIC_CLIPVERTEX:
1086             clipVertexOutput = i;
1087             break;
1088          case TGSI_SEMANTIC_CLIPDIST:
1089             info->io.clipDistanceMask |=
1090                decl->Declaration.UsageMask << (si * 4);
1091             info->io.genUserClip = -1;
1092             break;
1093          case TGSI_SEMANTIC_SAMPLEMASK:
1094             info->io.sampleMask = i;
1095             break;
1096          case TGSI_SEMANTIC_VIEWPORT_INDEX:
1097             info->io.viewportId = i;
1098             break;
1099          case TGSI_SEMANTIC_PATCH:
1100             info->numPatchConstants = MAX2(info->numPatchConstants, si + 1);
1101             /* fallthrough */
1102          case TGSI_SEMANTIC_TESSOUTER:
1103          case TGSI_SEMANTIC_TESSINNER:
1104             info->out[i].patch = 1;
1105             break;
1106          default:
1107             break;
1108          }
1109          info->out[i].id = i;
1110          info->out[i].sn = sn;
1111          info->out[i].si = si;
1112       }
1113       break;
1114    case TGSI_FILE_SYSTEM_VALUE:
1115       switch (sn) {
1116       case TGSI_SEMANTIC_INSTANCEID:
1117          info->io.instanceId = first;
1118          break;
1119       case TGSI_SEMANTIC_VERTEXID:
1120          info->io.vertexId = first;
1121          break;
1122       default:
1123          break;
1124       }
1125       for (i = first; i <= last; ++i, ++si) {
1126          info->sv[i].sn = sn;
1127          info->sv[i].si = si;
1128          info->sv[i].input = inferSysValDirection(sn);
1129
1130          switch (sn) {
1131          case TGSI_SEMANTIC_TESSOUTER:
1132          case TGSI_SEMANTIC_TESSINNER:
1133             info->sv[i].patch = 1;
1134             break;
1135          }
1136       }
1137       break;
1138    case TGSI_FILE_RESOURCE:
1139       for (i = first; i <= last; ++i) {
1140          resources[i].target = decl->Resource.Resource;
1141          resources[i].raw = decl->Resource.Raw;
1142          resources[i].slot = i;
1143       }
1144       break;
1145    case TGSI_FILE_SAMPLER_VIEW:
1146       for (i = first; i <= last; ++i)
1147          textureViews[i].target = decl->SamplerView.Resource;
1148       break;
1149    case TGSI_FILE_NULL:
1150    case TGSI_FILE_TEMPORARY:
1151    case TGSI_FILE_ADDRESS:
1152    case TGSI_FILE_CONSTANT:
1153    case TGSI_FILE_IMMEDIATE:
1154    case TGSI_FILE_PREDICATE:
1155    case TGSI_FILE_SAMPLER:
1156       break;
1157    default:
1158       ERROR("unhandled TGSI_FILE %d\n", decl->Declaration.File);
1159       return false;
1160    }
1161    return true;
1162 }
1163
1164 inline bool Source::isEdgeFlagPassthrough(const Instruction& insn) const
1165 {
1166    return insn.getOpcode() == TGSI_OPCODE_MOV &&
1167       insn.getDst(0).getIndex(0) == info->io.edgeFlagOut &&
1168       insn.getSrc(0).getFile() == TGSI_FILE_INPUT;
1169 }
1170
1171 bool Source::scanInstruction(const struct tgsi_full_instruction *inst)
1172 {
1173    Instruction insn(inst);
1174
1175    if (insn.getOpcode() == TGSI_OPCODE_BARRIER)
1176       info->numBarriers = 1;
1177
1178    if (insn.dstCount()) {
1179       if (insn.getDst(0).getFile() == TGSI_FILE_OUTPUT) {
1180          Instruction::DstRegister dst = insn.getDst(0);
1181
1182          if (dst.isIndirect(0))
1183             for (unsigned i = 0; i < info->numOutputs; ++i)
1184                info->out[i].mask = 0xf;
1185          else
1186             info->out[dst.getIndex(0)].mask |= dst.getMask();
1187
1188          if (info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PSIZE ||
1189              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PRIMID ||
1190              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_LAYER ||
1191              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_VIEWPORT_INDEX ||
1192              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_FOG)
1193             info->out[dst.getIndex(0)].mask &= 1;
1194
1195          if (isEdgeFlagPassthrough(insn))
1196             info->io.edgeFlagIn = insn.getSrc(0).getIndex(0);
1197       } else
1198       if (insn.getDst(0).getFile() == TGSI_FILE_TEMPORARY) {
1199          if (insn.getDst(0).isIndirect(0))
1200             mainTempsInLMem = true;
1201       }
1202    }
1203
1204    for (unsigned s = 0; s < insn.srcCount(); ++s) {
1205       Instruction::SrcRegister src = insn.getSrc(s);
1206       if (src.getFile() == TGSI_FILE_TEMPORARY) {
1207          if (src.isIndirect(0))
1208             mainTempsInLMem = true;
1209       } else
1210       if (src.getFile() == TGSI_FILE_RESOURCE) {
1211          if (src.getIndex(0) == TGSI_RESOURCE_GLOBAL)
1212             info->io.globalAccess |= (insn.getOpcode() == TGSI_OPCODE_LOAD) ?
1213                0x1 : 0x2;
1214       } else
1215       if (src.getFile() == TGSI_FILE_OUTPUT) {
1216          if (src.isIndirect(0)) {
1217             // We don't know which one is accessed, just mark everything for
1218             // reading. This is an extremely unlikely occurrence.
1219             for (unsigned i = 0; i < info->numOutputs; ++i)
1220                info->out[i].oread = 1;
1221          } else {
1222             info->out[src.getIndex(0)].oread = 1;
1223          }
1224       }
1225       if (src.getFile() != TGSI_FILE_INPUT)
1226          continue;
1227       unsigned mask = insn.srcMask(s);
1228
1229       if (src.isIndirect(0)) {
1230          for (unsigned i = 0; i < info->numInputs; ++i)
1231             info->in[i].mask = 0xf;
1232       } else {
1233          const int i = src.getIndex(0);
1234          for (unsigned c = 0; c < 4; ++c) {
1235             if (!(mask & (1 << c)))
1236                continue;
1237             int k = src.getSwizzle(c);
1238             if (k <= TGSI_SWIZZLE_W)
1239                info->in[i].mask |= 1 << k;
1240          }
1241          switch (info->in[i].sn) {
1242          case TGSI_SEMANTIC_PSIZE:
1243          case TGSI_SEMANTIC_PRIMID:
1244          case TGSI_SEMANTIC_FOG:
1245             info->in[i].mask &= 0x1;
1246             break;
1247          case TGSI_SEMANTIC_PCOORD:
1248             info->in[i].mask &= 0x3;
1249             break;
1250          default:
1251             break;
1252          }
1253       }
1254    }
1255    return true;
1256 }
1257
1258 nv50_ir::TexInstruction::Target
1259 Instruction::getTexture(const tgsi::Source *code, int s) const
1260 {
1261    // XXX: indirect access
1262    unsigned int r;
1263
1264    switch (getSrc(s).getFile()) {
1265    case TGSI_FILE_RESOURCE:
1266       r = getSrc(s).getIndex(0);
1267       return translateTexture(code->resources.at(r).target);
1268    case TGSI_FILE_SAMPLER_VIEW:
1269       r = getSrc(s).getIndex(0);
1270       return translateTexture(code->textureViews.at(r).target);
1271    default:
1272       return translateTexture(insn->Texture.Texture);
1273    }
1274 }
1275
1276 } // namespace tgsi
1277
1278 namespace {
1279
1280 using namespace nv50_ir;
1281
1282 class Converter : public BuildUtil
1283 {
1284 public:
1285    Converter(Program *, const tgsi::Source *);
1286    ~Converter();
1287
1288    bool run();
1289
1290 private:
1291    struct Subroutine
1292    {
1293       Subroutine(Function *f) : f(f) { }
1294       Function *f;
1295       ValueMap values;
1296    };
1297
1298    Value *shiftAddress(Value *);
1299    Value *getVertexBase(int s);
1300    Value *getOutputBase(int s);
1301    DataArray *getArrayForFile(unsigned file, int idx);
1302    Value *fetchSrc(int s, int c);
1303    Value *acquireDst(int d, int c);
1304    void storeDst(int d, int c, Value *);
1305
1306    Value *fetchSrc(const tgsi::Instruction::SrcRegister src, int c, Value *ptr);
1307    void storeDst(const tgsi::Instruction::DstRegister dst, int c,
1308                  Value *val, Value *ptr);
1309
1310    Value *applySrcMod(Value *, int s, int c);
1311
1312    Symbol *makeSym(uint file, int fileIndex, int idx, int c, uint32_t addr);
1313    Symbol *srcToSym(tgsi::Instruction::SrcRegister, int c);
1314    Symbol *dstToSym(tgsi::Instruction::DstRegister, int c);
1315
1316    bool handleInstruction(const struct tgsi_full_instruction *);
1317    void exportOutputs();
1318    inline Subroutine *getSubroutine(unsigned ip);
1319    inline Subroutine *getSubroutine(Function *);
1320    inline bool isEndOfSubroutine(uint ip);
1321
1322    void loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask);
1323
1324    // R,S,L,C,Dx,Dy encode TGSI sources for respective values (0xSf for auto)
1325    void setTexRS(TexInstruction *, unsigned int& s, int R, int S);
1326    void handleTEX(Value *dst0[4], int R, int S, int L, int C, int Dx, int Dy);
1327    void handleTXF(Value *dst0[4], int R, int L_M);
1328    void handleTXQ(Value *dst0[4], enum TexQuery, int R);
1329    void handleLIT(Value *dst0[4]);
1330    void handleUserClipPlanes();
1331
1332    Symbol *getResourceBase(int r);
1333    void getResourceCoords(std::vector<Value *>&, int r, int s);
1334
1335    void handleLOAD(Value *dst0[4]);
1336    void handleSTORE();
1337    void handleATOM(Value *dst0[4], DataType, uint16_t subOp);
1338
1339    void handleINTERP(Value *dst0[4]);
1340
1341    Value *interpolate(tgsi::Instruction::SrcRegister, int c, Value *ptr);
1342
1343    void insertConvergenceOps(BasicBlock *conv, BasicBlock *fork);
1344
1345    Value *buildDot(int dim);
1346
1347    class BindArgumentsPass : public Pass {
1348    public:
1349       BindArgumentsPass(Converter &conv) : conv(conv) { }
1350
1351    private:
1352       Converter &conv;
1353       Subroutine *sub;
1354
1355       inline const Location *getValueLocation(Subroutine *, Value *);
1356
1357       template<typename T> inline void
1358       updateCallArgs(Instruction *i, void (Instruction::*setArg)(int, Value *),
1359                      T (Function::*proto));
1360
1361       template<typename T> inline void
1362       updatePrototype(BitSet *set, void (Function::*updateSet)(),
1363                       T (Function::*proto));
1364
1365    protected:
1366       bool visit(Function *);
1367       bool visit(BasicBlock *bb) { return false; }
1368    };
1369
1370 private:
1371    const tgsi::Source *code;
1372    const struct nv50_ir_prog_info *info;
1373
1374    struct {
1375       std::map<unsigned, Subroutine> map;
1376       Subroutine *cur;
1377    } sub;
1378
1379    uint ip; // instruction pointer
1380
1381    tgsi::Instruction tgsi;
1382
1383    DataType dstTy;
1384    DataType srcTy;
1385
1386    DataArray tData; // TGSI_FILE_TEMPORARY
1387    DataArray aData; // TGSI_FILE_ADDRESS
1388    DataArray pData; // TGSI_FILE_PREDICATE
1389    DataArray oData; // TGSI_FILE_OUTPUT (if outputs in registers)
1390
1391    Value *zero;
1392    Value *fragCoord[4];
1393    Value *clipVtx[4];
1394
1395    Value *vtxBase[5]; // base address of vertex in primitive (for TP/GP)
1396    uint8_t vtxBaseValid;
1397
1398    Value *outBase; // base address of vertex out patch (for TCP)
1399
1400    Stack condBBs;  // fork BB, then else clause BB
1401    Stack joinBBs;  // fork BB, for inserting join ops on ENDIF
1402    Stack loopBBs;  // loop headers
1403    Stack breakBBs; // end of / after loop
1404
1405    Value *viewport;
1406 };
1407
1408 Symbol *
1409 Converter::srcToSym(tgsi::Instruction::SrcRegister src, int c)
1410 {
1411    const int swz = src.getSwizzle(c);
1412
1413    /* TODO: Use Array ID when it's available for the index */
1414    return makeSym(src.getFile(),
1415                   src.is2D() ? src.getIndex(1) : 0,
1416                   src.getIndex(0), swz,
1417                   src.getIndex(0) * 16 + swz * 4);
1418 }
1419
1420 Symbol *
1421 Converter::dstToSym(tgsi::Instruction::DstRegister dst, int c)
1422 {
1423    /* TODO: Use Array ID when it's available for the index */
1424    return makeSym(dst.getFile(),
1425                   dst.is2D() ? dst.getIndex(1) : 0,
1426                   dst.getIndex(0), c,
1427                   dst.getIndex(0) * 16 + c * 4);
1428 }
1429
1430 Symbol *
1431 Converter::makeSym(uint tgsiFile, int fileIdx, int idx, int c, uint32_t address)
1432 {
1433    Symbol *sym = new_Symbol(prog, tgsi::translateFile(tgsiFile));
1434
1435    sym->reg.fileIndex = fileIdx;
1436
1437    if (idx >= 0) {
1438       if (sym->reg.file == FILE_SHADER_INPUT)
1439          sym->setOffset(info->in[idx].slot[c] * 4);
1440       else
1441       if (sym->reg.file == FILE_SHADER_OUTPUT)
1442          sym->setOffset(info->out[idx].slot[c] * 4);
1443       else
1444       if (sym->reg.file == FILE_SYSTEM_VALUE)
1445          sym->setSV(tgsi::translateSysVal(info->sv[idx].sn), c);
1446       else
1447          sym->setOffset(address);
1448    } else {
1449       sym->setOffset(address);
1450    }
1451    return sym;
1452 }
1453
1454 static inline uint8_t
1455 translateInterpMode(const struct nv50_ir_varying *var, operation& op)
1456 {
1457    uint8_t mode = NV50_IR_INTERP_PERSPECTIVE;
1458
1459    if (var->flat)
1460       mode = NV50_IR_INTERP_FLAT;
1461    else
1462    if (var->linear)
1463       mode = NV50_IR_INTERP_LINEAR;
1464    else
1465    if (var->sc)
1466       mode = NV50_IR_INTERP_SC;
1467
1468    op = (mode == NV50_IR_INTERP_PERSPECTIVE || mode == NV50_IR_INTERP_SC)
1469       ? OP_PINTERP : OP_LINTERP;
1470
1471    if (var->centroid)
1472       mode |= NV50_IR_INTERP_CENTROID;
1473
1474    return mode;
1475 }
1476
1477 Value *
1478 Converter::interpolate(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
1479 {
1480    operation op;
1481
1482    // XXX: no way to know interpolation mode if we don't know what's accessed
1483    const uint8_t mode = translateInterpMode(&info->in[ptr ? 0 :
1484                                                       src.getIndex(0)], op);
1485
1486    Instruction *insn = new_Instruction(func, op, TYPE_F32);
1487
1488    insn->setDef(0, getScratch());
1489    insn->setSrc(0, srcToSym(src, c));
1490    if (op == OP_PINTERP)
1491       insn->setSrc(1, fragCoord[3]);
1492    if (ptr)
1493       insn->setIndirect(0, 0, ptr);
1494
1495    insn->setInterpolate(mode);
1496
1497    bb->insertTail(insn);
1498    return insn->getDef(0);
1499 }
1500
1501 Value *
1502 Converter::applySrcMod(Value *val, int s, int c)
1503 {
1504    Modifier m = tgsi.getSrc(s).getMod(c);
1505    DataType ty = tgsi.inferSrcType();
1506
1507    if (m & Modifier(NV50_IR_MOD_ABS))
1508       val = mkOp1v(OP_ABS, ty, getScratch(), val);
1509
1510    if (m & Modifier(NV50_IR_MOD_NEG))
1511       val = mkOp1v(OP_NEG, ty, getScratch(), val);
1512
1513    return val;
1514 }
1515
1516 Value *
1517 Converter::getVertexBase(int s)
1518 {
1519    assert(s < 5);
1520    if (!(vtxBaseValid & (1 << s))) {
1521       const int index = tgsi.getSrc(s).getIndex(1);
1522       Value *rel = NULL;
1523       if (tgsi.getSrc(s).isIndirect(1))
1524          rel = fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL);
1525       vtxBaseValid |= 1 << s;
1526       vtxBase[s] = mkOp2v(OP_PFETCH, TYPE_U32, getSSA(4, FILE_ADDRESS),
1527                           mkImm(index), rel);
1528    }
1529    return vtxBase[s];
1530 }
1531
1532 Value *
1533 Converter::getOutputBase(int s)
1534 {
1535    assert(s < 5);
1536    if (!(vtxBaseValid & (1 << s))) {
1537       Value *offset = loadImm(NULL, tgsi.getSrc(s).getIndex(1));
1538       if (tgsi.getSrc(s).isIndirect(1))
1539          offset = mkOp2v(OP_ADD, TYPE_U32, getSSA(),
1540                          fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL),
1541                          offset);
1542       vtxBaseValid |= 1 << s;
1543       vtxBase[s] = mkOp2v(OP_ADD, TYPE_U32, getSSA(), outBase, offset);
1544    }
1545    return vtxBase[s];
1546 }
1547
1548 Value *
1549 Converter::fetchSrc(int s, int c)
1550 {
1551    Value *res;
1552    Value *ptr = NULL, *dimRel = NULL;
1553
1554    tgsi::Instruction::SrcRegister src = tgsi.getSrc(s);
1555
1556    if (src.isIndirect(0))
1557       ptr = fetchSrc(src.getIndirect(0), 0, NULL);
1558
1559    if (src.is2D()) {
1560       switch (src.getFile()) {
1561       case TGSI_FILE_OUTPUT:
1562          dimRel = getOutputBase(s);
1563          break;
1564       case TGSI_FILE_INPUT:
1565          dimRel = getVertexBase(s);
1566          break;
1567       case TGSI_FILE_CONSTANT:
1568          // on NVC0, this is valid and c{I+J}[k] == cI[(J << 16) + k]
1569          if (src.isIndirect(1))
1570             dimRel = fetchSrc(src.getIndirect(1), 0, 0);
1571          break;
1572       default:
1573          break;
1574       }
1575    }
1576
1577    res = fetchSrc(src, c, ptr);
1578
1579    if (dimRel)
1580       res->getInsn()->setIndirect(0, 1, dimRel);
1581
1582    return applySrcMod(res, s, c);
1583 }
1584
1585 Converter::DataArray *
1586 Converter::getArrayForFile(unsigned file, int idx)
1587 {
1588    switch (file) {
1589    case TGSI_FILE_TEMPORARY:
1590       return &tData;
1591    case TGSI_FILE_PREDICATE:
1592       return &pData;
1593    case TGSI_FILE_ADDRESS:
1594       return &aData;
1595    case TGSI_FILE_OUTPUT:
1596       assert(prog->getType() == Program::TYPE_FRAGMENT);
1597       return &oData;
1598    default:
1599       assert(!"invalid/unhandled TGSI source file");
1600       return NULL;
1601    }
1602 }
1603
1604 Value *
1605 Converter::shiftAddress(Value *index)
1606 {
1607    if (!index)
1608       return NULL;
1609    return mkOp2v(OP_SHL, TYPE_U32, getSSA(4, FILE_ADDRESS), index, mkImm(4));
1610 }
1611
1612 Value *
1613 Converter::fetchSrc(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
1614 {
1615    const int idx2d = src.is2D() ? src.getIndex(1) : 0;
1616    const int idx = src.getIndex(0);
1617    const int swz = src.getSwizzle(c);
1618    Instruction *ld;
1619
1620    switch (src.getFile()) {
1621    case TGSI_FILE_IMMEDIATE:
1622       assert(!ptr);
1623       return loadImm(NULL, info->immd.data[idx * 4 + swz]);
1624    case TGSI_FILE_CONSTANT:
1625       return mkLoadv(TYPE_U32, srcToSym(src, c), shiftAddress(ptr));
1626    case TGSI_FILE_INPUT:
1627       if (prog->getType() == Program::TYPE_FRAGMENT) {
1628          // don't load masked inputs, won't be assigned a slot
1629          if (!ptr && !(info->in[idx].mask & (1 << swz)))
1630             return loadImm(NULL, swz == TGSI_SWIZZLE_W ? 1.0f : 0.0f);
1631          if (!ptr && info->in[idx].sn == TGSI_SEMANTIC_FACE)
1632             return mkOp1v(OP_RDSV, TYPE_F32, getSSA(), mkSysVal(SV_FACE, 0));
1633          return interpolate(src, c, shiftAddress(ptr));
1634       } else
1635       if (prog->getType() == Program::TYPE_GEOMETRY) {
1636          if (!ptr && info->in[idx].sn == TGSI_SEMANTIC_PRIMID)
1637             return mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_PRIMITIVE_ID, 0));
1638          // XXX: This is going to be a problem with scalar arrays, i.e. when
1639          // we cannot assume that the address is given in units of vec4.
1640          //
1641          // nv50 and nvc0 need different things here, so let the lowering
1642          // passes decide what to do with the address
1643          if (ptr)
1644             return mkLoadv(TYPE_U32, srcToSym(src, c), ptr);
1645       }
1646       ld = mkLoad(TYPE_U32, getSSA(), srcToSym(src, c), shiftAddress(ptr));
1647       ld->perPatch = info->in[idx].patch;
1648       return ld->getDef(0);
1649    case TGSI_FILE_OUTPUT:
1650       assert(prog->getType() == Program::TYPE_TESSELLATION_CONTROL);
1651       ld = mkLoad(TYPE_U32, getSSA(), srcToSym(src, c), shiftAddress(ptr));
1652       ld->perPatch = info->out[idx].patch;
1653       return ld->getDef(0);
1654    case TGSI_FILE_SYSTEM_VALUE:
1655       assert(!ptr);
1656       ld = mkOp1(OP_RDSV, TYPE_U32, getSSA(), srcToSym(src, c));
1657       ld->perPatch = info->sv[idx].patch;
1658       return ld->getDef(0);
1659    default:
1660       return getArrayForFile(src.getFile(), idx2d)->load(
1661          sub.cur->values, idx, swz, shiftAddress(ptr));
1662    }
1663 }
1664
1665 Value *
1666 Converter::acquireDst(int d, int c)
1667 {
1668    const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
1669    const unsigned f = dst.getFile();
1670    const int idx = dst.getIndex(0);
1671    const int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
1672
1673    if (dst.isMasked(c) || f == TGSI_FILE_RESOURCE)
1674       return NULL;
1675
1676    if (dst.isIndirect(0) ||
1677        f == TGSI_FILE_SYSTEM_VALUE ||
1678        (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT))
1679       return getScratch();
1680
1681    return getArrayForFile(f, idx2d)-> acquire(sub.cur->values, idx, c);
1682 }
1683
1684 void
1685 Converter::storeDst(int d, int c, Value *val)
1686 {
1687    const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
1688
1689    if (tgsi.getSaturate()) {
1690       mkOp1(OP_SAT, dstTy, val, val);
1691    }
1692
1693    Value *ptr = NULL;
1694    if (dst.isIndirect(0))
1695       ptr = shiftAddress(fetchSrc(dst.getIndirect(0), 0, NULL));
1696
1697    if (info->io.genUserClip > 0 &&
1698        dst.getFile() == TGSI_FILE_OUTPUT &&
1699        !dst.isIndirect(0) && dst.getIndex(0) == code->clipVertexOutput) {
1700       mkMov(clipVtx[c], val);
1701       val = clipVtx[c];
1702    }
1703
1704    storeDst(dst, c, val, ptr);
1705 }
1706
1707 void
1708 Converter::storeDst(const tgsi::Instruction::DstRegister dst, int c,
1709                     Value *val, Value *ptr)
1710 {
1711    const unsigned f = dst.getFile();
1712    const int idx = dst.getIndex(0);
1713    const int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
1714
1715    if (f == TGSI_FILE_SYSTEM_VALUE) {
1716       assert(!ptr);
1717       mkOp2(OP_WRSV, TYPE_U32, NULL, dstToSym(dst, c), val);
1718    } else
1719    if (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT) {
1720
1721       if (ptr || (info->out[idx].mask & (1 << c))) {
1722          /* Save the viewport index into a scratch register so that it can be
1723             exported at EMIT time */
1724          if (info->out[idx].sn == TGSI_SEMANTIC_VIEWPORT_INDEX &&
1725              viewport != NULL)
1726             mkOp1(OP_MOV, TYPE_U32, viewport, val);
1727          else
1728             mkStore(OP_EXPORT, TYPE_U32, dstToSym(dst, c), ptr, val)->perPatch =
1729                info->out[idx].patch;
1730       }
1731    } else
1732    if (f == TGSI_FILE_TEMPORARY ||
1733        f == TGSI_FILE_PREDICATE ||
1734        f == TGSI_FILE_ADDRESS ||
1735        f == TGSI_FILE_OUTPUT) {
1736       getArrayForFile(f, idx2d)->store(sub.cur->values, idx, c, ptr, val);
1737    } else {
1738       assert(!"invalid dst file");
1739    }
1740 }
1741
1742 #define FOR_EACH_DST_ENABLED_CHANNEL(d, chan, inst) \
1743    for (chan = 0; chan < 4; ++chan)                 \
1744       if (!inst.getDst(d).isMasked(chan))
1745
1746 Value *
1747 Converter::buildDot(int dim)
1748 {
1749    assert(dim > 0);
1750
1751    Value *src0 = fetchSrc(0, 0), *src1 = fetchSrc(1, 0);
1752    Value *dotp = getScratch();
1753
1754    mkOp2(OP_MUL, TYPE_F32, dotp, src0, src1);
1755
1756    for (int c = 1; c < dim; ++c) {
1757       src0 = fetchSrc(0, c);
1758       src1 = fetchSrc(1, c);
1759       mkOp3(OP_MAD, TYPE_F32, dotp, src0, src1, dotp);
1760    }
1761    return dotp;
1762 }
1763
1764 void
1765 Converter::insertConvergenceOps(BasicBlock *conv, BasicBlock *fork)
1766 {
1767    FlowInstruction *join = new_FlowInstruction(func, OP_JOIN, NULL);
1768    join->fixed = 1;
1769    conv->insertHead(join);
1770
1771    assert(!fork->joinAt);
1772    fork->joinAt = new_FlowInstruction(func, OP_JOINAT, conv);
1773    fork->insertBefore(fork->getExit(), fork->joinAt);
1774 }
1775
1776 void
1777 Converter::setTexRS(TexInstruction *tex, unsigned int& s, int R, int S)
1778 {
1779    unsigned rIdx = 0, sIdx = 0;
1780
1781    if (R >= 0)
1782       rIdx = tgsi.getSrc(R).getIndex(0);
1783    if (S >= 0)
1784       sIdx = tgsi.getSrc(S).getIndex(0);
1785
1786    tex->setTexture(tgsi.getTexture(code, R), rIdx, sIdx);
1787
1788    if (tgsi.getSrc(R).isIndirect(0)) {
1789       tex->tex.rIndirectSrc = s;
1790       tex->setSrc(s++, fetchSrc(tgsi.getSrc(R).getIndirect(0), 0, NULL));
1791    }
1792    if (S >= 0 && tgsi.getSrc(S).isIndirect(0)) {
1793       tex->tex.sIndirectSrc = s;
1794       tex->setSrc(s++, fetchSrc(tgsi.getSrc(S).getIndirect(0), 0, NULL));
1795    }
1796 }
1797
1798 void
1799 Converter::handleTXQ(Value *dst0[4], enum TexQuery query, int R)
1800 {
1801    TexInstruction *tex = new_TexInstruction(func, OP_TXQ);
1802    tex->tex.query = query;
1803    unsigned int c, d;
1804
1805    for (d = 0, c = 0; c < 4; ++c) {
1806       if (!dst0[c])
1807          continue;
1808       tex->tex.mask |= 1 << c;
1809       tex->setDef(d++, dst0[c]);
1810    }
1811    if (query == TXQ_DIMS)
1812       tex->setSrc((c = 0), fetchSrc(0, 0)); // mip level
1813    else
1814       tex->setSrc((c = 0), zero);
1815
1816    setTexRS(tex, ++c, R, -1);
1817
1818    bb->insertTail(tex);
1819 }
1820
1821 void
1822 Converter::loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask)
1823 {
1824    Value *proj = fetchSrc(0, 3);
1825    Instruction *insn = proj->getUniqueInsn();
1826    int c;
1827
1828    if (insn->op == OP_PINTERP) {
1829       bb->insertTail(insn = cloneForward(func, insn));
1830       insn->op = OP_LINTERP;
1831       insn->setInterpolate(NV50_IR_INTERP_LINEAR | insn->getSampleMode());
1832       insn->setSrc(1, NULL);
1833       proj = insn->getDef(0);
1834    }
1835    proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), proj);
1836
1837    for (c = 0; c < 4; ++c) {
1838       if (!(mask & (1 << c)))
1839          continue;
1840       if ((insn = src[c]->getUniqueInsn())->op != OP_PINTERP)
1841          continue;
1842       mask &= ~(1 << c);
1843
1844       bb->insertTail(insn = cloneForward(func, insn));
1845       insn->setInterpolate(NV50_IR_INTERP_PERSPECTIVE | insn->getSampleMode());
1846       insn->setSrc(1, proj);
1847       dst[c] = insn->getDef(0);
1848    }
1849    if (!mask)
1850       return;
1851
1852    proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), fetchSrc(0, 3));
1853
1854    for (c = 0; c < 4; ++c)
1855       if (mask & (1 << c))
1856          dst[c] = mkOp2v(OP_MUL, TYPE_F32, getSSA(), src[c], proj);
1857 }
1858
1859 // order of nv50 ir sources: x y z layer lod/bias shadow
1860 // order of TGSI TEX sources: x y z layer shadow lod/bias
1861 //  lowering will finally set the hw specific order (like array first on nvc0)
1862 void
1863 Converter::handleTEX(Value *dst[4], int R, int S, int L, int C, int Dx, int Dy)
1864 {
1865    Value *val;
1866    Value *arg[4], *src[8];
1867    Value *lod = NULL, *shd = NULL;
1868    unsigned int s, c, d;
1869    TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
1870
1871    TexInstruction::Target tgt = tgsi.getTexture(code, R);
1872
1873    for (s = 0; s < tgt.getArgCount(); ++s)
1874       arg[s] = src[s] = fetchSrc(0, s);
1875
1876    if (texi->op == OP_TXL || texi->op == OP_TXB)
1877       lod = fetchSrc(L >> 4, L & 3);
1878
1879    if (C == 0x0f)
1880       C = 0x00 | MAX2(tgt.getArgCount(), 2); // guess DC src
1881
1882    if (tgsi.getOpcode() == TGSI_OPCODE_TG4 &&
1883        tgt == TEX_TARGET_CUBE_ARRAY_SHADOW)
1884       shd = fetchSrc(1, 0);
1885    else if (tgt.isShadow())
1886       shd = fetchSrc(C >> 4, C & 3);
1887
1888    if (texi->op == OP_TXD) {
1889       for (c = 0; c < tgt.getDim(); ++c) {
1890          texi->dPdx[c].set(fetchSrc(Dx >> 4, (Dx & 3) + c));
1891          texi->dPdy[c].set(fetchSrc(Dy >> 4, (Dy & 3) + c));
1892       }
1893    }
1894
1895    // cube textures don't care about projection value, it's divided out
1896    if (tgsi.getOpcode() == TGSI_OPCODE_TXP && !tgt.isCube() && !tgt.isArray()) {
1897       unsigned int n = tgt.getDim();
1898       if (shd) {
1899          arg[n] = shd;
1900          ++n;
1901          assert(tgt.getDim() == tgt.getArgCount());
1902       }
1903       loadProjTexCoords(src, arg, (1 << n) - 1);
1904       if (shd)
1905          shd = src[n - 1];
1906    }
1907
1908    if (tgt.isCube()) {
1909       for (c = 0; c < 3; ++c)
1910          src[c] = mkOp1v(OP_ABS, TYPE_F32, getSSA(), arg[c]);
1911       val = getScratch();
1912       mkOp2(OP_MAX, TYPE_F32, val, src[0], src[1]);
1913       mkOp2(OP_MAX, TYPE_F32, val, src[2], val);
1914       mkOp1(OP_RCP, TYPE_F32, val, val);
1915       for (c = 0; c < 3; ++c)
1916          src[c] = mkOp2v(OP_MUL, TYPE_F32, getSSA(), arg[c], val);
1917    }
1918
1919    for (c = 0, d = 0; c < 4; ++c) {
1920       if (dst[c]) {
1921          texi->setDef(d++, dst[c]);
1922          texi->tex.mask |= 1 << c;
1923       } else {
1924          // NOTE: maybe hook up def too, for CSE
1925       }
1926    }
1927    for (s = 0; s < tgt.getArgCount(); ++s)
1928       texi->setSrc(s, src[s]);
1929    if (lod)
1930       texi->setSrc(s++, lod);
1931    if (shd)
1932       texi->setSrc(s++, shd);
1933
1934    setTexRS(texi, s, R, S);
1935
1936    if (tgsi.getOpcode() == TGSI_OPCODE_SAMPLE_C_LZ)
1937       texi->tex.levelZero = true;
1938    if (tgsi.getOpcode() == TGSI_OPCODE_TG4 && !tgt.isShadow())
1939       texi->tex.gatherComp = tgsi.getSrc(1).getValueU32(0, info);
1940
1941    texi->tex.useOffsets = tgsi.getNumTexOffsets();
1942    for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
1943       for (c = 0; c < 3; ++c) {
1944          texi->offset[s][c].set(fetchSrc(tgsi.getTexOffset(s), c, NULL));
1945          texi->offset[s][c].setInsn(texi);
1946       }
1947    }
1948
1949    bb->insertTail(texi);
1950 }
1951
1952 // 1st source: xyz = coordinates, w = lod/sample
1953 // 2nd source: offset
1954 void
1955 Converter::handleTXF(Value *dst[4], int R, int L_M)
1956 {
1957    TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
1958    int ms;
1959    unsigned int c, d, s;
1960
1961    texi->tex.target = tgsi.getTexture(code, R);
1962
1963    ms = texi->tex.target.isMS() ? 1 : 0;
1964    texi->tex.levelZero = ms; /* MS textures don't have mip-maps */
1965
1966    for (c = 0, d = 0; c < 4; ++c) {
1967       if (dst[c]) {
1968          texi->setDef(d++, dst[c]);
1969          texi->tex.mask |= 1 << c;
1970       }
1971    }
1972    for (c = 0; c < (texi->tex.target.getArgCount() - ms); ++c)
1973       texi->setSrc(c, fetchSrc(0, c));
1974    texi->setSrc(c++, fetchSrc(L_M >> 4, L_M & 3)); // lod or ms
1975
1976    setTexRS(texi, c, R, -1);
1977
1978    texi->tex.useOffsets = tgsi.getNumTexOffsets();
1979    for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
1980       for (c = 0; c < 3; ++c) {
1981          texi->offset[s][c].set(fetchSrc(tgsi.getTexOffset(s), c, NULL));
1982          texi->offset[s][c].setInsn(texi);
1983       }
1984    }
1985
1986    bb->insertTail(texi);
1987 }
1988
1989 void
1990 Converter::handleLIT(Value *dst0[4])
1991 {
1992    Value *val0 = NULL;
1993    unsigned int mask = tgsi.getDst(0).getMask();
1994
1995    if (mask & (1 << 0))
1996       loadImm(dst0[0], 1.0f);
1997
1998    if (mask & (1 << 3))
1999       loadImm(dst0[3], 1.0f);
2000
2001    if (mask & (3 << 1)) {
2002       val0 = getScratch();
2003       mkOp2(OP_MAX, TYPE_F32, val0, fetchSrc(0, 0), zero);
2004       if (mask & (1 << 1))
2005          mkMov(dst0[1], val0);
2006    }
2007
2008    if (mask & (1 << 2)) {
2009       Value *src1 = fetchSrc(0, 1), *src3 = fetchSrc(0, 3);
2010       Value *val1 = getScratch(), *val3 = getScratch();
2011
2012       Value *pos128 = loadImm(NULL, +127.999999f);
2013       Value *neg128 = loadImm(NULL, -127.999999f);
2014
2015       mkOp2(OP_MAX, TYPE_F32, val1, src1, zero);
2016       mkOp2(OP_MAX, TYPE_F32, val3, src3, neg128);
2017       mkOp2(OP_MIN, TYPE_F32, val3, val3, pos128);
2018       mkOp2(OP_POW, TYPE_F32, val3, val1, val3);
2019
2020       mkCmp(OP_SLCT, CC_GT, TYPE_F32, dst0[2], TYPE_F32, val3, zero, val0);
2021    }
2022 }
2023
2024 static inline bool
2025 isResourceSpecial(const int r)
2026 {
2027    return (r == TGSI_RESOURCE_GLOBAL ||
2028            r == TGSI_RESOURCE_LOCAL ||
2029            r == TGSI_RESOURCE_PRIVATE ||
2030            r == TGSI_RESOURCE_INPUT);
2031 }
2032
2033 static inline bool
2034 isResourceRaw(const tgsi::Source *code, const int r)
2035 {
2036    return isResourceSpecial(r) || code->resources[r].raw;
2037 }
2038
2039 static inline nv50_ir::TexTarget
2040 getResourceTarget(const tgsi::Source *code, int r)
2041 {
2042    if (isResourceSpecial(r))
2043       return nv50_ir::TEX_TARGET_BUFFER;
2044    return tgsi::translateTexture(code->resources.at(r).target);
2045 }
2046
2047 Symbol *
2048 Converter::getResourceBase(const int r)
2049 {
2050    Symbol *sym = NULL;
2051
2052    switch (r) {
2053    case TGSI_RESOURCE_GLOBAL:
2054       sym = new_Symbol(prog, nv50_ir::FILE_MEMORY_GLOBAL, 15);
2055       break;
2056    case TGSI_RESOURCE_LOCAL:
2057       assert(prog->getType() == Program::TYPE_COMPUTE);
2058       sym = mkSymbol(nv50_ir::FILE_MEMORY_SHARED, 0, TYPE_U32,
2059                      info->prop.cp.sharedOffset);
2060       break;
2061    case TGSI_RESOURCE_PRIVATE:
2062       sym = mkSymbol(nv50_ir::FILE_MEMORY_LOCAL, 0, TYPE_U32,
2063                      info->bin.tlsSpace);
2064       break;
2065    case TGSI_RESOURCE_INPUT:
2066       assert(prog->getType() == Program::TYPE_COMPUTE);
2067       sym = mkSymbol(nv50_ir::FILE_SHADER_INPUT, 0, TYPE_U32,
2068                      info->prop.cp.inputOffset);
2069       break;
2070    default:
2071       sym = new_Symbol(prog,
2072                        nv50_ir::FILE_MEMORY_GLOBAL, code->resources.at(r).slot);
2073       break;
2074    }
2075    return sym;
2076 }
2077
2078 void
2079 Converter::getResourceCoords(std::vector<Value *> &coords, int r, int s)
2080 {
2081    const int arg =
2082       TexInstruction::Target(getResourceTarget(code, r)).getArgCount();
2083
2084    for (int c = 0; c < arg; ++c)
2085       coords.push_back(fetchSrc(s, c));
2086
2087    // NOTE: TGSI_RESOURCE_GLOBAL needs FILE_GPR; this is an nv50 quirk
2088    if (r == TGSI_RESOURCE_LOCAL ||
2089        r == TGSI_RESOURCE_PRIVATE ||
2090        r == TGSI_RESOURCE_INPUT)
2091       coords[0] = mkOp1v(OP_MOV, TYPE_U32, getScratch(4, FILE_ADDRESS),
2092                          coords[0]);
2093 }
2094
2095 static inline int
2096 partitionLoadStore(uint8_t comp[2], uint8_t size[2], uint8_t mask)
2097 {
2098    int n = 0;
2099
2100    while (mask) {
2101       if (mask & 1) {
2102          size[n]++;
2103       } else {
2104          if (size[n])
2105             comp[n = 1] = size[0] + 1;
2106          else
2107             comp[n]++;
2108       }
2109       mask >>= 1;
2110    }
2111    if (size[0] == 3) {
2112       n = 1;
2113       size[0] = (comp[0] == 1) ? 1 : 2;
2114       size[1] = 3 - size[0];
2115       comp[1] = comp[0] + size[0];
2116    }
2117    return n + 1;
2118 }
2119
2120 // For raw loads, granularity is 4 byte.
2121 // Usage of the texture read mask on OP_SULDP is not allowed.
2122 void
2123 Converter::handleLOAD(Value *dst0[4])
2124 {
2125    const int r = tgsi.getSrc(0).getIndex(0);
2126    int c;
2127    std::vector<Value *> off, src, ldv, def;
2128
2129    getResourceCoords(off, r, 1);
2130
2131    if (isResourceRaw(code, r)) {
2132       uint8_t mask = 0;
2133       uint8_t comp[2] = { 0, 0 };
2134       uint8_t size[2] = { 0, 0 };
2135
2136       Symbol *base = getResourceBase(r);
2137
2138       // determine the base and size of the at most 2 load ops
2139       for (c = 0; c < 4; ++c)
2140          if (!tgsi.getDst(0).isMasked(c))
2141             mask |= 1 << (tgsi.getSrc(0).getSwizzle(c) - TGSI_SWIZZLE_X);
2142
2143       int n = partitionLoadStore(comp, size, mask);
2144
2145       src = off;
2146
2147       def.resize(4); // index by component, the ones we need will be non-NULL
2148       for (c = 0; c < 4; ++c) {
2149          if (dst0[c] && tgsi.getSrc(0).getSwizzle(c) == (TGSI_SWIZZLE_X + c))
2150             def[c] = dst0[c];
2151          else
2152          if (mask & (1 << c))
2153             def[c] = getScratch();
2154       }
2155
2156       const bool useLd = isResourceSpecial(r) ||
2157          (info->io.nv50styleSurfaces &&
2158           code->resources[r].target == TGSI_TEXTURE_BUFFER);
2159
2160       for (int i = 0; i < n; ++i) {
2161          ldv.assign(def.begin() + comp[i], def.begin() + comp[i] + size[i]);
2162
2163          if (comp[i]) // adjust x component of source address if necessary
2164             src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
2165                             off[0], mkImm(comp[i] * 4));
2166          else
2167             src[0] = off[0];
2168
2169          if (useLd) {
2170             Instruction *ld =
2171                mkLoad(typeOfSize(size[i] * 4), ldv[0], base, src[0]);
2172             for (size_t c = 1; c < ldv.size(); ++c)
2173                ld->setDef(c, ldv[c]);
2174          } else {
2175             mkTex(OP_SULDB, getResourceTarget(code, r), code->resources[r].slot,
2176                   0, ldv, src)->dType = typeOfSize(size[i] * 4);
2177          }
2178       }
2179    } else {
2180       def.resize(4);
2181       for (c = 0; c < 4; ++c) {
2182          if (!dst0[c] || tgsi.getSrc(0).getSwizzle(c) != (TGSI_SWIZZLE_X + c))
2183             def[c] = getScratch();
2184          else
2185             def[c] = dst0[c];
2186       }
2187
2188       mkTex(OP_SULDP, getResourceTarget(code, r), code->resources[r].slot, 0,
2189             def, off);
2190    }
2191    FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2192       if (dst0[c] != def[c])
2193          mkMov(dst0[c], def[tgsi.getSrc(0).getSwizzle(c)]);
2194 }
2195
2196 // For formatted stores, the write mask on OP_SUSTP can be used.
2197 // Raw stores have to be split.
2198 void
2199 Converter::handleSTORE()
2200 {
2201    const int r = tgsi.getDst(0).getIndex(0);
2202    int c;
2203    std::vector<Value *> off, src, dummy;
2204
2205    getResourceCoords(off, r, 0);
2206    src = off;
2207    const int s = src.size();
2208
2209    if (isResourceRaw(code, r)) {
2210       uint8_t comp[2] = { 0, 0 };
2211       uint8_t size[2] = { 0, 0 };
2212
2213       int n = partitionLoadStore(comp, size, tgsi.getDst(0).getMask());
2214
2215       Symbol *base = getResourceBase(r);
2216
2217       const bool useSt = isResourceSpecial(r) ||
2218          (info->io.nv50styleSurfaces &&
2219           code->resources[r].target == TGSI_TEXTURE_BUFFER);
2220
2221       for (int i = 0; i < n; ++i) {
2222          if (comp[i]) // adjust x component of source address if necessary
2223             src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
2224                             off[0], mkImm(comp[i] * 4));
2225          else
2226             src[0] = off[0];
2227
2228          const DataType stTy = typeOfSize(size[i] * 4);
2229
2230          if (useSt) {
2231             Instruction *st =
2232                mkStore(OP_STORE, stTy, base, NULL, fetchSrc(1, comp[i]));
2233             for (c = 1; c < size[i]; ++c)
2234                st->setSrc(1 + c, fetchSrc(1, comp[i] + c));
2235             st->setIndirect(0, 0, src[0]);
2236          } else {
2237             // attach values to be stored
2238             src.resize(s + size[i]);
2239             for (c = 0; c < size[i]; ++c)
2240                src[s + c] = fetchSrc(1, comp[i] + c);
2241             mkTex(OP_SUSTB, getResourceTarget(code, r), code->resources[r].slot,
2242                   0, dummy, src)->setType(stTy);
2243          }
2244       }
2245    } else {
2246       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2247          src.push_back(fetchSrc(1, c));
2248
2249       mkTex(OP_SUSTP, getResourceTarget(code, r), code->resources[r].slot, 0,
2250             dummy, src)->tex.mask = tgsi.getDst(0).getMask();
2251    }
2252 }
2253
2254 // XXX: These only work on resources with the single-component u32/s32 formats.
2255 // Therefore the result is replicated. This might not be intended by TGSI, but
2256 // operating on more than 1 component would produce undefined results because
2257 // they do not exist.
2258 void
2259 Converter::handleATOM(Value *dst0[4], DataType ty, uint16_t subOp)
2260 {
2261    const int r = tgsi.getSrc(0).getIndex(0);
2262    std::vector<Value *> srcv;
2263    std::vector<Value *> defv;
2264    LValue *dst = getScratch();
2265
2266    getResourceCoords(srcv, r, 1);
2267
2268    if (isResourceSpecial(r)) {
2269       assert(r != TGSI_RESOURCE_INPUT);
2270       Instruction *insn;
2271       insn = mkOp2(OP_ATOM, ty, dst, getResourceBase(r), fetchSrc(2, 0));
2272       insn->subOp = subOp;
2273       if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2274          insn->setSrc(2, fetchSrc(3, 0));
2275       insn->setIndirect(0, 0, srcv.at(0));
2276    } else {
2277       operation op = isResourceRaw(code, r) ? OP_SUREDB : OP_SUREDP;
2278       TexTarget targ = getResourceTarget(code, r);
2279       int idx = code->resources[r].slot;
2280       defv.push_back(dst);
2281       srcv.push_back(fetchSrc(2, 0));
2282       if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2283          srcv.push_back(fetchSrc(3, 0));
2284       TexInstruction *tex = mkTex(op, targ, idx, 0, defv, srcv);
2285       tex->subOp = subOp;
2286       tex->tex.mask = 1;
2287       tex->setType(ty);
2288    }
2289
2290    for (int c = 0; c < 4; ++c)
2291       if (dst0[c])
2292          dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov
2293 }
2294
2295 void
2296 Converter::handleINTERP(Value *dst[4])
2297 {
2298    // Check whether the input is linear. All other attributes ignored.
2299    Instruction *insn;
2300    Value *offset = NULL, *ptr = NULL, *w = NULL;
2301    bool linear;
2302    operation op;
2303    int c, mode;
2304
2305    tgsi::Instruction::SrcRegister src = tgsi.getSrc(0);
2306    assert(src.getFile() == TGSI_FILE_INPUT);
2307
2308    if (src.isIndirect(0))
2309       ptr = fetchSrc(src.getIndirect(0), 0, NULL);
2310
2311    // XXX: no way to know interp mode if we don't know the index
2312    linear = info->in[ptr ? 0 : src.getIndex(0)].linear;
2313    if (linear) {
2314       op = OP_LINTERP;
2315       mode = NV50_IR_INTERP_LINEAR;
2316    } else {
2317       op = OP_PINTERP;
2318       mode = NV50_IR_INTERP_PERSPECTIVE;
2319    }
2320
2321    switch (tgsi.getOpcode()) {
2322    case TGSI_OPCODE_INTERP_CENTROID:
2323       mode |= NV50_IR_INTERP_CENTROID;
2324       break;
2325    case TGSI_OPCODE_INTERP_SAMPLE:
2326       insn = mkOp1(OP_PIXLD, TYPE_U32, (offset = getScratch()), fetchSrc(1, 0));
2327       insn->subOp = NV50_IR_SUBOP_PIXLD_OFFSET;
2328       mode |= NV50_IR_INTERP_OFFSET;
2329       break;
2330    case TGSI_OPCODE_INTERP_OFFSET: {
2331       // The input in src1.xy is float, but we need a single 32-bit value
2332       // where the upper and lower 16 bits are encoded in S0.12 format. We need
2333       // to clamp the input coordinates to (-0.5, 0.4375), multiply by 4096,
2334       // and then convert to s32.
2335       Value *offs[2];
2336       for (c = 0; c < 2; c++) {
2337          offs[c] = fetchSrc(1, c);
2338          mkOp2(OP_MIN, TYPE_F32, offs[c], offs[c], loadImm(NULL, 0.4375f));
2339          mkOp2(OP_MAX, TYPE_F32, offs[c], offs[c], loadImm(NULL, -0.5f));
2340          mkOp2(OP_MUL, TYPE_F32, offs[c], offs[c], loadImm(NULL, 4096.0f));
2341          mkCvt(OP_CVT, TYPE_S32, offs[c], TYPE_F32, offs[c]);
2342       }
2343       offset = mkOp3v(OP_INSBF, TYPE_U32, getScratch(),
2344                       offs[1], mkImm(0x1010), offs[0]);
2345       mode |= NV50_IR_INTERP_OFFSET;
2346       break;
2347    }
2348    }
2349
2350    if (op == OP_PINTERP) {
2351       if (offset) {
2352          w = mkOp2v(OP_RDSV, TYPE_F32, getSSA(), mkSysVal(SV_POSITION, 3), offset);
2353          mkOp1(OP_RCP, TYPE_F32, w, w);
2354       } else {
2355          w = fragCoord[3];
2356       }
2357    }
2358
2359
2360    FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2361       insn = mkOp1(op, TYPE_F32, dst[c], srcToSym(src, c));
2362       if (op == OP_PINTERP)
2363          insn->setSrc(1, w);
2364       if (ptr)
2365          insn->setIndirect(0, 0, ptr);
2366       if (offset)
2367          insn->setSrc(op == OP_PINTERP ? 2 : 1, offset);
2368
2369       insn->setInterpolate(mode);
2370    }
2371 }
2372
2373 Converter::Subroutine *
2374 Converter::getSubroutine(unsigned ip)
2375 {
2376    std::map<unsigned, Subroutine>::iterator it = sub.map.find(ip);
2377
2378    if (it == sub.map.end())
2379       it = sub.map.insert(std::make_pair(
2380               ip, Subroutine(new Function(prog, "SUB", ip)))).first;
2381
2382    return &it->second;
2383 }
2384
2385 Converter::Subroutine *
2386 Converter::getSubroutine(Function *f)
2387 {
2388    unsigned ip = f->getLabel();
2389    std::map<unsigned, Subroutine>::iterator it = sub.map.find(ip);
2390
2391    if (it == sub.map.end())
2392       it = sub.map.insert(std::make_pair(ip, Subroutine(f))).first;
2393
2394    return &it->second;
2395 }
2396
2397 bool
2398 Converter::isEndOfSubroutine(uint ip)
2399 {
2400    assert(ip < code->scan.num_instructions);
2401    tgsi::Instruction insn(&code->insns[ip]);
2402    return (insn.getOpcode() == TGSI_OPCODE_END ||
2403            insn.getOpcode() == TGSI_OPCODE_ENDSUB ||
2404            // does END occur at end of main or the very end ?
2405            insn.getOpcode() == TGSI_OPCODE_BGNSUB);
2406 }
2407
2408 bool
2409 Converter::handleInstruction(const struct tgsi_full_instruction *insn)
2410 {
2411    Instruction *geni;
2412
2413    Value *dst0[4], *rDst0[4];
2414    Value *src0, *src1, *src2, *src3;
2415    Value *val0, *val1;
2416    int c;
2417
2418    tgsi = tgsi::Instruction(insn);
2419
2420    bool useScratchDst = tgsi.checkDstSrcAliasing();
2421
2422    operation op = tgsi.getOP();
2423    dstTy = tgsi.inferDstType();
2424    srcTy = tgsi.inferSrcType();
2425
2426    unsigned int mask = tgsi.dstCount() ? tgsi.getDst(0).getMask() : 0;
2427
2428    if (tgsi.dstCount()) {
2429       for (c = 0; c < 4; ++c) {
2430          rDst0[c] = acquireDst(0, c);
2431          dst0[c] = (useScratchDst && rDst0[c]) ? getScratch() : rDst0[c];
2432       }
2433    }
2434
2435    switch (tgsi.getOpcode()) {
2436    case TGSI_OPCODE_ADD:
2437    case TGSI_OPCODE_UADD:
2438    case TGSI_OPCODE_AND:
2439    case TGSI_OPCODE_DIV:
2440    case TGSI_OPCODE_IDIV:
2441    case TGSI_OPCODE_UDIV:
2442    case TGSI_OPCODE_MAX:
2443    case TGSI_OPCODE_MIN:
2444    case TGSI_OPCODE_IMAX:
2445    case TGSI_OPCODE_IMIN:
2446    case TGSI_OPCODE_UMAX:
2447    case TGSI_OPCODE_UMIN:
2448    case TGSI_OPCODE_MOD:
2449    case TGSI_OPCODE_UMOD:
2450    case TGSI_OPCODE_MUL:
2451    case TGSI_OPCODE_UMUL:
2452    case TGSI_OPCODE_IMUL_HI:
2453    case TGSI_OPCODE_UMUL_HI:
2454    case TGSI_OPCODE_OR:
2455    case TGSI_OPCODE_SHL:
2456    case TGSI_OPCODE_ISHR:
2457    case TGSI_OPCODE_USHR:
2458    case TGSI_OPCODE_SUB:
2459    case TGSI_OPCODE_XOR:
2460       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2461          src0 = fetchSrc(0, c);
2462          src1 = fetchSrc(1, c);
2463          geni = mkOp2(op, dstTy, dst0[c], src0, src1);
2464          geni->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode());
2465       }
2466       break;
2467    case TGSI_OPCODE_MAD:
2468    case TGSI_OPCODE_UMAD:
2469    case TGSI_OPCODE_SAD:
2470       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2471          src0 = fetchSrc(0, c);
2472          src1 = fetchSrc(1, c);
2473          src2 = fetchSrc(2, c);
2474          mkOp3(op, dstTy, dst0[c], src0, src1, src2);
2475       }
2476       break;
2477    case TGSI_OPCODE_MOV:
2478    case TGSI_OPCODE_ABS:
2479    case TGSI_OPCODE_CEIL:
2480    case TGSI_OPCODE_FLR:
2481    case TGSI_OPCODE_TRUNC:
2482    case TGSI_OPCODE_RCP:
2483    case TGSI_OPCODE_IABS:
2484    case TGSI_OPCODE_INEG:
2485    case TGSI_OPCODE_NOT:
2486    case TGSI_OPCODE_DDX:
2487    case TGSI_OPCODE_DDY:
2488    case TGSI_OPCODE_DDX_FINE:
2489    case TGSI_OPCODE_DDY_FINE:
2490       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2491          mkOp1(op, dstTy, dst0[c], fetchSrc(0, c));
2492       break;
2493    case TGSI_OPCODE_RSQ:
2494       src0 = fetchSrc(0, 0);
2495       val0 = getScratch();
2496       mkOp1(OP_ABS, TYPE_F32, val0, src0);
2497       mkOp1(OP_RSQ, TYPE_F32, val0, val0);
2498       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2499          mkMov(dst0[c], val0);
2500       break;
2501    case TGSI_OPCODE_ARL:
2502    case TGSI_OPCODE_ARR:
2503       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2504          const RoundMode rnd =
2505             tgsi.getOpcode() == TGSI_OPCODE_ARR ? ROUND_N : ROUND_M;
2506          src0 = fetchSrc(0, c);
2507          mkCvt(OP_CVT, TYPE_S32, dst0[c], TYPE_F32, src0)->rnd = rnd;
2508       }
2509       break;
2510    case TGSI_OPCODE_UARL:
2511       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2512          mkOp1(OP_MOV, TYPE_U32, dst0[c], fetchSrc(0, c));
2513       break;
2514    case TGSI_OPCODE_POW:
2515       val0 = mkOp2v(op, TYPE_F32, getScratch(), fetchSrc(0, 0), fetchSrc(1, 0));
2516       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2517          mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
2518       break;
2519    case TGSI_OPCODE_EX2:
2520    case TGSI_OPCODE_LG2:
2521       val0 = mkOp1(op, TYPE_F32, getScratch(), fetchSrc(0, 0))->getDef(0);
2522       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2523          mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
2524       break;
2525    case TGSI_OPCODE_COS:
2526    case TGSI_OPCODE_SIN:
2527       val0 = getScratch();
2528       if (mask & 7) {
2529          mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 0));
2530          mkOp1(op, TYPE_F32, val0, val0);
2531          for (c = 0; c < 3; ++c)
2532             if (dst0[c])
2533                mkMov(dst0[c], val0);
2534       }
2535       if (dst0[3]) {
2536          mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 3));
2537          mkOp1(op, TYPE_F32, dst0[3], val0);
2538       }
2539       break;
2540    case TGSI_OPCODE_SCS:
2541       if (mask & 3) {
2542          val0 = mkOp1v(OP_PRESIN, TYPE_F32, getSSA(), fetchSrc(0, 0));
2543          if (dst0[0])
2544             mkOp1(OP_COS, TYPE_F32, dst0[0], val0);
2545          if (dst0[1])
2546             mkOp1(OP_SIN, TYPE_F32, dst0[1], val0);
2547       }
2548       if (dst0[2])
2549          loadImm(dst0[2], 0.0f);
2550       if (dst0[3])
2551          loadImm(dst0[3], 1.0f);
2552       break;
2553    case TGSI_OPCODE_EXP:
2554       src0 = fetchSrc(0, 0);
2555       val0 = mkOp1v(OP_FLOOR, TYPE_F32, getSSA(), src0);
2556       if (dst0[1])
2557          mkOp2(OP_SUB, TYPE_F32, dst0[1], src0, val0);
2558       if (dst0[0])
2559          mkOp1(OP_EX2, TYPE_F32, dst0[0], val0);
2560       if (dst0[2])
2561          mkOp1(OP_EX2, TYPE_F32, dst0[2], src0);
2562       if (dst0[3])
2563          loadImm(dst0[3], 1.0f);
2564       break;
2565    case TGSI_OPCODE_LOG:
2566       src0 = mkOp1v(OP_ABS, TYPE_F32, getSSA(), fetchSrc(0, 0));
2567       val0 = mkOp1v(OP_LG2, TYPE_F32, dst0[2] ? dst0[2] : getSSA(), src0);
2568       if (dst0[0] || dst0[1])
2569          val1 = mkOp1v(OP_FLOOR, TYPE_F32, dst0[0] ? dst0[0] : getSSA(), val0);
2570       if (dst0[1]) {
2571          mkOp1(OP_EX2, TYPE_F32, dst0[1], val1);
2572          mkOp1(OP_RCP, TYPE_F32, dst0[1], dst0[1]);
2573          mkOp2(OP_MUL, TYPE_F32, dst0[1], dst0[1], src0);
2574       }
2575       if (dst0[3])
2576          loadImm(dst0[3], 1.0f);
2577       break;
2578    case TGSI_OPCODE_DP2:
2579       val0 = buildDot(2);
2580       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2581          mkMov(dst0[c], val0);
2582       break;
2583    case TGSI_OPCODE_DP3:
2584       val0 = buildDot(3);
2585       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2586          mkMov(dst0[c], val0);
2587       break;
2588    case TGSI_OPCODE_DP4:
2589       val0 = buildDot(4);
2590       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2591          mkMov(dst0[c], val0);
2592       break;
2593    case TGSI_OPCODE_DPH:
2594       val0 = buildDot(3);
2595       src1 = fetchSrc(1, 3);
2596       mkOp2(OP_ADD, TYPE_F32, val0, val0, src1);
2597       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2598          mkMov(dst0[c], val0);
2599       break;
2600    case TGSI_OPCODE_DST:
2601       if (dst0[0])
2602          loadImm(dst0[0], 1.0f);
2603       if (dst0[1]) {
2604          src0 = fetchSrc(0, 1);
2605          src1 = fetchSrc(1, 1);
2606          mkOp2(OP_MUL, TYPE_F32, dst0[1], src0, src1);
2607       }
2608       if (dst0[2])
2609          mkMov(dst0[2], fetchSrc(0, 2));
2610       if (dst0[3])
2611          mkMov(dst0[3], fetchSrc(1, 3));
2612       break;
2613    case TGSI_OPCODE_LRP:
2614       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2615          src0 = fetchSrc(0, c);
2616          src1 = fetchSrc(1, c);
2617          src2 = fetchSrc(2, c);
2618          mkOp3(OP_MAD, TYPE_F32, dst0[c],
2619                mkOp2v(OP_SUB, TYPE_F32, getSSA(), src1, src2), src0, src2);
2620       }
2621       break;
2622    case TGSI_OPCODE_LIT:
2623       handleLIT(dst0);
2624       break;
2625    case TGSI_OPCODE_XPD:
2626       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2627          if (c < 3) {
2628             val0 = getSSA();
2629             src0 = fetchSrc(1, (c + 1) % 3);
2630             src1 = fetchSrc(0, (c + 2) % 3);
2631             mkOp2(OP_MUL, TYPE_F32, val0, src0, src1);
2632             mkOp1(OP_NEG, TYPE_F32, val0, val0);
2633
2634             src0 = fetchSrc(0, (c + 1) % 3);
2635             src1 = fetchSrc(1, (c + 2) % 3);
2636             mkOp3(OP_MAD, TYPE_F32, dst0[c], src0, src1, val0);
2637          } else {
2638             loadImm(dst0[c], 1.0f);
2639          }
2640       }
2641       break;
2642    case TGSI_OPCODE_ISSG:
2643    case TGSI_OPCODE_SSG:
2644       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2645          src0 = fetchSrc(0, c);
2646          val0 = getScratch();
2647          val1 = getScratch();
2648          mkCmp(OP_SET, CC_GT, srcTy, val0, srcTy, src0, zero);
2649          mkCmp(OP_SET, CC_LT, srcTy, val1, srcTy, src0, zero);
2650          if (srcTy == TYPE_F32)
2651             mkOp2(OP_SUB, TYPE_F32, dst0[c], val0, val1);
2652          else
2653             mkOp2(OP_SUB, TYPE_S32, dst0[c], val1, val0);
2654       }
2655       break;
2656    case TGSI_OPCODE_UCMP:
2657       srcTy = TYPE_U32;
2658       /* fallthrough */
2659    case TGSI_OPCODE_CMP:
2660       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2661          src0 = fetchSrc(0, c);
2662          src1 = fetchSrc(1, c);
2663          src2 = fetchSrc(2, c);
2664          if (src1 == src2)
2665             mkMov(dst0[c], src1);
2666          else
2667             mkCmp(OP_SLCT, (srcTy == TYPE_F32) ? CC_LT : CC_NE,
2668                   srcTy, dst0[c], srcTy, src1, src2, src0);
2669       }
2670       break;
2671    case TGSI_OPCODE_FRC:
2672       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2673          src0 = fetchSrc(0, c);
2674          val0 = getScratch();
2675          mkOp1(OP_FLOOR, TYPE_F32, val0, src0);
2676          mkOp2(OP_SUB, TYPE_F32, dst0[c], src0, val0);
2677       }
2678       break;
2679    case TGSI_OPCODE_ROUND:
2680       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2681          mkCvt(OP_CVT, TYPE_F32, dst0[c], TYPE_F32, fetchSrc(0, c))
2682          ->rnd = ROUND_NI;
2683       break;
2684    case TGSI_OPCODE_CLAMP:
2685       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2686          src0 = fetchSrc(0, c);
2687          src1 = fetchSrc(1, c);
2688          src2 = fetchSrc(2, c);
2689          val0 = getScratch();
2690          mkOp2(OP_MIN, TYPE_F32, val0, src0, src1);
2691          mkOp2(OP_MAX, TYPE_F32, dst0[c], val0, src2);
2692       }
2693       break;
2694    case TGSI_OPCODE_SLT:
2695    case TGSI_OPCODE_SGE:
2696    case TGSI_OPCODE_SEQ:
2697    case TGSI_OPCODE_SGT:
2698    case TGSI_OPCODE_SLE:
2699    case TGSI_OPCODE_SNE:
2700    case TGSI_OPCODE_FSEQ:
2701    case TGSI_OPCODE_FSGE:
2702    case TGSI_OPCODE_FSLT:
2703    case TGSI_OPCODE_FSNE:
2704    case TGSI_OPCODE_ISGE:
2705    case TGSI_OPCODE_ISLT:
2706    case TGSI_OPCODE_USEQ:
2707    case TGSI_OPCODE_USGE:
2708    case TGSI_OPCODE_USLT:
2709    case TGSI_OPCODE_USNE:
2710       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2711          src0 = fetchSrc(0, c);
2712          src1 = fetchSrc(1, c);
2713          mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], srcTy, src0, src1);
2714       }
2715       break;
2716    case TGSI_OPCODE_KILL_IF:
2717       val0 = new_LValue(func, FILE_PREDICATE);
2718       mask = 0;
2719       for (c = 0; c < 4; ++c) {
2720          const int s = tgsi.getSrc(0).getSwizzle(c);
2721          if (mask & (1 << s))
2722             continue;
2723          mask |= 1 << s;
2724          mkCmp(OP_SET, CC_LT, TYPE_F32, val0, TYPE_F32, fetchSrc(0, c), zero);
2725          mkOp(OP_DISCARD, TYPE_NONE, NULL)->setPredicate(CC_P, val0);
2726       }
2727       break;
2728    case TGSI_OPCODE_KILL:
2729       mkOp(OP_DISCARD, TYPE_NONE, NULL);
2730       break;
2731    case TGSI_OPCODE_TEX:
2732    case TGSI_OPCODE_TXB:
2733    case TGSI_OPCODE_TXL:
2734    case TGSI_OPCODE_TXP:
2735    case TGSI_OPCODE_LODQ:
2736       //              R  S     L     C    Dx    Dy
2737       handleTEX(dst0, 1, 1, 0x03, 0x0f, 0x00, 0x00);
2738       break;
2739    case TGSI_OPCODE_TXD:
2740       handleTEX(dst0, 3, 3, 0x03, 0x0f, 0x10, 0x20);
2741       break;
2742    case TGSI_OPCODE_TG4:
2743       handleTEX(dst0, 2, 2, 0x03, 0x0f, 0x00, 0x00);
2744       break;
2745    case TGSI_OPCODE_TEX2:
2746       handleTEX(dst0, 2, 2, 0x03, 0x10, 0x00, 0x00);
2747       break;
2748    case TGSI_OPCODE_TXB2:
2749    case TGSI_OPCODE_TXL2:
2750       handleTEX(dst0, 2, 2, 0x10, 0x0f, 0x00, 0x00);
2751       break;
2752    case TGSI_OPCODE_SAMPLE:
2753    case TGSI_OPCODE_SAMPLE_B:
2754    case TGSI_OPCODE_SAMPLE_D:
2755    case TGSI_OPCODE_SAMPLE_L:
2756    case TGSI_OPCODE_SAMPLE_C:
2757    case TGSI_OPCODE_SAMPLE_C_LZ:
2758       handleTEX(dst0, 1, 2, 0x30, 0x30, 0x30, 0x40);
2759       break;
2760    case TGSI_OPCODE_TXF:
2761       handleTXF(dst0, 1, 0x03);
2762       break;
2763    case TGSI_OPCODE_SAMPLE_I:
2764       handleTXF(dst0, 1, 0x03);
2765       break;
2766    case TGSI_OPCODE_SAMPLE_I_MS:
2767       handleTXF(dst0, 1, 0x20);
2768       break;
2769    case TGSI_OPCODE_TXQ:
2770    case TGSI_OPCODE_SVIEWINFO:
2771       handleTXQ(dst0, TXQ_DIMS, 1);
2772       break;
2773    case TGSI_OPCODE_TXQS:
2774       // The TXQ_TYPE query returns samples in its 3rd arg, but we need it to
2775       // be in .x
2776       dst0[1] = dst0[2] = dst0[3] = NULL;
2777       std::swap(dst0[0], dst0[2]);
2778       handleTXQ(dst0, TXQ_TYPE, 0);
2779       std::swap(dst0[0], dst0[2]);
2780       break;
2781    case TGSI_OPCODE_F2I:
2782    case TGSI_OPCODE_F2U:
2783       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2784          mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c))->rnd = ROUND_Z;
2785       break;
2786    case TGSI_OPCODE_I2F:
2787    case TGSI_OPCODE_U2F:
2788       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2789          mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c));
2790       break;
2791    case TGSI_OPCODE_EMIT:
2792       /* export the saved viewport index */
2793       if (viewport != NULL) {
2794          Symbol *vpSym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_U32,
2795                                   info->out[info->io.viewportId].slot[0] * 4);
2796          mkStore(OP_EXPORT, TYPE_U32, vpSym, NULL, viewport);
2797       }
2798       /* fallthrough */
2799    case TGSI_OPCODE_ENDPRIM:
2800    {
2801       // get vertex stream (must be immediate)
2802       unsigned int stream = tgsi.getSrc(0).getValueU32(0, info);
2803       if (stream && op == OP_RESTART)
2804          break;
2805       src0 = mkImm(stream);
2806       mkOp1(op, TYPE_U32, NULL, src0)->fixed = 1;
2807       break;
2808    }
2809    case TGSI_OPCODE_IF:
2810    case TGSI_OPCODE_UIF:
2811    {
2812       BasicBlock *ifBB = new BasicBlock(func);
2813
2814       bb->cfg.attach(&ifBB->cfg, Graph::Edge::TREE);
2815       condBBs.push(bb);
2816       joinBBs.push(bb);
2817
2818       mkFlow(OP_BRA, NULL, CC_NOT_P, fetchSrc(0, 0))->setType(srcTy);
2819
2820       setPosition(ifBB, true);
2821    }
2822       break;
2823    case TGSI_OPCODE_ELSE:
2824    {
2825       BasicBlock *elseBB = new BasicBlock(func);
2826       BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
2827
2828       forkBB->cfg.attach(&elseBB->cfg, Graph::Edge::TREE);
2829       condBBs.push(bb);
2830
2831       forkBB->getExit()->asFlow()->target.bb = elseBB;
2832       if (!bb->isTerminated())
2833          mkFlow(OP_BRA, NULL, CC_ALWAYS, NULL);
2834
2835       setPosition(elseBB, true);
2836    }
2837       break;
2838    case TGSI_OPCODE_ENDIF:
2839    {
2840       BasicBlock *convBB = new BasicBlock(func);
2841       BasicBlock *prevBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
2842       BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(joinBBs.pop().u.p);
2843
2844       if (!bb->isTerminated()) {
2845          // we only want join if none of the clauses ended with CONT/BREAK/RET
2846          if (prevBB->getExit()->op == OP_BRA && joinBBs.getSize() < 6)
2847             insertConvergenceOps(convBB, forkBB);
2848          mkFlow(OP_BRA, convBB, CC_ALWAYS, NULL);
2849          bb->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
2850       }
2851
2852       if (prevBB->getExit()->op == OP_BRA) {
2853          prevBB->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
2854          prevBB->getExit()->asFlow()->target.bb = convBB;
2855       }
2856       setPosition(convBB, true);
2857    }
2858       break;
2859    case TGSI_OPCODE_BGNLOOP:
2860    {
2861       BasicBlock *lbgnBB = new BasicBlock(func);
2862       BasicBlock *lbrkBB = new BasicBlock(func);
2863
2864       loopBBs.push(lbgnBB);
2865       breakBBs.push(lbrkBB);
2866       if (loopBBs.getSize() > func->loopNestingBound)
2867          func->loopNestingBound++;
2868
2869       mkFlow(OP_PREBREAK, lbrkBB, CC_ALWAYS, NULL);
2870
2871       bb->cfg.attach(&lbgnBB->cfg, Graph::Edge::TREE);
2872       setPosition(lbgnBB, true);
2873       mkFlow(OP_PRECONT, lbgnBB, CC_ALWAYS, NULL);
2874    }
2875       break;
2876    case TGSI_OPCODE_ENDLOOP:
2877    {
2878       BasicBlock *loopBB = reinterpret_cast<BasicBlock *>(loopBBs.pop().u.p);
2879
2880       if (!bb->isTerminated()) {
2881          mkFlow(OP_CONT, loopBB, CC_ALWAYS, NULL);
2882          bb->cfg.attach(&loopBB->cfg, Graph::Edge::BACK);
2883       }
2884       setPosition(reinterpret_cast<BasicBlock *>(breakBBs.pop().u.p), true);
2885    }
2886       break;
2887    case TGSI_OPCODE_BRK:
2888    {
2889       if (bb->isTerminated())
2890          break;
2891       BasicBlock *brkBB = reinterpret_cast<BasicBlock *>(breakBBs.peek().u.p);
2892       mkFlow(OP_BREAK, brkBB, CC_ALWAYS, NULL);
2893       bb->cfg.attach(&brkBB->cfg, Graph::Edge::CROSS);
2894    }
2895       break;
2896    case TGSI_OPCODE_CONT:
2897    {
2898       if (bb->isTerminated())
2899          break;
2900       BasicBlock *contBB = reinterpret_cast<BasicBlock *>(loopBBs.peek().u.p);
2901       mkFlow(OP_CONT, contBB, CC_ALWAYS, NULL);
2902       contBB->explicitCont = true;
2903       bb->cfg.attach(&contBB->cfg, Graph::Edge::BACK);
2904    }
2905       break;
2906    case TGSI_OPCODE_BGNSUB:
2907    {
2908       Subroutine *s = getSubroutine(ip);
2909       BasicBlock *entry = new BasicBlock(s->f);
2910       BasicBlock *leave = new BasicBlock(s->f);
2911
2912       // multiple entrypoints possible, keep the graph connected
2913       if (prog->getType() == Program::TYPE_COMPUTE)
2914          prog->main->call.attach(&s->f->call, Graph::Edge::TREE);
2915
2916       sub.cur = s;
2917       s->f->setEntry(entry);
2918       s->f->setExit(leave);
2919       setPosition(entry, true);
2920       return true;
2921    }
2922    case TGSI_OPCODE_ENDSUB:
2923    {
2924       sub.cur = getSubroutine(prog->main);
2925       setPosition(BasicBlock::get(sub.cur->f->cfg.getRoot()), true);
2926       return true;
2927    }
2928    case TGSI_OPCODE_CAL:
2929    {
2930       Subroutine *s = getSubroutine(tgsi.getLabel());
2931       mkFlow(OP_CALL, s->f, CC_ALWAYS, NULL);
2932       func->call.attach(&s->f->call, Graph::Edge::TREE);
2933       return true;
2934    }
2935    case TGSI_OPCODE_RET:
2936    {
2937       if (bb->isTerminated())
2938          return true;
2939       BasicBlock *leave = BasicBlock::get(func->cfgExit);
2940
2941       if (!isEndOfSubroutine(ip + 1)) {
2942          // insert a PRERET at the entry if this is an early return
2943          // (only needed for sharing code in the epilogue)
2944          BasicBlock *pos = getBB();
2945          setPosition(BasicBlock::get(func->cfg.getRoot()), false);
2946          mkFlow(OP_PRERET, leave, CC_ALWAYS, NULL)->fixed = 1;
2947          setPosition(pos, true);
2948       }
2949       mkFlow(OP_RET, NULL, CC_ALWAYS, NULL)->fixed = 1;
2950       bb->cfg.attach(&leave->cfg, Graph::Edge::CROSS);
2951    }
2952       break;
2953    case TGSI_OPCODE_END:
2954    {
2955       // attach and generate epilogue code
2956       BasicBlock *epilogue = BasicBlock::get(func->cfgExit);
2957       bb->cfg.attach(&epilogue->cfg, Graph::Edge::TREE);
2958       setPosition(epilogue, true);
2959       if (prog->getType() == Program::TYPE_FRAGMENT)
2960          exportOutputs();
2961       if (info->io.genUserClip > 0)
2962          handleUserClipPlanes();
2963       mkOp(OP_EXIT, TYPE_NONE, NULL)->terminator = 1;
2964    }
2965       break;
2966    case TGSI_OPCODE_SWITCH:
2967    case TGSI_OPCODE_CASE:
2968       ERROR("switch/case opcode encountered, should have been lowered\n");
2969       abort();
2970       break;
2971    case TGSI_OPCODE_LOAD:
2972       handleLOAD(dst0);
2973       break;
2974    case TGSI_OPCODE_STORE:
2975       handleSTORE();
2976       break;
2977    case TGSI_OPCODE_BARRIER:
2978       geni = mkOp2(OP_BAR, TYPE_U32, NULL, mkImm(0), mkImm(0));
2979       geni->fixed = 1;
2980       geni->subOp = NV50_IR_SUBOP_BAR_SYNC;
2981       break;
2982    case TGSI_OPCODE_MFENCE:
2983    case TGSI_OPCODE_LFENCE:
2984    case TGSI_OPCODE_SFENCE:
2985       geni = mkOp(OP_MEMBAR, TYPE_NONE, NULL);
2986       geni->fixed = 1;
2987       geni->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode());
2988       break;
2989    case TGSI_OPCODE_ATOMUADD:
2990    case TGSI_OPCODE_ATOMXCHG:
2991    case TGSI_OPCODE_ATOMCAS:
2992    case TGSI_OPCODE_ATOMAND:
2993    case TGSI_OPCODE_ATOMOR:
2994    case TGSI_OPCODE_ATOMXOR:
2995    case TGSI_OPCODE_ATOMUMIN:
2996    case TGSI_OPCODE_ATOMIMIN:
2997    case TGSI_OPCODE_ATOMUMAX:
2998    case TGSI_OPCODE_ATOMIMAX:
2999       handleATOM(dst0, dstTy, tgsi::opcodeToSubOp(tgsi.getOpcode()));
3000       break;
3001    case TGSI_OPCODE_IBFE:
3002    case TGSI_OPCODE_UBFE:
3003       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3004          src0 = fetchSrc(0, c);
3005          if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE &&
3006              tgsi.getSrc(2).getFile() == TGSI_FILE_IMMEDIATE) {
3007             src1 = loadImm(NULL, tgsi.getSrc(2).getValueU32(c, info) << 8 |
3008                            tgsi.getSrc(1).getValueU32(c, info));
3009          } else {
3010             src1 = fetchSrc(1, c);
3011             src2 = fetchSrc(2, c);
3012             mkOp3(OP_INSBF, TYPE_U32, src1, src2, mkImm(0x808), src1);
3013          }
3014          mkOp2(OP_EXTBF, dstTy, dst0[c], src0, src1);
3015       }
3016       break;
3017    case TGSI_OPCODE_BFI:
3018       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3019          src0 = fetchSrc(0, c);
3020          src1 = fetchSrc(1, c);
3021          src2 = fetchSrc(2, c);
3022          src3 = fetchSrc(3, c);
3023          mkOp3(OP_INSBF, TYPE_U32, src2, src3, mkImm(0x808), src2);
3024          mkOp3(OP_INSBF, TYPE_U32, dst0[c], src1, src2, src0);
3025       }
3026       break;
3027    case TGSI_OPCODE_LSB:
3028       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3029          src0 = fetchSrc(0, c);
3030          geni = mkOp2(OP_EXTBF, TYPE_U32, src0, src0, mkImm(0x2000));
3031          geni->subOp = NV50_IR_SUBOP_EXTBF_REV;
3032          geni = mkOp1(OP_BFIND, TYPE_U32, dst0[c], src0);
3033          geni->subOp = NV50_IR_SUBOP_BFIND_SAMT;
3034       }
3035       break;
3036    case TGSI_OPCODE_IMSB:
3037    case TGSI_OPCODE_UMSB:
3038       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3039          src0 = fetchSrc(0, c);
3040          mkOp1(OP_BFIND, srcTy, dst0[c], src0);
3041       }
3042       break;
3043    case TGSI_OPCODE_BREV:
3044       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3045          src0 = fetchSrc(0, c);
3046          geni = mkOp2(OP_EXTBF, TYPE_U32, dst0[c], src0, mkImm(0x2000));
3047          geni->subOp = NV50_IR_SUBOP_EXTBF_REV;
3048       }
3049       break;
3050    case TGSI_OPCODE_POPC:
3051       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3052          src0 = fetchSrc(0, c);
3053          mkOp2(OP_POPCNT, TYPE_U32, dst0[c], src0, src0);
3054       }
3055       break;
3056    case TGSI_OPCODE_INTERP_CENTROID:
3057    case TGSI_OPCODE_INTERP_SAMPLE:
3058    case TGSI_OPCODE_INTERP_OFFSET:
3059       handleINTERP(dst0);
3060       break;
3061    case TGSI_OPCODE_D2I:
3062    case TGSI_OPCODE_D2U:
3063    case TGSI_OPCODE_D2F: {
3064       int pos = 0;
3065       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3066          Value *dreg = getSSA(8);
3067          src0 = fetchSrc(0, pos);
3068          src1 = fetchSrc(0, pos + 1);
3069          mkOp2(OP_MERGE, TYPE_U64, dreg, src0, src1);
3070          mkCvt(OP_CVT, dstTy, dst0[c], srcTy, dreg);
3071          pos += 2;
3072       }
3073       break;
3074    }
3075    case TGSI_OPCODE_I2D:
3076    case TGSI_OPCODE_U2D:
3077    case TGSI_OPCODE_F2D:
3078       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3079          Value *dreg = getSSA(8);
3080          mkCvt(OP_CVT, dstTy, dreg, srcTy, fetchSrc(0, c / 2));
3081          mkSplit(&dst0[c], 4, dreg);
3082          c++;
3083       }
3084       break;
3085    case TGSI_OPCODE_DABS:
3086    case TGSI_OPCODE_DNEG:
3087    case TGSI_OPCODE_DRCP:
3088    case TGSI_OPCODE_DSQRT:
3089    case TGSI_OPCODE_DRSQ:
3090    case TGSI_OPCODE_DTRUNC:
3091    case TGSI_OPCODE_DCEIL:
3092    case TGSI_OPCODE_DFLR:
3093       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3094          src0 = getSSA(8);
3095          Value *dst = getSSA(8), *tmp[2];
3096          tmp[0] = fetchSrc(0, c);
3097          tmp[1] = fetchSrc(0, c + 1);
3098          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3099          mkOp1(op, dstTy, dst, src0);
3100          mkSplit(&dst0[c], 4, dst);
3101          c++;
3102       }
3103       break;
3104    case TGSI_OPCODE_DFRAC:
3105       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3106          src0 = getSSA(8);
3107          Value *dst = getSSA(8), *tmp[2];
3108          tmp[0] = fetchSrc(0, c);
3109          tmp[1] = fetchSrc(0, c + 1);
3110          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3111          mkOp1(OP_FLOOR, TYPE_F64, dst, src0);
3112          mkOp2(OP_SUB, TYPE_F64, dst, src0, dst);
3113          mkSplit(&dst0[c], 4, dst);
3114          c++;
3115       }
3116       break;
3117    case TGSI_OPCODE_DSLT:
3118    case TGSI_OPCODE_DSGE:
3119    case TGSI_OPCODE_DSEQ:
3120    case TGSI_OPCODE_DSNE: {
3121       int pos = 0;
3122       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3123          Value *tmp[2];
3124
3125          src0 = getSSA(8);
3126          src1 = getSSA(8);
3127          tmp[0] = fetchSrc(0, pos);
3128          tmp[1] = fetchSrc(0, pos + 1);
3129          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3130          tmp[0] = fetchSrc(1, pos);
3131          tmp[1] = fetchSrc(1, pos + 1);
3132          mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
3133          mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], srcTy, src0, src1);
3134          pos += 2;
3135       }
3136       break;
3137    }
3138    case TGSI_OPCODE_DADD:
3139    case TGSI_OPCODE_DMUL:
3140    case TGSI_OPCODE_DMAX:
3141    case TGSI_OPCODE_DMIN:
3142       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3143          src0 = getSSA(8);
3144          src1 = getSSA(8);
3145          Value *dst = getSSA(8), *tmp[2];
3146          tmp[0] = fetchSrc(0, c);
3147          tmp[1] = fetchSrc(0, c + 1);
3148          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3149          tmp[0] = fetchSrc(1, c);
3150          tmp[1] = fetchSrc(1, c + 1);
3151          mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
3152          mkOp2(op, dstTy, dst, src0, src1);
3153          mkSplit(&dst0[c], 4, dst);
3154          c++;
3155       }
3156       break;
3157    case TGSI_OPCODE_DMAD:
3158       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3159          src0 = getSSA(8);
3160          src1 = getSSA(8);
3161          src2 = getSSA(8);
3162          Value *dst = getSSA(8), *tmp[2];
3163          tmp[0] = fetchSrc(0, c);
3164          tmp[1] = fetchSrc(0, c + 1);
3165          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3166          tmp[0] = fetchSrc(1, c);
3167          tmp[1] = fetchSrc(1, c + 1);
3168          mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
3169          tmp[0] = fetchSrc(2, c);
3170          tmp[1] = fetchSrc(2, c + 1);
3171          mkOp2(OP_MERGE, TYPE_U64, src2, tmp[0], tmp[1]);
3172          mkOp3(op, dstTy, dst, src0, src1, src2);
3173          mkSplit(&dst0[c], 4, dst);
3174          c++;
3175       }
3176       break;
3177    case TGSI_OPCODE_DROUND:
3178       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3179          src0 = getSSA(8);
3180          Value *dst = getSSA(8), *tmp[2];
3181          tmp[0] = fetchSrc(0, c);
3182          tmp[1] = fetchSrc(0, c + 1);
3183          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3184          mkCvt(OP_CVT, TYPE_F64, dst, TYPE_F64, src0)
3185          ->rnd = ROUND_NI;
3186          mkSplit(&dst0[c], 4, dst);
3187          c++;
3188       }
3189       break;
3190    case TGSI_OPCODE_DSSG:
3191       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3192          src0 = getSSA(8);
3193          Value *dst = getSSA(8), *dstF32 = getSSA(), *tmp[2];
3194          tmp[0] = fetchSrc(0, c);
3195          tmp[1] = fetchSrc(0, c + 1);
3196          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3197
3198          val0 = getScratch();
3199          val1 = getScratch();
3200          // The zero is wrong here since it's only 32-bit, but it works out in
3201          // the end since it gets replaced with $r63.
3202          mkCmp(OP_SET, CC_GT, TYPE_F32, val0, TYPE_F64, src0, zero);
3203          mkCmp(OP_SET, CC_LT, TYPE_F32, val1, TYPE_F64, src0, zero);
3204          mkOp2(OP_SUB, TYPE_F32, dstF32, val0, val1);
3205          mkCvt(OP_CVT, TYPE_F64, dst, TYPE_F32, dstF32);
3206          mkSplit(&dst0[c], 4, dst);
3207          c++;
3208       }
3209       break;
3210    default:
3211       ERROR("unhandled TGSI opcode: %u\n", tgsi.getOpcode());
3212       assert(0);
3213       break;
3214    }
3215
3216    if (tgsi.dstCount()) {
3217       for (c = 0; c < 4; ++c) {
3218          if (!dst0[c])
3219             continue;
3220          if (dst0[c] != rDst0[c])
3221             mkMov(rDst0[c], dst0[c]);
3222          storeDst(0, c, rDst0[c]);
3223       }
3224    }
3225    vtxBaseValid = 0;
3226
3227    return true;
3228 }
3229
3230 void
3231 Converter::handleUserClipPlanes()
3232 {
3233    Value *res[8];
3234    int n, i, c;
3235
3236    for (c = 0; c < 4; ++c) {
3237       for (i = 0; i < info->io.genUserClip; ++i) {
3238          Symbol *sym = mkSymbol(FILE_MEMORY_CONST, info->io.ucpCBSlot,
3239                                 TYPE_F32, info->io.ucpBase + i * 16 + c * 4);
3240          Value *ucp = mkLoadv(TYPE_F32, sym, NULL);
3241          if (c == 0)
3242             res[i] = mkOp2v(OP_MUL, TYPE_F32, getScratch(), clipVtx[c], ucp);
3243          else
3244             mkOp3(OP_MAD, TYPE_F32, res[i], clipVtx[c], ucp, res[i]);
3245       }
3246    }
3247
3248    const int first = info->numOutputs - (info->io.genUserClip + 3) / 4;
3249
3250    for (i = 0; i < info->io.genUserClip; ++i) {
3251       n = i / 4 + first;
3252       c = i % 4;
3253       Symbol *sym =
3254          mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32, info->out[n].slot[c] * 4);
3255       mkStore(OP_EXPORT, TYPE_F32, sym, NULL, res[i]);
3256    }
3257 }
3258
3259 void
3260 Converter::exportOutputs()
3261 {
3262    for (unsigned int i = 0; i < info->numOutputs; ++i) {
3263       for (unsigned int c = 0; c < 4; ++c) {
3264          if (!oData.exists(sub.cur->values, i, c))
3265             continue;
3266          Symbol *sym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32,
3267                                 info->out[i].slot[c] * 4);
3268          Value *val = oData.load(sub.cur->values, i, c, NULL);
3269          if (val)
3270             mkStore(OP_EXPORT, TYPE_F32, sym, NULL, val);
3271       }
3272    }
3273 }
3274
3275 Converter::Converter(Program *ir, const tgsi::Source *code) : BuildUtil(ir),
3276      code(code),
3277      tgsi(NULL),
3278      tData(this), aData(this), pData(this), oData(this)
3279 {
3280    info = code->info;
3281
3282    const DataFile tFile = code->mainTempsInLMem ? FILE_MEMORY_LOCAL : FILE_GPR;
3283
3284    const unsigned tSize = code->fileSize(TGSI_FILE_TEMPORARY);
3285    const unsigned pSize = code->fileSize(TGSI_FILE_PREDICATE);
3286    const unsigned aSize = code->fileSize(TGSI_FILE_ADDRESS);
3287    const unsigned oSize = code->fileSize(TGSI_FILE_OUTPUT);
3288
3289    tData.setup(TGSI_FILE_TEMPORARY, 0, 0, tSize, 4, 4, tFile, 0);
3290    pData.setup(TGSI_FILE_PREDICATE, 0, 0, pSize, 4, 4, FILE_PREDICATE, 0);
3291    aData.setup(TGSI_FILE_ADDRESS, 0, 0, aSize, 4, 4, FILE_GPR, 0);
3292    oData.setup(TGSI_FILE_OUTPUT, 0, 0, oSize, 4, 4, FILE_GPR, 0);
3293
3294    zero = mkImm((uint32_t)0);
3295
3296    vtxBaseValid = 0;
3297 }
3298
3299 Converter::~Converter()
3300 {
3301 }
3302
3303 inline const Converter::Location *
3304 Converter::BindArgumentsPass::getValueLocation(Subroutine *s, Value *v)
3305 {
3306    ValueMap::l_iterator it = s->values.l.find(v);
3307    return it == s->values.l.end() ? NULL : &it->second;
3308 }
3309
3310 template<typename T> inline void
3311 Converter::BindArgumentsPass::updateCallArgs(
3312    Instruction *i, void (Instruction::*setArg)(int, Value *),
3313    T (Function::*proto))
3314 {
3315    Function *g = i->asFlow()->target.fn;
3316    Subroutine *subg = conv.getSubroutine(g);
3317
3318    for (unsigned a = 0; a < (g->*proto).size(); ++a) {
3319       Value *v = (g->*proto)[a].get();
3320       const Converter::Location &l = *getValueLocation(subg, v);
3321       Converter::DataArray *array = conv.getArrayForFile(l.array, l.arrayIdx);
3322
3323       (i->*setArg)(a, array->acquire(sub->values, l.i, l.c));
3324    }
3325 }
3326
3327 template<typename T> inline void
3328 Converter::BindArgumentsPass::updatePrototype(
3329    BitSet *set, void (Function::*updateSet)(), T (Function::*proto))
3330 {
3331    (func->*updateSet)();
3332
3333    for (unsigned i = 0; i < set->getSize(); ++i) {
3334       Value *v = func->getLValue(i);
3335       const Converter::Location *l = getValueLocation(sub, v);
3336
3337       // only include values with a matching TGSI register
3338       if (set->test(i) && l && !conv.code->locals.count(*l))
3339          (func->*proto).push_back(v);
3340    }
3341 }
3342
3343 bool
3344 Converter::BindArgumentsPass::visit(Function *f)
3345 {
3346    sub = conv.getSubroutine(f);
3347
3348    for (ArrayList::Iterator bi = f->allBBlocks.iterator();
3349         !bi.end(); bi.next()) {
3350       for (Instruction *i = BasicBlock::get(bi)->getFirst();
3351            i; i = i->next) {
3352          if (i->op == OP_CALL && !i->asFlow()->builtin) {
3353             updateCallArgs(i, &Instruction::setSrc, &Function::ins);
3354             updateCallArgs(i, &Instruction::setDef, &Function::outs);
3355          }
3356       }
3357    }
3358
3359    if (func == prog->main && prog->getType() != Program::TYPE_COMPUTE)
3360       return true;
3361    updatePrototype(&BasicBlock::get(f->cfg.getRoot())->liveSet,
3362                    &Function::buildLiveSets, &Function::ins);
3363    updatePrototype(&BasicBlock::get(f->cfgExit)->defSet,
3364                    &Function::buildDefSets, &Function::outs);
3365
3366    return true;
3367 }
3368
3369 bool
3370 Converter::run()
3371 {
3372    BasicBlock *entry = new BasicBlock(prog->main);
3373    BasicBlock *leave = new BasicBlock(prog->main);
3374
3375    prog->main->setEntry(entry);
3376    prog->main->setExit(leave);
3377
3378    setPosition(entry, true);
3379    sub.cur = getSubroutine(prog->main);
3380
3381    if (info->io.genUserClip > 0) {
3382       for (int c = 0; c < 4; ++c)
3383          clipVtx[c] = getScratch();
3384    }
3385
3386    switch (prog->getType()) {
3387    case Program::TYPE_TESSELLATION_CONTROL:
3388       outBase = mkOp2v(
3389          OP_SUB, TYPE_U32, getSSA(),
3390          mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_LANEID, 0)),
3391          mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_INVOCATION_ID, 0)));
3392       break;
3393    case Program::TYPE_FRAGMENT: {
3394       Symbol *sv = mkSysVal(SV_POSITION, 3);
3395       fragCoord[3] = mkOp1v(OP_RDSV, TYPE_F32, getSSA(), sv);
3396       mkOp1(OP_RCP, TYPE_F32, fragCoord[3], fragCoord[3]);
3397       break;
3398    }
3399    default:
3400       break;
3401    }
3402
3403    if (info->io.viewportId >= 0)
3404       viewport = getScratch();
3405    else
3406       viewport = NULL;
3407
3408    for (ip = 0; ip < code->scan.num_instructions; ++ip) {
3409       if (!handleInstruction(&code->insns[ip]))
3410          return false;
3411    }
3412
3413    if (!BindArgumentsPass(*this).run(prog))
3414       return false;
3415
3416    return true;
3417 }
3418
3419 } // unnamed namespace
3420
3421 namespace nv50_ir {
3422
3423 bool
3424 Program::makeFromTGSI(struct nv50_ir_prog_info *info)
3425 {
3426    tgsi::Source src(info);
3427    if (!src.scanSource())
3428       return false;
3429    tlsSize = info->bin.tlsSpace;
3430
3431    Converter builder(this, &src);
3432    return builder.run();
3433 }
3434
3435 } // namespace nv50_ir