src/gallium/drivers/nouveau/codegen/nv50_ir_from_tgsi.cpp

   1 /*
   2  * Copyright 2011 Christoph Bumiller
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 #include "tgsi/tgsi_dump.h"
  24 #include "tgsi/tgsi_scan.h"
  25 #include "tgsi/tgsi_util.h"
  26
  27 #include <set>
  28
  29 #include "codegen/nv50_ir.h"
  30 #include "codegen/nv50_ir_util.h"
  31 #include "codegen/nv50_ir_build_util.h"
  32
  33 namespace tgsi {
  34
  35 class Source;
  36
  37 static nv50_ir::operation translateOpcode(uint opcode);
  38 static nv50_ir::DataFile translateFile(uint file);
  39 static nv50_ir::TexTarget translateTexture(uint texTarg);
  40 static nv50_ir::SVSemantic translateSysVal(uint sysval);
  41
  42 class Instruction
  43 {
  44 public:
  45    Instruction(const struct tgsi_full_instruction *inst) : insn(inst) { }
  46
  47    class SrcRegister
  48    {
  49    public:
  50       SrcRegister(const struct tgsi_full_src_register *src)
  51          : reg(src->Register),
  52            fsr(src)
  53       { }
  54
  55       SrcRegister(const struct tgsi_src_register& src) : reg(src), fsr(NULL) { }
  56
  57       SrcRegister(const struct tgsi_ind_register& ind)
  58          : reg(tgsi_util_get_src_from_ind(&ind)),
  59            fsr(NULL)
  60       { }
  61
  62       struct tgsi_src_register offsetToSrc(struct tgsi_texture_offset off)
  63       {
  64          struct tgsi_src_register reg;
  65          memset(&reg, 0, sizeof(reg));
  66          reg.Index = off.Index;
  67          reg.File = off.File;
  68          reg.SwizzleX = off.SwizzleX;
  69          reg.SwizzleY = off.SwizzleY;
  70          reg.SwizzleZ = off.SwizzleZ;
  71          return reg;
  72       }
  73
  74       SrcRegister(const struct tgsi_texture_offset& off) :
  75          reg(offsetToSrc(off)),
  76          fsr(NULL)
  77       { }
  78
  79       uint getFile() const { return reg.File; }
  80
  81       bool is2D() const { return reg.Dimension; }
  82
  83       bool isIndirect(int dim) const
  84       {
  85          return (dim && fsr) ? fsr->Dimension.Indirect : reg.Indirect;
  86       }
  87
  88       int getIndex(int dim) const
  89       {
  90          return (dim && fsr) ? fsr->Dimension.Index : reg.Index;
  91       }
  92
  93       int getSwizzle(int chan) const
  94       {
  95          return tgsi_util_get_src_register_swizzle(&reg, chan);
  96       }
  97
  98       int getArrayId() const
  99       {
 100          if (isIndirect(0))
 101             return fsr->Indirect.ArrayID;
 102          return 0;
 103       }
 104
 105       nv50_ir::Modifier getMod(int chan) const;
 106
 107       SrcRegister getIndirect(int dim) const
 108       {
 109          assert(fsr && isIndirect(dim));
 110          if (dim)
 111             return SrcRegister(fsr->DimIndirect);
 112          return SrcRegister(fsr->Indirect);
 113       }
 114
 115       uint32_t getValueU32(int c, const struct nv50_ir_prog_info *info) const
 116       {
 117          assert(reg.File == TGSI_FILE_IMMEDIATE);
 118          assert(!reg.Absolute);
 119          assert(!reg.Negate);
 120          return info->immd.data[reg.Index * 4 + getSwizzle(c)];
 121       }
 122
 123    private:
 124       const struct tgsi_src_register reg;
 125       const struct tgsi_full_src_register *fsr;
 126    };
 127
 128    class DstRegister
 129    {
 130    public:
 131       DstRegister(const struct tgsi_full_dst_register *dst)
 132          : reg(dst->Register),
 133            fdr(dst)
 134       { }
 135
 136       DstRegister(const struct tgsi_dst_register& dst) : reg(dst), fdr(NULL) { }
 137
 138       uint getFile() const { return reg.File; }
 139
 140       bool is2D() const { return reg.Dimension; }
 141
 142       bool isIndirect(int dim) const
 143       {
 144          return (dim && fdr) ? fdr->Dimension.Indirect : reg.Indirect;
 145       }
 146
 147       int getIndex(int dim) const
 148       {
 149          return (dim && fdr) ? fdr->Dimension.Dimension : reg.Index;
 150       }
 151
 152       unsigned int getMask() const { return reg.WriteMask; }
 153
 154       bool isMasked(int chan) const { return !(getMask() & (1 << chan)); }
 155
 156       SrcRegister getIndirect(int dim) const
 157       {
 158          assert(fdr && isIndirect(dim));
 159          if (dim)
 160             return SrcRegister(fdr->DimIndirect);
 161          return SrcRegister(fdr->Indirect);
 162       }
 163
 164       int getArrayId() const
 165       {
 166          if (isIndirect(0))
 167             return fdr->Indirect.ArrayID;
 168          return 0;
 169       }
 170
 171    private:
 172       const struct tgsi_dst_register reg;
 173       const struct tgsi_full_dst_register *fdr;
 174    };
 175
 176    inline uint getOpcode() const { return insn->Instruction.Opcode; }
 177
 178    unsigned int srcCount() const { return insn->Instruction.NumSrcRegs; }
 179    unsigned int dstCount() const { return insn->Instruction.NumDstRegs; }
 180
 181    // mask of used components of source s
 182    unsigned int srcMask(unsigned int s) const;
 183
 184    SrcRegister getSrc(unsigned int s) const
 185    {
 186       assert(s < srcCount());
 187       return SrcRegister(&insn->Src[s]);
 188    }
 189
 190    DstRegister getDst(unsigned int d) const
 191    {
 192       assert(d < dstCount());
 193       return DstRegister(&insn->Dst[d]);
 194    }
 195
 196    SrcRegister getTexOffset(unsigned int i) const
 197    {
 198       assert(i < TGSI_FULL_MAX_TEX_OFFSETS);
 199       return SrcRegister(insn->TexOffsets[i]);
 200    }
 201
 202    unsigned int getNumTexOffsets() const { return insn->Texture.NumOffsets; }
 203
 204    bool checkDstSrcAliasing() const;
 205
 206    inline nv50_ir::operation getOP() const {
 207       return translateOpcode(getOpcode()); }
 208
 209    nv50_ir::DataType inferSrcType() const;
 210    nv50_ir::DataType inferDstType() const;
 211
 212    nv50_ir::CondCode getSetCond() const;
 213
 214    nv50_ir::TexInstruction::Target getTexture(const Source *, int s) const;
 215
 216    inline uint getLabel() { return insn->Label.Label; }
 217
 218    unsigned getSaturate() const { return insn->Instruction.Saturate; }
 219
 220    void print() const
 221    {
 222       tgsi_dump_instruction(insn, 1);
 223    }
 224
 225 private:
 226    const struct tgsi_full_instruction *insn;
 227 };
 228
 229 unsigned int Instruction::srcMask(unsigned int s) const
 230 {
 231    unsigned int mask = insn->Dst[0].Register.WriteMask;
 232
 233    switch (insn->Instruction.Opcode) {
 234    case TGSI_OPCODE_COS:
 235    case TGSI_OPCODE_SIN:
 236       return (mask & 0x8) | ((mask & 0x7) ? 0x1 : 0x0);
 237    case TGSI_OPCODE_DP2:
 238       return 0x3;
 239    case TGSI_OPCODE_DP3:
 240       return 0x7;
 241    case TGSI_OPCODE_DP4:
 242    case TGSI_OPCODE_DPH:
 243    case TGSI_OPCODE_KILL_IF: /* WriteMask ignored */
 244       return 0xf;
 245    case TGSI_OPCODE_DST:
 246       return mask & (s ? 0xa : 0x6);
 247    case TGSI_OPCODE_EX2:
 248    case TGSI_OPCODE_EXP:
 249    case TGSI_OPCODE_LG2:
 250    case TGSI_OPCODE_LOG:
 251    case TGSI_OPCODE_POW:
 252    case TGSI_OPCODE_RCP:
 253    case TGSI_OPCODE_RSQ:
 254    case TGSI_OPCODE_SCS:
 255       return 0x1;
 256    case TGSI_OPCODE_IF:
 257    case TGSI_OPCODE_UIF:
 258       return 0x1;
 259    case TGSI_OPCODE_LIT:
 260       return 0xb;
 261    case TGSI_OPCODE_TEX2:
 262    case TGSI_OPCODE_TXB2:
 263    case TGSI_OPCODE_TXL2:
 264       return (s == 0) ? 0xf : 0x3;
 265    case TGSI_OPCODE_TEX:
 266    case TGSI_OPCODE_TXB:
 267    case TGSI_OPCODE_TXD:
 268    case TGSI_OPCODE_TXL:
 269    case TGSI_OPCODE_TXP:
 270    case TGSI_OPCODE_LODQ:
 271    {
 272       const struct tgsi_instruction_texture *tex = &insn->Texture;
 273
 274       assert(insn->Instruction.Texture);
 275
 276       mask = 0x7;
 277       if (insn->Instruction.Opcode != TGSI_OPCODE_TEX &&
 278           insn->Instruction.Opcode != TGSI_OPCODE_TXD)
 279          mask |= 0x8; /* bias, lod or proj */
 280
 281       switch (tex->Texture) {
 282       case TGSI_TEXTURE_1D:
 283          mask &= 0x9;
 284          break;
 285       case TGSI_TEXTURE_SHADOW1D:
 286          mask &= 0xd;
 287          break;
 288       case TGSI_TEXTURE_1D_ARRAY:
 289       case TGSI_TEXTURE_2D:
 290       case TGSI_TEXTURE_RECT:
 291          mask &= 0xb;
 292          break;
 293       case TGSI_TEXTURE_CUBE_ARRAY:
 294       case TGSI_TEXTURE_SHADOW2D_ARRAY:
 295       case TGSI_TEXTURE_SHADOWCUBE:
 296       case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
 297          mask |= 0x8;
 298          break;
 299       default:
 300          break;
 301       }
 302    }
 303       return mask;
 304    case TGSI_OPCODE_XPD:
 305    {
 306       unsigned int x = 0;
 307       if (mask & 1) x |= 0x6;
 308       if (mask & 2) x |= 0x5;
 309       if (mask & 4) x |= 0x3;
 310       return x;
 311    }
 312    case TGSI_OPCODE_D2I:
 313    case TGSI_OPCODE_D2U:
 314    case TGSI_OPCODE_D2F:
 315    case TGSI_OPCODE_DSLT:
 316    case TGSI_OPCODE_DSGE:
 317    case TGSI_OPCODE_DSEQ:
 318    case TGSI_OPCODE_DSNE:
 319       switch (util_bitcount(mask)) {
 320       case 1: return 0x3;
 321       case 2: return 0xf;
 322       default:
 323          assert(!"unexpected mask");
 324          return 0xf;
 325       }
 326    case TGSI_OPCODE_I2D:
 327    case TGSI_OPCODE_U2D:
 328    case TGSI_OPCODE_F2D: {
 329       unsigned int x = 0;
 330       if ((mask & 0x3) == 0x3)
 331          x |= 1;
 332       if ((mask & 0xc) == 0xc)
 333          x |= 2;
 334       return x;
 335    }
 336    case TGSI_OPCODE_PK2H:
 337       return 0x3;
 338    case TGSI_OPCODE_UP2H:
 339       return 0x1;
 340    default:
 341       break;
 342    }
 343
 344    return mask;
 345 }
 346
 347 nv50_ir::Modifier Instruction::SrcRegister::getMod(int chan) const
 348 {
 349    nv50_ir::Modifier m(0);
 350
 351    if (reg.Absolute)
 352       m = m | nv50_ir::Modifier(NV50_IR_MOD_ABS);
 353    if (reg.Negate)
 354       m = m | nv50_ir::Modifier(NV50_IR_MOD_NEG);
 355    return m;
 356 }
 357
 358 static nv50_ir::DataFile translateFile(uint file)
 359 {
 360    switch (file) {
 361    case TGSI_FILE_CONSTANT:        return nv50_ir::FILE_MEMORY_CONST;
 362    case TGSI_FILE_INPUT:           return nv50_ir::FILE_SHADER_INPUT;
 363    case TGSI_FILE_OUTPUT:          return nv50_ir::FILE_SHADER_OUTPUT;
 364    case TGSI_FILE_TEMPORARY:       return nv50_ir::FILE_GPR;
 365    case TGSI_FILE_ADDRESS:         return nv50_ir::FILE_ADDRESS;
 366    case TGSI_FILE_PREDICATE:       return nv50_ir::FILE_PREDICATE;
 367    case TGSI_FILE_IMMEDIATE:       return nv50_ir::FILE_IMMEDIATE;
 368    case TGSI_FILE_SYSTEM_VALUE:    return nv50_ir::FILE_SYSTEM_VALUE;
 369    //case TGSI_FILE_RESOURCE:        return nv50_ir::FILE_MEMORY_GLOBAL;
 370    case TGSI_FILE_SAMPLER:
 371    case TGSI_FILE_NULL:
 372    default:
 373       return nv50_ir::FILE_NULL;
 374    }
 375 }
 376
 377 static nv50_ir::SVSemantic translateSysVal(uint sysval)
 378 {
 379    switch (sysval) {
 380    case TGSI_SEMANTIC_FACE:       return nv50_ir::SV_FACE;
 381    case TGSI_SEMANTIC_PSIZE:      return nv50_ir::SV_POINT_SIZE;
 382    case TGSI_SEMANTIC_PRIMID:     return nv50_ir::SV_PRIMITIVE_ID;
 383    case TGSI_SEMANTIC_INSTANCEID: return nv50_ir::SV_INSTANCE_ID;
 384    case TGSI_SEMANTIC_VERTEXID:   return nv50_ir::SV_VERTEX_ID;
 385    case TGSI_SEMANTIC_GRID_SIZE:  return nv50_ir::SV_NCTAID;
 386    case TGSI_SEMANTIC_BLOCK_ID:   return nv50_ir::SV_CTAID;
 387    case TGSI_SEMANTIC_BLOCK_SIZE: return nv50_ir::SV_NTID;
 388    case TGSI_SEMANTIC_THREAD_ID:  return nv50_ir::SV_TID;
 389    case TGSI_SEMANTIC_SAMPLEID:   return nv50_ir::SV_SAMPLE_INDEX;
 390    case TGSI_SEMANTIC_SAMPLEPOS:  return nv50_ir::SV_SAMPLE_POS;
 391    case TGSI_SEMANTIC_SAMPLEMASK: return nv50_ir::SV_SAMPLE_MASK;
 392    case TGSI_SEMANTIC_INVOCATIONID: return nv50_ir::SV_INVOCATION_ID;
 393    case TGSI_SEMANTIC_TESSCOORD:  return nv50_ir::SV_TESS_COORD;
 394    case TGSI_SEMANTIC_TESSOUTER:  return nv50_ir::SV_TESS_OUTER;
 395    case TGSI_SEMANTIC_TESSINNER:  return nv50_ir::SV_TESS_INNER;
 396    case TGSI_SEMANTIC_VERTICESIN: return nv50_ir::SV_VERTEX_COUNT;
 397    case TGSI_SEMANTIC_HELPER_INVOCATION: return nv50_ir::SV_THREAD_KILL;
 398    case TGSI_SEMANTIC_BASEVERTEX: return nv50_ir::SV_BASEVERTEX;
 399    case TGSI_SEMANTIC_BASEINSTANCE: return nv50_ir::SV_BASEINSTANCE;
 400    case TGSI_SEMANTIC_DRAWID:     return nv50_ir::SV_DRAWID;
 401    default:
 402       assert(0);
 403       return nv50_ir::SV_CLOCK;
 404    }
 405 }
 406
 407 #define NV50_IR_TEX_TARG_CASE(a, b) \
 408    case TGSI_TEXTURE_##a: return nv50_ir::TEX_TARGET_##b;
 409
 410 static nv50_ir::TexTarget translateTexture(uint tex)
 411 {
 412    switch (tex) {
 413    NV50_IR_TEX_TARG_CASE(1D, 1D);
 414    NV50_IR_TEX_TARG_CASE(2D, 2D);
 415    NV50_IR_TEX_TARG_CASE(2D_MSAA, 2D_MS);
 416    NV50_IR_TEX_TARG_CASE(3D, 3D);
 417    NV50_IR_TEX_TARG_CASE(CUBE, CUBE);
 418    NV50_IR_TEX_TARG_CASE(RECT, RECT);
 419    NV50_IR_TEX_TARG_CASE(1D_ARRAY, 1D_ARRAY);
 420    NV50_IR_TEX_TARG_CASE(2D_ARRAY, 2D_ARRAY);
 421    NV50_IR_TEX_TARG_CASE(2D_ARRAY_MSAA, 2D_MS_ARRAY);
 422    NV50_IR_TEX_TARG_CASE(CUBE_ARRAY, CUBE_ARRAY);
 423    NV50_IR_TEX_TARG_CASE(SHADOW1D, 1D_SHADOW);
 424    NV50_IR_TEX_TARG_CASE(SHADOW2D, 2D_SHADOW);
 425    NV50_IR_TEX_TARG_CASE(SHADOWCUBE, CUBE_SHADOW);
 426    NV50_IR_TEX_TARG_CASE(SHADOWRECT, RECT_SHADOW);
 427    NV50_IR_TEX_TARG_CASE(SHADOW1D_ARRAY, 1D_ARRAY_SHADOW);
 428    NV50_IR_TEX_TARG_CASE(SHADOW2D_ARRAY, 2D_ARRAY_SHADOW);
 429    NV50_IR_TEX_TARG_CASE(SHADOWCUBE_ARRAY, CUBE_ARRAY_SHADOW);
 430    NV50_IR_TEX_TARG_CASE(BUFFER, BUFFER);
 431
 432    case TGSI_TEXTURE_UNKNOWN:
 433    default:
 434       assert(!"invalid texture target");
 435       return nv50_ir::TEX_TARGET_2D;
 436    }
 437 }
 438
 439 nv50_ir::DataType Instruction::inferSrcType() const
 440 {
 441    switch (getOpcode()) {
 442    case TGSI_OPCODE_UIF:
 443    case TGSI_OPCODE_AND:
 444    case TGSI_OPCODE_OR:
 445    case TGSI_OPCODE_XOR:
 446    case TGSI_OPCODE_NOT:
 447    case TGSI_OPCODE_SHL:
 448    case TGSI_OPCODE_U2F:
 449    case TGSI_OPCODE_U2D:
 450    case TGSI_OPCODE_UADD:
 451    case TGSI_OPCODE_UDIV:
 452    case TGSI_OPCODE_UMOD:
 453    case TGSI_OPCODE_UMAD:
 454    case TGSI_OPCODE_UMUL:
 455    case TGSI_OPCODE_UMUL_HI:
 456    case TGSI_OPCODE_UMAX:
 457    case TGSI_OPCODE_UMIN:
 458    case TGSI_OPCODE_USEQ:
 459    case TGSI_OPCODE_USGE:
 460    case TGSI_OPCODE_USLT:
 461    case TGSI_OPCODE_USNE:
 462    case TGSI_OPCODE_USHR:
 463    case TGSI_OPCODE_ATOMUADD:
 464    case TGSI_OPCODE_ATOMXCHG:
 465    case TGSI_OPCODE_ATOMCAS:
 466    case TGSI_OPCODE_ATOMAND:
 467    case TGSI_OPCODE_ATOMOR:
 468    case TGSI_OPCODE_ATOMXOR:
 469    case TGSI_OPCODE_ATOMUMIN:
 470    case TGSI_OPCODE_ATOMUMAX:
 471    case TGSI_OPCODE_UBFE:
 472    case TGSI_OPCODE_UMSB:
 473    case TGSI_OPCODE_UP2H:
 474       return nv50_ir::TYPE_U32;
 475    case TGSI_OPCODE_I2F:
 476    case TGSI_OPCODE_I2D:
 477    case TGSI_OPCODE_IDIV:
 478    case TGSI_OPCODE_IMUL_HI:
 479    case TGSI_OPCODE_IMAX:
 480    case TGSI_OPCODE_IMIN:
 481    case TGSI_OPCODE_IABS:
 482    case TGSI_OPCODE_INEG:
 483    case TGSI_OPCODE_ISGE:
 484    case TGSI_OPCODE_ISHR:
 485    case TGSI_OPCODE_ISLT:
 486    case TGSI_OPCODE_ISSG:
 487    case TGSI_OPCODE_SAD: // not sure about SAD, but no one has a float version
 488    case TGSI_OPCODE_MOD:
 489    case TGSI_OPCODE_UARL:
 490    case TGSI_OPCODE_ATOMIMIN:
 491    case TGSI_OPCODE_ATOMIMAX:
 492    case TGSI_OPCODE_IBFE:
 493    case TGSI_OPCODE_IMSB:
 494       return nv50_ir::TYPE_S32;
 495    case TGSI_OPCODE_D2F:
 496    case TGSI_OPCODE_D2I:
 497    case TGSI_OPCODE_D2U:
 498    case TGSI_OPCODE_DABS:
 499    case TGSI_OPCODE_DNEG:
 500    case TGSI_OPCODE_DADD:
 501    case TGSI_OPCODE_DMUL:
 502    case TGSI_OPCODE_DMAX:
 503    case TGSI_OPCODE_DMIN:
 504    case TGSI_OPCODE_DSLT:
 505    case TGSI_OPCODE_DSGE:
 506    case TGSI_OPCODE_DSEQ:
 507    case TGSI_OPCODE_DSNE:
 508    case TGSI_OPCODE_DRCP:
 509    case TGSI_OPCODE_DSQRT:
 510    case TGSI_OPCODE_DMAD:
 511    case TGSI_OPCODE_DFRAC:
 512    case TGSI_OPCODE_DRSQ:
 513    case TGSI_OPCODE_DTRUNC:
 514    case TGSI_OPCODE_DCEIL:
 515    case TGSI_OPCODE_DFLR:
 516    case TGSI_OPCODE_DROUND:
 517       return nv50_ir::TYPE_F64;
 518    default:
 519       return nv50_ir::TYPE_F32;
 520    }
 521 }
 522
 523 nv50_ir::DataType Instruction::inferDstType() const
 524 {
 525    switch (getOpcode()) {
 526    case TGSI_OPCODE_D2U:
 527    case TGSI_OPCODE_F2U: return nv50_ir::TYPE_U32;
 528    case TGSI_OPCODE_D2I:
 529    case TGSI_OPCODE_F2I: return nv50_ir::TYPE_S32;
 530    case TGSI_OPCODE_FSEQ:
 531    case TGSI_OPCODE_FSGE:
 532    case TGSI_OPCODE_FSLT:
 533    case TGSI_OPCODE_FSNE:
 534    case TGSI_OPCODE_DSEQ:
 535    case TGSI_OPCODE_DSGE:
 536    case TGSI_OPCODE_DSLT:
 537    case TGSI_OPCODE_DSNE:
 538    case TGSI_OPCODE_PK2H:
 539       return nv50_ir::TYPE_U32;
 540    case TGSI_OPCODE_I2F:
 541    case TGSI_OPCODE_U2F:
 542    case TGSI_OPCODE_D2F:
 543    case TGSI_OPCODE_UP2H:
 544       return nv50_ir::TYPE_F32;
 545    case TGSI_OPCODE_I2D:
 546    case TGSI_OPCODE_U2D:
 547    case TGSI_OPCODE_F2D:
 548       return nv50_ir::TYPE_F64;
 549    default:
 550       return inferSrcType();
 551    }
 552 }
 553
 554 nv50_ir::CondCode Instruction::getSetCond() const
 555 {
 556    using namespace nv50_ir;
 557
 558    switch (getOpcode()) {
 559    case TGSI_OPCODE_SLT:
 560    case TGSI_OPCODE_ISLT:
 561    case TGSI_OPCODE_USLT:
 562    case TGSI_OPCODE_FSLT:
 563    case TGSI_OPCODE_DSLT:
 564       return CC_LT;
 565    case TGSI_OPCODE_SLE:
 566       return CC_LE;
 567    case TGSI_OPCODE_SGE:
 568    case TGSI_OPCODE_ISGE:
 569    case TGSI_OPCODE_USGE:
 570    case TGSI_OPCODE_FSGE:
 571    case TGSI_OPCODE_DSGE:
 572       return CC_GE;
 573    case TGSI_OPCODE_SGT:
 574       return CC_GT;
 575    case TGSI_OPCODE_SEQ:
 576    case TGSI_OPCODE_USEQ:
 577    case TGSI_OPCODE_FSEQ:
 578    case TGSI_OPCODE_DSEQ:
 579       return CC_EQ;
 580    case TGSI_OPCODE_SNE:
 581    case TGSI_OPCODE_FSNE:
 582    case TGSI_OPCODE_DSNE:
 583       return CC_NEU;
 584    case TGSI_OPCODE_USNE:
 585       return CC_NE;
 586    default:
 587       return CC_ALWAYS;
 588    }
 589 }
 590
 591 #define NV50_IR_OPCODE_CASE(a, b) case TGSI_OPCODE_##a: return nv50_ir::OP_##b
 592
 593 static nv50_ir::operation translateOpcode(uint opcode)
 594 {
 595    switch (opcode) {
 596    NV50_IR_OPCODE_CASE(ARL, SHL);
 597    NV50_IR_OPCODE_CASE(MOV, MOV);
 598
 599    NV50_IR_OPCODE_CASE(RCP, RCP);
 600    NV50_IR_OPCODE_CASE(RSQ, RSQ);
 601
 602    NV50_IR_OPCODE_CASE(MUL, MUL);
 603    NV50_IR_OPCODE_CASE(ADD, ADD);
 604
 605    NV50_IR_OPCODE_CASE(MIN, MIN);
 606    NV50_IR_OPCODE_CASE(MAX, MAX);
 607    NV50_IR_OPCODE_CASE(SLT, SET);
 608    NV50_IR_OPCODE_CASE(SGE, SET);
 609    NV50_IR_OPCODE_CASE(MAD, MAD);
 610    NV50_IR_OPCODE_CASE(SUB, SUB);
 611
 612    NV50_IR_OPCODE_CASE(FLR, FLOOR);
 613    NV50_IR_OPCODE_CASE(ROUND, CVT);
 614    NV50_IR_OPCODE_CASE(EX2, EX2);
 615    NV50_IR_OPCODE_CASE(LG2, LG2);
 616    NV50_IR_OPCODE_CASE(POW, POW);
 617
 618    NV50_IR_OPCODE_CASE(ABS, ABS);
 619
 620    NV50_IR_OPCODE_CASE(COS, COS);
 621    NV50_IR_OPCODE_CASE(DDX, DFDX);
 622    NV50_IR_OPCODE_CASE(DDX_FINE, DFDX);
 623    NV50_IR_OPCODE_CASE(DDY, DFDY);
 624    NV50_IR_OPCODE_CASE(DDY_FINE, DFDY);
 625    NV50_IR_OPCODE_CASE(KILL, DISCARD);
 626
 627    NV50_IR_OPCODE_CASE(SEQ, SET);
 628    NV50_IR_OPCODE_CASE(SGT, SET);
 629    NV50_IR_OPCODE_CASE(SIN, SIN);
 630    NV50_IR_OPCODE_CASE(SLE, SET);
 631    NV50_IR_OPCODE_CASE(SNE, SET);
 632    NV50_IR_OPCODE_CASE(TEX, TEX);
 633    NV50_IR_OPCODE_CASE(TXD, TXD);
 634    NV50_IR_OPCODE_CASE(TXP, TEX);
 635
 636    NV50_IR_OPCODE_CASE(CAL, CALL);
 637    NV50_IR_OPCODE_CASE(RET, RET);
 638    NV50_IR_OPCODE_CASE(CMP, SLCT);
 639
 640    NV50_IR_OPCODE_CASE(TXB, TXB);
 641
 642    NV50_IR_OPCODE_CASE(DIV, DIV);
 643
 644    NV50_IR_OPCODE_CASE(TXL, TXL);
 645
 646    NV50_IR_OPCODE_CASE(CEIL, CEIL);
 647    NV50_IR_OPCODE_CASE(I2F, CVT);
 648    NV50_IR_OPCODE_CASE(NOT, NOT);
 649    NV50_IR_OPCODE_CASE(TRUNC, TRUNC);
 650    NV50_IR_OPCODE_CASE(SHL, SHL);
 651
 652    NV50_IR_OPCODE_CASE(AND, AND);
 653    NV50_IR_OPCODE_CASE(OR, OR);
 654    NV50_IR_OPCODE_CASE(MOD, MOD);
 655    NV50_IR_OPCODE_CASE(XOR, XOR);
 656    NV50_IR_OPCODE_CASE(SAD, SAD);
 657    NV50_IR_OPCODE_CASE(TXF, TXF);
 658    NV50_IR_OPCODE_CASE(TXQ, TXQ);
 659    NV50_IR_OPCODE_CASE(TXQS, TXQ);
 660    NV50_IR_OPCODE_CASE(TG4, TXG);
 661    NV50_IR_OPCODE_CASE(LODQ, TXLQ);
 662
 663    NV50_IR_OPCODE_CASE(EMIT, EMIT);
 664    NV50_IR_OPCODE_CASE(ENDPRIM, RESTART);
 665
 666    NV50_IR_OPCODE_CASE(KILL_IF, DISCARD);
 667
 668    NV50_IR_OPCODE_CASE(F2I, CVT);
 669    NV50_IR_OPCODE_CASE(FSEQ, SET);
 670    NV50_IR_OPCODE_CASE(FSGE, SET);
 671    NV50_IR_OPCODE_CASE(FSLT, SET);
 672    NV50_IR_OPCODE_CASE(FSNE, SET);
 673    NV50_IR_OPCODE_CASE(IDIV, DIV);
 674    NV50_IR_OPCODE_CASE(IMAX, MAX);
 675    NV50_IR_OPCODE_CASE(IMIN, MIN);
 676    NV50_IR_OPCODE_CASE(IABS, ABS);
 677    NV50_IR_OPCODE_CASE(INEG, NEG);
 678    NV50_IR_OPCODE_CASE(ISGE, SET);
 679    NV50_IR_OPCODE_CASE(ISHR, SHR);
 680    NV50_IR_OPCODE_CASE(ISLT, SET);
 681    NV50_IR_OPCODE_CASE(F2U, CVT);
 682    NV50_IR_OPCODE_CASE(U2F, CVT);
 683    NV50_IR_OPCODE_CASE(UADD, ADD);
 684    NV50_IR_OPCODE_CASE(UDIV, DIV);
 685    NV50_IR_OPCODE_CASE(UMAD, MAD);
 686    NV50_IR_OPCODE_CASE(UMAX, MAX);
 687    NV50_IR_OPCODE_CASE(UMIN, MIN);
 688    NV50_IR_OPCODE_CASE(UMOD, MOD);
 689    NV50_IR_OPCODE_CASE(UMUL, MUL);
 690    NV50_IR_OPCODE_CASE(USEQ, SET);
 691    NV50_IR_OPCODE_CASE(USGE, SET);
 692    NV50_IR_OPCODE_CASE(USHR, SHR);
 693    NV50_IR_OPCODE_CASE(USLT, SET);
 694    NV50_IR_OPCODE_CASE(USNE, SET);
 695
 696    NV50_IR_OPCODE_CASE(DABS, ABS);
 697    NV50_IR_OPCODE_CASE(DNEG, NEG);
 698    NV50_IR_OPCODE_CASE(DADD, ADD);
 699    NV50_IR_OPCODE_CASE(DMUL, MUL);
 700    NV50_IR_OPCODE_CASE(DMAX, MAX);
 701    NV50_IR_OPCODE_CASE(DMIN, MIN);
 702    NV50_IR_OPCODE_CASE(DSLT, SET);
 703    NV50_IR_OPCODE_CASE(DSGE, SET);
 704    NV50_IR_OPCODE_CASE(DSEQ, SET);
 705    NV50_IR_OPCODE_CASE(DSNE, SET);
 706    NV50_IR_OPCODE_CASE(DRCP, RCP);
 707    NV50_IR_OPCODE_CASE(DSQRT, SQRT);
 708    NV50_IR_OPCODE_CASE(DMAD, MAD);
 709    NV50_IR_OPCODE_CASE(D2I, CVT);
 710    NV50_IR_OPCODE_CASE(D2U, CVT);
 711    NV50_IR_OPCODE_CASE(I2D, CVT);
 712    NV50_IR_OPCODE_CASE(U2D, CVT);
 713    NV50_IR_OPCODE_CASE(DRSQ, RSQ);
 714    NV50_IR_OPCODE_CASE(DTRUNC, TRUNC);
 715    NV50_IR_OPCODE_CASE(DCEIL, CEIL);
 716    NV50_IR_OPCODE_CASE(DFLR, FLOOR);
 717    NV50_IR_OPCODE_CASE(DROUND, CVT);
 718
 719    NV50_IR_OPCODE_CASE(IMUL_HI, MUL);
 720    NV50_IR_OPCODE_CASE(UMUL_HI, MUL);
 721
 722    NV50_IR_OPCODE_CASE(SAMPLE, TEX);
 723    NV50_IR_OPCODE_CASE(SAMPLE_B, TXB);
 724    NV50_IR_OPCODE_CASE(SAMPLE_C, TEX);
 725    NV50_IR_OPCODE_CASE(SAMPLE_C_LZ, TEX);
 726    NV50_IR_OPCODE_CASE(SAMPLE_D, TXD);
 727    NV50_IR_OPCODE_CASE(SAMPLE_L, TXL);
 728    NV50_IR_OPCODE_CASE(SAMPLE_I, TXF);
 729    NV50_IR_OPCODE_CASE(SAMPLE_I_MS, TXF);
 730    NV50_IR_OPCODE_CASE(GATHER4, TXG);
 731    NV50_IR_OPCODE_CASE(SVIEWINFO, TXQ);
 732
 733    NV50_IR_OPCODE_CASE(ATOMUADD, ATOM);
 734    NV50_IR_OPCODE_CASE(ATOMXCHG, ATOM);
 735    NV50_IR_OPCODE_CASE(ATOMCAS, ATOM);
 736    NV50_IR_OPCODE_CASE(ATOMAND, ATOM);
 737    NV50_IR_OPCODE_CASE(ATOMOR, ATOM);
 738    NV50_IR_OPCODE_CASE(ATOMXOR, ATOM);
 739    NV50_IR_OPCODE_CASE(ATOMUMIN, ATOM);
 740    NV50_IR_OPCODE_CASE(ATOMUMAX, ATOM);
 741    NV50_IR_OPCODE_CASE(ATOMIMIN, ATOM);
 742    NV50_IR_OPCODE_CASE(ATOMIMAX, ATOM);
 743
 744    NV50_IR_OPCODE_CASE(TEX2, TEX);
 745    NV50_IR_OPCODE_CASE(TXB2, TXB);
 746    NV50_IR_OPCODE_CASE(TXL2, TXL);
 747
 748    NV50_IR_OPCODE_CASE(IBFE, EXTBF);
 749    NV50_IR_OPCODE_CASE(UBFE, EXTBF);
 750    NV50_IR_OPCODE_CASE(BFI, INSBF);
 751    NV50_IR_OPCODE_CASE(BREV, EXTBF);
 752    NV50_IR_OPCODE_CASE(POPC, POPCNT);
 753    NV50_IR_OPCODE_CASE(LSB, BFIND);
 754    NV50_IR_OPCODE_CASE(IMSB, BFIND);
 755    NV50_IR_OPCODE_CASE(UMSB, BFIND);
 756
 757    NV50_IR_OPCODE_CASE(END, EXIT);
 758
 759    default:
 760       return nv50_ir::OP_NOP;
 761    }
 762 }
 763
 764 static uint16_t opcodeToSubOp(uint opcode)
 765 {
 766    switch (opcode) {
 767    case TGSI_OPCODE_LFENCE:   return NV50_IR_SUBOP_MEMBAR(L, GL);
 768    case TGSI_OPCODE_SFENCE:   return NV50_IR_SUBOP_MEMBAR(S, GL);
 769    case TGSI_OPCODE_MFENCE:   return NV50_IR_SUBOP_MEMBAR(M, GL);
 770    case TGSI_OPCODE_ATOMUADD: return NV50_IR_SUBOP_ATOM_ADD;
 771    case TGSI_OPCODE_ATOMXCHG: return NV50_IR_SUBOP_ATOM_EXCH;
 772    case TGSI_OPCODE_ATOMCAS:  return NV50_IR_SUBOP_ATOM_CAS;
 773    case TGSI_OPCODE_ATOMAND:  return NV50_IR_SUBOP_ATOM_AND;
 774    case TGSI_OPCODE_ATOMOR:   return NV50_IR_SUBOP_ATOM_OR;
 775    case TGSI_OPCODE_ATOMXOR:  return NV50_IR_SUBOP_ATOM_XOR;
 776    case TGSI_OPCODE_ATOMUMIN: return NV50_IR_SUBOP_ATOM_MIN;
 777    case TGSI_OPCODE_ATOMIMIN: return NV50_IR_SUBOP_ATOM_MIN;
 778    case TGSI_OPCODE_ATOMUMAX: return NV50_IR_SUBOP_ATOM_MAX;
 779    case TGSI_OPCODE_ATOMIMAX: return NV50_IR_SUBOP_ATOM_MAX;
 780    case TGSI_OPCODE_IMUL_HI:
 781    case TGSI_OPCODE_UMUL_HI:
 782       return NV50_IR_SUBOP_MUL_HIGH;
 783    default:
 784       return 0;
 785    }
 786 }
 787
 788 bool Instruction::checkDstSrcAliasing() const
 789 {
 790    if (insn->Dst[0].Register.Indirect) // no danger if indirect, using memory
 791       return false;
 792
 793    for (int s = 0; s < TGSI_FULL_MAX_SRC_REGISTERS; ++s) {
 794       if (insn->Src[s].Register.File == TGSI_FILE_NULL)
 795          break;
 796       if (insn->Src[s].Register.File == insn->Dst[0].Register.File &&
 797           insn->Src[s].Register.Index == insn->Dst[0].Register.Index)
 798          return true;
 799    }
 800    return false;
 801 }
 802
 803 class Source
 804 {
 805 public:
 806    Source(struct nv50_ir_prog_info *);
 807    ~Source();
 808
 809 public:
 810    bool scanSource();
 811    unsigned fileSize(unsigned file) const { return scan.file_max[file] + 1; }
 812
 813 public:
 814    struct tgsi_shader_info scan;
 815    struct tgsi_full_instruction *insns;
 816    const struct tgsi_token *tokens;
 817    struct nv50_ir_prog_info *info;
 818
 819    nv50_ir::DynArray tempArrays;
 820    nv50_ir::DynArray immdArrays;
 821
 822    typedef nv50_ir::BuildUtil::Location Location;
 823    // these registers are per-subroutine, cannot be used for parameter passing
 824    std::set<Location> locals;
 825
 826    std::set<int> indirectTempArrays;
 827    std::vector<int> tempArrayId;
 828
 829    int clipVertexOutput;
 830
 831    struct TextureView {
 832       uint8_t target; // TGSI_TEXTURE_*
 833    };
 834    std::vector<TextureView> textureViews;
 835
 836    struct Resource {
 837       uint8_t target; // TGSI_TEXTURE_*
 838       bool raw;
 839       uint8_t slot; // $surface index
 840    };
 841    std::vector<Resource> resources;
 842
 843 private:
 844    int inferSysValDirection(unsigned sn) const;
 845    bool scanDeclaration(const struct tgsi_full_declaration *);
 846    bool scanInstruction(const struct tgsi_full_instruction *);
 847    void scanProperty(const struct tgsi_full_property *);
 848    void scanImmediate(const struct tgsi_full_immediate *);
 849
 850    inline bool isEdgeFlagPassthrough(const Instruction&) const;
 851 };
 852
 853 Source::Source(struct nv50_ir_prog_info *prog) : info(prog)
 854 {
 855    tokens = (const struct tgsi_token *)info->bin.source;
 856
 857    if (prog->dbgFlags & NV50_IR_DEBUG_BASIC)
 858       tgsi_dump(tokens, 0);
 859 }
 860
 861 Source::~Source()
 862 {
 863    if (insns)
 864       FREE(insns);
 865
 866    if (info->immd.data)
 867       FREE(info->immd.data);
 868    if (info->immd.type)
 869       FREE(info->immd.type);
 870 }
 871
 872 bool Source::scanSource()
 873 {
 874    unsigned insnCount = 0;
 875    struct tgsi_parse_context parse;
 876
 877    tgsi_scan_shader(tokens, &scan);
 878
 879    insns = (struct tgsi_full_instruction *)MALLOC(scan.num_instructions *
 880                                                   sizeof(insns[0]));
 881    if (!insns)
 882       return false;
 883
 884    clipVertexOutput = -1;
 885
 886    textureViews.resize(scan.file_max[TGSI_FILE_SAMPLER_VIEW] + 1);
 887    //resources.resize(scan.file_max[TGSI_FILE_RESOURCE] + 1);
 888    tempArrayId.resize(scan.file_max[TGSI_FILE_TEMPORARY] + 1);
 889
 890    info->immd.bufSize = 0;
 891
 892    info->numInputs = scan.file_max[TGSI_FILE_INPUT] + 1;
 893    info->numOutputs = scan.file_max[TGSI_FILE_OUTPUT] + 1;
 894    info->numSysVals = scan.file_max[TGSI_FILE_SYSTEM_VALUE] + 1;
 895
 896    if (info->type == PIPE_SHADER_FRAGMENT) {
 897       info->prop.fp.writesDepth = scan.writes_z;
 898       info->prop.fp.usesDiscard = scan.uses_kill;
 899    } else
 900    if (info->type == PIPE_SHADER_GEOMETRY) {
 901       info->prop.gp.instanceCount = 1; // default value
 902    }
 903
 904    info->io.viewportId = -1;
 905
 906    info->immd.data = (uint32_t *)MALLOC(scan.immediate_count * 16);
 907    info->immd.type = (ubyte *)MALLOC(scan.immediate_count * sizeof(ubyte));
 908
 909    tgsi_parse_init(&parse, tokens);
 910    while (!tgsi_parse_end_of_tokens(&parse)) {
 911       tgsi_parse_token(&parse);
 912
 913       switch (parse.FullToken.Token.Type) {
 914       case TGSI_TOKEN_TYPE_IMMEDIATE:
 915          scanImmediate(&parse.FullToken.FullImmediate);
 916          break;
 917       case TGSI_TOKEN_TYPE_DECLARATION:
 918          scanDeclaration(&parse.FullToken.FullDeclaration);
 919          break;
 920       case TGSI_TOKEN_TYPE_INSTRUCTION:
 921          insns[insnCount++] = parse.FullToken.FullInstruction;
 922          scanInstruction(&parse.FullToken.FullInstruction);
 923          break;
 924       case TGSI_TOKEN_TYPE_PROPERTY:
 925          scanProperty(&parse.FullToken.FullProperty);
 926          break;
 927       default:
 928          INFO("unknown TGSI token type: %d\n", parse.FullToken.Token.Type);
 929          break;
 930       }
 931    }
 932    tgsi_parse_free(&parse);
 933
 934    // TODO: Compute based on relevant array sizes
 935    if (indirectTempArrays.size())
 936       info->bin.tlsSpace += (scan.file_max[TGSI_FILE_TEMPORARY] + 1) * 16;
 937
 938    if (info->io.genUserClip > 0) {
 939       info->io.clipDistances = info->io.genUserClip;
 940
 941       const unsigned int nOut = (info->io.genUserClip + 3) / 4;
 942
 943       for (unsigned int n = 0; n < nOut; ++n) {
 944          unsigned int i = info->numOutputs++;
 945          info->out[i].id = i;
 946          info->out[i].sn = TGSI_SEMANTIC_CLIPDIST;
 947          info->out[i].si = n;
 948          info->out[i].mask = ((1 << info->io.clipDistances) - 1) >> (n * 4);
 949       }
 950    }
 951
 952    return info->assignSlots(info) == 0;
 953 }
 954
 955 void Source::scanProperty(const struct tgsi_full_property *prop)
 956 {
 957    switch (prop->Property.PropertyName) {
 958    case TGSI_PROPERTY_GS_OUTPUT_PRIM:
 959       info->prop.gp.outputPrim = prop->u[0].Data;
 960       break;
 961    case TGSI_PROPERTY_GS_INPUT_PRIM:
 962       info->prop.gp.inputPrim = prop->u[0].Data;
 963       break;
 964    case TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES:
 965       info->prop.gp.maxVertices = prop->u[0].Data;
 966       break;
 967    case TGSI_PROPERTY_GS_INVOCATIONS:
 968       info->prop.gp.instanceCount = prop->u[0].Data;
 969       break;
 970    case TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS:
 971       info->prop.fp.separateFragData = true;
 972       break;
 973    case TGSI_PROPERTY_FS_COORD_ORIGIN:
 974    case TGSI_PROPERTY_FS_COORD_PIXEL_CENTER:
 975       // we don't care
 976       break;
 977    case TGSI_PROPERTY_VS_PROHIBIT_UCPS:
 978       info->io.genUserClip = -1;
 979       break;
 980    case TGSI_PROPERTY_TCS_VERTICES_OUT:
 981       info->prop.tp.outputPatchSize = prop->u[0].Data;
 982       break;
 983    case TGSI_PROPERTY_TES_PRIM_MODE:
 984       info->prop.tp.domain = prop->u[0].Data;
 985       break;
 986    case TGSI_PROPERTY_TES_SPACING:
 987       info->prop.tp.partitioning = prop->u[0].Data;
 988       break;
 989    case TGSI_PROPERTY_TES_VERTEX_ORDER_CW:
 990       info->prop.tp.winding = prop->u[0].Data;
 991       break;
 992    case TGSI_PROPERTY_TES_POINT_MODE:
 993       if (prop->u[0].Data)
 994          info->prop.tp.outputPrim = PIPE_PRIM_POINTS;
 995       else
 996          info->prop.tp.outputPrim = PIPE_PRIM_TRIANGLES; /* anything but points */
 997       break;
 998    case TGSI_PROPERTY_NUM_CLIPDIST_ENABLED:
 999       info->io.clipDistances = prop->u[0].Data;
1000       break;
1001    case TGSI_PROPERTY_NUM_CULLDIST_ENABLED:
1002       info->io.cullDistances = prop->u[0].Data;
1003       break;
1004    default:
1005       INFO("unhandled TGSI property %d\n", prop->Property.PropertyName);
1006       break;
1007    }
1008 }
1009
1010 void Source::scanImmediate(const struct tgsi_full_immediate *imm)
1011 {
1012    const unsigned n = info->immd.count++;
1013
1014    assert(n < scan.immediate_count);
1015
1016    for (int c = 0; c < 4; ++c)
1017       info->immd.data[n * 4 + c] = imm->u[c].Uint;
1018
1019    info->immd.type[n] = imm->Immediate.DataType;
1020 }
1021
1022 int Source::inferSysValDirection(unsigned sn) const
1023 {
1024    switch (sn) {
1025    case TGSI_SEMANTIC_INSTANCEID:
1026    case TGSI_SEMANTIC_VERTEXID:
1027       return 1;
1028    case TGSI_SEMANTIC_LAYER:
1029 #if 0
1030    case TGSI_SEMANTIC_VIEWPORTINDEX:
1031       return 0;
1032 #endif
1033    case TGSI_SEMANTIC_PRIMID:
1034       return (info->type == PIPE_SHADER_FRAGMENT) ? 1 : 0;
1035    default:
1036       return 0;
1037    }
1038 }
1039
1040 bool Source::scanDeclaration(const struct tgsi_full_declaration *decl)
1041 {
1042    unsigned i, c;
1043    unsigned sn = TGSI_SEMANTIC_GENERIC;
1044    unsigned si = 0;
1045    const unsigned first = decl->Range.First, last = decl->Range.Last;
1046    const int arrayId = decl->Array.ArrayID;
1047
1048    if (decl->Declaration.Semantic) {
1049       sn = decl->Semantic.Name;
1050       si = decl->Semantic.Index;
1051    }
1052
1053    if (decl->Declaration.Local) {
1054       for (i = first; i <= last; ++i) {
1055          for (c = 0; c < 4; ++c) {
1056             locals.insert(
1057                Location(decl->Declaration.File, decl->Dim.Index2D, i, c));
1058          }
1059       }
1060    }
1061
1062    switch (decl->Declaration.File) {
1063    case TGSI_FILE_INPUT:
1064       if (info->type == PIPE_SHADER_VERTEX) {
1065          // all vertex attributes are equal
1066          for (i = first; i <= last; ++i) {
1067             info->in[i].sn = TGSI_SEMANTIC_GENERIC;
1068             info->in[i].si = i;
1069          }
1070       } else {
1071          for (i = first; i <= last; ++i, ++si) {
1072             info->in[i].id = i;
1073             info->in[i].sn = sn;
1074             info->in[i].si = si;
1075             if (info->type == PIPE_SHADER_FRAGMENT) {
1076                // translate interpolation mode
1077                switch (decl->Interp.Interpolate) {
1078                case TGSI_INTERPOLATE_CONSTANT:
1079                   info->in[i].flat = 1;
1080                   break;
1081                case TGSI_INTERPOLATE_COLOR:
1082                   info->in[i].sc = 1;
1083                   break;
1084                case TGSI_INTERPOLATE_LINEAR:
1085                   info->in[i].linear = 1;
1086                   break;
1087                default:
1088                   break;
1089                }
1090                if (decl->Interp.Location)
1091                   info->in[i].centroid = 1;
1092             }
1093
1094             if (sn == TGSI_SEMANTIC_PATCH)
1095                info->in[i].patch = 1;
1096             if (sn == TGSI_SEMANTIC_PATCH)
1097                info->numPatchConstants = MAX2(info->numPatchConstants, si + 1);
1098          }
1099       }
1100       break;
1101    case TGSI_FILE_OUTPUT:
1102       for (i = first; i <= last; ++i, ++si) {
1103          switch (sn) {
1104          case TGSI_SEMANTIC_POSITION:
1105             if (info->type == PIPE_SHADER_FRAGMENT)
1106                info->io.fragDepth = i;
1107             else
1108             if (clipVertexOutput < 0)
1109                clipVertexOutput = i;
1110             break;
1111          case TGSI_SEMANTIC_COLOR:
1112             if (info->type == PIPE_SHADER_FRAGMENT)
1113                info->prop.fp.numColourResults++;
1114             break;
1115          case TGSI_SEMANTIC_EDGEFLAG:
1116             info->io.edgeFlagOut = i;
1117             break;
1118          case TGSI_SEMANTIC_CLIPVERTEX:
1119             clipVertexOutput = i;
1120             break;
1121          case TGSI_SEMANTIC_CLIPDIST:
1122             info->io.genUserClip = -1;
1123             break;
1124          case TGSI_SEMANTIC_SAMPLEMASK:
1125             info->io.sampleMask = i;
1126             break;
1127          case TGSI_SEMANTIC_VIEWPORT_INDEX:
1128             info->io.viewportId = i;
1129             break;
1130          case TGSI_SEMANTIC_PATCH:
1131             info->numPatchConstants = MAX2(info->numPatchConstants, si + 1);
1132             /* fallthrough */
1133          case TGSI_SEMANTIC_TESSOUTER:
1134          case TGSI_SEMANTIC_TESSINNER:
1135             info->out[i].patch = 1;
1136             break;
1137          default:
1138             break;
1139          }
1140          info->out[i].id = i;
1141          info->out[i].sn = sn;
1142          info->out[i].si = si;
1143       }
1144       break;
1145    case TGSI_FILE_SYSTEM_VALUE:
1146       switch (sn) {
1147       case TGSI_SEMANTIC_INSTANCEID:
1148          info->io.instanceId = first;
1149          break;
1150       case TGSI_SEMANTIC_VERTEXID:
1151          info->io.vertexId = first;
1152          break;
1153       case TGSI_SEMANTIC_SAMPLEID:
1154       case TGSI_SEMANTIC_SAMPLEPOS:
1155          info->prop.fp.sampleInterp = 1;
1156          break;
1157       case TGSI_SEMANTIC_BASEVERTEX:
1158       case TGSI_SEMANTIC_BASEINSTANCE:
1159       case TGSI_SEMANTIC_DRAWID:
1160          info->prop.vp.usesDrawParameters = true;
1161          break;
1162       default:
1163          break;
1164       }
1165       for (i = first; i <= last; ++i, ++si) {
1166          info->sv[i].sn = sn;
1167          info->sv[i].si = si;
1168          info->sv[i].input = inferSysValDirection(sn);
1169
1170          switch (sn) {
1171          case TGSI_SEMANTIC_TESSOUTER:
1172          case TGSI_SEMANTIC_TESSINNER:
1173             info->sv[i].patch = 1;
1174             break;
1175          }
1176       }
1177       break;
1178 /*
1179    case TGSI_FILE_RESOURCE:
1180       for (i = first; i <= last; ++i) {
1181          resources[i].target = decl->Resource.Resource;
1182          resources[i].raw = decl->Resource.Raw;
1183          resources[i].slot = i;
1184       }
1185       break;
1186 */
1187    case TGSI_FILE_SAMPLER_VIEW:
1188       for (i = first; i <= last; ++i)
1189          textureViews[i].target = decl->SamplerView.Resource;
1190       break;
1191    case TGSI_FILE_TEMPORARY:
1192       for (i = first; i <= last; ++i)
1193          tempArrayId[i] = arrayId;
1194       break;
1195    case TGSI_FILE_NULL:
1196    case TGSI_FILE_ADDRESS:
1197    case TGSI_FILE_CONSTANT:
1198    case TGSI_FILE_IMMEDIATE:
1199    case TGSI_FILE_PREDICATE:
1200    case TGSI_FILE_SAMPLER:
1201       break;
1202    default:
1203       ERROR("unhandled TGSI_FILE %d\n", decl->Declaration.File);
1204       return false;
1205    }
1206    return true;
1207 }
1208
1209 inline bool Source::isEdgeFlagPassthrough(const Instruction& insn) const
1210 {
1211    return insn.getOpcode() == TGSI_OPCODE_MOV &&
1212       insn.getDst(0).getIndex(0) == info->io.edgeFlagOut &&
1213       insn.getSrc(0).getFile() == TGSI_FILE_INPUT;
1214 }
1215
1216 bool Source::scanInstruction(const struct tgsi_full_instruction *inst)
1217 {
1218    Instruction insn(inst);
1219
1220    if (insn.getOpcode() == TGSI_OPCODE_BARRIER)
1221       info->numBarriers = 1;
1222
1223    if (insn.dstCount()) {
1224       if (insn.getDst(0).getFile() == TGSI_FILE_OUTPUT) {
1225          Instruction::DstRegister dst = insn.getDst(0);
1226
1227          if (dst.isIndirect(0))
1228             for (unsigned i = 0; i < info->numOutputs; ++i)
1229                info->out[i].mask = 0xf;
1230          else
1231             info->out[dst.getIndex(0)].mask |= dst.getMask();
1232
1233          if (info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PSIZE ||
1234              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PRIMID ||
1235              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_LAYER ||
1236              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_VIEWPORT_INDEX ||
1237              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_FOG)
1238             info->out[dst.getIndex(0)].mask &= 1;
1239
1240          if (isEdgeFlagPassthrough(insn))
1241             info->io.edgeFlagIn = insn.getSrc(0).getIndex(0);
1242       } else
1243       if (insn.getDst(0).getFile() == TGSI_FILE_TEMPORARY) {
1244          if (insn.getDst(0).isIndirect(0))
1245             indirectTempArrays.insert(insn.getDst(0).getArrayId());
1246       }
1247    }
1248
1249    for (unsigned s = 0; s < insn.srcCount(); ++s) {
1250       Instruction::SrcRegister src = insn.getSrc(s);
1251       if (src.getFile() == TGSI_FILE_TEMPORARY) {
1252          if (src.isIndirect(0))
1253             indirectTempArrays.insert(src.getArrayId());
1254       } else
1255 /*
1256       if (src.getFile() == TGSI_FILE_RESOURCE) {
1257          if (src.getIndex(0) == TGSI_RESOURCE_GLOBAL)
1258             info->io.globalAccess |= (insn.getOpcode() == TGSI_OPCODE_LOAD) ?
1259                0x1 : 0x2;
1260       } else
1261 */
1262       if (src.getFile() == TGSI_FILE_OUTPUT) {
1263          if (src.isIndirect(0)) {
1264             // We don't know which one is accessed, just mark everything for
1265             // reading. This is an extremely unlikely occurrence.
1266             for (unsigned i = 0; i < info->numOutputs; ++i)
1267                info->out[i].oread = 1;
1268          } else {
1269             info->out[src.getIndex(0)].oread = 1;
1270          }
1271       }
1272       if (src.getFile() != TGSI_FILE_INPUT)
1273          continue;
1274       unsigned mask = insn.srcMask(s);
1275
1276       if (src.isIndirect(0)) {
1277          for (unsigned i = 0; i < info->numInputs; ++i)
1278             info->in[i].mask = 0xf;
1279       } else {
1280          const int i = src.getIndex(0);
1281          for (unsigned c = 0; c < 4; ++c) {
1282             if (!(mask & (1 << c)))
1283                continue;
1284             int k = src.getSwizzle(c);
1285             if (k <= TGSI_SWIZZLE_W)
1286                info->in[i].mask |= 1 << k;
1287          }
1288          switch (info->in[i].sn) {
1289          case TGSI_SEMANTIC_PSIZE:
1290          case TGSI_SEMANTIC_PRIMID:
1291          case TGSI_SEMANTIC_FOG:
1292             info->in[i].mask &= 0x1;
1293             break;
1294          case TGSI_SEMANTIC_PCOORD:
1295             info->in[i].mask &= 0x3;
1296             break;
1297          default:
1298             break;
1299          }
1300       }
1301    }
1302    return true;
1303 }
1304
1305 nv50_ir::TexInstruction::Target
1306 Instruction::getTexture(const tgsi::Source *code, int s) const
1307 {
1308    // XXX: indirect access
1309    unsigned int r;
1310
1311    switch (getSrc(s).getFile()) {
1312 /*
1313    case TGSI_FILE_RESOURCE:
1314       r = getSrc(s).getIndex(0);
1315       return translateTexture(code->resources.at(r).target);
1316 */
1317    case TGSI_FILE_SAMPLER_VIEW:
1318       r = getSrc(s).getIndex(0);
1319       return translateTexture(code->textureViews.at(r).target);
1320    default:
1321       return translateTexture(insn->Texture.Texture);
1322    }
1323 }
1324
1325 } // namespace tgsi
1326
1327 namespace {
1328
1329 using namespace nv50_ir;
1330
1331 class Converter : public BuildUtil
1332 {
1333 public:
1334    Converter(Program *, const tgsi::Source *);
1335    ~Converter();
1336
1337    bool run();
1338
1339 private:
1340    struct Subroutine
1341    {
1342       Subroutine(Function *f) : f(f) { }
1343       Function *f;
1344       ValueMap values;
1345    };
1346
1347    Value *shiftAddress(Value *);
1348    Value *getVertexBase(int s);
1349    Value *getOutputBase(int s);
1350    DataArray *getArrayForFile(unsigned file, int idx);
1351    Value *fetchSrc(int s, int c);
1352    Value *acquireDst(int d, int c);
1353    void storeDst(int d, int c, Value *);
1354
1355    Value *fetchSrc(const tgsi::Instruction::SrcRegister src, int c, Value *ptr);
1356    void storeDst(const tgsi::Instruction::DstRegister dst, int c,
1357                  Value *val, Value *ptr);
1358
1359    Value *applySrcMod(Value *, int s, int c);
1360
1361    Symbol *makeSym(uint file, int fileIndex, int idx, int c, uint32_t addr);
1362    Symbol *srcToSym(tgsi::Instruction::SrcRegister, int c);
1363    Symbol *dstToSym(tgsi::Instruction::DstRegister, int c);
1364
1365    bool handleInstruction(const struct tgsi_full_instruction *);
1366    void exportOutputs();
1367    inline Subroutine *getSubroutine(unsigned ip);
1368    inline Subroutine *getSubroutine(Function *);
1369    inline bool isEndOfSubroutine(uint ip);
1370
1371    void loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask);
1372
1373    // R,S,L,C,Dx,Dy encode TGSI sources for respective values (0xSf for auto)
1374    void setTexRS(TexInstruction *, unsigned int& s, int R, int S);
1375    void handleTEX(Value *dst0[4], int R, int S, int L, int C, int Dx, int Dy);
1376    void handleTXF(Value *dst0[4], int R, int L_M);
1377    void handleTXQ(Value *dst0[4], enum TexQuery, int R);
1378    void handleLIT(Value *dst0[4]);
1379    void handleUserClipPlanes();
1380
1381    Symbol *getResourceBase(int r);
1382    void getResourceCoords(std::vector<Value *>&, int r, int s);
1383
1384    void handleLOAD(Value *dst0[4]);
1385    void handleSTORE();
1386    void handleATOM(Value *dst0[4], DataType, uint16_t subOp);
1387
1388    void handleINTERP(Value *dst0[4]);
1389
1390    uint8_t translateInterpMode(const struct nv50_ir_varying *var,
1391                                operation& op);
1392    Value *interpolate(tgsi::Instruction::SrcRegister, int c, Value *ptr);
1393
1394    void insertConvergenceOps(BasicBlock *conv, BasicBlock *fork);
1395
1396    Value *buildDot(int dim);
1397
1398    class BindArgumentsPass : public Pass {
1399    public:
1400       BindArgumentsPass(Converter &conv) : conv(conv) { }
1401
1402    private:
1403       Converter &conv;
1404       Subroutine *sub;
1405
1406       inline const Location *getValueLocation(Subroutine *, Value *);
1407
1408       template<typename T> inline void
1409       updateCallArgs(Instruction *i, void (Instruction::*setArg)(int, Value *),
1410                      T (Function::*proto));
1411
1412       template<typename T> inline void
1413       updatePrototype(BitSet *set, void (Function::*updateSet)(),
1414                       T (Function::*proto));
1415
1416    protected:
1417       bool visit(Function *);
1418       bool visit(BasicBlock *bb) { return false; }
1419    };
1420
1421 private:
1422    const tgsi::Source *code;
1423    const struct nv50_ir_prog_info *info;
1424
1425    struct {
1426       std::map<unsigned, Subroutine> map;
1427       Subroutine *cur;
1428    } sub;
1429
1430    uint ip; // instruction pointer
1431
1432    tgsi::Instruction tgsi;
1433
1434    DataType dstTy;
1435    DataType srcTy;
1436
1437    DataArray tData; // TGSI_FILE_TEMPORARY
1438    DataArray lData; // TGSI_FILE_TEMPORARY, for indirect arrays
1439    DataArray aData; // TGSI_FILE_ADDRESS
1440    DataArray pData; // TGSI_FILE_PREDICATE
1441    DataArray oData; // TGSI_FILE_OUTPUT (if outputs in registers)
1442
1443    Value *zero;
1444    Value *fragCoord[4];
1445    Value *clipVtx[4];
1446
1447    Value *vtxBase[5]; // base address of vertex in primitive (for TP/GP)
1448    uint8_t vtxBaseValid;
1449
1450    Value *outBase; // base address of vertex out patch (for TCP)
1451
1452    Stack condBBs;  // fork BB, then else clause BB
1453    Stack joinBBs;  // fork BB, for inserting join ops on ENDIF
1454    Stack loopBBs;  // loop headers
1455    Stack breakBBs; // end of / after loop
1456
1457    Value *viewport;
1458 };
1459
1460 Symbol *
1461 Converter::srcToSym(tgsi::Instruction::SrcRegister src, int c)
1462 {
1463    const int swz = src.getSwizzle(c);
1464
1465    /* TODO: Use Array ID when it's available for the index */
1466    return makeSym(src.getFile(),
1467                   src.is2D() ? src.getIndex(1) : 0,
1468                   src.getIndex(0), swz,
1469                   src.getIndex(0) * 16 + swz * 4);
1470 }
1471
1472 Symbol *
1473 Converter::dstToSym(tgsi::Instruction::DstRegister dst, int c)
1474 {
1475    /* TODO: Use Array ID when it's available for the index */
1476    return makeSym(dst.getFile(),
1477                   dst.is2D() ? dst.getIndex(1) : 0,
1478                   dst.getIndex(0), c,
1479                   dst.getIndex(0) * 16 + c * 4);
1480 }
1481
1482 Symbol *
1483 Converter::makeSym(uint tgsiFile, int fileIdx, int idx, int c, uint32_t address)
1484 {
1485    Symbol *sym = new_Symbol(prog, tgsi::translateFile(tgsiFile));
1486
1487    sym->reg.fileIndex = fileIdx;
1488
1489    if (idx >= 0) {
1490       if (sym->reg.file == FILE_SHADER_INPUT)
1491          sym->setOffset(info->in[idx].slot[c] * 4);
1492       else
1493       if (sym->reg.file == FILE_SHADER_OUTPUT)
1494          sym->setOffset(info->out[idx].slot[c] * 4);
1495       else
1496       if (sym->reg.file == FILE_SYSTEM_VALUE)
1497          sym->setSV(tgsi::translateSysVal(info->sv[idx].sn), c);
1498       else
1499          sym->setOffset(address);
1500    } else {
1501       sym->setOffset(address);
1502    }
1503    return sym;
1504 }
1505
1506 uint8_t
1507 Converter::translateInterpMode(const struct nv50_ir_varying *var, operation& op)
1508 {
1509    uint8_t mode = NV50_IR_INTERP_PERSPECTIVE;
1510
1511    if (var->flat)
1512       mode = NV50_IR_INTERP_FLAT;
1513    else
1514    if (var->linear)
1515       mode = NV50_IR_INTERP_LINEAR;
1516    else
1517    if (var->sc)
1518       mode = NV50_IR_INTERP_SC;
1519
1520    op = (mode == NV50_IR_INTERP_PERSPECTIVE || mode == NV50_IR_INTERP_SC)
1521       ? OP_PINTERP : OP_LINTERP;
1522
1523    if (var->centroid || info->prop.fp.sampleInterp)
1524       mode |= NV50_IR_INTERP_CENTROID;
1525
1526    return mode;
1527 }
1528
1529 Value *
1530 Converter::interpolate(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
1531 {
1532    operation op;
1533
1534    // XXX: no way to know interpolation mode if we don't know what's accessed
1535    const uint8_t mode = translateInterpMode(&info->in[ptr ? 0 :
1536                                                       src.getIndex(0)], op);
1537
1538    Instruction *insn = new_Instruction(func, op, TYPE_F32);
1539
1540    insn->setDef(0, getScratch());
1541    insn->setSrc(0, srcToSym(src, c));
1542    if (op == OP_PINTERP)
1543       insn->setSrc(1, fragCoord[3]);
1544    if (ptr)
1545       insn->setIndirect(0, 0, ptr);
1546
1547    insn->setInterpolate(mode);
1548
1549    bb->insertTail(insn);
1550    return insn->getDef(0);
1551 }
1552
1553 Value *
1554 Converter::applySrcMod(Value *val, int s, int c)
1555 {
1556    Modifier m = tgsi.getSrc(s).getMod(c);
1557    DataType ty = tgsi.inferSrcType();
1558
1559    if (m & Modifier(NV50_IR_MOD_ABS))
1560       val = mkOp1v(OP_ABS, ty, getScratch(), val);
1561
1562    if (m & Modifier(NV50_IR_MOD_NEG))
1563       val = mkOp1v(OP_NEG, ty, getScratch(), val);
1564
1565    return val;
1566 }
1567
1568 Value *
1569 Converter::getVertexBase(int s)
1570 {
1571    assert(s < 5);
1572    if (!(vtxBaseValid & (1 << s))) {
1573       const int index = tgsi.getSrc(s).getIndex(1);
1574       Value *rel = NULL;
1575       if (tgsi.getSrc(s).isIndirect(1))
1576          rel = fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL);
1577       vtxBaseValid |= 1 << s;
1578       vtxBase[s] = mkOp2v(OP_PFETCH, TYPE_U32, getSSA(4, FILE_ADDRESS),
1579                           mkImm(index), rel);
1580    }
1581    return vtxBase[s];
1582 }
1583
1584 Value *
1585 Converter::getOutputBase(int s)
1586 {
1587    assert(s < 5);
1588    if (!(vtxBaseValid & (1 << s))) {
1589       Value *offset = loadImm(NULL, tgsi.getSrc(s).getIndex(1));
1590       if (tgsi.getSrc(s).isIndirect(1))
1591          offset = mkOp2v(OP_ADD, TYPE_U32, getSSA(),
1592                          fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL),
1593                          offset);
1594       vtxBaseValid |= 1 << s;
1595       vtxBase[s] = mkOp2v(OP_ADD, TYPE_U32, getSSA(), outBase, offset);
1596    }
1597    return vtxBase[s];
1598 }
1599
1600 Value *
1601 Converter::fetchSrc(int s, int c)
1602 {
1603    Value *res;
1604    Value *ptr = NULL, *dimRel = NULL;
1605
1606    tgsi::Instruction::SrcRegister src = tgsi.getSrc(s);
1607
1608    if (src.isIndirect(0))
1609       ptr = fetchSrc(src.getIndirect(0), 0, NULL);
1610
1611    if (src.is2D()) {
1612       switch (src.getFile()) {
1613       case TGSI_FILE_OUTPUT:
1614          dimRel = getOutputBase(s);
1615          break;
1616       case TGSI_FILE_INPUT:
1617          dimRel = getVertexBase(s);
1618          break;
1619       case TGSI_FILE_CONSTANT:
1620          // on NVC0, this is valid and c{I+J}[k] == cI[(J << 16) + k]
1621          if (src.isIndirect(1))
1622             dimRel = fetchSrc(src.getIndirect(1), 0, 0);
1623          break;
1624       default:
1625          break;
1626       }
1627    }
1628
1629    res = fetchSrc(src, c, ptr);
1630
1631    if (dimRel)
1632       res->getInsn()->setIndirect(0, 1, dimRel);
1633
1634    return applySrcMod(res, s, c);
1635 }
1636
1637 Converter::DataArray *
1638 Converter::getArrayForFile(unsigned file, int idx)
1639 {
1640    switch (file) {
1641    case TGSI_FILE_TEMPORARY:
1642       return idx == 0 ? &tData : &lData;
1643    case TGSI_FILE_PREDICATE:
1644       return &pData;
1645    case TGSI_FILE_ADDRESS:
1646       return &aData;
1647    case TGSI_FILE_OUTPUT:
1648       assert(prog->getType() == Program::TYPE_FRAGMENT);
1649       return &oData;
1650    default:
1651       assert(!"invalid/unhandled TGSI source file");
1652       return NULL;
1653    }
1654 }
1655
1656 Value *
1657 Converter::shiftAddress(Value *index)
1658 {
1659    if (!index)
1660       return NULL;
1661    return mkOp2v(OP_SHL, TYPE_U32, getSSA(4, FILE_ADDRESS), index, mkImm(4));
1662 }
1663
1664 Value *
1665 Converter::fetchSrc(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
1666 {
1667    int idx2d = src.is2D() ? src.getIndex(1) : 0;
1668    const int idx = src.getIndex(0);
1669    const int swz = src.getSwizzle(c);
1670    Instruction *ld;
1671
1672    switch (src.getFile()) {
1673    case TGSI_FILE_IMMEDIATE:
1674       assert(!ptr);
1675       return loadImm(NULL, info->immd.data[idx * 4 + swz]);
1676    case TGSI_FILE_CONSTANT:
1677       return mkLoadv(TYPE_U32, srcToSym(src, c), shiftAddress(ptr));
1678    case TGSI_FILE_INPUT:
1679       if (prog->getType() == Program::TYPE_FRAGMENT) {
1680          // don't load masked inputs, won't be assigned a slot
1681          if (!ptr && !(info->in[idx].mask & (1 << swz)))
1682             return loadImm(NULL, swz == TGSI_SWIZZLE_W ? 1.0f : 0.0f);
1683          return interpolate(src, c, shiftAddress(ptr));
1684       } else
1685       if (prog->getType() == Program::TYPE_GEOMETRY) {
1686          if (!ptr && info->in[idx].sn == TGSI_SEMANTIC_PRIMID)
1687             return mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_PRIMITIVE_ID, 0));
1688          // XXX: This is going to be a problem with scalar arrays, i.e. when
1689          // we cannot assume that the address is given in units of vec4.
1690          //
1691          // nv50 and nvc0 need different things here, so let the lowering
1692          // passes decide what to do with the address
1693          if (ptr)
1694             return mkLoadv(TYPE_U32, srcToSym(src, c), ptr);
1695       }
1696       ld = mkLoad(TYPE_U32, getSSA(), srcToSym(src, c), shiftAddress(ptr));
1697       ld->perPatch = info->in[idx].patch;
1698       return ld->getDef(0);
1699    case TGSI_FILE_OUTPUT:
1700       assert(prog->getType() == Program::TYPE_TESSELLATION_CONTROL);
1701       ld = mkLoad(TYPE_U32, getSSA(), srcToSym(src, c), shiftAddress(ptr));
1702       ld->perPatch = info->out[idx].patch;
1703       return ld->getDef(0);
1704    case TGSI_FILE_SYSTEM_VALUE:
1705       assert(!ptr);
1706       ld = mkOp1(OP_RDSV, TYPE_U32, getSSA(), srcToSym(src, c));
1707       ld->perPatch = info->sv[idx].patch;
1708       return ld->getDef(0);
1709    case TGSI_FILE_TEMPORARY: {
1710       int arrayid = src.getArrayId();
1711       if (!arrayid)
1712          arrayid = code->tempArrayId[idx];
1713       idx2d = (code->indirectTempArrays.find(arrayid) !=
1714                code->indirectTempArrays.end());
1715    }
1716       /* fallthrough */
1717    default:
1718       return getArrayForFile(src.getFile(), idx2d)->load(
1719          sub.cur->values, idx, swz, shiftAddress(ptr));
1720    }
1721 }
1722
1723 Value *
1724 Converter::acquireDst(int d, int c)
1725 {
1726    const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
1727    const unsigned f = dst.getFile();
1728    const int idx = dst.getIndex(0);
1729    int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
1730
1731    if (dst.isMasked(c)/* || f == TGSI_FILE_RESOURCE*/)
1732       return NULL;
1733
1734    if (dst.isIndirect(0) ||
1735        f == TGSI_FILE_SYSTEM_VALUE ||
1736        (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT))
1737       return getScratch();
1738
1739    if (f == TGSI_FILE_TEMPORARY)
1740       idx2d = code->indirectTempArrays.find(code->tempArrayId[idx]) !=
1741          code->indirectTempArrays.end();
1742
1743    return getArrayForFile(f, idx2d)-> acquire(sub.cur->values, idx, c);
1744 }
1745
1746 void
1747 Converter::storeDst(int d, int c, Value *val)
1748 {
1749    const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
1750
1751    if (tgsi.getSaturate()) {
1752       mkOp1(OP_SAT, dstTy, val, val);
1753    }
1754
1755    Value *ptr = NULL;
1756    if (dst.isIndirect(0))
1757       ptr = shiftAddress(fetchSrc(dst.getIndirect(0), 0, NULL));
1758
1759    if (info->io.genUserClip > 0 &&
1760        dst.getFile() == TGSI_FILE_OUTPUT &&
1761        !dst.isIndirect(0) && dst.getIndex(0) == code->clipVertexOutput) {
1762       mkMov(clipVtx[c], val);
1763       val = clipVtx[c];
1764    }
1765
1766    storeDst(dst, c, val, ptr);
1767 }
1768
1769 void
1770 Converter::storeDst(const tgsi::Instruction::DstRegister dst, int c,
1771                     Value *val, Value *ptr)
1772 {
1773    const unsigned f = dst.getFile();
1774    const int idx = dst.getIndex(0);
1775    int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
1776
1777    if (f == TGSI_FILE_SYSTEM_VALUE) {
1778       assert(!ptr);
1779       mkOp2(OP_WRSV, TYPE_U32, NULL, dstToSym(dst, c), val);
1780    } else
1781    if (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT) {
1782
1783       if (ptr || (info->out[idx].mask & (1 << c))) {
1784          /* Save the viewport index into a scratch register so that it can be
1785             exported at EMIT time */
1786          if (info->out[idx].sn == TGSI_SEMANTIC_VIEWPORT_INDEX &&
1787              viewport != NULL)
1788             mkOp1(OP_MOV, TYPE_U32, viewport, val);
1789          else
1790             mkStore(OP_EXPORT, TYPE_U32, dstToSym(dst, c), ptr, val)->perPatch =
1791                info->out[idx].patch;
1792       }
1793    } else
1794    if (f == TGSI_FILE_TEMPORARY ||
1795        f == TGSI_FILE_PREDICATE ||
1796        f == TGSI_FILE_ADDRESS ||
1797        f == TGSI_FILE_OUTPUT) {
1798       if (f == TGSI_FILE_TEMPORARY)
1799          idx2d = code->indirectTempArrays.find(code->tempArrayId[idx]) !=
1800             code->indirectTempArrays.end();
1801
1802       getArrayForFile(f, idx2d)->store(sub.cur->values, idx, c, ptr, val);
1803    } else {
1804       assert(!"invalid dst file");
1805    }
1806 }
1807
1808 #define FOR_EACH_DST_ENABLED_CHANNEL(d, chan, inst) \
1809    for (chan = 0; chan < 4; ++chan)                 \
1810       if (!inst.getDst(d).isMasked(chan))
1811
1812 Value *
1813 Converter::buildDot(int dim)
1814 {
1815    assert(dim > 0);
1816
1817    Value *src0 = fetchSrc(0, 0), *src1 = fetchSrc(1, 0);
1818    Value *dotp = getScratch();
1819
1820    mkOp2(OP_MUL, TYPE_F32, dotp, src0, src1);
1821
1822    for (int c = 1; c < dim; ++c) {
1823       src0 = fetchSrc(0, c);
1824       src1 = fetchSrc(1, c);
1825       mkOp3(OP_MAD, TYPE_F32, dotp, src0, src1, dotp);
1826    }
1827    return dotp;
1828 }
1829
1830 void
1831 Converter::insertConvergenceOps(BasicBlock *conv, BasicBlock *fork)
1832 {
1833    FlowInstruction *join = new_FlowInstruction(func, OP_JOIN, NULL);
1834    join->fixed = 1;
1835    conv->insertHead(join);
1836
1837    assert(!fork->joinAt);
1838    fork->joinAt = new_FlowInstruction(func, OP_JOINAT, conv);
1839    fork->insertBefore(fork->getExit(), fork->joinAt);
1840 }
1841
1842 void
1843 Converter::setTexRS(TexInstruction *tex, unsigned int& s, int R, int S)
1844 {
1845    unsigned rIdx = 0, sIdx = 0;
1846
1847    if (R >= 0)
1848       rIdx = tgsi.getSrc(R).getIndex(0);
1849    if (S >= 0)
1850       sIdx = tgsi.getSrc(S).getIndex(0);
1851
1852    tex->setTexture(tgsi.getTexture(code, R), rIdx, sIdx);
1853
1854    if (tgsi.getSrc(R).isIndirect(0)) {
1855       tex->tex.rIndirectSrc = s;
1856       tex->setSrc(s++, fetchSrc(tgsi.getSrc(R).getIndirect(0), 0, NULL));
1857    }
1858    if (S >= 0 && tgsi.getSrc(S).isIndirect(0)) {
1859       tex->tex.sIndirectSrc = s;
1860       tex->setSrc(s++, fetchSrc(tgsi.getSrc(S).getIndirect(0), 0, NULL));
1861    }
1862 }
1863
1864 void
1865 Converter::handleTXQ(Value *dst0[4], enum TexQuery query, int R)
1866 {
1867    TexInstruction *tex = new_TexInstruction(func, OP_TXQ);
1868    tex->tex.query = query;
1869    unsigned int c, d;
1870
1871    for (d = 0, c = 0; c < 4; ++c) {
1872       if (!dst0[c])
1873          continue;
1874       tex->tex.mask |= 1 << c;
1875       tex->setDef(d++, dst0[c]);
1876    }
1877    if (query == TXQ_DIMS)
1878       tex->setSrc((c = 0), fetchSrc(0, 0)); // mip level
1879    else
1880       tex->setSrc((c = 0), zero);
1881
1882    setTexRS(tex, ++c, R, -1);
1883
1884    bb->insertTail(tex);
1885 }
1886
1887 void
1888 Converter::loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask)
1889 {
1890    Value *proj = fetchSrc(0, 3);
1891    Instruction *insn = proj->getUniqueInsn();
1892    int c;
1893
1894    if (insn->op == OP_PINTERP) {
1895       bb->insertTail(insn = cloneForward(func, insn));
1896       insn->op = OP_LINTERP;
1897       insn->setInterpolate(NV50_IR_INTERP_LINEAR | insn->getSampleMode());
1898       insn->setSrc(1, NULL);
1899       proj = insn->getDef(0);
1900    }
1901    proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), proj);
1902
1903    for (c = 0; c < 4; ++c) {
1904       if (!(mask & (1 << c)))
1905          continue;
1906       if ((insn = src[c]->getUniqueInsn())->op != OP_PINTERP)
1907          continue;
1908       mask &= ~(1 << c);
1909
1910       bb->insertTail(insn = cloneForward(func, insn));
1911       insn->setInterpolate(NV50_IR_INTERP_PERSPECTIVE | insn->getSampleMode());
1912       insn->setSrc(1, proj);
1913       dst[c] = insn->getDef(0);
1914    }
1915    if (!mask)
1916       return;
1917
1918    proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), fetchSrc(0, 3));
1919
1920    for (c = 0; c < 4; ++c)
1921       if (mask & (1 << c))
1922          dst[c] = mkOp2v(OP_MUL, TYPE_F32, getSSA(), src[c], proj);
1923 }
1924
1925 // order of nv50 ir sources: x y z layer lod/bias shadow
1926 // order of TGSI TEX sources: x y z layer shadow lod/bias
1927 //  lowering will finally set the hw specific order (like array first on nvc0)
1928 void
1929 Converter::handleTEX(Value *dst[4], int R, int S, int L, int C, int Dx, int Dy)
1930 {
1931    Value *val;
1932    Value *arg[4], *src[8];
1933    Value *lod = NULL, *shd = NULL;
1934    unsigned int s, c, d;
1935    TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
1936
1937    TexInstruction::Target tgt = tgsi.getTexture(code, R);
1938
1939    for (s = 0; s < tgt.getArgCount(); ++s)
1940       arg[s] = src[s] = fetchSrc(0, s);
1941
1942    if (texi->op == OP_TXL || texi->op == OP_TXB)
1943       lod = fetchSrc(L >> 4, L & 3);
1944
1945    if (C == 0x0f)
1946       C = 0x00 | MAX2(tgt.getArgCount(), 2); // guess DC src
1947
1948    if (tgsi.getOpcode() == TGSI_OPCODE_TG4 &&
1949        tgt == TEX_TARGET_CUBE_ARRAY_SHADOW)
1950       shd = fetchSrc(1, 0);
1951    else if (tgt.isShadow())
1952       shd = fetchSrc(C >> 4, C & 3);
1953
1954    if (texi->op == OP_TXD) {
1955       for (c = 0; c < tgt.getDim() + tgt.isCube(); ++c) {
1956          texi->dPdx[c].set(fetchSrc(Dx >> 4, (Dx & 3) + c));
1957          texi->dPdy[c].set(fetchSrc(Dy >> 4, (Dy & 3) + c));
1958       }
1959    }
1960
1961    // cube textures don't care about projection value, it's divided out
1962    if (tgsi.getOpcode() == TGSI_OPCODE_TXP && !tgt.isCube() && !tgt.isArray()) {
1963       unsigned int n = tgt.getDim();
1964       if (shd) {
1965          arg[n] = shd;
1966          ++n;
1967          assert(tgt.getDim() == tgt.getArgCount());
1968       }
1969       loadProjTexCoords(src, arg, (1 << n) - 1);
1970       if (shd)
1971          shd = src[n - 1];
1972    }
1973
1974    if (tgt.isCube()) {
1975       for (c = 0; c < 3; ++c)
1976          src[c] = mkOp1v(OP_ABS, TYPE_F32, getSSA(), arg[c]);
1977       val = getScratch();
1978       mkOp2(OP_MAX, TYPE_F32, val, src[0], src[1]);
1979       mkOp2(OP_MAX, TYPE_F32, val, src[2], val);
1980       mkOp1(OP_RCP, TYPE_F32, val, val);
1981       for (c = 0; c < 3; ++c)
1982          src[c] = mkOp2v(OP_MUL, TYPE_F32, getSSA(), arg[c], val);
1983    }
1984
1985    for (c = 0, d = 0; c < 4; ++c) {
1986       if (dst[c]) {
1987          texi->setDef(d++, dst[c]);
1988          texi->tex.mask |= 1 << c;
1989       } else {
1990          // NOTE: maybe hook up def too, for CSE
1991       }
1992    }
1993    for (s = 0; s < tgt.getArgCount(); ++s)
1994       texi->setSrc(s, src[s]);
1995    if (lod)
1996       texi->setSrc(s++, lod);
1997    if (shd)
1998       texi->setSrc(s++, shd);
1999
2000    setTexRS(texi, s, R, S);
2001
2002    if (tgsi.getOpcode() == TGSI_OPCODE_SAMPLE_C_LZ)
2003       texi->tex.levelZero = true;
2004    if (tgsi.getOpcode() == TGSI_OPCODE_TG4 && !tgt.isShadow())
2005       texi->tex.gatherComp = tgsi.getSrc(1).getValueU32(0, info);
2006
2007    texi->tex.useOffsets = tgsi.getNumTexOffsets();
2008    for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
2009       for (c = 0; c < 3; ++c) {
2010          texi->offset[s][c].set(fetchSrc(tgsi.getTexOffset(s), c, NULL));
2011          texi->offset[s][c].setInsn(texi);
2012       }
2013    }
2014
2015    bb->insertTail(texi);
2016 }
2017
2018 // 1st source: xyz = coordinates, w = lod/sample
2019 // 2nd source: offset
2020 void
2021 Converter::handleTXF(Value *dst[4], int R, int L_M)
2022 {
2023    TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
2024    int ms;
2025    unsigned int c, d, s;
2026
2027    texi->tex.target = tgsi.getTexture(code, R);
2028
2029    ms = texi->tex.target.isMS() ? 1 : 0;
2030    texi->tex.levelZero = ms; /* MS textures don't have mip-maps */
2031
2032    for (c = 0, d = 0; c < 4; ++c) {
2033       if (dst[c]) {
2034          texi->setDef(d++, dst[c]);
2035          texi->tex.mask |= 1 << c;
2036       }
2037    }
2038    for (c = 0; c < (texi->tex.target.getArgCount() - ms); ++c)
2039       texi->setSrc(c, fetchSrc(0, c));
2040    texi->setSrc(c++, fetchSrc(L_M >> 4, L_M & 3)); // lod or ms
2041
2042    setTexRS(texi, c, R, -1);
2043
2044    texi->tex.useOffsets = tgsi.getNumTexOffsets();
2045    for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
2046       for (c = 0; c < 3; ++c) {
2047          texi->offset[s][c].set(fetchSrc(tgsi.getTexOffset(s), c, NULL));
2048          texi->offset[s][c].setInsn(texi);
2049       }
2050    }
2051
2052    bb->insertTail(texi);
2053 }
2054
2055 void
2056 Converter::handleLIT(Value *dst0[4])
2057 {
2058    Value *val0 = NULL;
2059    unsigned int mask = tgsi.getDst(0).getMask();
2060
2061    if (mask & (1 << 0))
2062       loadImm(dst0[0], 1.0f);
2063
2064    if (mask & (1 << 3))
2065       loadImm(dst0[3], 1.0f);
2066
2067    if (mask & (3 << 1)) {
2068       val0 = getScratch();
2069       mkOp2(OP_MAX, TYPE_F32, val0, fetchSrc(0, 0), zero);
2070       if (mask & (1 << 1))
2071          mkMov(dst0[1], val0);
2072    }
2073
2074    if (mask & (1 << 2)) {
2075       Value *src1 = fetchSrc(0, 1), *src3 = fetchSrc(0, 3);
2076       Value *val1 = getScratch(), *val3 = getScratch();
2077
2078       Value *pos128 = loadImm(NULL, +127.999999f);
2079       Value *neg128 = loadImm(NULL, -127.999999f);
2080
2081       mkOp2(OP_MAX, TYPE_F32, val1, src1, zero);
2082       mkOp2(OP_MAX, TYPE_F32, val3, src3, neg128);
2083       mkOp2(OP_MIN, TYPE_F32, val3, val3, pos128);
2084       mkOp2(OP_POW, TYPE_F32, val3, val1, val3);
2085
2086       mkCmp(OP_SLCT, CC_GT, TYPE_F32, dst0[2], TYPE_F32, val3, zero, val0);
2087    }
2088 }
2089
2090 static inline bool
2091 isResourceSpecial(const int r)
2092 {
2093    return (r == TGSI_RESOURCE_GLOBAL ||
2094            r == TGSI_RESOURCE_LOCAL ||
2095            r == TGSI_RESOURCE_PRIVATE ||
2096            r == TGSI_RESOURCE_INPUT);
2097 }
2098
2099 static inline bool
2100 isResourceRaw(const tgsi::Source *code, const int r)
2101 {
2102    return isResourceSpecial(r) || code->resources[r].raw;
2103 }
2104
2105 static inline nv50_ir::TexTarget
2106 getResourceTarget(const tgsi::Source *code, int r)
2107 {
2108    if (isResourceSpecial(r))
2109       return nv50_ir::TEX_TARGET_BUFFER;
2110    return tgsi::translateTexture(code->resources.at(r).target);
2111 }
2112
2113 Symbol *
2114 Converter::getResourceBase(const int r)
2115 {
2116    Symbol *sym = NULL;
2117
2118    switch (r) {
2119    case TGSI_RESOURCE_GLOBAL:
2120       sym = new_Symbol(prog, nv50_ir::FILE_MEMORY_GLOBAL, 15);
2121       break;
2122    case TGSI_RESOURCE_LOCAL:
2123       assert(prog->getType() == Program::TYPE_COMPUTE);
2124       sym = mkSymbol(nv50_ir::FILE_MEMORY_SHARED, 0, TYPE_U32,
2125                      info->prop.cp.sharedOffset);
2126       break;
2127    case TGSI_RESOURCE_PRIVATE:
2128       sym = mkSymbol(nv50_ir::FILE_MEMORY_LOCAL, 0, TYPE_U32,
2129                      info->bin.tlsSpace);
2130       break;
2131    case TGSI_RESOURCE_INPUT:
2132       assert(prog->getType() == Program::TYPE_COMPUTE);
2133       sym = mkSymbol(nv50_ir::FILE_SHADER_INPUT, 0, TYPE_U32,
2134                      info->prop.cp.inputOffset);
2135       break;
2136    default:
2137       sym = new_Symbol(prog,
2138                        nv50_ir::FILE_MEMORY_GLOBAL, code->resources.at(r).slot);
2139       break;
2140    }
2141    return sym;
2142 }
2143
2144 void
2145 Converter::getResourceCoords(std::vector<Value *> &coords, int r, int s)
2146 {
2147    const int arg =
2148       TexInstruction::Target(getResourceTarget(code, r)).getArgCount();
2149
2150    for (int c = 0; c < arg; ++c)
2151       coords.push_back(fetchSrc(s, c));
2152
2153    // NOTE: TGSI_RESOURCE_GLOBAL needs FILE_GPR; this is an nv50 quirk
2154    if (r == TGSI_RESOURCE_LOCAL ||
2155        r == TGSI_RESOURCE_PRIVATE ||
2156        r == TGSI_RESOURCE_INPUT)
2157       coords[0] = mkOp1v(OP_MOV, TYPE_U32, getScratch(4, FILE_ADDRESS),
2158                          coords[0]);
2159 }
2160
2161 static inline int
2162 partitionLoadStore(uint8_t comp[2], uint8_t size[2], uint8_t mask)
2163 {
2164    int n = 0;
2165
2166    while (mask) {
2167       if (mask & 1) {
2168          size[n]++;
2169       } else {
2170          if (size[n])
2171             comp[n = 1] = size[0] + 1;
2172          else
2173             comp[n]++;
2174       }
2175       mask >>= 1;
2176    }
2177    if (size[0] == 3) {
2178       n = 1;
2179       size[0] = (comp[0] == 1) ? 1 : 2;
2180       size[1] = 3 - size[0];
2181       comp[1] = comp[0] + size[0];
2182    }
2183    return n + 1;
2184 }
2185
2186 // For raw loads, granularity is 4 byte.
2187 // Usage of the texture read mask on OP_SULDP is not allowed.
2188 void
2189 Converter::handleLOAD(Value *dst0[4])
2190 {
2191    const int r = tgsi.getSrc(0).getIndex(0);
2192    int c;
2193    std::vector<Value *> off, src, ldv, def;
2194
2195    getResourceCoords(off, r, 1);
2196
2197    if (isResourceRaw(code, r)) {
2198       uint8_t mask = 0;
2199       uint8_t comp[2] = { 0, 0 };
2200       uint8_t size[2] = { 0, 0 };
2201
2202       Symbol *base = getResourceBase(r);
2203
2204       // determine the base and size of the at most 2 load ops
2205       for (c = 0; c < 4; ++c)
2206          if (!tgsi.getDst(0).isMasked(c))
2207             mask |= 1 << (tgsi.getSrc(0).getSwizzle(c) - TGSI_SWIZZLE_X);
2208
2209       int n = partitionLoadStore(comp, size, mask);
2210
2211       src = off;
2212
2213       def.resize(4); // index by component, the ones we need will be non-NULL
2214       for (c = 0; c < 4; ++c) {
2215          if (dst0[c] && tgsi.getSrc(0).getSwizzle(c) == (TGSI_SWIZZLE_X + c))
2216             def[c] = dst0[c];
2217          else
2218          if (mask & (1 << c))
2219             def[c] = getScratch();
2220       }
2221
2222       const bool useLd = isResourceSpecial(r) ||
2223          (info->io.nv50styleSurfaces &&
2224           code->resources[r].target == TGSI_TEXTURE_BUFFER);
2225
2226       for (int i = 0; i < n; ++i) {
2227          ldv.assign(def.begin() + comp[i], def.begin() + comp[i] + size[i]);
2228
2229          if (comp[i]) // adjust x component of source address if necessary
2230             src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
2231                             off[0], mkImm(comp[i] * 4));
2232          else
2233             src[0] = off[0];
2234
2235          if (useLd) {
2236             Instruction *ld =
2237                mkLoad(typeOfSize(size[i] * 4), ldv[0], base, src[0]);
2238             for (size_t c = 1; c < ldv.size(); ++c)
2239                ld->setDef(c, ldv[c]);
2240          } else {
2241             mkTex(OP_SULDB, getResourceTarget(code, r), code->resources[r].slot,
2242                   0, ldv, src)->dType = typeOfSize(size[i] * 4);
2243          }
2244       }
2245    } else {
2246       def.resize(4);
2247       for (c = 0; c < 4; ++c) {
2248          if (!dst0[c] || tgsi.getSrc(0).getSwizzle(c) != (TGSI_SWIZZLE_X + c))
2249             def[c] = getScratch();
2250          else
2251             def[c] = dst0[c];
2252       }
2253
2254       mkTex(OP_SULDP, getResourceTarget(code, r), code->resources[r].slot, 0,
2255             def, off);
2256    }
2257    FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2258       if (dst0[c] != def[c])
2259          mkMov(dst0[c], def[tgsi.getSrc(0).getSwizzle(c)]);
2260 }
2261
2262 // For formatted stores, the write mask on OP_SUSTP can be used.
2263 // Raw stores have to be split.
2264 void
2265 Converter::handleSTORE()
2266 {
2267    const int r = tgsi.getDst(0).getIndex(0);
2268    int c;
2269    std::vector<Value *> off, src, dummy;
2270
2271    getResourceCoords(off, r, 0);
2272    src = off;
2273    const int s = src.size();
2274
2275    if (isResourceRaw(code, r)) {
2276       uint8_t comp[2] = { 0, 0 };
2277       uint8_t size[2] = { 0, 0 };
2278
2279       int n = partitionLoadStore(comp, size, tgsi.getDst(0).getMask());
2280
2281       Symbol *base = getResourceBase(r);
2282
2283       const bool useSt = isResourceSpecial(r) ||
2284          (info->io.nv50styleSurfaces &&
2285           code->resources[r].target == TGSI_TEXTURE_BUFFER);
2286
2287       for (int i = 0; i < n; ++i) {
2288          if (comp[i]) // adjust x component of source address if necessary
2289             src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
2290                             off[0], mkImm(comp[i] * 4));
2291          else
2292             src[0] = off[0];
2293
2294          const DataType stTy = typeOfSize(size[i] * 4);
2295
2296          if (useSt) {
2297             Instruction *st =
2298                mkStore(OP_STORE, stTy, base, NULL, fetchSrc(1, comp[i]));
2299             for (c = 1; c < size[i]; ++c)
2300                st->setSrc(1 + c, fetchSrc(1, comp[i] + c));
2301             st->setIndirect(0, 0, src[0]);
2302          } else {
2303             // attach values to be stored
2304             src.resize(s + size[i]);
2305             for (c = 0; c < size[i]; ++c)
2306                src[s + c] = fetchSrc(1, comp[i] + c);
2307             mkTex(OP_SUSTB, getResourceTarget(code, r), code->resources[r].slot,
2308                   0, dummy, src)->setType(stTy);
2309          }
2310       }
2311    } else {
2312       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2313          src.push_back(fetchSrc(1, c));
2314
2315       mkTex(OP_SUSTP, getResourceTarget(code, r), code->resources[r].slot, 0,
2316             dummy, src)->tex.mask = tgsi.getDst(0).getMask();
2317    }
2318 }
2319
2320 // XXX: These only work on resources with the single-component u32/s32 formats.
2321 // Therefore the result is replicated. This might not be intended by TGSI, but
2322 // operating on more than 1 component would produce undefined results because
2323 // they do not exist.
2324 void
2325 Converter::handleATOM(Value *dst0[4], DataType ty, uint16_t subOp)
2326 {
2327    const int r = tgsi.getSrc(0).getIndex(0);
2328    std::vector<Value *> srcv;
2329    std::vector<Value *> defv;
2330    LValue *dst = getScratch();
2331
2332    getResourceCoords(srcv, r, 1);
2333
2334    if (isResourceSpecial(r)) {
2335       assert(r != TGSI_RESOURCE_INPUT);
2336       Instruction *insn;
2337       insn = mkOp2(OP_ATOM, ty, dst, getResourceBase(r), fetchSrc(2, 0));
2338       insn->subOp = subOp;
2339       if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2340          insn->setSrc(2, fetchSrc(3, 0));
2341       insn->setIndirect(0, 0, srcv.at(0));
2342    } else {
2343       operation op = isResourceRaw(code, r) ? OP_SUREDB : OP_SUREDP;
2344       TexTarget targ = getResourceTarget(code, r);
2345       int idx = code->resources[r].slot;
2346       defv.push_back(dst);
2347       srcv.push_back(fetchSrc(2, 0));
2348       if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2349          srcv.push_back(fetchSrc(3, 0));
2350       TexInstruction *tex = mkTex(op, targ, idx, 0, defv, srcv);
2351       tex->subOp = subOp;
2352       tex->tex.mask = 1;
2353       tex->setType(ty);
2354    }
2355
2356    for (int c = 0; c < 4; ++c)
2357       if (dst0[c])
2358          dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov
2359 }
2360
2361 void
2362 Converter::handleINTERP(Value *dst[4])
2363 {
2364    // Check whether the input is linear. All other attributes ignored.
2365    Instruction *insn;
2366    Value *offset = NULL, *ptr = NULL, *w = NULL;
2367    bool linear;
2368    operation op;
2369    int c, mode;
2370
2371    tgsi::Instruction::SrcRegister src = tgsi.getSrc(0);
2372    assert(src.getFile() == TGSI_FILE_INPUT);
2373
2374    if (src.isIndirect(0))
2375       ptr = fetchSrc(src.getIndirect(0), 0, NULL);
2376
2377    // XXX: no way to know interp mode if we don't know the index
2378    linear = info->in[ptr ? 0 : src.getIndex(0)].linear;
2379    if (linear) {
2380       op = OP_LINTERP;
2381       mode = NV50_IR_INTERP_LINEAR;
2382    } else {
2383       op = OP_PINTERP;
2384       mode = NV50_IR_INTERP_PERSPECTIVE;
2385    }
2386
2387    switch (tgsi.getOpcode()) {
2388    case TGSI_OPCODE_INTERP_CENTROID:
2389       mode |= NV50_IR_INTERP_CENTROID;
2390       break;
2391    case TGSI_OPCODE_INTERP_SAMPLE:
2392       insn = mkOp1(OP_PIXLD, TYPE_U32, (offset = getScratch()), fetchSrc(1, 0));
2393       insn->subOp = NV50_IR_SUBOP_PIXLD_OFFSET;
2394       mode |= NV50_IR_INTERP_OFFSET;
2395       break;
2396    case TGSI_OPCODE_INTERP_OFFSET: {
2397       // The input in src1.xy is float, but we need a single 32-bit value
2398       // where the upper and lower 16 bits are encoded in S0.12 format. We need
2399       // to clamp the input coordinates to (-0.5, 0.4375), multiply by 4096,
2400       // and then convert to s32.
2401       Value *offs[2];
2402       for (c = 0; c < 2; c++) {
2403          offs[c] = fetchSrc(1, c);
2404          mkOp2(OP_MIN, TYPE_F32, offs[c], offs[c], loadImm(NULL, 0.4375f));
2405          mkOp2(OP_MAX, TYPE_F32, offs[c], offs[c], loadImm(NULL, -0.5f));
2406          mkOp2(OP_MUL, TYPE_F32, offs[c], offs[c], loadImm(NULL, 4096.0f));
2407          mkCvt(OP_CVT, TYPE_S32, offs[c], TYPE_F32, offs[c]);
2408       }
2409       offset = mkOp3v(OP_INSBF, TYPE_U32, getScratch(),
2410                       offs[1], mkImm(0x1010), offs[0]);
2411       mode |= NV50_IR_INTERP_OFFSET;
2412       break;
2413    }
2414    }
2415
2416    if (op == OP_PINTERP) {
2417       if (offset) {
2418          w = mkOp2v(OP_RDSV, TYPE_F32, getSSA(), mkSysVal(SV_POSITION, 3), offset);
2419          mkOp1(OP_RCP, TYPE_F32, w, w);
2420       } else {
2421          w = fragCoord[3];
2422       }
2423    }
2424
2425
2426    FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2427       insn = mkOp1(op, TYPE_F32, dst[c], srcToSym(src, c));
2428       if (op == OP_PINTERP)
2429          insn->setSrc(1, w);
2430       if (ptr)
2431          insn->setIndirect(0, 0, ptr);
2432       if (offset)
2433          insn->setSrc(op == OP_PINTERP ? 2 : 1, offset);
2434
2435       insn->setInterpolate(mode);
2436    }
2437 }
2438
2439 Converter::Subroutine *
2440 Converter::getSubroutine(unsigned ip)
2441 {
2442    std::map<unsigned, Subroutine>::iterator it = sub.map.find(ip);
2443
2444    if (it == sub.map.end())
2445       it = sub.map.insert(std::make_pair(
2446               ip, Subroutine(new Function(prog, "SUB", ip)))).first;
2447
2448    return &it->second;
2449 }
2450
2451 Converter::Subroutine *
2452 Converter::getSubroutine(Function *f)
2453 {
2454    unsigned ip = f->getLabel();
2455    std::map<unsigned, Subroutine>::iterator it = sub.map.find(ip);
2456
2457    if (it == sub.map.end())
2458       it = sub.map.insert(std::make_pair(ip, Subroutine(f))).first;
2459
2460    return &it->second;
2461 }
2462
2463 bool
2464 Converter::isEndOfSubroutine(uint ip)
2465 {
2466    assert(ip < code->scan.num_instructions);
2467    tgsi::Instruction insn(&code->insns[ip]);
2468    return (insn.getOpcode() == TGSI_OPCODE_END ||
2469            insn.getOpcode() == TGSI_OPCODE_ENDSUB ||
2470            // does END occur at end of main or the very end ?
2471            insn.getOpcode() == TGSI_OPCODE_BGNSUB);
2472 }
2473
2474 bool
2475 Converter::handleInstruction(const struct tgsi_full_instruction *insn)
2476 {
2477    Instruction *geni;
2478
2479    Value *dst0[4], *rDst0[4];
2480    Value *src0, *src1, *src2, *src3;
2481    Value *val0, *val1;
2482    int c;
2483
2484    tgsi = tgsi::Instruction(insn);
2485
2486    bool useScratchDst = tgsi.checkDstSrcAliasing();
2487
2488    operation op = tgsi.getOP();
2489    dstTy = tgsi.inferDstType();
2490    srcTy = tgsi.inferSrcType();
2491
2492    unsigned int mask = tgsi.dstCount() ? tgsi.getDst(0).getMask() : 0;
2493
2494    if (tgsi.dstCount()) {
2495       for (c = 0; c < 4; ++c) {
2496          rDst0[c] = acquireDst(0, c);
2497          dst0[c] = (useScratchDst && rDst0[c]) ? getScratch() : rDst0[c];
2498       }
2499    }
2500
2501    switch (tgsi.getOpcode()) {
2502    case TGSI_OPCODE_ADD:
2503    case TGSI_OPCODE_UADD:
2504    case TGSI_OPCODE_AND:
2505    case TGSI_OPCODE_DIV:
2506    case TGSI_OPCODE_IDIV:
2507    case TGSI_OPCODE_UDIV:
2508    case TGSI_OPCODE_MAX:
2509    case TGSI_OPCODE_MIN:
2510    case TGSI_OPCODE_IMAX:
2511    case TGSI_OPCODE_IMIN:
2512    case TGSI_OPCODE_UMAX:
2513    case TGSI_OPCODE_UMIN:
2514    case TGSI_OPCODE_MOD:
2515    case TGSI_OPCODE_UMOD:
2516    case TGSI_OPCODE_MUL:
2517    case TGSI_OPCODE_UMUL:
2518    case TGSI_OPCODE_IMUL_HI:
2519    case TGSI_OPCODE_UMUL_HI:
2520    case TGSI_OPCODE_OR:
2521    case TGSI_OPCODE_SHL:
2522    case TGSI_OPCODE_ISHR:
2523    case TGSI_OPCODE_USHR:
2524    case TGSI_OPCODE_SUB:
2525    case TGSI_OPCODE_XOR:
2526       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2527          src0 = fetchSrc(0, c);
2528          src1 = fetchSrc(1, c);
2529          geni = mkOp2(op, dstTy, dst0[c], src0, src1);
2530          geni->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode());
2531       }
2532       break;
2533    case TGSI_OPCODE_MAD:
2534    case TGSI_OPCODE_UMAD:
2535    case TGSI_OPCODE_SAD:
2536       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2537          src0 = fetchSrc(0, c);
2538          src1 = fetchSrc(1, c);
2539          src2 = fetchSrc(2, c);
2540          mkOp3(op, dstTy, dst0[c], src0, src1, src2);
2541       }
2542       break;
2543    case TGSI_OPCODE_MOV:
2544    case TGSI_OPCODE_ABS:
2545    case TGSI_OPCODE_CEIL:
2546    case TGSI_OPCODE_FLR:
2547    case TGSI_OPCODE_TRUNC:
2548    case TGSI_OPCODE_RCP:
2549    case TGSI_OPCODE_IABS:
2550    case TGSI_OPCODE_INEG:
2551    case TGSI_OPCODE_NOT:
2552    case TGSI_OPCODE_DDX:
2553    case TGSI_OPCODE_DDY:
2554    case TGSI_OPCODE_DDX_FINE:
2555    case TGSI_OPCODE_DDY_FINE:
2556       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2557          mkOp1(op, dstTy, dst0[c], fetchSrc(0, c));
2558       break;
2559    case TGSI_OPCODE_RSQ:
2560       src0 = fetchSrc(0, 0);
2561       val0 = getScratch();
2562       mkOp1(OP_ABS, TYPE_F32, val0, src0);
2563       mkOp1(OP_RSQ, TYPE_F32, val0, val0);
2564       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2565          mkMov(dst0[c], val0);
2566       break;
2567    case TGSI_OPCODE_ARL:
2568    case TGSI_OPCODE_ARR:
2569       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2570          const RoundMode rnd =
2571             tgsi.getOpcode() == TGSI_OPCODE_ARR ? ROUND_N : ROUND_M;
2572          src0 = fetchSrc(0, c);
2573          mkCvt(OP_CVT, TYPE_S32, dst0[c], TYPE_F32, src0)->rnd = rnd;
2574       }
2575       break;
2576    case TGSI_OPCODE_UARL:
2577       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2578          mkOp1(OP_MOV, TYPE_U32, dst0[c], fetchSrc(0, c));
2579       break;
2580    case TGSI_OPCODE_POW:
2581       val0 = mkOp2v(op, TYPE_F32, getScratch(), fetchSrc(0, 0), fetchSrc(1, 0));
2582       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2583          mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
2584       break;
2585    case TGSI_OPCODE_EX2:
2586    case TGSI_OPCODE_LG2:
2587       val0 = mkOp1(op, TYPE_F32, getScratch(), fetchSrc(0, 0))->getDef(0);
2588       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2589          mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
2590       break;
2591    case TGSI_OPCODE_COS:
2592    case TGSI_OPCODE_SIN:
2593       val0 = getScratch();
2594       if (mask & 7) {
2595          mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 0));
2596          mkOp1(op, TYPE_F32, val0, val0);
2597          for (c = 0; c < 3; ++c)
2598             if (dst0[c])
2599                mkMov(dst0[c], val0);
2600       }
2601       if (dst0[3]) {
2602          mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 3));
2603          mkOp1(op, TYPE_F32, dst0[3], val0);
2604       }
2605       break;
2606    case TGSI_OPCODE_SCS:
2607       if (mask & 3) {
2608          val0 = mkOp1v(OP_PRESIN, TYPE_F32, getSSA(), fetchSrc(0, 0));
2609          if (dst0[0])
2610             mkOp1(OP_COS, TYPE_F32, dst0[0], val0);
2611          if (dst0[1])
2612             mkOp1(OP_SIN, TYPE_F32, dst0[1], val0);
2613       }
2614       if (dst0[2])
2615          loadImm(dst0[2], 0.0f);
2616       if (dst0[3])
2617          loadImm(dst0[3], 1.0f);
2618       break;
2619    case TGSI_OPCODE_EXP:
2620       src0 = fetchSrc(0, 0);
2621       val0 = mkOp1v(OP_FLOOR, TYPE_F32, getSSA(), src0);
2622       if (dst0[1])
2623          mkOp2(OP_SUB, TYPE_F32, dst0[1], src0, val0);
2624       if (dst0[0])
2625          mkOp1(OP_EX2, TYPE_F32, dst0[0], val0);
2626       if (dst0[2])
2627          mkOp1(OP_EX2, TYPE_F32, dst0[2], src0);
2628       if (dst0[3])
2629          loadImm(dst0[3], 1.0f);
2630       break;
2631    case TGSI_OPCODE_LOG:
2632       src0 = mkOp1v(OP_ABS, TYPE_F32, getSSA(), fetchSrc(0, 0));
2633       val0 = mkOp1v(OP_LG2, TYPE_F32, dst0[2] ? dst0[2] : getSSA(), src0);
2634       if (dst0[0] || dst0[1])
2635          val1 = mkOp1v(OP_FLOOR, TYPE_F32, dst0[0] ? dst0[0] : getSSA(), val0);
2636       if (dst0[1]) {
2637          mkOp1(OP_EX2, TYPE_F32, dst0[1], val1);
2638          mkOp1(OP_RCP, TYPE_F32, dst0[1], dst0[1]);
2639          mkOp2(OP_MUL, TYPE_F32, dst0[1], dst0[1], src0);
2640       }
2641       if (dst0[3])
2642          loadImm(dst0[3], 1.0f);
2643       break;
2644    case TGSI_OPCODE_DP2:
2645       val0 = buildDot(2);
2646       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2647          mkMov(dst0[c], val0);
2648       break;
2649    case TGSI_OPCODE_DP3:
2650       val0 = buildDot(3);
2651       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2652          mkMov(dst0[c], val0);
2653       break;
2654    case TGSI_OPCODE_DP4:
2655       val0 = buildDot(4);
2656       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2657          mkMov(dst0[c], val0);
2658       break;
2659    case TGSI_OPCODE_DPH:
2660       val0 = buildDot(3);
2661       src1 = fetchSrc(1, 3);
2662       mkOp2(OP_ADD, TYPE_F32, val0, val0, src1);
2663       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2664          mkMov(dst0[c], val0);
2665       break;
2666    case TGSI_OPCODE_DST:
2667       if (dst0[0])
2668          loadImm(dst0[0], 1.0f);
2669       if (dst0[1]) {
2670          src0 = fetchSrc(0, 1);
2671          src1 = fetchSrc(1, 1);
2672          mkOp2(OP_MUL, TYPE_F32, dst0[1], src0, src1);
2673       }
2674       if (dst0[2])
2675          mkMov(dst0[2], fetchSrc(0, 2));
2676       if (dst0[3])
2677          mkMov(dst0[3], fetchSrc(1, 3));
2678       break;
2679    case TGSI_OPCODE_LRP:
2680       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2681          src0 = fetchSrc(0, c);
2682          src1 = fetchSrc(1, c);
2683          src2 = fetchSrc(2, c);
2684          mkOp3(OP_MAD, TYPE_F32, dst0[c],
2685                mkOp2v(OP_SUB, TYPE_F32, getSSA(), src1, src2), src0, src2);
2686       }
2687       break;
2688    case TGSI_OPCODE_LIT:
2689       handleLIT(dst0);
2690       break;
2691    case TGSI_OPCODE_XPD:
2692       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2693          if (c < 3) {
2694             val0 = getSSA();
2695             src0 = fetchSrc(1, (c + 1) % 3);
2696             src1 = fetchSrc(0, (c + 2) % 3);
2697             mkOp2(OP_MUL, TYPE_F32, val0, src0, src1);
2698             mkOp1(OP_NEG, TYPE_F32, val0, val0);
2699
2700             src0 = fetchSrc(0, (c + 1) % 3);
2701             src1 = fetchSrc(1, (c + 2) % 3);
2702             mkOp3(OP_MAD, TYPE_F32, dst0[c], src0, src1, val0);
2703          } else {
2704             loadImm(dst0[c], 1.0f);
2705          }
2706       }
2707       break;
2708    case TGSI_OPCODE_ISSG:
2709    case TGSI_OPCODE_SSG:
2710       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2711          src0 = fetchSrc(0, c);
2712          val0 = getScratch();
2713          val1 = getScratch();
2714          mkCmp(OP_SET, CC_GT, srcTy, val0, srcTy, src0, zero);
2715          mkCmp(OP_SET, CC_LT, srcTy, val1, srcTy, src0, zero);
2716          if (srcTy == TYPE_F32)
2717             mkOp2(OP_SUB, TYPE_F32, dst0[c], val0, val1);
2718          else
2719             mkOp2(OP_SUB, TYPE_S32, dst0[c], val1, val0);
2720       }
2721       break;
2722    case TGSI_OPCODE_UCMP:
2723       srcTy = TYPE_U32;
2724       /* fallthrough */
2725    case TGSI_OPCODE_CMP:
2726       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2727          src0 = fetchSrc(0, c);
2728          src1 = fetchSrc(1, c);
2729          src2 = fetchSrc(2, c);
2730          if (src1 == src2)
2731             mkMov(dst0[c], src1);
2732          else
2733             mkCmp(OP_SLCT, (srcTy == TYPE_F32) ? CC_LT : CC_NE,
2734                   srcTy, dst0[c], srcTy, src1, src2, src0);
2735       }
2736       break;
2737    case TGSI_OPCODE_FRC:
2738       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2739          src0 = fetchSrc(0, c);
2740          val0 = getScratch();
2741          mkOp1(OP_FLOOR, TYPE_F32, val0, src0);
2742          mkOp2(OP_SUB, TYPE_F32, dst0[c], src0, val0);
2743       }
2744       break;
2745    case TGSI_OPCODE_ROUND:
2746       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2747          mkCvt(OP_CVT, TYPE_F32, dst0[c], TYPE_F32, fetchSrc(0, c))
2748          ->rnd = ROUND_NI;
2749       break;
2750    case TGSI_OPCODE_CLAMP:
2751       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2752          src0 = fetchSrc(0, c);
2753          src1 = fetchSrc(1, c);
2754          src2 = fetchSrc(2, c);
2755          val0 = getScratch();
2756          mkOp2(OP_MIN, TYPE_F32, val0, src0, src1);
2757          mkOp2(OP_MAX, TYPE_F32, dst0[c], val0, src2);
2758       }
2759       break;
2760    case TGSI_OPCODE_SLT:
2761    case TGSI_OPCODE_SGE:
2762    case TGSI_OPCODE_SEQ:
2763    case TGSI_OPCODE_SGT:
2764    case TGSI_OPCODE_SLE:
2765    case TGSI_OPCODE_SNE:
2766    case TGSI_OPCODE_FSEQ:
2767    case TGSI_OPCODE_FSGE:
2768    case TGSI_OPCODE_FSLT:
2769    case TGSI_OPCODE_FSNE:
2770    case TGSI_OPCODE_ISGE:
2771    case TGSI_OPCODE_ISLT:
2772    case TGSI_OPCODE_USEQ:
2773    case TGSI_OPCODE_USGE:
2774    case TGSI_OPCODE_USLT:
2775    case TGSI_OPCODE_USNE:
2776       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2777          src0 = fetchSrc(0, c);
2778          src1 = fetchSrc(1, c);
2779          mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], srcTy, src0, src1);
2780       }
2781       break;
2782    case TGSI_OPCODE_KILL_IF:
2783       val0 = new_LValue(func, FILE_PREDICATE);
2784       mask = 0;
2785       for (c = 0; c < 4; ++c) {
2786          const int s = tgsi.getSrc(0).getSwizzle(c);
2787          if (mask & (1 << s))
2788             continue;
2789          mask |= 1 << s;
2790          mkCmp(OP_SET, CC_LT, TYPE_F32, val0, TYPE_F32, fetchSrc(0, c), zero);
2791          mkOp(OP_DISCARD, TYPE_NONE, NULL)->setPredicate(CC_P, val0);
2792       }
2793       break;
2794    case TGSI_OPCODE_KILL:
2795       mkOp(OP_DISCARD, TYPE_NONE, NULL);
2796       break;
2797    case TGSI_OPCODE_TEX:
2798    case TGSI_OPCODE_TXB:
2799    case TGSI_OPCODE_TXL:
2800    case TGSI_OPCODE_TXP:
2801    case TGSI_OPCODE_LODQ:
2802       //              R  S     L     C    Dx    Dy
2803       handleTEX(dst0, 1, 1, 0x03, 0x0f, 0x00, 0x00);
2804       break;
2805    case TGSI_OPCODE_TXD:
2806       handleTEX(dst0, 3, 3, 0x03, 0x0f, 0x10, 0x20);
2807       break;
2808    case TGSI_OPCODE_TG4:
2809       handleTEX(dst0, 2, 2, 0x03, 0x0f, 0x00, 0x00);
2810       break;
2811    case TGSI_OPCODE_TEX2:
2812       handleTEX(dst0, 2, 2, 0x03, 0x10, 0x00, 0x00);
2813       break;
2814    case TGSI_OPCODE_TXB2:
2815    case TGSI_OPCODE_TXL2:
2816       handleTEX(dst0, 2, 2, 0x10, 0x0f, 0x00, 0x00);
2817       break;
2818    case TGSI_OPCODE_SAMPLE:
2819    case TGSI_OPCODE_SAMPLE_B:
2820    case TGSI_OPCODE_SAMPLE_D:
2821    case TGSI_OPCODE_SAMPLE_L:
2822    case TGSI_OPCODE_SAMPLE_C:
2823    case TGSI_OPCODE_SAMPLE_C_LZ:
2824       handleTEX(dst0, 1, 2, 0x30, 0x30, 0x30, 0x40);
2825       break;
2826    case TGSI_OPCODE_TXF:
2827       handleTXF(dst0, 1, 0x03);
2828       break;
2829    case TGSI_OPCODE_SAMPLE_I:
2830       handleTXF(dst0, 1, 0x03);
2831       break;
2832    case TGSI_OPCODE_SAMPLE_I_MS:
2833       handleTXF(dst0, 1, 0x20);
2834       break;
2835    case TGSI_OPCODE_TXQ:
2836    case TGSI_OPCODE_SVIEWINFO:
2837       handleTXQ(dst0, TXQ_DIMS, 1);
2838       break;
2839    case TGSI_OPCODE_TXQS:
2840       // The TXQ_TYPE query returns samples in its 3rd arg, but we need it to
2841       // be in .x
2842       dst0[1] = dst0[2] = dst0[3] = NULL;
2843       std::swap(dst0[0], dst0[2]);
2844       handleTXQ(dst0, TXQ_TYPE, 0);
2845       std::swap(dst0[0], dst0[2]);
2846       break;
2847    case TGSI_OPCODE_F2I:
2848    case TGSI_OPCODE_F2U:
2849       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2850          mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c))->rnd = ROUND_Z;
2851       break;
2852    case TGSI_OPCODE_I2F:
2853    case TGSI_OPCODE_U2F:
2854       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2855          mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c));
2856       break;
2857    case TGSI_OPCODE_PK2H:
2858       val0 = getScratch();
2859       val1 = getScratch();
2860       mkCvt(OP_CVT, TYPE_F16, val0, TYPE_F32, fetchSrc(0, 0));
2861       mkCvt(OP_CVT, TYPE_F16, val1, TYPE_F32, fetchSrc(0, 1));
2862       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2863          mkOp3(OP_INSBF, TYPE_U32, dst0[c], val1, mkImm(0x1010), val0);
2864       break;
2865    case TGSI_OPCODE_UP2H:
2866       src0 = fetchSrc(0, 0);
2867       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2868          geni = mkCvt(OP_CVT, TYPE_F32, dst0[c], TYPE_F16, src0);
2869          geni->subOp = c & 1;
2870       }
2871       break;
2872    case TGSI_OPCODE_EMIT:
2873       /* export the saved viewport index */
2874       if (viewport != NULL) {
2875          Symbol *vpSym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_U32,
2876                                   info->out[info->io.viewportId].slot[0] * 4);
2877          mkStore(OP_EXPORT, TYPE_U32, vpSym, NULL, viewport);
2878       }
2879       /* fallthrough */
2880    case TGSI_OPCODE_ENDPRIM:
2881    {
2882       // get vertex stream (must be immediate)
2883       unsigned int stream = tgsi.getSrc(0).getValueU32(0, info);
2884       if (stream && op == OP_RESTART)
2885          break;
2886       src0 = mkImm(stream);
2887       mkOp1(op, TYPE_U32, NULL, src0)->fixed = 1;
2888       break;
2889    }
2890    case TGSI_OPCODE_IF:
2891    case TGSI_OPCODE_UIF:
2892    {
2893       BasicBlock *ifBB = new BasicBlock(func);
2894
2895       bb->cfg.attach(&ifBB->cfg, Graph::Edge::TREE);
2896       condBBs.push(bb);
2897       joinBBs.push(bb);
2898
2899       mkFlow(OP_BRA, NULL, CC_NOT_P, fetchSrc(0, 0))->setType(srcTy);
2900
2901       setPosition(ifBB, true);
2902    }
2903       break;
2904    case TGSI_OPCODE_ELSE:
2905    {
2906       BasicBlock *elseBB = new BasicBlock(func);
2907       BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
2908
2909       forkBB->cfg.attach(&elseBB->cfg, Graph::Edge::TREE);
2910       condBBs.push(bb);
2911
2912       forkBB->getExit()->asFlow()->target.bb = elseBB;
2913       if (!bb->isTerminated())
2914          mkFlow(OP_BRA, NULL, CC_ALWAYS, NULL);
2915
2916       setPosition(elseBB, true);
2917    }
2918       break;
2919    case TGSI_OPCODE_ENDIF:
2920    {
2921       BasicBlock *convBB = new BasicBlock(func);
2922       BasicBlock *prevBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
2923       BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(joinBBs.pop().u.p);
2924
2925       if (!bb->isTerminated()) {
2926          // we only want join if none of the clauses ended with CONT/BREAK/RET
2927          if (prevBB->getExit()->op == OP_BRA && joinBBs.getSize() < 6)
2928             insertConvergenceOps(convBB, forkBB);
2929          mkFlow(OP_BRA, convBB, CC_ALWAYS, NULL);
2930          bb->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
2931       }
2932
2933       if (prevBB->getExit()->op == OP_BRA) {
2934          prevBB->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
2935          prevBB->getExit()->asFlow()->target.bb = convBB;
2936       }
2937       setPosition(convBB, true);
2938    }
2939       break;
2940    case TGSI_OPCODE_BGNLOOP:
2941    {
2942       BasicBlock *lbgnBB = new BasicBlock(func);
2943       BasicBlock *lbrkBB = new BasicBlock(func);
2944
2945       loopBBs.push(lbgnBB);
2946       breakBBs.push(lbrkBB);
2947       if (loopBBs.getSize() > func->loopNestingBound)
2948          func->loopNestingBound++;
2949
2950       mkFlow(OP_PREBREAK, lbrkBB, CC_ALWAYS, NULL);
2951
2952       bb->cfg.attach(&lbgnBB->cfg, Graph::Edge::TREE);
2953       setPosition(lbgnBB, true);
2954       mkFlow(OP_PRECONT, lbgnBB, CC_ALWAYS, NULL);
2955    }
2956       break;
2957    case TGSI_OPCODE_ENDLOOP:
2958    {
2959       BasicBlock *loopBB = reinterpret_cast<BasicBlock *>(loopBBs.pop().u.p);
2960
2961       if (!bb->isTerminated()) {
2962          mkFlow(OP_CONT, loopBB, CC_ALWAYS, NULL);
2963          bb->cfg.attach(&loopBB->cfg, Graph::Edge::BACK);
2964       }
2965       setPosition(reinterpret_cast<BasicBlock *>(breakBBs.pop().u.p), true);
2966
2967       // If the loop never breaks (e.g. only has RET's inside), then there
2968       // will be no way to get to the break bb. However BGNLOOP will have
2969       // already made a PREBREAK to it, so it must be in the CFG.
2970       if (getBB()->cfg.incidentCount() == 0)
2971          loopBB->cfg.attach(&getBB()->cfg, Graph::Edge::TREE);
2972    }
2973       break;
2974    case TGSI_OPCODE_BRK:
2975    {
2976       if (bb->isTerminated())
2977          break;
2978       BasicBlock *brkBB = reinterpret_cast<BasicBlock *>(breakBBs.peek().u.p);
2979       mkFlow(OP_BREAK, brkBB, CC_ALWAYS, NULL);
2980       bb->cfg.attach(&brkBB->cfg, Graph::Edge::CROSS);
2981    }
2982       break;
2983    case TGSI_OPCODE_CONT:
2984    {
2985       if (bb->isTerminated())
2986          break;
2987       BasicBlock *contBB = reinterpret_cast<BasicBlock *>(loopBBs.peek().u.p);
2988       mkFlow(OP_CONT, contBB, CC_ALWAYS, NULL);
2989       contBB->explicitCont = true;
2990       bb->cfg.attach(&contBB->cfg, Graph::Edge::BACK);
2991    }
2992       break;
2993    case TGSI_OPCODE_BGNSUB:
2994    {
2995       Subroutine *s = getSubroutine(ip);
2996       BasicBlock *entry = new BasicBlock(s->f);
2997       BasicBlock *leave = new BasicBlock(s->f);
2998
2999       // multiple entrypoints possible, keep the graph connected
3000       if (prog->getType() == Program::TYPE_COMPUTE)
3001          prog->main->call.attach(&s->f->call, Graph::Edge::TREE);
3002
3003       sub.cur = s;
3004       s->f->setEntry(entry);
3005       s->f->setExit(leave);
3006       setPosition(entry, true);
3007       return true;
3008    }
3009    case TGSI_OPCODE_ENDSUB:
3010    {
3011       sub.cur = getSubroutine(prog->main);
3012       setPosition(BasicBlock::get(sub.cur->f->cfg.getRoot()), true);
3013       return true;
3014    }
3015    case TGSI_OPCODE_CAL:
3016    {
3017       Subroutine *s = getSubroutine(tgsi.getLabel());
3018       mkFlow(OP_CALL, s->f, CC_ALWAYS, NULL);
3019       func->call.attach(&s->f->call, Graph::Edge::TREE);
3020       return true;
3021    }
3022    case TGSI_OPCODE_RET:
3023    {
3024       if (bb->isTerminated())
3025          return true;
3026       BasicBlock *leave = BasicBlock::get(func->cfgExit);
3027
3028       if (!isEndOfSubroutine(ip + 1)) {
3029          // insert a PRERET at the entry if this is an early return
3030          // (only needed for sharing code in the epilogue)
3031          BasicBlock *pos = getBB();
3032          setPosition(BasicBlock::get(func->cfg.getRoot()), false);
3033          mkFlow(OP_PRERET, leave, CC_ALWAYS, NULL)->fixed = 1;
3034          setPosition(pos, true);
3035       }
3036       mkFlow(OP_RET, NULL, CC_ALWAYS, NULL)->fixed = 1;
3037       bb->cfg.attach(&leave->cfg, Graph::Edge::CROSS);
3038    }
3039       break;
3040    case TGSI_OPCODE_END:
3041    {
3042       // attach and generate epilogue code
3043       BasicBlock *epilogue = BasicBlock::get(func->cfgExit);
3044       bb->cfg.attach(&epilogue->cfg, Graph::Edge::TREE);
3045       setPosition(epilogue, true);
3046       if (prog->getType() == Program::TYPE_FRAGMENT)
3047          exportOutputs();
3048       if (info->io.genUserClip > 0)
3049          handleUserClipPlanes();
3050       mkOp(OP_EXIT, TYPE_NONE, NULL)->terminator = 1;
3051    }
3052       break;
3053    case TGSI_OPCODE_SWITCH:
3054    case TGSI_OPCODE_CASE:
3055       ERROR("switch/case opcode encountered, should have been lowered\n");
3056       abort();
3057       break;
3058    case TGSI_OPCODE_LOAD:
3059       handleLOAD(dst0);
3060       break;
3061    case TGSI_OPCODE_STORE:
3062       handleSTORE();
3063       break;
3064    case TGSI_OPCODE_BARRIER:
3065       geni = mkOp2(OP_BAR, TYPE_U32, NULL, mkImm(0), mkImm(0));
3066       geni->fixed = 1;
3067       geni->subOp = NV50_IR_SUBOP_BAR_SYNC;
3068       break;
3069    case TGSI_OPCODE_MFENCE:
3070    case TGSI_OPCODE_LFENCE:
3071    case TGSI_OPCODE_SFENCE:
3072       geni = mkOp(OP_MEMBAR, TYPE_NONE, NULL);
3073       geni->fixed = 1;
3074       geni->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode());
3075       break;
3076    case TGSI_OPCODE_ATOMUADD:
3077    case TGSI_OPCODE_ATOMXCHG:
3078    case TGSI_OPCODE_ATOMCAS:
3079    case TGSI_OPCODE_ATOMAND:
3080    case TGSI_OPCODE_ATOMOR:
3081    case TGSI_OPCODE_ATOMXOR:
3082    case TGSI_OPCODE_ATOMUMIN:
3083    case TGSI_OPCODE_ATOMIMIN:
3084    case TGSI_OPCODE_ATOMUMAX:
3085    case TGSI_OPCODE_ATOMIMAX:
3086       handleATOM(dst0, dstTy, tgsi::opcodeToSubOp(tgsi.getOpcode()));
3087       break;
3088    case TGSI_OPCODE_IBFE:
3089    case TGSI_OPCODE_UBFE:
3090       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3091          src0 = fetchSrc(0, c);
3092          if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE &&
3093              tgsi.getSrc(2).getFile() == TGSI_FILE_IMMEDIATE) {
3094             src1 = loadImm(NULL, tgsi.getSrc(2).getValueU32(c, info) << 8 |
3095                            tgsi.getSrc(1).getValueU32(c, info));
3096          } else {
3097             src1 = fetchSrc(1, c);
3098             src2 = fetchSrc(2, c);
3099             mkOp3(OP_INSBF, TYPE_U32, src1, src2, mkImm(0x808), src1);
3100          }
3101          mkOp2(OP_EXTBF, dstTy, dst0[c], src0, src1);
3102       }
3103       break;
3104    case TGSI_OPCODE_BFI:
3105       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3106          src0 = fetchSrc(0, c);
3107          src1 = fetchSrc(1, c);
3108          src2 = fetchSrc(2, c);
3109          src3 = fetchSrc(3, c);
3110          mkOp3(OP_INSBF, TYPE_U32, src2, src3, mkImm(0x808), src2);
3111          mkOp3(OP_INSBF, TYPE_U32, dst0[c], src1, src2, src0);
3112       }
3113       break;
3114    case TGSI_OPCODE_LSB:
3115       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3116          src0 = fetchSrc(0, c);
3117          geni = mkOp2(OP_EXTBF, TYPE_U32, src0, src0, mkImm(0x2000));
3118          geni->subOp = NV50_IR_SUBOP_EXTBF_REV;
3119          geni = mkOp1(OP_BFIND, TYPE_U32, dst0[c], src0);
3120          geni->subOp = NV50_IR_SUBOP_BFIND_SAMT;
3121       }
3122       break;
3123    case TGSI_OPCODE_IMSB:
3124    case TGSI_OPCODE_UMSB:
3125       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3126          src0 = fetchSrc(0, c);
3127          mkOp1(OP_BFIND, srcTy, dst0[c], src0);
3128       }
3129       break;
3130    case TGSI_OPCODE_BREV:
3131       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3132          src0 = fetchSrc(0, c);
3133          geni = mkOp2(OP_EXTBF, TYPE_U32, dst0[c], src0, mkImm(0x2000));
3134          geni->subOp = NV50_IR_SUBOP_EXTBF_REV;
3135       }
3136       break;
3137    case TGSI_OPCODE_POPC:
3138       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3139          src0 = fetchSrc(0, c);
3140          mkOp2(OP_POPCNT, TYPE_U32, dst0[c], src0, src0);
3141       }
3142       break;
3143    case TGSI_OPCODE_INTERP_CENTROID:
3144    case TGSI_OPCODE_INTERP_SAMPLE:
3145    case TGSI_OPCODE_INTERP_OFFSET:
3146       handleINTERP(dst0);
3147       break;
3148    case TGSI_OPCODE_D2I:
3149    case TGSI_OPCODE_D2U:
3150    case TGSI_OPCODE_D2F: {
3151       int pos = 0;
3152       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3153          Value *dreg = getSSA(8);
3154          src0 = fetchSrc(0, pos);
3155          src1 = fetchSrc(0, pos + 1);
3156          mkOp2(OP_MERGE, TYPE_U64, dreg, src0, src1);
3157          mkCvt(OP_CVT, dstTy, dst0[c], srcTy, dreg);
3158          pos += 2;
3159       }
3160       break;
3161    }
3162    case TGSI_OPCODE_I2D:
3163    case TGSI_OPCODE_U2D:
3164    case TGSI_OPCODE_F2D:
3165       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3166          Value *dreg = getSSA(8);
3167          mkCvt(OP_CVT, dstTy, dreg, srcTy, fetchSrc(0, c / 2));
3168          mkSplit(&dst0[c], 4, dreg);
3169          c++;
3170       }
3171       break;
3172    case TGSI_OPCODE_DABS:
3173    case TGSI_OPCODE_DNEG:
3174    case TGSI_OPCODE_DRCP:
3175    case TGSI_OPCODE_DSQRT:
3176    case TGSI_OPCODE_DRSQ:
3177    case TGSI_OPCODE_DTRUNC:
3178    case TGSI_OPCODE_DCEIL:
3179    case TGSI_OPCODE_DFLR:
3180       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3181          src0 = getSSA(8);
3182          Value *dst = getSSA(8), *tmp[2];
3183          tmp[0] = fetchSrc(0, c);
3184          tmp[1] = fetchSrc(0, c + 1);
3185          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3186          mkOp1(op, dstTy, dst, src0);
3187          mkSplit(&dst0[c], 4, dst);
3188          c++;
3189       }
3190       break;
3191    case TGSI_OPCODE_DFRAC:
3192       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3193          src0 = getSSA(8);
3194          Value *dst = getSSA(8), *tmp[2];
3195          tmp[0] = fetchSrc(0, c);
3196          tmp[1] = fetchSrc(0, c + 1);
3197          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3198          mkOp1(OP_FLOOR, TYPE_F64, dst, src0);
3199          mkOp2(OP_SUB, TYPE_F64, dst, src0, dst);
3200          mkSplit(&dst0[c], 4, dst);
3201          c++;
3202       }
3203       break;
3204    case TGSI_OPCODE_DSLT:
3205    case TGSI_OPCODE_DSGE:
3206    case TGSI_OPCODE_DSEQ:
3207    case TGSI_OPCODE_DSNE: {
3208       int pos = 0;
3209       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3210          Value *tmp[2];
3211
3212          src0 = getSSA(8);
3213          src1 = getSSA(8);
3214          tmp[0] = fetchSrc(0, pos);
3215          tmp[1] = fetchSrc(0, pos + 1);
3216          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3217          tmp[0] = fetchSrc(1, pos);
3218          tmp[1] = fetchSrc(1, pos + 1);
3219          mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
3220          mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], srcTy, src0, src1);
3221          pos += 2;
3222       }
3223       break;
3224    }
3225    case TGSI_OPCODE_DADD:
3226    case TGSI_OPCODE_DMUL:
3227    case TGSI_OPCODE_DMAX:
3228    case TGSI_OPCODE_DMIN:
3229       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3230          src0 = getSSA(8);
3231          src1 = getSSA(8);
3232          Value *dst = getSSA(8), *tmp[2];
3233          tmp[0] = fetchSrc(0, c);
3234          tmp[1] = fetchSrc(0, c + 1);
3235          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3236          tmp[0] = fetchSrc(1, c);
3237          tmp[1] = fetchSrc(1, c + 1);
3238          mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
3239          mkOp2(op, dstTy, dst, src0, src1);
3240          mkSplit(&dst0[c], 4, dst);
3241          c++;
3242       }
3243       break;
3244    case TGSI_OPCODE_DMAD:
3245       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3246          src0 = getSSA(8);
3247          src1 = getSSA(8);
3248          src2 = getSSA(8);
3249          Value *dst = getSSA(8), *tmp[2];
3250          tmp[0] = fetchSrc(0, c);
3251          tmp[1] = fetchSrc(0, c + 1);
3252          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3253          tmp[0] = fetchSrc(1, c);
3254          tmp[1] = fetchSrc(1, c + 1);
3255          mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
3256          tmp[0] = fetchSrc(2, c);
3257          tmp[1] = fetchSrc(2, c + 1);
3258          mkOp2(OP_MERGE, TYPE_U64, src2, tmp[0], tmp[1]);
3259          mkOp3(op, dstTy, dst, src0, src1, src2);
3260          mkSplit(&dst0[c], 4, dst);
3261          c++;
3262       }
3263       break;
3264    case TGSI_OPCODE_DROUND:
3265       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3266          src0 = getSSA(8);
3267          Value *dst = getSSA(8), *tmp[2];
3268          tmp[0] = fetchSrc(0, c);
3269          tmp[1] = fetchSrc(0, c + 1);
3270          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3271          mkCvt(OP_CVT, TYPE_F64, dst, TYPE_F64, src0)
3272          ->rnd = ROUND_NI;
3273          mkSplit(&dst0[c], 4, dst);
3274          c++;
3275       }
3276       break;
3277    case TGSI_OPCODE_DSSG:
3278       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3279          src0 = getSSA(8);
3280          Value *dst = getSSA(8), *dstF32 = getSSA(), *tmp[2];
3281          tmp[0] = fetchSrc(0, c);
3282          tmp[1] = fetchSrc(0, c + 1);
3283          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3284
3285          val0 = getScratch();
3286          val1 = getScratch();
3287          // The zero is wrong here since it's only 32-bit, but it works out in
3288          // the end since it gets replaced with $r63.
3289          mkCmp(OP_SET, CC_GT, TYPE_F32, val0, TYPE_F64, src0, zero);
3290          mkCmp(OP_SET, CC_LT, TYPE_F32, val1, TYPE_F64, src0, zero);
3291          mkOp2(OP_SUB, TYPE_F32, dstF32, val0, val1);
3292          mkCvt(OP_CVT, TYPE_F64, dst, TYPE_F32, dstF32);
3293          mkSplit(&dst0[c], 4, dst);
3294          c++;
3295       }
3296       break;
3297    default:
3298       ERROR("unhandled TGSI opcode: %u\n", tgsi.getOpcode());
3299       assert(0);
3300       break;
3301    }
3302
3303    if (tgsi.dstCount()) {
3304       for (c = 0; c < 4; ++c) {
3305          if (!dst0[c])
3306             continue;
3307          if (dst0[c] != rDst0[c])
3308             mkMov(rDst0[c], dst0[c]);
3309          storeDst(0, c, rDst0[c]);
3310       }
3311    }
3312    vtxBaseValid = 0;
3313
3314    return true;
3315 }
3316
3317 void
3318 Converter::handleUserClipPlanes()
3319 {
3320    Value *res[8];
3321    int n, i, c;
3322
3323    for (c = 0; c < 4; ++c) {
3324       for (i = 0; i < info->io.genUserClip; ++i) {
3325          Symbol *sym = mkSymbol(FILE_MEMORY_CONST, info->io.auxCBSlot,
3326                                 TYPE_F32, info->io.ucpBase + i * 16 + c * 4);
3327          Value *ucp = mkLoadv(TYPE_F32, sym, NULL);
3328          if (c == 0)
3329             res[i] = mkOp2v(OP_MUL, TYPE_F32, getScratch(), clipVtx[c], ucp);
3330          else
3331             mkOp3(OP_MAD, TYPE_F32, res[i], clipVtx[c], ucp, res[i]);
3332       }
3333    }
3334
3335    const int first = info->numOutputs - (info->io.genUserClip + 3) / 4;
3336
3337    for (i = 0; i < info->io.genUserClip; ++i) {
3338       n = i / 4 + first;
3339       c = i % 4;
3340       Symbol *sym =
3341          mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32, info->out[n].slot[c] * 4);
3342       mkStore(OP_EXPORT, TYPE_F32, sym, NULL, res[i]);
3343    }
3344 }
3345
3346 void
3347 Converter::exportOutputs()
3348 {
3349    for (unsigned int i = 0; i < info->numOutputs; ++i) {
3350       for (unsigned int c = 0; c < 4; ++c) {
3351          if (!oData.exists(sub.cur->values, i, c))
3352             continue;
3353          Symbol *sym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32,
3354                                 info->out[i].slot[c] * 4);
3355          Value *val = oData.load(sub.cur->values, i, c, NULL);
3356          if (val)
3357             mkStore(OP_EXPORT, TYPE_F32, sym, NULL, val);
3358       }
3359    }
3360 }
3361
3362 Converter::Converter(Program *ir, const tgsi::Source *code) : BuildUtil(ir),
3363      code(code),
3364      tgsi(NULL),
3365      tData(this), lData(this), aData(this), pData(this), oData(this)
3366 {
3367    info = code->info;
3368
3369    const unsigned tSize = code->fileSize(TGSI_FILE_TEMPORARY);
3370    const unsigned pSize = code->fileSize(TGSI_FILE_PREDICATE);
3371    const unsigned aSize = code->fileSize(TGSI_FILE_ADDRESS);
3372    const unsigned oSize = code->fileSize(TGSI_FILE_OUTPUT);
3373
3374    tData.setup(TGSI_FILE_TEMPORARY, 0, 0, tSize, 4, 4, FILE_GPR, 0);
3375    lData.setup(TGSI_FILE_TEMPORARY, 1, 0, tSize, 4, 4, FILE_MEMORY_LOCAL, 0);
3376    pData.setup(TGSI_FILE_PREDICATE, 0, 0, pSize, 4, 4, FILE_PREDICATE, 0);
3377    aData.setup(TGSI_FILE_ADDRESS, 0, 0, aSize, 4, 4, FILE_GPR, 0);
3378    oData.setup(TGSI_FILE_OUTPUT, 0, 0, oSize, 4, 4, FILE_GPR, 0);
3379
3380    zero = mkImm((uint32_t)0);
3381
3382    vtxBaseValid = 0;
3383 }
3384
3385 Converter::~Converter()
3386 {
3387 }
3388
3389 inline const Converter::Location *
3390 Converter::BindArgumentsPass::getValueLocation(Subroutine *s, Value *v)
3391 {
3392    ValueMap::l_iterator it = s->values.l.find(v);
3393    return it == s->values.l.end() ? NULL : &it->second;
3394 }
3395
3396 template<typename T> inline void
3397 Converter::BindArgumentsPass::updateCallArgs(
3398    Instruction *i, void (Instruction::*setArg)(int, Value *),
3399    T (Function::*proto))
3400 {
3401    Function *g = i->asFlow()->target.fn;
3402    Subroutine *subg = conv.getSubroutine(g);
3403
3404    for (unsigned a = 0; a < (g->*proto).size(); ++a) {
3405       Value *v = (g->*proto)[a].get();
3406       const Converter::Location &l = *getValueLocation(subg, v);
3407       Converter::DataArray *array = conv.getArrayForFile(l.array, l.arrayIdx);
3408
3409       (i->*setArg)(a, array->acquire(sub->values, l.i, l.c));
3410    }
3411 }
3412
3413 template<typename T> inline void
3414 Converter::BindArgumentsPass::updatePrototype(
3415    BitSet *set, void (Function::*updateSet)(), T (Function::*proto))
3416 {
3417    (func->*updateSet)();
3418
3419    for (unsigned i = 0; i < set->getSize(); ++i) {
3420       Value *v = func->getLValue(i);
3421       const Converter::Location *l = getValueLocation(sub, v);
3422
3423       // only include values with a matching TGSI register
3424       if (set->test(i) && l && !conv.code->locals.count(*l))
3425          (func->*proto).push_back(v);
3426    }
3427 }
3428
3429 bool
3430 Converter::BindArgumentsPass::visit(Function *f)
3431 {
3432    sub = conv.getSubroutine(f);
3433
3434    for (ArrayList::Iterator bi = f->allBBlocks.iterator();
3435         !bi.end(); bi.next()) {
3436       for (Instruction *i = BasicBlock::get(bi)->getFirst();
3437            i; i = i->next) {
3438          if (i->op == OP_CALL && !i->asFlow()->builtin) {
3439             updateCallArgs(i, &Instruction::setSrc, &Function::ins);
3440             updateCallArgs(i, &Instruction::setDef, &Function::outs);
3441          }
3442       }
3443    }
3444
3445    if (func == prog->main && prog->getType() != Program::TYPE_COMPUTE)
3446       return true;
3447    updatePrototype(&BasicBlock::get(f->cfg.getRoot())->liveSet,
3448                    &Function::buildLiveSets, &Function::ins);
3449    updatePrototype(&BasicBlock::get(f->cfgExit)->defSet,
3450                    &Function::buildDefSets, &Function::outs);
3451
3452    return true;
3453 }
3454
3455 bool
3456 Converter::run()
3457 {
3458    BasicBlock *entry = new BasicBlock(prog->main);
3459    BasicBlock *leave = new BasicBlock(prog->main);
3460
3461    prog->main->setEntry(entry);
3462    prog->main->setExit(leave);
3463
3464    setPosition(entry, true);
3465    sub.cur = getSubroutine(prog->main);
3466
3467    if (info->io.genUserClip > 0) {
3468       for (int c = 0; c < 4; ++c)
3469          clipVtx[c] = getScratch();
3470    }
3471
3472    switch (prog->getType()) {
3473    case Program::TYPE_TESSELLATION_CONTROL:
3474       outBase = mkOp2v(
3475          OP_SUB, TYPE_U32, getSSA(),
3476          mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_LANEID, 0)),
3477          mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_INVOCATION_ID, 0)));
3478       break;
3479    case Program::TYPE_FRAGMENT: {
3480       Symbol *sv = mkSysVal(SV_POSITION, 3);
3481       fragCoord[3] = mkOp1v(OP_RDSV, TYPE_F32, getSSA(), sv);
3482       mkOp1(OP_RCP, TYPE_F32, fragCoord[3], fragCoord[3]);
3483       break;
3484    }
3485    default:
3486       break;
3487    }
3488
3489    if (info->io.viewportId >= 0)
3490       viewport = getScratch();
3491    else
3492       viewport = NULL;
3493
3494    for (ip = 0; ip < code->scan.num_instructions; ++ip) {
3495       if (!handleInstruction(&code->insns[ip]))
3496          return false;
3497    }
3498
3499    if (!BindArgumentsPass(*this).run(prog))
3500       return false;
3501
3502    return true;
3503 }
3504
3505 } // unnamed namespace
3506
3507 namespace nv50_ir {
3508
3509 bool
3510 Program::makeFromTGSI(struct nv50_ir_prog_info *info)
3511 {
3512    tgsi::Source src(info);
3513    if (!src.scanSource())
3514       return false;
3515    tlsSize = info->bin.tlsSpace;
3516
3517    Converter builder(this, &src);
3518    return builder.run();
3519 }
3520
3521 } // namespace nv50_ir