src/gallium/drivers/nouveau/codegen/nv50_ir_from_tgsi.cpp

   1 /*
   2  * Copyright 2011 Christoph Bumiller
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 #include "tgsi/tgsi_dump.h"
  24 #include "tgsi/tgsi_scan.h"
  25 #include "tgsi/tgsi_util.h"
  26
  27 #include <set>
  28
  29 #include "codegen/nv50_ir.h"
  30 #include "codegen/nv50_ir_util.h"
  31 #include "codegen/nv50_ir_build_util.h"
  32
  33 namespace tgsi {
  34
  35 class Source;
  36
  37 static nv50_ir::operation translateOpcode(uint opcode);
  38 static nv50_ir::DataFile translateFile(uint file);
  39 static nv50_ir::TexTarget translateTexture(uint texTarg);
  40 static nv50_ir::SVSemantic translateSysVal(uint sysval);
  41 static nv50_ir::CacheMode translateCacheMode(uint qualifier);
  42
  43 class Instruction
  44 {
  45 public:
  46    Instruction(const struct tgsi_full_instruction *inst) : insn(inst) { }
  47
  48    class SrcRegister
  49    {
  50    public:
  51       SrcRegister(const struct tgsi_full_src_register *src)
  52          : reg(src->Register),
  53            fsr(src)
  54       { }
  55
  56       SrcRegister(const struct tgsi_src_register& src) : reg(src), fsr(NULL) { }
  57
  58       SrcRegister(const struct tgsi_ind_register& ind)
  59          : reg(tgsi_util_get_src_from_ind(&ind)),
  60            fsr(NULL)
  61       { }
  62
  63       struct tgsi_src_register offsetToSrc(struct tgsi_texture_offset off)
  64       {
  65          struct tgsi_src_register reg;
  66          memset(&reg, 0, sizeof(reg));
  67          reg.Index = off.Index;
  68          reg.File = off.File;
  69          reg.SwizzleX = off.SwizzleX;
  70          reg.SwizzleY = off.SwizzleY;
  71          reg.SwizzleZ = off.SwizzleZ;
  72          return reg;
  73       }
  74
  75       SrcRegister(const struct tgsi_texture_offset& off) :
  76          reg(offsetToSrc(off)),
  77          fsr(NULL)
  78       { }
  79
  80       uint getFile() const { return reg.File; }
  81
  82       bool is2D() const { return reg.Dimension; }
  83
  84       bool isIndirect(int dim) const
  85       {
  86          return (dim && fsr) ? fsr->Dimension.Indirect : reg.Indirect;
  87       }
  88
  89       int getIndex(int dim) const
  90       {
  91          return (dim && fsr) ? fsr->Dimension.Index : reg.Index;
  92       }
  93
  94       int getSwizzle(int chan) const
  95       {
  96          return tgsi_util_get_src_register_swizzle(&reg, chan);
  97       }
  98
  99       int getArrayId() const
 100       {
 101          if (isIndirect(0))
 102             return fsr->Indirect.ArrayID;
 103          return 0;
 104       }
 105
 106       nv50_ir::Modifier getMod(int chan) const;
 107
 108       SrcRegister getIndirect(int dim) const
 109       {
 110          assert(fsr && isIndirect(dim));
 111          if (dim)
 112             return SrcRegister(fsr->DimIndirect);
 113          return SrcRegister(fsr->Indirect);
 114       }
 115
 116       uint32_t getValueU32(int c, const struct nv50_ir_prog_info *info) const
 117       {
 118          assert(reg.File == TGSI_FILE_IMMEDIATE);
 119          assert(!reg.Absolute);
 120          assert(!reg.Negate);
 121          return info->immd.data[reg.Index * 4 + getSwizzle(c)];
 122       }
 123
 124    private:
 125       const struct tgsi_src_register reg;
 126       const struct tgsi_full_src_register *fsr;
 127    };
 128
 129    class DstRegister
 130    {
 131    public:
 132       DstRegister(const struct tgsi_full_dst_register *dst)
 133          : reg(dst->Register),
 134            fdr(dst)
 135       { }
 136
 137       DstRegister(const struct tgsi_dst_register& dst) : reg(dst), fdr(NULL) { }
 138
 139       uint getFile() const { return reg.File; }
 140
 141       bool is2D() const { return reg.Dimension; }
 142
 143       bool isIndirect(int dim) const
 144       {
 145          return (dim && fdr) ? fdr->Dimension.Indirect : reg.Indirect;
 146       }
 147
 148       int getIndex(int dim) const
 149       {
 150          return (dim && fdr) ? fdr->Dimension.Dimension : reg.Index;
 151       }
 152
 153       unsigned int getMask() const { return reg.WriteMask; }
 154
 155       bool isMasked(int chan) const { return !(getMask() & (1 << chan)); }
 156
 157       SrcRegister getIndirect(int dim) const
 158       {
 159          assert(fdr && isIndirect(dim));
 160          if (dim)
 161             return SrcRegister(fdr->DimIndirect);
 162          return SrcRegister(fdr->Indirect);
 163       }
 164
 165       int getArrayId() const
 166       {
 167          if (isIndirect(0))
 168             return fdr->Indirect.ArrayID;
 169          return 0;
 170       }
 171
 172    private:
 173       const struct tgsi_dst_register reg;
 174       const struct tgsi_full_dst_register *fdr;
 175    };
 176
 177    inline uint getOpcode() const { return insn->Instruction.Opcode; }
 178
 179    unsigned int srcCount() const { return insn->Instruction.NumSrcRegs; }
 180    unsigned int dstCount() const { return insn->Instruction.NumDstRegs; }
 181
 182    // mask of used components of source s
 183    unsigned int srcMask(unsigned int s) const;
 184
 185    SrcRegister getSrc(unsigned int s) const
 186    {
 187       assert(s < srcCount());
 188       return SrcRegister(&insn->Src[s]);
 189    }
 190
 191    DstRegister getDst(unsigned int d) const
 192    {
 193       assert(d < dstCount());
 194       return DstRegister(&insn->Dst[d]);
 195    }
 196
 197    SrcRegister getTexOffset(unsigned int i) const
 198    {
 199       assert(i < TGSI_FULL_MAX_TEX_OFFSETS);
 200       return SrcRegister(insn->TexOffsets[i]);
 201    }
 202
 203    unsigned int getNumTexOffsets() const { return insn->Texture.NumOffsets; }
 204
 205    bool checkDstSrcAliasing() const;
 206
 207    inline nv50_ir::operation getOP() const {
 208       return translateOpcode(getOpcode()); }
 209
 210    nv50_ir::DataType inferSrcType() const;
 211    nv50_ir::DataType inferDstType() const;
 212
 213    nv50_ir::CondCode getSetCond() const;
 214
 215    nv50_ir::TexInstruction::Target getTexture(const Source *, int s) const;
 216
 217    nv50_ir::CacheMode getCacheMode() const {
 218       if (!insn->Instruction.Memory)
 219          return nv50_ir::CACHE_CA;
 220       return translateCacheMode(insn->Memory.Qualifier);
 221    }
 222
 223    inline uint getLabel() { return insn->Label.Label; }
 224
 225    unsigned getSaturate() const { return insn->Instruction.Saturate; }
 226
 227    void print() const
 228    {
 229       tgsi_dump_instruction(insn, 1);
 230    }
 231
 232 private:
 233    const struct tgsi_full_instruction *insn;
 234 };
 235
 236 unsigned int Instruction::srcMask(unsigned int s) const
 237 {
 238    unsigned int mask = insn->Dst[0].Register.WriteMask;
 239
 240    switch (insn->Instruction.Opcode) {
 241    case TGSI_OPCODE_COS:
 242    case TGSI_OPCODE_SIN:
 243       return (mask & 0x8) | ((mask & 0x7) ? 0x1 : 0x0);
 244    case TGSI_OPCODE_DP2:
 245       return 0x3;
 246    case TGSI_OPCODE_DP3:
 247       return 0x7;
 248    case TGSI_OPCODE_DP4:
 249    case TGSI_OPCODE_DPH:
 250    case TGSI_OPCODE_KILL_IF: /* WriteMask ignored */
 251       return 0xf;
 252    case TGSI_OPCODE_DST:
 253       return mask & (s ? 0xa : 0x6);
 254    case TGSI_OPCODE_EX2:
 255    case TGSI_OPCODE_EXP:
 256    case TGSI_OPCODE_LG2:
 257    case TGSI_OPCODE_LOG:
 258    case TGSI_OPCODE_POW:
 259    case TGSI_OPCODE_RCP:
 260    case TGSI_OPCODE_RSQ:
 261    case TGSI_OPCODE_SCS:
 262       return 0x1;
 263    case TGSI_OPCODE_IF:
 264    case TGSI_OPCODE_UIF:
 265       return 0x1;
 266    case TGSI_OPCODE_LIT:
 267       return 0xb;
 268    case TGSI_OPCODE_TEX2:
 269    case TGSI_OPCODE_TXB2:
 270    case TGSI_OPCODE_TXL2:
 271       return (s == 0) ? 0xf : 0x3;
 272    case TGSI_OPCODE_TEX:
 273    case TGSI_OPCODE_TXB:
 274    case TGSI_OPCODE_TXD:
 275    case TGSI_OPCODE_TXL:
 276    case TGSI_OPCODE_TXP:
 277    case TGSI_OPCODE_LODQ:
 278    {
 279       const struct tgsi_instruction_texture *tex = &insn->Texture;
 280
 281       assert(insn->Instruction.Texture);
 282
 283       mask = 0x7;
 284       if (insn->Instruction.Opcode != TGSI_OPCODE_TEX &&
 285           insn->Instruction.Opcode != TGSI_OPCODE_TXD)
 286          mask |= 0x8; /* bias, lod or proj */
 287
 288       switch (tex->Texture) {
 289       case TGSI_TEXTURE_1D:
 290          mask &= 0x9;
 291          break;
 292       case TGSI_TEXTURE_SHADOW1D:
 293          mask &= 0xd;
 294          break;
 295       case TGSI_TEXTURE_1D_ARRAY:
 296       case TGSI_TEXTURE_2D:
 297       case TGSI_TEXTURE_RECT:
 298          mask &= 0xb;
 299          break;
 300       case TGSI_TEXTURE_CUBE_ARRAY:
 301       case TGSI_TEXTURE_SHADOW2D_ARRAY:
 302       case TGSI_TEXTURE_SHADOWCUBE:
 303       case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
 304          mask |= 0x8;
 305          break;
 306       default:
 307          break;
 308       }
 309    }
 310       return mask;
 311    case TGSI_OPCODE_XPD:
 312    {
 313       unsigned int x = 0;
 314       if (mask & 1) x |= 0x6;
 315       if (mask & 2) x |= 0x5;
 316       if (mask & 4) x |= 0x3;
 317       return x;
 318    }
 319    case TGSI_OPCODE_D2I:
 320    case TGSI_OPCODE_D2U:
 321    case TGSI_OPCODE_D2F:
 322    case TGSI_OPCODE_DSLT:
 323    case TGSI_OPCODE_DSGE:
 324    case TGSI_OPCODE_DSEQ:
 325    case TGSI_OPCODE_DSNE:
 326       switch (util_bitcount(mask)) {
 327       case 1: return 0x3;
 328       case 2: return 0xf;
 329       default:
 330          assert(!"unexpected mask");
 331          return 0xf;
 332       }
 333    case TGSI_OPCODE_I2D:
 334    case TGSI_OPCODE_U2D:
 335    case TGSI_OPCODE_F2D: {
 336       unsigned int x = 0;
 337       if ((mask & 0x3) == 0x3)
 338          x |= 1;
 339       if ((mask & 0xc) == 0xc)
 340          x |= 2;
 341       return x;
 342    }
 343    case TGSI_OPCODE_PK2H:
 344       return 0x3;
 345    case TGSI_OPCODE_UP2H:
 346       return 0x1;
 347    default:
 348       break;
 349    }
 350
 351    return mask;
 352 }
 353
 354 nv50_ir::Modifier Instruction::SrcRegister::getMod(int chan) const
 355 {
 356    nv50_ir::Modifier m(0);
 357
 358    if (reg.Absolute)
 359       m = m | nv50_ir::Modifier(NV50_IR_MOD_ABS);
 360    if (reg.Negate)
 361       m = m | nv50_ir::Modifier(NV50_IR_MOD_NEG);
 362    return m;
 363 }
 364
 365 static nv50_ir::DataFile translateFile(uint file)
 366 {
 367    switch (file) {
 368    case TGSI_FILE_CONSTANT:        return nv50_ir::FILE_MEMORY_CONST;
 369    case TGSI_FILE_INPUT:           return nv50_ir::FILE_SHADER_INPUT;
 370    case TGSI_FILE_OUTPUT:          return nv50_ir::FILE_SHADER_OUTPUT;
 371    case TGSI_FILE_TEMPORARY:       return nv50_ir::FILE_GPR;
 372    case TGSI_FILE_ADDRESS:         return nv50_ir::FILE_ADDRESS;
 373    case TGSI_FILE_PREDICATE:       return nv50_ir::FILE_PREDICATE;
 374    case TGSI_FILE_IMMEDIATE:       return nv50_ir::FILE_IMMEDIATE;
 375    case TGSI_FILE_SYSTEM_VALUE:    return nv50_ir::FILE_SYSTEM_VALUE;
 376    case TGSI_FILE_BUFFER:          return nv50_ir::FILE_MEMORY_GLOBAL;
 377    case TGSI_FILE_MEMORY:          return nv50_ir::FILE_MEMORY_GLOBAL;
 378    case TGSI_FILE_SAMPLER:
 379    case TGSI_FILE_NULL:
 380    default:
 381       return nv50_ir::FILE_NULL;
 382    }
 383 }
 384
 385 static nv50_ir::SVSemantic translateSysVal(uint sysval)
 386 {
 387    switch (sysval) {
 388    case TGSI_SEMANTIC_FACE:       return nv50_ir::SV_FACE;
 389    case TGSI_SEMANTIC_PSIZE:      return nv50_ir::SV_POINT_SIZE;
 390    case TGSI_SEMANTIC_PRIMID:     return nv50_ir::SV_PRIMITIVE_ID;
 391    case TGSI_SEMANTIC_INSTANCEID: return nv50_ir::SV_INSTANCE_ID;
 392    case TGSI_SEMANTIC_VERTEXID:   return nv50_ir::SV_VERTEX_ID;
 393    case TGSI_SEMANTIC_GRID_SIZE:  return nv50_ir::SV_NCTAID;
 394    case TGSI_SEMANTIC_BLOCK_ID:   return nv50_ir::SV_CTAID;
 395    case TGSI_SEMANTIC_BLOCK_SIZE: return nv50_ir::SV_NTID;
 396    case TGSI_SEMANTIC_THREAD_ID:  return nv50_ir::SV_TID;
 397    case TGSI_SEMANTIC_SAMPLEID:   return nv50_ir::SV_SAMPLE_INDEX;
 398    case TGSI_SEMANTIC_SAMPLEPOS:  return nv50_ir::SV_SAMPLE_POS;
 399    case TGSI_SEMANTIC_SAMPLEMASK: return nv50_ir::SV_SAMPLE_MASK;
 400    case TGSI_SEMANTIC_INVOCATIONID: return nv50_ir::SV_INVOCATION_ID;
 401    case TGSI_SEMANTIC_TESSCOORD:  return nv50_ir::SV_TESS_COORD;
 402    case TGSI_SEMANTIC_TESSOUTER:  return nv50_ir::SV_TESS_OUTER;
 403    case TGSI_SEMANTIC_TESSINNER:  return nv50_ir::SV_TESS_INNER;
 404    case TGSI_SEMANTIC_VERTICESIN: return nv50_ir::SV_VERTEX_COUNT;
 405    case TGSI_SEMANTIC_HELPER_INVOCATION: return nv50_ir::SV_THREAD_KILL;
 406    case TGSI_SEMANTIC_BASEVERTEX: return nv50_ir::SV_BASEVERTEX;
 407    case TGSI_SEMANTIC_BASEINSTANCE: return nv50_ir::SV_BASEINSTANCE;
 408    case TGSI_SEMANTIC_DRAWID:     return nv50_ir::SV_DRAWID;
 409    default:
 410       assert(0);
 411       return nv50_ir::SV_CLOCK;
 412    }
 413 }
 414
 415 #define NV50_IR_TEX_TARG_CASE(a, b) \
 416    case TGSI_TEXTURE_##a: return nv50_ir::TEX_TARGET_##b;
 417
 418 static nv50_ir::TexTarget translateTexture(uint tex)
 419 {
 420    switch (tex) {
 421    NV50_IR_TEX_TARG_CASE(1D, 1D);
 422    NV50_IR_TEX_TARG_CASE(2D, 2D);
 423    NV50_IR_TEX_TARG_CASE(2D_MSAA, 2D_MS);
 424    NV50_IR_TEX_TARG_CASE(3D, 3D);
 425    NV50_IR_TEX_TARG_CASE(CUBE, CUBE);
 426    NV50_IR_TEX_TARG_CASE(RECT, RECT);
 427    NV50_IR_TEX_TARG_CASE(1D_ARRAY, 1D_ARRAY);
 428    NV50_IR_TEX_TARG_CASE(2D_ARRAY, 2D_ARRAY);
 429    NV50_IR_TEX_TARG_CASE(2D_ARRAY_MSAA, 2D_MS_ARRAY);
 430    NV50_IR_TEX_TARG_CASE(CUBE_ARRAY, CUBE_ARRAY);
 431    NV50_IR_TEX_TARG_CASE(SHADOW1D, 1D_SHADOW);
 432    NV50_IR_TEX_TARG_CASE(SHADOW2D, 2D_SHADOW);
 433    NV50_IR_TEX_TARG_CASE(SHADOWCUBE, CUBE_SHADOW);
 434    NV50_IR_TEX_TARG_CASE(SHADOWRECT, RECT_SHADOW);
 435    NV50_IR_TEX_TARG_CASE(SHADOW1D_ARRAY, 1D_ARRAY_SHADOW);
 436    NV50_IR_TEX_TARG_CASE(SHADOW2D_ARRAY, 2D_ARRAY_SHADOW);
 437    NV50_IR_TEX_TARG_CASE(SHADOWCUBE_ARRAY, CUBE_ARRAY_SHADOW);
 438    NV50_IR_TEX_TARG_CASE(BUFFER, BUFFER);
 439
 440    case TGSI_TEXTURE_UNKNOWN:
 441    default:
 442       assert(!"invalid texture target");
 443       return nv50_ir::TEX_TARGET_2D;
 444    }
 445 }
 446
 447 static nv50_ir::CacheMode translateCacheMode(uint qualifier)
 448 {
 449    if (qualifier & TGSI_MEMORY_VOLATILE)
 450       return nv50_ir::CACHE_CV;
 451    if (qualifier & TGSI_MEMORY_COHERENT)
 452       return nv50_ir::CACHE_CG;
 453    return nv50_ir::CACHE_CA;
 454 }
 455
 456 nv50_ir::DataType Instruction::inferSrcType() const
 457 {
 458    switch (getOpcode()) {
 459    case TGSI_OPCODE_UIF:
 460    case TGSI_OPCODE_AND:
 461    case TGSI_OPCODE_OR:
 462    case TGSI_OPCODE_XOR:
 463    case TGSI_OPCODE_NOT:
 464    case TGSI_OPCODE_SHL:
 465    case TGSI_OPCODE_U2F:
 466    case TGSI_OPCODE_U2D:
 467    case TGSI_OPCODE_UADD:
 468    case TGSI_OPCODE_UDIV:
 469    case TGSI_OPCODE_UMOD:
 470    case TGSI_OPCODE_UMAD:
 471    case TGSI_OPCODE_UMUL:
 472    case TGSI_OPCODE_UMUL_HI:
 473    case TGSI_OPCODE_UMAX:
 474    case TGSI_OPCODE_UMIN:
 475    case TGSI_OPCODE_USEQ:
 476    case TGSI_OPCODE_USGE:
 477    case TGSI_OPCODE_USLT:
 478    case TGSI_OPCODE_USNE:
 479    case TGSI_OPCODE_USHR:
 480    case TGSI_OPCODE_ATOMUADD:
 481    case TGSI_OPCODE_ATOMXCHG:
 482    case TGSI_OPCODE_ATOMCAS:
 483    case TGSI_OPCODE_ATOMAND:
 484    case TGSI_OPCODE_ATOMOR:
 485    case TGSI_OPCODE_ATOMXOR:
 486    case TGSI_OPCODE_ATOMUMIN:
 487    case TGSI_OPCODE_ATOMUMAX:
 488    case TGSI_OPCODE_UBFE:
 489    case TGSI_OPCODE_UMSB:
 490    case TGSI_OPCODE_UP2H:
 491       return nv50_ir::TYPE_U32;
 492    case TGSI_OPCODE_I2F:
 493    case TGSI_OPCODE_I2D:
 494    case TGSI_OPCODE_IDIV:
 495    case TGSI_OPCODE_IMUL_HI:
 496    case TGSI_OPCODE_IMAX:
 497    case TGSI_OPCODE_IMIN:
 498    case TGSI_OPCODE_IABS:
 499    case TGSI_OPCODE_INEG:
 500    case TGSI_OPCODE_ISGE:
 501    case TGSI_OPCODE_ISHR:
 502    case TGSI_OPCODE_ISLT:
 503    case TGSI_OPCODE_ISSG:
 504    case TGSI_OPCODE_SAD: // not sure about SAD, but no one has a float version
 505    case TGSI_OPCODE_MOD:
 506    case TGSI_OPCODE_UARL:
 507    case TGSI_OPCODE_ATOMIMIN:
 508    case TGSI_OPCODE_ATOMIMAX:
 509    case TGSI_OPCODE_IBFE:
 510    case TGSI_OPCODE_IMSB:
 511       return nv50_ir::TYPE_S32;
 512    case TGSI_OPCODE_D2F:
 513    case TGSI_OPCODE_D2I:
 514    case TGSI_OPCODE_D2U:
 515    case TGSI_OPCODE_DABS:
 516    case TGSI_OPCODE_DNEG:
 517    case TGSI_OPCODE_DADD:
 518    case TGSI_OPCODE_DMUL:
 519    case TGSI_OPCODE_DMAX:
 520    case TGSI_OPCODE_DMIN:
 521    case TGSI_OPCODE_DSLT:
 522    case TGSI_OPCODE_DSGE:
 523    case TGSI_OPCODE_DSEQ:
 524    case TGSI_OPCODE_DSNE:
 525    case TGSI_OPCODE_DRCP:
 526    case TGSI_OPCODE_DSQRT:
 527    case TGSI_OPCODE_DMAD:
 528    case TGSI_OPCODE_DFRAC:
 529    case TGSI_OPCODE_DRSQ:
 530    case TGSI_OPCODE_DTRUNC:
 531    case TGSI_OPCODE_DCEIL:
 532    case TGSI_OPCODE_DFLR:
 533    case TGSI_OPCODE_DROUND:
 534       return nv50_ir::TYPE_F64;
 535    default:
 536       return nv50_ir::TYPE_F32;
 537    }
 538 }
 539
 540 nv50_ir::DataType Instruction::inferDstType() const
 541 {
 542    switch (getOpcode()) {
 543    case TGSI_OPCODE_D2U:
 544    case TGSI_OPCODE_F2U: return nv50_ir::TYPE_U32;
 545    case TGSI_OPCODE_D2I:
 546    case TGSI_OPCODE_F2I: return nv50_ir::TYPE_S32;
 547    case TGSI_OPCODE_FSEQ:
 548    case TGSI_OPCODE_FSGE:
 549    case TGSI_OPCODE_FSLT:
 550    case TGSI_OPCODE_FSNE:
 551    case TGSI_OPCODE_DSEQ:
 552    case TGSI_OPCODE_DSGE:
 553    case TGSI_OPCODE_DSLT:
 554    case TGSI_OPCODE_DSNE:
 555    case TGSI_OPCODE_PK2H:
 556       return nv50_ir::TYPE_U32;
 557    case TGSI_OPCODE_I2F:
 558    case TGSI_OPCODE_U2F:
 559    case TGSI_OPCODE_D2F:
 560    case TGSI_OPCODE_UP2H:
 561       return nv50_ir::TYPE_F32;
 562    case TGSI_OPCODE_I2D:
 563    case TGSI_OPCODE_U2D:
 564    case TGSI_OPCODE_F2D:
 565       return nv50_ir::TYPE_F64;
 566    default:
 567       return inferSrcType();
 568    }
 569 }
 570
 571 nv50_ir::CondCode Instruction::getSetCond() const
 572 {
 573    using namespace nv50_ir;
 574
 575    switch (getOpcode()) {
 576    case TGSI_OPCODE_SLT:
 577    case TGSI_OPCODE_ISLT:
 578    case TGSI_OPCODE_USLT:
 579    case TGSI_OPCODE_FSLT:
 580    case TGSI_OPCODE_DSLT:
 581       return CC_LT;
 582    case TGSI_OPCODE_SLE:
 583       return CC_LE;
 584    case TGSI_OPCODE_SGE:
 585    case TGSI_OPCODE_ISGE:
 586    case TGSI_OPCODE_USGE:
 587    case TGSI_OPCODE_FSGE:
 588    case TGSI_OPCODE_DSGE:
 589       return CC_GE;
 590    case TGSI_OPCODE_SGT:
 591       return CC_GT;
 592    case TGSI_OPCODE_SEQ:
 593    case TGSI_OPCODE_USEQ:
 594    case TGSI_OPCODE_FSEQ:
 595    case TGSI_OPCODE_DSEQ:
 596       return CC_EQ;
 597    case TGSI_OPCODE_SNE:
 598    case TGSI_OPCODE_FSNE:
 599    case TGSI_OPCODE_DSNE:
 600       return CC_NEU;
 601    case TGSI_OPCODE_USNE:
 602       return CC_NE;
 603    default:
 604       return CC_ALWAYS;
 605    }
 606 }
 607
 608 #define NV50_IR_OPCODE_CASE(a, b) case TGSI_OPCODE_##a: return nv50_ir::OP_##b
 609
 610 static nv50_ir::operation translateOpcode(uint opcode)
 611 {
 612    switch (opcode) {
 613    NV50_IR_OPCODE_CASE(ARL, SHL);
 614    NV50_IR_OPCODE_CASE(MOV, MOV);
 615
 616    NV50_IR_OPCODE_CASE(RCP, RCP);
 617    NV50_IR_OPCODE_CASE(RSQ, RSQ);
 618
 619    NV50_IR_OPCODE_CASE(MUL, MUL);
 620    NV50_IR_OPCODE_CASE(ADD, ADD);
 621
 622    NV50_IR_OPCODE_CASE(MIN, MIN);
 623    NV50_IR_OPCODE_CASE(MAX, MAX);
 624    NV50_IR_OPCODE_CASE(SLT, SET);
 625    NV50_IR_OPCODE_CASE(SGE, SET);
 626    NV50_IR_OPCODE_CASE(MAD, MAD);
 627    NV50_IR_OPCODE_CASE(SUB, SUB);
 628
 629    NV50_IR_OPCODE_CASE(FLR, FLOOR);
 630    NV50_IR_OPCODE_CASE(ROUND, CVT);
 631    NV50_IR_OPCODE_CASE(EX2, EX2);
 632    NV50_IR_OPCODE_CASE(LG2, LG2);
 633    NV50_IR_OPCODE_CASE(POW, POW);
 634
 635    NV50_IR_OPCODE_CASE(ABS, ABS);
 636
 637    NV50_IR_OPCODE_CASE(COS, COS);
 638    NV50_IR_OPCODE_CASE(DDX, DFDX);
 639    NV50_IR_OPCODE_CASE(DDX_FINE, DFDX);
 640    NV50_IR_OPCODE_CASE(DDY, DFDY);
 641    NV50_IR_OPCODE_CASE(DDY_FINE, DFDY);
 642    NV50_IR_OPCODE_CASE(KILL, DISCARD);
 643
 644    NV50_IR_OPCODE_CASE(SEQ, SET);
 645    NV50_IR_OPCODE_CASE(SGT, SET);
 646    NV50_IR_OPCODE_CASE(SIN, SIN);
 647    NV50_IR_OPCODE_CASE(SLE, SET);
 648    NV50_IR_OPCODE_CASE(SNE, SET);
 649    NV50_IR_OPCODE_CASE(TEX, TEX);
 650    NV50_IR_OPCODE_CASE(TXD, TXD);
 651    NV50_IR_OPCODE_CASE(TXP, TEX);
 652
 653    NV50_IR_OPCODE_CASE(CAL, CALL);
 654    NV50_IR_OPCODE_CASE(RET, RET);
 655    NV50_IR_OPCODE_CASE(CMP, SLCT);
 656
 657    NV50_IR_OPCODE_CASE(TXB, TXB);
 658
 659    NV50_IR_OPCODE_CASE(DIV, DIV);
 660
 661    NV50_IR_OPCODE_CASE(TXL, TXL);
 662
 663    NV50_IR_OPCODE_CASE(CEIL, CEIL);
 664    NV50_IR_OPCODE_CASE(I2F, CVT);
 665    NV50_IR_OPCODE_CASE(NOT, NOT);
 666    NV50_IR_OPCODE_CASE(TRUNC, TRUNC);
 667    NV50_IR_OPCODE_CASE(SHL, SHL);
 668
 669    NV50_IR_OPCODE_CASE(AND, AND);
 670    NV50_IR_OPCODE_CASE(OR, OR);
 671    NV50_IR_OPCODE_CASE(MOD, MOD);
 672    NV50_IR_OPCODE_CASE(XOR, XOR);
 673    NV50_IR_OPCODE_CASE(SAD, SAD);
 674    NV50_IR_OPCODE_CASE(TXF, TXF);
 675    NV50_IR_OPCODE_CASE(TXQ, TXQ);
 676    NV50_IR_OPCODE_CASE(TXQS, TXQ);
 677    NV50_IR_OPCODE_CASE(TG4, TXG);
 678    NV50_IR_OPCODE_CASE(LODQ, TXLQ);
 679
 680    NV50_IR_OPCODE_CASE(EMIT, EMIT);
 681    NV50_IR_OPCODE_CASE(ENDPRIM, RESTART);
 682
 683    NV50_IR_OPCODE_CASE(KILL_IF, DISCARD);
 684
 685    NV50_IR_OPCODE_CASE(F2I, CVT);
 686    NV50_IR_OPCODE_CASE(FSEQ, SET);
 687    NV50_IR_OPCODE_CASE(FSGE, SET);
 688    NV50_IR_OPCODE_CASE(FSLT, SET);
 689    NV50_IR_OPCODE_CASE(FSNE, SET);
 690    NV50_IR_OPCODE_CASE(IDIV, DIV);
 691    NV50_IR_OPCODE_CASE(IMAX, MAX);
 692    NV50_IR_OPCODE_CASE(IMIN, MIN);
 693    NV50_IR_OPCODE_CASE(IABS, ABS);
 694    NV50_IR_OPCODE_CASE(INEG, NEG);
 695    NV50_IR_OPCODE_CASE(ISGE, SET);
 696    NV50_IR_OPCODE_CASE(ISHR, SHR);
 697    NV50_IR_OPCODE_CASE(ISLT, SET);
 698    NV50_IR_OPCODE_CASE(F2U, CVT);
 699    NV50_IR_OPCODE_CASE(U2F, CVT);
 700    NV50_IR_OPCODE_CASE(UADD, ADD);
 701    NV50_IR_OPCODE_CASE(UDIV, DIV);
 702    NV50_IR_OPCODE_CASE(UMAD, MAD);
 703    NV50_IR_OPCODE_CASE(UMAX, MAX);
 704    NV50_IR_OPCODE_CASE(UMIN, MIN);
 705    NV50_IR_OPCODE_CASE(UMOD, MOD);
 706    NV50_IR_OPCODE_CASE(UMUL, MUL);
 707    NV50_IR_OPCODE_CASE(USEQ, SET);
 708    NV50_IR_OPCODE_CASE(USGE, SET);
 709    NV50_IR_OPCODE_CASE(USHR, SHR);
 710    NV50_IR_OPCODE_CASE(USLT, SET);
 711    NV50_IR_OPCODE_CASE(USNE, SET);
 712
 713    NV50_IR_OPCODE_CASE(DABS, ABS);
 714    NV50_IR_OPCODE_CASE(DNEG, NEG);
 715    NV50_IR_OPCODE_CASE(DADD, ADD);
 716    NV50_IR_OPCODE_CASE(DMUL, MUL);
 717    NV50_IR_OPCODE_CASE(DMAX, MAX);
 718    NV50_IR_OPCODE_CASE(DMIN, MIN);
 719    NV50_IR_OPCODE_CASE(DSLT, SET);
 720    NV50_IR_OPCODE_CASE(DSGE, SET);
 721    NV50_IR_OPCODE_CASE(DSEQ, SET);
 722    NV50_IR_OPCODE_CASE(DSNE, SET);
 723    NV50_IR_OPCODE_CASE(DRCP, RCP);
 724    NV50_IR_OPCODE_CASE(DSQRT, SQRT);
 725    NV50_IR_OPCODE_CASE(DMAD, MAD);
 726    NV50_IR_OPCODE_CASE(D2I, CVT);
 727    NV50_IR_OPCODE_CASE(D2U, CVT);
 728    NV50_IR_OPCODE_CASE(I2D, CVT);
 729    NV50_IR_OPCODE_CASE(U2D, CVT);
 730    NV50_IR_OPCODE_CASE(DRSQ, RSQ);
 731    NV50_IR_OPCODE_CASE(DTRUNC, TRUNC);
 732    NV50_IR_OPCODE_CASE(DCEIL, CEIL);
 733    NV50_IR_OPCODE_CASE(DFLR, FLOOR);
 734    NV50_IR_OPCODE_CASE(DROUND, CVT);
 735
 736    NV50_IR_OPCODE_CASE(IMUL_HI, MUL);
 737    NV50_IR_OPCODE_CASE(UMUL_HI, MUL);
 738
 739    NV50_IR_OPCODE_CASE(SAMPLE, TEX);
 740    NV50_IR_OPCODE_CASE(SAMPLE_B, TXB);
 741    NV50_IR_OPCODE_CASE(SAMPLE_C, TEX);
 742    NV50_IR_OPCODE_CASE(SAMPLE_C_LZ, TEX);
 743    NV50_IR_OPCODE_CASE(SAMPLE_D, TXD);
 744    NV50_IR_OPCODE_CASE(SAMPLE_L, TXL);
 745    NV50_IR_OPCODE_CASE(SAMPLE_I, TXF);
 746    NV50_IR_OPCODE_CASE(SAMPLE_I_MS, TXF);
 747    NV50_IR_OPCODE_CASE(GATHER4, TXG);
 748    NV50_IR_OPCODE_CASE(SVIEWINFO, TXQ);
 749
 750    NV50_IR_OPCODE_CASE(ATOMUADD, ATOM);
 751    NV50_IR_OPCODE_CASE(ATOMXCHG, ATOM);
 752    NV50_IR_OPCODE_CASE(ATOMCAS, ATOM);
 753    NV50_IR_OPCODE_CASE(ATOMAND, ATOM);
 754    NV50_IR_OPCODE_CASE(ATOMOR, ATOM);
 755    NV50_IR_OPCODE_CASE(ATOMXOR, ATOM);
 756    NV50_IR_OPCODE_CASE(ATOMUMIN, ATOM);
 757    NV50_IR_OPCODE_CASE(ATOMUMAX, ATOM);
 758    NV50_IR_OPCODE_CASE(ATOMIMIN, ATOM);
 759    NV50_IR_OPCODE_CASE(ATOMIMAX, ATOM);
 760
 761    NV50_IR_OPCODE_CASE(TEX2, TEX);
 762    NV50_IR_OPCODE_CASE(TXB2, TXB);
 763    NV50_IR_OPCODE_CASE(TXL2, TXL);
 764
 765    NV50_IR_OPCODE_CASE(IBFE, EXTBF);
 766    NV50_IR_OPCODE_CASE(UBFE, EXTBF);
 767    NV50_IR_OPCODE_CASE(BFI, INSBF);
 768    NV50_IR_OPCODE_CASE(BREV, EXTBF);
 769    NV50_IR_OPCODE_CASE(POPC, POPCNT);
 770    NV50_IR_OPCODE_CASE(LSB, BFIND);
 771    NV50_IR_OPCODE_CASE(IMSB, BFIND);
 772    NV50_IR_OPCODE_CASE(UMSB, BFIND);
 773
 774    NV50_IR_OPCODE_CASE(END, EXIT);
 775
 776    default:
 777       return nv50_ir::OP_NOP;
 778    }
 779 }
 780
 781 static uint16_t opcodeToSubOp(uint opcode)
 782 {
 783    switch (opcode) {
 784    case TGSI_OPCODE_LFENCE:   return NV50_IR_SUBOP_MEMBAR(L, GL);
 785    case TGSI_OPCODE_SFENCE:   return NV50_IR_SUBOP_MEMBAR(S, GL);
 786    case TGSI_OPCODE_MFENCE:   return NV50_IR_SUBOP_MEMBAR(M, GL);
 787    case TGSI_OPCODE_ATOMUADD: return NV50_IR_SUBOP_ATOM_ADD;
 788    case TGSI_OPCODE_ATOMXCHG: return NV50_IR_SUBOP_ATOM_EXCH;
 789    case TGSI_OPCODE_ATOMCAS:  return NV50_IR_SUBOP_ATOM_CAS;
 790    case TGSI_OPCODE_ATOMAND:  return NV50_IR_SUBOP_ATOM_AND;
 791    case TGSI_OPCODE_ATOMOR:   return NV50_IR_SUBOP_ATOM_OR;
 792    case TGSI_OPCODE_ATOMXOR:  return NV50_IR_SUBOP_ATOM_XOR;
 793    case TGSI_OPCODE_ATOMUMIN: return NV50_IR_SUBOP_ATOM_MIN;
 794    case TGSI_OPCODE_ATOMIMIN: return NV50_IR_SUBOP_ATOM_MIN;
 795    case TGSI_OPCODE_ATOMUMAX: return NV50_IR_SUBOP_ATOM_MAX;
 796    case TGSI_OPCODE_ATOMIMAX: return NV50_IR_SUBOP_ATOM_MAX;
 797    case TGSI_OPCODE_IMUL_HI:
 798    case TGSI_OPCODE_UMUL_HI:
 799       return NV50_IR_SUBOP_MUL_HIGH;
 800    default:
 801       return 0;
 802    }
 803 }
 804
 805 bool Instruction::checkDstSrcAliasing() const
 806 {
 807    if (insn->Dst[0].Register.Indirect) // no danger if indirect, using memory
 808       return false;
 809
 810    for (int s = 0; s < TGSI_FULL_MAX_SRC_REGISTERS; ++s) {
 811       if (insn->Src[s].Register.File == TGSI_FILE_NULL)
 812          break;
 813       if (insn->Src[s].Register.File == insn->Dst[0].Register.File &&
 814           insn->Src[s].Register.Index == insn->Dst[0].Register.Index)
 815          return true;
 816    }
 817    return false;
 818 }
 819
 820 class Source
 821 {
 822 public:
 823    Source(struct nv50_ir_prog_info *);
 824    ~Source();
 825
 826 public:
 827    bool scanSource();
 828    unsigned fileSize(unsigned file) const { return scan.file_max[file] + 1; }
 829
 830 public:
 831    struct tgsi_shader_info scan;
 832    struct tgsi_full_instruction *insns;
 833    const struct tgsi_token *tokens;
 834    struct nv50_ir_prog_info *info;
 835
 836    nv50_ir::DynArray tempArrays;
 837    nv50_ir::DynArray immdArrays;
 838
 839    typedef nv50_ir::BuildUtil::Location Location;
 840    // these registers are per-subroutine, cannot be used for parameter passing
 841    std::set<Location> locals;
 842
 843    std::set<int> indirectTempArrays;
 844    std::map<int, int> indirectTempOffsets;
 845    std::map<int, std::pair<int, int> > tempArrayInfo;
 846    std::vector<int> tempArrayId;
 847
 848    int clipVertexOutput;
 849
 850    struct TextureView {
 851       uint8_t target; // TGSI_TEXTURE_*
 852    };
 853    std::vector<TextureView> textureViews;
 854
 855    struct Resource {
 856       uint8_t target; // TGSI_TEXTURE_*
 857       bool raw;
 858       uint8_t slot; // $surface index
 859    };
 860    std::vector<Resource> resources;
 861
 862    struct MemoryFile {
 863       bool shared;
 864    };
 865    std::vector<MemoryFile> memoryFiles;
 866
 867 private:
 868    int inferSysValDirection(unsigned sn) const;
 869    bool scanDeclaration(const struct tgsi_full_declaration *);
 870    bool scanInstruction(const struct tgsi_full_instruction *);
 871    void scanProperty(const struct tgsi_full_property *);
 872    void scanImmediate(const struct tgsi_full_immediate *);
 873
 874    inline bool isEdgeFlagPassthrough(const Instruction&) const;
 875 };
 876
 877 Source::Source(struct nv50_ir_prog_info *prog) : info(prog)
 878 {
 879    tokens = (const struct tgsi_token *)info->bin.source;
 880
 881    if (prog->dbgFlags & NV50_IR_DEBUG_BASIC)
 882       tgsi_dump(tokens, 0);
 883 }
 884
 885 Source::~Source()
 886 {
 887    if (insns)
 888       FREE(insns);
 889
 890    if (info->immd.data)
 891       FREE(info->immd.data);
 892    if (info->immd.type)
 893       FREE(info->immd.type);
 894 }
 895
 896 bool Source::scanSource()
 897 {
 898    unsigned insnCount = 0;
 899    struct tgsi_parse_context parse;
 900
 901    tgsi_scan_shader(tokens, &scan);
 902
 903    insns = (struct tgsi_full_instruction *)MALLOC(scan.num_instructions *
 904                                                   sizeof(insns[0]));
 905    if (!insns)
 906       return false;
 907
 908    clipVertexOutput = -1;
 909
 910    textureViews.resize(scan.file_max[TGSI_FILE_SAMPLER_VIEW] + 1);
 911    //resources.resize(scan.file_max[TGSI_FILE_RESOURCE] + 1);
 912    tempArrayId.resize(scan.file_max[TGSI_FILE_TEMPORARY] + 1);
 913    memoryFiles.resize(scan.file_max[TGSI_FILE_MEMORY] + 1);
 914
 915    info->immd.bufSize = 0;
 916
 917    info->numInputs = scan.file_max[TGSI_FILE_INPUT] + 1;
 918    info->numOutputs = scan.file_max[TGSI_FILE_OUTPUT] + 1;
 919    info->numSysVals = scan.file_max[TGSI_FILE_SYSTEM_VALUE] + 1;
 920
 921    if (info->type == PIPE_SHADER_FRAGMENT) {
 922       info->prop.fp.writesDepth = scan.writes_z;
 923       info->prop.fp.usesDiscard = scan.uses_kill;
 924    } else
 925    if (info->type == PIPE_SHADER_GEOMETRY) {
 926       info->prop.gp.instanceCount = 1; // default value
 927    }
 928
 929    info->io.viewportId = -1;
 930
 931    info->immd.data = (uint32_t *)MALLOC(scan.immediate_count * 16);
 932    info->immd.type = (ubyte *)MALLOC(scan.immediate_count * sizeof(ubyte));
 933
 934    tgsi_parse_init(&parse, tokens);
 935    while (!tgsi_parse_end_of_tokens(&parse)) {
 936       tgsi_parse_token(&parse);
 937
 938       switch (parse.FullToken.Token.Type) {
 939       case TGSI_TOKEN_TYPE_IMMEDIATE:
 940          scanImmediate(&parse.FullToken.FullImmediate);
 941          break;
 942       case TGSI_TOKEN_TYPE_DECLARATION:
 943          scanDeclaration(&parse.FullToken.FullDeclaration);
 944          break;
 945       case TGSI_TOKEN_TYPE_INSTRUCTION:
 946          insns[insnCount++] = parse.FullToken.FullInstruction;
 947          scanInstruction(&parse.FullToken.FullInstruction);
 948          break;
 949       case TGSI_TOKEN_TYPE_PROPERTY:
 950          scanProperty(&parse.FullToken.FullProperty);
 951          break;
 952       default:
 953          INFO("unknown TGSI token type: %d\n", parse.FullToken.Token.Type);
 954          break;
 955       }
 956    }
 957    tgsi_parse_free(&parse);
 958
 959    if (indirectTempArrays.size()) {
 960       int tempBase = 0;
 961       for (std::set<int>::const_iterator it = indirectTempArrays.begin();
 962            it != indirectTempArrays.end(); ++it) {
 963          std::pair<int, int>& info = tempArrayInfo[*it];
 964          indirectTempOffsets.insert(std::make_pair(*it, tempBase - info.first));
 965          tempBase += info.second;
 966       }
 967       info->bin.tlsSpace += tempBase * 16;
 968    }
 969
 970    if (info->io.genUserClip > 0) {
 971       info->io.clipDistances = info->io.genUserClip;
 972
 973       const unsigned int nOut = (info->io.genUserClip + 3) / 4;
 974
 975       for (unsigned int n = 0; n < nOut; ++n) {
 976          unsigned int i = info->numOutputs++;
 977          info->out[i].id = i;
 978          info->out[i].sn = TGSI_SEMANTIC_CLIPDIST;
 979          info->out[i].si = n;
 980          info->out[i].mask = ((1 << info->io.clipDistances) - 1) >> (n * 4);
 981       }
 982    }
 983
 984    return info->assignSlots(info) == 0;
 985 }
 986
 987 void Source::scanProperty(const struct tgsi_full_property *prop)
 988 {
 989    switch (prop->Property.PropertyName) {
 990    case TGSI_PROPERTY_GS_OUTPUT_PRIM:
 991       info->prop.gp.outputPrim = prop->u[0].Data;
 992       break;
 993    case TGSI_PROPERTY_GS_INPUT_PRIM:
 994       info->prop.gp.inputPrim = prop->u[0].Data;
 995       break;
 996    case TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES:
 997       info->prop.gp.maxVertices = prop->u[0].Data;
 998       break;
 999    case TGSI_PROPERTY_GS_INVOCATIONS:
1000       info->prop.gp.instanceCount = prop->u[0].Data;
1001       break;
1002    case TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS:
1003       info->prop.fp.separateFragData = true;
1004       break;
1005    case TGSI_PROPERTY_FS_COORD_ORIGIN:
1006    case TGSI_PROPERTY_FS_COORD_PIXEL_CENTER:
1007       // we don't care
1008       break;
1009    case TGSI_PROPERTY_VS_PROHIBIT_UCPS:
1010       info->io.genUserClip = -1;
1011       break;
1012    case TGSI_PROPERTY_TCS_VERTICES_OUT:
1013       info->prop.tp.outputPatchSize = prop->u[0].Data;
1014       break;
1015    case TGSI_PROPERTY_TES_PRIM_MODE:
1016       info->prop.tp.domain = prop->u[0].Data;
1017       break;
1018    case TGSI_PROPERTY_TES_SPACING:
1019       info->prop.tp.partitioning = prop->u[0].Data;
1020       break;
1021    case TGSI_PROPERTY_TES_VERTEX_ORDER_CW:
1022       info->prop.tp.winding = prop->u[0].Data;
1023       break;
1024    case TGSI_PROPERTY_TES_POINT_MODE:
1025       if (prop->u[0].Data)
1026          info->prop.tp.outputPrim = PIPE_PRIM_POINTS;
1027       else
1028          info->prop.tp.outputPrim = PIPE_PRIM_TRIANGLES; /* anything but points */
1029       break;
1030    case TGSI_PROPERTY_NUM_CLIPDIST_ENABLED:
1031       info->io.clipDistances = prop->u[0].Data;
1032       break;
1033    case TGSI_PROPERTY_NUM_CULLDIST_ENABLED:
1034       info->io.cullDistances = prop->u[0].Data;
1035       break;
1036    default:
1037       INFO("unhandled TGSI property %d\n", prop->Property.PropertyName);
1038       break;
1039    }
1040 }
1041
1042 void Source::scanImmediate(const struct tgsi_full_immediate *imm)
1043 {
1044    const unsigned n = info->immd.count++;
1045
1046    assert(n < scan.immediate_count);
1047
1048    for (int c = 0; c < 4; ++c)
1049       info->immd.data[n * 4 + c] = imm->u[c].Uint;
1050
1051    info->immd.type[n] = imm->Immediate.DataType;
1052 }
1053
1054 int Source::inferSysValDirection(unsigned sn) const
1055 {
1056    switch (sn) {
1057    case TGSI_SEMANTIC_INSTANCEID:
1058    case TGSI_SEMANTIC_VERTEXID:
1059       return 1;
1060    case TGSI_SEMANTIC_LAYER:
1061 #if 0
1062    case TGSI_SEMANTIC_VIEWPORTINDEX:
1063       return 0;
1064 #endif
1065    case TGSI_SEMANTIC_PRIMID:
1066       return (info->type == PIPE_SHADER_FRAGMENT) ? 1 : 0;
1067    default:
1068       return 0;
1069    }
1070 }
1071
1072 bool Source::scanDeclaration(const struct tgsi_full_declaration *decl)
1073 {
1074    unsigned i, c;
1075    unsigned sn = TGSI_SEMANTIC_GENERIC;
1076    unsigned si = 0;
1077    const unsigned first = decl->Range.First, last = decl->Range.Last;
1078    const int arrayId = decl->Array.ArrayID;
1079
1080    if (decl->Declaration.Semantic) {
1081       sn = decl->Semantic.Name;
1082       si = decl->Semantic.Index;
1083    }
1084
1085    if (decl->Declaration.Local) {
1086       for (i = first; i <= last; ++i) {
1087          for (c = 0; c < 4; ++c) {
1088             locals.insert(
1089                Location(decl->Declaration.File, decl->Dim.Index2D, i, c));
1090          }
1091       }
1092    }
1093
1094    switch (decl->Declaration.File) {
1095    case TGSI_FILE_INPUT:
1096       if (info->type == PIPE_SHADER_VERTEX) {
1097          // all vertex attributes are equal
1098          for (i = first; i <= last; ++i) {
1099             info->in[i].sn = TGSI_SEMANTIC_GENERIC;
1100             info->in[i].si = i;
1101          }
1102       } else {
1103          for (i = first; i <= last; ++i, ++si) {
1104             info->in[i].id = i;
1105             info->in[i].sn = sn;
1106             info->in[i].si = si;
1107             if (info->type == PIPE_SHADER_FRAGMENT) {
1108                // translate interpolation mode
1109                switch (decl->Interp.Interpolate) {
1110                case TGSI_INTERPOLATE_CONSTANT:
1111                   info->in[i].flat = 1;
1112                   break;
1113                case TGSI_INTERPOLATE_COLOR:
1114                   info->in[i].sc = 1;
1115                   break;
1116                case TGSI_INTERPOLATE_LINEAR:
1117                   info->in[i].linear = 1;
1118                   break;
1119                default:
1120                   break;
1121                }
1122                if (decl->Interp.Location)
1123                   info->in[i].centroid = 1;
1124             }
1125
1126             if (sn == TGSI_SEMANTIC_PATCH)
1127                info->in[i].patch = 1;
1128             if (sn == TGSI_SEMANTIC_PATCH)
1129                info->numPatchConstants = MAX2(info->numPatchConstants, si + 1);
1130          }
1131       }
1132       break;
1133    case TGSI_FILE_OUTPUT:
1134       for (i = first; i <= last; ++i, ++si) {
1135          switch (sn) {
1136          case TGSI_SEMANTIC_POSITION:
1137             if (info->type == PIPE_SHADER_FRAGMENT)
1138                info->io.fragDepth = i;
1139             else
1140             if (clipVertexOutput < 0)
1141                clipVertexOutput = i;
1142             break;
1143          case TGSI_SEMANTIC_COLOR:
1144             if (info->type == PIPE_SHADER_FRAGMENT)
1145                info->prop.fp.numColourResults++;
1146             break;
1147          case TGSI_SEMANTIC_EDGEFLAG:
1148             info->io.edgeFlagOut = i;
1149             break;
1150          case TGSI_SEMANTIC_CLIPVERTEX:
1151             clipVertexOutput = i;
1152             break;
1153          case TGSI_SEMANTIC_CLIPDIST:
1154             info->io.genUserClip = -1;
1155             break;
1156          case TGSI_SEMANTIC_SAMPLEMASK:
1157             info->io.sampleMask = i;
1158             break;
1159          case TGSI_SEMANTIC_VIEWPORT_INDEX:
1160             info->io.viewportId = i;
1161             break;
1162          case TGSI_SEMANTIC_PATCH:
1163             info->numPatchConstants = MAX2(info->numPatchConstants, si + 1);
1164             /* fallthrough */
1165          case TGSI_SEMANTIC_TESSOUTER:
1166          case TGSI_SEMANTIC_TESSINNER:
1167             info->out[i].patch = 1;
1168             break;
1169          default:
1170             break;
1171          }
1172          info->out[i].id = i;
1173          info->out[i].sn = sn;
1174          info->out[i].si = si;
1175       }
1176       break;
1177    case TGSI_FILE_SYSTEM_VALUE:
1178       switch (sn) {
1179       case TGSI_SEMANTIC_INSTANCEID:
1180          info->io.instanceId = first;
1181          break;
1182       case TGSI_SEMANTIC_VERTEXID:
1183          info->io.vertexId = first;
1184          break;
1185       case TGSI_SEMANTIC_BASEVERTEX:
1186       case TGSI_SEMANTIC_BASEINSTANCE:
1187       case TGSI_SEMANTIC_DRAWID:
1188          info->prop.vp.usesDrawParameters = true;
1189          break;
1190       default:
1191          break;
1192       }
1193       for (i = first; i <= last; ++i, ++si) {
1194          info->sv[i].sn = sn;
1195          info->sv[i].si = si;
1196          info->sv[i].input = inferSysValDirection(sn);
1197
1198          switch (sn) {
1199          case TGSI_SEMANTIC_TESSOUTER:
1200          case TGSI_SEMANTIC_TESSINNER:
1201             info->sv[i].patch = 1;
1202             break;
1203          }
1204       }
1205       break;
1206 /*
1207    case TGSI_FILE_RESOURCE:
1208       for (i = first; i <= last; ++i) {
1209          resources[i].target = decl->Resource.Resource;
1210          resources[i].raw = decl->Resource.Raw;
1211          resources[i].slot = i;
1212       }
1213       break;
1214 */
1215    case TGSI_FILE_SAMPLER_VIEW:
1216       for (i = first; i <= last; ++i)
1217          textureViews[i].target = decl->SamplerView.Resource;
1218       break;
1219    case TGSI_FILE_MEMORY:
1220       for (i = first; i <= last; ++i)
1221          memoryFiles[i].shared = decl->Declaration.Shared;
1222       break;
1223    case TGSI_FILE_NULL:
1224    case TGSI_FILE_TEMPORARY:
1225       for (i = first; i <= last; ++i)
1226          tempArrayId[i] = arrayId;
1227       if (arrayId)
1228          tempArrayInfo.insert(std::make_pair(arrayId, std::make_pair(
1229                                                    first, last - first + 1)));
1230       break;
1231    case TGSI_FILE_ADDRESS:
1232    case TGSI_FILE_CONSTANT:
1233    case TGSI_FILE_IMMEDIATE:
1234    case TGSI_FILE_PREDICATE:
1235    case TGSI_FILE_SAMPLER:
1236    case TGSI_FILE_BUFFER:
1237       break;
1238    default:
1239       ERROR("unhandled TGSI_FILE %d\n", decl->Declaration.File);
1240       return false;
1241    }
1242    return true;
1243 }
1244
1245 inline bool Source::isEdgeFlagPassthrough(const Instruction& insn) const
1246 {
1247    return insn.getOpcode() == TGSI_OPCODE_MOV &&
1248       insn.getDst(0).getIndex(0) == info->io.edgeFlagOut &&
1249       insn.getSrc(0).getFile() == TGSI_FILE_INPUT;
1250 }
1251
1252 bool Source::scanInstruction(const struct tgsi_full_instruction *inst)
1253 {
1254    Instruction insn(inst);
1255
1256    if (insn.getOpcode() == TGSI_OPCODE_BARRIER)
1257       info->numBarriers = 1;
1258
1259    if (insn.dstCount()) {
1260       if (insn.getDst(0).getFile() == TGSI_FILE_OUTPUT) {
1261          Instruction::DstRegister dst = insn.getDst(0);
1262
1263          if (dst.isIndirect(0))
1264             for (unsigned i = 0; i < info->numOutputs; ++i)
1265                info->out[i].mask = 0xf;
1266          else
1267             info->out[dst.getIndex(0)].mask |= dst.getMask();
1268
1269          if (info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PSIZE ||
1270              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PRIMID ||
1271              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_LAYER ||
1272              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_VIEWPORT_INDEX ||
1273              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_FOG)
1274             info->out[dst.getIndex(0)].mask &= 1;
1275
1276          if (isEdgeFlagPassthrough(insn))
1277             info->io.edgeFlagIn = insn.getSrc(0).getIndex(0);
1278       } else
1279       if (insn.getDst(0).getFile() == TGSI_FILE_TEMPORARY) {
1280          if (insn.getDst(0).isIndirect(0))
1281             indirectTempArrays.insert(insn.getDst(0).getArrayId());
1282       } else
1283       if (insn.getDst(0).getFile() == TGSI_FILE_BUFFER) {
1284          info->io.globalAccess |= 0x2;
1285       }
1286    }
1287
1288    for (unsigned s = 0; s < insn.srcCount(); ++s) {
1289       Instruction::SrcRegister src = insn.getSrc(s);
1290       if (src.getFile() == TGSI_FILE_TEMPORARY) {
1291          if (src.isIndirect(0))
1292             indirectTempArrays.insert(src.getArrayId());
1293       } else
1294       if (src.getFile() == TGSI_FILE_BUFFER) {
1295          info->io.globalAccess |= (insn.getOpcode() == TGSI_OPCODE_LOAD) ?
1296                0x1 : 0x2;
1297       } else
1298       if (src.getFile() == TGSI_FILE_OUTPUT) {
1299          if (src.isIndirect(0)) {
1300             // We don't know which one is accessed, just mark everything for
1301             // reading. This is an extremely unlikely occurrence.
1302             for (unsigned i = 0; i < info->numOutputs; ++i)
1303                info->out[i].oread = 1;
1304          } else {
1305             info->out[src.getIndex(0)].oread = 1;
1306          }
1307       }
1308       if (src.getFile() != TGSI_FILE_INPUT)
1309          continue;
1310       unsigned mask = insn.srcMask(s);
1311
1312       if (src.isIndirect(0)) {
1313          for (unsigned i = 0; i < info->numInputs; ++i)
1314             info->in[i].mask = 0xf;
1315       } else {
1316          const int i = src.getIndex(0);
1317          for (unsigned c = 0; c < 4; ++c) {
1318             if (!(mask & (1 << c)))
1319                continue;
1320             int k = src.getSwizzle(c);
1321             if (k <= TGSI_SWIZZLE_W)
1322                info->in[i].mask |= 1 << k;
1323          }
1324          switch (info->in[i].sn) {
1325          case TGSI_SEMANTIC_PSIZE:
1326          case TGSI_SEMANTIC_PRIMID:
1327          case TGSI_SEMANTIC_FOG:
1328             info->in[i].mask &= 0x1;
1329             break;
1330          case TGSI_SEMANTIC_PCOORD:
1331             info->in[i].mask &= 0x3;
1332             break;
1333          default:
1334             break;
1335          }
1336       }
1337    }
1338    return true;
1339 }
1340
1341 nv50_ir::TexInstruction::Target
1342 Instruction::getTexture(const tgsi::Source *code, int s) const
1343 {
1344    // XXX: indirect access
1345    unsigned int r;
1346
1347    switch (getSrc(s).getFile()) {
1348 /*
1349    case TGSI_FILE_RESOURCE:
1350       r = getSrc(s).getIndex(0);
1351       return translateTexture(code->resources.at(r).target);
1352 */
1353    case TGSI_FILE_SAMPLER_VIEW:
1354       r = getSrc(s).getIndex(0);
1355       return translateTexture(code->textureViews.at(r).target);
1356    default:
1357       return translateTexture(insn->Texture.Texture);
1358    }
1359 }
1360
1361 } // namespace tgsi
1362
1363 namespace {
1364
1365 using namespace nv50_ir;
1366
1367 class Converter : public BuildUtil
1368 {
1369 public:
1370    Converter(Program *, const tgsi::Source *);
1371    ~Converter();
1372
1373    bool run();
1374
1375 private:
1376    struct Subroutine
1377    {
1378       Subroutine(Function *f) : f(f) { }
1379       Function *f;
1380       ValueMap values;
1381    };
1382
1383    Value *shiftAddress(Value *);
1384    Value *getVertexBase(int s);
1385    Value *getOutputBase(int s);
1386    DataArray *getArrayForFile(unsigned file, int idx);
1387    Value *fetchSrc(int s, int c);
1388    Value *acquireDst(int d, int c);
1389    void storeDst(int d, int c, Value *);
1390
1391    Value *fetchSrc(const tgsi::Instruction::SrcRegister src, int c, Value *ptr);
1392    void storeDst(const tgsi::Instruction::DstRegister dst, int c,
1393                  Value *val, Value *ptr);
1394
1395    void adjustTempIndex(int arrayId, int &idx, int &idx2d) const;
1396    Value *applySrcMod(Value *, int s, int c);
1397
1398    Symbol *makeSym(uint file, int fileIndex, int idx, int c, uint32_t addr);
1399    Symbol *srcToSym(tgsi::Instruction::SrcRegister, int c);
1400    Symbol *dstToSym(tgsi::Instruction::DstRegister, int c);
1401
1402    bool handleInstruction(const struct tgsi_full_instruction *);
1403    void exportOutputs();
1404    inline Subroutine *getSubroutine(unsigned ip);
1405    inline Subroutine *getSubroutine(Function *);
1406    inline bool isEndOfSubroutine(uint ip);
1407
1408    void loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask);
1409
1410    // R,S,L,C,Dx,Dy encode TGSI sources for respective values (0xSf for auto)
1411    void setTexRS(TexInstruction *, unsigned int& s, int R, int S);
1412    void handleTEX(Value *dst0[4], int R, int S, int L, int C, int Dx, int Dy);
1413    void handleTXF(Value *dst0[4], int R, int L_M);
1414    void handleTXQ(Value *dst0[4], enum TexQuery, int R);
1415    void handleLIT(Value *dst0[4]);
1416    void handleUserClipPlanes();
1417
1418    Symbol *getResourceBase(int r);
1419    void getResourceCoords(std::vector<Value *>&, int r, int s);
1420
1421    void handleLOAD(Value *dst0[4]);
1422    void handleSTORE();
1423    void handleATOM(Value *dst0[4], DataType, uint16_t subOp);
1424
1425    void handleINTERP(Value *dst0[4]);
1426
1427    uint8_t translateInterpMode(const struct nv50_ir_varying *var,
1428                                operation& op);
1429    Value *interpolate(tgsi::Instruction::SrcRegister, int c, Value *ptr);
1430
1431    void insertConvergenceOps(BasicBlock *conv, BasicBlock *fork);
1432
1433    Value *buildDot(int dim);
1434
1435    class BindArgumentsPass : public Pass {
1436    public:
1437       BindArgumentsPass(Converter &conv) : conv(conv) { }
1438
1439    private:
1440       Converter &conv;
1441       Subroutine *sub;
1442
1443       inline const Location *getValueLocation(Subroutine *, Value *);
1444
1445       template<typename T> inline void
1446       updateCallArgs(Instruction *i, void (Instruction::*setArg)(int, Value *),
1447                      T (Function::*proto));
1448
1449       template<typename T> inline void
1450       updatePrototype(BitSet *set, void (Function::*updateSet)(),
1451                       T (Function::*proto));
1452
1453    protected:
1454       bool visit(Function *);
1455       bool visit(BasicBlock *bb) { return false; }
1456    };
1457
1458 private:
1459    const tgsi::Source *code;
1460    const struct nv50_ir_prog_info *info;
1461
1462    struct {
1463       std::map<unsigned, Subroutine> map;
1464       Subroutine *cur;
1465    } sub;
1466
1467    uint ip; // instruction pointer
1468
1469    tgsi::Instruction tgsi;
1470
1471    DataType dstTy;
1472    DataType srcTy;
1473
1474    DataArray tData; // TGSI_FILE_TEMPORARY
1475    DataArray lData; // TGSI_FILE_TEMPORARY, for indirect arrays
1476    DataArray aData; // TGSI_FILE_ADDRESS
1477    DataArray pData; // TGSI_FILE_PREDICATE
1478    DataArray oData; // TGSI_FILE_OUTPUT (if outputs in registers)
1479
1480    Value *zero;
1481    Value *fragCoord[4];
1482    Value *clipVtx[4];
1483
1484    Value *vtxBase[5]; // base address of vertex in primitive (for TP/GP)
1485    uint8_t vtxBaseValid;
1486
1487    Value *outBase; // base address of vertex out patch (for TCP)
1488
1489    Stack condBBs;  // fork BB, then else clause BB
1490    Stack joinBBs;  // fork BB, for inserting join ops on ENDIF
1491    Stack loopBBs;  // loop headers
1492    Stack breakBBs; // end of / after loop
1493
1494    Value *viewport;
1495 };
1496
1497 Symbol *
1498 Converter::srcToSym(tgsi::Instruction::SrcRegister src, int c)
1499 {
1500    const int swz = src.getSwizzle(c);
1501
1502    /* TODO: Use Array ID when it's available for the index */
1503    return makeSym(src.getFile(),
1504                   src.is2D() ? src.getIndex(1) : 0,
1505                   src.getIndex(0), swz,
1506                   src.getIndex(0) * 16 + swz * 4);
1507 }
1508
1509 Symbol *
1510 Converter::dstToSym(tgsi::Instruction::DstRegister dst, int c)
1511 {
1512    /* TODO: Use Array ID when it's available for the index */
1513    return makeSym(dst.getFile(),
1514                   dst.is2D() ? dst.getIndex(1) : 0,
1515                   dst.getIndex(0), c,
1516                   dst.getIndex(0) * 16 + c * 4);
1517 }
1518
1519 Symbol *
1520 Converter::makeSym(uint tgsiFile, int fileIdx, int idx, int c, uint32_t address)
1521 {
1522    Symbol *sym = new_Symbol(prog, tgsi::translateFile(tgsiFile));
1523
1524    sym->reg.fileIndex = fileIdx;
1525
1526    if (tgsiFile == TGSI_FILE_MEMORY && code->memoryFiles[fileIdx].shared)
1527       sym->setFile(FILE_MEMORY_SHARED);
1528
1529    if (idx >= 0) {
1530       if (sym->reg.file == FILE_SHADER_INPUT)
1531          sym->setOffset(info->in[idx].slot[c] * 4);
1532       else
1533       if (sym->reg.file == FILE_SHADER_OUTPUT)
1534          sym->setOffset(info->out[idx].slot[c] * 4);
1535       else
1536       if (sym->reg.file == FILE_SYSTEM_VALUE)
1537          sym->setSV(tgsi::translateSysVal(info->sv[idx].sn), c);
1538       else
1539          sym->setOffset(address);
1540    } else {
1541       sym->setOffset(address);
1542    }
1543    return sym;
1544 }
1545
1546 uint8_t
1547 Converter::translateInterpMode(const struct nv50_ir_varying *var, operation& op)
1548 {
1549    uint8_t mode = NV50_IR_INTERP_PERSPECTIVE;
1550
1551    if (var->flat)
1552       mode = NV50_IR_INTERP_FLAT;
1553    else
1554    if (var->linear)
1555       mode = NV50_IR_INTERP_LINEAR;
1556    else
1557    if (var->sc)
1558       mode = NV50_IR_INTERP_SC;
1559
1560    op = (mode == NV50_IR_INTERP_PERSPECTIVE || mode == NV50_IR_INTERP_SC)
1561       ? OP_PINTERP : OP_LINTERP;
1562
1563    if (var->centroid)
1564       mode |= NV50_IR_INTERP_CENTROID;
1565
1566    return mode;
1567 }
1568
1569 Value *
1570 Converter::interpolate(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
1571 {
1572    operation op;
1573
1574    // XXX: no way to know interpolation mode if we don't know what's accessed
1575    const uint8_t mode = translateInterpMode(&info->in[ptr ? 0 :
1576                                                       src.getIndex(0)], op);
1577
1578    Instruction *insn = new_Instruction(func, op, TYPE_F32);
1579
1580    insn->setDef(0, getScratch());
1581    insn->setSrc(0, srcToSym(src, c));
1582    if (op == OP_PINTERP)
1583       insn->setSrc(1, fragCoord[3]);
1584    if (ptr)
1585       insn->setIndirect(0, 0, ptr);
1586
1587    insn->setInterpolate(mode);
1588
1589    bb->insertTail(insn);
1590    return insn->getDef(0);
1591 }
1592
1593 Value *
1594 Converter::applySrcMod(Value *val, int s, int c)
1595 {
1596    Modifier m = tgsi.getSrc(s).getMod(c);
1597    DataType ty = tgsi.inferSrcType();
1598
1599    if (m & Modifier(NV50_IR_MOD_ABS))
1600       val = mkOp1v(OP_ABS, ty, getScratch(), val);
1601
1602    if (m & Modifier(NV50_IR_MOD_NEG))
1603       val = mkOp1v(OP_NEG, ty, getScratch(), val);
1604
1605    return val;
1606 }
1607
1608 Value *
1609 Converter::getVertexBase(int s)
1610 {
1611    assert(s < 5);
1612    if (!(vtxBaseValid & (1 << s))) {
1613       const int index = tgsi.getSrc(s).getIndex(1);
1614       Value *rel = NULL;
1615       if (tgsi.getSrc(s).isIndirect(1))
1616          rel = fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL);
1617       vtxBaseValid |= 1 << s;
1618       vtxBase[s] = mkOp2v(OP_PFETCH, TYPE_U32, getSSA(4, FILE_ADDRESS),
1619                           mkImm(index), rel);
1620    }
1621    return vtxBase[s];
1622 }
1623
1624 Value *
1625 Converter::getOutputBase(int s)
1626 {
1627    assert(s < 5);
1628    if (!(vtxBaseValid & (1 << s))) {
1629       Value *offset = loadImm(NULL, tgsi.getSrc(s).getIndex(1));
1630       if (tgsi.getSrc(s).isIndirect(1))
1631          offset = mkOp2v(OP_ADD, TYPE_U32, getSSA(),
1632                          fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL),
1633                          offset);
1634       vtxBaseValid |= 1 << s;
1635       vtxBase[s] = mkOp2v(OP_ADD, TYPE_U32, getSSA(), outBase, offset);
1636    }
1637    return vtxBase[s];
1638 }
1639
1640 Value *
1641 Converter::fetchSrc(int s, int c)
1642 {
1643    Value *res;
1644    Value *ptr = NULL, *dimRel = NULL;
1645
1646    tgsi::Instruction::SrcRegister src = tgsi.getSrc(s);
1647
1648    if (src.isIndirect(0))
1649       ptr = fetchSrc(src.getIndirect(0), 0, NULL);
1650
1651    if (src.is2D()) {
1652       switch (src.getFile()) {
1653       case TGSI_FILE_OUTPUT:
1654          dimRel = getOutputBase(s);
1655          break;
1656       case TGSI_FILE_INPUT:
1657          dimRel = getVertexBase(s);
1658          break;
1659       case TGSI_FILE_CONSTANT:
1660          // on NVC0, this is valid and c{I+J}[k] == cI[(J << 16) + k]
1661          if (src.isIndirect(1))
1662             dimRel = fetchSrc(src.getIndirect(1), 0, 0);
1663          break;
1664       default:
1665          break;
1666       }
1667    }
1668
1669    res = fetchSrc(src, c, ptr);
1670
1671    if (dimRel)
1672       res->getInsn()->setIndirect(0, 1, dimRel);
1673
1674    return applySrcMod(res, s, c);
1675 }
1676
1677 Converter::DataArray *
1678 Converter::getArrayForFile(unsigned file, int idx)
1679 {
1680    switch (file) {
1681    case TGSI_FILE_TEMPORARY:
1682       return idx == 0 ? &tData : &lData;
1683    case TGSI_FILE_PREDICATE:
1684       return &pData;
1685    case TGSI_FILE_ADDRESS:
1686       return &aData;
1687    case TGSI_FILE_OUTPUT:
1688       assert(prog->getType() == Program::TYPE_FRAGMENT);
1689       return &oData;
1690    default:
1691       assert(!"invalid/unhandled TGSI source file");
1692       return NULL;
1693    }
1694 }
1695
1696 Value *
1697 Converter::shiftAddress(Value *index)
1698 {
1699    if (!index)
1700       return NULL;
1701    return mkOp2v(OP_SHL, TYPE_U32, getSSA(4, FILE_ADDRESS), index, mkImm(4));
1702 }
1703
1704 void
1705 Converter::adjustTempIndex(int arrayId, int &idx, int &idx2d) const
1706 {
1707    std::map<int, int>::const_iterator it =
1708       code->indirectTempOffsets.find(arrayId);
1709    if (it == code->indirectTempOffsets.end())
1710       return;
1711
1712    idx2d = 1;
1713    idx += it->second;
1714 }
1715
1716 Value *
1717 Converter::fetchSrc(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
1718 {
1719    int idx2d = src.is2D() ? src.getIndex(1) : 0;
1720    int idx = src.getIndex(0);
1721    const int swz = src.getSwizzle(c);
1722    Instruction *ld;
1723
1724    switch (src.getFile()) {
1725    case TGSI_FILE_IMMEDIATE:
1726       assert(!ptr);
1727       return loadImm(NULL, info->immd.data[idx * 4 + swz]);
1728    case TGSI_FILE_CONSTANT:
1729       return mkLoadv(TYPE_U32, srcToSym(src, c), shiftAddress(ptr));
1730    case TGSI_FILE_INPUT:
1731       if (prog->getType() == Program::TYPE_FRAGMENT) {
1732          // don't load masked inputs, won't be assigned a slot
1733          if (!ptr && !(info->in[idx].mask & (1 << swz)))
1734             return loadImm(NULL, swz == TGSI_SWIZZLE_W ? 1.0f : 0.0f);
1735          return interpolate(src, c, shiftAddress(ptr));
1736       } else
1737       if (prog->getType() == Program::TYPE_GEOMETRY) {
1738          if (!ptr && info->in[idx].sn == TGSI_SEMANTIC_PRIMID)
1739             return mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_PRIMITIVE_ID, 0));
1740          // XXX: This is going to be a problem with scalar arrays, i.e. when
1741          // we cannot assume that the address is given in units of vec4.
1742          //
1743          // nv50 and nvc0 need different things here, so let the lowering
1744          // passes decide what to do with the address
1745          if (ptr)
1746             return mkLoadv(TYPE_U32, srcToSym(src, c), ptr);
1747       }
1748       ld = mkLoad(TYPE_U32, getSSA(), srcToSym(src, c), shiftAddress(ptr));
1749       ld->perPatch = info->in[idx].patch;
1750       return ld->getDef(0);
1751    case TGSI_FILE_OUTPUT:
1752       assert(prog->getType() == Program::TYPE_TESSELLATION_CONTROL);
1753       ld = mkLoad(TYPE_U32, getSSA(), srcToSym(src, c), shiftAddress(ptr));
1754       ld->perPatch = info->out[idx].patch;
1755       return ld->getDef(0);
1756    case TGSI_FILE_SYSTEM_VALUE:
1757       assert(!ptr);
1758       ld = mkOp1(OP_RDSV, TYPE_U32, getSSA(), srcToSym(src, c));
1759       ld->perPatch = info->sv[idx].patch;
1760       return ld->getDef(0);
1761    case TGSI_FILE_TEMPORARY: {
1762       int arrayid = src.getArrayId();
1763       if (!arrayid)
1764          arrayid = code->tempArrayId[idx];
1765       adjustTempIndex(arrayid, idx, idx2d);
1766    }
1767       /* fallthrough */
1768    default:
1769       return getArrayForFile(src.getFile(), idx2d)->load(
1770          sub.cur->values, idx, swz, shiftAddress(ptr));
1771    }
1772 }
1773
1774 Value *
1775 Converter::acquireDst(int d, int c)
1776 {
1777    const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
1778    const unsigned f = dst.getFile();
1779    int idx = dst.getIndex(0);
1780    int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
1781
1782    if (dst.isMasked(c) || f == TGSI_FILE_BUFFER || f == TGSI_FILE_MEMORY)
1783       return NULL;
1784
1785    if (dst.isIndirect(0) ||
1786        f == TGSI_FILE_SYSTEM_VALUE ||
1787        (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT))
1788       return getScratch();
1789
1790    if (f == TGSI_FILE_TEMPORARY) {
1791       int arrayid = dst.getArrayId();
1792       if (!arrayid)
1793          arrayid = code->tempArrayId[idx];
1794       adjustTempIndex(arrayid, idx, idx2d);
1795    }
1796
1797    return getArrayForFile(f, idx2d)-> acquire(sub.cur->values, idx, c);
1798 }
1799
1800 void
1801 Converter::storeDst(int d, int c, Value *val)
1802 {
1803    const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
1804
1805    if (tgsi.getSaturate()) {
1806       mkOp1(OP_SAT, dstTy, val, val);
1807    }
1808
1809    Value *ptr = NULL;
1810    if (dst.isIndirect(0))
1811       ptr = shiftAddress(fetchSrc(dst.getIndirect(0), 0, NULL));
1812
1813    if (info->io.genUserClip > 0 &&
1814        dst.getFile() == TGSI_FILE_OUTPUT &&
1815        !dst.isIndirect(0) && dst.getIndex(0) == code->clipVertexOutput) {
1816       mkMov(clipVtx[c], val);
1817       val = clipVtx[c];
1818    }
1819
1820    storeDst(dst, c, val, ptr);
1821 }
1822
1823 void
1824 Converter::storeDst(const tgsi::Instruction::DstRegister dst, int c,
1825                     Value *val, Value *ptr)
1826 {
1827    const unsigned f = dst.getFile();
1828    int idx = dst.getIndex(0);
1829    int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
1830
1831    if (f == TGSI_FILE_SYSTEM_VALUE) {
1832       assert(!ptr);
1833       mkOp2(OP_WRSV, TYPE_U32, NULL, dstToSym(dst, c), val);
1834    } else
1835    if (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT) {
1836
1837       if (ptr || (info->out[idx].mask & (1 << c))) {
1838          /* Save the viewport index into a scratch register so that it can be
1839             exported at EMIT time */
1840          if (info->out[idx].sn == TGSI_SEMANTIC_VIEWPORT_INDEX &&
1841              viewport != NULL)
1842             mkOp1(OP_MOV, TYPE_U32, viewport, val);
1843          else
1844             mkStore(OP_EXPORT, TYPE_U32, dstToSym(dst, c), ptr, val)->perPatch =
1845                info->out[idx].patch;
1846       }
1847    } else
1848    if (f == TGSI_FILE_TEMPORARY ||
1849        f == TGSI_FILE_PREDICATE ||
1850        f == TGSI_FILE_ADDRESS ||
1851        f == TGSI_FILE_OUTPUT) {
1852       if (f == TGSI_FILE_TEMPORARY) {
1853          int arrayid = dst.getArrayId();
1854          if (!arrayid)
1855             arrayid = code->tempArrayId[idx];
1856          adjustTempIndex(arrayid, idx, idx2d);
1857       }
1858
1859       getArrayForFile(f, idx2d)->store(sub.cur->values, idx, c, ptr, val);
1860    } else {
1861       assert(!"invalid dst file");
1862    }
1863 }
1864
1865 #define FOR_EACH_DST_ENABLED_CHANNEL(d, chan, inst) \
1866    for (chan = 0; chan < 4; ++chan)                 \
1867       if (!inst.getDst(d).isMasked(chan))
1868
1869 Value *
1870 Converter::buildDot(int dim)
1871 {
1872    assert(dim > 0);
1873
1874    Value *src0 = fetchSrc(0, 0), *src1 = fetchSrc(1, 0);
1875    Value *dotp = getScratch();
1876
1877    mkOp2(OP_MUL, TYPE_F32, dotp, src0, src1);
1878
1879    for (int c = 1; c < dim; ++c) {
1880       src0 = fetchSrc(0, c);
1881       src1 = fetchSrc(1, c);
1882       mkOp3(OP_MAD, TYPE_F32, dotp, src0, src1, dotp);
1883    }
1884    return dotp;
1885 }
1886
1887 void
1888 Converter::insertConvergenceOps(BasicBlock *conv, BasicBlock *fork)
1889 {
1890    FlowInstruction *join = new_FlowInstruction(func, OP_JOIN, NULL);
1891    join->fixed = 1;
1892    conv->insertHead(join);
1893
1894    assert(!fork->joinAt);
1895    fork->joinAt = new_FlowInstruction(func, OP_JOINAT, conv);
1896    fork->insertBefore(fork->getExit(), fork->joinAt);
1897 }
1898
1899 void
1900 Converter::setTexRS(TexInstruction *tex, unsigned int& s, int R, int S)
1901 {
1902    unsigned rIdx = 0, sIdx = 0;
1903
1904    if (R >= 0)
1905       rIdx = tgsi.getSrc(R).getIndex(0);
1906    if (S >= 0)
1907       sIdx = tgsi.getSrc(S).getIndex(0);
1908
1909    tex->setTexture(tgsi.getTexture(code, R), rIdx, sIdx);
1910
1911    if (tgsi.getSrc(R).isIndirect(0)) {
1912       tex->tex.rIndirectSrc = s;
1913       tex->setSrc(s++, fetchSrc(tgsi.getSrc(R).getIndirect(0), 0, NULL));
1914    }
1915    if (S >= 0 && tgsi.getSrc(S).isIndirect(0)) {
1916       tex->tex.sIndirectSrc = s;
1917       tex->setSrc(s++, fetchSrc(tgsi.getSrc(S).getIndirect(0), 0, NULL));
1918    }
1919 }
1920
1921 void
1922 Converter::handleTXQ(Value *dst0[4], enum TexQuery query, int R)
1923 {
1924    TexInstruction *tex = new_TexInstruction(func, OP_TXQ);
1925    tex->tex.query = query;
1926    unsigned int c, d;
1927
1928    for (d = 0, c = 0; c < 4; ++c) {
1929       if (!dst0[c])
1930          continue;
1931       tex->tex.mask |= 1 << c;
1932       tex->setDef(d++, dst0[c]);
1933    }
1934    if (query == TXQ_DIMS)
1935       tex->setSrc((c = 0), fetchSrc(0, 0)); // mip level
1936    else
1937       tex->setSrc((c = 0), zero);
1938
1939    setTexRS(tex, ++c, R, -1);
1940
1941    bb->insertTail(tex);
1942 }
1943
1944 void
1945 Converter::loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask)
1946 {
1947    Value *proj = fetchSrc(0, 3);
1948    Instruction *insn = proj->getUniqueInsn();
1949    int c;
1950
1951    if (insn->op == OP_PINTERP) {
1952       bb->insertTail(insn = cloneForward(func, insn));
1953       insn->op = OP_LINTERP;
1954       insn->setInterpolate(NV50_IR_INTERP_LINEAR | insn->getSampleMode());
1955       insn->setSrc(1, NULL);
1956       proj = insn->getDef(0);
1957    }
1958    proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), proj);
1959
1960    for (c = 0; c < 4; ++c) {
1961       if (!(mask & (1 << c)))
1962          continue;
1963       if ((insn = src[c]->getUniqueInsn())->op != OP_PINTERP)
1964          continue;
1965       mask &= ~(1 << c);
1966
1967       bb->insertTail(insn = cloneForward(func, insn));
1968       insn->setInterpolate(NV50_IR_INTERP_PERSPECTIVE | insn->getSampleMode());
1969       insn->setSrc(1, proj);
1970       dst[c] = insn->getDef(0);
1971    }
1972    if (!mask)
1973       return;
1974
1975    proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), fetchSrc(0, 3));
1976
1977    for (c = 0; c < 4; ++c)
1978       if (mask & (1 << c))
1979          dst[c] = mkOp2v(OP_MUL, TYPE_F32, getSSA(), src[c], proj);
1980 }
1981
1982 // order of nv50 ir sources: x y z layer lod/bias shadow
1983 // order of TGSI TEX sources: x y z layer shadow lod/bias
1984 //  lowering will finally set the hw specific order (like array first on nvc0)
1985 void
1986 Converter::handleTEX(Value *dst[4], int R, int S, int L, int C, int Dx, int Dy)
1987 {
1988    Value *val;
1989    Value *arg[4], *src[8];
1990    Value *lod = NULL, *shd = NULL;
1991    unsigned int s, c, d;
1992    TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
1993
1994    TexInstruction::Target tgt = tgsi.getTexture(code, R);
1995
1996    for (s = 0; s < tgt.getArgCount(); ++s)
1997       arg[s] = src[s] = fetchSrc(0, s);
1998
1999    if (texi->op == OP_TXL || texi->op == OP_TXB)
2000       lod = fetchSrc(L >> 4, L & 3);
2001
2002    if (C == 0x0f)
2003       C = 0x00 | MAX2(tgt.getArgCount(), 2); // guess DC src
2004
2005    if (tgsi.getOpcode() == TGSI_OPCODE_TG4 &&
2006        tgt == TEX_TARGET_CUBE_ARRAY_SHADOW)
2007       shd = fetchSrc(1, 0);
2008    else if (tgt.isShadow())
2009       shd = fetchSrc(C >> 4, C & 3);
2010
2011    if (texi->op == OP_TXD) {
2012       for (c = 0; c < tgt.getDim() + tgt.isCube(); ++c) {
2013          texi->dPdx[c].set(fetchSrc(Dx >> 4, (Dx & 3) + c));
2014          texi->dPdy[c].set(fetchSrc(Dy >> 4, (Dy & 3) + c));
2015       }
2016    }
2017
2018    // cube textures don't care about projection value, it's divided out
2019    if (tgsi.getOpcode() == TGSI_OPCODE_TXP && !tgt.isCube() && !tgt.isArray()) {
2020       unsigned int n = tgt.getDim();
2021       if (shd) {
2022          arg[n] = shd;
2023          ++n;
2024          assert(tgt.getDim() == tgt.getArgCount());
2025       }
2026       loadProjTexCoords(src, arg, (1 << n) - 1);
2027       if (shd)
2028          shd = src[n - 1];
2029    }
2030
2031    if (tgt.isCube()) {
2032       for (c = 0; c < 3; ++c)
2033          src[c] = mkOp1v(OP_ABS, TYPE_F32, getSSA(), arg[c]);
2034       val = getScratch();
2035       mkOp2(OP_MAX, TYPE_F32, val, src[0], src[1]);
2036       mkOp2(OP_MAX, TYPE_F32, val, src[2], val);
2037       mkOp1(OP_RCP, TYPE_F32, val, val);
2038       for (c = 0; c < 3; ++c)
2039          src[c] = mkOp2v(OP_MUL, TYPE_F32, getSSA(), arg[c], val);
2040    }
2041
2042    for (c = 0, d = 0; c < 4; ++c) {
2043       if (dst[c]) {
2044          texi->setDef(d++, dst[c]);
2045          texi->tex.mask |= 1 << c;
2046       } else {
2047          // NOTE: maybe hook up def too, for CSE
2048       }
2049    }
2050    for (s = 0; s < tgt.getArgCount(); ++s)
2051       texi->setSrc(s, src[s]);
2052    if (lod)
2053       texi->setSrc(s++, lod);
2054    if (shd)
2055       texi->setSrc(s++, shd);
2056
2057    setTexRS(texi, s, R, S);
2058
2059    if (tgsi.getOpcode() == TGSI_OPCODE_SAMPLE_C_LZ)
2060       texi->tex.levelZero = true;
2061    if (tgsi.getOpcode() == TGSI_OPCODE_TG4 && !tgt.isShadow())
2062       texi->tex.gatherComp = tgsi.getSrc(1).getValueU32(0, info);
2063
2064    texi->tex.useOffsets = tgsi.getNumTexOffsets();
2065    for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
2066       for (c = 0; c < 3; ++c) {
2067          texi->offset[s][c].set(fetchSrc(tgsi.getTexOffset(s), c, NULL));
2068          texi->offset[s][c].setInsn(texi);
2069       }
2070    }
2071
2072    bb->insertTail(texi);
2073 }
2074
2075 // 1st source: xyz = coordinates, w = lod/sample
2076 // 2nd source: offset
2077 void
2078 Converter::handleTXF(Value *dst[4], int R, int L_M)
2079 {
2080    TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
2081    int ms;
2082    unsigned int c, d, s;
2083
2084    texi->tex.target = tgsi.getTexture(code, R);
2085
2086    ms = texi->tex.target.isMS() ? 1 : 0;
2087    texi->tex.levelZero = ms; /* MS textures don't have mip-maps */
2088
2089    for (c = 0, d = 0; c < 4; ++c) {
2090       if (dst[c]) {
2091          texi->setDef(d++, dst[c]);
2092          texi->tex.mask |= 1 << c;
2093       }
2094    }
2095    for (c = 0; c < (texi->tex.target.getArgCount() - ms); ++c)
2096       texi->setSrc(c, fetchSrc(0, c));
2097    texi->setSrc(c++, fetchSrc(L_M >> 4, L_M & 3)); // lod or ms
2098
2099    setTexRS(texi, c, R, -1);
2100
2101    texi->tex.useOffsets = tgsi.getNumTexOffsets();
2102    for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
2103       for (c = 0; c < 3; ++c) {
2104          texi->offset[s][c].set(fetchSrc(tgsi.getTexOffset(s), c, NULL));
2105          texi->offset[s][c].setInsn(texi);
2106       }
2107    }
2108
2109    bb->insertTail(texi);
2110 }
2111
2112 void
2113 Converter::handleLIT(Value *dst0[4])
2114 {
2115    Value *val0 = NULL;
2116    unsigned int mask = tgsi.getDst(0).getMask();
2117
2118    if (mask & (1 << 0))
2119       loadImm(dst0[0], 1.0f);
2120
2121    if (mask & (1 << 3))
2122       loadImm(dst0[3], 1.0f);
2123
2124    if (mask & (3 << 1)) {
2125       val0 = getScratch();
2126       mkOp2(OP_MAX, TYPE_F32, val0, fetchSrc(0, 0), zero);
2127       if (mask & (1 << 1))
2128          mkMov(dst0[1], val0);
2129    }
2130
2131    if (mask & (1 << 2)) {
2132       Value *src1 = fetchSrc(0, 1), *src3 = fetchSrc(0, 3);
2133       Value *val1 = getScratch(), *val3 = getScratch();
2134
2135       Value *pos128 = loadImm(NULL, +127.999999f);
2136       Value *neg128 = loadImm(NULL, -127.999999f);
2137
2138       mkOp2(OP_MAX, TYPE_F32, val1, src1, zero);
2139       mkOp2(OP_MAX, TYPE_F32, val3, src3, neg128);
2140       mkOp2(OP_MIN, TYPE_F32, val3, val3, pos128);
2141       mkOp2(OP_POW, TYPE_F32, val3, val1, val3);
2142
2143       mkCmp(OP_SLCT, CC_GT, TYPE_F32, dst0[2], TYPE_F32, val3, zero, val0);
2144    }
2145 }
2146
2147 static inline bool
2148 isResourceSpecial(const int r)
2149 {
2150    return (r == TGSI_RESOURCE_GLOBAL ||
2151            r == TGSI_RESOURCE_LOCAL ||
2152            r == TGSI_RESOURCE_PRIVATE ||
2153            r == TGSI_RESOURCE_INPUT);
2154 }
2155
2156 static inline bool
2157 isResourceRaw(const tgsi::Source *code, const int r)
2158 {
2159    return isResourceSpecial(r) || code->resources[r].raw;
2160 }
2161
2162 static inline nv50_ir::TexTarget
2163 getResourceTarget(const tgsi::Source *code, int r)
2164 {
2165    if (isResourceSpecial(r))
2166       return nv50_ir::TEX_TARGET_BUFFER;
2167    return tgsi::translateTexture(code->resources.at(r).target);
2168 }
2169
2170 Symbol *
2171 Converter::getResourceBase(const int r)
2172 {
2173    Symbol *sym = NULL;
2174
2175    switch (r) {
2176    case TGSI_RESOURCE_GLOBAL:
2177       sym = new_Symbol(prog, nv50_ir::FILE_MEMORY_GLOBAL, 15);
2178       break;
2179    case TGSI_RESOURCE_LOCAL:
2180       assert(prog->getType() == Program::TYPE_COMPUTE);
2181       sym = mkSymbol(nv50_ir::FILE_MEMORY_SHARED, 0, TYPE_U32,
2182                      info->prop.cp.sharedOffset);
2183       break;
2184    case TGSI_RESOURCE_PRIVATE:
2185       sym = mkSymbol(nv50_ir::FILE_MEMORY_LOCAL, 0, TYPE_U32,
2186                      info->bin.tlsSpace);
2187       break;
2188    case TGSI_RESOURCE_INPUT:
2189       assert(prog->getType() == Program::TYPE_COMPUTE);
2190       sym = mkSymbol(nv50_ir::FILE_SHADER_INPUT, 0, TYPE_U32,
2191                      info->prop.cp.inputOffset);
2192       break;
2193    default:
2194       sym = new_Symbol(prog,
2195                        nv50_ir::FILE_MEMORY_GLOBAL, code->resources.at(r).slot);
2196       break;
2197    }
2198    return sym;
2199 }
2200
2201 void
2202 Converter::getResourceCoords(std::vector<Value *> &coords, int r, int s)
2203 {
2204    const int arg =
2205       TexInstruction::Target(getResourceTarget(code, r)).getArgCount();
2206
2207    for (int c = 0; c < arg; ++c)
2208       coords.push_back(fetchSrc(s, c));
2209
2210    // NOTE: TGSI_RESOURCE_GLOBAL needs FILE_GPR; this is an nv50 quirk
2211    if (r == TGSI_RESOURCE_LOCAL ||
2212        r == TGSI_RESOURCE_PRIVATE ||
2213        r == TGSI_RESOURCE_INPUT)
2214       coords[0] = mkOp1v(OP_MOV, TYPE_U32, getScratch(4, FILE_ADDRESS),
2215                          coords[0]);
2216 }
2217
2218 static inline int
2219 partitionLoadStore(uint8_t comp[2], uint8_t size[2], uint8_t mask)
2220 {
2221    int n = 0;
2222
2223    while (mask) {
2224       if (mask & 1) {
2225          size[n]++;
2226       } else {
2227          if (size[n])
2228             comp[n = 1] = size[0] + 1;
2229          else
2230             comp[n]++;
2231       }
2232       mask >>= 1;
2233    }
2234    if (size[0] == 3) {
2235       n = 1;
2236       size[0] = (comp[0] == 1) ? 1 : 2;
2237       size[1] = 3 - size[0];
2238       comp[1] = comp[0] + size[0];
2239    }
2240    return n + 1;
2241 }
2242
2243 // For raw loads, granularity is 4 byte.
2244 // Usage of the texture read mask on OP_SULDP is not allowed.
2245 void
2246 Converter::handleLOAD(Value *dst0[4])
2247 {
2248    const int r = tgsi.getSrc(0).getIndex(0);
2249    int c;
2250    std::vector<Value *> off, src, ldv, def;
2251
2252    if (tgsi.getSrc(0).getFile() == TGSI_FILE_BUFFER ||
2253        tgsi.getSrc(0).getFile() == TGSI_FILE_MEMORY) {
2254       for (c = 0; c < 4; ++c) {
2255          if (!dst0[c])
2256             continue;
2257
2258          Value *off = fetchSrc(1, c);
2259          Symbol *sym;
2260          if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE) {
2261             off = NULL;
2262             sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c,
2263                           tgsi.getSrc(1).getValueU32(0, info) + 4 * c);
2264          } else {
2265             sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c, 4 * c);
2266          }
2267
2268          Instruction *ld = mkLoad(TYPE_U32, dst0[c], sym, off);
2269          ld->cache = tgsi.getCacheMode();
2270          if (tgsi.getSrc(0).isIndirect(0))
2271             ld->setIndirect(0, 1, fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, 0));
2272       }
2273       return;
2274    }
2275
2276    getResourceCoords(off, r, 1);
2277
2278    if (isResourceRaw(code, r)) {
2279       uint8_t mask = 0;
2280       uint8_t comp[2] = { 0, 0 };
2281       uint8_t size[2] = { 0, 0 };
2282
2283       Symbol *base = getResourceBase(r);
2284
2285       // determine the base and size of the at most 2 load ops
2286       for (c = 0; c < 4; ++c)
2287          if (!tgsi.getDst(0).isMasked(c))
2288             mask |= 1 << (tgsi.getSrc(0).getSwizzle(c) - TGSI_SWIZZLE_X);
2289
2290       int n = partitionLoadStore(comp, size, mask);
2291
2292       src = off;
2293
2294       def.resize(4); // index by component, the ones we need will be non-NULL
2295       for (c = 0; c < 4; ++c) {
2296          if (dst0[c] && tgsi.getSrc(0).getSwizzle(c) == (TGSI_SWIZZLE_X + c))
2297             def[c] = dst0[c];
2298          else
2299          if (mask & (1 << c))
2300             def[c] = getScratch();
2301       }
2302
2303       const bool useLd = isResourceSpecial(r) ||
2304          (info->io.nv50styleSurfaces &&
2305           code->resources[r].target == TGSI_TEXTURE_BUFFER);
2306
2307       for (int i = 0; i < n; ++i) {
2308          ldv.assign(def.begin() + comp[i], def.begin() + comp[i] + size[i]);
2309
2310          if (comp[i]) // adjust x component of source address if necessary
2311             src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
2312                             off[0], mkImm(comp[i] * 4));
2313          else
2314             src[0] = off[0];
2315
2316          if (useLd) {
2317             Instruction *ld =
2318                mkLoad(typeOfSize(size[i] * 4), ldv[0], base, src[0]);
2319             for (size_t c = 1; c < ldv.size(); ++c)
2320                ld->setDef(c, ldv[c]);
2321          } else {
2322             mkTex(OP_SULDB, getResourceTarget(code, r), code->resources[r].slot,
2323                   0, ldv, src)->dType = typeOfSize(size[i] * 4);
2324          }
2325       }
2326    } else {
2327       def.resize(4);
2328       for (c = 0; c < 4; ++c) {
2329          if (!dst0[c] || tgsi.getSrc(0).getSwizzle(c) != (TGSI_SWIZZLE_X + c))
2330             def[c] = getScratch();
2331          else
2332             def[c] = dst0[c];
2333       }
2334
2335       mkTex(OP_SULDP, getResourceTarget(code, r), code->resources[r].slot, 0,
2336             def, off);
2337    }
2338    FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2339       if (dst0[c] != def[c])
2340          mkMov(dst0[c], def[tgsi.getSrc(0).getSwizzle(c)]);
2341 }
2342
2343 // For formatted stores, the write mask on OP_SUSTP can be used.
2344 // Raw stores have to be split.
2345 void
2346 Converter::handleSTORE()
2347 {
2348    const int r = tgsi.getDst(0).getIndex(0);
2349    int c;
2350    std::vector<Value *> off, src, dummy;
2351
2352    if (tgsi.getDst(0).getFile() == TGSI_FILE_BUFFER ||
2353        tgsi.getDst(0).getFile() == TGSI_FILE_MEMORY) {
2354       for (c = 0; c < 4; ++c) {
2355          if (!(tgsi.getDst(0).getMask() & (1 << c)))
2356             continue;
2357
2358          Symbol *sym;
2359          Value *off;
2360          if (tgsi.getSrc(0).getFile() == TGSI_FILE_IMMEDIATE) {
2361             off = NULL;
2362             sym = makeSym(tgsi.getDst(0).getFile(), r, -1, c,
2363                           tgsi.getSrc(0).getValueU32(0, info) + 4 * c);
2364          } else {
2365             off = fetchSrc(0, 0);
2366             sym = makeSym(tgsi.getDst(0).getFile(), r, -1, c, 4 * c);
2367          }
2368
2369          Instruction *st = mkStore(OP_STORE, TYPE_U32, sym, off, fetchSrc(1, c));
2370          st->cache = tgsi.getCacheMode();
2371          if (tgsi.getDst(0).isIndirect(0))
2372             st->setIndirect(0, 1, fetchSrc(tgsi.getDst(0).getIndirect(0), 0, 0));
2373       }
2374       return;
2375    }
2376
2377    getResourceCoords(off, r, 0);
2378    src = off;
2379    const int s = src.size();
2380
2381    if (isResourceRaw(code, r)) {
2382       uint8_t comp[2] = { 0, 0 };
2383       uint8_t size[2] = { 0, 0 };
2384
2385       int n = partitionLoadStore(comp, size, tgsi.getDst(0).getMask());
2386
2387       Symbol *base = getResourceBase(r);
2388
2389       const bool useSt = isResourceSpecial(r) ||
2390          (info->io.nv50styleSurfaces &&
2391           code->resources[r].target == TGSI_TEXTURE_BUFFER);
2392
2393       for (int i = 0; i < n; ++i) {
2394          if (comp[i]) // adjust x component of source address if necessary
2395             src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
2396                             off[0], mkImm(comp[i] * 4));
2397          else
2398             src[0] = off[0];
2399
2400          const DataType stTy = typeOfSize(size[i] * 4);
2401
2402          if (useSt) {
2403             Instruction *st =
2404                mkStore(OP_STORE, stTy, base, NULL, fetchSrc(1, comp[i]));
2405             for (c = 1; c < size[i]; ++c)
2406                st->setSrc(1 + c, fetchSrc(1, comp[i] + c));
2407             st->setIndirect(0, 0, src[0]);
2408          } else {
2409             // attach values to be stored
2410             src.resize(s + size[i]);
2411             for (c = 0; c < size[i]; ++c)
2412                src[s + c] = fetchSrc(1, comp[i] + c);
2413             mkTex(OP_SUSTB, getResourceTarget(code, r), code->resources[r].slot,
2414                   0, dummy, src)->setType(stTy);
2415          }
2416       }
2417    } else {
2418       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2419          src.push_back(fetchSrc(1, c));
2420
2421       mkTex(OP_SUSTP, getResourceTarget(code, r), code->resources[r].slot, 0,
2422             dummy, src)->tex.mask = tgsi.getDst(0).getMask();
2423    }
2424 }
2425
2426 // XXX: These only work on resources with the single-component u32/s32 formats.
2427 // Therefore the result is replicated. This might not be intended by TGSI, but
2428 // operating on more than 1 component would produce undefined results because
2429 // they do not exist.
2430 void
2431 Converter::handleATOM(Value *dst0[4], DataType ty, uint16_t subOp)
2432 {
2433    const int r = tgsi.getSrc(0).getIndex(0);
2434    std::vector<Value *> srcv;
2435    std::vector<Value *> defv;
2436    LValue *dst = getScratch();
2437
2438    if (tgsi.getSrc(0).getFile() == TGSI_FILE_BUFFER ||
2439        tgsi.getSrc(0).getFile() == TGSI_FILE_MEMORY) {
2440       for (int c = 0; c < 4; ++c) {
2441          if (!dst0[c])
2442             continue;
2443
2444          Instruction *insn;
2445          Value *off = fetchSrc(1, c), *off2 = NULL;
2446          Value *sym;
2447          if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE)
2448             sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c,
2449                           tgsi.getSrc(1).getValueU32(c, info));
2450          else
2451             sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c, 0);
2452          if (tgsi.getSrc(0).isIndirect(0))
2453             off2 = fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, 0);
2454          if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2455             insn = mkOp3(OP_ATOM, ty, dst, sym, fetchSrc(2, c), fetchSrc(3, c));
2456          else
2457             insn = mkOp2(OP_ATOM, ty, dst, sym, fetchSrc(2, c));
2458          if (tgsi.getSrc(1).getFile() != TGSI_FILE_IMMEDIATE)
2459             insn->setIndirect(0, 0, off);
2460          if (off2)
2461             insn->setIndirect(0, 1, off2);
2462          insn->subOp = subOp;
2463       }
2464       for (int c = 0; c < 4; ++c)
2465          if (dst0[c])
2466             dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov
2467       return;
2468    }
2469
2470
2471    getResourceCoords(srcv, r, 1);
2472
2473    if (isResourceSpecial(r)) {
2474       assert(r != TGSI_RESOURCE_INPUT);
2475       Instruction *insn;
2476       insn = mkOp2(OP_ATOM, ty, dst, getResourceBase(r), fetchSrc(2, 0));
2477       insn->subOp = subOp;
2478       if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2479          insn->setSrc(2, fetchSrc(3, 0));
2480       insn->setIndirect(0, 0, srcv.at(0));
2481    } else {
2482       operation op = isResourceRaw(code, r) ? OP_SUREDB : OP_SUREDP;
2483       TexTarget targ = getResourceTarget(code, r);
2484       int idx = code->resources[r].slot;
2485       defv.push_back(dst);
2486       srcv.push_back(fetchSrc(2, 0));
2487       if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2488          srcv.push_back(fetchSrc(3, 0));
2489       TexInstruction *tex = mkTex(op, targ, idx, 0, defv, srcv);
2490       tex->subOp = subOp;
2491       tex->tex.mask = 1;
2492       tex->setType(ty);
2493    }
2494
2495    for (int c = 0; c < 4; ++c)
2496       if (dst0[c])
2497          dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov
2498 }
2499
2500 void
2501 Converter::handleINTERP(Value *dst[4])
2502 {
2503    // Check whether the input is linear. All other attributes ignored.
2504    Instruction *insn;
2505    Value *offset = NULL, *ptr = NULL, *w = NULL;
2506    bool linear;
2507    operation op;
2508    int c, mode;
2509
2510    tgsi::Instruction::SrcRegister src = tgsi.getSrc(0);
2511    assert(src.getFile() == TGSI_FILE_INPUT);
2512
2513    if (src.isIndirect(0))
2514       ptr = fetchSrc(src.getIndirect(0), 0, NULL);
2515
2516    // XXX: no way to know interp mode if we don't know the index
2517    linear = info->in[ptr ? 0 : src.getIndex(0)].linear;
2518    if (linear) {
2519       op = OP_LINTERP;
2520       mode = NV50_IR_INTERP_LINEAR;
2521    } else {
2522       op = OP_PINTERP;
2523       mode = NV50_IR_INTERP_PERSPECTIVE;
2524    }
2525
2526    switch (tgsi.getOpcode()) {
2527    case TGSI_OPCODE_INTERP_CENTROID:
2528       mode |= NV50_IR_INTERP_CENTROID;
2529       break;
2530    case TGSI_OPCODE_INTERP_SAMPLE:
2531       insn = mkOp1(OP_PIXLD, TYPE_U32, (offset = getScratch()), fetchSrc(1, 0));
2532       insn->subOp = NV50_IR_SUBOP_PIXLD_OFFSET;
2533       mode |= NV50_IR_INTERP_OFFSET;
2534       break;
2535    case TGSI_OPCODE_INTERP_OFFSET: {
2536       // The input in src1.xy is float, but we need a single 32-bit value
2537       // where the upper and lower 16 bits are encoded in S0.12 format. We need
2538       // to clamp the input coordinates to (-0.5, 0.4375), multiply by 4096,
2539       // and then convert to s32.
2540       Value *offs[2];
2541       for (c = 0; c < 2; c++) {
2542          offs[c] = fetchSrc(1, c);
2543          mkOp2(OP_MIN, TYPE_F32, offs[c], offs[c], loadImm(NULL, 0.4375f));
2544          mkOp2(OP_MAX, TYPE_F32, offs[c], offs[c], loadImm(NULL, -0.5f));
2545          mkOp2(OP_MUL, TYPE_F32, offs[c], offs[c], loadImm(NULL, 4096.0f));
2546          mkCvt(OP_CVT, TYPE_S32, offs[c], TYPE_F32, offs[c]);
2547       }
2548       offset = mkOp3v(OP_INSBF, TYPE_U32, getScratch(),
2549                       offs[1], mkImm(0x1010), offs[0]);
2550       mode |= NV50_IR_INTERP_OFFSET;
2551       break;
2552    }
2553    }
2554
2555    if (op == OP_PINTERP) {
2556       if (offset) {
2557          w = mkOp2v(OP_RDSV, TYPE_F32, getSSA(), mkSysVal(SV_POSITION, 3), offset);
2558          mkOp1(OP_RCP, TYPE_F32, w, w);
2559       } else {
2560          w = fragCoord[3];
2561       }
2562    }
2563
2564
2565    FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2566       insn = mkOp1(op, TYPE_F32, dst[c], srcToSym(src, c));
2567       if (op == OP_PINTERP)
2568          insn->setSrc(1, w);
2569       if (ptr)
2570          insn->setIndirect(0, 0, ptr);
2571       if (offset)
2572          insn->setSrc(op == OP_PINTERP ? 2 : 1, offset);
2573
2574       insn->setInterpolate(mode);
2575    }
2576 }
2577
2578 Converter::Subroutine *
2579 Converter::getSubroutine(unsigned ip)
2580 {
2581    std::map<unsigned, Subroutine>::iterator it = sub.map.find(ip);
2582
2583    if (it == sub.map.end())
2584       it = sub.map.insert(std::make_pair(
2585               ip, Subroutine(new Function(prog, "SUB", ip)))).first;
2586
2587    return &it->second;
2588 }
2589
2590 Converter::Subroutine *
2591 Converter::getSubroutine(Function *f)
2592 {
2593    unsigned ip = f->getLabel();
2594    std::map<unsigned, Subroutine>::iterator it = sub.map.find(ip);
2595
2596    if (it == sub.map.end())
2597       it = sub.map.insert(std::make_pair(ip, Subroutine(f))).first;
2598
2599    return &it->second;
2600 }
2601
2602 bool
2603 Converter::isEndOfSubroutine(uint ip)
2604 {
2605    assert(ip < code->scan.num_instructions);
2606    tgsi::Instruction insn(&code->insns[ip]);
2607    return (insn.getOpcode() == TGSI_OPCODE_END ||
2608            insn.getOpcode() == TGSI_OPCODE_ENDSUB ||
2609            // does END occur at end of main or the very end ?
2610            insn.getOpcode() == TGSI_OPCODE_BGNSUB);
2611 }
2612
2613 bool
2614 Converter::handleInstruction(const struct tgsi_full_instruction *insn)
2615 {
2616    Instruction *geni;
2617
2618    Value *dst0[4], *rDst0[4];
2619    Value *src0, *src1, *src2, *src3;
2620    Value *val0, *val1;
2621    int c;
2622
2623    tgsi = tgsi::Instruction(insn);
2624
2625    bool useScratchDst = tgsi.checkDstSrcAliasing();
2626
2627    operation op = tgsi.getOP();
2628    dstTy = tgsi.inferDstType();
2629    srcTy = tgsi.inferSrcType();
2630
2631    unsigned int mask = tgsi.dstCount() ? tgsi.getDst(0).getMask() : 0;
2632
2633    if (tgsi.dstCount()) {
2634       for (c = 0; c < 4; ++c) {
2635          rDst0[c] = acquireDst(0, c);
2636          dst0[c] = (useScratchDst && rDst0[c]) ? getScratch() : rDst0[c];
2637       }
2638    }
2639
2640    switch (tgsi.getOpcode()) {
2641    case TGSI_OPCODE_ADD:
2642    case TGSI_OPCODE_UADD:
2643    case TGSI_OPCODE_AND:
2644    case TGSI_OPCODE_DIV:
2645    case TGSI_OPCODE_IDIV:
2646    case TGSI_OPCODE_UDIV:
2647    case TGSI_OPCODE_MAX:
2648    case TGSI_OPCODE_MIN:
2649    case TGSI_OPCODE_IMAX:
2650    case TGSI_OPCODE_IMIN:
2651    case TGSI_OPCODE_UMAX:
2652    case TGSI_OPCODE_UMIN:
2653    case TGSI_OPCODE_MOD:
2654    case TGSI_OPCODE_UMOD:
2655    case TGSI_OPCODE_MUL:
2656    case TGSI_OPCODE_UMUL:
2657    case TGSI_OPCODE_IMUL_HI:
2658    case TGSI_OPCODE_UMUL_HI:
2659    case TGSI_OPCODE_OR:
2660    case TGSI_OPCODE_SHL:
2661    case TGSI_OPCODE_ISHR:
2662    case TGSI_OPCODE_USHR:
2663    case TGSI_OPCODE_SUB:
2664    case TGSI_OPCODE_XOR:
2665       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2666          src0 = fetchSrc(0, c);
2667          src1 = fetchSrc(1, c);
2668          geni = mkOp2(op, dstTy, dst0[c], src0, src1);
2669          geni->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode());
2670       }
2671       break;
2672    case TGSI_OPCODE_MAD:
2673    case TGSI_OPCODE_UMAD:
2674    case TGSI_OPCODE_SAD:
2675       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2676          src0 = fetchSrc(0, c);
2677          src1 = fetchSrc(1, c);
2678          src2 = fetchSrc(2, c);
2679          mkOp3(op, dstTy, dst0[c], src0, src1, src2);
2680       }
2681       break;
2682    case TGSI_OPCODE_MOV:
2683    case TGSI_OPCODE_ABS:
2684    case TGSI_OPCODE_CEIL:
2685    case TGSI_OPCODE_FLR:
2686    case TGSI_OPCODE_TRUNC:
2687    case TGSI_OPCODE_RCP:
2688    case TGSI_OPCODE_IABS:
2689    case TGSI_OPCODE_INEG:
2690    case TGSI_OPCODE_NOT:
2691    case TGSI_OPCODE_DDX:
2692    case TGSI_OPCODE_DDY:
2693    case TGSI_OPCODE_DDX_FINE:
2694    case TGSI_OPCODE_DDY_FINE:
2695       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2696          mkOp1(op, dstTy, dst0[c], fetchSrc(0, c));
2697       break;
2698    case TGSI_OPCODE_RSQ:
2699       src0 = fetchSrc(0, 0);
2700       val0 = getScratch();
2701       mkOp1(OP_ABS, TYPE_F32, val0, src0);
2702       mkOp1(OP_RSQ, TYPE_F32, val0, val0);
2703       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2704          mkMov(dst0[c], val0);
2705       break;
2706    case TGSI_OPCODE_ARL:
2707    case TGSI_OPCODE_ARR:
2708       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2709          const RoundMode rnd =
2710             tgsi.getOpcode() == TGSI_OPCODE_ARR ? ROUND_N : ROUND_M;
2711          src0 = fetchSrc(0, c);
2712          mkCvt(OP_CVT, TYPE_S32, dst0[c], TYPE_F32, src0)->rnd = rnd;
2713       }
2714       break;
2715    case TGSI_OPCODE_UARL:
2716       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2717          mkOp1(OP_MOV, TYPE_U32, dst0[c], fetchSrc(0, c));
2718       break;
2719    case TGSI_OPCODE_POW:
2720       val0 = mkOp2v(op, TYPE_F32, getScratch(), fetchSrc(0, 0), fetchSrc(1, 0));
2721       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2722          mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
2723       break;
2724    case TGSI_OPCODE_EX2:
2725    case TGSI_OPCODE_LG2:
2726       val0 = mkOp1(op, TYPE_F32, getScratch(), fetchSrc(0, 0))->getDef(0);
2727       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2728          mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
2729       break;
2730    case TGSI_OPCODE_COS:
2731    case TGSI_OPCODE_SIN:
2732       val0 = getScratch();
2733       if (mask & 7) {
2734          mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 0));
2735          mkOp1(op, TYPE_F32, val0, val0);
2736          for (c = 0; c < 3; ++c)
2737             if (dst0[c])
2738                mkMov(dst0[c], val0);
2739       }
2740       if (dst0[3]) {
2741          mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 3));
2742          mkOp1(op, TYPE_F32, dst0[3], val0);
2743       }
2744       break;
2745    case TGSI_OPCODE_SCS:
2746       if (mask & 3) {
2747          val0 = mkOp1v(OP_PRESIN, TYPE_F32, getSSA(), fetchSrc(0, 0));
2748          if (dst0[0])
2749             mkOp1(OP_COS, TYPE_F32, dst0[0], val0);
2750          if (dst0[1])
2751             mkOp1(OP_SIN, TYPE_F32, dst0[1], val0);
2752       }
2753       if (dst0[2])
2754          loadImm(dst0[2], 0.0f);
2755       if (dst0[3])
2756          loadImm(dst0[3], 1.0f);
2757       break;
2758    case TGSI_OPCODE_EXP:
2759       src0 = fetchSrc(0, 0);
2760       val0 = mkOp1v(OP_FLOOR, TYPE_F32, getSSA(), src0);
2761       if (dst0[1])
2762          mkOp2(OP_SUB, TYPE_F32, dst0[1], src0, val0);
2763       if (dst0[0])
2764          mkOp1(OP_EX2, TYPE_F32, dst0[0], val0);
2765       if (dst0[2])
2766          mkOp1(OP_EX2, TYPE_F32, dst0[2], src0);
2767       if (dst0[3])
2768          loadImm(dst0[3], 1.0f);
2769       break;
2770    case TGSI_OPCODE_LOG:
2771       src0 = mkOp1v(OP_ABS, TYPE_F32, getSSA(), fetchSrc(0, 0));
2772       val0 = mkOp1v(OP_LG2, TYPE_F32, dst0[2] ? dst0[2] : getSSA(), src0);
2773       if (dst0[0] || dst0[1])
2774          val1 = mkOp1v(OP_FLOOR, TYPE_F32, dst0[0] ? dst0[0] : getSSA(), val0);
2775       if (dst0[1]) {
2776          mkOp1(OP_EX2, TYPE_F32, dst0[1], val1);
2777          mkOp1(OP_RCP, TYPE_F32, dst0[1], dst0[1]);
2778          mkOp2(OP_MUL, TYPE_F32, dst0[1], dst0[1], src0);
2779       }
2780       if (dst0[3])
2781          loadImm(dst0[3], 1.0f);
2782       break;
2783    case TGSI_OPCODE_DP2:
2784       val0 = buildDot(2);
2785       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2786          mkMov(dst0[c], val0);
2787       break;
2788    case TGSI_OPCODE_DP3:
2789       val0 = buildDot(3);
2790       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2791          mkMov(dst0[c], val0);
2792       break;
2793    case TGSI_OPCODE_DP4:
2794       val0 = buildDot(4);
2795       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2796          mkMov(dst0[c], val0);
2797       break;
2798    case TGSI_OPCODE_DPH:
2799       val0 = buildDot(3);
2800       src1 = fetchSrc(1, 3);
2801       mkOp2(OP_ADD, TYPE_F32, val0, val0, src1);
2802       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2803          mkMov(dst0[c], val0);
2804       break;
2805    case TGSI_OPCODE_DST:
2806       if (dst0[0])
2807          loadImm(dst0[0], 1.0f);
2808       if (dst0[1]) {
2809          src0 = fetchSrc(0, 1);
2810          src1 = fetchSrc(1, 1);
2811          mkOp2(OP_MUL, TYPE_F32, dst0[1], src0, src1);
2812       }
2813       if (dst0[2])
2814          mkMov(dst0[2], fetchSrc(0, 2));
2815       if (dst0[3])
2816          mkMov(dst0[3], fetchSrc(1, 3));
2817       break;
2818    case TGSI_OPCODE_LRP:
2819       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2820          src0 = fetchSrc(0, c);
2821          src1 = fetchSrc(1, c);
2822          src2 = fetchSrc(2, c);
2823          mkOp3(OP_MAD, TYPE_F32, dst0[c],
2824                mkOp2v(OP_SUB, TYPE_F32, getSSA(), src1, src2), src0, src2);
2825       }
2826       break;
2827    case TGSI_OPCODE_LIT:
2828       handleLIT(dst0);
2829       break;
2830    case TGSI_OPCODE_XPD:
2831       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2832          if (c < 3) {
2833             val0 = getSSA();
2834             src0 = fetchSrc(1, (c + 1) % 3);
2835             src1 = fetchSrc(0, (c + 2) % 3);
2836             mkOp2(OP_MUL, TYPE_F32, val0, src0, src1);
2837             mkOp1(OP_NEG, TYPE_F32, val0, val0);
2838
2839             src0 = fetchSrc(0, (c + 1) % 3);
2840             src1 = fetchSrc(1, (c + 2) % 3);
2841             mkOp3(OP_MAD, TYPE_F32, dst0[c], src0, src1, val0);
2842          } else {
2843             loadImm(dst0[c], 1.0f);
2844          }
2845       }
2846       break;
2847    case TGSI_OPCODE_ISSG:
2848    case TGSI_OPCODE_SSG:
2849       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2850          src0 = fetchSrc(0, c);
2851          val0 = getScratch();
2852          val1 = getScratch();
2853          mkCmp(OP_SET, CC_GT, srcTy, val0, srcTy, src0, zero);
2854          mkCmp(OP_SET, CC_LT, srcTy, val1, srcTy, src0, zero);
2855          if (srcTy == TYPE_F32)
2856             mkOp2(OP_SUB, TYPE_F32, dst0[c], val0, val1);
2857          else
2858             mkOp2(OP_SUB, TYPE_S32, dst0[c], val1, val0);
2859       }
2860       break;
2861    case TGSI_OPCODE_UCMP:
2862       srcTy = TYPE_U32;
2863       /* fallthrough */
2864    case TGSI_OPCODE_CMP:
2865       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2866          src0 = fetchSrc(0, c);
2867          src1 = fetchSrc(1, c);
2868          src2 = fetchSrc(2, c);
2869          if (src1 == src2)
2870             mkMov(dst0[c], src1);
2871          else
2872             mkCmp(OP_SLCT, (srcTy == TYPE_F32) ? CC_LT : CC_NE,
2873                   srcTy, dst0[c], srcTy, src1, src2, src0);
2874       }
2875       break;
2876    case TGSI_OPCODE_FRC:
2877       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2878          src0 = fetchSrc(0, c);
2879          val0 = getScratch();
2880          mkOp1(OP_FLOOR, TYPE_F32, val0, src0);
2881          mkOp2(OP_SUB, TYPE_F32, dst0[c], src0, val0);
2882       }
2883       break;
2884    case TGSI_OPCODE_ROUND:
2885       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2886          mkCvt(OP_CVT, TYPE_F32, dst0[c], TYPE_F32, fetchSrc(0, c))
2887          ->rnd = ROUND_NI;
2888       break;
2889    case TGSI_OPCODE_CLAMP:
2890       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2891          src0 = fetchSrc(0, c);
2892          src1 = fetchSrc(1, c);
2893          src2 = fetchSrc(2, c);
2894          val0 = getScratch();
2895          mkOp2(OP_MIN, TYPE_F32, val0, src0, src1);
2896          mkOp2(OP_MAX, TYPE_F32, dst0[c], val0, src2);
2897       }
2898       break;
2899    case TGSI_OPCODE_SLT:
2900    case TGSI_OPCODE_SGE:
2901    case TGSI_OPCODE_SEQ:
2902    case TGSI_OPCODE_SGT:
2903    case TGSI_OPCODE_SLE:
2904    case TGSI_OPCODE_SNE:
2905    case TGSI_OPCODE_FSEQ:
2906    case TGSI_OPCODE_FSGE:
2907    case TGSI_OPCODE_FSLT:
2908    case TGSI_OPCODE_FSNE:
2909    case TGSI_OPCODE_ISGE:
2910    case TGSI_OPCODE_ISLT:
2911    case TGSI_OPCODE_USEQ:
2912    case TGSI_OPCODE_USGE:
2913    case TGSI_OPCODE_USLT:
2914    case TGSI_OPCODE_USNE:
2915       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2916          src0 = fetchSrc(0, c);
2917          src1 = fetchSrc(1, c);
2918          mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], srcTy, src0, src1);
2919       }
2920       break;
2921    case TGSI_OPCODE_KILL_IF:
2922       val0 = new_LValue(func, FILE_PREDICATE);
2923       mask = 0;
2924       for (c = 0; c < 4; ++c) {
2925          const int s = tgsi.getSrc(0).getSwizzle(c);
2926          if (mask & (1 << s))
2927             continue;
2928          mask |= 1 << s;
2929          mkCmp(OP_SET, CC_LT, TYPE_F32, val0, TYPE_F32, fetchSrc(0, c), zero);
2930          mkOp(OP_DISCARD, TYPE_NONE, NULL)->setPredicate(CC_P, val0);
2931       }
2932       break;
2933    case TGSI_OPCODE_KILL:
2934       mkOp(OP_DISCARD, TYPE_NONE, NULL);
2935       break;
2936    case TGSI_OPCODE_TEX:
2937    case TGSI_OPCODE_TXB:
2938    case TGSI_OPCODE_TXL:
2939    case TGSI_OPCODE_TXP:
2940    case TGSI_OPCODE_LODQ:
2941       //              R  S     L     C    Dx    Dy
2942       handleTEX(dst0, 1, 1, 0x03, 0x0f, 0x00, 0x00);
2943       break;
2944    case TGSI_OPCODE_TXD:
2945       handleTEX(dst0, 3, 3, 0x03, 0x0f, 0x10, 0x20);
2946       break;
2947    case TGSI_OPCODE_TG4:
2948       handleTEX(dst0, 2, 2, 0x03, 0x0f, 0x00, 0x00);
2949       break;
2950    case TGSI_OPCODE_TEX2:
2951       handleTEX(dst0, 2, 2, 0x03, 0x10, 0x00, 0x00);
2952       break;
2953    case TGSI_OPCODE_TXB2:
2954    case TGSI_OPCODE_TXL2:
2955       handleTEX(dst0, 2, 2, 0x10, 0x0f, 0x00, 0x00);
2956       break;
2957    case TGSI_OPCODE_SAMPLE:
2958    case TGSI_OPCODE_SAMPLE_B:
2959    case TGSI_OPCODE_SAMPLE_D:
2960    case TGSI_OPCODE_SAMPLE_L:
2961    case TGSI_OPCODE_SAMPLE_C:
2962    case TGSI_OPCODE_SAMPLE_C_LZ:
2963       handleTEX(dst0, 1, 2, 0x30, 0x30, 0x30, 0x40);
2964       break;
2965    case TGSI_OPCODE_TXF:
2966       handleTXF(dst0, 1, 0x03);
2967       break;
2968    case TGSI_OPCODE_SAMPLE_I:
2969       handleTXF(dst0, 1, 0x03);
2970       break;
2971    case TGSI_OPCODE_SAMPLE_I_MS:
2972       handleTXF(dst0, 1, 0x20);
2973       break;
2974    case TGSI_OPCODE_TXQ:
2975    case TGSI_OPCODE_SVIEWINFO:
2976       handleTXQ(dst0, TXQ_DIMS, 1);
2977       break;
2978    case TGSI_OPCODE_TXQS:
2979       // The TXQ_TYPE query returns samples in its 3rd arg, but we need it to
2980       // be in .x
2981       dst0[1] = dst0[2] = dst0[3] = NULL;
2982       std::swap(dst0[0], dst0[2]);
2983       handleTXQ(dst0, TXQ_TYPE, 0);
2984       std::swap(dst0[0], dst0[2]);
2985       break;
2986    case TGSI_OPCODE_F2I:
2987    case TGSI_OPCODE_F2U:
2988       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2989          mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c))->rnd = ROUND_Z;
2990       break;
2991    case TGSI_OPCODE_I2F:
2992    case TGSI_OPCODE_U2F:
2993       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2994          mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c));
2995       break;
2996    case TGSI_OPCODE_PK2H:
2997       val0 = getScratch();
2998       val1 = getScratch();
2999       mkCvt(OP_CVT, TYPE_F16, val0, TYPE_F32, fetchSrc(0, 0));
3000       mkCvt(OP_CVT, TYPE_F16, val1, TYPE_F32, fetchSrc(0, 1));
3001       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3002          mkOp3(OP_INSBF, TYPE_U32, dst0[c], val1, mkImm(0x1010), val0);
3003       break;
3004    case TGSI_OPCODE_UP2H:
3005       src0 = fetchSrc(0, 0);
3006       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3007          geni = mkCvt(OP_CVT, TYPE_F32, dst0[c], TYPE_F16, src0);
3008          geni->subOp = c & 1;
3009       }
3010       break;
3011    case TGSI_OPCODE_EMIT:
3012       /* export the saved viewport index */
3013       if (viewport != NULL) {
3014          Symbol *vpSym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_U32,
3015                                   info->out[info->io.viewportId].slot[0] * 4);
3016          mkStore(OP_EXPORT, TYPE_U32, vpSym, NULL, viewport);
3017       }
3018       /* fallthrough */
3019    case TGSI_OPCODE_ENDPRIM:
3020    {
3021       // get vertex stream (must be immediate)
3022       unsigned int stream = tgsi.getSrc(0).getValueU32(0, info);
3023       if (stream && op == OP_RESTART)
3024          break;
3025       src0 = mkImm(stream);
3026       mkOp1(op, TYPE_U32, NULL, src0)->fixed = 1;
3027       break;
3028    }
3029    case TGSI_OPCODE_IF:
3030    case TGSI_OPCODE_UIF:
3031    {
3032       BasicBlock *ifBB = new BasicBlock(func);
3033
3034       bb->cfg.attach(&ifBB->cfg, Graph::Edge::TREE);
3035       condBBs.push(bb);
3036       joinBBs.push(bb);
3037
3038       mkFlow(OP_BRA, NULL, CC_NOT_P, fetchSrc(0, 0))->setType(srcTy);
3039
3040       setPosition(ifBB, true);
3041    }
3042       break;
3043    case TGSI_OPCODE_ELSE:
3044    {
3045       BasicBlock *elseBB = new BasicBlock(func);
3046       BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
3047
3048       forkBB->cfg.attach(&elseBB->cfg, Graph::Edge::TREE);
3049       condBBs.push(bb);
3050
3051       forkBB->getExit()->asFlow()->target.bb = elseBB;
3052       if (!bb->isTerminated())
3053          mkFlow(OP_BRA, NULL, CC_ALWAYS, NULL);
3054
3055       setPosition(elseBB, true);
3056    }
3057       break;
3058    case TGSI_OPCODE_ENDIF:
3059    {
3060       BasicBlock *convBB = new BasicBlock(func);
3061       BasicBlock *prevBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
3062       BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(joinBBs.pop().u.p);
3063
3064       if (!bb->isTerminated()) {
3065          // we only want join if none of the clauses ended with CONT/BREAK/RET
3066          if (prevBB->getExit()->op == OP_BRA && joinBBs.getSize() < 6)
3067             insertConvergenceOps(convBB, forkBB);
3068          mkFlow(OP_BRA, convBB, CC_ALWAYS, NULL);
3069          bb->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
3070       }
3071
3072       if (prevBB->getExit()->op == OP_BRA) {
3073          prevBB->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
3074          prevBB->getExit()->asFlow()->target.bb = convBB;
3075       }
3076       setPosition(convBB, true);
3077    }
3078       break;
3079    case TGSI_OPCODE_BGNLOOP:
3080    {
3081       BasicBlock *lbgnBB = new BasicBlock(func);
3082       BasicBlock *lbrkBB = new BasicBlock(func);
3083
3084       loopBBs.push(lbgnBB);
3085       breakBBs.push(lbrkBB);
3086       if (loopBBs.getSize() > func->loopNestingBound)
3087          func->loopNestingBound++;
3088
3089       mkFlow(OP_PREBREAK, lbrkBB, CC_ALWAYS, NULL);
3090
3091       bb->cfg.attach(&lbgnBB->cfg, Graph::Edge::TREE);
3092       setPosition(lbgnBB, true);
3093       mkFlow(OP_PRECONT, lbgnBB, CC_ALWAYS, NULL);
3094    }
3095       break;
3096    case TGSI_OPCODE_ENDLOOP:
3097    {
3098       BasicBlock *loopBB = reinterpret_cast<BasicBlock *>(loopBBs.pop().u.p);
3099
3100       if (!bb->isTerminated()) {
3101          mkFlow(OP_CONT, loopBB, CC_ALWAYS, NULL);
3102          bb->cfg.attach(&loopBB->cfg, Graph::Edge::BACK);
3103       }
3104       setPosition(reinterpret_cast<BasicBlock *>(breakBBs.pop().u.p), true);
3105
3106       // If the loop never breaks (e.g. only has RET's inside), then there
3107       // will be no way to get to the break bb. However BGNLOOP will have
3108       // already made a PREBREAK to it, so it must be in the CFG.
3109       if (getBB()->cfg.incidentCount() == 0)
3110          loopBB->cfg.attach(&getBB()->cfg, Graph::Edge::TREE);
3111    }
3112       break;
3113    case TGSI_OPCODE_BRK:
3114    {
3115       if (bb->isTerminated())
3116          break;
3117       BasicBlock *brkBB = reinterpret_cast<BasicBlock *>(breakBBs.peek().u.p);
3118       mkFlow(OP_BREAK, brkBB, CC_ALWAYS, NULL);
3119       bb->cfg.attach(&brkBB->cfg, Graph::Edge::CROSS);
3120    }
3121       break;
3122    case TGSI_OPCODE_CONT:
3123    {
3124       if (bb->isTerminated())
3125          break;
3126       BasicBlock *contBB = reinterpret_cast<BasicBlock *>(loopBBs.peek().u.p);
3127       mkFlow(OP_CONT, contBB, CC_ALWAYS, NULL);
3128       contBB->explicitCont = true;
3129       bb->cfg.attach(&contBB->cfg, Graph::Edge::BACK);
3130    }
3131       break;
3132    case TGSI_OPCODE_BGNSUB:
3133    {
3134       Subroutine *s = getSubroutine(ip);
3135       BasicBlock *entry = new BasicBlock(s->f);
3136       BasicBlock *leave = new BasicBlock(s->f);
3137
3138       // multiple entrypoints possible, keep the graph connected
3139       if (prog->getType() == Program::TYPE_COMPUTE)
3140          prog->main->call.attach(&s->f->call, Graph::Edge::TREE);
3141
3142       sub.cur = s;
3143       s->f->setEntry(entry);
3144       s->f->setExit(leave);
3145       setPosition(entry, true);
3146       return true;
3147    }
3148    case TGSI_OPCODE_ENDSUB:
3149    {
3150       sub.cur = getSubroutine(prog->main);
3151       setPosition(BasicBlock::get(sub.cur->f->cfg.getRoot()), true);
3152       return true;
3153    }
3154    case TGSI_OPCODE_CAL:
3155    {
3156       Subroutine *s = getSubroutine(tgsi.getLabel());
3157       mkFlow(OP_CALL, s->f, CC_ALWAYS, NULL);
3158       func->call.attach(&s->f->call, Graph::Edge::TREE);
3159       return true;
3160    }
3161    case TGSI_OPCODE_RET:
3162    {
3163       if (bb->isTerminated())
3164          return true;
3165       BasicBlock *leave = BasicBlock::get(func->cfgExit);
3166
3167       if (!isEndOfSubroutine(ip + 1)) {
3168          // insert a PRERET at the entry if this is an early return
3169          // (only needed for sharing code in the epilogue)
3170          BasicBlock *pos = getBB();
3171          setPosition(BasicBlock::get(func->cfg.getRoot()), false);
3172          mkFlow(OP_PRERET, leave, CC_ALWAYS, NULL)->fixed = 1;
3173          setPosition(pos, true);
3174       }
3175       mkFlow(OP_RET, NULL, CC_ALWAYS, NULL)->fixed = 1;
3176       bb->cfg.attach(&leave->cfg, Graph::Edge::CROSS);
3177    }
3178       break;
3179    case TGSI_OPCODE_END:
3180    {
3181       // attach and generate epilogue code
3182       BasicBlock *epilogue = BasicBlock::get(func->cfgExit);
3183       bb->cfg.attach(&epilogue->cfg, Graph::Edge::TREE);
3184       setPosition(epilogue, true);
3185       if (prog->getType() == Program::TYPE_FRAGMENT)
3186          exportOutputs();
3187       if (info->io.genUserClip > 0)
3188          handleUserClipPlanes();
3189       mkOp(OP_EXIT, TYPE_NONE, NULL)->terminator = 1;
3190    }
3191       break;
3192    case TGSI_OPCODE_SWITCH:
3193    case TGSI_OPCODE_CASE:
3194       ERROR("switch/case opcode encountered, should have been lowered\n");
3195       abort();
3196       break;
3197    case TGSI_OPCODE_LOAD:
3198       handleLOAD(dst0);
3199       break;
3200    case TGSI_OPCODE_STORE:
3201       handleSTORE();
3202       break;
3203    case TGSI_OPCODE_BARRIER:
3204       geni = mkOp2(OP_BAR, TYPE_U32, NULL, mkImm(0), mkImm(0));
3205       geni->fixed = 1;
3206       geni->subOp = NV50_IR_SUBOP_BAR_SYNC;
3207       break;
3208    case TGSI_OPCODE_MFENCE:
3209    case TGSI_OPCODE_LFENCE:
3210    case TGSI_OPCODE_SFENCE:
3211       geni = mkOp(OP_MEMBAR, TYPE_NONE, NULL);
3212       geni->fixed = 1;
3213       geni->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode());
3214       break;
3215    case TGSI_OPCODE_MEMBAR:
3216       geni = mkOp(OP_MEMBAR, TYPE_NONE, NULL);
3217       geni->fixed = 1;
3218       if (tgsi.getSrc(0).getValueU32(0, info) & TGSI_MEMBAR_THREAD_GROUP)
3219          geni->subOp = NV50_IR_SUBOP_MEMBAR(M, CTA);
3220       else
3221          geni->subOp = NV50_IR_SUBOP_MEMBAR(M, GL);
3222       break;
3223    case TGSI_OPCODE_ATOMUADD:
3224    case TGSI_OPCODE_ATOMXCHG:
3225    case TGSI_OPCODE_ATOMCAS:
3226    case TGSI_OPCODE_ATOMAND:
3227    case TGSI_OPCODE_ATOMOR:
3228    case TGSI_OPCODE_ATOMXOR:
3229    case TGSI_OPCODE_ATOMUMIN:
3230    case TGSI_OPCODE_ATOMIMIN:
3231    case TGSI_OPCODE_ATOMUMAX:
3232    case TGSI_OPCODE_ATOMIMAX:
3233       handleATOM(dst0, dstTy, tgsi::opcodeToSubOp(tgsi.getOpcode()));
3234       break;
3235    case TGSI_OPCODE_RESQ:
3236       geni = mkOp1(OP_SUQ, TYPE_U32, dst0[0],
3237                    makeSym(TGSI_FILE_BUFFER, tgsi.getSrc(0).getIndex(0), -1, 0, 0));
3238       if (tgsi.getSrc(0).isIndirect(0))
3239          geni->setIndirect(0, 1, fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, 0));
3240       break;
3241    case TGSI_OPCODE_IBFE:
3242    case TGSI_OPCODE_UBFE:
3243       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3244          src0 = fetchSrc(0, c);
3245          if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE &&
3246              tgsi.getSrc(2).getFile() == TGSI_FILE_IMMEDIATE) {
3247             src1 = loadImm(NULL, tgsi.getSrc(2).getValueU32(c, info) << 8 |
3248                            tgsi.getSrc(1).getValueU32(c, info));
3249          } else {
3250             src1 = fetchSrc(1, c);
3251             src2 = fetchSrc(2, c);
3252             mkOp3(OP_INSBF, TYPE_U32, src1, src2, mkImm(0x808), src1);
3253          }
3254          mkOp2(OP_EXTBF, dstTy, dst0[c], src0, src1);
3255       }
3256       break;
3257    case TGSI_OPCODE_BFI:
3258       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3259          src0 = fetchSrc(0, c);
3260          src1 = fetchSrc(1, c);
3261          src2 = fetchSrc(2, c);
3262          src3 = fetchSrc(3, c);
3263          mkOp3(OP_INSBF, TYPE_U32, src2, src3, mkImm(0x808), src2);
3264          mkOp3(OP_INSBF, TYPE_U32, dst0[c], src1, src2, src0);
3265       }
3266       break;
3267    case TGSI_OPCODE_LSB:
3268       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3269          src0 = fetchSrc(0, c);
3270          geni = mkOp2(OP_EXTBF, TYPE_U32, src0, src0, mkImm(0x2000));
3271          geni->subOp = NV50_IR_SUBOP_EXTBF_REV;
3272          geni = mkOp1(OP_BFIND, TYPE_U32, dst0[c], src0);
3273          geni->subOp = NV50_IR_SUBOP_BFIND_SAMT;
3274       }
3275       break;
3276    case TGSI_OPCODE_IMSB:
3277    case TGSI_OPCODE_UMSB:
3278       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3279          src0 = fetchSrc(0, c);
3280          mkOp1(OP_BFIND, srcTy, dst0[c], src0);
3281       }
3282       break;
3283    case TGSI_OPCODE_BREV:
3284       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3285          src0 = fetchSrc(0, c);
3286          geni = mkOp2(OP_EXTBF, TYPE_U32, dst0[c], src0, mkImm(0x2000));
3287          geni->subOp = NV50_IR_SUBOP_EXTBF_REV;
3288       }
3289       break;
3290    case TGSI_OPCODE_POPC:
3291       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3292          src0 = fetchSrc(0, c);
3293          mkOp2(OP_POPCNT, TYPE_U32, dst0[c], src0, src0);
3294       }
3295       break;
3296    case TGSI_OPCODE_INTERP_CENTROID:
3297    case TGSI_OPCODE_INTERP_SAMPLE:
3298    case TGSI_OPCODE_INTERP_OFFSET:
3299       handleINTERP(dst0);
3300       break;
3301    case TGSI_OPCODE_D2I:
3302    case TGSI_OPCODE_D2U:
3303    case TGSI_OPCODE_D2F: {
3304       int pos = 0;
3305       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3306          Value *dreg = getSSA(8);
3307          src0 = fetchSrc(0, pos);
3308          src1 = fetchSrc(0, pos + 1);
3309          mkOp2(OP_MERGE, TYPE_U64, dreg, src0, src1);
3310          mkCvt(OP_CVT, dstTy, dst0[c], srcTy, dreg);
3311          pos += 2;
3312       }
3313       break;
3314    }
3315    case TGSI_OPCODE_I2D:
3316    case TGSI_OPCODE_U2D:
3317    case TGSI_OPCODE_F2D:
3318       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3319          Value *dreg = getSSA(8);
3320          mkCvt(OP_CVT, dstTy, dreg, srcTy, fetchSrc(0, c / 2));
3321          mkSplit(&dst0[c], 4, dreg);
3322          c++;
3323       }
3324       break;
3325    case TGSI_OPCODE_DABS:
3326    case TGSI_OPCODE_DNEG:
3327    case TGSI_OPCODE_DRCP:
3328    case TGSI_OPCODE_DSQRT:
3329    case TGSI_OPCODE_DRSQ:
3330    case TGSI_OPCODE_DTRUNC:
3331    case TGSI_OPCODE_DCEIL:
3332    case TGSI_OPCODE_DFLR:
3333       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3334          src0 = getSSA(8);
3335          Value *dst = getSSA(8), *tmp[2];
3336          tmp[0] = fetchSrc(0, c);
3337          tmp[1] = fetchSrc(0, c + 1);
3338          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3339          mkOp1(op, dstTy, dst, src0);
3340          mkSplit(&dst0[c], 4, dst);
3341          c++;
3342       }
3343       break;
3344    case TGSI_OPCODE_DFRAC:
3345       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3346          src0 = getSSA(8);
3347          Value *dst = getSSA(8), *tmp[2];
3348          tmp[0] = fetchSrc(0, c);
3349          tmp[1] = fetchSrc(0, c + 1);
3350          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3351          mkOp1(OP_FLOOR, TYPE_F64, dst, src0);
3352          mkOp2(OP_SUB, TYPE_F64, dst, src0, dst);
3353          mkSplit(&dst0[c], 4, dst);
3354          c++;
3355       }
3356       break;
3357    case TGSI_OPCODE_DSLT:
3358    case TGSI_OPCODE_DSGE:
3359    case TGSI_OPCODE_DSEQ:
3360    case TGSI_OPCODE_DSNE: {
3361       int pos = 0;
3362       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3363          Value *tmp[2];
3364
3365          src0 = getSSA(8);
3366          src1 = getSSA(8);
3367          tmp[0] = fetchSrc(0, pos);
3368          tmp[1] = fetchSrc(0, pos + 1);
3369          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3370          tmp[0] = fetchSrc(1, pos);
3371          tmp[1] = fetchSrc(1, pos + 1);
3372          mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
3373          mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], srcTy, src0, src1);
3374          pos += 2;
3375       }
3376       break;
3377    }
3378    case TGSI_OPCODE_DADD:
3379    case TGSI_OPCODE_DMUL:
3380    case TGSI_OPCODE_DMAX:
3381    case TGSI_OPCODE_DMIN:
3382       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3383          src0 = getSSA(8);
3384          src1 = getSSA(8);
3385          Value *dst = getSSA(8), *tmp[2];
3386          tmp[0] = fetchSrc(0, c);
3387          tmp[1] = fetchSrc(0, c + 1);
3388          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3389          tmp[0] = fetchSrc(1, c);
3390          tmp[1] = fetchSrc(1, c + 1);
3391          mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
3392          mkOp2(op, dstTy, dst, src0, src1);
3393          mkSplit(&dst0[c], 4, dst);
3394          c++;
3395       }
3396       break;
3397    case TGSI_OPCODE_DMAD:
3398       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3399          src0 = getSSA(8);
3400          src1 = getSSA(8);
3401          src2 = getSSA(8);
3402          Value *dst = getSSA(8), *tmp[2];
3403          tmp[0] = fetchSrc(0, c);
3404          tmp[1] = fetchSrc(0, c + 1);
3405          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3406          tmp[0] = fetchSrc(1, c);
3407          tmp[1] = fetchSrc(1, c + 1);
3408          mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
3409          tmp[0] = fetchSrc(2, c);
3410          tmp[1] = fetchSrc(2, c + 1);
3411          mkOp2(OP_MERGE, TYPE_U64, src2, tmp[0], tmp[1]);
3412          mkOp3(op, dstTy, dst, src0, src1, src2);
3413          mkSplit(&dst0[c], 4, dst);
3414          c++;
3415       }
3416       break;
3417    case TGSI_OPCODE_DROUND:
3418       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3419          src0 = getSSA(8);
3420          Value *dst = getSSA(8), *tmp[2];
3421          tmp[0] = fetchSrc(0, c);
3422          tmp[1] = fetchSrc(0, c + 1);
3423          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3424          mkCvt(OP_CVT, TYPE_F64, dst, TYPE_F64, src0)
3425          ->rnd = ROUND_NI;
3426          mkSplit(&dst0[c], 4, dst);
3427          c++;
3428       }
3429       break;
3430    case TGSI_OPCODE_DSSG:
3431       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3432          src0 = getSSA(8);
3433          Value *dst = getSSA(8), *dstF32 = getSSA(), *tmp[2];
3434          tmp[0] = fetchSrc(0, c);
3435          tmp[1] = fetchSrc(0, c + 1);
3436          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3437
3438          val0 = getScratch();
3439          val1 = getScratch();
3440          // The zero is wrong here since it's only 32-bit, but it works out in
3441          // the end since it gets replaced with $r63.
3442          mkCmp(OP_SET, CC_GT, TYPE_F32, val0, TYPE_F64, src0, zero);
3443          mkCmp(OP_SET, CC_LT, TYPE_F32, val1, TYPE_F64, src0, zero);
3444          mkOp2(OP_SUB, TYPE_F32, dstF32, val0, val1);
3445          mkCvt(OP_CVT, TYPE_F64, dst, TYPE_F32, dstF32);
3446          mkSplit(&dst0[c], 4, dst);
3447          c++;
3448       }
3449       break;
3450    default:
3451       ERROR("unhandled TGSI opcode: %u\n", tgsi.getOpcode());
3452       assert(0);
3453       break;
3454    }
3455
3456    if (tgsi.dstCount()) {
3457       for (c = 0; c < 4; ++c) {
3458          if (!dst0[c])
3459             continue;
3460          if (dst0[c] != rDst0[c])
3461             mkMov(rDst0[c], dst0[c]);
3462          storeDst(0, c, rDst0[c]);
3463       }
3464    }
3465    vtxBaseValid = 0;
3466
3467    return true;
3468 }
3469
3470 void
3471 Converter::handleUserClipPlanes()
3472 {
3473    Value *res[8];
3474    int n, i, c;
3475
3476    for (c = 0; c < 4; ++c) {
3477       for (i = 0; i < info->io.genUserClip; ++i) {
3478          Symbol *sym = mkSymbol(FILE_MEMORY_CONST, info->io.auxCBSlot,
3479                                 TYPE_F32, info->io.ucpBase + i * 16 + c * 4);
3480          Value *ucp = mkLoadv(TYPE_F32, sym, NULL);
3481          if (c == 0)
3482             res[i] = mkOp2v(OP_MUL, TYPE_F32, getScratch(), clipVtx[c], ucp);
3483          else
3484             mkOp3(OP_MAD, TYPE_F32, res[i], clipVtx[c], ucp, res[i]);
3485       }
3486    }
3487
3488    const int first = info->numOutputs - (info->io.genUserClip + 3) / 4;
3489
3490    for (i = 0; i < info->io.genUserClip; ++i) {
3491       n = i / 4 + first;
3492       c = i % 4;
3493       Symbol *sym =
3494          mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32, info->out[n].slot[c] * 4);
3495       mkStore(OP_EXPORT, TYPE_F32, sym, NULL, res[i]);
3496    }
3497 }
3498
3499 void
3500 Converter::exportOutputs()
3501 {
3502    for (unsigned int i = 0; i < info->numOutputs; ++i) {
3503       for (unsigned int c = 0; c < 4; ++c) {
3504          if (!oData.exists(sub.cur->values, i, c))
3505             continue;
3506          Symbol *sym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32,
3507                                 info->out[i].slot[c] * 4);
3508          Value *val = oData.load(sub.cur->values, i, c, NULL);
3509          if (val)
3510             mkStore(OP_EXPORT, TYPE_F32, sym, NULL, val);
3511       }
3512    }
3513 }
3514
3515 Converter::Converter(Program *ir, const tgsi::Source *code) : BuildUtil(ir),
3516      code(code),
3517      tgsi(NULL),
3518      tData(this), lData(this), aData(this), pData(this), oData(this)
3519 {
3520    info = code->info;
3521
3522    const unsigned tSize = code->fileSize(TGSI_FILE_TEMPORARY);
3523    const unsigned pSize = code->fileSize(TGSI_FILE_PREDICATE);
3524    const unsigned aSize = code->fileSize(TGSI_FILE_ADDRESS);
3525    const unsigned oSize = code->fileSize(TGSI_FILE_OUTPUT);
3526
3527    tData.setup(TGSI_FILE_TEMPORARY, 0, 0, tSize, 4, 4, FILE_GPR, 0);
3528    lData.setup(TGSI_FILE_TEMPORARY, 1, 0, tSize, 4, 4, FILE_MEMORY_LOCAL, 0);
3529    pData.setup(TGSI_FILE_PREDICATE, 0, 0, pSize, 4, 4, FILE_PREDICATE, 0);
3530    aData.setup(TGSI_FILE_ADDRESS, 0, 0, aSize, 4, 4, FILE_GPR, 0);
3531    oData.setup(TGSI_FILE_OUTPUT, 0, 0, oSize, 4, 4, FILE_GPR, 0);
3532
3533    zero = mkImm((uint32_t)0);
3534
3535    vtxBaseValid = 0;
3536 }
3537
3538 Converter::~Converter()
3539 {
3540 }
3541
3542 inline const Converter::Location *
3543 Converter::BindArgumentsPass::getValueLocation(Subroutine *s, Value *v)
3544 {
3545    ValueMap::l_iterator it = s->values.l.find(v);
3546    return it == s->values.l.end() ? NULL : &it->second;
3547 }
3548
3549 template<typename T> inline void
3550 Converter::BindArgumentsPass::updateCallArgs(
3551    Instruction *i, void (Instruction::*setArg)(int, Value *),
3552    T (Function::*proto))
3553 {
3554    Function *g = i->asFlow()->target.fn;
3555    Subroutine *subg = conv.getSubroutine(g);
3556
3557    for (unsigned a = 0; a < (g->*proto).size(); ++a) {
3558       Value *v = (g->*proto)[a].get();
3559       const Converter::Location &l = *getValueLocation(subg, v);
3560       Converter::DataArray *array = conv.getArrayForFile(l.array, l.arrayIdx);
3561
3562       (i->*setArg)(a, array->acquire(sub->values, l.i, l.c));
3563    }
3564 }
3565
3566 template<typename T> inline void
3567 Converter::BindArgumentsPass::updatePrototype(
3568    BitSet *set, void (Function::*updateSet)(), T (Function::*proto))
3569 {
3570    (func->*updateSet)();
3571
3572    for (unsigned i = 0; i < set->getSize(); ++i) {
3573       Value *v = func->getLValue(i);
3574       const Converter::Location *l = getValueLocation(sub, v);
3575
3576       // only include values with a matching TGSI register
3577       if (set->test(i) && l && !conv.code->locals.count(*l))
3578          (func->*proto).push_back(v);
3579    }
3580 }
3581
3582 bool
3583 Converter::BindArgumentsPass::visit(Function *f)
3584 {
3585    sub = conv.getSubroutine(f);
3586
3587    for (ArrayList::Iterator bi = f->allBBlocks.iterator();
3588         !bi.end(); bi.next()) {
3589       for (Instruction *i = BasicBlock::get(bi)->getFirst();
3590            i; i = i->next) {
3591          if (i->op == OP_CALL && !i->asFlow()->builtin) {
3592             updateCallArgs(i, &Instruction::setSrc, &Function::ins);
3593             updateCallArgs(i, &Instruction::setDef, &Function::outs);
3594          }
3595       }
3596    }
3597
3598    if (func == prog->main && prog->getType() != Program::TYPE_COMPUTE)
3599       return true;
3600    updatePrototype(&BasicBlock::get(f->cfg.getRoot())->liveSet,
3601                    &Function::buildLiveSets, &Function::ins);
3602    updatePrototype(&BasicBlock::get(f->cfgExit)->defSet,
3603                    &Function::buildDefSets, &Function::outs);
3604
3605    return true;
3606 }
3607
3608 bool
3609 Converter::run()
3610 {
3611    BasicBlock *entry = new BasicBlock(prog->main);
3612    BasicBlock *leave = new BasicBlock(prog->main);
3613
3614    prog->main->setEntry(entry);
3615    prog->main->setExit(leave);
3616
3617    setPosition(entry, true);
3618    sub.cur = getSubroutine(prog->main);
3619
3620    if (info->io.genUserClip > 0) {
3621       for (int c = 0; c < 4; ++c)
3622          clipVtx[c] = getScratch();
3623    }
3624
3625    switch (prog->getType()) {
3626    case Program::TYPE_TESSELLATION_CONTROL:
3627       outBase = mkOp2v(
3628          OP_SUB, TYPE_U32, getSSA(),
3629          mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_LANEID, 0)),
3630          mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_INVOCATION_ID, 0)));
3631       break;
3632    case Program::TYPE_FRAGMENT: {
3633       Symbol *sv = mkSysVal(SV_POSITION, 3);
3634       fragCoord[3] = mkOp1v(OP_RDSV, TYPE_F32, getSSA(), sv);
3635       mkOp1(OP_RCP, TYPE_F32, fragCoord[3], fragCoord[3]);
3636       break;
3637    }
3638    default:
3639       break;
3640    }
3641
3642    if (info->io.viewportId >= 0)
3643       viewport = getScratch();
3644    else
3645       viewport = NULL;
3646
3647    for (ip = 0; ip < code->scan.num_instructions; ++ip) {
3648       if (!handleInstruction(&code->insns[ip]))
3649          return false;
3650    }
3651
3652    if (!BindArgumentsPass(*this).run(prog))
3653       return false;
3654
3655    return true;
3656 }
3657
3658 } // unnamed namespace
3659
3660 namespace nv50_ir {
3661
3662 bool
3663 Program::makeFromTGSI(struct nv50_ir_prog_info *info)
3664 {
3665    tgsi::Source src(info);
3666    if (!src.scanSource())
3667       return false;
3668    tlsSize = info->bin.tlsSpace;
3669
3670    Converter builder(this, &src);
3671    return builder.run();
3672 }
3673
3674 } // namespace nv50_ir