src/gallium/drivers/nouveau/codegen/nv50_ir_from_tgsi.cpp

   1 /*
   2  * Copyright 2011 Christoph Bumiller
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 #include "tgsi/tgsi_dump.h"
  24 #include "tgsi/tgsi_scan.h"
  25 #include "tgsi/tgsi_util.h"
  26
  27 #include <set>
  28
  29 #include "codegen/nv50_ir.h"
  30 #include "codegen/nv50_ir_util.h"
  31 #include "codegen/nv50_ir_build_util.h"
  32
  33 namespace tgsi {
  34
  35 class Source;
  36
  37 static nv50_ir::operation translateOpcode(uint opcode);
  38 static nv50_ir::DataFile translateFile(uint file);
  39 static nv50_ir::TexTarget translateTexture(uint texTarg);
  40 static nv50_ir::SVSemantic translateSysVal(uint sysval);
  41 static nv50_ir::CacheMode translateCacheMode(uint qualifier);
  42
  43 class Instruction
  44 {
  45 public:
  46    Instruction(const struct tgsi_full_instruction *inst) : insn(inst) { }
  47
  48    class SrcRegister
  49    {
  50    public:
  51       SrcRegister(const struct tgsi_full_src_register *src)
  52          : reg(src->Register),
  53            fsr(src)
  54       { }
  55
  56       SrcRegister(const struct tgsi_src_register& src) : reg(src), fsr(NULL) { }
  57
  58       SrcRegister(const struct tgsi_ind_register& ind)
  59          : reg(tgsi_util_get_src_from_ind(&ind)),
  60            fsr(NULL)
  61       { }
  62
  63       struct tgsi_src_register offsetToSrc(struct tgsi_texture_offset off)
  64       {
  65          struct tgsi_src_register reg;
  66          memset(&reg, 0, sizeof(reg));
  67          reg.Index = off.Index;
  68          reg.File = off.File;
  69          reg.SwizzleX = off.SwizzleX;
  70          reg.SwizzleY = off.SwizzleY;
  71          reg.SwizzleZ = off.SwizzleZ;
  72          return reg;
  73       }
  74
  75       SrcRegister(const struct tgsi_texture_offset& off) :
  76          reg(offsetToSrc(off)),
  77          fsr(NULL)
  78       { }
  79
  80       uint getFile() const { return reg.File; }
  81
  82       bool is2D() const { return reg.Dimension; }
  83
  84       bool isIndirect(int dim) const
  85       {
  86          return (dim && fsr) ? fsr->Dimension.Indirect : reg.Indirect;
  87       }
  88
  89       int getIndex(int dim) const
  90       {
  91          return (dim && fsr) ? fsr->Dimension.Index : reg.Index;
  92       }
  93
  94       int getSwizzle(int chan) const
  95       {
  96          return tgsi_util_get_src_register_swizzle(&reg, chan);
  97       }
  98
  99       int getArrayId() const
 100       {
 101          if (isIndirect(0))
 102             return fsr->Indirect.ArrayID;
 103          return 0;
 104       }
 105
 106       nv50_ir::Modifier getMod(int chan) const;
 107
 108       SrcRegister getIndirect(int dim) const
 109       {
 110          assert(fsr && isIndirect(dim));
 111          if (dim)
 112             return SrcRegister(fsr->DimIndirect);
 113          return SrcRegister(fsr->Indirect);
 114       }
 115
 116       uint32_t getValueU32(int c, const struct nv50_ir_prog_info *info) const
 117       {
 118          assert(reg.File == TGSI_FILE_IMMEDIATE);
 119          assert(!reg.Absolute);
 120          assert(!reg.Negate);
 121          return info->immd.data[reg.Index * 4 + getSwizzle(c)];
 122       }
 123
 124    private:
 125       const struct tgsi_src_register reg;
 126       const struct tgsi_full_src_register *fsr;
 127    };
 128
 129    class DstRegister
 130    {
 131    public:
 132       DstRegister(const struct tgsi_full_dst_register *dst)
 133          : reg(dst->Register),
 134            fdr(dst)
 135       { }
 136
 137       DstRegister(const struct tgsi_dst_register& dst) : reg(dst), fdr(NULL) { }
 138
 139       uint getFile() const { return reg.File; }
 140
 141       bool is2D() const { return reg.Dimension; }
 142
 143       bool isIndirect(int dim) const
 144       {
 145          return (dim && fdr) ? fdr->Dimension.Indirect : reg.Indirect;
 146       }
 147
 148       int getIndex(int dim) const
 149       {
 150          return (dim && fdr) ? fdr->Dimension.Dimension : reg.Index;
 151       }
 152
 153       unsigned int getMask() const { return reg.WriteMask; }
 154
 155       bool isMasked(int chan) const { return !(getMask() & (1 << chan)); }
 156
 157       SrcRegister getIndirect(int dim) const
 158       {
 159          assert(fdr && isIndirect(dim));
 160          if (dim)
 161             return SrcRegister(fdr->DimIndirect);
 162          return SrcRegister(fdr->Indirect);
 163       }
 164
 165       int getArrayId() const
 166       {
 167          if (isIndirect(0))
 168             return fdr->Indirect.ArrayID;
 169          return 0;
 170       }
 171
 172    private:
 173       const struct tgsi_dst_register reg;
 174       const struct tgsi_full_dst_register *fdr;
 175    };
 176
 177    inline uint getOpcode() const { return insn->Instruction.Opcode; }
 178
 179    unsigned int srcCount() const { return insn->Instruction.NumSrcRegs; }
 180    unsigned int dstCount() const { return insn->Instruction.NumDstRegs; }
 181
 182    // mask of used components of source s
 183    unsigned int srcMask(unsigned int s) const;
 184
 185    SrcRegister getSrc(unsigned int s) const
 186    {
 187       assert(s < srcCount());
 188       return SrcRegister(&insn->Src[s]);
 189    }
 190
 191    DstRegister getDst(unsigned int d) const
 192    {
 193       assert(d < dstCount());
 194       return DstRegister(&insn->Dst[d]);
 195    }
 196
 197    SrcRegister getTexOffset(unsigned int i) const
 198    {
 199       assert(i < TGSI_FULL_MAX_TEX_OFFSETS);
 200       return SrcRegister(insn->TexOffsets[i]);
 201    }
 202
 203    unsigned int getNumTexOffsets() const { return insn->Texture.NumOffsets; }
 204
 205    bool checkDstSrcAliasing() const;
 206
 207    inline nv50_ir::operation getOP() const {
 208       return translateOpcode(getOpcode()); }
 209
 210    nv50_ir::DataType inferSrcType() const;
 211    nv50_ir::DataType inferDstType() const;
 212
 213    nv50_ir::CondCode getSetCond() const;
 214
 215    nv50_ir::TexInstruction::Target getTexture(const Source *, int s) const;
 216
 217    nv50_ir::CacheMode getCacheMode() const {
 218       if (!insn->Instruction.Memory)
 219          return nv50_ir::CACHE_CA;
 220       return translateCacheMode(insn->Memory.Qualifier);
 221    }
 222
 223    inline uint getLabel() { return insn->Label.Label; }
 224
 225    unsigned getSaturate() const { return insn->Instruction.Saturate; }
 226
 227    void print() const
 228    {
 229       tgsi_dump_instruction(insn, 1);
 230    }
 231
 232 private:
 233    const struct tgsi_full_instruction *insn;
 234 };
 235
 236 unsigned int Instruction::srcMask(unsigned int s) const
 237 {
 238    unsigned int mask = insn->Dst[0].Register.WriteMask;
 239
 240    switch (insn->Instruction.Opcode) {
 241    case TGSI_OPCODE_COS:
 242    case TGSI_OPCODE_SIN:
 243       return (mask & 0x8) | ((mask & 0x7) ? 0x1 : 0x0);
 244    case TGSI_OPCODE_DP2:
 245       return 0x3;
 246    case TGSI_OPCODE_DP3:
 247       return 0x7;
 248    case TGSI_OPCODE_DP4:
 249    case TGSI_OPCODE_DPH:
 250    case TGSI_OPCODE_KILL_IF: /* WriteMask ignored */
 251       return 0xf;
 252    case TGSI_OPCODE_DST:
 253       return mask & (s ? 0xa : 0x6);
 254    case TGSI_OPCODE_EX2:
 255    case TGSI_OPCODE_EXP:
 256    case TGSI_OPCODE_LG2:
 257    case TGSI_OPCODE_LOG:
 258    case TGSI_OPCODE_POW:
 259    case TGSI_OPCODE_RCP:
 260    case TGSI_OPCODE_RSQ:
 261    case TGSI_OPCODE_SCS:
 262       return 0x1;
 263    case TGSI_OPCODE_IF:
 264    case TGSI_OPCODE_UIF:
 265       return 0x1;
 266    case TGSI_OPCODE_LIT:
 267       return 0xb;
 268    case TGSI_OPCODE_TEX2:
 269    case TGSI_OPCODE_TXB2:
 270    case TGSI_OPCODE_TXL2:
 271       return (s == 0) ? 0xf : 0x3;
 272    case TGSI_OPCODE_TEX:
 273    case TGSI_OPCODE_TXB:
 274    case TGSI_OPCODE_TXD:
 275    case TGSI_OPCODE_TXL:
 276    case TGSI_OPCODE_TXP:
 277    case TGSI_OPCODE_LODQ:
 278    {
 279       const struct tgsi_instruction_texture *tex = &insn->Texture;
 280
 281       assert(insn->Instruction.Texture);
 282
 283       mask = 0x7;
 284       if (insn->Instruction.Opcode != TGSI_OPCODE_TEX &&
 285           insn->Instruction.Opcode != TGSI_OPCODE_TXD)
 286          mask |= 0x8; /* bias, lod or proj */
 287
 288       switch (tex->Texture) {
 289       case TGSI_TEXTURE_1D:
 290          mask &= 0x9;
 291          break;
 292       case TGSI_TEXTURE_SHADOW1D:
 293          mask &= 0xd;
 294          break;
 295       case TGSI_TEXTURE_1D_ARRAY:
 296       case TGSI_TEXTURE_2D:
 297       case TGSI_TEXTURE_RECT:
 298          mask &= 0xb;
 299          break;
 300       case TGSI_TEXTURE_CUBE_ARRAY:
 301       case TGSI_TEXTURE_SHADOW2D_ARRAY:
 302       case TGSI_TEXTURE_SHADOWCUBE:
 303       case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
 304          mask |= 0x8;
 305          break;
 306       default:
 307          break;
 308       }
 309    }
 310       return mask;
 311    case TGSI_OPCODE_XPD:
 312    {
 313       unsigned int x = 0;
 314       if (mask & 1) x |= 0x6;
 315       if (mask & 2) x |= 0x5;
 316       if (mask & 4) x |= 0x3;
 317       return x;
 318    }
 319    case TGSI_OPCODE_D2I:
 320    case TGSI_OPCODE_D2U:
 321    case TGSI_OPCODE_D2F:
 322    case TGSI_OPCODE_DSLT:
 323    case TGSI_OPCODE_DSGE:
 324    case TGSI_OPCODE_DSEQ:
 325    case TGSI_OPCODE_DSNE:
 326       switch (util_bitcount(mask)) {
 327       case 1: return 0x3;
 328       case 2: return 0xf;
 329       default:
 330          assert(!"unexpected mask");
 331          return 0xf;
 332       }
 333    case TGSI_OPCODE_I2D:
 334    case TGSI_OPCODE_U2D:
 335    case TGSI_OPCODE_F2D: {
 336       unsigned int x = 0;
 337       if ((mask & 0x3) == 0x3)
 338          x |= 1;
 339       if ((mask & 0xc) == 0xc)
 340          x |= 2;
 341       return x;
 342    }
 343    case TGSI_OPCODE_PK2H:
 344       return 0x3;
 345    case TGSI_OPCODE_UP2H:
 346       return 0x1;
 347    default:
 348       break;
 349    }
 350
 351    return mask;
 352 }
 353
 354 nv50_ir::Modifier Instruction::SrcRegister::getMod(int chan) const
 355 {
 356    nv50_ir::Modifier m(0);
 357
 358    if (reg.Absolute)
 359       m = m | nv50_ir::Modifier(NV50_IR_MOD_ABS);
 360    if (reg.Negate)
 361       m = m | nv50_ir::Modifier(NV50_IR_MOD_NEG);
 362    return m;
 363 }
 364
 365 static nv50_ir::DataFile translateFile(uint file)
 366 {
 367    switch (file) {
 368    case TGSI_FILE_CONSTANT:        return nv50_ir::FILE_MEMORY_CONST;
 369    case TGSI_FILE_INPUT:           return nv50_ir::FILE_SHADER_INPUT;
 370    case TGSI_FILE_OUTPUT:          return nv50_ir::FILE_SHADER_OUTPUT;
 371    case TGSI_FILE_TEMPORARY:       return nv50_ir::FILE_GPR;
 372    case TGSI_FILE_ADDRESS:         return nv50_ir::FILE_ADDRESS;
 373    case TGSI_FILE_PREDICATE:       return nv50_ir::FILE_PREDICATE;
 374    case TGSI_FILE_IMMEDIATE:       return nv50_ir::FILE_IMMEDIATE;
 375    case TGSI_FILE_SYSTEM_VALUE:    return nv50_ir::FILE_SYSTEM_VALUE;
 376    case TGSI_FILE_BUFFER:          return nv50_ir::FILE_MEMORY_GLOBAL;
 377    case TGSI_FILE_SAMPLER:
 378    case TGSI_FILE_NULL:
 379    default:
 380       return nv50_ir::FILE_NULL;
 381    }
 382 }
 383
 384 static nv50_ir::SVSemantic translateSysVal(uint sysval)
 385 {
 386    switch (sysval) {
 387    case TGSI_SEMANTIC_FACE:       return nv50_ir::SV_FACE;
 388    case TGSI_SEMANTIC_PSIZE:      return nv50_ir::SV_POINT_SIZE;
 389    case TGSI_SEMANTIC_PRIMID:     return nv50_ir::SV_PRIMITIVE_ID;
 390    case TGSI_SEMANTIC_INSTANCEID: return nv50_ir::SV_INSTANCE_ID;
 391    case TGSI_SEMANTIC_VERTEXID:   return nv50_ir::SV_VERTEX_ID;
 392    case TGSI_SEMANTIC_GRID_SIZE:  return nv50_ir::SV_NCTAID;
 393    case TGSI_SEMANTIC_BLOCK_ID:   return nv50_ir::SV_CTAID;
 394    case TGSI_SEMANTIC_BLOCK_SIZE: return nv50_ir::SV_NTID;
 395    case TGSI_SEMANTIC_THREAD_ID:  return nv50_ir::SV_TID;
 396    case TGSI_SEMANTIC_SAMPLEID:   return nv50_ir::SV_SAMPLE_INDEX;
 397    case TGSI_SEMANTIC_SAMPLEPOS:  return nv50_ir::SV_SAMPLE_POS;
 398    case TGSI_SEMANTIC_SAMPLEMASK: return nv50_ir::SV_SAMPLE_MASK;
 399    case TGSI_SEMANTIC_INVOCATIONID: return nv50_ir::SV_INVOCATION_ID;
 400    case TGSI_SEMANTIC_TESSCOORD:  return nv50_ir::SV_TESS_COORD;
 401    case TGSI_SEMANTIC_TESSOUTER:  return nv50_ir::SV_TESS_OUTER;
 402    case TGSI_SEMANTIC_TESSINNER:  return nv50_ir::SV_TESS_INNER;
 403    case TGSI_SEMANTIC_VERTICESIN: return nv50_ir::SV_VERTEX_COUNT;
 404    case TGSI_SEMANTIC_HELPER_INVOCATION: return nv50_ir::SV_THREAD_KILL;
 405    case TGSI_SEMANTIC_BASEVERTEX: return nv50_ir::SV_BASEVERTEX;
 406    case TGSI_SEMANTIC_BASEINSTANCE: return nv50_ir::SV_BASEINSTANCE;
 407    case TGSI_SEMANTIC_DRAWID:     return nv50_ir::SV_DRAWID;
 408    default:
 409       assert(0);
 410       return nv50_ir::SV_CLOCK;
 411    }
 412 }
 413
 414 #define NV50_IR_TEX_TARG_CASE(a, b) \
 415    case TGSI_TEXTURE_##a: return nv50_ir::TEX_TARGET_##b;
 416
 417 static nv50_ir::TexTarget translateTexture(uint tex)
 418 {
 419    switch (tex) {
 420    NV50_IR_TEX_TARG_CASE(1D, 1D);
 421    NV50_IR_TEX_TARG_CASE(2D, 2D);
 422    NV50_IR_TEX_TARG_CASE(2D_MSAA, 2D_MS);
 423    NV50_IR_TEX_TARG_CASE(3D, 3D);
 424    NV50_IR_TEX_TARG_CASE(CUBE, CUBE);
 425    NV50_IR_TEX_TARG_CASE(RECT, RECT);
 426    NV50_IR_TEX_TARG_CASE(1D_ARRAY, 1D_ARRAY);
 427    NV50_IR_TEX_TARG_CASE(2D_ARRAY, 2D_ARRAY);
 428    NV50_IR_TEX_TARG_CASE(2D_ARRAY_MSAA, 2D_MS_ARRAY);
 429    NV50_IR_TEX_TARG_CASE(CUBE_ARRAY, CUBE_ARRAY);
 430    NV50_IR_TEX_TARG_CASE(SHADOW1D, 1D_SHADOW);
 431    NV50_IR_TEX_TARG_CASE(SHADOW2D, 2D_SHADOW);
 432    NV50_IR_TEX_TARG_CASE(SHADOWCUBE, CUBE_SHADOW);
 433    NV50_IR_TEX_TARG_CASE(SHADOWRECT, RECT_SHADOW);
 434    NV50_IR_TEX_TARG_CASE(SHADOW1D_ARRAY, 1D_ARRAY_SHADOW);
 435    NV50_IR_TEX_TARG_CASE(SHADOW2D_ARRAY, 2D_ARRAY_SHADOW);
 436    NV50_IR_TEX_TARG_CASE(SHADOWCUBE_ARRAY, CUBE_ARRAY_SHADOW);
 437    NV50_IR_TEX_TARG_CASE(BUFFER, BUFFER);
 438
 439    case TGSI_TEXTURE_UNKNOWN:
 440    default:
 441       assert(!"invalid texture target");
 442       return nv50_ir::TEX_TARGET_2D;
 443    }
 444 }
 445
 446 static nv50_ir::CacheMode translateCacheMode(uint qualifier)
 447 {
 448    if (qualifier & TGSI_MEMORY_VOLATILE)
 449       return nv50_ir::CACHE_CV;
 450    if (qualifier & TGSI_MEMORY_COHERENT)
 451       return nv50_ir::CACHE_CG;
 452    return nv50_ir::CACHE_CA;
 453 }
 454
 455 nv50_ir::DataType Instruction::inferSrcType() const
 456 {
 457    switch (getOpcode()) {
 458    case TGSI_OPCODE_UIF:
 459    case TGSI_OPCODE_AND:
 460    case TGSI_OPCODE_OR:
 461    case TGSI_OPCODE_XOR:
 462    case TGSI_OPCODE_NOT:
 463    case TGSI_OPCODE_SHL:
 464    case TGSI_OPCODE_U2F:
 465    case TGSI_OPCODE_U2D:
 466    case TGSI_OPCODE_UADD:
 467    case TGSI_OPCODE_UDIV:
 468    case TGSI_OPCODE_UMOD:
 469    case TGSI_OPCODE_UMAD:
 470    case TGSI_OPCODE_UMUL:
 471    case TGSI_OPCODE_UMUL_HI:
 472    case TGSI_OPCODE_UMAX:
 473    case TGSI_OPCODE_UMIN:
 474    case TGSI_OPCODE_USEQ:
 475    case TGSI_OPCODE_USGE:
 476    case TGSI_OPCODE_USLT:
 477    case TGSI_OPCODE_USNE:
 478    case TGSI_OPCODE_USHR:
 479    case TGSI_OPCODE_ATOMUADD:
 480    case TGSI_OPCODE_ATOMXCHG:
 481    case TGSI_OPCODE_ATOMCAS:
 482    case TGSI_OPCODE_ATOMAND:
 483    case TGSI_OPCODE_ATOMOR:
 484    case TGSI_OPCODE_ATOMXOR:
 485    case TGSI_OPCODE_ATOMUMIN:
 486    case TGSI_OPCODE_ATOMUMAX:
 487    case TGSI_OPCODE_UBFE:
 488    case TGSI_OPCODE_UMSB:
 489    case TGSI_OPCODE_UP2H:
 490       return nv50_ir::TYPE_U32;
 491    case TGSI_OPCODE_I2F:
 492    case TGSI_OPCODE_I2D:
 493    case TGSI_OPCODE_IDIV:
 494    case TGSI_OPCODE_IMUL_HI:
 495    case TGSI_OPCODE_IMAX:
 496    case TGSI_OPCODE_IMIN:
 497    case TGSI_OPCODE_IABS:
 498    case TGSI_OPCODE_INEG:
 499    case TGSI_OPCODE_ISGE:
 500    case TGSI_OPCODE_ISHR:
 501    case TGSI_OPCODE_ISLT:
 502    case TGSI_OPCODE_ISSG:
 503    case TGSI_OPCODE_SAD: // not sure about SAD, but no one has a float version
 504    case TGSI_OPCODE_MOD:
 505    case TGSI_OPCODE_UARL:
 506    case TGSI_OPCODE_ATOMIMIN:
 507    case TGSI_OPCODE_ATOMIMAX:
 508    case TGSI_OPCODE_IBFE:
 509    case TGSI_OPCODE_IMSB:
 510       return nv50_ir::TYPE_S32;
 511    case TGSI_OPCODE_D2F:
 512    case TGSI_OPCODE_D2I:
 513    case TGSI_OPCODE_D2U:
 514    case TGSI_OPCODE_DABS:
 515    case TGSI_OPCODE_DNEG:
 516    case TGSI_OPCODE_DADD:
 517    case TGSI_OPCODE_DMUL:
 518    case TGSI_OPCODE_DMAX:
 519    case TGSI_OPCODE_DMIN:
 520    case TGSI_OPCODE_DSLT:
 521    case TGSI_OPCODE_DSGE:
 522    case TGSI_OPCODE_DSEQ:
 523    case TGSI_OPCODE_DSNE:
 524    case TGSI_OPCODE_DRCP:
 525    case TGSI_OPCODE_DSQRT:
 526    case TGSI_OPCODE_DMAD:
 527    case TGSI_OPCODE_DFRAC:
 528    case TGSI_OPCODE_DRSQ:
 529    case TGSI_OPCODE_DTRUNC:
 530    case TGSI_OPCODE_DCEIL:
 531    case TGSI_OPCODE_DFLR:
 532    case TGSI_OPCODE_DROUND:
 533       return nv50_ir::TYPE_F64;
 534    default:
 535       return nv50_ir::TYPE_F32;
 536    }
 537 }
 538
 539 nv50_ir::DataType Instruction::inferDstType() const
 540 {
 541    switch (getOpcode()) {
 542    case TGSI_OPCODE_D2U:
 543    case TGSI_OPCODE_F2U: return nv50_ir::TYPE_U32;
 544    case TGSI_OPCODE_D2I:
 545    case TGSI_OPCODE_F2I: return nv50_ir::TYPE_S32;
 546    case TGSI_OPCODE_FSEQ:
 547    case TGSI_OPCODE_FSGE:
 548    case TGSI_OPCODE_FSLT:
 549    case TGSI_OPCODE_FSNE:
 550    case TGSI_OPCODE_DSEQ:
 551    case TGSI_OPCODE_DSGE:
 552    case TGSI_OPCODE_DSLT:
 553    case TGSI_OPCODE_DSNE:
 554    case TGSI_OPCODE_PK2H:
 555       return nv50_ir::TYPE_U32;
 556    case TGSI_OPCODE_I2F:
 557    case TGSI_OPCODE_U2F:
 558    case TGSI_OPCODE_D2F:
 559    case TGSI_OPCODE_UP2H:
 560       return nv50_ir::TYPE_F32;
 561    case TGSI_OPCODE_I2D:
 562    case TGSI_OPCODE_U2D:
 563    case TGSI_OPCODE_F2D:
 564       return nv50_ir::TYPE_F64;
 565    default:
 566       return inferSrcType();
 567    }
 568 }
 569
 570 nv50_ir::CondCode Instruction::getSetCond() const
 571 {
 572    using namespace nv50_ir;
 573
 574    switch (getOpcode()) {
 575    case TGSI_OPCODE_SLT:
 576    case TGSI_OPCODE_ISLT:
 577    case TGSI_OPCODE_USLT:
 578    case TGSI_OPCODE_FSLT:
 579    case TGSI_OPCODE_DSLT:
 580       return CC_LT;
 581    case TGSI_OPCODE_SLE:
 582       return CC_LE;
 583    case TGSI_OPCODE_SGE:
 584    case TGSI_OPCODE_ISGE:
 585    case TGSI_OPCODE_USGE:
 586    case TGSI_OPCODE_FSGE:
 587    case TGSI_OPCODE_DSGE:
 588       return CC_GE;
 589    case TGSI_OPCODE_SGT:
 590       return CC_GT;
 591    case TGSI_OPCODE_SEQ:
 592    case TGSI_OPCODE_USEQ:
 593    case TGSI_OPCODE_FSEQ:
 594    case TGSI_OPCODE_DSEQ:
 595       return CC_EQ;
 596    case TGSI_OPCODE_SNE:
 597    case TGSI_OPCODE_FSNE:
 598    case TGSI_OPCODE_DSNE:
 599       return CC_NEU;
 600    case TGSI_OPCODE_USNE:
 601       return CC_NE;
 602    default:
 603       return CC_ALWAYS;
 604    }
 605 }
 606
 607 #define NV50_IR_OPCODE_CASE(a, b) case TGSI_OPCODE_##a: return nv50_ir::OP_##b
 608
 609 static nv50_ir::operation translateOpcode(uint opcode)
 610 {
 611    switch (opcode) {
 612    NV50_IR_OPCODE_CASE(ARL, SHL);
 613    NV50_IR_OPCODE_CASE(MOV, MOV);
 614
 615    NV50_IR_OPCODE_CASE(RCP, RCP);
 616    NV50_IR_OPCODE_CASE(RSQ, RSQ);
 617
 618    NV50_IR_OPCODE_CASE(MUL, MUL);
 619    NV50_IR_OPCODE_CASE(ADD, ADD);
 620
 621    NV50_IR_OPCODE_CASE(MIN, MIN);
 622    NV50_IR_OPCODE_CASE(MAX, MAX);
 623    NV50_IR_OPCODE_CASE(SLT, SET);
 624    NV50_IR_OPCODE_CASE(SGE, SET);
 625    NV50_IR_OPCODE_CASE(MAD, MAD);
 626    NV50_IR_OPCODE_CASE(SUB, SUB);
 627
 628    NV50_IR_OPCODE_CASE(FLR, FLOOR);
 629    NV50_IR_OPCODE_CASE(ROUND, CVT);
 630    NV50_IR_OPCODE_CASE(EX2, EX2);
 631    NV50_IR_OPCODE_CASE(LG2, LG2);
 632    NV50_IR_OPCODE_CASE(POW, POW);
 633
 634    NV50_IR_OPCODE_CASE(ABS, ABS);
 635
 636    NV50_IR_OPCODE_CASE(COS, COS);
 637    NV50_IR_OPCODE_CASE(DDX, DFDX);
 638    NV50_IR_OPCODE_CASE(DDX_FINE, DFDX);
 639    NV50_IR_OPCODE_CASE(DDY, DFDY);
 640    NV50_IR_OPCODE_CASE(DDY_FINE, DFDY);
 641    NV50_IR_OPCODE_CASE(KILL, DISCARD);
 642
 643    NV50_IR_OPCODE_CASE(SEQ, SET);
 644    NV50_IR_OPCODE_CASE(SGT, SET);
 645    NV50_IR_OPCODE_CASE(SIN, SIN);
 646    NV50_IR_OPCODE_CASE(SLE, SET);
 647    NV50_IR_OPCODE_CASE(SNE, SET);
 648    NV50_IR_OPCODE_CASE(TEX, TEX);
 649    NV50_IR_OPCODE_CASE(TXD, TXD);
 650    NV50_IR_OPCODE_CASE(TXP, TEX);
 651
 652    NV50_IR_OPCODE_CASE(CAL, CALL);
 653    NV50_IR_OPCODE_CASE(RET, RET);
 654    NV50_IR_OPCODE_CASE(CMP, SLCT);
 655
 656    NV50_IR_OPCODE_CASE(TXB, TXB);
 657
 658    NV50_IR_OPCODE_CASE(DIV, DIV);
 659
 660    NV50_IR_OPCODE_CASE(TXL, TXL);
 661
 662    NV50_IR_OPCODE_CASE(CEIL, CEIL);
 663    NV50_IR_OPCODE_CASE(I2F, CVT);
 664    NV50_IR_OPCODE_CASE(NOT, NOT);
 665    NV50_IR_OPCODE_CASE(TRUNC, TRUNC);
 666    NV50_IR_OPCODE_CASE(SHL, SHL);
 667
 668    NV50_IR_OPCODE_CASE(AND, AND);
 669    NV50_IR_OPCODE_CASE(OR, OR);
 670    NV50_IR_OPCODE_CASE(MOD, MOD);
 671    NV50_IR_OPCODE_CASE(XOR, XOR);
 672    NV50_IR_OPCODE_CASE(SAD, SAD);
 673    NV50_IR_OPCODE_CASE(TXF, TXF);
 674    NV50_IR_OPCODE_CASE(TXQ, TXQ);
 675    NV50_IR_OPCODE_CASE(TXQS, TXQ);
 676    NV50_IR_OPCODE_CASE(TG4, TXG);
 677    NV50_IR_OPCODE_CASE(LODQ, TXLQ);
 678
 679    NV50_IR_OPCODE_CASE(EMIT, EMIT);
 680    NV50_IR_OPCODE_CASE(ENDPRIM, RESTART);
 681
 682    NV50_IR_OPCODE_CASE(KILL_IF, DISCARD);
 683
 684    NV50_IR_OPCODE_CASE(F2I, CVT);
 685    NV50_IR_OPCODE_CASE(FSEQ, SET);
 686    NV50_IR_OPCODE_CASE(FSGE, SET);
 687    NV50_IR_OPCODE_CASE(FSLT, SET);
 688    NV50_IR_OPCODE_CASE(FSNE, SET);
 689    NV50_IR_OPCODE_CASE(IDIV, DIV);
 690    NV50_IR_OPCODE_CASE(IMAX, MAX);
 691    NV50_IR_OPCODE_CASE(IMIN, MIN);
 692    NV50_IR_OPCODE_CASE(IABS, ABS);
 693    NV50_IR_OPCODE_CASE(INEG, NEG);
 694    NV50_IR_OPCODE_CASE(ISGE, SET);
 695    NV50_IR_OPCODE_CASE(ISHR, SHR);
 696    NV50_IR_OPCODE_CASE(ISLT, SET);
 697    NV50_IR_OPCODE_CASE(F2U, CVT);
 698    NV50_IR_OPCODE_CASE(U2F, CVT);
 699    NV50_IR_OPCODE_CASE(UADD, ADD);
 700    NV50_IR_OPCODE_CASE(UDIV, DIV);
 701    NV50_IR_OPCODE_CASE(UMAD, MAD);
 702    NV50_IR_OPCODE_CASE(UMAX, MAX);
 703    NV50_IR_OPCODE_CASE(UMIN, MIN);
 704    NV50_IR_OPCODE_CASE(UMOD, MOD);
 705    NV50_IR_OPCODE_CASE(UMUL, MUL);
 706    NV50_IR_OPCODE_CASE(USEQ, SET);
 707    NV50_IR_OPCODE_CASE(USGE, SET);
 708    NV50_IR_OPCODE_CASE(USHR, SHR);
 709    NV50_IR_OPCODE_CASE(USLT, SET);
 710    NV50_IR_OPCODE_CASE(USNE, SET);
 711
 712    NV50_IR_OPCODE_CASE(DABS, ABS);
 713    NV50_IR_OPCODE_CASE(DNEG, NEG);
 714    NV50_IR_OPCODE_CASE(DADD, ADD);
 715    NV50_IR_OPCODE_CASE(DMUL, MUL);
 716    NV50_IR_OPCODE_CASE(DMAX, MAX);
 717    NV50_IR_OPCODE_CASE(DMIN, MIN);
 718    NV50_IR_OPCODE_CASE(DSLT, SET);
 719    NV50_IR_OPCODE_CASE(DSGE, SET);
 720    NV50_IR_OPCODE_CASE(DSEQ, SET);
 721    NV50_IR_OPCODE_CASE(DSNE, SET);
 722    NV50_IR_OPCODE_CASE(DRCP, RCP);
 723    NV50_IR_OPCODE_CASE(DSQRT, SQRT);
 724    NV50_IR_OPCODE_CASE(DMAD, MAD);
 725    NV50_IR_OPCODE_CASE(D2I, CVT);
 726    NV50_IR_OPCODE_CASE(D2U, CVT);
 727    NV50_IR_OPCODE_CASE(I2D, CVT);
 728    NV50_IR_OPCODE_CASE(U2D, CVT);
 729    NV50_IR_OPCODE_CASE(DRSQ, RSQ);
 730    NV50_IR_OPCODE_CASE(DTRUNC, TRUNC);
 731    NV50_IR_OPCODE_CASE(DCEIL, CEIL);
 732    NV50_IR_OPCODE_CASE(DFLR, FLOOR);
 733    NV50_IR_OPCODE_CASE(DROUND, CVT);
 734
 735    NV50_IR_OPCODE_CASE(IMUL_HI, MUL);
 736    NV50_IR_OPCODE_CASE(UMUL_HI, MUL);
 737
 738    NV50_IR_OPCODE_CASE(SAMPLE, TEX);
 739    NV50_IR_OPCODE_CASE(SAMPLE_B, TXB);
 740    NV50_IR_OPCODE_CASE(SAMPLE_C, TEX);
 741    NV50_IR_OPCODE_CASE(SAMPLE_C_LZ, TEX);
 742    NV50_IR_OPCODE_CASE(SAMPLE_D, TXD);
 743    NV50_IR_OPCODE_CASE(SAMPLE_L, TXL);
 744    NV50_IR_OPCODE_CASE(SAMPLE_I, TXF);
 745    NV50_IR_OPCODE_CASE(SAMPLE_I_MS, TXF);
 746    NV50_IR_OPCODE_CASE(GATHER4, TXG);
 747    NV50_IR_OPCODE_CASE(SVIEWINFO, TXQ);
 748
 749    NV50_IR_OPCODE_CASE(ATOMUADD, ATOM);
 750    NV50_IR_OPCODE_CASE(ATOMXCHG, ATOM);
 751    NV50_IR_OPCODE_CASE(ATOMCAS, ATOM);
 752    NV50_IR_OPCODE_CASE(ATOMAND, ATOM);
 753    NV50_IR_OPCODE_CASE(ATOMOR, ATOM);
 754    NV50_IR_OPCODE_CASE(ATOMXOR, ATOM);
 755    NV50_IR_OPCODE_CASE(ATOMUMIN, ATOM);
 756    NV50_IR_OPCODE_CASE(ATOMUMAX, ATOM);
 757    NV50_IR_OPCODE_CASE(ATOMIMIN, ATOM);
 758    NV50_IR_OPCODE_CASE(ATOMIMAX, ATOM);
 759
 760    NV50_IR_OPCODE_CASE(TEX2, TEX);
 761    NV50_IR_OPCODE_CASE(TXB2, TXB);
 762    NV50_IR_OPCODE_CASE(TXL2, TXL);
 763
 764    NV50_IR_OPCODE_CASE(IBFE, EXTBF);
 765    NV50_IR_OPCODE_CASE(UBFE, EXTBF);
 766    NV50_IR_OPCODE_CASE(BFI, INSBF);
 767    NV50_IR_OPCODE_CASE(BREV, EXTBF);
 768    NV50_IR_OPCODE_CASE(POPC, POPCNT);
 769    NV50_IR_OPCODE_CASE(LSB, BFIND);
 770    NV50_IR_OPCODE_CASE(IMSB, BFIND);
 771    NV50_IR_OPCODE_CASE(UMSB, BFIND);
 772
 773    NV50_IR_OPCODE_CASE(END, EXIT);
 774
 775    default:
 776       return nv50_ir::OP_NOP;
 777    }
 778 }
 779
 780 static uint16_t opcodeToSubOp(uint opcode)
 781 {
 782    switch (opcode) {
 783    case TGSI_OPCODE_LFENCE:   return NV50_IR_SUBOP_MEMBAR(L, GL);
 784    case TGSI_OPCODE_SFENCE:   return NV50_IR_SUBOP_MEMBAR(S, GL);
 785    case TGSI_OPCODE_MFENCE:   return NV50_IR_SUBOP_MEMBAR(M, GL);
 786    case TGSI_OPCODE_ATOMUADD: return NV50_IR_SUBOP_ATOM_ADD;
 787    case TGSI_OPCODE_ATOMXCHG: return NV50_IR_SUBOP_ATOM_EXCH;
 788    case TGSI_OPCODE_ATOMCAS:  return NV50_IR_SUBOP_ATOM_CAS;
 789    case TGSI_OPCODE_ATOMAND:  return NV50_IR_SUBOP_ATOM_AND;
 790    case TGSI_OPCODE_ATOMOR:   return NV50_IR_SUBOP_ATOM_OR;
 791    case TGSI_OPCODE_ATOMXOR:  return NV50_IR_SUBOP_ATOM_XOR;
 792    case TGSI_OPCODE_ATOMUMIN: return NV50_IR_SUBOP_ATOM_MIN;
 793    case TGSI_OPCODE_ATOMIMIN: return NV50_IR_SUBOP_ATOM_MIN;
 794    case TGSI_OPCODE_ATOMUMAX: return NV50_IR_SUBOP_ATOM_MAX;
 795    case TGSI_OPCODE_ATOMIMAX: return NV50_IR_SUBOP_ATOM_MAX;
 796    case TGSI_OPCODE_IMUL_HI:
 797    case TGSI_OPCODE_UMUL_HI:
 798       return NV50_IR_SUBOP_MUL_HIGH;
 799    default:
 800       return 0;
 801    }
 802 }
 803
 804 bool Instruction::checkDstSrcAliasing() const
 805 {
 806    if (insn->Dst[0].Register.Indirect) // no danger if indirect, using memory
 807       return false;
 808
 809    for (int s = 0; s < TGSI_FULL_MAX_SRC_REGISTERS; ++s) {
 810       if (insn->Src[s].Register.File == TGSI_FILE_NULL)
 811          break;
 812       if (insn->Src[s].Register.File == insn->Dst[0].Register.File &&
 813           insn->Src[s].Register.Index == insn->Dst[0].Register.Index)
 814          return true;
 815    }
 816    return false;
 817 }
 818
 819 class Source
 820 {
 821 public:
 822    Source(struct nv50_ir_prog_info *);
 823    ~Source();
 824
 825 public:
 826    bool scanSource();
 827    unsigned fileSize(unsigned file) const { return scan.file_max[file] + 1; }
 828
 829 public:
 830    struct tgsi_shader_info scan;
 831    struct tgsi_full_instruction *insns;
 832    const struct tgsi_token *tokens;
 833    struct nv50_ir_prog_info *info;
 834
 835    nv50_ir::DynArray tempArrays;
 836    nv50_ir::DynArray immdArrays;
 837
 838    typedef nv50_ir::BuildUtil::Location Location;
 839    // these registers are per-subroutine, cannot be used for parameter passing
 840    std::set<Location> locals;
 841
 842    std::set<int> indirectTempArrays;
 843    std::map<int, int> indirectTempOffsets;
 844    std::map<int, std::pair<int, int> > tempArrayInfo;
 845    std::vector<int> tempArrayId;
 846
 847    int clipVertexOutput;
 848
 849    struct TextureView {
 850       uint8_t target; // TGSI_TEXTURE_*
 851    };
 852    std::vector<TextureView> textureViews;
 853
 854    struct Resource {
 855       uint8_t target; // TGSI_TEXTURE_*
 856       bool raw;
 857       uint8_t slot; // $surface index
 858    };
 859    std::vector<Resource> resources;
 860
 861 private:
 862    int inferSysValDirection(unsigned sn) const;
 863    bool scanDeclaration(const struct tgsi_full_declaration *);
 864    bool scanInstruction(const struct tgsi_full_instruction *);
 865    void scanProperty(const struct tgsi_full_property *);
 866    void scanImmediate(const struct tgsi_full_immediate *);
 867
 868    inline bool isEdgeFlagPassthrough(const Instruction&) const;
 869 };
 870
 871 Source::Source(struct nv50_ir_prog_info *prog) : info(prog)
 872 {
 873    tokens = (const struct tgsi_token *)info->bin.source;
 874
 875    if (prog->dbgFlags & NV50_IR_DEBUG_BASIC)
 876       tgsi_dump(tokens, 0);
 877 }
 878
 879 Source::~Source()
 880 {
 881    if (insns)
 882       FREE(insns);
 883
 884    if (info->immd.data)
 885       FREE(info->immd.data);
 886    if (info->immd.type)
 887       FREE(info->immd.type);
 888 }
 889
 890 bool Source::scanSource()
 891 {
 892    unsigned insnCount = 0;
 893    struct tgsi_parse_context parse;
 894
 895    tgsi_scan_shader(tokens, &scan);
 896
 897    insns = (struct tgsi_full_instruction *)MALLOC(scan.num_instructions *
 898                                                   sizeof(insns[0]));
 899    if (!insns)
 900       return false;
 901
 902    clipVertexOutput = -1;
 903
 904    textureViews.resize(scan.file_max[TGSI_FILE_SAMPLER_VIEW] + 1);
 905    //resources.resize(scan.file_max[TGSI_FILE_RESOURCE] + 1);
 906    tempArrayId.resize(scan.file_max[TGSI_FILE_TEMPORARY] + 1);
 907
 908    info->immd.bufSize = 0;
 909
 910    info->numInputs = scan.file_max[TGSI_FILE_INPUT] + 1;
 911    info->numOutputs = scan.file_max[TGSI_FILE_OUTPUT] + 1;
 912    info->numSysVals = scan.file_max[TGSI_FILE_SYSTEM_VALUE] + 1;
 913
 914    if (info->type == PIPE_SHADER_FRAGMENT) {
 915       info->prop.fp.writesDepth = scan.writes_z;
 916       info->prop.fp.usesDiscard = scan.uses_kill;
 917    } else
 918    if (info->type == PIPE_SHADER_GEOMETRY) {
 919       info->prop.gp.instanceCount = 1; // default value
 920    }
 921
 922    info->io.viewportId = -1;
 923
 924    info->immd.data = (uint32_t *)MALLOC(scan.immediate_count * 16);
 925    info->immd.type = (ubyte *)MALLOC(scan.immediate_count * sizeof(ubyte));
 926
 927    tgsi_parse_init(&parse, tokens);
 928    while (!tgsi_parse_end_of_tokens(&parse)) {
 929       tgsi_parse_token(&parse);
 930
 931       switch (parse.FullToken.Token.Type) {
 932       case TGSI_TOKEN_TYPE_IMMEDIATE:
 933          scanImmediate(&parse.FullToken.FullImmediate);
 934          break;
 935       case TGSI_TOKEN_TYPE_DECLARATION:
 936          scanDeclaration(&parse.FullToken.FullDeclaration);
 937          break;
 938       case TGSI_TOKEN_TYPE_INSTRUCTION:
 939          insns[insnCount++] = parse.FullToken.FullInstruction;
 940          scanInstruction(&parse.FullToken.FullInstruction);
 941          break;
 942       case TGSI_TOKEN_TYPE_PROPERTY:
 943          scanProperty(&parse.FullToken.FullProperty);
 944          break;
 945       default:
 946          INFO("unknown TGSI token type: %d\n", parse.FullToken.Token.Type);
 947          break;
 948       }
 949    }
 950    tgsi_parse_free(&parse);
 951
 952    if (indirectTempArrays.size()) {
 953       int tempBase = 0;
 954       for (std::set<int>::const_iterator it = indirectTempArrays.begin();
 955            it != indirectTempArrays.end(); ++it) {
 956          std::pair<int, int>& info = tempArrayInfo[*it];
 957          indirectTempOffsets.insert(std::make_pair(*it, tempBase - info.first));
 958          tempBase += info.second;
 959       }
 960       info->bin.tlsSpace += tempBase * 16;
 961    }
 962
 963    if (info->io.genUserClip > 0) {
 964       info->io.clipDistances = info->io.genUserClip;
 965
 966       const unsigned int nOut = (info->io.genUserClip + 3) / 4;
 967
 968       for (unsigned int n = 0; n < nOut; ++n) {
 969          unsigned int i = info->numOutputs++;
 970          info->out[i].id = i;
 971          info->out[i].sn = TGSI_SEMANTIC_CLIPDIST;
 972          info->out[i].si = n;
 973          info->out[i].mask = ((1 << info->io.clipDistances) - 1) >> (n * 4);
 974       }
 975    }
 976
 977    return info->assignSlots(info) == 0;
 978 }
 979
 980 void Source::scanProperty(const struct tgsi_full_property *prop)
 981 {
 982    switch (prop->Property.PropertyName) {
 983    case TGSI_PROPERTY_GS_OUTPUT_PRIM:
 984       info->prop.gp.outputPrim = prop->u[0].Data;
 985       break;
 986    case TGSI_PROPERTY_GS_INPUT_PRIM:
 987       info->prop.gp.inputPrim = prop->u[0].Data;
 988       break;
 989    case TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES:
 990       info->prop.gp.maxVertices = prop->u[0].Data;
 991       break;
 992    case TGSI_PROPERTY_GS_INVOCATIONS:
 993       info->prop.gp.instanceCount = prop->u[0].Data;
 994       break;
 995    case TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS:
 996       info->prop.fp.separateFragData = true;
 997       break;
 998    case TGSI_PROPERTY_FS_COORD_ORIGIN:
 999    case TGSI_PROPERTY_FS_COORD_PIXEL_CENTER:
1000       // we don't care
1001       break;
1002    case TGSI_PROPERTY_VS_PROHIBIT_UCPS:
1003       info->io.genUserClip = -1;
1004       break;
1005    case TGSI_PROPERTY_TCS_VERTICES_OUT:
1006       info->prop.tp.outputPatchSize = prop->u[0].Data;
1007       break;
1008    case TGSI_PROPERTY_TES_PRIM_MODE:
1009       info->prop.tp.domain = prop->u[0].Data;
1010       break;
1011    case TGSI_PROPERTY_TES_SPACING:
1012       info->prop.tp.partitioning = prop->u[0].Data;
1013       break;
1014    case TGSI_PROPERTY_TES_VERTEX_ORDER_CW:
1015       info->prop.tp.winding = prop->u[0].Data;
1016       break;
1017    case TGSI_PROPERTY_TES_POINT_MODE:
1018       if (prop->u[0].Data)
1019          info->prop.tp.outputPrim = PIPE_PRIM_POINTS;
1020       else
1021          info->prop.tp.outputPrim = PIPE_PRIM_TRIANGLES; /* anything but points */
1022       break;
1023    case TGSI_PROPERTY_NUM_CLIPDIST_ENABLED:
1024       info->io.clipDistances = prop->u[0].Data;
1025       break;
1026    case TGSI_PROPERTY_NUM_CULLDIST_ENABLED:
1027       info->io.cullDistances = prop->u[0].Data;
1028       break;
1029    default:
1030       INFO("unhandled TGSI property %d\n", prop->Property.PropertyName);
1031       break;
1032    }
1033 }
1034
1035 void Source::scanImmediate(const struct tgsi_full_immediate *imm)
1036 {
1037    const unsigned n = info->immd.count++;
1038
1039    assert(n < scan.immediate_count);
1040
1041    for (int c = 0; c < 4; ++c)
1042       info->immd.data[n * 4 + c] = imm->u[c].Uint;
1043
1044    info->immd.type[n] = imm->Immediate.DataType;
1045 }
1046
1047 int Source::inferSysValDirection(unsigned sn) const
1048 {
1049    switch (sn) {
1050    case TGSI_SEMANTIC_INSTANCEID:
1051    case TGSI_SEMANTIC_VERTEXID:
1052       return 1;
1053    case TGSI_SEMANTIC_LAYER:
1054 #if 0
1055    case TGSI_SEMANTIC_VIEWPORTINDEX:
1056       return 0;
1057 #endif
1058    case TGSI_SEMANTIC_PRIMID:
1059       return (info->type == PIPE_SHADER_FRAGMENT) ? 1 : 0;
1060    default:
1061       return 0;
1062    }
1063 }
1064
1065 bool Source::scanDeclaration(const struct tgsi_full_declaration *decl)
1066 {
1067    unsigned i, c;
1068    unsigned sn = TGSI_SEMANTIC_GENERIC;
1069    unsigned si = 0;
1070    const unsigned first = decl->Range.First, last = decl->Range.Last;
1071    const int arrayId = decl->Array.ArrayID;
1072
1073    if (decl->Declaration.Semantic) {
1074       sn = decl->Semantic.Name;
1075       si = decl->Semantic.Index;
1076    }
1077
1078    if (decl->Declaration.Local) {
1079       for (i = first; i <= last; ++i) {
1080          for (c = 0; c < 4; ++c) {
1081             locals.insert(
1082                Location(decl->Declaration.File, decl->Dim.Index2D, i, c));
1083          }
1084       }
1085    }
1086
1087    switch (decl->Declaration.File) {
1088    case TGSI_FILE_INPUT:
1089       if (info->type == PIPE_SHADER_VERTEX) {
1090          // all vertex attributes are equal
1091          for (i = first; i <= last; ++i) {
1092             info->in[i].sn = TGSI_SEMANTIC_GENERIC;
1093             info->in[i].si = i;
1094          }
1095       } else {
1096          for (i = first; i <= last; ++i, ++si) {
1097             info->in[i].id = i;
1098             info->in[i].sn = sn;
1099             info->in[i].si = si;
1100             if (info->type == PIPE_SHADER_FRAGMENT) {
1101                // translate interpolation mode
1102                switch (decl->Interp.Interpolate) {
1103                case TGSI_INTERPOLATE_CONSTANT:
1104                   info->in[i].flat = 1;
1105                   break;
1106                case TGSI_INTERPOLATE_COLOR:
1107                   info->in[i].sc = 1;
1108                   break;
1109                case TGSI_INTERPOLATE_LINEAR:
1110                   info->in[i].linear = 1;
1111                   break;
1112                default:
1113                   break;
1114                }
1115                if (decl->Interp.Location)
1116                   info->in[i].centroid = 1;
1117             }
1118
1119             if (sn == TGSI_SEMANTIC_PATCH)
1120                info->in[i].patch = 1;
1121             if (sn == TGSI_SEMANTIC_PATCH)
1122                info->numPatchConstants = MAX2(info->numPatchConstants, si + 1);
1123          }
1124       }
1125       break;
1126    case TGSI_FILE_OUTPUT:
1127       for (i = first; i <= last; ++i, ++si) {
1128          switch (sn) {
1129          case TGSI_SEMANTIC_POSITION:
1130             if (info->type == PIPE_SHADER_FRAGMENT)
1131                info->io.fragDepth = i;
1132             else
1133             if (clipVertexOutput < 0)
1134                clipVertexOutput = i;
1135             break;
1136          case TGSI_SEMANTIC_COLOR:
1137             if (info->type == PIPE_SHADER_FRAGMENT)
1138                info->prop.fp.numColourResults++;
1139             break;
1140          case TGSI_SEMANTIC_EDGEFLAG:
1141             info->io.edgeFlagOut = i;
1142             break;
1143          case TGSI_SEMANTIC_CLIPVERTEX:
1144             clipVertexOutput = i;
1145             break;
1146          case TGSI_SEMANTIC_CLIPDIST:
1147             info->io.genUserClip = -1;
1148             break;
1149          case TGSI_SEMANTIC_SAMPLEMASK:
1150             info->io.sampleMask = i;
1151             break;
1152          case TGSI_SEMANTIC_VIEWPORT_INDEX:
1153             info->io.viewportId = i;
1154             break;
1155          case TGSI_SEMANTIC_PATCH:
1156             info->numPatchConstants = MAX2(info->numPatchConstants, si + 1);
1157             /* fallthrough */
1158          case TGSI_SEMANTIC_TESSOUTER:
1159          case TGSI_SEMANTIC_TESSINNER:
1160             info->out[i].patch = 1;
1161             break;
1162          default:
1163             break;
1164          }
1165          info->out[i].id = i;
1166          info->out[i].sn = sn;
1167          info->out[i].si = si;
1168       }
1169       break;
1170    case TGSI_FILE_SYSTEM_VALUE:
1171       switch (sn) {
1172       case TGSI_SEMANTIC_INSTANCEID:
1173          info->io.instanceId = first;
1174          break;
1175       case TGSI_SEMANTIC_VERTEXID:
1176          info->io.vertexId = first;
1177          break;
1178       case TGSI_SEMANTIC_SAMPLEID:
1179       case TGSI_SEMANTIC_SAMPLEPOS:
1180          info->prop.fp.sampleInterp = 1;
1181          break;
1182       case TGSI_SEMANTIC_BASEVERTEX:
1183       case TGSI_SEMANTIC_BASEINSTANCE:
1184       case TGSI_SEMANTIC_DRAWID:
1185          info->prop.vp.usesDrawParameters = true;
1186          break;
1187       default:
1188          break;
1189       }
1190       for (i = first; i <= last; ++i, ++si) {
1191          info->sv[i].sn = sn;
1192          info->sv[i].si = si;
1193          info->sv[i].input = inferSysValDirection(sn);
1194
1195          switch (sn) {
1196          case TGSI_SEMANTIC_TESSOUTER:
1197          case TGSI_SEMANTIC_TESSINNER:
1198             info->sv[i].patch = 1;
1199             break;
1200          }
1201       }
1202       break;
1203 /*
1204    case TGSI_FILE_RESOURCE:
1205       for (i = first; i <= last; ++i) {
1206          resources[i].target = decl->Resource.Resource;
1207          resources[i].raw = decl->Resource.Raw;
1208          resources[i].slot = i;
1209       }
1210       break;
1211 */
1212    case TGSI_FILE_SAMPLER_VIEW:
1213       for (i = first; i <= last; ++i)
1214          textureViews[i].target = decl->SamplerView.Resource;
1215       break;
1216    case TGSI_FILE_TEMPORARY:
1217       for (i = first; i <= last; ++i)
1218          tempArrayId[i] = arrayId;
1219       if (arrayId)
1220          tempArrayInfo.insert(std::make_pair(arrayId, std::make_pair(
1221                                                    first, last - first + 1)));
1222       break;
1223    case TGSI_FILE_NULL:
1224    case TGSI_FILE_ADDRESS:
1225    case TGSI_FILE_CONSTANT:
1226    case TGSI_FILE_IMMEDIATE:
1227    case TGSI_FILE_PREDICATE:
1228    case TGSI_FILE_SAMPLER:
1229    case TGSI_FILE_BUFFER:
1230       break;
1231    default:
1232       ERROR("unhandled TGSI_FILE %d\n", decl->Declaration.File);
1233       return false;
1234    }
1235    return true;
1236 }
1237
1238 inline bool Source::isEdgeFlagPassthrough(const Instruction& insn) const
1239 {
1240    return insn.getOpcode() == TGSI_OPCODE_MOV &&
1241       insn.getDst(0).getIndex(0) == info->io.edgeFlagOut &&
1242       insn.getSrc(0).getFile() == TGSI_FILE_INPUT;
1243 }
1244
1245 bool Source::scanInstruction(const struct tgsi_full_instruction *inst)
1246 {
1247    Instruction insn(inst);
1248
1249    if (insn.getOpcode() == TGSI_OPCODE_BARRIER)
1250       info->numBarriers = 1;
1251
1252    if (insn.dstCount()) {
1253       if (insn.getDst(0).getFile() == TGSI_FILE_OUTPUT) {
1254          Instruction::DstRegister dst = insn.getDst(0);
1255
1256          if (dst.isIndirect(0))
1257             for (unsigned i = 0; i < info->numOutputs; ++i)
1258                info->out[i].mask = 0xf;
1259          else
1260             info->out[dst.getIndex(0)].mask |= dst.getMask();
1261
1262          if (info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PSIZE ||
1263              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PRIMID ||
1264              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_LAYER ||
1265              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_VIEWPORT_INDEX ||
1266              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_FOG)
1267             info->out[dst.getIndex(0)].mask &= 1;
1268
1269          if (isEdgeFlagPassthrough(insn))
1270             info->io.edgeFlagIn = insn.getSrc(0).getIndex(0);
1271       } else
1272       if (insn.getDst(0).getFile() == TGSI_FILE_TEMPORARY) {
1273          if (insn.getDst(0).isIndirect(0))
1274             indirectTempArrays.insert(insn.getDst(0).getArrayId());
1275       } else
1276       if (insn.getDst(0).getFile() == TGSI_FILE_BUFFER) {
1277          info->io.globalAccess |= 0x2;
1278       }
1279    }
1280
1281    for (unsigned s = 0; s < insn.srcCount(); ++s) {
1282       Instruction::SrcRegister src = insn.getSrc(s);
1283       if (src.getFile() == TGSI_FILE_TEMPORARY) {
1284          if (src.isIndirect(0))
1285             indirectTempArrays.insert(src.getArrayId());
1286       } else
1287       if (src.getFile() == TGSI_FILE_BUFFER) {
1288          info->io.globalAccess |= (insn.getOpcode() == TGSI_OPCODE_LOAD) ?
1289                0x1 : 0x2;
1290       } else
1291       if (src.getFile() == TGSI_FILE_OUTPUT) {
1292          if (src.isIndirect(0)) {
1293             // We don't know which one is accessed, just mark everything for
1294             // reading. This is an extremely unlikely occurrence.
1295             for (unsigned i = 0; i < info->numOutputs; ++i)
1296                info->out[i].oread = 1;
1297          } else {
1298             info->out[src.getIndex(0)].oread = 1;
1299          }
1300       }
1301       if (src.getFile() != TGSI_FILE_INPUT)
1302          continue;
1303       unsigned mask = insn.srcMask(s);
1304
1305       if (src.isIndirect(0)) {
1306          for (unsigned i = 0; i < info->numInputs; ++i)
1307             info->in[i].mask = 0xf;
1308       } else {
1309          const int i = src.getIndex(0);
1310          for (unsigned c = 0; c < 4; ++c) {
1311             if (!(mask & (1 << c)))
1312                continue;
1313             int k = src.getSwizzle(c);
1314             if (k <= TGSI_SWIZZLE_W)
1315                info->in[i].mask |= 1 << k;
1316          }
1317          switch (info->in[i].sn) {
1318          case TGSI_SEMANTIC_PSIZE:
1319          case TGSI_SEMANTIC_PRIMID:
1320          case TGSI_SEMANTIC_FOG:
1321             info->in[i].mask &= 0x1;
1322             break;
1323          case TGSI_SEMANTIC_PCOORD:
1324             info->in[i].mask &= 0x3;
1325             break;
1326          default:
1327             break;
1328          }
1329       }
1330    }
1331    return true;
1332 }
1333
1334 nv50_ir::TexInstruction::Target
1335 Instruction::getTexture(const tgsi::Source *code, int s) const
1336 {
1337    // XXX: indirect access
1338    unsigned int r;
1339
1340    switch (getSrc(s).getFile()) {
1341 /*
1342    case TGSI_FILE_RESOURCE:
1343       r = getSrc(s).getIndex(0);
1344       return translateTexture(code->resources.at(r).target);
1345 */
1346    case TGSI_FILE_SAMPLER_VIEW:
1347       r = getSrc(s).getIndex(0);
1348       return translateTexture(code->textureViews.at(r).target);
1349    default:
1350       return translateTexture(insn->Texture.Texture);
1351    }
1352 }
1353
1354 } // namespace tgsi
1355
1356 namespace {
1357
1358 using namespace nv50_ir;
1359
1360 class Converter : public BuildUtil
1361 {
1362 public:
1363    Converter(Program *, const tgsi::Source *);
1364    ~Converter();
1365
1366    bool run();
1367
1368 private:
1369    struct Subroutine
1370    {
1371       Subroutine(Function *f) : f(f) { }
1372       Function *f;
1373       ValueMap values;
1374    };
1375
1376    Value *shiftAddress(Value *);
1377    Value *getVertexBase(int s);
1378    Value *getOutputBase(int s);
1379    DataArray *getArrayForFile(unsigned file, int idx);
1380    Value *fetchSrc(int s, int c);
1381    Value *acquireDst(int d, int c);
1382    void storeDst(int d, int c, Value *);
1383
1384    Value *fetchSrc(const tgsi::Instruction::SrcRegister src, int c, Value *ptr);
1385    void storeDst(const tgsi::Instruction::DstRegister dst, int c,
1386                  Value *val, Value *ptr);
1387
1388    void adjustTempIndex(int arrayId, int &idx, int &idx2d) const;
1389    Value *applySrcMod(Value *, int s, int c);
1390
1391    Symbol *makeSym(uint file, int fileIndex, int idx, int c, uint32_t addr);
1392    Symbol *srcToSym(tgsi::Instruction::SrcRegister, int c);
1393    Symbol *dstToSym(tgsi::Instruction::DstRegister, int c);
1394
1395    bool handleInstruction(const struct tgsi_full_instruction *);
1396    void exportOutputs();
1397    inline Subroutine *getSubroutine(unsigned ip);
1398    inline Subroutine *getSubroutine(Function *);
1399    inline bool isEndOfSubroutine(uint ip);
1400
1401    void loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask);
1402
1403    // R,S,L,C,Dx,Dy encode TGSI sources for respective values (0xSf for auto)
1404    void setTexRS(TexInstruction *, unsigned int& s, int R, int S);
1405    void handleTEX(Value *dst0[4], int R, int S, int L, int C, int Dx, int Dy);
1406    void handleTXF(Value *dst0[4], int R, int L_M);
1407    void handleTXQ(Value *dst0[4], enum TexQuery, int R);
1408    void handleLIT(Value *dst0[4]);
1409    void handleUserClipPlanes();
1410
1411    Symbol *getResourceBase(int r);
1412    void getResourceCoords(std::vector<Value *>&, int r, int s);
1413
1414    void handleLOAD(Value *dst0[4]);
1415    void handleSTORE();
1416    void handleATOM(Value *dst0[4], DataType, uint16_t subOp);
1417
1418    void handleINTERP(Value *dst0[4]);
1419
1420    uint8_t translateInterpMode(const struct nv50_ir_varying *var,
1421                                operation& op);
1422    Value *interpolate(tgsi::Instruction::SrcRegister, int c, Value *ptr);
1423
1424    void insertConvergenceOps(BasicBlock *conv, BasicBlock *fork);
1425
1426    Value *buildDot(int dim);
1427
1428    class BindArgumentsPass : public Pass {
1429    public:
1430       BindArgumentsPass(Converter &conv) : conv(conv) { }
1431
1432    private:
1433       Converter &conv;
1434       Subroutine *sub;
1435
1436       inline const Location *getValueLocation(Subroutine *, Value *);
1437
1438       template<typename T> inline void
1439       updateCallArgs(Instruction *i, void (Instruction::*setArg)(int, Value *),
1440                      T (Function::*proto));
1441
1442       template<typename T> inline void
1443       updatePrototype(BitSet *set, void (Function::*updateSet)(),
1444                       T (Function::*proto));
1445
1446    protected:
1447       bool visit(Function *);
1448       bool visit(BasicBlock *bb) { return false; }
1449    };
1450
1451 private:
1452    const tgsi::Source *code;
1453    const struct nv50_ir_prog_info *info;
1454
1455    struct {
1456       std::map<unsigned, Subroutine> map;
1457       Subroutine *cur;
1458    } sub;
1459
1460    uint ip; // instruction pointer
1461
1462    tgsi::Instruction tgsi;
1463
1464    DataType dstTy;
1465    DataType srcTy;
1466
1467    DataArray tData; // TGSI_FILE_TEMPORARY
1468    DataArray lData; // TGSI_FILE_TEMPORARY, for indirect arrays
1469    DataArray aData; // TGSI_FILE_ADDRESS
1470    DataArray pData; // TGSI_FILE_PREDICATE
1471    DataArray oData; // TGSI_FILE_OUTPUT (if outputs in registers)
1472
1473    Value *zero;
1474    Value *fragCoord[4];
1475    Value *clipVtx[4];
1476
1477    Value *vtxBase[5]; // base address of vertex in primitive (for TP/GP)
1478    uint8_t vtxBaseValid;
1479
1480    Value *outBase; // base address of vertex out patch (for TCP)
1481
1482    Stack condBBs;  // fork BB, then else clause BB
1483    Stack joinBBs;  // fork BB, for inserting join ops on ENDIF
1484    Stack loopBBs;  // loop headers
1485    Stack breakBBs; // end of / after loop
1486
1487    Value *viewport;
1488 };
1489
1490 Symbol *
1491 Converter::srcToSym(tgsi::Instruction::SrcRegister src, int c)
1492 {
1493    const int swz = src.getSwizzle(c);
1494
1495    /* TODO: Use Array ID when it's available for the index */
1496    return makeSym(src.getFile(),
1497                   src.is2D() ? src.getIndex(1) : 0,
1498                   src.getIndex(0), swz,
1499                   src.getIndex(0) * 16 + swz * 4);
1500 }
1501
1502 Symbol *
1503 Converter::dstToSym(tgsi::Instruction::DstRegister dst, int c)
1504 {
1505    /* TODO: Use Array ID when it's available for the index */
1506    return makeSym(dst.getFile(),
1507                   dst.is2D() ? dst.getIndex(1) : 0,
1508                   dst.getIndex(0), c,
1509                   dst.getIndex(0) * 16 + c * 4);
1510 }
1511
1512 Symbol *
1513 Converter::makeSym(uint tgsiFile, int fileIdx, int idx, int c, uint32_t address)
1514 {
1515    Symbol *sym = new_Symbol(prog, tgsi::translateFile(tgsiFile));
1516
1517    sym->reg.fileIndex = fileIdx;
1518
1519    if (idx >= 0) {
1520       if (sym->reg.file == FILE_SHADER_INPUT)
1521          sym->setOffset(info->in[idx].slot[c] * 4);
1522       else
1523       if (sym->reg.file == FILE_SHADER_OUTPUT)
1524          sym->setOffset(info->out[idx].slot[c] * 4);
1525       else
1526       if (sym->reg.file == FILE_SYSTEM_VALUE)
1527          sym->setSV(tgsi::translateSysVal(info->sv[idx].sn), c);
1528       else
1529          sym->setOffset(address);
1530    } else {
1531       sym->setOffset(address);
1532    }
1533    return sym;
1534 }
1535
1536 uint8_t
1537 Converter::translateInterpMode(const struct nv50_ir_varying *var, operation& op)
1538 {
1539    uint8_t mode = NV50_IR_INTERP_PERSPECTIVE;
1540
1541    if (var->flat)
1542       mode = NV50_IR_INTERP_FLAT;
1543    else
1544    if (var->linear)
1545       mode = NV50_IR_INTERP_LINEAR;
1546    else
1547    if (var->sc)
1548       mode = NV50_IR_INTERP_SC;
1549
1550    op = (mode == NV50_IR_INTERP_PERSPECTIVE || mode == NV50_IR_INTERP_SC)
1551       ? OP_PINTERP : OP_LINTERP;
1552
1553    if (var->centroid || info->prop.fp.sampleInterp)
1554       mode |= NV50_IR_INTERP_CENTROID;
1555
1556    return mode;
1557 }
1558
1559 Value *
1560 Converter::interpolate(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
1561 {
1562    operation op;
1563
1564    // XXX: no way to know interpolation mode if we don't know what's accessed
1565    const uint8_t mode = translateInterpMode(&info->in[ptr ? 0 :
1566                                                       src.getIndex(0)], op);
1567
1568    Instruction *insn = new_Instruction(func, op, TYPE_F32);
1569
1570    insn->setDef(0, getScratch());
1571    insn->setSrc(0, srcToSym(src, c));
1572    if (op == OP_PINTERP)
1573       insn->setSrc(1, fragCoord[3]);
1574    if (ptr)
1575       insn->setIndirect(0, 0, ptr);
1576
1577    insn->setInterpolate(mode);
1578
1579    bb->insertTail(insn);
1580    return insn->getDef(0);
1581 }
1582
1583 Value *
1584 Converter::applySrcMod(Value *val, int s, int c)
1585 {
1586    Modifier m = tgsi.getSrc(s).getMod(c);
1587    DataType ty = tgsi.inferSrcType();
1588
1589    if (m & Modifier(NV50_IR_MOD_ABS))
1590       val = mkOp1v(OP_ABS, ty, getScratch(), val);
1591
1592    if (m & Modifier(NV50_IR_MOD_NEG))
1593       val = mkOp1v(OP_NEG, ty, getScratch(), val);
1594
1595    return val;
1596 }
1597
1598 Value *
1599 Converter::getVertexBase(int s)
1600 {
1601    assert(s < 5);
1602    if (!(vtxBaseValid & (1 << s))) {
1603       const int index = tgsi.getSrc(s).getIndex(1);
1604       Value *rel = NULL;
1605       if (tgsi.getSrc(s).isIndirect(1))
1606          rel = fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL);
1607       vtxBaseValid |= 1 << s;
1608       vtxBase[s] = mkOp2v(OP_PFETCH, TYPE_U32, getSSA(4, FILE_ADDRESS),
1609                           mkImm(index), rel);
1610    }
1611    return vtxBase[s];
1612 }
1613
1614 Value *
1615 Converter::getOutputBase(int s)
1616 {
1617    assert(s < 5);
1618    if (!(vtxBaseValid & (1 << s))) {
1619       Value *offset = loadImm(NULL, tgsi.getSrc(s).getIndex(1));
1620       if (tgsi.getSrc(s).isIndirect(1))
1621          offset = mkOp2v(OP_ADD, TYPE_U32, getSSA(),
1622                          fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL),
1623                          offset);
1624       vtxBaseValid |= 1 << s;
1625       vtxBase[s] = mkOp2v(OP_ADD, TYPE_U32, getSSA(), outBase, offset);
1626    }
1627    return vtxBase[s];
1628 }
1629
1630 Value *
1631 Converter::fetchSrc(int s, int c)
1632 {
1633    Value *res;
1634    Value *ptr = NULL, *dimRel = NULL;
1635
1636    tgsi::Instruction::SrcRegister src = tgsi.getSrc(s);
1637
1638    if (src.isIndirect(0))
1639       ptr = fetchSrc(src.getIndirect(0), 0, NULL);
1640
1641    if (src.is2D()) {
1642       switch (src.getFile()) {
1643       case TGSI_FILE_OUTPUT:
1644          dimRel = getOutputBase(s);
1645          break;
1646       case TGSI_FILE_INPUT:
1647          dimRel = getVertexBase(s);
1648          break;
1649       case TGSI_FILE_CONSTANT:
1650          // on NVC0, this is valid and c{I+J}[k] == cI[(J << 16) + k]
1651          if (src.isIndirect(1))
1652             dimRel = fetchSrc(src.getIndirect(1), 0, 0);
1653          break;
1654       default:
1655          break;
1656       }
1657    }
1658
1659    res = fetchSrc(src, c, ptr);
1660
1661    if (dimRel)
1662       res->getInsn()->setIndirect(0, 1, dimRel);
1663
1664    return applySrcMod(res, s, c);
1665 }
1666
1667 Converter::DataArray *
1668 Converter::getArrayForFile(unsigned file, int idx)
1669 {
1670    switch (file) {
1671    case TGSI_FILE_TEMPORARY:
1672       return idx == 0 ? &tData : &lData;
1673    case TGSI_FILE_PREDICATE:
1674       return &pData;
1675    case TGSI_FILE_ADDRESS:
1676       return &aData;
1677    case TGSI_FILE_OUTPUT:
1678       assert(prog->getType() == Program::TYPE_FRAGMENT);
1679       return &oData;
1680    default:
1681       assert(!"invalid/unhandled TGSI source file");
1682       return NULL;
1683    }
1684 }
1685
1686 Value *
1687 Converter::shiftAddress(Value *index)
1688 {
1689    if (!index)
1690       return NULL;
1691    return mkOp2v(OP_SHL, TYPE_U32, getSSA(4, FILE_ADDRESS), index, mkImm(4));
1692 }
1693
1694 void
1695 Converter::adjustTempIndex(int arrayId, int &idx, int &idx2d) const
1696 {
1697    std::map<int, int>::const_iterator it =
1698       code->indirectTempOffsets.find(arrayId);
1699    if (it == code->indirectTempOffsets.end())
1700       return;
1701
1702    idx2d = 1;
1703    idx += it->second;
1704 }
1705
1706 Value *
1707 Converter::fetchSrc(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
1708 {
1709    int idx2d = src.is2D() ? src.getIndex(1) : 0;
1710    int idx = src.getIndex(0);
1711    const int swz = src.getSwizzle(c);
1712    Instruction *ld;
1713
1714    switch (src.getFile()) {
1715    case TGSI_FILE_IMMEDIATE:
1716       assert(!ptr);
1717       return loadImm(NULL, info->immd.data[idx * 4 + swz]);
1718    case TGSI_FILE_CONSTANT:
1719       return mkLoadv(TYPE_U32, srcToSym(src, c), shiftAddress(ptr));
1720    case TGSI_FILE_INPUT:
1721       if (prog->getType() == Program::TYPE_FRAGMENT) {
1722          // don't load masked inputs, won't be assigned a slot
1723          if (!ptr && !(info->in[idx].mask & (1 << swz)))
1724             return loadImm(NULL, swz == TGSI_SWIZZLE_W ? 1.0f : 0.0f);
1725          return interpolate(src, c, shiftAddress(ptr));
1726       } else
1727       if (prog->getType() == Program::TYPE_GEOMETRY) {
1728          if (!ptr && info->in[idx].sn == TGSI_SEMANTIC_PRIMID)
1729             return mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_PRIMITIVE_ID, 0));
1730          // XXX: This is going to be a problem with scalar arrays, i.e. when
1731          // we cannot assume that the address is given in units of vec4.
1732          //
1733          // nv50 and nvc0 need different things here, so let the lowering
1734          // passes decide what to do with the address
1735          if (ptr)
1736             return mkLoadv(TYPE_U32, srcToSym(src, c), ptr);
1737       }
1738       ld = mkLoad(TYPE_U32, getSSA(), srcToSym(src, c), shiftAddress(ptr));
1739       ld->perPatch = info->in[idx].patch;
1740       return ld->getDef(0);
1741    case TGSI_FILE_OUTPUT:
1742       assert(prog->getType() == Program::TYPE_TESSELLATION_CONTROL);
1743       ld = mkLoad(TYPE_U32, getSSA(), srcToSym(src, c), shiftAddress(ptr));
1744       ld->perPatch = info->out[idx].patch;
1745       return ld->getDef(0);
1746    case TGSI_FILE_SYSTEM_VALUE:
1747       assert(!ptr);
1748       ld = mkOp1(OP_RDSV, TYPE_U32, getSSA(), srcToSym(src, c));
1749       ld->perPatch = info->sv[idx].patch;
1750       return ld->getDef(0);
1751    case TGSI_FILE_TEMPORARY: {
1752       int arrayid = src.getArrayId();
1753       if (!arrayid)
1754          arrayid = code->tempArrayId[idx];
1755       adjustTempIndex(arrayid, idx, idx2d);
1756    }
1757       /* fallthrough */
1758    default:
1759       return getArrayForFile(src.getFile(), idx2d)->load(
1760          sub.cur->values, idx, swz, shiftAddress(ptr));
1761    }
1762 }
1763
1764 Value *
1765 Converter::acquireDst(int d, int c)
1766 {
1767    const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
1768    const unsigned f = dst.getFile();
1769    int idx = dst.getIndex(0);
1770    int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
1771
1772    if (dst.isMasked(c) || f == TGSI_FILE_BUFFER)
1773       return NULL;
1774
1775    if (dst.isIndirect(0) ||
1776        f == TGSI_FILE_SYSTEM_VALUE ||
1777        (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT))
1778       return getScratch();
1779
1780    if (f == TGSI_FILE_TEMPORARY) {
1781       int arrayid = dst.getArrayId();
1782       if (!arrayid)
1783          arrayid = code->tempArrayId[idx];
1784       adjustTempIndex(arrayid, idx, idx2d);
1785    }
1786
1787    return getArrayForFile(f, idx2d)-> acquire(sub.cur->values, idx, c);
1788 }
1789
1790 void
1791 Converter::storeDst(int d, int c, Value *val)
1792 {
1793    const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
1794
1795    if (tgsi.getSaturate()) {
1796       mkOp1(OP_SAT, dstTy, val, val);
1797    }
1798
1799    Value *ptr = NULL;
1800    if (dst.isIndirect(0))
1801       ptr = shiftAddress(fetchSrc(dst.getIndirect(0), 0, NULL));
1802
1803    if (info->io.genUserClip > 0 &&
1804        dst.getFile() == TGSI_FILE_OUTPUT &&
1805        !dst.isIndirect(0) && dst.getIndex(0) == code->clipVertexOutput) {
1806       mkMov(clipVtx[c], val);
1807       val = clipVtx[c];
1808    }
1809
1810    storeDst(dst, c, val, ptr);
1811 }
1812
1813 void
1814 Converter::storeDst(const tgsi::Instruction::DstRegister dst, int c,
1815                     Value *val, Value *ptr)
1816 {
1817    const unsigned f = dst.getFile();
1818    int idx = dst.getIndex(0);
1819    int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
1820
1821    if (f == TGSI_FILE_SYSTEM_VALUE) {
1822       assert(!ptr);
1823       mkOp2(OP_WRSV, TYPE_U32, NULL, dstToSym(dst, c), val);
1824    } else
1825    if (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT) {
1826
1827       if (ptr || (info->out[idx].mask & (1 << c))) {
1828          /* Save the viewport index into a scratch register so that it can be
1829             exported at EMIT time */
1830          if (info->out[idx].sn == TGSI_SEMANTIC_VIEWPORT_INDEX &&
1831              viewport != NULL)
1832             mkOp1(OP_MOV, TYPE_U32, viewport, val);
1833          else
1834             mkStore(OP_EXPORT, TYPE_U32, dstToSym(dst, c), ptr, val)->perPatch =
1835                info->out[idx].patch;
1836       }
1837    } else
1838    if (f == TGSI_FILE_TEMPORARY ||
1839        f == TGSI_FILE_PREDICATE ||
1840        f == TGSI_FILE_ADDRESS ||
1841        f == TGSI_FILE_OUTPUT) {
1842       if (f == TGSI_FILE_TEMPORARY) {
1843          int arrayid = dst.getArrayId();
1844          if (!arrayid)
1845             arrayid = code->tempArrayId[idx];
1846          adjustTempIndex(arrayid, idx, idx2d);
1847       }
1848
1849       getArrayForFile(f, idx2d)->store(sub.cur->values, idx, c, ptr, val);
1850    } else {
1851       assert(!"invalid dst file");
1852    }
1853 }
1854
1855 #define FOR_EACH_DST_ENABLED_CHANNEL(d, chan, inst) \
1856    for (chan = 0; chan < 4; ++chan)                 \
1857       if (!inst.getDst(d).isMasked(chan))
1858
1859 Value *
1860 Converter::buildDot(int dim)
1861 {
1862    assert(dim > 0);
1863
1864    Value *src0 = fetchSrc(0, 0), *src1 = fetchSrc(1, 0);
1865    Value *dotp = getScratch();
1866
1867    mkOp2(OP_MUL, TYPE_F32, dotp, src0, src1);
1868
1869    for (int c = 1; c < dim; ++c) {
1870       src0 = fetchSrc(0, c);
1871       src1 = fetchSrc(1, c);
1872       mkOp3(OP_MAD, TYPE_F32, dotp, src0, src1, dotp);
1873    }
1874    return dotp;
1875 }
1876
1877 void
1878 Converter::insertConvergenceOps(BasicBlock *conv, BasicBlock *fork)
1879 {
1880    FlowInstruction *join = new_FlowInstruction(func, OP_JOIN, NULL);
1881    join->fixed = 1;
1882    conv->insertHead(join);
1883
1884    assert(!fork->joinAt);
1885    fork->joinAt = new_FlowInstruction(func, OP_JOINAT, conv);
1886    fork->insertBefore(fork->getExit(), fork->joinAt);
1887 }
1888
1889 void
1890 Converter::setTexRS(TexInstruction *tex, unsigned int& s, int R, int S)
1891 {
1892    unsigned rIdx = 0, sIdx = 0;
1893
1894    if (R >= 0)
1895       rIdx = tgsi.getSrc(R).getIndex(0);
1896    if (S >= 0)
1897       sIdx = tgsi.getSrc(S).getIndex(0);
1898
1899    tex->setTexture(tgsi.getTexture(code, R), rIdx, sIdx);
1900
1901    if (tgsi.getSrc(R).isIndirect(0)) {
1902       tex->tex.rIndirectSrc = s;
1903       tex->setSrc(s++, fetchSrc(tgsi.getSrc(R).getIndirect(0), 0, NULL));
1904    }
1905    if (S >= 0 && tgsi.getSrc(S).isIndirect(0)) {
1906       tex->tex.sIndirectSrc = s;
1907       tex->setSrc(s++, fetchSrc(tgsi.getSrc(S).getIndirect(0), 0, NULL));
1908    }
1909 }
1910
1911 void
1912 Converter::handleTXQ(Value *dst0[4], enum TexQuery query, int R)
1913 {
1914    TexInstruction *tex = new_TexInstruction(func, OP_TXQ);
1915    tex->tex.query = query;
1916    unsigned int c, d;
1917
1918    for (d = 0, c = 0; c < 4; ++c) {
1919       if (!dst0[c])
1920          continue;
1921       tex->tex.mask |= 1 << c;
1922       tex->setDef(d++, dst0[c]);
1923    }
1924    if (query == TXQ_DIMS)
1925       tex->setSrc((c = 0), fetchSrc(0, 0)); // mip level
1926    else
1927       tex->setSrc((c = 0), zero);
1928
1929    setTexRS(tex, ++c, R, -1);
1930
1931    bb->insertTail(tex);
1932 }
1933
1934 void
1935 Converter::loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask)
1936 {
1937    Value *proj = fetchSrc(0, 3);
1938    Instruction *insn = proj->getUniqueInsn();
1939    int c;
1940
1941    if (insn->op == OP_PINTERP) {
1942       bb->insertTail(insn = cloneForward(func, insn));
1943       insn->op = OP_LINTERP;
1944       insn->setInterpolate(NV50_IR_INTERP_LINEAR | insn->getSampleMode());
1945       insn->setSrc(1, NULL);
1946       proj = insn->getDef(0);
1947    }
1948    proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), proj);
1949
1950    for (c = 0; c < 4; ++c) {
1951       if (!(mask & (1 << c)))
1952          continue;
1953       if ((insn = src[c]->getUniqueInsn())->op != OP_PINTERP)
1954          continue;
1955       mask &= ~(1 << c);
1956
1957       bb->insertTail(insn = cloneForward(func, insn));
1958       insn->setInterpolate(NV50_IR_INTERP_PERSPECTIVE | insn->getSampleMode());
1959       insn->setSrc(1, proj);
1960       dst[c] = insn->getDef(0);
1961    }
1962    if (!mask)
1963       return;
1964
1965    proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), fetchSrc(0, 3));
1966
1967    for (c = 0; c < 4; ++c)
1968       if (mask & (1 << c))
1969          dst[c] = mkOp2v(OP_MUL, TYPE_F32, getSSA(), src[c], proj);
1970 }
1971
1972 // order of nv50 ir sources: x y z layer lod/bias shadow
1973 // order of TGSI TEX sources: x y z layer shadow lod/bias
1974 //  lowering will finally set the hw specific order (like array first on nvc0)
1975 void
1976 Converter::handleTEX(Value *dst[4], int R, int S, int L, int C, int Dx, int Dy)
1977 {
1978    Value *val;
1979    Value *arg[4], *src[8];
1980    Value *lod = NULL, *shd = NULL;
1981    unsigned int s, c, d;
1982    TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
1983
1984    TexInstruction::Target tgt = tgsi.getTexture(code, R);
1985
1986    for (s = 0; s < tgt.getArgCount(); ++s)
1987       arg[s] = src[s] = fetchSrc(0, s);
1988
1989    if (texi->op == OP_TXL || texi->op == OP_TXB)
1990       lod = fetchSrc(L >> 4, L & 3);
1991
1992    if (C == 0x0f)
1993       C = 0x00 | MAX2(tgt.getArgCount(), 2); // guess DC src
1994
1995    if (tgsi.getOpcode() == TGSI_OPCODE_TG4 &&
1996        tgt == TEX_TARGET_CUBE_ARRAY_SHADOW)
1997       shd = fetchSrc(1, 0);
1998    else if (tgt.isShadow())
1999       shd = fetchSrc(C >> 4, C & 3);
2000
2001    if (texi->op == OP_TXD) {
2002       for (c = 0; c < tgt.getDim() + tgt.isCube(); ++c) {
2003          texi->dPdx[c].set(fetchSrc(Dx >> 4, (Dx & 3) + c));
2004          texi->dPdy[c].set(fetchSrc(Dy >> 4, (Dy & 3) + c));
2005       }
2006    }
2007
2008    // cube textures don't care about projection value, it's divided out
2009    if (tgsi.getOpcode() == TGSI_OPCODE_TXP && !tgt.isCube() && !tgt.isArray()) {
2010       unsigned int n = tgt.getDim();
2011       if (shd) {
2012          arg[n] = shd;
2013          ++n;
2014          assert(tgt.getDim() == tgt.getArgCount());
2015       }
2016       loadProjTexCoords(src, arg, (1 << n) - 1);
2017       if (shd)
2018          shd = src[n - 1];
2019    }
2020
2021    if (tgt.isCube()) {
2022       for (c = 0; c < 3; ++c)
2023          src[c] = mkOp1v(OP_ABS, TYPE_F32, getSSA(), arg[c]);
2024       val = getScratch();
2025       mkOp2(OP_MAX, TYPE_F32, val, src[0], src[1]);
2026       mkOp2(OP_MAX, TYPE_F32, val, src[2], val);
2027       mkOp1(OP_RCP, TYPE_F32, val, val);
2028       for (c = 0; c < 3; ++c)
2029          src[c] = mkOp2v(OP_MUL, TYPE_F32, getSSA(), arg[c], val);
2030    }
2031
2032    for (c = 0, d = 0; c < 4; ++c) {
2033       if (dst[c]) {
2034          texi->setDef(d++, dst[c]);
2035          texi->tex.mask |= 1 << c;
2036       } else {
2037          // NOTE: maybe hook up def too, for CSE
2038       }
2039    }
2040    for (s = 0; s < tgt.getArgCount(); ++s)
2041       texi->setSrc(s, src[s]);
2042    if (lod)
2043       texi->setSrc(s++, lod);
2044    if (shd)
2045       texi->setSrc(s++, shd);
2046
2047    setTexRS(texi, s, R, S);
2048
2049    if (tgsi.getOpcode() == TGSI_OPCODE_SAMPLE_C_LZ)
2050       texi->tex.levelZero = true;
2051    if (tgsi.getOpcode() == TGSI_OPCODE_TG4 && !tgt.isShadow())
2052       texi->tex.gatherComp = tgsi.getSrc(1).getValueU32(0, info);
2053
2054    texi->tex.useOffsets = tgsi.getNumTexOffsets();
2055    for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
2056       for (c = 0; c < 3; ++c) {
2057          texi->offset[s][c].set(fetchSrc(tgsi.getTexOffset(s), c, NULL));
2058          texi->offset[s][c].setInsn(texi);
2059       }
2060    }
2061
2062    bb->insertTail(texi);
2063 }
2064
2065 // 1st source: xyz = coordinates, w = lod/sample
2066 // 2nd source: offset
2067 void
2068 Converter::handleTXF(Value *dst[4], int R, int L_M)
2069 {
2070    TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
2071    int ms;
2072    unsigned int c, d, s;
2073
2074    texi->tex.target = tgsi.getTexture(code, R);
2075
2076    ms = texi->tex.target.isMS() ? 1 : 0;
2077    texi->tex.levelZero = ms; /* MS textures don't have mip-maps */
2078
2079    for (c = 0, d = 0; c < 4; ++c) {
2080       if (dst[c]) {
2081          texi->setDef(d++, dst[c]);
2082          texi->tex.mask |= 1 << c;
2083       }
2084    }
2085    for (c = 0; c < (texi->tex.target.getArgCount() - ms); ++c)
2086       texi->setSrc(c, fetchSrc(0, c));
2087    texi->setSrc(c++, fetchSrc(L_M >> 4, L_M & 3)); // lod or ms
2088
2089    setTexRS(texi, c, R, -1);
2090
2091    texi->tex.useOffsets = tgsi.getNumTexOffsets();
2092    for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
2093       for (c = 0; c < 3; ++c) {
2094          texi->offset[s][c].set(fetchSrc(tgsi.getTexOffset(s), c, NULL));
2095          texi->offset[s][c].setInsn(texi);
2096       }
2097    }
2098
2099    bb->insertTail(texi);
2100 }
2101
2102 void
2103 Converter::handleLIT(Value *dst0[4])
2104 {
2105    Value *val0 = NULL;
2106    unsigned int mask = tgsi.getDst(0).getMask();
2107
2108    if (mask & (1 << 0))
2109       loadImm(dst0[0], 1.0f);
2110
2111    if (mask & (1 << 3))
2112       loadImm(dst0[3], 1.0f);
2113
2114    if (mask & (3 << 1)) {
2115       val0 = getScratch();
2116       mkOp2(OP_MAX, TYPE_F32, val0, fetchSrc(0, 0), zero);
2117       if (mask & (1 << 1))
2118          mkMov(dst0[1], val0);
2119    }
2120
2121    if (mask & (1 << 2)) {
2122       Value *src1 = fetchSrc(0, 1), *src3 = fetchSrc(0, 3);
2123       Value *val1 = getScratch(), *val3 = getScratch();
2124
2125       Value *pos128 = loadImm(NULL, +127.999999f);
2126       Value *neg128 = loadImm(NULL, -127.999999f);
2127
2128       mkOp2(OP_MAX, TYPE_F32, val1, src1, zero);
2129       mkOp2(OP_MAX, TYPE_F32, val3, src3, neg128);
2130       mkOp2(OP_MIN, TYPE_F32, val3, val3, pos128);
2131       mkOp2(OP_POW, TYPE_F32, val3, val1, val3);
2132
2133       mkCmp(OP_SLCT, CC_GT, TYPE_F32, dst0[2], TYPE_F32, val3, zero, val0);
2134    }
2135 }
2136
2137 static inline bool
2138 isResourceSpecial(const int r)
2139 {
2140    return (r == TGSI_RESOURCE_GLOBAL ||
2141            r == TGSI_RESOURCE_LOCAL ||
2142            r == TGSI_RESOURCE_PRIVATE ||
2143            r == TGSI_RESOURCE_INPUT);
2144 }
2145
2146 static inline bool
2147 isResourceRaw(const tgsi::Source *code, const int r)
2148 {
2149    return isResourceSpecial(r) || code->resources[r].raw;
2150 }
2151
2152 static inline nv50_ir::TexTarget
2153 getResourceTarget(const tgsi::Source *code, int r)
2154 {
2155    if (isResourceSpecial(r))
2156       return nv50_ir::TEX_TARGET_BUFFER;
2157    return tgsi::translateTexture(code->resources.at(r).target);
2158 }
2159
2160 Symbol *
2161 Converter::getResourceBase(const int r)
2162 {
2163    Symbol *sym = NULL;
2164
2165    switch (r) {
2166    case TGSI_RESOURCE_GLOBAL:
2167       sym = new_Symbol(prog, nv50_ir::FILE_MEMORY_GLOBAL, 15);
2168       break;
2169    case TGSI_RESOURCE_LOCAL:
2170       assert(prog->getType() == Program::TYPE_COMPUTE);
2171       sym = mkSymbol(nv50_ir::FILE_MEMORY_SHARED, 0, TYPE_U32,
2172                      info->prop.cp.sharedOffset);
2173       break;
2174    case TGSI_RESOURCE_PRIVATE:
2175       sym = mkSymbol(nv50_ir::FILE_MEMORY_LOCAL, 0, TYPE_U32,
2176                      info->bin.tlsSpace);
2177       break;
2178    case TGSI_RESOURCE_INPUT:
2179       assert(prog->getType() == Program::TYPE_COMPUTE);
2180       sym = mkSymbol(nv50_ir::FILE_SHADER_INPUT, 0, TYPE_U32,
2181                      info->prop.cp.inputOffset);
2182       break;
2183    default:
2184       sym = new_Symbol(prog,
2185                        nv50_ir::FILE_MEMORY_GLOBAL, code->resources.at(r).slot);
2186       break;
2187    }
2188    return sym;
2189 }
2190
2191 void
2192 Converter::getResourceCoords(std::vector<Value *> &coords, int r, int s)
2193 {
2194    const int arg =
2195       TexInstruction::Target(getResourceTarget(code, r)).getArgCount();
2196
2197    for (int c = 0; c < arg; ++c)
2198       coords.push_back(fetchSrc(s, c));
2199
2200    // NOTE: TGSI_RESOURCE_GLOBAL needs FILE_GPR; this is an nv50 quirk
2201    if (r == TGSI_RESOURCE_LOCAL ||
2202        r == TGSI_RESOURCE_PRIVATE ||
2203        r == TGSI_RESOURCE_INPUT)
2204       coords[0] = mkOp1v(OP_MOV, TYPE_U32, getScratch(4, FILE_ADDRESS),
2205                          coords[0]);
2206 }
2207
2208 static inline int
2209 partitionLoadStore(uint8_t comp[2], uint8_t size[2], uint8_t mask)
2210 {
2211    int n = 0;
2212
2213    while (mask) {
2214       if (mask & 1) {
2215          size[n]++;
2216       } else {
2217          if (size[n])
2218             comp[n = 1] = size[0] + 1;
2219          else
2220             comp[n]++;
2221       }
2222       mask >>= 1;
2223    }
2224    if (size[0] == 3) {
2225       n = 1;
2226       size[0] = (comp[0] == 1) ? 1 : 2;
2227       size[1] = 3 - size[0];
2228       comp[1] = comp[0] + size[0];
2229    }
2230    return n + 1;
2231 }
2232
2233 // For raw loads, granularity is 4 byte.
2234 // Usage of the texture read mask on OP_SULDP is not allowed.
2235 void
2236 Converter::handleLOAD(Value *dst0[4])
2237 {
2238    const int r = tgsi.getSrc(0).getIndex(0);
2239    int c;
2240    std::vector<Value *> off, src, ldv, def;
2241
2242    if (tgsi.getSrc(0).getFile() == TGSI_FILE_BUFFER) {
2243       for (c = 0; c < 4; ++c) {
2244          if (!dst0[c])
2245             continue;
2246
2247          Value *off = fetchSrc(1, c);
2248          Symbol *sym;
2249          if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE) {
2250             off = NULL;
2251             sym = makeSym(TGSI_FILE_BUFFER, r, -1, c, tgsi.getSrc(1).getValueU32(0, info) + 4 * c);
2252          } else {
2253             sym = makeSym(TGSI_FILE_BUFFER, r, -1, c, 4 * c);
2254          }
2255
2256          Instruction *ld = mkLoad(TYPE_U32, dst0[c], sym, off);
2257          ld->cache = tgsi.getCacheMode();
2258          if (tgsi.getSrc(0).isIndirect(0))
2259             ld->setIndirect(0, 1, fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, 0));
2260       }
2261       return;
2262    }
2263
2264    getResourceCoords(off, r, 1);
2265
2266    if (isResourceRaw(code, r)) {
2267       uint8_t mask = 0;
2268       uint8_t comp[2] = { 0, 0 };
2269       uint8_t size[2] = { 0, 0 };
2270
2271       Symbol *base = getResourceBase(r);
2272
2273       // determine the base and size of the at most 2 load ops
2274       for (c = 0; c < 4; ++c)
2275          if (!tgsi.getDst(0).isMasked(c))
2276             mask |= 1 << (tgsi.getSrc(0).getSwizzle(c) - TGSI_SWIZZLE_X);
2277
2278       int n = partitionLoadStore(comp, size, mask);
2279
2280       src = off;
2281
2282       def.resize(4); // index by component, the ones we need will be non-NULL
2283       for (c = 0; c < 4; ++c) {
2284          if (dst0[c] && tgsi.getSrc(0).getSwizzle(c) == (TGSI_SWIZZLE_X + c))
2285             def[c] = dst0[c];
2286          else
2287          if (mask & (1 << c))
2288             def[c] = getScratch();
2289       }
2290
2291       const bool useLd = isResourceSpecial(r) ||
2292          (info->io.nv50styleSurfaces &&
2293           code->resources[r].target == TGSI_TEXTURE_BUFFER);
2294
2295       for (int i = 0; i < n; ++i) {
2296          ldv.assign(def.begin() + comp[i], def.begin() + comp[i] + size[i]);
2297
2298          if (comp[i]) // adjust x component of source address if necessary
2299             src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
2300                             off[0], mkImm(comp[i] * 4));
2301          else
2302             src[0] = off[0];
2303
2304          if (useLd) {
2305             Instruction *ld =
2306                mkLoad(typeOfSize(size[i] * 4), ldv[0], base, src[0]);
2307             for (size_t c = 1; c < ldv.size(); ++c)
2308                ld->setDef(c, ldv[c]);
2309          } else {
2310             mkTex(OP_SULDB, getResourceTarget(code, r), code->resources[r].slot,
2311                   0, ldv, src)->dType = typeOfSize(size[i] * 4);
2312          }
2313       }
2314    } else {
2315       def.resize(4);
2316       for (c = 0; c < 4; ++c) {
2317          if (!dst0[c] || tgsi.getSrc(0).getSwizzle(c) != (TGSI_SWIZZLE_X + c))
2318             def[c] = getScratch();
2319          else
2320             def[c] = dst0[c];
2321       }
2322
2323       mkTex(OP_SULDP, getResourceTarget(code, r), code->resources[r].slot, 0,
2324             def, off);
2325    }
2326    FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2327       if (dst0[c] != def[c])
2328          mkMov(dst0[c], def[tgsi.getSrc(0).getSwizzle(c)]);
2329 }
2330
2331 // For formatted stores, the write mask on OP_SUSTP can be used.
2332 // Raw stores have to be split.
2333 void
2334 Converter::handleSTORE()
2335 {
2336    const int r = tgsi.getDst(0).getIndex(0);
2337    int c;
2338    std::vector<Value *> off, src, dummy;
2339
2340    if (tgsi.getDst(0).getFile() == TGSI_FILE_BUFFER) {
2341       for (c = 0; c < 4; ++c) {
2342          if (!(tgsi.getDst(0).getMask() & (1 << c)))
2343             continue;
2344
2345          Symbol *sym;
2346          Value *off;
2347          if (tgsi.getSrc(0).getFile() == TGSI_FILE_IMMEDIATE) {
2348             off = NULL;
2349             sym = makeSym(TGSI_FILE_BUFFER, r, -1, c,
2350                           tgsi.getSrc(0).getValueU32(0, info) + 4 * c);
2351          } else {
2352             off = fetchSrc(0, 0);
2353             sym = makeSym(TGSI_FILE_BUFFER, r, -1, c, 4 * c);
2354          }
2355
2356          Instruction *st = mkStore(OP_STORE, TYPE_U32, sym, off, fetchSrc(1, c));
2357          st->cache = tgsi.getCacheMode();
2358          if (tgsi.getDst(0).isIndirect(0))
2359             st->setIndirect(0, 1, fetchSrc(tgsi.getDst(0).getIndirect(0), 0, 0));
2360       }
2361       return;
2362    }
2363
2364    getResourceCoords(off, r, 0);
2365    src = off;
2366    const int s = src.size();
2367
2368    if (isResourceRaw(code, r)) {
2369       uint8_t comp[2] = { 0, 0 };
2370       uint8_t size[2] = { 0, 0 };
2371
2372       int n = partitionLoadStore(comp, size, tgsi.getDst(0).getMask());
2373
2374       Symbol *base = getResourceBase(r);
2375
2376       const bool useSt = isResourceSpecial(r) ||
2377          (info->io.nv50styleSurfaces &&
2378           code->resources[r].target == TGSI_TEXTURE_BUFFER);
2379
2380       for (int i = 0; i < n; ++i) {
2381          if (comp[i]) // adjust x component of source address if necessary
2382             src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
2383                             off[0], mkImm(comp[i] * 4));
2384          else
2385             src[0] = off[0];
2386
2387          const DataType stTy = typeOfSize(size[i] * 4);
2388
2389          if (useSt) {
2390             Instruction *st =
2391                mkStore(OP_STORE, stTy, base, NULL, fetchSrc(1, comp[i]));
2392             for (c = 1; c < size[i]; ++c)
2393                st->setSrc(1 + c, fetchSrc(1, comp[i] + c));
2394             st->setIndirect(0, 0, src[0]);
2395          } else {
2396             // attach values to be stored
2397             src.resize(s + size[i]);
2398             for (c = 0; c < size[i]; ++c)
2399                src[s + c] = fetchSrc(1, comp[i] + c);
2400             mkTex(OP_SUSTB, getResourceTarget(code, r), code->resources[r].slot,
2401                   0, dummy, src)->setType(stTy);
2402          }
2403       }
2404    } else {
2405       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2406          src.push_back(fetchSrc(1, c));
2407
2408       mkTex(OP_SUSTP, getResourceTarget(code, r), code->resources[r].slot, 0,
2409             dummy, src)->tex.mask = tgsi.getDst(0).getMask();
2410    }
2411 }
2412
2413 // XXX: These only work on resources with the single-component u32/s32 formats.
2414 // Therefore the result is replicated. This might not be intended by TGSI, but
2415 // operating on more than 1 component would produce undefined results because
2416 // they do not exist.
2417 void
2418 Converter::handleATOM(Value *dst0[4], DataType ty, uint16_t subOp)
2419 {
2420    const int r = tgsi.getSrc(0).getIndex(0);
2421    std::vector<Value *> srcv;
2422    std::vector<Value *> defv;
2423    LValue *dst = getScratch();
2424
2425    if (tgsi.getSrc(0).getFile() == TGSI_FILE_BUFFER) {
2426       for (int c = 0; c < 4; ++c) {
2427          if (!dst0[c])
2428             continue;
2429
2430          Instruction *insn;
2431          Value *off = fetchSrc(1, c);
2432          Value *sym;
2433          if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE)
2434             sym = makeSym(TGSI_FILE_BUFFER, r, -1, c, tgsi.getSrc(1).getValueU32(c, info));
2435          else
2436             sym = makeSym(TGSI_FILE_BUFFER, r, -1, c, 0);
2437          insn = mkOp2(OP_ATOM, ty, dst, sym, fetchSrc(2, c));
2438          if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2439             insn->setSrc(2, fetchSrc(3, 0));
2440          if (tgsi.getSrc(1).getFile() != TGSI_FILE_IMMEDIATE)
2441             insn->setIndirect(0, 0, off);
2442          if (tgsi.getSrc(0).isIndirect(0))
2443             insn->setIndirect(0, 1, fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, 0));
2444          insn->subOp = subOp;
2445       }
2446       for (int c = 0; c < 4; ++c)
2447          if (dst0[c])
2448             dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov
2449       return;
2450    }
2451
2452
2453    getResourceCoords(srcv, r, 1);
2454
2455    if (isResourceSpecial(r)) {
2456       assert(r != TGSI_RESOURCE_INPUT);
2457       Instruction *insn;
2458       insn = mkOp2(OP_ATOM, ty, dst, getResourceBase(r), fetchSrc(2, 0));
2459       insn->subOp = subOp;
2460       if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2461          insn->setSrc(2, fetchSrc(3, 0));
2462       insn->setIndirect(0, 0, srcv.at(0));
2463    } else {
2464       operation op = isResourceRaw(code, r) ? OP_SUREDB : OP_SUREDP;
2465       TexTarget targ = getResourceTarget(code, r);
2466       int idx = code->resources[r].slot;
2467       defv.push_back(dst);
2468       srcv.push_back(fetchSrc(2, 0));
2469       if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2470          srcv.push_back(fetchSrc(3, 0));
2471       TexInstruction *tex = mkTex(op, targ, idx, 0, defv, srcv);
2472       tex->subOp = subOp;
2473       tex->tex.mask = 1;
2474       tex->setType(ty);
2475    }
2476
2477    for (int c = 0; c < 4; ++c)
2478       if (dst0[c])
2479          dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov
2480 }
2481
2482 void
2483 Converter::handleINTERP(Value *dst[4])
2484 {
2485    // Check whether the input is linear. All other attributes ignored.
2486    Instruction *insn;
2487    Value *offset = NULL, *ptr = NULL, *w = NULL;
2488    bool linear;
2489    operation op;
2490    int c, mode;
2491
2492    tgsi::Instruction::SrcRegister src = tgsi.getSrc(0);
2493    assert(src.getFile() == TGSI_FILE_INPUT);
2494
2495    if (src.isIndirect(0))
2496       ptr = fetchSrc(src.getIndirect(0), 0, NULL);
2497
2498    // XXX: no way to know interp mode if we don't know the index
2499    linear = info->in[ptr ? 0 : src.getIndex(0)].linear;
2500    if (linear) {
2501       op = OP_LINTERP;
2502       mode = NV50_IR_INTERP_LINEAR;
2503    } else {
2504       op = OP_PINTERP;
2505       mode = NV50_IR_INTERP_PERSPECTIVE;
2506    }
2507
2508    switch (tgsi.getOpcode()) {
2509    case TGSI_OPCODE_INTERP_CENTROID:
2510       mode |= NV50_IR_INTERP_CENTROID;
2511       break;
2512    case TGSI_OPCODE_INTERP_SAMPLE:
2513       insn = mkOp1(OP_PIXLD, TYPE_U32, (offset = getScratch()), fetchSrc(1, 0));
2514       insn->subOp = NV50_IR_SUBOP_PIXLD_OFFSET;
2515       mode |= NV50_IR_INTERP_OFFSET;
2516       break;
2517    case TGSI_OPCODE_INTERP_OFFSET: {
2518       // The input in src1.xy is float, but we need a single 32-bit value
2519       // where the upper and lower 16 bits are encoded in S0.12 format. We need
2520       // to clamp the input coordinates to (-0.5, 0.4375), multiply by 4096,
2521       // and then convert to s32.
2522       Value *offs[2];
2523       for (c = 0; c < 2; c++) {
2524          offs[c] = fetchSrc(1, c);
2525          mkOp2(OP_MIN, TYPE_F32, offs[c], offs[c], loadImm(NULL, 0.4375f));
2526          mkOp2(OP_MAX, TYPE_F32, offs[c], offs[c], loadImm(NULL, -0.5f));
2527          mkOp2(OP_MUL, TYPE_F32, offs[c], offs[c], loadImm(NULL, 4096.0f));
2528          mkCvt(OP_CVT, TYPE_S32, offs[c], TYPE_F32, offs[c]);
2529       }
2530       offset = mkOp3v(OP_INSBF, TYPE_U32, getScratch(),
2531                       offs[1], mkImm(0x1010), offs[0]);
2532       mode |= NV50_IR_INTERP_OFFSET;
2533       break;
2534    }
2535    }
2536
2537    if (op == OP_PINTERP) {
2538       if (offset) {
2539          w = mkOp2v(OP_RDSV, TYPE_F32, getSSA(), mkSysVal(SV_POSITION, 3), offset);
2540          mkOp1(OP_RCP, TYPE_F32, w, w);
2541       } else {
2542          w = fragCoord[3];
2543       }
2544    }
2545
2546
2547    FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2548       insn = mkOp1(op, TYPE_F32, dst[c], srcToSym(src, c));
2549       if (op == OP_PINTERP)
2550          insn->setSrc(1, w);
2551       if (ptr)
2552          insn->setIndirect(0, 0, ptr);
2553       if (offset)
2554          insn->setSrc(op == OP_PINTERP ? 2 : 1, offset);
2555
2556       insn->setInterpolate(mode);
2557    }
2558 }
2559
2560 Converter::Subroutine *
2561 Converter::getSubroutine(unsigned ip)
2562 {
2563    std::map<unsigned, Subroutine>::iterator it = sub.map.find(ip);
2564
2565    if (it == sub.map.end())
2566       it = sub.map.insert(std::make_pair(
2567               ip, Subroutine(new Function(prog, "SUB", ip)))).first;
2568
2569    return &it->second;
2570 }
2571
2572 Converter::Subroutine *
2573 Converter::getSubroutine(Function *f)
2574 {
2575    unsigned ip = f->getLabel();
2576    std::map<unsigned, Subroutine>::iterator it = sub.map.find(ip);
2577
2578    if (it == sub.map.end())
2579       it = sub.map.insert(std::make_pair(ip, Subroutine(f))).first;
2580
2581    return &it->second;
2582 }
2583
2584 bool
2585 Converter::isEndOfSubroutine(uint ip)
2586 {
2587    assert(ip < code->scan.num_instructions);
2588    tgsi::Instruction insn(&code->insns[ip]);
2589    return (insn.getOpcode() == TGSI_OPCODE_END ||
2590            insn.getOpcode() == TGSI_OPCODE_ENDSUB ||
2591            // does END occur at end of main or the very end ?
2592            insn.getOpcode() == TGSI_OPCODE_BGNSUB);
2593 }
2594
2595 bool
2596 Converter::handleInstruction(const struct tgsi_full_instruction *insn)
2597 {
2598    Instruction *geni;
2599
2600    Value *dst0[4], *rDst0[4];
2601    Value *src0, *src1, *src2, *src3;
2602    Value *val0, *val1;
2603    int c;
2604
2605    tgsi = tgsi::Instruction(insn);
2606
2607    bool useScratchDst = tgsi.checkDstSrcAliasing();
2608
2609    operation op = tgsi.getOP();
2610    dstTy = tgsi.inferDstType();
2611    srcTy = tgsi.inferSrcType();
2612
2613    unsigned int mask = tgsi.dstCount() ? tgsi.getDst(0).getMask() : 0;
2614
2615    if (tgsi.dstCount()) {
2616       for (c = 0; c < 4; ++c) {
2617          rDst0[c] = acquireDst(0, c);
2618          dst0[c] = (useScratchDst && rDst0[c]) ? getScratch() : rDst0[c];
2619       }
2620    }
2621
2622    switch (tgsi.getOpcode()) {
2623    case TGSI_OPCODE_ADD:
2624    case TGSI_OPCODE_UADD:
2625    case TGSI_OPCODE_AND:
2626    case TGSI_OPCODE_DIV:
2627    case TGSI_OPCODE_IDIV:
2628    case TGSI_OPCODE_UDIV:
2629    case TGSI_OPCODE_MAX:
2630    case TGSI_OPCODE_MIN:
2631    case TGSI_OPCODE_IMAX:
2632    case TGSI_OPCODE_IMIN:
2633    case TGSI_OPCODE_UMAX:
2634    case TGSI_OPCODE_UMIN:
2635    case TGSI_OPCODE_MOD:
2636    case TGSI_OPCODE_UMOD:
2637    case TGSI_OPCODE_MUL:
2638    case TGSI_OPCODE_UMUL:
2639    case TGSI_OPCODE_IMUL_HI:
2640    case TGSI_OPCODE_UMUL_HI:
2641    case TGSI_OPCODE_OR:
2642    case TGSI_OPCODE_SHL:
2643    case TGSI_OPCODE_ISHR:
2644    case TGSI_OPCODE_USHR:
2645    case TGSI_OPCODE_SUB:
2646    case TGSI_OPCODE_XOR:
2647       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2648          src0 = fetchSrc(0, c);
2649          src1 = fetchSrc(1, c);
2650          geni = mkOp2(op, dstTy, dst0[c], src0, src1);
2651          geni->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode());
2652       }
2653       break;
2654    case TGSI_OPCODE_MAD:
2655    case TGSI_OPCODE_UMAD:
2656    case TGSI_OPCODE_SAD:
2657       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2658          src0 = fetchSrc(0, c);
2659          src1 = fetchSrc(1, c);
2660          src2 = fetchSrc(2, c);
2661          mkOp3(op, dstTy, dst0[c], src0, src1, src2);
2662       }
2663       break;
2664    case TGSI_OPCODE_MOV:
2665    case TGSI_OPCODE_ABS:
2666    case TGSI_OPCODE_CEIL:
2667    case TGSI_OPCODE_FLR:
2668    case TGSI_OPCODE_TRUNC:
2669    case TGSI_OPCODE_RCP:
2670    case TGSI_OPCODE_IABS:
2671    case TGSI_OPCODE_INEG:
2672    case TGSI_OPCODE_NOT:
2673    case TGSI_OPCODE_DDX:
2674    case TGSI_OPCODE_DDY:
2675    case TGSI_OPCODE_DDX_FINE:
2676    case TGSI_OPCODE_DDY_FINE:
2677       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2678          mkOp1(op, dstTy, dst0[c], fetchSrc(0, c));
2679       break;
2680    case TGSI_OPCODE_RSQ:
2681       src0 = fetchSrc(0, 0);
2682       val0 = getScratch();
2683       mkOp1(OP_ABS, TYPE_F32, val0, src0);
2684       mkOp1(OP_RSQ, TYPE_F32, val0, val0);
2685       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2686          mkMov(dst0[c], val0);
2687       break;
2688    case TGSI_OPCODE_ARL:
2689    case TGSI_OPCODE_ARR:
2690       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2691          const RoundMode rnd =
2692             tgsi.getOpcode() == TGSI_OPCODE_ARR ? ROUND_N : ROUND_M;
2693          src0 = fetchSrc(0, c);
2694          mkCvt(OP_CVT, TYPE_S32, dst0[c], TYPE_F32, src0)->rnd = rnd;
2695       }
2696       break;
2697    case TGSI_OPCODE_UARL:
2698       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2699          mkOp1(OP_MOV, TYPE_U32, dst0[c], fetchSrc(0, c));
2700       break;
2701    case TGSI_OPCODE_POW:
2702       val0 = mkOp2v(op, TYPE_F32, getScratch(), fetchSrc(0, 0), fetchSrc(1, 0));
2703       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2704          mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
2705       break;
2706    case TGSI_OPCODE_EX2:
2707    case TGSI_OPCODE_LG2:
2708       val0 = mkOp1(op, TYPE_F32, getScratch(), fetchSrc(0, 0))->getDef(0);
2709       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2710          mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
2711       break;
2712    case TGSI_OPCODE_COS:
2713    case TGSI_OPCODE_SIN:
2714       val0 = getScratch();
2715       if (mask & 7) {
2716          mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 0));
2717          mkOp1(op, TYPE_F32, val0, val0);
2718          for (c = 0; c < 3; ++c)
2719             if (dst0[c])
2720                mkMov(dst0[c], val0);
2721       }
2722       if (dst0[3]) {
2723          mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 3));
2724          mkOp1(op, TYPE_F32, dst0[3], val0);
2725       }
2726       break;
2727    case TGSI_OPCODE_SCS:
2728       if (mask & 3) {
2729          val0 = mkOp1v(OP_PRESIN, TYPE_F32, getSSA(), fetchSrc(0, 0));
2730          if (dst0[0])
2731             mkOp1(OP_COS, TYPE_F32, dst0[0], val0);
2732          if (dst0[1])
2733             mkOp1(OP_SIN, TYPE_F32, dst0[1], val0);
2734       }
2735       if (dst0[2])
2736          loadImm(dst0[2], 0.0f);
2737       if (dst0[3])
2738          loadImm(dst0[3], 1.0f);
2739       break;
2740    case TGSI_OPCODE_EXP:
2741       src0 = fetchSrc(0, 0);
2742       val0 = mkOp1v(OP_FLOOR, TYPE_F32, getSSA(), src0);
2743       if (dst0[1])
2744          mkOp2(OP_SUB, TYPE_F32, dst0[1], src0, val0);
2745       if (dst0[0])
2746          mkOp1(OP_EX2, TYPE_F32, dst0[0], val0);
2747       if (dst0[2])
2748          mkOp1(OP_EX2, TYPE_F32, dst0[2], src0);
2749       if (dst0[3])
2750          loadImm(dst0[3], 1.0f);
2751       break;
2752    case TGSI_OPCODE_LOG:
2753       src0 = mkOp1v(OP_ABS, TYPE_F32, getSSA(), fetchSrc(0, 0));
2754       val0 = mkOp1v(OP_LG2, TYPE_F32, dst0[2] ? dst0[2] : getSSA(), src0);
2755       if (dst0[0] || dst0[1])
2756          val1 = mkOp1v(OP_FLOOR, TYPE_F32, dst0[0] ? dst0[0] : getSSA(), val0);
2757       if (dst0[1]) {
2758          mkOp1(OP_EX2, TYPE_F32, dst0[1], val1);
2759          mkOp1(OP_RCP, TYPE_F32, dst0[1], dst0[1]);
2760          mkOp2(OP_MUL, TYPE_F32, dst0[1], dst0[1], src0);
2761       }
2762       if (dst0[3])
2763          loadImm(dst0[3], 1.0f);
2764       break;
2765    case TGSI_OPCODE_DP2:
2766       val0 = buildDot(2);
2767       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2768          mkMov(dst0[c], val0);
2769       break;
2770    case TGSI_OPCODE_DP3:
2771       val0 = buildDot(3);
2772       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2773          mkMov(dst0[c], val0);
2774       break;
2775    case TGSI_OPCODE_DP4:
2776       val0 = buildDot(4);
2777       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2778          mkMov(dst0[c], val0);
2779       break;
2780    case TGSI_OPCODE_DPH:
2781       val0 = buildDot(3);
2782       src1 = fetchSrc(1, 3);
2783       mkOp2(OP_ADD, TYPE_F32, val0, val0, src1);
2784       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2785          mkMov(dst0[c], val0);
2786       break;
2787    case TGSI_OPCODE_DST:
2788       if (dst0[0])
2789          loadImm(dst0[0], 1.0f);
2790       if (dst0[1]) {
2791          src0 = fetchSrc(0, 1);
2792          src1 = fetchSrc(1, 1);
2793          mkOp2(OP_MUL, TYPE_F32, dst0[1], src0, src1);
2794       }
2795       if (dst0[2])
2796          mkMov(dst0[2], fetchSrc(0, 2));
2797       if (dst0[3])
2798          mkMov(dst0[3], fetchSrc(1, 3));
2799       break;
2800    case TGSI_OPCODE_LRP:
2801       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2802          src0 = fetchSrc(0, c);
2803          src1 = fetchSrc(1, c);
2804          src2 = fetchSrc(2, c);
2805          mkOp3(OP_MAD, TYPE_F32, dst0[c],
2806                mkOp2v(OP_SUB, TYPE_F32, getSSA(), src1, src2), src0, src2);
2807       }
2808       break;
2809    case TGSI_OPCODE_LIT:
2810       handleLIT(dst0);
2811       break;
2812    case TGSI_OPCODE_XPD:
2813       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2814          if (c < 3) {
2815             val0 = getSSA();
2816             src0 = fetchSrc(1, (c + 1) % 3);
2817             src1 = fetchSrc(0, (c + 2) % 3);
2818             mkOp2(OP_MUL, TYPE_F32, val0, src0, src1);
2819             mkOp1(OP_NEG, TYPE_F32, val0, val0);
2820
2821             src0 = fetchSrc(0, (c + 1) % 3);
2822             src1 = fetchSrc(1, (c + 2) % 3);
2823             mkOp3(OP_MAD, TYPE_F32, dst0[c], src0, src1, val0);
2824          } else {
2825             loadImm(dst0[c], 1.0f);
2826          }
2827       }
2828       break;
2829    case TGSI_OPCODE_ISSG:
2830    case TGSI_OPCODE_SSG:
2831       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2832          src0 = fetchSrc(0, c);
2833          val0 = getScratch();
2834          val1 = getScratch();
2835          mkCmp(OP_SET, CC_GT, srcTy, val0, srcTy, src0, zero);
2836          mkCmp(OP_SET, CC_LT, srcTy, val1, srcTy, src0, zero);
2837          if (srcTy == TYPE_F32)
2838             mkOp2(OP_SUB, TYPE_F32, dst0[c], val0, val1);
2839          else
2840             mkOp2(OP_SUB, TYPE_S32, dst0[c], val1, val0);
2841       }
2842       break;
2843    case TGSI_OPCODE_UCMP:
2844       srcTy = TYPE_U32;
2845       /* fallthrough */
2846    case TGSI_OPCODE_CMP:
2847       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2848          src0 = fetchSrc(0, c);
2849          src1 = fetchSrc(1, c);
2850          src2 = fetchSrc(2, c);
2851          if (src1 == src2)
2852             mkMov(dst0[c], src1);
2853          else
2854             mkCmp(OP_SLCT, (srcTy == TYPE_F32) ? CC_LT : CC_NE,
2855                   srcTy, dst0[c], srcTy, src1, src2, src0);
2856       }
2857       break;
2858    case TGSI_OPCODE_FRC:
2859       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2860          src0 = fetchSrc(0, c);
2861          val0 = getScratch();
2862          mkOp1(OP_FLOOR, TYPE_F32, val0, src0);
2863          mkOp2(OP_SUB, TYPE_F32, dst0[c], src0, val0);
2864       }
2865       break;
2866    case TGSI_OPCODE_ROUND:
2867       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2868          mkCvt(OP_CVT, TYPE_F32, dst0[c], TYPE_F32, fetchSrc(0, c))
2869          ->rnd = ROUND_NI;
2870       break;
2871    case TGSI_OPCODE_CLAMP:
2872       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2873          src0 = fetchSrc(0, c);
2874          src1 = fetchSrc(1, c);
2875          src2 = fetchSrc(2, c);
2876          val0 = getScratch();
2877          mkOp2(OP_MIN, TYPE_F32, val0, src0, src1);
2878          mkOp2(OP_MAX, TYPE_F32, dst0[c], val0, src2);
2879       }
2880       break;
2881    case TGSI_OPCODE_SLT:
2882    case TGSI_OPCODE_SGE:
2883    case TGSI_OPCODE_SEQ:
2884    case TGSI_OPCODE_SGT:
2885    case TGSI_OPCODE_SLE:
2886    case TGSI_OPCODE_SNE:
2887    case TGSI_OPCODE_FSEQ:
2888    case TGSI_OPCODE_FSGE:
2889    case TGSI_OPCODE_FSLT:
2890    case TGSI_OPCODE_FSNE:
2891    case TGSI_OPCODE_ISGE:
2892    case TGSI_OPCODE_ISLT:
2893    case TGSI_OPCODE_USEQ:
2894    case TGSI_OPCODE_USGE:
2895    case TGSI_OPCODE_USLT:
2896    case TGSI_OPCODE_USNE:
2897       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2898          src0 = fetchSrc(0, c);
2899          src1 = fetchSrc(1, c);
2900          mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], srcTy, src0, src1);
2901       }
2902       break;
2903    case TGSI_OPCODE_KILL_IF:
2904       val0 = new_LValue(func, FILE_PREDICATE);
2905       mask = 0;
2906       for (c = 0; c < 4; ++c) {
2907          const int s = tgsi.getSrc(0).getSwizzle(c);
2908          if (mask & (1 << s))
2909             continue;
2910          mask |= 1 << s;
2911          mkCmp(OP_SET, CC_LT, TYPE_F32, val0, TYPE_F32, fetchSrc(0, c), zero);
2912          mkOp(OP_DISCARD, TYPE_NONE, NULL)->setPredicate(CC_P, val0);
2913       }
2914       break;
2915    case TGSI_OPCODE_KILL:
2916       mkOp(OP_DISCARD, TYPE_NONE, NULL);
2917       break;
2918    case TGSI_OPCODE_TEX:
2919    case TGSI_OPCODE_TXB:
2920    case TGSI_OPCODE_TXL:
2921    case TGSI_OPCODE_TXP:
2922    case TGSI_OPCODE_LODQ:
2923       //              R  S     L     C    Dx    Dy
2924       handleTEX(dst0, 1, 1, 0x03, 0x0f, 0x00, 0x00);
2925       break;
2926    case TGSI_OPCODE_TXD:
2927       handleTEX(dst0, 3, 3, 0x03, 0x0f, 0x10, 0x20);
2928       break;
2929    case TGSI_OPCODE_TG4:
2930       handleTEX(dst0, 2, 2, 0x03, 0x0f, 0x00, 0x00);
2931       break;
2932    case TGSI_OPCODE_TEX2:
2933       handleTEX(dst0, 2, 2, 0x03, 0x10, 0x00, 0x00);
2934       break;
2935    case TGSI_OPCODE_TXB2:
2936    case TGSI_OPCODE_TXL2:
2937       handleTEX(dst0, 2, 2, 0x10, 0x0f, 0x00, 0x00);
2938       break;
2939    case TGSI_OPCODE_SAMPLE:
2940    case TGSI_OPCODE_SAMPLE_B:
2941    case TGSI_OPCODE_SAMPLE_D:
2942    case TGSI_OPCODE_SAMPLE_L:
2943    case TGSI_OPCODE_SAMPLE_C:
2944    case TGSI_OPCODE_SAMPLE_C_LZ:
2945       handleTEX(dst0, 1, 2, 0x30, 0x30, 0x30, 0x40);
2946       break;
2947    case TGSI_OPCODE_TXF:
2948       handleTXF(dst0, 1, 0x03);
2949       break;
2950    case TGSI_OPCODE_SAMPLE_I:
2951       handleTXF(dst0, 1, 0x03);
2952       break;
2953    case TGSI_OPCODE_SAMPLE_I_MS:
2954       handleTXF(dst0, 1, 0x20);
2955       break;
2956    case TGSI_OPCODE_TXQ:
2957    case TGSI_OPCODE_SVIEWINFO:
2958       handleTXQ(dst0, TXQ_DIMS, 1);
2959       break;
2960    case TGSI_OPCODE_TXQS:
2961       // The TXQ_TYPE query returns samples in its 3rd arg, but we need it to
2962       // be in .x
2963       dst0[1] = dst0[2] = dst0[3] = NULL;
2964       std::swap(dst0[0], dst0[2]);
2965       handleTXQ(dst0, TXQ_TYPE, 0);
2966       std::swap(dst0[0], dst0[2]);
2967       break;
2968    case TGSI_OPCODE_F2I:
2969    case TGSI_OPCODE_F2U:
2970       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2971          mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c))->rnd = ROUND_Z;
2972       break;
2973    case TGSI_OPCODE_I2F:
2974    case TGSI_OPCODE_U2F:
2975       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2976          mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c));
2977       break;
2978    case TGSI_OPCODE_PK2H:
2979       val0 = getScratch();
2980       val1 = getScratch();
2981       mkCvt(OP_CVT, TYPE_F16, val0, TYPE_F32, fetchSrc(0, 0));
2982       mkCvt(OP_CVT, TYPE_F16, val1, TYPE_F32, fetchSrc(0, 1));
2983       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2984          mkOp3(OP_INSBF, TYPE_U32, dst0[c], val1, mkImm(0x1010), val0);
2985       break;
2986    case TGSI_OPCODE_UP2H:
2987       src0 = fetchSrc(0, 0);
2988       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2989          geni = mkCvt(OP_CVT, TYPE_F32, dst0[c], TYPE_F16, src0);
2990          geni->subOp = c & 1;
2991       }
2992       break;
2993    case TGSI_OPCODE_EMIT:
2994       /* export the saved viewport index */
2995       if (viewport != NULL) {
2996          Symbol *vpSym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_U32,
2997                                   info->out[info->io.viewportId].slot[0] * 4);
2998          mkStore(OP_EXPORT, TYPE_U32, vpSym, NULL, viewport);
2999       }
3000       /* fallthrough */
3001    case TGSI_OPCODE_ENDPRIM:
3002    {
3003       // get vertex stream (must be immediate)
3004       unsigned int stream = tgsi.getSrc(0).getValueU32(0, info);
3005       if (stream && op == OP_RESTART)
3006          break;
3007       src0 = mkImm(stream);
3008       mkOp1(op, TYPE_U32, NULL, src0)->fixed = 1;
3009       break;
3010    }
3011    case TGSI_OPCODE_IF:
3012    case TGSI_OPCODE_UIF:
3013    {
3014       BasicBlock *ifBB = new BasicBlock(func);
3015
3016       bb->cfg.attach(&ifBB->cfg, Graph::Edge::TREE);
3017       condBBs.push(bb);
3018       joinBBs.push(bb);
3019
3020       mkFlow(OP_BRA, NULL, CC_NOT_P, fetchSrc(0, 0))->setType(srcTy);
3021
3022       setPosition(ifBB, true);
3023    }
3024       break;
3025    case TGSI_OPCODE_ELSE:
3026    {
3027       BasicBlock *elseBB = new BasicBlock(func);
3028       BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
3029
3030       forkBB->cfg.attach(&elseBB->cfg, Graph::Edge::TREE);
3031       condBBs.push(bb);
3032
3033       forkBB->getExit()->asFlow()->target.bb = elseBB;
3034       if (!bb->isTerminated())
3035          mkFlow(OP_BRA, NULL, CC_ALWAYS, NULL);
3036
3037       setPosition(elseBB, true);
3038    }
3039       break;
3040    case TGSI_OPCODE_ENDIF:
3041    {
3042       BasicBlock *convBB = new BasicBlock(func);
3043       BasicBlock *prevBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
3044       BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(joinBBs.pop().u.p);
3045
3046       if (!bb->isTerminated()) {
3047          // we only want join if none of the clauses ended with CONT/BREAK/RET
3048          if (prevBB->getExit()->op == OP_BRA && joinBBs.getSize() < 6)
3049             insertConvergenceOps(convBB, forkBB);
3050          mkFlow(OP_BRA, convBB, CC_ALWAYS, NULL);
3051          bb->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
3052       }
3053
3054       if (prevBB->getExit()->op == OP_BRA) {
3055          prevBB->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
3056          prevBB->getExit()->asFlow()->target.bb = convBB;
3057       }
3058       setPosition(convBB, true);
3059    }
3060       break;
3061    case TGSI_OPCODE_BGNLOOP:
3062    {
3063       BasicBlock *lbgnBB = new BasicBlock(func);
3064       BasicBlock *lbrkBB = new BasicBlock(func);
3065
3066       loopBBs.push(lbgnBB);
3067       breakBBs.push(lbrkBB);
3068       if (loopBBs.getSize() > func->loopNestingBound)
3069          func->loopNestingBound++;
3070
3071       mkFlow(OP_PREBREAK, lbrkBB, CC_ALWAYS, NULL);
3072
3073       bb->cfg.attach(&lbgnBB->cfg, Graph::Edge::TREE);
3074       setPosition(lbgnBB, true);
3075       mkFlow(OP_PRECONT, lbgnBB, CC_ALWAYS, NULL);
3076    }
3077       break;
3078    case TGSI_OPCODE_ENDLOOP:
3079    {
3080       BasicBlock *loopBB = reinterpret_cast<BasicBlock *>(loopBBs.pop().u.p);
3081
3082       if (!bb->isTerminated()) {
3083          mkFlow(OP_CONT, loopBB, CC_ALWAYS, NULL);
3084          bb->cfg.attach(&loopBB->cfg, Graph::Edge::BACK);
3085       }
3086       setPosition(reinterpret_cast<BasicBlock *>(breakBBs.pop().u.p), true);
3087
3088       // If the loop never breaks (e.g. only has RET's inside), then there
3089       // will be no way to get to the break bb. However BGNLOOP will have
3090       // already made a PREBREAK to it, so it must be in the CFG.
3091       if (getBB()->cfg.incidentCount() == 0)
3092          loopBB->cfg.attach(&getBB()->cfg, Graph::Edge::TREE);
3093    }
3094       break;
3095    case TGSI_OPCODE_BRK:
3096    {
3097       if (bb->isTerminated())
3098          break;
3099       BasicBlock *brkBB = reinterpret_cast<BasicBlock *>(breakBBs.peek().u.p);
3100       mkFlow(OP_BREAK, brkBB, CC_ALWAYS, NULL);
3101       bb->cfg.attach(&brkBB->cfg, Graph::Edge::CROSS);
3102    }
3103       break;
3104    case TGSI_OPCODE_CONT:
3105    {
3106       if (bb->isTerminated())
3107          break;
3108       BasicBlock *contBB = reinterpret_cast<BasicBlock *>(loopBBs.peek().u.p);
3109       mkFlow(OP_CONT, contBB, CC_ALWAYS, NULL);
3110       contBB->explicitCont = true;
3111       bb->cfg.attach(&contBB->cfg, Graph::Edge::BACK);
3112    }
3113       break;
3114    case TGSI_OPCODE_BGNSUB:
3115    {
3116       Subroutine *s = getSubroutine(ip);
3117       BasicBlock *entry = new BasicBlock(s->f);
3118       BasicBlock *leave = new BasicBlock(s->f);
3119
3120       // multiple entrypoints possible, keep the graph connected
3121       if (prog->getType() == Program::TYPE_COMPUTE)
3122          prog->main->call.attach(&s->f->call, Graph::Edge::TREE);
3123
3124       sub.cur = s;
3125       s->f->setEntry(entry);
3126       s->f->setExit(leave);
3127       setPosition(entry, true);
3128       return true;
3129    }
3130    case TGSI_OPCODE_ENDSUB:
3131    {
3132       sub.cur = getSubroutine(prog->main);
3133       setPosition(BasicBlock::get(sub.cur->f->cfg.getRoot()), true);
3134       return true;
3135    }
3136    case TGSI_OPCODE_CAL:
3137    {
3138       Subroutine *s = getSubroutine(tgsi.getLabel());
3139       mkFlow(OP_CALL, s->f, CC_ALWAYS, NULL);
3140       func->call.attach(&s->f->call, Graph::Edge::TREE);
3141       return true;
3142    }
3143    case TGSI_OPCODE_RET:
3144    {
3145       if (bb->isTerminated())
3146          return true;
3147       BasicBlock *leave = BasicBlock::get(func->cfgExit);
3148
3149       if (!isEndOfSubroutine(ip + 1)) {
3150          // insert a PRERET at the entry if this is an early return
3151          // (only needed for sharing code in the epilogue)
3152          BasicBlock *pos = getBB();
3153          setPosition(BasicBlock::get(func->cfg.getRoot()), false);
3154          mkFlow(OP_PRERET, leave, CC_ALWAYS, NULL)->fixed = 1;
3155          setPosition(pos, true);
3156       }
3157       mkFlow(OP_RET, NULL, CC_ALWAYS, NULL)->fixed = 1;
3158       bb->cfg.attach(&leave->cfg, Graph::Edge::CROSS);
3159    }
3160       break;
3161    case TGSI_OPCODE_END:
3162    {
3163       // attach and generate epilogue code
3164       BasicBlock *epilogue = BasicBlock::get(func->cfgExit);
3165       bb->cfg.attach(&epilogue->cfg, Graph::Edge::TREE);
3166       setPosition(epilogue, true);
3167       if (prog->getType() == Program::TYPE_FRAGMENT)
3168          exportOutputs();
3169       if (info->io.genUserClip > 0)
3170          handleUserClipPlanes();
3171       mkOp(OP_EXIT, TYPE_NONE, NULL)->terminator = 1;
3172    }
3173       break;
3174    case TGSI_OPCODE_SWITCH:
3175    case TGSI_OPCODE_CASE:
3176       ERROR("switch/case opcode encountered, should have been lowered\n");
3177       abort();
3178       break;
3179    case TGSI_OPCODE_LOAD:
3180       handleLOAD(dst0);
3181       break;
3182    case TGSI_OPCODE_STORE:
3183       handleSTORE();
3184       break;
3185    case TGSI_OPCODE_BARRIER:
3186       geni = mkOp2(OP_BAR, TYPE_U32, NULL, mkImm(0), mkImm(0));
3187       geni->fixed = 1;
3188       geni->subOp = NV50_IR_SUBOP_BAR_SYNC;
3189       break;
3190    case TGSI_OPCODE_MFENCE:
3191    case TGSI_OPCODE_LFENCE:
3192    case TGSI_OPCODE_SFENCE:
3193       geni = mkOp(OP_MEMBAR, TYPE_NONE, NULL);
3194       geni->fixed = 1;
3195       geni->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode());
3196       break;
3197    case TGSI_OPCODE_ATOMUADD:
3198    case TGSI_OPCODE_ATOMXCHG:
3199    case TGSI_OPCODE_ATOMCAS:
3200    case TGSI_OPCODE_ATOMAND:
3201    case TGSI_OPCODE_ATOMOR:
3202    case TGSI_OPCODE_ATOMXOR:
3203    case TGSI_OPCODE_ATOMUMIN:
3204    case TGSI_OPCODE_ATOMIMIN:
3205    case TGSI_OPCODE_ATOMUMAX:
3206    case TGSI_OPCODE_ATOMIMAX:
3207       handleATOM(dst0, dstTy, tgsi::opcodeToSubOp(tgsi.getOpcode()));
3208       break;
3209    case TGSI_OPCODE_RESQ:
3210       geni = mkOp1(OP_SUQ, TYPE_U32, dst0[0],
3211                    makeSym(TGSI_FILE_BUFFER, tgsi.getSrc(0).getIndex(0), -1, 0, 0));
3212       if (tgsi.getSrc(0).isIndirect(0))
3213          geni->setIndirect(0, 1, fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, 0));
3214       break;
3215    case TGSI_OPCODE_IBFE:
3216    case TGSI_OPCODE_UBFE:
3217       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3218          src0 = fetchSrc(0, c);
3219          if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE &&
3220              tgsi.getSrc(2).getFile() == TGSI_FILE_IMMEDIATE) {
3221             src1 = loadImm(NULL, tgsi.getSrc(2).getValueU32(c, info) << 8 |
3222                            tgsi.getSrc(1).getValueU32(c, info));
3223          } else {
3224             src1 = fetchSrc(1, c);
3225             src2 = fetchSrc(2, c);
3226             mkOp3(OP_INSBF, TYPE_U32, src1, src2, mkImm(0x808), src1);
3227          }
3228          mkOp2(OP_EXTBF, dstTy, dst0[c], src0, src1);
3229       }
3230       break;
3231    case TGSI_OPCODE_BFI:
3232       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3233          src0 = fetchSrc(0, c);
3234          src1 = fetchSrc(1, c);
3235          src2 = fetchSrc(2, c);
3236          src3 = fetchSrc(3, c);
3237          mkOp3(OP_INSBF, TYPE_U32, src2, src3, mkImm(0x808), src2);
3238          mkOp3(OP_INSBF, TYPE_U32, dst0[c], src1, src2, src0);
3239       }
3240       break;
3241    case TGSI_OPCODE_LSB:
3242       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3243          src0 = fetchSrc(0, c);
3244          geni = mkOp2(OP_EXTBF, TYPE_U32, src0, src0, mkImm(0x2000));
3245          geni->subOp = NV50_IR_SUBOP_EXTBF_REV;
3246          geni = mkOp1(OP_BFIND, TYPE_U32, dst0[c], src0);
3247          geni->subOp = NV50_IR_SUBOP_BFIND_SAMT;
3248       }
3249       break;
3250    case TGSI_OPCODE_IMSB:
3251    case TGSI_OPCODE_UMSB:
3252       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3253          src0 = fetchSrc(0, c);
3254          mkOp1(OP_BFIND, srcTy, dst0[c], src0);
3255       }
3256       break;
3257    case TGSI_OPCODE_BREV:
3258       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3259          src0 = fetchSrc(0, c);
3260          geni = mkOp2(OP_EXTBF, TYPE_U32, dst0[c], src0, mkImm(0x2000));
3261          geni->subOp = NV50_IR_SUBOP_EXTBF_REV;
3262       }
3263       break;
3264    case TGSI_OPCODE_POPC:
3265       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3266          src0 = fetchSrc(0, c);
3267          mkOp2(OP_POPCNT, TYPE_U32, dst0[c], src0, src0);
3268       }
3269       break;
3270    case TGSI_OPCODE_INTERP_CENTROID:
3271    case TGSI_OPCODE_INTERP_SAMPLE:
3272    case TGSI_OPCODE_INTERP_OFFSET:
3273       handleINTERP(dst0);
3274       break;
3275    case TGSI_OPCODE_D2I:
3276    case TGSI_OPCODE_D2U:
3277    case TGSI_OPCODE_D2F: {
3278       int pos = 0;
3279       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3280          Value *dreg = getSSA(8);
3281          src0 = fetchSrc(0, pos);
3282          src1 = fetchSrc(0, pos + 1);
3283          mkOp2(OP_MERGE, TYPE_U64, dreg, src0, src1);
3284          mkCvt(OP_CVT, dstTy, dst0[c], srcTy, dreg);
3285          pos += 2;
3286       }
3287       break;
3288    }
3289    case TGSI_OPCODE_I2D:
3290    case TGSI_OPCODE_U2D:
3291    case TGSI_OPCODE_F2D:
3292       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3293          Value *dreg = getSSA(8);
3294          mkCvt(OP_CVT, dstTy, dreg, srcTy, fetchSrc(0, c / 2));
3295          mkSplit(&dst0[c], 4, dreg);
3296          c++;
3297       }
3298       break;
3299    case TGSI_OPCODE_DABS:
3300    case TGSI_OPCODE_DNEG:
3301    case TGSI_OPCODE_DRCP:
3302    case TGSI_OPCODE_DSQRT:
3303    case TGSI_OPCODE_DRSQ:
3304    case TGSI_OPCODE_DTRUNC:
3305    case TGSI_OPCODE_DCEIL:
3306    case TGSI_OPCODE_DFLR:
3307       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3308          src0 = getSSA(8);
3309          Value *dst = getSSA(8), *tmp[2];
3310          tmp[0] = fetchSrc(0, c);
3311          tmp[1] = fetchSrc(0, c + 1);
3312          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3313          mkOp1(op, dstTy, dst, src0);
3314          mkSplit(&dst0[c], 4, dst);
3315          c++;
3316       }
3317       break;
3318    case TGSI_OPCODE_DFRAC:
3319       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3320          src0 = getSSA(8);
3321          Value *dst = getSSA(8), *tmp[2];
3322          tmp[0] = fetchSrc(0, c);
3323          tmp[1] = fetchSrc(0, c + 1);
3324          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3325          mkOp1(OP_FLOOR, TYPE_F64, dst, src0);
3326          mkOp2(OP_SUB, TYPE_F64, dst, src0, dst);
3327          mkSplit(&dst0[c], 4, dst);
3328          c++;
3329       }
3330       break;
3331    case TGSI_OPCODE_DSLT:
3332    case TGSI_OPCODE_DSGE:
3333    case TGSI_OPCODE_DSEQ:
3334    case TGSI_OPCODE_DSNE: {
3335       int pos = 0;
3336       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3337          Value *tmp[2];
3338
3339          src0 = getSSA(8);
3340          src1 = getSSA(8);
3341          tmp[0] = fetchSrc(0, pos);
3342          tmp[1] = fetchSrc(0, pos + 1);
3343          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3344          tmp[0] = fetchSrc(1, pos);
3345          tmp[1] = fetchSrc(1, pos + 1);
3346          mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
3347          mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], srcTy, src0, src1);
3348          pos += 2;
3349       }
3350       break;
3351    }
3352    case TGSI_OPCODE_DADD:
3353    case TGSI_OPCODE_DMUL:
3354    case TGSI_OPCODE_DMAX:
3355    case TGSI_OPCODE_DMIN:
3356       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3357          src0 = getSSA(8);
3358          src1 = getSSA(8);
3359          Value *dst = getSSA(8), *tmp[2];
3360          tmp[0] = fetchSrc(0, c);
3361          tmp[1] = fetchSrc(0, c + 1);
3362          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3363          tmp[0] = fetchSrc(1, c);
3364          tmp[1] = fetchSrc(1, c + 1);
3365          mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
3366          mkOp2(op, dstTy, dst, src0, src1);
3367          mkSplit(&dst0[c], 4, dst);
3368          c++;
3369       }
3370       break;
3371    case TGSI_OPCODE_DMAD:
3372       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3373          src0 = getSSA(8);
3374          src1 = getSSA(8);
3375          src2 = getSSA(8);
3376          Value *dst = getSSA(8), *tmp[2];
3377          tmp[0] = fetchSrc(0, c);
3378          tmp[1] = fetchSrc(0, c + 1);
3379          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3380          tmp[0] = fetchSrc(1, c);
3381          tmp[1] = fetchSrc(1, c + 1);
3382          mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
3383          tmp[0] = fetchSrc(2, c);
3384          tmp[1] = fetchSrc(2, c + 1);
3385          mkOp2(OP_MERGE, TYPE_U64, src2, tmp[0], tmp[1]);
3386          mkOp3(op, dstTy, dst, src0, src1, src2);
3387          mkSplit(&dst0[c], 4, dst);
3388          c++;
3389       }
3390       break;
3391    case TGSI_OPCODE_DROUND:
3392       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3393          src0 = getSSA(8);
3394          Value *dst = getSSA(8), *tmp[2];
3395          tmp[0] = fetchSrc(0, c);
3396          tmp[1] = fetchSrc(0, c + 1);
3397          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3398          mkCvt(OP_CVT, TYPE_F64, dst, TYPE_F64, src0)
3399          ->rnd = ROUND_NI;
3400          mkSplit(&dst0[c], 4, dst);
3401          c++;
3402       }
3403       break;
3404    case TGSI_OPCODE_DSSG:
3405       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3406          src0 = getSSA(8);
3407          Value *dst = getSSA(8), *dstF32 = getSSA(), *tmp[2];
3408          tmp[0] = fetchSrc(0, c);
3409          tmp[1] = fetchSrc(0, c + 1);
3410          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3411
3412          val0 = getScratch();
3413          val1 = getScratch();
3414          // The zero is wrong here since it's only 32-bit, but it works out in
3415          // the end since it gets replaced with $r63.
3416          mkCmp(OP_SET, CC_GT, TYPE_F32, val0, TYPE_F64, src0, zero);
3417          mkCmp(OP_SET, CC_LT, TYPE_F32, val1, TYPE_F64, src0, zero);
3418          mkOp2(OP_SUB, TYPE_F32, dstF32, val0, val1);
3419          mkCvt(OP_CVT, TYPE_F64, dst, TYPE_F32, dstF32);
3420          mkSplit(&dst0[c], 4, dst);
3421          c++;
3422       }
3423       break;
3424    default:
3425       ERROR("unhandled TGSI opcode: %u\n", tgsi.getOpcode());
3426       assert(0);
3427       break;
3428    }
3429
3430    if (tgsi.dstCount()) {
3431       for (c = 0; c < 4; ++c) {
3432          if (!dst0[c])
3433             continue;
3434          if (dst0[c] != rDst0[c])
3435             mkMov(rDst0[c], dst0[c]);
3436          storeDst(0, c, rDst0[c]);
3437       }
3438    }
3439    vtxBaseValid = 0;
3440
3441    return true;
3442 }
3443
3444 void
3445 Converter::handleUserClipPlanes()
3446 {
3447    Value *res[8];
3448    int n, i, c;
3449
3450    for (c = 0; c < 4; ++c) {
3451       for (i = 0; i < info->io.genUserClip; ++i) {
3452          Symbol *sym = mkSymbol(FILE_MEMORY_CONST, info->io.auxCBSlot,
3453                                 TYPE_F32, info->io.ucpBase + i * 16 + c * 4);
3454          Value *ucp = mkLoadv(TYPE_F32, sym, NULL);
3455          if (c == 0)
3456             res[i] = mkOp2v(OP_MUL, TYPE_F32, getScratch(), clipVtx[c], ucp);
3457          else
3458             mkOp3(OP_MAD, TYPE_F32, res[i], clipVtx[c], ucp, res[i]);
3459       }
3460    }
3461
3462    const int first = info->numOutputs - (info->io.genUserClip + 3) / 4;
3463
3464    for (i = 0; i < info->io.genUserClip; ++i) {
3465       n = i / 4 + first;
3466       c = i % 4;
3467       Symbol *sym =
3468          mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32, info->out[n].slot[c] * 4);
3469       mkStore(OP_EXPORT, TYPE_F32, sym, NULL, res[i]);
3470    }
3471 }
3472
3473 void
3474 Converter::exportOutputs()
3475 {
3476    for (unsigned int i = 0; i < info->numOutputs; ++i) {
3477       for (unsigned int c = 0; c < 4; ++c) {
3478          if (!oData.exists(sub.cur->values, i, c))
3479             continue;
3480          Symbol *sym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32,
3481                                 info->out[i].slot[c] * 4);
3482          Value *val = oData.load(sub.cur->values, i, c, NULL);
3483          if (val)
3484             mkStore(OP_EXPORT, TYPE_F32, sym, NULL, val);
3485       }
3486    }
3487 }
3488
3489 Converter::Converter(Program *ir, const tgsi::Source *code) : BuildUtil(ir),
3490      code(code),
3491      tgsi(NULL),
3492      tData(this), lData(this), aData(this), pData(this), oData(this)
3493 {
3494    info = code->info;
3495
3496    const unsigned tSize = code->fileSize(TGSI_FILE_TEMPORARY);
3497    const unsigned pSize = code->fileSize(TGSI_FILE_PREDICATE);
3498    const unsigned aSize = code->fileSize(TGSI_FILE_ADDRESS);
3499    const unsigned oSize = code->fileSize(TGSI_FILE_OUTPUT);
3500
3501    tData.setup(TGSI_FILE_TEMPORARY, 0, 0, tSize, 4, 4, FILE_GPR, 0);
3502    lData.setup(TGSI_FILE_TEMPORARY, 1, 0, tSize, 4, 4, FILE_MEMORY_LOCAL, 0);
3503    pData.setup(TGSI_FILE_PREDICATE, 0, 0, pSize, 4, 4, FILE_PREDICATE, 0);
3504    aData.setup(TGSI_FILE_ADDRESS, 0, 0, aSize, 4, 4, FILE_GPR, 0);
3505    oData.setup(TGSI_FILE_OUTPUT, 0, 0, oSize, 4, 4, FILE_GPR, 0);
3506
3507    zero = mkImm((uint32_t)0);
3508
3509    vtxBaseValid = 0;
3510 }
3511
3512 Converter::~Converter()
3513 {
3514 }
3515
3516 inline const Converter::Location *
3517 Converter::BindArgumentsPass::getValueLocation(Subroutine *s, Value *v)
3518 {
3519    ValueMap::l_iterator it = s->values.l.find(v);
3520    return it == s->values.l.end() ? NULL : &it->second;
3521 }
3522
3523 template<typename T> inline void
3524 Converter::BindArgumentsPass::updateCallArgs(
3525    Instruction *i, void (Instruction::*setArg)(int, Value *),
3526    T (Function::*proto))
3527 {
3528    Function *g = i->asFlow()->target.fn;
3529    Subroutine *subg = conv.getSubroutine(g);
3530
3531    for (unsigned a = 0; a < (g->*proto).size(); ++a) {
3532       Value *v = (g->*proto)[a].get();
3533       const Converter::Location &l = *getValueLocation(subg, v);
3534       Converter::DataArray *array = conv.getArrayForFile(l.array, l.arrayIdx);
3535
3536       (i->*setArg)(a, array->acquire(sub->values, l.i, l.c));
3537    }
3538 }
3539
3540 template<typename T> inline void
3541 Converter::BindArgumentsPass::updatePrototype(
3542    BitSet *set, void (Function::*updateSet)(), T (Function::*proto))
3543 {
3544    (func->*updateSet)();
3545
3546    for (unsigned i = 0; i < set->getSize(); ++i) {
3547       Value *v = func->getLValue(i);
3548       const Converter::Location *l = getValueLocation(sub, v);
3549
3550       // only include values with a matching TGSI register
3551       if (set->test(i) && l && !conv.code->locals.count(*l))
3552          (func->*proto).push_back(v);
3553    }
3554 }
3555
3556 bool
3557 Converter::BindArgumentsPass::visit(Function *f)
3558 {
3559    sub = conv.getSubroutine(f);
3560
3561    for (ArrayList::Iterator bi = f->allBBlocks.iterator();
3562         !bi.end(); bi.next()) {
3563       for (Instruction *i = BasicBlock::get(bi)->getFirst();
3564            i; i = i->next) {
3565          if (i->op == OP_CALL && !i->asFlow()->builtin) {
3566             updateCallArgs(i, &Instruction::setSrc, &Function::ins);
3567             updateCallArgs(i, &Instruction::setDef, &Function::outs);
3568          }
3569       }
3570    }
3571
3572    if (func == prog->main && prog->getType() != Program::TYPE_COMPUTE)
3573       return true;
3574    updatePrototype(&BasicBlock::get(f->cfg.getRoot())->liveSet,
3575                    &Function::buildLiveSets, &Function::ins);
3576    updatePrototype(&BasicBlock::get(f->cfgExit)->defSet,
3577                    &Function::buildDefSets, &Function::outs);
3578
3579    return true;
3580 }
3581
3582 bool
3583 Converter::run()
3584 {
3585    BasicBlock *entry = new BasicBlock(prog->main);
3586    BasicBlock *leave = new BasicBlock(prog->main);
3587
3588    prog->main->setEntry(entry);
3589    prog->main->setExit(leave);
3590
3591    setPosition(entry, true);
3592    sub.cur = getSubroutine(prog->main);
3593
3594    if (info->io.genUserClip > 0) {
3595       for (int c = 0; c < 4; ++c)
3596          clipVtx[c] = getScratch();
3597    }
3598
3599    switch (prog->getType()) {
3600    case Program::TYPE_TESSELLATION_CONTROL:
3601       outBase = mkOp2v(
3602          OP_SUB, TYPE_U32, getSSA(),
3603          mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_LANEID, 0)),
3604          mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_INVOCATION_ID, 0)));
3605       break;
3606    case Program::TYPE_FRAGMENT: {
3607       Symbol *sv = mkSysVal(SV_POSITION, 3);
3608       fragCoord[3] = mkOp1v(OP_RDSV, TYPE_F32, getSSA(), sv);
3609       mkOp1(OP_RCP, TYPE_F32, fragCoord[3], fragCoord[3]);
3610       break;
3611    }
3612    default:
3613       break;
3614    }
3615
3616    if (info->io.viewportId >= 0)
3617       viewport = getScratch();
3618    else
3619       viewport = NULL;
3620
3621    for (ip = 0; ip < code->scan.num_instructions; ++ip) {
3622       if (!handleInstruction(&code->insns[ip]))
3623          return false;
3624    }
3625
3626    if (!BindArgumentsPass(*this).run(prog))
3627       return false;
3628
3629    return true;
3630 }
3631
3632 } // unnamed namespace
3633
3634 namespace nv50_ir {
3635
3636 bool
3637 Program::makeFromTGSI(struct nv50_ir_prog_info *info)
3638 {
3639    tgsi::Source src(info);
3640    if (!src.scanSource())
3641       return false;
3642    tlsSize = info->bin.tlsSpace;
3643
3644    Converter builder(this, &src);
3645    return builder.run();
3646 }
3647
3648 } // namespace nv50_ir