src/gallium/drivers/nouveau/codegen/nv50_ir_from_tgsi.cpp

   1 /*
   2  * Copyright 2011 Christoph Bumiller
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 #include "tgsi/tgsi_build.h"
  24 #include "tgsi/tgsi_dump.h"
  25 #include "tgsi/tgsi_scan.h"
  26 #include "tgsi/tgsi_util.h"
  27
  28 #include <set>
  29
  30 #include "codegen/nv50_ir.h"
  31 #include "codegen/nv50_ir_from_common.h"
  32 #include "codegen/nv50_ir_util.h"
  33
  34 namespace tgsi {
  35
  36 class Source;
  37
  38 static nv50_ir::operation translateOpcode(uint opcode);
  39 static nv50_ir::DataFile translateFile(uint file);
  40 static nv50_ir::TexTarget translateTexture(uint texTarg);
  41 static nv50_ir::SVSemantic translateSysVal(uint sysval);
  42 static nv50_ir::CacheMode translateCacheMode(uint qualifier);
  43
  44 class Instruction
  45 {
  46 public:
  47    Instruction(const struct tgsi_full_instruction *inst) : insn(inst) { }
  48
  49    class SrcRegister
  50    {
  51    public:
  52       SrcRegister(const struct tgsi_full_src_register *src)
  53          : reg(src->Register),
  54            fsr(src)
  55       { }
  56
  57       SrcRegister(const struct tgsi_src_register& src) : reg(src), fsr(NULL) { }
  58
  59       SrcRegister(const struct tgsi_ind_register& ind)
  60          : reg(tgsi_util_get_src_from_ind(&ind)),
  61            fsr(NULL)
  62       { }
  63
  64       struct tgsi_src_register offsetToSrc(struct tgsi_texture_offset off)
  65       {
  66          struct tgsi_src_register reg;
  67          memset(&reg, 0, sizeof(reg));
  68          reg.Index = off.Index;
  69          reg.File = off.File;
  70          reg.SwizzleX = off.SwizzleX;
  71          reg.SwizzleY = off.SwizzleY;
  72          reg.SwizzleZ = off.SwizzleZ;
  73          return reg;
  74       }
  75
  76       SrcRegister(const struct tgsi_texture_offset& off) :
  77          reg(offsetToSrc(off)),
  78          fsr(NULL)
  79       { }
  80
  81       uint getFile() const { return reg.File; }
  82
  83       bool is2D() const { return reg.Dimension; }
  84
  85       bool isIndirect(int dim) const
  86       {
  87          return (dim && fsr) ? fsr->Dimension.Indirect : reg.Indirect;
  88       }
  89
  90       int getIndex(int dim) const
  91       {
  92          return (dim && fsr) ? fsr->Dimension.Index : reg.Index;
  93       }
  94
  95       int getSwizzle(int chan) const
  96       {
  97          return tgsi_util_get_src_register_swizzle(&reg, chan);
  98       }
  99
 100       int getArrayId() const
 101       {
 102          if (isIndirect(0))
 103             return fsr->Indirect.ArrayID;
 104          return 0;
 105       }
 106
 107       nv50_ir::Modifier getMod(int chan) const;
 108
 109       SrcRegister getIndirect(int dim) const
 110       {
 111          assert(fsr && isIndirect(dim));
 112          if (dim)
 113             return SrcRegister(fsr->DimIndirect);
 114          return SrcRegister(fsr->Indirect);
 115       }
 116
 117       uint32_t getValueU32(int c, const struct nv50_ir_prog_info *info) const
 118       {
 119          assert(reg.File == TGSI_FILE_IMMEDIATE);
 120          assert(!reg.Absolute);
 121          assert(!reg.Negate);
 122          return info->immd.data[reg.Index * 4 + getSwizzle(c)];
 123       }
 124
 125    private:
 126       const struct tgsi_src_register reg;
 127       const struct tgsi_full_src_register *fsr;
 128    };
 129
 130    class DstRegister
 131    {
 132    public:
 133       DstRegister(const struct tgsi_full_dst_register *dst)
 134          : reg(dst->Register),
 135            fdr(dst)
 136       { }
 137
 138       DstRegister(const struct tgsi_dst_register& dst) : reg(dst), fdr(NULL) { }
 139
 140       uint getFile() const { return reg.File; }
 141
 142       bool is2D() const { return reg.Dimension; }
 143
 144       bool isIndirect(int dim) const
 145       {
 146          return (dim && fdr) ? fdr->Dimension.Indirect : reg.Indirect;
 147       }
 148
 149       int getIndex(int dim) const
 150       {
 151          return (dim && fdr) ? fdr->Dimension.Dimension : reg.Index;
 152       }
 153
 154       unsigned int getMask() const { return reg.WriteMask; }
 155
 156       bool isMasked(int chan) const { return !(getMask() & (1 << chan)); }
 157
 158       SrcRegister getIndirect(int dim) const
 159       {
 160          assert(fdr && isIndirect(dim));
 161          if (dim)
 162             return SrcRegister(fdr->DimIndirect);
 163          return SrcRegister(fdr->Indirect);
 164       }
 165
 166       struct tgsi_full_src_register asSrc()
 167       {
 168          assert(fdr);
 169          return tgsi_full_src_register_from_dst(fdr);
 170       }
 171
 172       int getArrayId() const
 173       {
 174          if (isIndirect(0))
 175             return fdr->Indirect.ArrayID;
 176          return 0;
 177       }
 178
 179    private:
 180       const struct tgsi_dst_register reg;
 181       const struct tgsi_full_dst_register *fdr;
 182    };
 183
 184    inline uint getOpcode() const { return insn->Instruction.Opcode; }
 185
 186    unsigned int srcCount() const { return insn->Instruction.NumSrcRegs; }
 187    unsigned int dstCount() const { return insn->Instruction.NumDstRegs; }
 188
 189    // mask of used components of source s
 190    unsigned int srcMask(unsigned int s) const;
 191    unsigned int texOffsetMask() const;
 192
 193    SrcRegister getSrc(unsigned int s) const
 194    {
 195       assert(s < srcCount());
 196       return SrcRegister(&insn->Src[s]);
 197    }
 198
 199    DstRegister getDst(unsigned int d) const
 200    {
 201       assert(d < dstCount());
 202       return DstRegister(&insn->Dst[d]);
 203    }
 204
 205    SrcRegister getTexOffset(unsigned int i) const
 206    {
 207       assert(i < TGSI_FULL_MAX_TEX_OFFSETS);
 208       return SrcRegister(insn->TexOffsets[i]);
 209    }
 210
 211    unsigned int getNumTexOffsets() const { return insn->Texture.NumOffsets; }
 212
 213    bool checkDstSrcAliasing() const;
 214
 215    inline nv50_ir::operation getOP() const {
 216       return translateOpcode(getOpcode()); }
 217
 218    nv50_ir::DataType inferSrcType() const;
 219    nv50_ir::DataType inferDstType() const;
 220
 221    nv50_ir::CondCode getSetCond() const;
 222
 223    nv50_ir::TexInstruction::Target getTexture(const Source *, int s) const;
 224
 225    const nv50_ir::TexInstruction::ImgFormatDesc *getImageFormat() const {
 226       return nv50_ir::TexInstruction::translateImgFormat((enum pipe_format)insn->Memory.Format);
 227    }
 228
 229    nv50_ir::TexTarget getImageTarget() const {
 230       return translateTexture(insn->Memory.Texture);
 231    }
 232
 233    nv50_ir::CacheMode getCacheMode() const {
 234       if (!insn->Instruction.Memory)
 235          return nv50_ir::CACHE_CA;
 236       return translateCacheMode(insn->Memory.Qualifier);
 237    }
 238
 239    inline uint getLabel() { return insn->Label.Label; }
 240
 241    unsigned getSaturate() const { return insn->Instruction.Saturate; }
 242
 243    void print() const
 244    {
 245       tgsi_dump_instruction(insn, 1);
 246    }
 247
 248 private:
 249    const struct tgsi_full_instruction *insn;
 250 };
 251
 252 unsigned int Instruction::texOffsetMask() const
 253 {
 254    const struct tgsi_instruction_texture *tex = &insn->Texture;
 255    assert(insn->Instruction.Texture);
 256
 257    switch (tex->Texture) {
 258    case TGSI_TEXTURE_BUFFER:
 259    case TGSI_TEXTURE_1D:
 260    case TGSI_TEXTURE_SHADOW1D:
 261    case TGSI_TEXTURE_1D_ARRAY:
 262    case TGSI_TEXTURE_SHADOW1D_ARRAY:
 263       return 0x1;
 264    case TGSI_TEXTURE_2D:
 265    case TGSI_TEXTURE_SHADOW2D:
 266    case TGSI_TEXTURE_2D_ARRAY:
 267    case TGSI_TEXTURE_SHADOW2D_ARRAY:
 268    case TGSI_TEXTURE_RECT:
 269    case TGSI_TEXTURE_SHADOWRECT:
 270    case TGSI_TEXTURE_2D_MSAA:
 271    case TGSI_TEXTURE_2D_ARRAY_MSAA:
 272       return 0x3;
 273    case TGSI_TEXTURE_3D:
 274       return 0x7;
 275    default:
 276       assert(!"Unexpected texture target");
 277       return 0xf;
 278    }
 279 }
 280
 281 unsigned int Instruction::srcMask(unsigned int s) const
 282 {
 283    unsigned int mask = insn->Dst[0].Register.WriteMask;
 284
 285    switch (insn->Instruction.Opcode) {
 286    case TGSI_OPCODE_COS:
 287    case TGSI_OPCODE_SIN:
 288       return (mask & 0x8) | ((mask & 0x7) ? 0x1 : 0x0);
 289    case TGSI_OPCODE_DP2:
 290       return 0x3;
 291    case TGSI_OPCODE_DP3:
 292       return 0x7;
 293    case TGSI_OPCODE_DP4:
 294    case TGSI_OPCODE_KILL_IF: /* WriteMask ignored */
 295       return 0xf;
 296    case TGSI_OPCODE_DST:
 297       return mask & (s ? 0xa : 0x6);
 298    case TGSI_OPCODE_EX2:
 299    case TGSI_OPCODE_EXP:
 300    case TGSI_OPCODE_LG2:
 301    case TGSI_OPCODE_LOG:
 302    case TGSI_OPCODE_POW:
 303    case TGSI_OPCODE_RCP:
 304    case TGSI_OPCODE_RSQ:
 305       return 0x1;
 306    case TGSI_OPCODE_IF:
 307    case TGSI_OPCODE_UIF:
 308       return 0x1;
 309    case TGSI_OPCODE_LIT:
 310       return 0xb;
 311    case TGSI_OPCODE_TEX2:
 312    case TGSI_OPCODE_TXB2:
 313    case TGSI_OPCODE_TXL2:
 314       return (s == 0) ? 0xf : 0x3;
 315    case TGSI_OPCODE_TEX:
 316    case TGSI_OPCODE_TXB:
 317    case TGSI_OPCODE_TXD:
 318    case TGSI_OPCODE_TXL:
 319    case TGSI_OPCODE_TXP:
 320    case TGSI_OPCODE_TXF:
 321    case TGSI_OPCODE_TG4:
 322    case TGSI_OPCODE_TEX_LZ:
 323    case TGSI_OPCODE_TXF_LZ:
 324    case TGSI_OPCODE_LODQ:
 325    {
 326       const struct tgsi_instruction_texture *tex = &insn->Texture;
 327
 328       assert(insn->Instruction.Texture);
 329
 330       mask = 0x7;
 331       if (insn->Instruction.Opcode != TGSI_OPCODE_TEX &&
 332           insn->Instruction.Opcode != TGSI_OPCODE_TEX_LZ &&
 333           insn->Instruction.Opcode != TGSI_OPCODE_TXF_LZ &&
 334           insn->Instruction.Opcode != TGSI_OPCODE_TXD)
 335          mask |= 0x8; /* bias, lod or proj */
 336
 337       switch (tex->Texture) {
 338       case TGSI_TEXTURE_1D:
 339          mask &= 0x9;
 340          break;
 341       case TGSI_TEXTURE_SHADOW1D:
 342          mask &= 0xd;
 343          break;
 344       case TGSI_TEXTURE_1D_ARRAY:
 345       case TGSI_TEXTURE_2D:
 346       case TGSI_TEXTURE_RECT:
 347          mask &= 0xb;
 348          break;
 349       case TGSI_TEXTURE_CUBE_ARRAY:
 350       case TGSI_TEXTURE_SHADOW2D_ARRAY:
 351       case TGSI_TEXTURE_SHADOWCUBE:
 352       case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
 353          mask |= 0x8;
 354          break;
 355       default:
 356          break;
 357       }
 358    }
 359       return mask;
 360    case TGSI_OPCODE_TXQ:
 361       return 1;
 362    case TGSI_OPCODE_D2I:
 363    case TGSI_OPCODE_D2U:
 364    case TGSI_OPCODE_D2F:
 365    case TGSI_OPCODE_DSLT:
 366    case TGSI_OPCODE_DSGE:
 367    case TGSI_OPCODE_DSEQ:
 368    case TGSI_OPCODE_DSNE:
 369    case TGSI_OPCODE_U64SEQ:
 370    case TGSI_OPCODE_U64SNE:
 371    case TGSI_OPCODE_I64SLT:
 372    case TGSI_OPCODE_U64SLT:
 373    case TGSI_OPCODE_I64SGE:
 374    case TGSI_OPCODE_U64SGE:
 375    case TGSI_OPCODE_I642F:
 376    case TGSI_OPCODE_U642F:
 377       switch (util_bitcount(mask)) {
 378       case 1: return 0x3;
 379       case 2: return 0xf;
 380       default:
 381          assert(!"unexpected mask");
 382          return 0xf;
 383       }
 384    case TGSI_OPCODE_I2D:
 385    case TGSI_OPCODE_U2D:
 386    case TGSI_OPCODE_F2D: {
 387       unsigned int x = 0;
 388       if ((mask & 0x3) == 0x3)
 389          x |= 1;
 390       if ((mask & 0xc) == 0xc)
 391          x |= 2;
 392       return x;
 393    }
 394    case TGSI_OPCODE_PK2H:
 395       return 0x3;
 396    case TGSI_OPCODE_UP2H:
 397       return 0x1;
 398    default:
 399       break;
 400    }
 401
 402    return mask;
 403 }
 404
 405 nv50_ir::Modifier Instruction::SrcRegister::getMod(int chan) const
 406 {
 407    nv50_ir::Modifier m(0);
 408
 409    if (reg.Absolute)
 410       m = m | nv50_ir::Modifier(NV50_IR_MOD_ABS);
 411    if (reg.Negate)
 412       m = m | nv50_ir::Modifier(NV50_IR_MOD_NEG);
 413    return m;
 414 }
 415
 416 static nv50_ir::DataFile translateFile(uint file)
 417 {
 418    switch (file) {
 419    case TGSI_FILE_CONSTANT:        return nv50_ir::FILE_MEMORY_CONST;
 420    case TGSI_FILE_INPUT:           return nv50_ir::FILE_SHADER_INPUT;
 421    case TGSI_FILE_OUTPUT:          return nv50_ir::FILE_SHADER_OUTPUT;
 422    case TGSI_FILE_TEMPORARY:       return nv50_ir::FILE_GPR;
 423    case TGSI_FILE_ADDRESS:         return nv50_ir::FILE_ADDRESS;
 424    case TGSI_FILE_IMMEDIATE:       return nv50_ir::FILE_IMMEDIATE;
 425    case TGSI_FILE_SYSTEM_VALUE:    return nv50_ir::FILE_SYSTEM_VALUE;
 426    case TGSI_FILE_BUFFER:          return nv50_ir::FILE_MEMORY_BUFFER;
 427    case TGSI_FILE_IMAGE:           return nv50_ir::FILE_MEMORY_GLOBAL;
 428    case TGSI_FILE_MEMORY:          return nv50_ir::FILE_MEMORY_GLOBAL;
 429    case TGSI_FILE_SAMPLER:
 430    case TGSI_FILE_NULL:
 431    default:
 432       return nv50_ir::FILE_NULL;
 433    }
 434 }
 435
 436 static nv50_ir::SVSemantic translateSysVal(uint sysval)
 437 {
 438    switch (sysval) {
 439    case TGSI_SEMANTIC_FACE:       return nv50_ir::SV_FACE;
 440    case TGSI_SEMANTIC_PSIZE:      return nv50_ir::SV_POINT_SIZE;
 441    case TGSI_SEMANTIC_PRIMID:     return nv50_ir::SV_PRIMITIVE_ID;
 442    case TGSI_SEMANTIC_INSTANCEID: return nv50_ir::SV_INSTANCE_ID;
 443    case TGSI_SEMANTIC_VERTEXID:   return nv50_ir::SV_VERTEX_ID;
 444    case TGSI_SEMANTIC_GRID_SIZE:  return nv50_ir::SV_NCTAID;
 445    case TGSI_SEMANTIC_BLOCK_ID:   return nv50_ir::SV_CTAID;
 446    case TGSI_SEMANTIC_BLOCK_SIZE: return nv50_ir::SV_NTID;
 447    case TGSI_SEMANTIC_THREAD_ID:  return nv50_ir::SV_TID;
 448    case TGSI_SEMANTIC_SAMPLEID:   return nv50_ir::SV_SAMPLE_INDEX;
 449    case TGSI_SEMANTIC_SAMPLEPOS:  return nv50_ir::SV_SAMPLE_POS;
 450    case TGSI_SEMANTIC_SAMPLEMASK: return nv50_ir::SV_SAMPLE_MASK;
 451    case TGSI_SEMANTIC_INVOCATIONID: return nv50_ir::SV_INVOCATION_ID;
 452    case TGSI_SEMANTIC_TESSCOORD:  return nv50_ir::SV_TESS_COORD;
 453    case TGSI_SEMANTIC_TESSOUTER:  return nv50_ir::SV_TESS_OUTER;
 454    case TGSI_SEMANTIC_TESSINNER:  return nv50_ir::SV_TESS_INNER;
 455    case TGSI_SEMANTIC_VERTICESIN: return nv50_ir::SV_VERTEX_COUNT;
 456    case TGSI_SEMANTIC_HELPER_INVOCATION: return nv50_ir::SV_THREAD_KILL;
 457    case TGSI_SEMANTIC_BASEVERTEX: return nv50_ir::SV_BASEVERTEX;
 458    case TGSI_SEMANTIC_BASEINSTANCE: return nv50_ir::SV_BASEINSTANCE;
 459    case TGSI_SEMANTIC_DRAWID:     return nv50_ir::SV_DRAWID;
 460    case TGSI_SEMANTIC_WORK_DIM:   return nv50_ir::SV_WORK_DIM;
 461    case TGSI_SEMANTIC_SUBGROUP_INVOCATION: return nv50_ir::SV_LANEID;
 462    case TGSI_SEMANTIC_SUBGROUP_EQ_MASK: return nv50_ir::SV_LANEMASK_EQ;
 463    case TGSI_SEMANTIC_SUBGROUP_LT_MASK: return nv50_ir::SV_LANEMASK_LT;
 464    case TGSI_SEMANTIC_SUBGROUP_LE_MASK: return nv50_ir::SV_LANEMASK_LE;
 465    case TGSI_SEMANTIC_SUBGROUP_GT_MASK: return nv50_ir::SV_LANEMASK_GT;
 466    case TGSI_SEMANTIC_SUBGROUP_GE_MASK: return nv50_ir::SV_LANEMASK_GE;
 467    default:
 468       assert(0);
 469       return nv50_ir::SV_CLOCK;
 470    }
 471 }
 472
 473 #define NV50_IR_TEX_TARG_CASE(a, b) \
 474    case TGSI_TEXTURE_##a: return nv50_ir::TEX_TARGET_##b;
 475
 476 static nv50_ir::TexTarget translateTexture(uint tex)
 477 {
 478    switch (tex) {
 479    NV50_IR_TEX_TARG_CASE(1D, 1D);
 480    NV50_IR_TEX_TARG_CASE(2D, 2D);
 481    NV50_IR_TEX_TARG_CASE(2D_MSAA, 2D_MS);
 482    NV50_IR_TEX_TARG_CASE(3D, 3D);
 483    NV50_IR_TEX_TARG_CASE(CUBE, CUBE);
 484    NV50_IR_TEX_TARG_CASE(RECT, RECT);
 485    NV50_IR_TEX_TARG_CASE(1D_ARRAY, 1D_ARRAY);
 486    NV50_IR_TEX_TARG_CASE(2D_ARRAY, 2D_ARRAY);
 487    NV50_IR_TEX_TARG_CASE(2D_ARRAY_MSAA, 2D_MS_ARRAY);
 488    NV50_IR_TEX_TARG_CASE(CUBE_ARRAY, CUBE_ARRAY);
 489    NV50_IR_TEX_TARG_CASE(SHADOW1D, 1D_SHADOW);
 490    NV50_IR_TEX_TARG_CASE(SHADOW2D, 2D_SHADOW);
 491    NV50_IR_TEX_TARG_CASE(SHADOWCUBE, CUBE_SHADOW);
 492    NV50_IR_TEX_TARG_CASE(SHADOWRECT, RECT_SHADOW);
 493    NV50_IR_TEX_TARG_CASE(SHADOW1D_ARRAY, 1D_ARRAY_SHADOW);
 494    NV50_IR_TEX_TARG_CASE(SHADOW2D_ARRAY, 2D_ARRAY_SHADOW);
 495    NV50_IR_TEX_TARG_CASE(SHADOWCUBE_ARRAY, CUBE_ARRAY_SHADOW);
 496    NV50_IR_TEX_TARG_CASE(BUFFER, BUFFER);
 497
 498    case TGSI_TEXTURE_UNKNOWN:
 499    default:
 500       assert(!"invalid texture target");
 501       return nv50_ir::TEX_TARGET_2D;
 502    }
 503 }
 504
 505 static nv50_ir::CacheMode translateCacheMode(uint qualifier)
 506 {
 507    if (qualifier & TGSI_MEMORY_VOLATILE)
 508       return nv50_ir::CACHE_CV;
 509    if (qualifier & TGSI_MEMORY_COHERENT)
 510       return nv50_ir::CACHE_CG;
 511    return nv50_ir::CACHE_CA;
 512 }
 513
 514 nv50_ir::DataType Instruction::inferSrcType() const
 515 {
 516    switch (getOpcode()) {
 517    case TGSI_OPCODE_UIF:
 518    case TGSI_OPCODE_AND:
 519    case TGSI_OPCODE_OR:
 520    case TGSI_OPCODE_XOR:
 521    case TGSI_OPCODE_NOT:
 522    case TGSI_OPCODE_SHL:
 523    case TGSI_OPCODE_U2F:
 524    case TGSI_OPCODE_U2D:
 525    case TGSI_OPCODE_U2I64:
 526    case TGSI_OPCODE_UADD:
 527    case TGSI_OPCODE_UDIV:
 528    case TGSI_OPCODE_UMOD:
 529    case TGSI_OPCODE_UMAD:
 530    case TGSI_OPCODE_UMUL:
 531    case TGSI_OPCODE_UMUL_HI:
 532    case TGSI_OPCODE_UMAX:
 533    case TGSI_OPCODE_UMIN:
 534    case TGSI_OPCODE_USEQ:
 535    case TGSI_OPCODE_USGE:
 536    case TGSI_OPCODE_USLT:
 537    case TGSI_OPCODE_USNE:
 538    case TGSI_OPCODE_USHR:
 539    case TGSI_OPCODE_ATOMUADD:
 540    case TGSI_OPCODE_ATOMXCHG:
 541    case TGSI_OPCODE_ATOMCAS:
 542    case TGSI_OPCODE_ATOMAND:
 543    case TGSI_OPCODE_ATOMOR:
 544    case TGSI_OPCODE_ATOMXOR:
 545    case TGSI_OPCODE_ATOMUMIN:
 546    case TGSI_OPCODE_ATOMUMAX:
 547    case TGSI_OPCODE_ATOMDEC_WRAP:
 548    case TGSI_OPCODE_ATOMINC_WRAP:
 549    case TGSI_OPCODE_UBFE:
 550    case TGSI_OPCODE_UMSB:
 551    case TGSI_OPCODE_UP2H:
 552    case TGSI_OPCODE_VOTE_ALL:
 553    case TGSI_OPCODE_VOTE_ANY:
 554    case TGSI_OPCODE_VOTE_EQ:
 555       return nv50_ir::TYPE_U32;
 556    case TGSI_OPCODE_I2F:
 557    case TGSI_OPCODE_I2D:
 558    case TGSI_OPCODE_I2I64:
 559    case TGSI_OPCODE_IDIV:
 560    case TGSI_OPCODE_IMUL_HI:
 561    case TGSI_OPCODE_IMAX:
 562    case TGSI_OPCODE_IMIN:
 563    case TGSI_OPCODE_IABS:
 564    case TGSI_OPCODE_INEG:
 565    case TGSI_OPCODE_ISGE:
 566    case TGSI_OPCODE_ISHR:
 567    case TGSI_OPCODE_ISLT:
 568    case TGSI_OPCODE_ISSG:
 569    case TGSI_OPCODE_MOD:
 570    case TGSI_OPCODE_UARL:
 571    case TGSI_OPCODE_ATOMIMIN:
 572    case TGSI_OPCODE_ATOMIMAX:
 573    case TGSI_OPCODE_IBFE:
 574    case TGSI_OPCODE_IMSB:
 575       return nv50_ir::TYPE_S32;
 576    case TGSI_OPCODE_D2F:
 577    case TGSI_OPCODE_D2I:
 578    case TGSI_OPCODE_D2U:
 579    case TGSI_OPCODE_D2I64:
 580    case TGSI_OPCODE_D2U64:
 581    case TGSI_OPCODE_DABS:
 582    case TGSI_OPCODE_DNEG:
 583    case TGSI_OPCODE_DADD:
 584    case TGSI_OPCODE_DMUL:
 585    case TGSI_OPCODE_DDIV:
 586    case TGSI_OPCODE_DMAX:
 587    case TGSI_OPCODE_DMIN:
 588    case TGSI_OPCODE_DSLT:
 589    case TGSI_OPCODE_DSGE:
 590    case TGSI_OPCODE_DSEQ:
 591    case TGSI_OPCODE_DSNE:
 592    case TGSI_OPCODE_DRCP:
 593    case TGSI_OPCODE_DSQRT:
 594    case TGSI_OPCODE_DMAD:
 595    case TGSI_OPCODE_DFMA:
 596    case TGSI_OPCODE_DFRAC:
 597    case TGSI_OPCODE_DRSQ:
 598    case TGSI_OPCODE_DTRUNC:
 599    case TGSI_OPCODE_DCEIL:
 600    case TGSI_OPCODE_DFLR:
 601    case TGSI_OPCODE_DROUND:
 602       return nv50_ir::TYPE_F64;
 603    case TGSI_OPCODE_U64SEQ:
 604    case TGSI_OPCODE_U64SNE:
 605    case TGSI_OPCODE_U64SLT:
 606    case TGSI_OPCODE_U64SGE:
 607    case TGSI_OPCODE_U64MIN:
 608    case TGSI_OPCODE_U64MAX:
 609    case TGSI_OPCODE_U64ADD:
 610    case TGSI_OPCODE_U64MUL:
 611    case TGSI_OPCODE_U64SHL:
 612    case TGSI_OPCODE_U64SHR:
 613    case TGSI_OPCODE_U64DIV:
 614    case TGSI_OPCODE_U64MOD:
 615    case TGSI_OPCODE_U642F:
 616    case TGSI_OPCODE_U642D:
 617       return nv50_ir::TYPE_U64;
 618    case TGSI_OPCODE_I64ABS:
 619    case TGSI_OPCODE_I64SSG:
 620    case TGSI_OPCODE_I64NEG:
 621    case TGSI_OPCODE_I64SLT:
 622    case TGSI_OPCODE_I64SGE:
 623    case TGSI_OPCODE_I64MIN:
 624    case TGSI_OPCODE_I64MAX:
 625    case TGSI_OPCODE_I64SHR:
 626    case TGSI_OPCODE_I64DIV:
 627    case TGSI_OPCODE_I64MOD:
 628    case TGSI_OPCODE_I642F:
 629    case TGSI_OPCODE_I642D:
 630       return nv50_ir::TYPE_S64;
 631    default:
 632       return nv50_ir::TYPE_F32;
 633    }
 634 }
 635
 636 nv50_ir::DataType Instruction::inferDstType() const
 637 {
 638    switch (getOpcode()) {
 639    case TGSI_OPCODE_D2U:
 640    case TGSI_OPCODE_F2U: return nv50_ir::TYPE_U32;
 641    case TGSI_OPCODE_D2I:
 642    case TGSI_OPCODE_F2I: return nv50_ir::TYPE_S32;
 643    case TGSI_OPCODE_FSEQ:
 644    case TGSI_OPCODE_FSGE:
 645    case TGSI_OPCODE_FSLT:
 646    case TGSI_OPCODE_FSNE:
 647    case TGSI_OPCODE_DSEQ:
 648    case TGSI_OPCODE_DSGE:
 649    case TGSI_OPCODE_DSLT:
 650    case TGSI_OPCODE_DSNE:
 651    case TGSI_OPCODE_I64SLT:
 652    case TGSI_OPCODE_I64SGE:
 653    case TGSI_OPCODE_U64SEQ:
 654    case TGSI_OPCODE_U64SNE:
 655    case TGSI_OPCODE_U64SLT:
 656    case TGSI_OPCODE_U64SGE:
 657    case TGSI_OPCODE_PK2H:
 658       return nv50_ir::TYPE_U32;
 659    case TGSI_OPCODE_I2F:
 660    case TGSI_OPCODE_U2F:
 661    case TGSI_OPCODE_D2F:
 662    case TGSI_OPCODE_I642F:
 663    case TGSI_OPCODE_U642F:
 664    case TGSI_OPCODE_UP2H:
 665       return nv50_ir::TYPE_F32;
 666    case TGSI_OPCODE_I2D:
 667    case TGSI_OPCODE_U2D:
 668    case TGSI_OPCODE_F2D:
 669    case TGSI_OPCODE_I642D:
 670    case TGSI_OPCODE_U642D:
 671       return nv50_ir::TYPE_F64;
 672    case TGSI_OPCODE_I2I64:
 673    case TGSI_OPCODE_U2I64:
 674    case TGSI_OPCODE_F2I64:
 675    case TGSI_OPCODE_D2I64:
 676       return nv50_ir::TYPE_S64;
 677    case TGSI_OPCODE_F2U64:
 678    case TGSI_OPCODE_D2U64:
 679       return nv50_ir::TYPE_U64;
 680    default:
 681       return inferSrcType();
 682    }
 683 }
 684
 685 nv50_ir::CondCode Instruction::getSetCond() const
 686 {
 687    using namespace nv50_ir;
 688
 689    switch (getOpcode()) {
 690    case TGSI_OPCODE_SLT:
 691    case TGSI_OPCODE_ISLT:
 692    case TGSI_OPCODE_USLT:
 693    case TGSI_OPCODE_FSLT:
 694    case TGSI_OPCODE_DSLT:
 695    case TGSI_OPCODE_I64SLT:
 696    case TGSI_OPCODE_U64SLT:
 697       return CC_LT;
 698    case TGSI_OPCODE_SLE:
 699       return CC_LE;
 700    case TGSI_OPCODE_SGE:
 701    case TGSI_OPCODE_ISGE:
 702    case TGSI_OPCODE_USGE:
 703    case TGSI_OPCODE_FSGE:
 704    case TGSI_OPCODE_DSGE:
 705    case TGSI_OPCODE_I64SGE:
 706    case TGSI_OPCODE_U64SGE:
 707       return CC_GE;
 708    case TGSI_OPCODE_SGT:
 709       return CC_GT;
 710    case TGSI_OPCODE_SEQ:
 711    case TGSI_OPCODE_USEQ:
 712    case TGSI_OPCODE_FSEQ:
 713    case TGSI_OPCODE_DSEQ:
 714    case TGSI_OPCODE_U64SEQ:
 715       return CC_EQ;
 716    case TGSI_OPCODE_SNE:
 717    case TGSI_OPCODE_FSNE:
 718    case TGSI_OPCODE_DSNE:
 719    case TGSI_OPCODE_U64SNE:
 720       return CC_NEU;
 721    case TGSI_OPCODE_USNE:
 722       return CC_NE;
 723    default:
 724       return CC_ALWAYS;
 725    }
 726 }
 727
 728 #define NV50_IR_OPCODE_CASE(a, b) case TGSI_OPCODE_##a: return nv50_ir::OP_##b
 729
 730 static nv50_ir::operation translateOpcode(uint opcode)
 731 {
 732    switch (opcode) {
 733    NV50_IR_OPCODE_CASE(ARL, SHL);
 734    NV50_IR_OPCODE_CASE(MOV, MOV);
 735
 736    NV50_IR_OPCODE_CASE(RCP, RCP);
 737    NV50_IR_OPCODE_CASE(RSQ, RSQ);
 738    NV50_IR_OPCODE_CASE(SQRT, SQRT);
 739
 740    NV50_IR_OPCODE_CASE(MUL, MUL);
 741    NV50_IR_OPCODE_CASE(ADD, ADD);
 742
 743    NV50_IR_OPCODE_CASE(MIN, MIN);
 744    NV50_IR_OPCODE_CASE(MAX, MAX);
 745    NV50_IR_OPCODE_CASE(SLT, SET);
 746    NV50_IR_OPCODE_CASE(SGE, SET);
 747    NV50_IR_OPCODE_CASE(MAD, MAD);
 748    NV50_IR_OPCODE_CASE(FMA, FMA);
 749
 750    NV50_IR_OPCODE_CASE(FLR, FLOOR);
 751    NV50_IR_OPCODE_CASE(ROUND, CVT);
 752    NV50_IR_OPCODE_CASE(EX2, EX2);
 753    NV50_IR_OPCODE_CASE(LG2, LG2);
 754    NV50_IR_OPCODE_CASE(POW, POW);
 755
 756    NV50_IR_OPCODE_CASE(COS, COS);
 757    NV50_IR_OPCODE_CASE(DDX, DFDX);
 758    NV50_IR_OPCODE_CASE(DDX_FINE, DFDX);
 759    NV50_IR_OPCODE_CASE(DDY, DFDY);
 760    NV50_IR_OPCODE_CASE(DDY_FINE, DFDY);
 761    NV50_IR_OPCODE_CASE(KILL, DISCARD);
 762    NV50_IR_OPCODE_CASE(DEMOTE, DISCARD);
 763
 764    NV50_IR_OPCODE_CASE(SEQ, SET);
 765    NV50_IR_OPCODE_CASE(SGT, SET);
 766    NV50_IR_OPCODE_CASE(SIN, SIN);
 767    NV50_IR_OPCODE_CASE(SLE, SET);
 768    NV50_IR_OPCODE_CASE(SNE, SET);
 769    NV50_IR_OPCODE_CASE(TEX, TEX);
 770    NV50_IR_OPCODE_CASE(TXD, TXD);
 771    NV50_IR_OPCODE_CASE(TXP, TEX);
 772
 773    NV50_IR_OPCODE_CASE(CAL, CALL);
 774    NV50_IR_OPCODE_CASE(RET, RET);
 775    NV50_IR_OPCODE_CASE(CMP, SLCT);
 776
 777    NV50_IR_OPCODE_CASE(TXB, TXB);
 778
 779    NV50_IR_OPCODE_CASE(DIV, DIV);
 780
 781    NV50_IR_OPCODE_CASE(TXL, TXL);
 782    NV50_IR_OPCODE_CASE(TEX_LZ, TXL);
 783
 784    NV50_IR_OPCODE_CASE(CEIL, CEIL);
 785    NV50_IR_OPCODE_CASE(I2F, CVT);
 786    NV50_IR_OPCODE_CASE(NOT, NOT);
 787    NV50_IR_OPCODE_CASE(TRUNC, TRUNC);
 788    NV50_IR_OPCODE_CASE(SHL, SHL);
 789
 790    NV50_IR_OPCODE_CASE(AND, AND);
 791    NV50_IR_OPCODE_CASE(OR, OR);
 792    NV50_IR_OPCODE_CASE(MOD, MOD);
 793    NV50_IR_OPCODE_CASE(XOR, XOR);
 794    NV50_IR_OPCODE_CASE(TXF, TXF);
 795    NV50_IR_OPCODE_CASE(TXF_LZ, TXF);
 796    NV50_IR_OPCODE_CASE(TXQ, TXQ);
 797    NV50_IR_OPCODE_CASE(TXQS, TXQ);
 798    NV50_IR_OPCODE_CASE(TG4, TXG);
 799    NV50_IR_OPCODE_CASE(LODQ, TXLQ);
 800
 801    NV50_IR_OPCODE_CASE(EMIT, EMIT);
 802    NV50_IR_OPCODE_CASE(ENDPRIM, RESTART);
 803
 804    NV50_IR_OPCODE_CASE(KILL_IF, DISCARD);
 805
 806    NV50_IR_OPCODE_CASE(F2I, CVT);
 807    NV50_IR_OPCODE_CASE(FSEQ, SET);
 808    NV50_IR_OPCODE_CASE(FSGE, SET);
 809    NV50_IR_OPCODE_CASE(FSLT, SET);
 810    NV50_IR_OPCODE_CASE(FSNE, SET);
 811    NV50_IR_OPCODE_CASE(IDIV, DIV);
 812    NV50_IR_OPCODE_CASE(IMAX, MAX);
 813    NV50_IR_OPCODE_CASE(IMIN, MIN);
 814    NV50_IR_OPCODE_CASE(IABS, ABS);
 815    NV50_IR_OPCODE_CASE(INEG, NEG);
 816    NV50_IR_OPCODE_CASE(ISGE, SET);
 817    NV50_IR_OPCODE_CASE(ISHR, SHR);
 818    NV50_IR_OPCODE_CASE(ISLT, SET);
 819    NV50_IR_OPCODE_CASE(F2U, CVT);
 820    NV50_IR_OPCODE_CASE(U2F, CVT);
 821    NV50_IR_OPCODE_CASE(UADD, ADD);
 822    NV50_IR_OPCODE_CASE(UDIV, DIV);
 823    NV50_IR_OPCODE_CASE(UMAD, MAD);
 824    NV50_IR_OPCODE_CASE(UMAX, MAX);
 825    NV50_IR_OPCODE_CASE(UMIN, MIN);
 826    NV50_IR_OPCODE_CASE(UMOD, MOD);
 827    NV50_IR_OPCODE_CASE(UMUL, MUL);
 828    NV50_IR_OPCODE_CASE(USEQ, SET);
 829    NV50_IR_OPCODE_CASE(USGE, SET);
 830    NV50_IR_OPCODE_CASE(USHR, SHR);
 831    NV50_IR_OPCODE_CASE(USLT, SET);
 832    NV50_IR_OPCODE_CASE(USNE, SET);
 833
 834    NV50_IR_OPCODE_CASE(DABS, ABS);
 835    NV50_IR_OPCODE_CASE(DNEG, NEG);
 836    NV50_IR_OPCODE_CASE(DADD, ADD);
 837    NV50_IR_OPCODE_CASE(DMUL, MUL);
 838    NV50_IR_OPCODE_CASE(DDIV, DIV);
 839    NV50_IR_OPCODE_CASE(DMAX, MAX);
 840    NV50_IR_OPCODE_CASE(DMIN, MIN);
 841    NV50_IR_OPCODE_CASE(DSLT, SET);
 842    NV50_IR_OPCODE_CASE(DSGE, SET);
 843    NV50_IR_OPCODE_CASE(DSEQ, SET);
 844    NV50_IR_OPCODE_CASE(DSNE, SET);
 845    NV50_IR_OPCODE_CASE(DRCP, RCP);
 846    NV50_IR_OPCODE_CASE(DSQRT, SQRT);
 847    NV50_IR_OPCODE_CASE(DMAD, MAD);
 848    NV50_IR_OPCODE_CASE(DFMA, FMA);
 849    NV50_IR_OPCODE_CASE(D2I, CVT);
 850    NV50_IR_OPCODE_CASE(D2U, CVT);
 851    NV50_IR_OPCODE_CASE(I2D, CVT);
 852    NV50_IR_OPCODE_CASE(U2D, CVT);
 853    NV50_IR_OPCODE_CASE(DRSQ, RSQ);
 854    NV50_IR_OPCODE_CASE(DTRUNC, TRUNC);
 855    NV50_IR_OPCODE_CASE(DCEIL, CEIL);
 856    NV50_IR_OPCODE_CASE(DFLR, FLOOR);
 857    NV50_IR_OPCODE_CASE(DROUND, CVT);
 858
 859    NV50_IR_OPCODE_CASE(U64SEQ, SET);
 860    NV50_IR_OPCODE_CASE(U64SNE, SET);
 861    NV50_IR_OPCODE_CASE(U64SLT, SET);
 862    NV50_IR_OPCODE_CASE(U64SGE, SET);
 863    NV50_IR_OPCODE_CASE(I64SLT, SET);
 864    NV50_IR_OPCODE_CASE(I64SGE, SET);
 865    NV50_IR_OPCODE_CASE(I2I64, CVT);
 866    NV50_IR_OPCODE_CASE(U2I64, CVT);
 867    NV50_IR_OPCODE_CASE(F2I64, CVT);
 868    NV50_IR_OPCODE_CASE(F2U64, CVT);
 869    NV50_IR_OPCODE_CASE(D2I64, CVT);
 870    NV50_IR_OPCODE_CASE(D2U64, CVT);
 871    NV50_IR_OPCODE_CASE(I642F, CVT);
 872    NV50_IR_OPCODE_CASE(U642F, CVT);
 873    NV50_IR_OPCODE_CASE(I642D, CVT);
 874    NV50_IR_OPCODE_CASE(U642D, CVT);
 875
 876    NV50_IR_OPCODE_CASE(I64MIN, MIN);
 877    NV50_IR_OPCODE_CASE(U64MIN, MIN);
 878    NV50_IR_OPCODE_CASE(I64MAX, MAX);
 879    NV50_IR_OPCODE_CASE(U64MAX, MAX);
 880    NV50_IR_OPCODE_CASE(I64ABS, ABS);
 881    NV50_IR_OPCODE_CASE(I64NEG, NEG);
 882    NV50_IR_OPCODE_CASE(U64ADD, ADD);
 883    NV50_IR_OPCODE_CASE(U64MUL, MUL);
 884    NV50_IR_OPCODE_CASE(U64SHL, SHL);
 885    NV50_IR_OPCODE_CASE(I64SHR, SHR);
 886    NV50_IR_OPCODE_CASE(U64SHR, SHR);
 887
 888    NV50_IR_OPCODE_CASE(IMUL_HI, MUL);
 889    NV50_IR_OPCODE_CASE(UMUL_HI, MUL);
 890
 891    NV50_IR_OPCODE_CASE(SAMPLE, TEX);
 892    NV50_IR_OPCODE_CASE(SAMPLE_B, TXB);
 893    NV50_IR_OPCODE_CASE(SAMPLE_C, TEX);
 894    NV50_IR_OPCODE_CASE(SAMPLE_C_LZ, TEX);
 895    NV50_IR_OPCODE_CASE(SAMPLE_D, TXD);
 896    NV50_IR_OPCODE_CASE(SAMPLE_L, TXL);
 897    NV50_IR_OPCODE_CASE(SAMPLE_I, TXF);
 898    NV50_IR_OPCODE_CASE(SAMPLE_I_MS, TXF);
 899    NV50_IR_OPCODE_CASE(GATHER4, TXG);
 900    NV50_IR_OPCODE_CASE(SVIEWINFO, TXQ);
 901
 902    NV50_IR_OPCODE_CASE(ATOMUADD, ATOM);
 903    NV50_IR_OPCODE_CASE(ATOMXCHG, ATOM);
 904    NV50_IR_OPCODE_CASE(ATOMCAS, ATOM);
 905    NV50_IR_OPCODE_CASE(ATOMAND, ATOM);
 906    NV50_IR_OPCODE_CASE(ATOMOR, ATOM);
 907    NV50_IR_OPCODE_CASE(ATOMXOR, ATOM);
 908    NV50_IR_OPCODE_CASE(ATOMUMIN, ATOM);
 909    NV50_IR_OPCODE_CASE(ATOMUMAX, ATOM);
 910    NV50_IR_OPCODE_CASE(ATOMIMIN, ATOM);
 911    NV50_IR_OPCODE_CASE(ATOMIMAX, ATOM);
 912    NV50_IR_OPCODE_CASE(ATOMFADD, ATOM);
 913    NV50_IR_OPCODE_CASE(ATOMDEC_WRAP, ATOM);
 914    NV50_IR_OPCODE_CASE(ATOMINC_WRAP, ATOM);
 915
 916    NV50_IR_OPCODE_CASE(TEX2, TEX);
 917    NV50_IR_OPCODE_CASE(TXB2, TXB);
 918    NV50_IR_OPCODE_CASE(TXL2, TXL);
 919
 920    NV50_IR_OPCODE_CASE(IBFE, EXTBF);
 921    NV50_IR_OPCODE_CASE(UBFE, EXTBF);
 922    NV50_IR_OPCODE_CASE(BFI, INSBF);
 923    NV50_IR_OPCODE_CASE(BREV, EXTBF);
 924    NV50_IR_OPCODE_CASE(POPC, POPCNT);
 925    NV50_IR_OPCODE_CASE(LSB, BFIND);
 926    NV50_IR_OPCODE_CASE(IMSB, BFIND);
 927    NV50_IR_OPCODE_CASE(UMSB, BFIND);
 928
 929    NV50_IR_OPCODE_CASE(VOTE_ALL, VOTE);
 930    NV50_IR_OPCODE_CASE(VOTE_ANY, VOTE);
 931    NV50_IR_OPCODE_CASE(VOTE_EQ, VOTE);
 932
 933    NV50_IR_OPCODE_CASE(BALLOT, VOTE);
 934    NV50_IR_OPCODE_CASE(READ_INVOC, SHFL);
 935    NV50_IR_OPCODE_CASE(READ_FIRST, SHFL);
 936
 937    NV50_IR_OPCODE_CASE(END, EXIT);
 938
 939    default:
 940       return nv50_ir::OP_NOP;
 941    }
 942 }
 943
 944 static uint16_t opcodeToSubOp(uint opcode)
 945 {
 946    switch (opcode) {
 947    case TGSI_OPCODE_ATOMUADD: return NV50_IR_SUBOP_ATOM_ADD;
 948    case TGSI_OPCODE_ATOMXCHG: return NV50_IR_SUBOP_ATOM_EXCH;
 949    case TGSI_OPCODE_ATOMCAS:  return NV50_IR_SUBOP_ATOM_CAS;
 950    case TGSI_OPCODE_ATOMAND:  return NV50_IR_SUBOP_ATOM_AND;
 951    case TGSI_OPCODE_ATOMOR:   return NV50_IR_SUBOP_ATOM_OR;
 952    case TGSI_OPCODE_ATOMXOR:  return NV50_IR_SUBOP_ATOM_XOR;
 953    case TGSI_OPCODE_ATOMUMIN: return NV50_IR_SUBOP_ATOM_MIN;
 954    case TGSI_OPCODE_ATOMIMIN: return NV50_IR_SUBOP_ATOM_MIN;
 955    case TGSI_OPCODE_ATOMUMAX: return NV50_IR_SUBOP_ATOM_MAX;
 956    case TGSI_OPCODE_ATOMIMAX: return NV50_IR_SUBOP_ATOM_MAX;
 957    case TGSI_OPCODE_ATOMFADD: return NV50_IR_SUBOP_ATOM_ADD;
 958    case TGSI_OPCODE_ATOMDEC_WRAP: return NV50_IR_SUBOP_ATOM_DEC;
 959    case TGSI_OPCODE_ATOMINC_WRAP: return NV50_IR_SUBOP_ATOM_INC;
 960    case TGSI_OPCODE_IMUL_HI:
 961    case TGSI_OPCODE_UMUL_HI:
 962       return NV50_IR_SUBOP_MUL_HIGH;
 963    case TGSI_OPCODE_VOTE_ALL: return NV50_IR_SUBOP_VOTE_ALL;
 964    case TGSI_OPCODE_VOTE_ANY: return NV50_IR_SUBOP_VOTE_ANY;
 965    case TGSI_OPCODE_VOTE_EQ: return NV50_IR_SUBOP_VOTE_UNI;
 966    default:
 967       return 0;
 968    }
 969 }
 970
 971 bool Instruction::checkDstSrcAliasing() const
 972 {
 973    if (insn->Dst[0].Register.Indirect) // no danger if indirect, using memory
 974       return false;
 975
 976    for (int s = 0; s < TGSI_FULL_MAX_SRC_REGISTERS; ++s) {
 977       if (insn->Src[s].Register.File == TGSI_FILE_NULL)
 978          break;
 979       if (insn->Src[s].Register.File == insn->Dst[0].Register.File &&
 980           insn->Src[s].Register.Index == insn->Dst[0].Register.Index)
 981          return true;
 982    }
 983    return false;
 984 }
 985
 986 class Source
 987 {
 988 public:
 989    Source(struct nv50_ir_prog_info *);
 990    ~Source();
 991
 992 public:
 993    bool scanSource();
 994    unsigned fileSize(unsigned file) const { return scan.file_max[file] + 1; }
 995
 996 public:
 997    struct tgsi_shader_info scan;
 998    struct tgsi_full_instruction *insns;
 999    const struct tgsi_token *tokens;
1000    struct nv50_ir_prog_info *info;
1001
1002    nv50_ir::DynArray tempArrays;
1003    nv50_ir::DynArray immdArrays;
1004
1005    typedef nv50_ir::BuildUtil::Location Location;
1006    // these registers are per-subroutine, cannot be used for parameter passing
1007    std::set<Location> locals;
1008
1009    std::set<int> indirectTempArrays;
1010    std::map<int, int> indirectTempOffsets;
1011    std::map<int, std::pair<int, int> > tempArrayInfo;
1012    std::vector<int> tempArrayId;
1013
1014    int clipVertexOutput;
1015
1016    struct TextureView {
1017       uint8_t target; // TGSI_TEXTURE_*
1018    };
1019    std::vector<TextureView> textureViews;
1020
1021    /*
1022    struct Resource {
1023       uint8_t target; // TGSI_TEXTURE_*
1024       bool raw;
1025       uint8_t slot; // $surface index
1026    };
1027    std::vector<Resource> resources;
1028    */
1029
1030    struct MemoryFile {
1031       uint8_t mem_type; // TGSI_MEMORY_TYPE_*
1032    };
1033    std::vector<MemoryFile> memoryFiles;
1034
1035    std::vector<bool> bufferAtomics;
1036
1037 private:
1038    int inferSysValDirection(unsigned sn) const;
1039    bool scanDeclaration(const struct tgsi_full_declaration *);
1040    bool scanInstruction(const struct tgsi_full_instruction *);
1041    void scanInstructionSrc(const Instruction& insn,
1042                            const Instruction::SrcRegister& src,
1043                            unsigned mask);
1044    void scanProperty(const struct tgsi_full_property *);
1045    void scanImmediate(const struct tgsi_full_immediate *);
1046
1047    inline bool isEdgeFlagPassthrough(const Instruction&) const;
1048 };
1049
1050 Source::Source(struct nv50_ir_prog_info *prog) : info(prog)
1051 {
1052    tokens = (const struct tgsi_token *)info->bin.source;
1053
1054    if (prog->dbgFlags & NV50_IR_DEBUG_BASIC)
1055       tgsi_dump(tokens, 0);
1056 }
1057
1058 Source::~Source()
1059 {
1060    if (insns)
1061       FREE(insns);
1062
1063    if (info->immd.data)
1064       FREE(info->immd.data);
1065    if (info->immd.type)
1066       FREE(info->immd.type);
1067 }
1068
1069 bool Source::scanSource()
1070 {
1071    unsigned insnCount = 0;
1072    struct tgsi_parse_context parse;
1073
1074    tgsi_scan_shader(tokens, &scan);
1075
1076    insns = (struct tgsi_full_instruction *)MALLOC(scan.num_instructions *
1077                                                   sizeof(insns[0]));
1078    if (!insns)
1079       return false;
1080
1081    clipVertexOutput = -1;
1082
1083    textureViews.resize(scan.file_max[TGSI_FILE_SAMPLER_VIEW] + 1);
1084    //resources.resize(scan.file_max[TGSI_FILE_RESOURCE] + 1);
1085    tempArrayId.resize(scan.file_max[TGSI_FILE_TEMPORARY] + 1);
1086    memoryFiles.resize(scan.file_max[TGSI_FILE_MEMORY] + 1);
1087    bufferAtomics.resize(scan.file_max[TGSI_FILE_BUFFER] + 1);
1088
1089    info->immd.bufSize = 0;
1090
1091    info->numInputs = scan.file_max[TGSI_FILE_INPUT] + 1;
1092    info->numOutputs = scan.file_max[TGSI_FILE_OUTPUT] + 1;
1093    info->numSysVals = scan.file_max[TGSI_FILE_SYSTEM_VALUE] + 1;
1094
1095    if (info->type == PIPE_SHADER_FRAGMENT) {
1096       info->prop.fp.writesDepth = scan.writes_z;
1097       info->prop.fp.usesDiscard = scan.uses_kill || info->io.alphaRefBase;
1098    } else
1099    if (info->type == PIPE_SHADER_GEOMETRY) {
1100       info->prop.gp.instanceCount = 1; // default value
1101    }
1102
1103    info->io.viewportId = -1;
1104
1105    info->immd.data = (uint32_t *)MALLOC(scan.immediate_count * 16);
1106    info->immd.type = (ubyte *)MALLOC(scan.immediate_count * sizeof(ubyte));
1107
1108    tgsi_parse_init(&parse, tokens);
1109    while (!tgsi_parse_end_of_tokens(&parse)) {
1110       tgsi_parse_token(&parse);
1111
1112       switch (parse.FullToken.Token.Type) {
1113       case TGSI_TOKEN_TYPE_IMMEDIATE:
1114          scanImmediate(&parse.FullToken.FullImmediate);
1115          break;
1116       case TGSI_TOKEN_TYPE_DECLARATION:
1117          scanDeclaration(&parse.FullToken.FullDeclaration);
1118          break;
1119       case TGSI_TOKEN_TYPE_INSTRUCTION:
1120          insns[insnCount++] = parse.FullToken.FullInstruction;
1121          scanInstruction(&parse.FullToken.FullInstruction);
1122          break;
1123       case TGSI_TOKEN_TYPE_PROPERTY:
1124          scanProperty(&parse.FullToken.FullProperty);
1125          break;
1126       default:
1127          INFO("unknown TGSI token type: %d\n", parse.FullToken.Token.Type);
1128          break;
1129       }
1130    }
1131    tgsi_parse_free(&parse);
1132
1133    if (indirectTempArrays.size()) {
1134       int tempBase = 0;
1135       for (std::set<int>::const_iterator it = indirectTempArrays.begin();
1136            it != indirectTempArrays.end(); ++it) {
1137          std::pair<int, int>& info = tempArrayInfo[*it];
1138          indirectTempOffsets.insert(std::make_pair(*it, tempBase - info.first));
1139          tempBase += info.second;
1140       }
1141       info->bin.tlsSpace += tempBase * 16;
1142    }
1143
1144    if (info->io.genUserClip > 0) {
1145       info->io.clipDistances = info->io.genUserClip;
1146
1147       const unsigned int nOut = (info->io.genUserClip + 3) / 4;
1148
1149       for (unsigned int n = 0; n < nOut; ++n) {
1150          unsigned int i = info->numOutputs++;
1151          info->out[i].id = i;
1152          info->out[i].sn = TGSI_SEMANTIC_CLIPDIST;
1153          info->out[i].si = n;
1154          info->out[i].mask = ((1 << info->io.clipDistances) - 1) >> (n * 4);
1155       }
1156    }
1157
1158    return info->assignSlots(info) == 0;
1159 }
1160
1161 void Source::scanProperty(const struct tgsi_full_property *prop)
1162 {
1163    switch (prop->Property.PropertyName) {
1164    case TGSI_PROPERTY_GS_OUTPUT_PRIM:
1165       info->prop.gp.outputPrim = prop->u[0].Data;
1166       break;
1167    case TGSI_PROPERTY_GS_INPUT_PRIM:
1168       info->prop.gp.inputPrim = prop->u[0].Data;
1169       break;
1170    case TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES:
1171       info->prop.gp.maxVertices = prop->u[0].Data;
1172       break;
1173    case TGSI_PROPERTY_GS_INVOCATIONS:
1174       info->prop.gp.instanceCount = prop->u[0].Data;
1175       break;
1176    case TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS:
1177       info->prop.fp.separateFragData = true;
1178       break;
1179    case TGSI_PROPERTY_FS_COORD_ORIGIN:
1180    case TGSI_PROPERTY_FS_COORD_PIXEL_CENTER:
1181    case TGSI_PROPERTY_FS_DEPTH_LAYOUT:
1182       // we don't care
1183       break;
1184    case TGSI_PROPERTY_VS_PROHIBIT_UCPS:
1185       info->io.genUserClip = -1;
1186       break;
1187    case TGSI_PROPERTY_TCS_VERTICES_OUT:
1188       info->prop.tp.outputPatchSize = prop->u[0].Data;
1189       break;
1190    case TGSI_PROPERTY_TES_PRIM_MODE:
1191       info->prop.tp.domain = prop->u[0].Data;
1192       break;
1193    case TGSI_PROPERTY_TES_SPACING:
1194       info->prop.tp.partitioning = prop->u[0].Data;
1195       break;
1196    case TGSI_PROPERTY_TES_VERTEX_ORDER_CW:
1197       info->prop.tp.winding = prop->u[0].Data;
1198       break;
1199    case TGSI_PROPERTY_TES_POINT_MODE:
1200       if (prop->u[0].Data)
1201          info->prop.tp.outputPrim = PIPE_PRIM_POINTS;
1202       else
1203          info->prop.tp.outputPrim = PIPE_PRIM_TRIANGLES; /* anything but points */
1204       break;
1205    case TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH:
1206       info->prop.cp.numThreads[0] = prop->u[0].Data;
1207       break;
1208    case TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT:
1209       info->prop.cp.numThreads[1] = prop->u[0].Data;
1210       break;
1211    case TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH:
1212       info->prop.cp.numThreads[2] = prop->u[0].Data;
1213       break;
1214    case TGSI_PROPERTY_NUM_CLIPDIST_ENABLED:
1215       info->io.clipDistances = prop->u[0].Data;
1216       break;
1217    case TGSI_PROPERTY_NUM_CULLDIST_ENABLED:
1218       info->io.cullDistances = prop->u[0].Data;
1219       break;
1220    case TGSI_PROPERTY_NEXT_SHADER:
1221       /* Do not need to know the next shader stage. */
1222       break;
1223    case TGSI_PROPERTY_FS_EARLY_DEPTH_STENCIL:
1224       info->prop.fp.earlyFragTests = prop->u[0].Data;
1225       break;
1226    case TGSI_PROPERTY_FS_POST_DEPTH_COVERAGE:
1227       info->prop.fp.postDepthCoverage = prop->u[0].Data;
1228       break;
1229    case TGSI_PROPERTY_MUL_ZERO_WINS:
1230       info->io.mul_zero_wins = prop->u[0].Data;
1231       break;
1232    default:
1233       INFO("unhandled TGSI property %d\n", prop->Property.PropertyName);
1234       break;
1235    }
1236 }
1237
1238 void Source::scanImmediate(const struct tgsi_full_immediate *imm)
1239 {
1240    const unsigned n = info->immd.count++;
1241
1242    assert(n < scan.immediate_count);
1243
1244    for (int c = 0; c < 4; ++c)
1245       info->immd.data[n * 4 + c] = imm->u[c].Uint;
1246
1247    info->immd.type[n] = imm->Immediate.DataType;
1248 }
1249
1250 int Source::inferSysValDirection(unsigned sn) const
1251 {
1252    switch (sn) {
1253    case TGSI_SEMANTIC_INSTANCEID:
1254    case TGSI_SEMANTIC_VERTEXID:
1255       return 1;
1256    case TGSI_SEMANTIC_LAYER:
1257 #if 0
1258    case TGSI_SEMANTIC_VIEWPORTINDEX:
1259       return 0;
1260 #endif
1261    case TGSI_SEMANTIC_PRIMID:
1262       return (info->type == PIPE_SHADER_FRAGMENT) ? 1 : 0;
1263    default:
1264       return 0;
1265    }
1266 }
1267
1268 bool Source::scanDeclaration(const struct tgsi_full_declaration *decl)
1269 {
1270    unsigned i, c;
1271    unsigned sn = TGSI_SEMANTIC_GENERIC;
1272    unsigned si = 0;
1273    const unsigned first = decl->Range.First, last = decl->Range.Last;
1274    const int arrayId = decl->Array.ArrayID;
1275
1276    if (decl->Declaration.Semantic) {
1277       sn = decl->Semantic.Name;
1278       si = decl->Semantic.Index;
1279    }
1280
1281    if (decl->Declaration.Local || decl->Declaration.File == TGSI_FILE_ADDRESS) {
1282       for (i = first; i <= last; ++i) {
1283          for (c = 0; c < 4; ++c) {
1284             locals.insert(
1285                Location(decl->Declaration.File, decl->Dim.Index2D, i, c));
1286          }
1287       }
1288    }
1289
1290    switch (decl->Declaration.File) {
1291    case TGSI_FILE_INPUT:
1292       if (info->type == PIPE_SHADER_VERTEX) {
1293          // all vertex attributes are equal
1294          for (i = first; i <= last; ++i) {
1295             info->in[i].sn = TGSI_SEMANTIC_GENERIC;
1296             info->in[i].si = i;
1297          }
1298       } else {
1299          for (i = first; i <= last; ++i, ++si) {
1300             info->in[i].id = i;
1301             info->in[i].sn = sn;
1302             info->in[i].si = si;
1303             if (info->type == PIPE_SHADER_FRAGMENT) {
1304                // translate interpolation mode
1305                switch (decl->Interp.Interpolate) {
1306                case TGSI_INTERPOLATE_CONSTANT:
1307                   info->in[i].flat = 1;
1308                   break;
1309                case TGSI_INTERPOLATE_COLOR:
1310                   info->in[i].sc = 1;
1311                   break;
1312                case TGSI_INTERPOLATE_LINEAR:
1313                   info->in[i].linear = 1;
1314                   break;
1315                default:
1316                   break;
1317                }
1318                if (decl->Interp.Location)
1319                   info->in[i].centroid = 1;
1320             }
1321
1322             if (sn == TGSI_SEMANTIC_PATCH)
1323                info->in[i].patch = 1;
1324             if (sn == TGSI_SEMANTIC_PATCH)
1325                info->numPatchConstants = MAX2(info->numPatchConstants, si + 1);
1326          }
1327       }
1328       break;
1329    case TGSI_FILE_OUTPUT:
1330       for (i = first; i <= last; ++i, ++si) {
1331          switch (sn) {
1332          case TGSI_SEMANTIC_POSITION:
1333             if (info->type == PIPE_SHADER_FRAGMENT)
1334                info->io.fragDepth = i;
1335             else
1336             if (clipVertexOutput < 0)
1337                clipVertexOutput = i;
1338             break;
1339          case TGSI_SEMANTIC_COLOR:
1340             if (info->type == PIPE_SHADER_FRAGMENT)
1341                info->prop.fp.numColourResults++;
1342             break;
1343          case TGSI_SEMANTIC_EDGEFLAG:
1344             info->io.edgeFlagOut = i;
1345             break;
1346          case TGSI_SEMANTIC_CLIPVERTEX:
1347             clipVertexOutput = i;
1348             break;
1349          case TGSI_SEMANTIC_CLIPDIST:
1350             info->io.genUserClip = -1;
1351             break;
1352          case TGSI_SEMANTIC_SAMPLEMASK:
1353             info->io.sampleMask = i;
1354             break;
1355          case TGSI_SEMANTIC_VIEWPORT_INDEX:
1356             info->io.viewportId = i;
1357             break;
1358          case TGSI_SEMANTIC_PATCH:
1359             info->numPatchConstants = MAX2(info->numPatchConstants, si + 1);
1360             /* fallthrough */
1361          case TGSI_SEMANTIC_TESSOUTER:
1362          case TGSI_SEMANTIC_TESSINNER:
1363             info->out[i].patch = 1;
1364             break;
1365          default:
1366             break;
1367          }
1368          info->out[i].id = i;
1369          info->out[i].sn = sn;
1370          info->out[i].si = si;
1371       }
1372       break;
1373    case TGSI_FILE_SYSTEM_VALUE:
1374       switch (sn) {
1375       case TGSI_SEMANTIC_INSTANCEID:
1376          info->io.instanceId = first;
1377          break;
1378       case TGSI_SEMANTIC_VERTEXID:
1379          info->io.vertexId = first;
1380          break;
1381       case TGSI_SEMANTIC_BASEVERTEX:
1382       case TGSI_SEMANTIC_BASEINSTANCE:
1383       case TGSI_SEMANTIC_DRAWID:
1384          info->prop.vp.usesDrawParameters = true;
1385          break;
1386       case TGSI_SEMANTIC_SAMPLEID:
1387       case TGSI_SEMANTIC_SAMPLEPOS:
1388          info->prop.fp.persampleInvocation = true;
1389          break;
1390       case TGSI_SEMANTIC_SAMPLEMASK:
1391          info->prop.fp.usesSampleMaskIn = true;
1392          break;
1393       default:
1394          break;
1395       }
1396       for (i = first; i <= last; ++i, ++si) {
1397          info->sv[i].sn = sn;
1398          info->sv[i].si = si;
1399          info->sv[i].input = inferSysValDirection(sn);
1400
1401          switch (sn) {
1402          case TGSI_SEMANTIC_TESSOUTER:
1403          case TGSI_SEMANTIC_TESSINNER:
1404             info->sv[i].patch = 1;
1405             break;
1406          }
1407       }
1408       break;
1409 /*
1410    case TGSI_FILE_RESOURCE:
1411       for (i = first; i <= last; ++i) {
1412          resources[i].target = decl->Resource.Resource;
1413          resources[i].raw = decl->Resource.Raw;
1414          resources[i].slot = i;
1415       }
1416       break;
1417 */
1418    case TGSI_FILE_SAMPLER_VIEW:
1419       for (i = first; i <= last; ++i)
1420          textureViews[i].target = decl->SamplerView.Resource;
1421       break;
1422    case TGSI_FILE_MEMORY:
1423       for (i = first; i <= last; ++i)
1424          memoryFiles[i].mem_type = decl->Declaration.MemType;
1425       break;
1426    case TGSI_FILE_NULL:
1427    case TGSI_FILE_TEMPORARY:
1428       for (i = first; i <= last; ++i)
1429          tempArrayId[i] = arrayId;
1430       if (arrayId)
1431          tempArrayInfo.insert(std::make_pair(arrayId, std::make_pair(
1432                                                    first, last - first + 1)));
1433       break;
1434    case TGSI_FILE_BUFFER:
1435       for (i = first; i <= last; ++i)
1436          bufferAtomics[i] = decl->Declaration.Atomic;
1437       break;
1438    case TGSI_FILE_ADDRESS:
1439    case TGSI_FILE_CONSTANT:
1440    case TGSI_FILE_IMMEDIATE:
1441    case TGSI_FILE_SAMPLER:
1442    case TGSI_FILE_IMAGE:
1443       break;
1444    default:
1445       ERROR("unhandled TGSI_FILE %d\n", decl->Declaration.File);
1446       return false;
1447    }
1448    return true;
1449 }
1450
1451 inline bool Source::isEdgeFlagPassthrough(const Instruction& insn) const
1452 {
1453    return insn.getOpcode() == TGSI_OPCODE_MOV &&
1454       insn.getDst(0).getIndex(0) == info->io.edgeFlagOut &&
1455       insn.getSrc(0).getFile() == TGSI_FILE_INPUT;
1456 }
1457
1458 void Source::scanInstructionSrc(const Instruction& insn,
1459                                 const Instruction::SrcRegister& src,
1460                                 unsigned mask)
1461 {
1462    if (src.getFile() == TGSI_FILE_TEMPORARY) {
1463       if (src.isIndirect(0))
1464          indirectTempArrays.insert(src.getArrayId());
1465    } else
1466    if (src.getFile() == TGSI_FILE_OUTPUT) {
1467       if (src.isIndirect(0)) {
1468          // We don't know which one is accessed, just mark everything for
1469          // reading. This is an extremely unlikely occurrence.
1470          for (unsigned i = 0; i < info->numOutputs; ++i)
1471             info->out[i].oread = 1;
1472       } else {
1473          info->out[src.getIndex(0)].oread = 1;
1474       }
1475    }
1476    if (src.getFile() == TGSI_FILE_SYSTEM_VALUE) {
1477       if (info->sv[src.getIndex(0)].sn == TGSI_SEMANTIC_SAMPLEPOS)
1478          info->prop.fp.readsSampleLocations = true;
1479    }
1480    if (src.getFile() != TGSI_FILE_INPUT)
1481       return;
1482
1483    if (src.isIndirect(0)) {
1484       for (unsigned i = 0; i < info->numInputs; ++i)
1485          info->in[i].mask = 0xf;
1486    } else {
1487       const int i = src.getIndex(0);
1488       for (unsigned c = 0; c < 4; ++c) {
1489          if (!(mask & (1 << c)))
1490             continue;
1491          int k = src.getSwizzle(c);
1492          if (k <= TGSI_SWIZZLE_W)
1493             info->in[i].mask |= 1 << k;
1494       }
1495       switch (info->in[i].sn) {
1496       case TGSI_SEMANTIC_PSIZE:
1497       case TGSI_SEMANTIC_PRIMID:
1498       case TGSI_SEMANTIC_FOG:
1499          info->in[i].mask &= 0x1;
1500          break;
1501       case TGSI_SEMANTIC_PCOORD:
1502          info->in[i].mask &= 0x3;
1503          break;
1504       default:
1505          break;
1506       }
1507    }
1508 }
1509
1510 bool Source::scanInstruction(const struct tgsi_full_instruction *inst)
1511 {
1512    Instruction insn(inst);
1513
1514    if (insn.getOpcode() == TGSI_OPCODE_BARRIER)
1515       info->numBarriers = 1;
1516
1517    if (insn.getOpcode() == TGSI_OPCODE_FBFETCH)
1518       info->prop.fp.readsFramebuffer = true;
1519
1520    if (insn.getOpcode() == TGSI_OPCODE_INTERP_SAMPLE)
1521       info->prop.fp.readsSampleLocations = true;
1522
1523    if (insn.getOpcode() == TGSI_OPCODE_DEMOTE)
1524       info->prop.fp.usesDiscard = true;
1525
1526    if (insn.dstCount()) {
1527       Instruction::DstRegister dst = insn.getDst(0);
1528
1529       if (insn.getOpcode() == TGSI_OPCODE_STORE &&
1530           dst.getFile() != TGSI_FILE_MEMORY) {
1531          info->io.globalAccess |= 0x2;
1532
1533          if (dst.getFile() == TGSI_FILE_INPUT) {
1534             // TODO: Handle indirect somehow?
1535             const int i = dst.getIndex(0);
1536             info->in[i].mask |= 1;
1537          }
1538       }
1539
1540       if (dst.getFile() == TGSI_FILE_OUTPUT) {
1541          if (dst.isIndirect(0))
1542             for (unsigned i = 0; i < info->numOutputs; ++i)
1543                info->out[i].mask = 0xf;
1544          else
1545             info->out[dst.getIndex(0)].mask |= dst.getMask();
1546
1547          if (info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PSIZE ||
1548              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PRIMID ||
1549              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_LAYER ||
1550              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_VIEWPORT_INDEX ||
1551              info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_FOG)
1552             info->out[dst.getIndex(0)].mask &= 1;
1553
1554          if (isEdgeFlagPassthrough(insn))
1555             info->io.edgeFlagIn = insn.getSrc(0).getIndex(0);
1556       } else
1557       if (dst.getFile() == TGSI_FILE_TEMPORARY) {
1558          if (dst.isIndirect(0))
1559             indirectTempArrays.insert(dst.getArrayId());
1560       } else
1561       if (dst.getFile() == TGSI_FILE_BUFFER ||
1562           dst.getFile() == TGSI_FILE_IMAGE ||
1563           (dst.getFile() == TGSI_FILE_MEMORY &&
1564            memoryFiles[dst.getIndex(0)].mem_type == TGSI_MEMORY_TYPE_GLOBAL)) {
1565          info->io.globalAccess |= 0x2;
1566       }
1567    }
1568
1569    if (insn.srcCount() && (
1570              insn.getSrc(0).getFile() != TGSI_FILE_MEMORY ||
1571              memoryFiles[insn.getSrc(0).getIndex(0)].mem_type ==
1572              TGSI_MEMORY_TYPE_GLOBAL)) {
1573       switch (insn.getOpcode()) {
1574       case TGSI_OPCODE_ATOMUADD:
1575       case TGSI_OPCODE_ATOMXCHG:
1576       case TGSI_OPCODE_ATOMCAS:
1577       case TGSI_OPCODE_ATOMAND:
1578       case TGSI_OPCODE_ATOMOR:
1579       case TGSI_OPCODE_ATOMXOR:
1580       case TGSI_OPCODE_ATOMUMIN:
1581       case TGSI_OPCODE_ATOMIMIN:
1582       case TGSI_OPCODE_ATOMUMAX:
1583       case TGSI_OPCODE_ATOMIMAX:
1584       case TGSI_OPCODE_ATOMFADD:
1585       case TGSI_OPCODE_ATOMDEC_WRAP:
1586       case TGSI_OPCODE_ATOMINC_WRAP:
1587       case TGSI_OPCODE_LOAD:
1588          info->io.globalAccess |= (insn.getOpcode() == TGSI_OPCODE_LOAD) ?
1589             0x1 : 0x2;
1590          break;
1591       }
1592    }
1593
1594
1595    for (unsigned s = 0; s < insn.srcCount(); ++s)
1596       scanInstructionSrc(insn, insn.getSrc(s), insn.srcMask(s));
1597
1598    for (unsigned s = 0; s < insn.getNumTexOffsets(); ++s)
1599       scanInstructionSrc(insn, insn.getTexOffset(s), insn.texOffsetMask());
1600
1601    return true;
1602 }
1603
1604 nv50_ir::TexInstruction::Target
1605 Instruction::getTexture(const tgsi::Source *code, int s) const
1606 {
1607    // XXX: indirect access
1608    unsigned int r;
1609
1610    switch (getSrc(s).getFile()) {
1611 /*
1612    case TGSI_FILE_RESOURCE:
1613       r = getSrc(s).getIndex(0);
1614       return translateTexture(code->resources.at(r).target);
1615 */
1616    case TGSI_FILE_SAMPLER_VIEW:
1617       r = getSrc(s).getIndex(0);
1618       return translateTexture(code->textureViews.at(r).target);
1619    default:
1620       return translateTexture(insn->Texture.Texture);
1621    }
1622 }
1623
1624 } // namespace tgsi
1625
1626 namespace {
1627
1628 using namespace nv50_ir;
1629
1630 class Converter : public ConverterCommon
1631 {
1632 public:
1633    Converter(Program *, const tgsi::Source *);
1634    ~Converter();
1635
1636    bool run();
1637
1638 private:
1639    Value *shiftAddress(Value *);
1640    Value *getVertexBase(int s);
1641    Value *getOutputBase(int s);
1642    DataArray *getArrayForFile(unsigned file, int idx);
1643    Value *fetchSrc(int s, int c);
1644    Value *fetchDst(int d, int c);
1645    Value *acquireDst(int d, int c);
1646    void storeDst(int d, int c, Value *);
1647
1648    Value *fetchSrc(const tgsi::Instruction::SrcRegister src, int c, Value *ptr);
1649    void storeDst(const tgsi::Instruction::DstRegister dst, int c,
1650                  Value *val, Value *ptr);
1651
1652    void adjustTempIndex(int arrayId, int &idx, int &idx2d) const;
1653    Value *applySrcMod(Value *, int s, int c);
1654
1655    Symbol *makeSym(uint file, int fileIndex, int idx, int c, uint32_t addr);
1656    Symbol *srcToSym(tgsi::Instruction::SrcRegister, int c);
1657    Symbol *dstToSym(tgsi::Instruction::DstRegister, int c);
1658
1659    bool isSubGroupMask(uint8_t semantic);
1660
1661    bool handleInstruction(const struct tgsi_full_instruction *);
1662    void exportOutputs();
1663    inline bool isEndOfSubroutine(uint ip);
1664
1665    void loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask);
1666
1667    // R,S,L,C,Dx,Dy encode TGSI sources for respective values (0xSf for auto)
1668    void setTexRS(TexInstruction *, unsigned int& s, int R, int S);
1669    void handleTEX(Value *dst0[4], int R, int S, int L, int C, int Dx, int Dy);
1670    void handleTXF(Value *dst0[4], int R, int L_M);
1671    void handleTXQ(Value *dst0[4], enum TexQuery, int R);
1672    void handleFBFETCH(Value *dst0[4]);
1673    void handleLIT(Value *dst0[4]);
1674
1675    // Symbol *getResourceBase(int r);
1676    void getImageCoords(std::vector<Value *>&, int s);
1677
1678    void handleLOAD(Value *dst0[4]);
1679    void handleSTORE();
1680    void handleATOM(Value *dst0[4], DataType, uint16_t subOp);
1681
1682    void handleINTERP(Value *dst0[4]);
1683
1684    Value *interpolate(tgsi::Instruction::SrcRegister, int c, Value *ptr);
1685
1686    void insertConvergenceOps(BasicBlock *conv, BasicBlock *fork);
1687
1688    Value *buildDot(int dim);
1689
1690    class BindArgumentsPass : public Pass {
1691    public:
1692       BindArgumentsPass(Converter &conv) : conv(conv) { }
1693
1694    private:
1695       Converter &conv;
1696       Subroutine *sub;
1697
1698       inline const Location *getValueLocation(Subroutine *, Value *);
1699
1700       template<typename T> inline void
1701       updateCallArgs(Instruction *i, void (Instruction::*setArg)(int, Value *),
1702                      T (Function::*proto));
1703
1704       template<typename T> inline void
1705       updatePrototype(BitSet *set, void (Function::*updateSet)(),
1706                       T (Function::*proto));
1707
1708    protected:
1709       bool visit(Function *);
1710       bool visit(BasicBlock *bb) { return false; }
1711    };
1712
1713 private:
1714    const tgsi::Source *code;
1715
1716    uint ip; // instruction pointer
1717
1718    tgsi::Instruction tgsi;
1719
1720    DataType dstTy;
1721    DataType srcTy;
1722
1723    DataArray tData; // TGSI_FILE_TEMPORARY
1724    DataArray lData; // TGSI_FILE_TEMPORARY, for indirect arrays
1725    DataArray aData; // TGSI_FILE_ADDRESS
1726    DataArray oData; // TGSI_FILE_OUTPUT (if outputs in registers)
1727
1728    Value *zero;
1729
1730    Value *vtxBase[5]; // base address of vertex in primitive (for TP/GP)
1731    uint8_t vtxBaseValid;
1732
1733    Stack condBBs;  // fork BB, then else clause BB
1734    Stack joinBBs;  // fork BB, for inserting join ops on ENDIF
1735    Stack loopBBs;  // loop headers
1736    Stack breakBBs; // end of / after loop
1737
1738    Value *viewport;
1739 };
1740
1741 Symbol *
1742 Converter::srcToSym(tgsi::Instruction::SrcRegister src, int c)
1743 {
1744    const int swz = src.getSwizzle(c);
1745
1746    /* TODO: Use Array ID when it's available for the index */
1747    return makeSym(src.getFile(),
1748                   src.is2D() ? src.getIndex(1) : 0,
1749                   src.getIndex(0), swz,
1750                   src.getIndex(0) * 16 + swz * 4);
1751 }
1752
1753 Symbol *
1754 Converter::dstToSym(tgsi::Instruction::DstRegister dst, int c)
1755 {
1756    /* TODO: Use Array ID when it's available for the index */
1757    return makeSym(dst.getFile(),
1758                   dst.is2D() ? dst.getIndex(1) : 0,
1759                   dst.getIndex(0), c,
1760                   dst.getIndex(0) * 16 + c * 4);
1761 }
1762
1763 Symbol *
1764 Converter::makeSym(uint tgsiFile, int fileIdx, int idx, int c, uint32_t address)
1765 {
1766    Symbol *sym = new_Symbol(prog, tgsi::translateFile(tgsiFile));
1767
1768    sym->reg.fileIndex = fileIdx;
1769
1770    if (tgsiFile == TGSI_FILE_MEMORY) {
1771       switch (code->memoryFiles[fileIdx].mem_type) {
1772       case TGSI_MEMORY_TYPE_GLOBAL:
1773          /* No-op this is the default for TGSI_FILE_MEMORY */
1774          sym->setFile(FILE_MEMORY_GLOBAL);
1775          break;
1776       case TGSI_MEMORY_TYPE_SHARED:
1777          sym->setFile(FILE_MEMORY_SHARED);
1778          break;
1779       case TGSI_MEMORY_TYPE_INPUT:
1780          assert(prog->getType() == Program::TYPE_COMPUTE);
1781          assert(idx == -1);
1782          sym->setFile(FILE_SHADER_INPUT);
1783          address += info->prop.cp.inputOffset;
1784          break;
1785       default:
1786          assert(0); /* TODO: Add support for global and private memory */
1787       }
1788    }
1789
1790    if (idx >= 0) {
1791       if (sym->reg.file == FILE_SHADER_INPUT)
1792          sym->setOffset(info->in[idx].slot[c] * 4);
1793       else
1794       if (sym->reg.file == FILE_SHADER_OUTPUT)
1795          sym->setOffset(info->out[idx].slot[c] * 4);
1796       else
1797       if (sym->reg.file == FILE_SYSTEM_VALUE)
1798          sym->setSV(tgsi::translateSysVal(info->sv[idx].sn), c);
1799       else
1800          sym->setOffset(address);
1801    } else {
1802       sym->setOffset(address);
1803    }
1804    return sym;
1805 }
1806
1807 Value *
1808 Converter::interpolate(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
1809 {
1810    operation op;
1811
1812    // XXX: no way to know interpolation mode if we don't know what's accessed
1813    const uint8_t mode = translateInterpMode(&info->in[ptr ? 0 :
1814                                                       src.getIndex(0)], op);
1815
1816    Instruction *insn = new_Instruction(func, op, TYPE_F32);
1817
1818    insn->setDef(0, getScratch());
1819    insn->setSrc(0, srcToSym(src, c));
1820    if (op == OP_PINTERP)
1821       insn->setSrc(1, fragCoord[3]);
1822    if (ptr)
1823       insn->setIndirect(0, 0, ptr);
1824
1825    insn->setInterpolate(mode);
1826
1827    bb->insertTail(insn);
1828    return insn->getDef(0);
1829 }
1830
1831 Value *
1832 Converter::applySrcMod(Value *val, int s, int c)
1833 {
1834    Modifier m = tgsi.getSrc(s).getMod(c);
1835    DataType ty = tgsi.inferSrcType();
1836
1837    if (m & Modifier(NV50_IR_MOD_ABS))
1838       val = mkOp1v(OP_ABS, ty, getScratch(), val);
1839
1840    if (m & Modifier(NV50_IR_MOD_NEG))
1841       val = mkOp1v(OP_NEG, ty, getScratch(), val);
1842
1843    return val;
1844 }
1845
1846 Value *
1847 Converter::getVertexBase(int s)
1848 {
1849    assert(s < 5);
1850    if (!(vtxBaseValid & (1 << s))) {
1851       const int index = tgsi.getSrc(s).getIndex(1);
1852       Value *rel = NULL;
1853       if (tgsi.getSrc(s).isIndirect(1))
1854          rel = fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL);
1855       vtxBaseValid |= 1 << s;
1856       vtxBase[s] = mkOp2v(OP_PFETCH, TYPE_U32, getSSA(4, FILE_ADDRESS),
1857                           mkImm(index), rel);
1858    }
1859    return vtxBase[s];
1860 }
1861
1862 Value *
1863 Converter::getOutputBase(int s)
1864 {
1865    assert(s < 5);
1866    if (!(vtxBaseValid & (1 << s))) {
1867       Value *offset = loadImm(NULL, tgsi.getSrc(s).getIndex(1));
1868       if (tgsi.getSrc(s).isIndirect(1))
1869          offset = mkOp2v(OP_ADD, TYPE_U32, getSSA(),
1870                          fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL),
1871                          offset);
1872       vtxBaseValid |= 1 << s;
1873       vtxBase[s] = mkOp2v(OP_ADD, TYPE_U32, getSSA(), outBase, offset);
1874    }
1875    return vtxBase[s];
1876 }
1877
1878 Value *
1879 Converter::fetchSrc(int s, int c)
1880 {
1881    Value *res;
1882    Value *ptr = NULL, *dimRel = NULL;
1883
1884    tgsi::Instruction::SrcRegister src = tgsi.getSrc(s);
1885
1886    if (src.isIndirect(0))
1887       ptr = fetchSrc(src.getIndirect(0), 0, NULL);
1888
1889    if (src.is2D()) {
1890       switch (src.getFile()) {
1891       case TGSI_FILE_OUTPUT:
1892          dimRel = getOutputBase(s);
1893          break;
1894       case TGSI_FILE_INPUT:
1895          dimRel = getVertexBase(s);
1896          break;
1897       case TGSI_FILE_CONSTANT:
1898          // on NVC0, this is valid and c{I+J}[k] == cI[(J << 16) + k]
1899          if (src.isIndirect(1))
1900             dimRel = fetchSrc(src.getIndirect(1), 0, 0);
1901          break;
1902       default:
1903          break;
1904       }
1905    }
1906
1907    res = fetchSrc(src, c, ptr);
1908
1909    if (dimRel)
1910       res->getInsn()->setIndirect(0, 1, dimRel);
1911
1912    return applySrcMod(res, s, c);
1913 }
1914
1915 Value *
1916 Converter::fetchDst(int d, int c)
1917 {
1918    Value *res;
1919    Value *ptr = NULL, *dimRel = NULL;
1920
1921    tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
1922
1923    if (dst.isIndirect(0))
1924       ptr = fetchSrc(dst.getIndirect(0), 0, NULL);
1925
1926    if (dst.is2D()) {
1927       switch (dst.getFile()) {
1928       case TGSI_FILE_OUTPUT:
1929          assert(0); // TODO
1930          dimRel = NULL;
1931          break;
1932       case TGSI_FILE_INPUT:
1933          assert(0); // TODO
1934          dimRel = NULL;
1935          break;
1936       case TGSI_FILE_CONSTANT:
1937          // on NVC0, this is valid and c{I+J}[k] == cI[(J << 16) + k]
1938          if (dst.isIndirect(1))
1939             dimRel = fetchSrc(dst.getIndirect(1), 0, 0);
1940          break;
1941       default:
1942          break;
1943       }
1944    }
1945
1946    struct tgsi_full_src_register fsr = dst.asSrc();
1947    tgsi::Instruction::SrcRegister src(&fsr);
1948    res = fetchSrc(src, c, ptr);
1949
1950    if (dimRel)
1951       res->getInsn()->setIndirect(0, 1, dimRel);
1952
1953    return res;
1954 }
1955
1956 Converter::DataArray *
1957 Converter::getArrayForFile(unsigned file, int idx)
1958 {
1959    switch (file) {
1960    case TGSI_FILE_TEMPORARY:
1961       return idx == 0 ? &tData : &lData;
1962    case TGSI_FILE_ADDRESS:
1963       return &aData;
1964    case TGSI_FILE_OUTPUT:
1965       assert(prog->getType() == Program::TYPE_FRAGMENT);
1966       return &oData;
1967    default:
1968       assert(!"invalid/unhandled TGSI source file");
1969       return NULL;
1970    }
1971 }
1972
1973 Value *
1974 Converter::shiftAddress(Value *index)
1975 {
1976    if (!index)
1977       return NULL;
1978    return mkOp2v(OP_SHL, TYPE_U32, getSSA(4, FILE_ADDRESS), index, mkImm(4));
1979 }
1980
1981 void
1982 Converter::adjustTempIndex(int arrayId, int &idx, int &idx2d) const
1983 {
1984    std::map<int, int>::const_iterator it =
1985       code->indirectTempOffsets.find(arrayId);
1986    if (it == code->indirectTempOffsets.end())
1987       return;
1988
1989    idx2d = 1;
1990    idx += it->second;
1991 }
1992
1993 bool
1994 Converter::isSubGroupMask(uint8_t semantic)
1995 {
1996    switch (semantic) {
1997       case TGSI_SEMANTIC_SUBGROUP_EQ_MASK:
1998       case TGSI_SEMANTIC_SUBGROUP_LT_MASK:
1999       case TGSI_SEMANTIC_SUBGROUP_LE_MASK:
2000       case TGSI_SEMANTIC_SUBGROUP_GT_MASK:
2001       case TGSI_SEMANTIC_SUBGROUP_GE_MASK:
2002          return true;
2003       default:
2004          return false;
2005    }
2006 }
2007
2008 Value *
2009 Converter::fetchSrc(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
2010 {
2011    int idx2d = src.is2D() ? src.getIndex(1) : 0;
2012    int idx = src.getIndex(0);
2013    const int swz = src.getSwizzle(c);
2014    Instruction *ld;
2015
2016    switch (src.getFile()) {
2017    case TGSI_FILE_IMMEDIATE:
2018       assert(!ptr);
2019       return loadImm(NULL, info->immd.data[idx * 4 + swz]);
2020    case TGSI_FILE_CONSTANT:
2021       return mkLoadv(TYPE_U32, srcToSym(src, c), shiftAddress(ptr));
2022    case TGSI_FILE_INPUT:
2023       if (prog->getType() == Program::TYPE_FRAGMENT) {
2024          // don't load masked inputs, won't be assigned a slot
2025          if (!ptr && !(info->in[idx].mask & (1 << swz)))
2026             return loadImm(NULL, swz == TGSI_SWIZZLE_W ? 1.0f : 0.0f);
2027          return interpolate(src, c, shiftAddress(ptr));
2028       } else
2029       if (prog->getType() == Program::TYPE_GEOMETRY) {
2030          if (!ptr && info->in[idx].sn == TGSI_SEMANTIC_PRIMID)
2031             return mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_PRIMITIVE_ID, 0));
2032          // XXX: This is going to be a problem with scalar arrays, i.e. when
2033          // we cannot assume that the address is given in units of vec4.
2034          //
2035          // nv50 and nvc0 need different things here, so let the lowering
2036          // passes decide what to do with the address
2037          if (ptr)
2038             return mkLoadv(TYPE_U32, srcToSym(src, c), ptr);
2039       }
2040       ld = mkLoad(TYPE_U32, getSSA(), srcToSym(src, c), shiftAddress(ptr));
2041       ld->perPatch = info->in[idx].patch;
2042       return ld->getDef(0);
2043    case TGSI_FILE_OUTPUT:
2044       assert(prog->getType() == Program::TYPE_TESSELLATION_CONTROL);
2045       ld = mkLoad(TYPE_U32, getSSA(), srcToSym(src, c), shiftAddress(ptr));
2046       ld->perPatch = info->out[idx].patch;
2047       return ld->getDef(0);
2048    case TGSI_FILE_SYSTEM_VALUE:
2049       assert(!ptr);
2050       if (info->sv[idx].sn == TGSI_SEMANTIC_THREAD_ID &&
2051           info->prop.cp.numThreads[swz] == 1)
2052          return loadImm(NULL, 0u);
2053       if (isSubGroupMask(info->sv[idx].sn) && swz > 0)
2054          return loadImm(NULL, 0u);
2055       if (info->sv[idx].sn == TGSI_SEMANTIC_SUBGROUP_SIZE)
2056          return loadImm(NULL, 32u);
2057       ld = mkOp1(OP_RDSV, TYPE_U32, getSSA(), srcToSym(src, c));
2058       ld->perPatch = info->sv[idx].patch;
2059       return ld->getDef(0);
2060    case TGSI_FILE_TEMPORARY: {
2061       int arrayid = src.getArrayId();
2062       if (!arrayid)
2063          arrayid = code->tempArrayId[idx];
2064       adjustTempIndex(arrayid, idx, idx2d);
2065    }
2066       /* fallthrough */
2067    default:
2068       return getArrayForFile(src.getFile(), idx2d)->load(
2069          sub.cur->values, idx, swz, shiftAddress(ptr));
2070    }
2071 }
2072
2073 Value *
2074 Converter::acquireDst(int d, int c)
2075 {
2076    const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
2077    const unsigned f = dst.getFile();
2078    int idx = dst.getIndex(0);
2079    int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
2080
2081    if (dst.isMasked(c) || f == TGSI_FILE_BUFFER || f == TGSI_FILE_MEMORY ||
2082        f == TGSI_FILE_IMAGE)
2083       return NULL;
2084
2085    if (dst.isIndirect(0) ||
2086        f == TGSI_FILE_SYSTEM_VALUE ||
2087        (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT))
2088       return getScratch();
2089
2090    if (f == TGSI_FILE_TEMPORARY) {
2091       int arrayid = dst.getArrayId();
2092       if (!arrayid)
2093          arrayid = code->tempArrayId[idx];
2094       adjustTempIndex(arrayid, idx, idx2d);
2095    }
2096
2097    return getArrayForFile(f, idx2d)-> acquire(sub.cur->values, idx, c);
2098 }
2099
2100 void
2101 Converter::storeDst(int d, int c, Value *val)
2102 {
2103    const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
2104
2105    if (tgsi.getSaturate()) {
2106       mkOp1(OP_SAT, dstTy, val, val);
2107    }
2108
2109    Value *ptr = NULL;
2110    if (dst.isIndirect(0))
2111       ptr = shiftAddress(fetchSrc(dst.getIndirect(0), 0, NULL));
2112
2113    if (info->io.genUserClip > 0 &&
2114        dst.getFile() == TGSI_FILE_OUTPUT &&
2115        !dst.isIndirect(0) && dst.getIndex(0) == code->clipVertexOutput) {
2116       mkMov(clipVtx[c], val);
2117       val = clipVtx[c];
2118    }
2119
2120    storeDst(dst, c, val, ptr);
2121 }
2122
2123 void
2124 Converter::storeDst(const tgsi::Instruction::DstRegister dst, int c,
2125                     Value *val, Value *ptr)
2126 {
2127    const unsigned f = dst.getFile();
2128    int idx = dst.getIndex(0);
2129    int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
2130
2131    if (f == TGSI_FILE_SYSTEM_VALUE) {
2132       assert(!ptr);
2133       mkOp2(OP_WRSV, TYPE_U32, NULL, dstToSym(dst, c), val);
2134    } else
2135    if (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT) {
2136
2137       if (ptr || (info->out[idx].mask & (1 << c))) {
2138          /* Save the viewport index into a scratch register so that it can be
2139             exported at EMIT time */
2140          if (info->out[idx].sn == TGSI_SEMANTIC_VIEWPORT_INDEX &&
2141              prog->getType() == Program::TYPE_GEOMETRY &&
2142              viewport != NULL)
2143             mkOp1(OP_MOV, TYPE_U32, viewport, val);
2144          else
2145             mkStore(OP_EXPORT, TYPE_U32, dstToSym(dst, c), ptr, val)->perPatch =
2146                info->out[idx].patch;
2147       }
2148    } else
2149    if (f == TGSI_FILE_TEMPORARY ||
2150        f == TGSI_FILE_ADDRESS ||
2151        f == TGSI_FILE_OUTPUT) {
2152       if (f == TGSI_FILE_TEMPORARY) {
2153          int arrayid = dst.getArrayId();
2154          if (!arrayid)
2155             arrayid = code->tempArrayId[idx];
2156          adjustTempIndex(arrayid, idx, idx2d);
2157       }
2158
2159       getArrayForFile(f, idx2d)->store(sub.cur->values, idx, c, ptr, val);
2160    } else {
2161       assert(!"invalid dst file");
2162    }
2163 }
2164
2165 #define FOR_EACH_DST_ENABLED_CHANNEL(d, chan, inst) \
2166    for (chan = 0; chan < 4; ++chan)                 \
2167       if (!inst.getDst(d).isMasked(chan))
2168
2169 Value *
2170 Converter::buildDot(int dim)
2171 {
2172    assert(dim > 0);
2173
2174    Value *src0 = fetchSrc(0, 0), *src1 = fetchSrc(1, 0);
2175    Value *dotp = getScratch();
2176
2177    mkOp2(OP_MUL, TYPE_F32, dotp, src0, src1)
2178       ->dnz = info->io.mul_zero_wins;
2179
2180    for (int c = 1; c < dim; ++c) {
2181       src0 = fetchSrc(0, c);
2182       src1 = fetchSrc(1, c);
2183       mkOp3(OP_MAD, TYPE_F32, dotp, src0, src1, dotp)
2184          ->dnz = info->io.mul_zero_wins;
2185    }
2186    return dotp;
2187 }
2188
2189 void
2190 Converter::insertConvergenceOps(BasicBlock *conv, BasicBlock *fork)
2191 {
2192    FlowInstruction *join = new_FlowInstruction(func, OP_JOIN, NULL);
2193    join->fixed = 1;
2194    conv->insertHead(join);
2195
2196    assert(!fork->joinAt);
2197    fork->joinAt = new_FlowInstruction(func, OP_JOINAT, conv);
2198    fork->insertBefore(fork->getExit(), fork->joinAt);
2199 }
2200
2201 void
2202 Converter::setTexRS(TexInstruction *tex, unsigned int& s, int R, int S)
2203 {
2204    unsigned rIdx = 0, sIdx = 0;
2205
2206    if (R >= 0 && tgsi.getSrc(R).getFile() != TGSI_FILE_SAMPLER) {
2207       // This is the bindless case. We have to get the actual value and pass
2208       // it in. This will be the complete handle.
2209       tex->tex.rIndirectSrc = s;
2210       tex->setSrc(s++, fetchSrc(R, 0));
2211       tex->setTexture(tgsi.getTexture(code, R), 0xff, 0x1f);
2212       tex->tex.bindless = true;
2213       return;
2214    }
2215
2216    if (R >= 0)
2217       rIdx = tgsi.getSrc(R).getIndex(0);
2218    if (S >= 0)
2219       sIdx = tgsi.getSrc(S).getIndex(0);
2220
2221    tex->setTexture(tgsi.getTexture(code, R), rIdx, sIdx);
2222
2223    if (tgsi.getSrc(R).isIndirect(0)) {
2224       tex->tex.rIndirectSrc = s;
2225       tex->setSrc(s++, fetchSrc(tgsi.getSrc(R).getIndirect(0), 0, NULL));
2226    }
2227    if (S >= 0 && tgsi.getSrc(S).isIndirect(0)) {
2228       tex->tex.sIndirectSrc = s;
2229       tex->setSrc(s++, fetchSrc(tgsi.getSrc(S).getIndirect(0), 0, NULL));
2230    }
2231 }
2232
2233 void
2234 Converter::handleTXQ(Value *dst0[4], enum TexQuery query, int R)
2235 {
2236    TexInstruction *tex = new_TexInstruction(func, OP_TXQ);
2237    tex->tex.query = query;
2238    unsigned int c, d;
2239
2240    for (d = 0, c = 0; c < 4; ++c) {
2241       if (!dst0[c])
2242          continue;
2243       tex->tex.mask |= 1 << c;
2244       tex->setDef(d++, dst0[c]);
2245    }
2246    if (query == TXQ_DIMS)
2247       tex->setSrc((c = 0), fetchSrc(0, 0)); // mip level
2248    else
2249       tex->setSrc((c = 0), zero);
2250
2251    setTexRS(tex, ++c, R, -1);
2252
2253    bb->insertTail(tex);
2254 }
2255
2256 void
2257 Converter::loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask)
2258 {
2259    Value *proj = fetchSrc(0, 3);
2260    Instruction *insn = proj->getUniqueInsn();
2261    int c;
2262
2263    if (insn->op == OP_PINTERP) {
2264       bb->insertTail(insn = cloneForward(func, insn));
2265       insn->op = OP_LINTERP;
2266       insn->setInterpolate(NV50_IR_INTERP_LINEAR | insn->getSampleMode());
2267       insn->setSrc(1, NULL);
2268       proj = insn->getDef(0);
2269    }
2270    proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), proj);
2271
2272    for (c = 0; c < 4; ++c) {
2273       if (!(mask & (1 << c)))
2274          continue;
2275       if ((insn = src[c]->getUniqueInsn())->op != OP_PINTERP)
2276          continue;
2277       mask &= ~(1 << c);
2278
2279       bb->insertTail(insn = cloneForward(func, insn));
2280       insn->setInterpolate(NV50_IR_INTERP_PERSPECTIVE | insn->getSampleMode());
2281       insn->setSrc(1, proj);
2282       dst[c] = insn->getDef(0);
2283    }
2284    if (!mask)
2285       return;
2286
2287    proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), fetchSrc(0, 3));
2288
2289    for (c = 0; c < 4; ++c)
2290       if (mask & (1 << c))
2291          dst[c] = mkOp2v(OP_MUL, TYPE_F32, getSSA(), src[c], proj);
2292 }
2293
2294 // order of nv50 ir sources: x y z layer lod/bias shadow
2295 // order of TGSI TEX sources: x y z layer shadow lod/bias
2296 //  lowering will finally set the hw specific order (like array first on nvc0)
2297 void
2298 Converter::handleTEX(Value *dst[4], int R, int S, int L, int C, int Dx, int Dy)
2299 {
2300    Value *arg[4], *src[8];
2301    Value *lod = NULL, *shd = NULL;
2302    unsigned int s, c, d;
2303    TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
2304
2305    TexInstruction::Target tgt = tgsi.getTexture(code, R);
2306
2307    for (s = 0; s < tgt.getArgCount(); ++s)
2308       arg[s] = src[s] = fetchSrc(0, s);
2309
2310    if (tgsi.getOpcode() == TGSI_OPCODE_TEX_LZ)
2311       lod = loadImm(NULL, 0);
2312    else if (texi->op == OP_TXL || texi->op == OP_TXB)
2313       lod = fetchSrc(L >> 4, L & 3);
2314
2315    if (C == 0x0f)
2316       C = 0x00 | MAX2(tgt.getArgCount(), 2); // guess DC src
2317
2318    if (tgt == TEX_TARGET_CUBE_ARRAY_SHADOW) {
2319       switch (tgsi.getOpcode()) {
2320       case TGSI_OPCODE_TG4: shd = fetchSrc(1, 0); break;
2321       case TGSI_OPCODE_TEX2: shd = fetchSrc(1, 0); break;
2322       case TGSI_OPCODE_TXB2: shd = fetchSrc(1, 1); break;
2323       case TGSI_OPCODE_TXL2: shd = fetchSrc(1, 1); break;
2324       default: assert(!"unexpected opcode with cube array shadow"); break;
2325       }
2326    }
2327    else if (tgt.isShadow())
2328       shd = fetchSrc(C >> 4, C & 3);
2329
2330    if (texi->op == OP_TXD) {
2331       for (c = 0; c < tgt.getDim() + tgt.isCube(); ++c) {
2332          texi->dPdx[c].set(fetchSrc(Dx >> 4, (Dx & 3) + c));
2333          texi->dPdy[c].set(fetchSrc(Dy >> 4, (Dy & 3) + c));
2334       }
2335    }
2336
2337    // cube textures don't care about projection value, it's divided out
2338    if (tgsi.getOpcode() == TGSI_OPCODE_TXP && !tgt.isCube() && !tgt.isArray()) {
2339       unsigned int n = tgt.getDim();
2340       if (shd) {
2341          arg[n] = shd;
2342          ++n;
2343          assert(tgt.getDim() == tgt.getArgCount());
2344       }
2345       loadProjTexCoords(src, arg, (1 << n) - 1);
2346       if (shd)
2347          shd = src[n - 1];
2348    }
2349
2350    for (c = 0, d = 0; c < 4; ++c) {
2351       if (dst[c]) {
2352          texi->setDef(d++, dst[c]);
2353          texi->tex.mask |= 1 << c;
2354       } else {
2355          // NOTE: maybe hook up def too, for CSE
2356       }
2357    }
2358    for (s = 0; s < tgt.getArgCount(); ++s)
2359       texi->setSrc(s, src[s]);
2360    if (lod)
2361       texi->setSrc(s++, lod);
2362    if (shd)
2363       texi->setSrc(s++, shd);
2364
2365    setTexRS(texi, s, R, S);
2366
2367    if (tgsi.getOpcode() == TGSI_OPCODE_SAMPLE_C_LZ)
2368       texi->tex.levelZero = true;
2369    if (prog->getType() != Program::TYPE_FRAGMENT &&
2370        (tgsi.getOpcode() == TGSI_OPCODE_TEX ||
2371         tgsi.getOpcode() == TGSI_OPCODE_TEX2 ||
2372         tgsi.getOpcode() == TGSI_OPCODE_TXP))
2373       texi->tex.levelZero = true;
2374    if (tgsi.getOpcode() == TGSI_OPCODE_TG4 && !tgt.isShadow())
2375       texi->tex.gatherComp = tgsi.getSrc(1).getValueU32(0, info);
2376
2377    texi->tex.useOffsets = tgsi.getNumTexOffsets();
2378    for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
2379       for (c = 0; c < 3; ++c) {
2380          texi->offset[s][c].set(fetchSrc(tgsi.getTexOffset(s), c, NULL));
2381          texi->offset[s][c].setInsn(texi);
2382       }
2383    }
2384
2385    bb->insertTail(texi);
2386 }
2387
2388 // 1st source: xyz = coordinates, w = lod/sample
2389 // 2nd source: offset
2390 void
2391 Converter::handleTXF(Value *dst[4], int R, int L_M)
2392 {
2393    TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
2394    int ms;
2395    unsigned int c, d, s;
2396
2397    texi->tex.target = tgsi.getTexture(code, R);
2398
2399    ms = texi->tex.target.isMS() ? 1 : 0;
2400    texi->tex.levelZero = ms; /* MS textures don't have mip-maps */
2401
2402    for (c = 0, d = 0; c < 4; ++c) {
2403       if (dst[c]) {
2404          texi->setDef(d++, dst[c]);
2405          texi->tex.mask |= 1 << c;
2406       }
2407    }
2408    for (c = 0; c < (texi->tex.target.getArgCount() - ms); ++c)
2409       texi->setSrc(c, fetchSrc(0, c));
2410    if (!ms && tgsi.getOpcode() == TGSI_OPCODE_TXF_LZ)
2411       texi->setSrc(c++, loadImm(NULL, 0));
2412    else
2413       texi->setSrc(c++, fetchSrc(L_M >> 4, L_M & 3)); // lod or ms
2414
2415    setTexRS(texi, c, R, -1);
2416
2417    texi->tex.useOffsets = tgsi.getNumTexOffsets();
2418    for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
2419       for (c = 0; c < 3; ++c) {
2420          texi->offset[s][c].set(fetchSrc(tgsi.getTexOffset(s), c, NULL));
2421          texi->offset[s][c].setInsn(texi);
2422       }
2423    }
2424
2425    bb->insertTail(texi);
2426 }
2427
2428 void
2429 Converter::handleFBFETCH(Value *dst[4])
2430 {
2431    TexInstruction *texi = new_TexInstruction(func, OP_TXF);
2432    unsigned int c, d;
2433
2434    texi->tex.target = TEX_TARGET_2D_MS_ARRAY;
2435    texi->tex.levelZero = 1;
2436    texi->tex.useOffsets = 0;
2437
2438    for (c = 0, d = 0; c < 4; ++c) {
2439       if (dst[c]) {
2440          texi->setDef(d++, dst[c]);
2441          texi->tex.mask |= 1 << c;
2442       }
2443    }
2444
2445    Value *x = mkOp1v(OP_RDSV, TYPE_F32, getScratch(), mkSysVal(SV_POSITION, 0));
2446    Value *y = mkOp1v(OP_RDSV, TYPE_F32, getScratch(), mkSysVal(SV_POSITION, 1));
2447    Value *z = mkOp1v(OP_RDSV, TYPE_U32, getScratch(), mkSysVal(SV_LAYER, 0));
2448    Value *ms = mkOp1v(OP_RDSV, TYPE_U32, getScratch(), mkSysVal(SV_SAMPLE_INDEX, 0));
2449
2450    mkCvt(OP_CVT, TYPE_U32, x, TYPE_F32, x)->rnd = ROUND_Z;
2451    mkCvt(OP_CVT, TYPE_U32, y, TYPE_F32, y)->rnd = ROUND_Z;
2452    texi->setSrc(0, x);
2453    texi->setSrc(1, y);
2454    texi->setSrc(2, z);
2455    texi->setSrc(3, ms);
2456
2457    texi->tex.r = texi->tex.s = -1;
2458
2459    bb->insertTail(texi);
2460 }
2461
2462 void
2463 Converter::handleLIT(Value *dst0[4])
2464 {
2465    Value *val0 = NULL;
2466    unsigned int mask = tgsi.getDst(0).getMask();
2467
2468    if (mask & (1 << 0))
2469       loadImm(dst0[0], 1.0f);
2470
2471    if (mask & (1 << 3))
2472       loadImm(dst0[3], 1.0f);
2473
2474    if (mask & (3 << 1)) {
2475       val0 = getScratch();
2476       mkOp2(OP_MAX, TYPE_F32, val0, fetchSrc(0, 0), zero);
2477       if (mask & (1 << 1))
2478          mkMov(dst0[1], val0);
2479    }
2480
2481    if (mask & (1 << 2)) {
2482       Value *src1 = fetchSrc(0, 1), *src3 = fetchSrc(0, 3);
2483       Value *val1 = getScratch(), *val3 = getScratch();
2484
2485       Value *pos128 = loadImm(NULL, +127.999999f);
2486       Value *neg128 = loadImm(NULL, -127.999999f);
2487
2488       mkOp2(OP_MAX, TYPE_F32, val1, src1, zero);
2489       mkOp2(OP_MAX, TYPE_F32, val3, src3, neg128);
2490       mkOp2(OP_MIN, TYPE_F32, val3, val3, pos128);
2491       mkOp2(OP_POW, TYPE_F32, val3, val1, val3);
2492
2493       mkCmp(OP_SLCT, CC_GT, TYPE_F32, dst0[2], TYPE_F32, val3, zero, val0);
2494    }
2495 }
2496
2497 /* Keep this around for now as reference when adding img support
2498 static inline bool
2499 isResourceSpecial(const int r)
2500 {
2501    return (r == TGSI_RESOURCE_GLOBAL ||
2502            r == TGSI_RESOURCE_LOCAL ||
2503            r == TGSI_RESOURCE_PRIVATE ||
2504            r == TGSI_RESOURCE_INPUT);
2505 }
2506
2507 static inline bool
2508 isResourceRaw(const tgsi::Source *code, const int r)
2509 {
2510    return isResourceSpecial(r) || code->resources[r].raw;
2511 }
2512
2513 static inline nv50_ir::TexTarget
2514 getResourceTarget(const tgsi::Source *code, int r)
2515 {
2516    if (isResourceSpecial(r))
2517       return nv50_ir::TEX_TARGET_BUFFER;
2518    return tgsi::translateTexture(code->resources.at(r).target);
2519 }
2520
2521 Symbol *
2522 Converter::getResourceBase(const int r)
2523 {
2524    Symbol *sym = NULL;
2525
2526    switch (r) {
2527    case TGSI_RESOURCE_GLOBAL:
2528       sym = new_Symbol(prog, nv50_ir::FILE_MEMORY_GLOBAL,
2529                        info->io.auxCBSlot);
2530       break;
2531    case TGSI_RESOURCE_LOCAL:
2532       assert(prog->getType() == Program::TYPE_COMPUTE);
2533       sym = mkSymbol(nv50_ir::FILE_MEMORY_SHARED, 0, TYPE_U32,
2534                      info->prop.cp.sharedOffset);
2535       break;
2536    case TGSI_RESOURCE_PRIVATE:
2537       sym = mkSymbol(nv50_ir::FILE_MEMORY_LOCAL, 0, TYPE_U32,
2538                      info->bin.tlsSpace);
2539       break;
2540    case TGSI_RESOURCE_INPUT:
2541       assert(prog->getType() == Program::TYPE_COMPUTE);
2542       sym = mkSymbol(nv50_ir::FILE_SHADER_INPUT, 0, TYPE_U32,
2543                      info->prop.cp.inputOffset);
2544       break;
2545    default:
2546       sym = new_Symbol(prog,
2547                        nv50_ir::FILE_MEMORY_GLOBAL, code->resources.at(r).slot);
2548       break;
2549    }
2550    return sym;
2551 }
2552
2553 void
2554 Converter::getResourceCoords(std::vector<Value *> &coords, int r, int s)
2555 {
2556    const int arg =
2557       TexInstruction::Target(getResourceTarget(code, r)).getArgCount();
2558
2559    for (int c = 0; c < arg; ++c)
2560       coords.push_back(fetchSrc(s, c));
2561
2562    // NOTE: TGSI_RESOURCE_GLOBAL needs FILE_GPR; this is an nv50 quirk
2563    if (r == TGSI_RESOURCE_LOCAL ||
2564        r == TGSI_RESOURCE_PRIVATE ||
2565        r == TGSI_RESOURCE_INPUT)
2566       coords[0] = mkOp1v(OP_MOV, TYPE_U32, getScratch(4, FILE_ADDRESS),
2567                          coords[0]);
2568 }
2569
2570 static inline int
2571 partitionLoadStore(uint8_t comp[2], uint8_t size[2], uint8_t mask)
2572 {
2573    int n = 0;
2574
2575    while (mask) {
2576       if (mask & 1) {
2577          size[n]++;
2578       } else {
2579          if (size[n])
2580             comp[n = 1] = size[0] + 1;
2581          else
2582             comp[n]++;
2583       }
2584       mask >>= 1;
2585    }
2586    if (size[0] == 3) {
2587       n = 1;
2588       size[0] = (comp[0] == 1) ? 1 : 2;
2589       size[1] = 3 - size[0];
2590       comp[1] = comp[0] + size[0];
2591    }
2592    return n + 1;
2593 }
2594 */
2595 void
2596 Converter::getImageCoords(std::vector<Value *> &coords, int s)
2597 {
2598    TexInstruction::Target t =
2599       TexInstruction::Target(tgsi.getImageTarget());
2600    const int arg = t.getDim() + (t.isArray() || t.isCube());
2601
2602    for (int c = 0; c < arg; ++c)
2603       coords.push_back(fetchSrc(s, c));
2604
2605    if (t.isMS())
2606       coords.push_back(fetchSrc(s, 3));
2607 }
2608
2609 // For raw loads, granularity is 4 byte.
2610 // Usage of the texture read mask on OP_SULDP is not allowed.
2611 void
2612 Converter::handleLOAD(Value *dst0[4])
2613 {
2614    const int r = tgsi.getSrc(0).getIndex(0);
2615    int c;
2616    std::vector<Value *> off, src, ldv, def;
2617    Value *ind = NULL;
2618
2619    if (tgsi.getSrc(0).isIndirect(0))
2620       ind = fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, 0);
2621
2622    switch (tgsi.getSrc(0).getFile()) {
2623    case TGSI_FILE_BUFFER:
2624    case TGSI_FILE_MEMORY:
2625       for (c = 0; c < 4; ++c) {
2626          if (!dst0[c])
2627             continue;
2628
2629          Value *off;
2630          Symbol *sym;
2631          uint32_t src0_component_offset = tgsi.getSrc(0).getSwizzle(c) * 4;
2632
2633          if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE) {
2634             off = NULL;
2635             sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c,
2636                           tgsi.getSrc(1).getValueU32(0, info) +
2637                           src0_component_offset);
2638          } else {
2639             // yzw are ignored for buffers
2640             off = fetchSrc(1, 0);
2641             sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c,
2642                           src0_component_offset);
2643          }
2644
2645          Instruction *ld = mkLoad(TYPE_U32, dst0[c], sym, off);
2646          if (tgsi.getSrc(0).getFile() == TGSI_FILE_BUFFER &&
2647              code->bufferAtomics[r])
2648             ld->cache = nv50_ir::CACHE_CG;
2649          else
2650             ld->cache = tgsi.getCacheMode();
2651          if (ind)
2652             ld->setIndirect(0, 1, ind);
2653       }
2654       break;
2655    default: {
2656       getImageCoords(off, 1);
2657       def.resize(4);
2658
2659       for (c = 0; c < 4; ++c) {
2660          if (!dst0[c] || tgsi.getSrc(0).getSwizzle(c) != (TGSI_SWIZZLE_X + c))
2661             def[c] = getScratch();
2662          else
2663             def[c] = dst0[c];
2664       }
2665
2666       bool bindless = tgsi.getSrc(0).getFile() != TGSI_FILE_IMAGE;
2667       if (bindless)
2668          ind = fetchSrc(0, 0);
2669
2670       TexInstruction *ld =
2671          mkTex(OP_SULDP, tgsi.getImageTarget(), 0, 0, def, off);
2672       ld->tex.mask = tgsi.getDst(0).getMask();
2673       ld->tex.format = tgsi.getImageFormat();
2674       ld->cache = tgsi.getCacheMode();
2675       ld->tex.bindless = bindless;
2676       if (!bindless)
2677          ld->tex.r = r;
2678       if (ind)
2679          ld->setIndirectR(ind);
2680
2681       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2682          if (dst0[c] != def[c])
2683             mkMov(dst0[c], def[tgsi.getSrc(0).getSwizzle(c)]);
2684       break;
2685    }
2686    }
2687
2688
2689 /* Keep this around for now as reference when adding img support
2690    getResourceCoords(off, r, 1);
2691
2692    if (isResourceRaw(code, r)) {
2693       uint8_t mask = 0;
2694       uint8_t comp[2] = { 0, 0 };
2695       uint8_t size[2] = { 0, 0 };
2696
2697       Symbol *base = getResourceBase(r);
2698
2699       // determine the base and size of the at most 2 load ops
2700       for (c = 0; c < 4; ++c)
2701          if (!tgsi.getDst(0).isMasked(c))
2702             mask |= 1 << (tgsi.getSrc(0).getSwizzle(c) - TGSI_SWIZZLE_X);
2703
2704       int n = partitionLoadStore(comp, size, mask);
2705
2706       src = off;
2707
2708       def.resize(4); // index by component, the ones we need will be non-NULL
2709       for (c = 0; c < 4; ++c) {
2710          if (dst0[c] && tgsi.getSrc(0).getSwizzle(c) == (TGSI_SWIZZLE_X + c))
2711             def[c] = dst0[c];
2712          else
2713          if (mask & (1 << c))
2714             def[c] = getScratch();
2715       }
2716
2717       const bool useLd = isResourceSpecial(r) ||
2718          (info->io.nv50styleSurfaces &&
2719           code->resources[r].target == TGSI_TEXTURE_BUFFER);
2720
2721       for (int i = 0; i < n; ++i) {
2722          ldv.assign(def.begin() + comp[i], def.begin() + comp[i] + size[i]);
2723
2724          if (comp[i]) // adjust x component of source address if necessary
2725             src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
2726                             off[0], mkImm(comp[i] * 4));
2727          else
2728             src[0] = off[0];
2729
2730          if (useLd) {
2731             Instruction *ld =
2732                mkLoad(typeOfSize(size[i] * 4), ldv[0], base, src[0]);
2733             for (size_t c = 1; c < ldv.size(); ++c)
2734                ld->setDef(c, ldv[c]);
2735          } else {
2736             mkTex(OP_SULDB, getResourceTarget(code, r), code->resources[r].slot,
2737                   0, ldv, src)->dType = typeOfSize(size[i] * 4);
2738          }
2739       }
2740    } else {
2741       def.resize(4);
2742       for (c = 0; c < 4; ++c) {
2743          if (!dst0[c] || tgsi.getSrc(0).getSwizzle(c) != (TGSI_SWIZZLE_X + c))
2744             def[c] = getScratch();
2745          else
2746             def[c] = dst0[c];
2747       }
2748
2749       mkTex(OP_SULDP, getResourceTarget(code, r), code->resources[r].slot, 0,
2750             def, off);
2751    }
2752    FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2753       if (dst0[c] != def[c])
2754          mkMov(dst0[c], def[tgsi.getSrc(0).getSwizzle(c)]);
2755 */
2756 }
2757
2758 // For formatted stores, the write mask on OP_SUSTP can be used.
2759 // Raw stores have to be split.
2760 void
2761 Converter::handleSTORE()
2762 {
2763    const int r = tgsi.getDst(0).getIndex(0);
2764    int c;
2765    std::vector<Value *> off, src, dummy;
2766    Value *ind = NULL;
2767
2768    if (tgsi.getDst(0).isIndirect(0))
2769       ind = fetchSrc(tgsi.getDst(0).getIndirect(0), 0, 0);
2770
2771    switch (tgsi.getDst(0).getFile()) {
2772    case TGSI_FILE_BUFFER:
2773    case TGSI_FILE_MEMORY:
2774       for (c = 0; c < 4; ++c) {
2775          if (!(tgsi.getDst(0).getMask() & (1 << c)))
2776             continue;
2777
2778          Symbol *sym;
2779          Value *off;
2780          if (tgsi.getSrc(0).getFile() == TGSI_FILE_IMMEDIATE) {
2781             off = NULL;
2782             sym = makeSym(tgsi.getDst(0).getFile(), r, -1, c,
2783                           tgsi.getSrc(0).getValueU32(0, info) + 4 * c);
2784          } else {
2785             // yzw are ignored for buffers
2786             off = fetchSrc(0, 0);
2787             sym = makeSym(tgsi.getDst(0).getFile(), r, -1, c, 4 * c);
2788          }
2789
2790          Instruction *st = mkStore(OP_STORE, TYPE_U32, sym, off, fetchSrc(1, c));
2791          st->cache = tgsi.getCacheMode();
2792          if (ind)
2793             st->setIndirect(0, 1, ind);
2794       }
2795       break;
2796    default: {
2797       getImageCoords(off, 0);
2798       src = off;
2799
2800       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2801          src.push_back(fetchSrc(1, c));
2802
2803       bool bindless = tgsi.getDst(0).getFile() != TGSI_FILE_IMAGE;
2804       if (bindless)
2805          ind = fetchDst(0, 0);
2806
2807       TexInstruction *st =
2808          mkTex(OP_SUSTP, tgsi.getImageTarget(), 0, 0, dummy, src);
2809       st->tex.mask = tgsi.getDst(0).getMask();
2810       st->tex.format = tgsi.getImageFormat();
2811       st->cache = tgsi.getCacheMode();
2812       st->tex.bindless = bindless;
2813       if (!bindless)
2814          st->tex.r = r;
2815       if (ind)
2816          st->setIndirectR(ind);
2817
2818       break;
2819    }
2820    }
2821
2822 /* Keep this around for now as reference when adding img support
2823    getResourceCoords(off, r, 0);
2824    src = off;
2825    const int s = src.size();
2826
2827    if (isResourceRaw(code, r)) {
2828       uint8_t comp[2] = { 0, 0 };
2829       uint8_t size[2] = { 0, 0 };
2830
2831       int n = partitionLoadStore(comp, size, tgsi.getDst(0).getMask());
2832
2833       Symbol *base = getResourceBase(r);
2834
2835       const bool useSt = isResourceSpecial(r) ||
2836          (info->io.nv50styleSurfaces &&
2837           code->resources[r].target == TGSI_TEXTURE_BUFFER);
2838
2839       for (int i = 0; i < n; ++i) {
2840          if (comp[i]) // adjust x component of source address if necessary
2841             src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
2842                             off[0], mkImm(comp[i] * 4));
2843          else
2844             src[0] = off[0];
2845
2846          const DataType stTy = typeOfSize(size[i] * 4);
2847
2848          if (useSt) {
2849             Instruction *st =
2850                mkStore(OP_STORE, stTy, base, NULL, fetchSrc(1, comp[i]));
2851             for (c = 1; c < size[i]; ++c)
2852                st->setSrc(1 + c, fetchSrc(1, comp[i] + c));
2853             st->setIndirect(0, 0, src[0]);
2854          } else {
2855             // attach values to be stored
2856             src.resize(s + size[i]);
2857             for (c = 0; c < size[i]; ++c)
2858                src[s + c] = fetchSrc(1, comp[i] + c);
2859             mkTex(OP_SUSTB, getResourceTarget(code, r), code->resources[r].slot,
2860                   0, dummy, src)->setType(stTy);
2861          }
2862       }
2863    } else {
2864       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2865          src.push_back(fetchSrc(1, c));
2866
2867       mkTex(OP_SUSTP, getResourceTarget(code, r), code->resources[r].slot, 0,
2868             dummy, src)->tex.mask = tgsi.getDst(0).getMask();
2869    }
2870 */
2871 }
2872
2873 // XXX: These only work on resources with the single-component u32/s32 formats.
2874 // Therefore the result is replicated. This might not be intended by TGSI, but
2875 // operating on more than 1 component would produce undefined results because
2876 // they do not exist.
2877 void
2878 Converter::handleATOM(Value *dst0[4], DataType ty, uint16_t subOp)
2879 {
2880    const int r = tgsi.getSrc(0).getIndex(0);
2881    std::vector<Value *> srcv;
2882    std::vector<Value *> defv;
2883    LValue *dst = getScratch();
2884    Value *ind = NULL;
2885
2886    if (tgsi.getSrc(0).isIndirect(0))
2887       ind = fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, 0);
2888
2889    switch (tgsi.getSrc(0).getFile()) {
2890    case TGSI_FILE_BUFFER:
2891    case TGSI_FILE_MEMORY:
2892       for (int c = 0; c < 4; ++c) {
2893          if (!dst0[c])
2894             continue;
2895
2896          Instruction *insn;
2897          Value *off = fetchSrc(1, c);
2898          Value *sym;
2899          if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE)
2900             sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c,
2901                           tgsi.getSrc(1).getValueU32(c, info));
2902          else
2903             sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c, 0);
2904          if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2905             insn = mkOp3(OP_ATOM, ty, dst, sym, fetchSrc(2, c), fetchSrc(3, c));
2906          else
2907             insn = mkOp2(OP_ATOM, ty, dst, sym, fetchSrc(2, c));
2908          if (tgsi.getSrc(1).getFile() != TGSI_FILE_IMMEDIATE)
2909             insn->setIndirect(0, 0, off);
2910          if (ind)
2911             insn->setIndirect(0, 1, ind);
2912          insn->subOp = subOp;
2913       }
2914       for (int c = 0; c < 4; ++c)
2915          if (dst0[c])
2916             dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov
2917       break;
2918    default: {
2919       getImageCoords(srcv, 1);
2920       defv.push_back(dst);
2921       srcv.push_back(fetchSrc(2, 0));
2922
2923       if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2924          srcv.push_back(fetchSrc(3, 0));
2925
2926       bool bindless = tgsi.getSrc(0).getFile() != TGSI_FILE_IMAGE;
2927       if (bindless)
2928          ind = fetchSrc(0, 0);
2929
2930       TexInstruction *tex = mkTex(OP_SUREDP, tgsi.getImageTarget(),
2931                                   0, 0, defv, srcv);
2932       tex->subOp = subOp;
2933       tex->tex.mask = 1;
2934       tex->tex.format = tgsi.getImageFormat();
2935       tex->setType(ty);
2936       tex->tex.bindless = bindless;
2937       if (!bindless)
2938          tex->tex.r = r;
2939       if (ind)
2940          tex->setIndirectR(ind);
2941
2942       for (int c = 0; c < 4; ++c)
2943          if (dst0[c])
2944             dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov
2945       break;
2946    }
2947    }
2948
2949 /* Keep this around for now as reference when adding img support
2950    getResourceCoords(srcv, r, 1);
2951
2952    if (isResourceSpecial(r)) {
2953       assert(r != TGSI_RESOURCE_INPUT);
2954       Instruction *insn;
2955       insn = mkOp2(OP_ATOM, ty, dst, getResourceBase(r), fetchSrc(2, 0));
2956       insn->subOp = subOp;
2957       if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2958          insn->setSrc(2, fetchSrc(3, 0));
2959       insn->setIndirect(0, 0, srcv.at(0));
2960    } else {
2961       operation op = isResourceRaw(code, r) ? OP_SUREDB : OP_SUREDP;
2962       TexTarget targ = getResourceTarget(code, r);
2963       int idx = code->resources[r].slot;
2964       defv.push_back(dst);
2965       srcv.push_back(fetchSrc(2, 0));
2966       if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2967          srcv.push_back(fetchSrc(3, 0));
2968       TexInstruction *tex = mkTex(op, targ, idx, 0, defv, srcv);
2969       tex->subOp = subOp;
2970       tex->tex.mask = 1;
2971       tex->setType(ty);
2972    }
2973
2974    for (int c = 0; c < 4; ++c)
2975       if (dst0[c])
2976          dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov
2977 */
2978 }
2979
2980 void
2981 Converter::handleINTERP(Value *dst[4])
2982 {
2983    // Check whether the input is linear. All other attributes ignored.
2984    Instruction *insn;
2985    Value *offset = NULL, *ptr = NULL, *w = NULL;
2986    Symbol *sym[4] = { NULL };
2987    bool linear;
2988    operation op = OP_NOP;
2989    int c, mode = 0;
2990
2991    tgsi::Instruction::SrcRegister src = tgsi.getSrc(0);
2992
2993    // In some odd cases, in large part due to varying packing, the source
2994    // might not actually be an input. This is illegal TGSI, but it's easier to
2995    // account for it here than it is to fix it where the TGSI is being
2996    // generated. In that case, it's going to be a straight up mov (or sequence
2997    // of mov's) from the input in question. We follow the mov chain to see
2998    // which input we need to use.
2999    if (src.getFile() != TGSI_FILE_INPUT) {
3000       if (src.isIndirect(0)) {
3001          ERROR("Ignoring indirect input interpolation\n");
3002          return;
3003       }
3004       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3005          Value *val = fetchSrc(0, c);
3006          assert(val->defs.size() == 1);
3007          insn = val->getInsn();
3008          while (insn->op == OP_MOV) {
3009             assert(insn->getSrc(0)->defs.size() == 1);
3010             insn = insn->getSrc(0)->getInsn();
3011             if (!insn) {
3012                ERROR("Miscompiling shader due to unhandled INTERP\n");
3013                return;
3014             }
3015          }
3016          if (insn->op != OP_LINTERP && insn->op != OP_PINTERP) {
3017             ERROR("Trying to interpolate non-input, this is not allowed.\n");
3018             return;
3019          }
3020          sym[c] = insn->getSrc(0)->asSym();
3021          assert(sym[c]);
3022          op = insn->op;
3023          mode = insn->ipa;
3024          ptr = insn->getIndirect(0, 0);
3025       }
3026    } else {
3027       if (src.isIndirect(0))
3028          ptr = shiftAddress(fetchSrc(src.getIndirect(0), 0, NULL));
3029
3030       // We can assume that the fixed index will point to an input of the same
3031       // interpolation type in case of an indirect.
3032       // TODO: Make use of ArrayID.
3033       linear = info->in[src.getIndex(0)].linear;
3034       if (linear) {
3035          op = OP_LINTERP;
3036          mode = NV50_IR_INTERP_LINEAR;
3037       } else {
3038          op = OP_PINTERP;
3039          mode = NV50_IR_INTERP_PERSPECTIVE;
3040       }
3041    }
3042
3043    switch (tgsi.getOpcode()) {
3044    case TGSI_OPCODE_INTERP_CENTROID:
3045       mode |= NV50_IR_INTERP_CENTROID;
3046       break;
3047    case TGSI_OPCODE_INTERP_SAMPLE:
3048       insn = mkOp1(OP_PIXLD, TYPE_U32, (offset = getScratch()), fetchSrc(1, 0));
3049       insn->subOp = NV50_IR_SUBOP_PIXLD_OFFSET;
3050       mode |= NV50_IR_INTERP_OFFSET;
3051       break;
3052    case TGSI_OPCODE_INTERP_OFFSET: {
3053       // The input in src1.xy is float, but we need a single 32-bit value
3054       // where the upper and lower 16 bits are encoded in S0.12 format. We need
3055       // to clamp the input coordinates to (-0.5, 0.4375), multiply by 4096,
3056       // and then convert to s32.
3057       Value *offs[2];
3058       for (c = 0; c < 2; c++) {
3059          offs[c] = getScratch();
3060          mkOp2(OP_MIN, TYPE_F32, offs[c], fetchSrc(1, c), loadImm(NULL, 0.4375f));
3061          mkOp2(OP_MAX, TYPE_F32, offs[c], offs[c], loadImm(NULL, -0.5f));
3062          mkOp2(OP_MUL, TYPE_F32, offs[c], offs[c], loadImm(NULL, 4096.0f));
3063          mkCvt(OP_CVT, TYPE_S32, offs[c], TYPE_F32, offs[c]);
3064       }
3065       offset = mkOp3v(OP_INSBF, TYPE_U32, getScratch(),
3066                       offs[1], mkImm(0x1010), offs[0]);
3067       mode |= NV50_IR_INTERP_OFFSET;
3068       break;
3069    }
3070    }
3071
3072    if (op == OP_PINTERP) {
3073       if (offset) {
3074          w = mkOp2v(OP_RDSV, TYPE_F32, getSSA(), mkSysVal(SV_POSITION, 3), offset);
3075          mkOp1(OP_RCP, TYPE_F32, w, w);
3076       } else {
3077          w = fragCoord[3];
3078       }
3079    }
3080
3081
3082    FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3083       insn = mkOp1(op, TYPE_F32, dst[c], sym[c] ? sym[c] : srcToSym(src, c));
3084       if (op == OP_PINTERP)
3085          insn->setSrc(1, w);
3086       if (offset)
3087          insn->setSrc(op == OP_PINTERP ? 2 : 1, offset);
3088       if (ptr)
3089          insn->setIndirect(0, 0, ptr);
3090
3091       insn->setInterpolate(mode);
3092    }
3093 }
3094
3095 bool
3096 Converter::isEndOfSubroutine(uint ip)
3097 {
3098    assert(ip < code->scan.num_instructions);
3099    tgsi::Instruction insn(&code->insns[ip]);
3100    return (insn.getOpcode() == TGSI_OPCODE_END ||
3101            insn.getOpcode() == TGSI_OPCODE_ENDSUB ||
3102            // does END occur at end of main or the very end ?
3103            insn.getOpcode() == TGSI_OPCODE_BGNSUB);
3104 }
3105
3106 bool
3107 Converter::handleInstruction(const struct tgsi_full_instruction *insn)
3108 {
3109    Instruction *geni;
3110
3111    Value *dst0[4], *rDst0[4];
3112    Value *src0, *src1, *src2, *src3;
3113    Value *val0, *val1;
3114    int c;
3115
3116    tgsi = tgsi::Instruction(insn);
3117
3118    bool useScratchDst = tgsi.checkDstSrcAliasing();
3119
3120    operation op = tgsi.getOP();
3121    dstTy = tgsi.inferDstType();
3122    srcTy = tgsi.inferSrcType();
3123
3124    unsigned int mask = tgsi.dstCount() ? tgsi.getDst(0).getMask() : 0;
3125
3126    if (tgsi.dstCount() && tgsi.getOpcode() != TGSI_OPCODE_STORE) {
3127       for (c = 0; c < 4; ++c) {
3128          rDst0[c] = acquireDst(0, c);
3129          dst0[c] = (useScratchDst && rDst0[c]) ? getScratch() : rDst0[c];
3130       }
3131    }
3132
3133    switch (tgsi.getOpcode()) {
3134    case TGSI_OPCODE_ADD:
3135    case TGSI_OPCODE_UADD:
3136    case TGSI_OPCODE_AND:
3137    case TGSI_OPCODE_DIV:
3138    case TGSI_OPCODE_IDIV:
3139    case TGSI_OPCODE_UDIV:
3140    case TGSI_OPCODE_MAX:
3141    case TGSI_OPCODE_MIN:
3142    case TGSI_OPCODE_IMAX:
3143    case TGSI_OPCODE_IMIN:
3144    case TGSI_OPCODE_UMAX:
3145    case TGSI_OPCODE_UMIN:
3146    case TGSI_OPCODE_MOD:
3147    case TGSI_OPCODE_UMOD:
3148    case TGSI_OPCODE_MUL:
3149    case TGSI_OPCODE_UMUL:
3150    case TGSI_OPCODE_IMUL_HI:
3151    case TGSI_OPCODE_UMUL_HI:
3152    case TGSI_OPCODE_OR:
3153    case TGSI_OPCODE_SHL:
3154    case TGSI_OPCODE_ISHR:
3155    case TGSI_OPCODE_USHR:
3156    case TGSI_OPCODE_XOR:
3157       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3158          src0 = fetchSrc(0, c);
3159          src1 = fetchSrc(1, c);
3160          geni = mkOp2(op, dstTy, dst0[c], src0, src1);
3161          geni->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode());
3162          if (op == OP_MUL && dstTy == TYPE_F32)
3163             geni->dnz = info->io.mul_zero_wins;
3164          geni->precise = insn->Instruction.Precise;
3165       }
3166       break;
3167    case TGSI_OPCODE_MAD:
3168    case TGSI_OPCODE_UMAD:
3169    case TGSI_OPCODE_FMA:
3170       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3171          src0 = fetchSrc(0, c);
3172          src1 = fetchSrc(1, c);
3173          src2 = fetchSrc(2, c);
3174          geni = mkOp3(op, dstTy, dst0[c], src0, src1, src2);
3175          if (dstTy == TYPE_F32)
3176             geni->dnz = info->io.mul_zero_wins;
3177          geni->precise = insn->Instruction.Precise;
3178       }
3179       break;
3180    case TGSI_OPCODE_MOV:
3181    case TGSI_OPCODE_CEIL:
3182    case TGSI_OPCODE_FLR:
3183    case TGSI_OPCODE_TRUNC:
3184    case TGSI_OPCODE_RCP:
3185    case TGSI_OPCODE_SQRT:
3186    case TGSI_OPCODE_IABS:
3187    case TGSI_OPCODE_INEG:
3188    case TGSI_OPCODE_NOT:
3189    case TGSI_OPCODE_DDX:
3190    case TGSI_OPCODE_DDY:
3191    case TGSI_OPCODE_DDX_FINE:
3192    case TGSI_OPCODE_DDY_FINE:
3193       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3194          mkOp1(op, dstTy, dst0[c], fetchSrc(0, c));
3195       break;
3196    case TGSI_OPCODE_RSQ:
3197       src0 = fetchSrc(0, 0);
3198       val0 = getScratch();
3199       mkOp1(OP_ABS, TYPE_F32, val0, src0);
3200       mkOp1(OP_RSQ, TYPE_F32, val0, val0);
3201       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3202          mkMov(dst0[c], val0);
3203       break;
3204    case TGSI_OPCODE_ARL:
3205    case TGSI_OPCODE_ARR:
3206       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3207          const RoundMode rnd =
3208             tgsi.getOpcode() == TGSI_OPCODE_ARR ? ROUND_N : ROUND_M;
3209          src0 = fetchSrc(0, c);
3210          mkCvt(OP_CVT, TYPE_S32, dst0[c], TYPE_F32, src0)->rnd = rnd;
3211       }
3212       break;
3213    case TGSI_OPCODE_UARL:
3214       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3215          mkOp1(OP_MOV, TYPE_U32, dst0[c], fetchSrc(0, c));
3216       break;
3217    case TGSI_OPCODE_POW:
3218       val0 = mkOp2v(op, TYPE_F32, getScratch(), fetchSrc(0, 0), fetchSrc(1, 0));
3219       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3220          mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
3221       break;
3222    case TGSI_OPCODE_EX2:
3223    case TGSI_OPCODE_LG2:
3224       val0 = mkOp1(op, TYPE_F32, getScratch(), fetchSrc(0, 0))->getDef(0);
3225       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3226          mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
3227       break;
3228    case TGSI_OPCODE_COS:
3229    case TGSI_OPCODE_SIN:
3230       val0 = getScratch();
3231       if (mask & 7) {
3232          mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 0));
3233          mkOp1(op, TYPE_F32, val0, val0);
3234          for (c = 0; c < 3; ++c)
3235             if (dst0[c])
3236                mkMov(dst0[c], val0);
3237       }
3238       if (dst0[3]) {
3239          mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 3));
3240          mkOp1(op, TYPE_F32, dst0[3], val0);
3241       }
3242       break;
3243    case TGSI_OPCODE_EXP:
3244       src0 = fetchSrc(0, 0);
3245       val0 = mkOp1v(OP_FLOOR, TYPE_F32, getSSA(), src0);
3246       if (dst0[1])
3247          mkOp2(OP_SUB, TYPE_F32, dst0[1], src0, val0);
3248       if (dst0[0])
3249          mkOp1(OP_EX2, TYPE_F32, dst0[0], val0);
3250       if (dst0[2])
3251          mkOp1(OP_EX2, TYPE_F32, dst0[2], src0);
3252       if (dst0[3])
3253          loadImm(dst0[3], 1.0f);
3254       break;
3255    case TGSI_OPCODE_LOG:
3256       src0 = mkOp1v(OP_ABS, TYPE_F32, getSSA(), fetchSrc(0, 0));
3257       val0 = mkOp1v(OP_LG2, TYPE_F32, dst0[2] ? dst0[2] : getSSA(), src0);
3258       if (dst0[0] || dst0[1])
3259          val1 = mkOp1v(OP_FLOOR, TYPE_F32, dst0[0] ? dst0[0] : getSSA(), val0);
3260       if (dst0[1]) {
3261          mkOp1(OP_EX2, TYPE_F32, dst0[1], val1);
3262          mkOp1(OP_RCP, TYPE_F32, dst0[1], dst0[1]);
3263          mkOp2(OP_MUL, TYPE_F32, dst0[1], dst0[1], src0)
3264             ->dnz = info->io.mul_zero_wins;
3265       }
3266       if (dst0[3])
3267          loadImm(dst0[3], 1.0f);
3268       break;
3269    case TGSI_OPCODE_DP2:
3270       val0 = buildDot(2);
3271       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3272          mkMov(dst0[c], val0);
3273       break;
3274    case TGSI_OPCODE_DP3:
3275       val0 = buildDot(3);
3276       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3277          mkMov(dst0[c], val0);
3278       break;
3279    case TGSI_OPCODE_DP4:
3280       val0 = buildDot(4);
3281       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3282          mkMov(dst0[c], val0);
3283       break;
3284    case TGSI_OPCODE_DST:
3285       if (dst0[0])
3286          loadImm(dst0[0], 1.0f);
3287       if (dst0[1]) {
3288          src0 = fetchSrc(0, 1);
3289          src1 = fetchSrc(1, 1);
3290          mkOp2(OP_MUL, TYPE_F32, dst0[1], src0, src1)
3291             ->dnz = info->io.mul_zero_wins;
3292       }
3293       if (dst0[2])
3294          mkMov(dst0[2], fetchSrc(0, 2));
3295       if (dst0[3])
3296          mkMov(dst0[3], fetchSrc(1, 3));
3297       break;
3298    case TGSI_OPCODE_LRP:
3299       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3300          src0 = fetchSrc(0, c);
3301          src1 = fetchSrc(1, c);
3302          src2 = fetchSrc(2, c);
3303          mkOp3(OP_MAD, TYPE_F32, dst0[c],
3304                mkOp2v(OP_SUB, TYPE_F32, getSSA(), src1, src2), src0, src2)
3305             ->dnz = info->io.mul_zero_wins;
3306       }
3307       break;
3308    case TGSI_OPCODE_LIT:
3309       handleLIT(dst0);
3310       break;
3311    case TGSI_OPCODE_ISSG:
3312    case TGSI_OPCODE_SSG:
3313       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3314          src0 = fetchSrc(0, c);
3315          val0 = getScratch();
3316          val1 = getScratch();
3317          mkCmp(OP_SET, CC_GT, srcTy, val0, srcTy, src0, zero);
3318          mkCmp(OP_SET, CC_LT, srcTy, val1, srcTy, src0, zero);
3319          if (srcTy == TYPE_F32)
3320             mkOp2(OP_SUB, TYPE_F32, dst0[c], val0, val1);
3321          else
3322             mkOp2(OP_SUB, TYPE_S32, dst0[c], val1, val0);
3323       }
3324       break;
3325    case TGSI_OPCODE_UCMP:
3326       srcTy = TYPE_U32;
3327       /* fallthrough */
3328    case TGSI_OPCODE_CMP:
3329       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3330          src0 = fetchSrc(0, c);
3331          src1 = fetchSrc(1, c);
3332          src2 = fetchSrc(2, c);
3333          if (src1 == src2)
3334             mkMov(dst0[c], src1);
3335          else
3336             mkCmp(OP_SLCT, (srcTy == TYPE_F32) ? CC_LT : CC_NE,
3337                   srcTy, dst0[c], srcTy, src1, src2, src0);
3338       }
3339       break;
3340    case TGSI_OPCODE_FRC:
3341       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3342          src0 = fetchSrc(0, c);
3343          val0 = getScratch();
3344          mkOp1(OP_FLOOR, TYPE_F32, val0, src0);
3345          mkOp2(OP_SUB, TYPE_F32, dst0[c], src0, val0);
3346       }
3347       break;
3348    case TGSI_OPCODE_ROUND:
3349       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3350          mkCvt(OP_CVT, TYPE_F32, dst0[c], TYPE_F32, fetchSrc(0, c))
3351          ->rnd = ROUND_NI;
3352       break;
3353    case TGSI_OPCODE_SLT:
3354    case TGSI_OPCODE_SGE:
3355    case TGSI_OPCODE_SEQ:
3356    case TGSI_OPCODE_SGT:
3357    case TGSI_OPCODE_SLE:
3358    case TGSI_OPCODE_SNE:
3359    case TGSI_OPCODE_FSEQ:
3360    case TGSI_OPCODE_FSGE:
3361    case TGSI_OPCODE_FSLT:
3362    case TGSI_OPCODE_FSNE:
3363    case TGSI_OPCODE_ISGE:
3364    case TGSI_OPCODE_ISLT:
3365    case TGSI_OPCODE_USEQ:
3366    case TGSI_OPCODE_USGE:
3367    case TGSI_OPCODE_USLT:
3368    case TGSI_OPCODE_USNE:
3369       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3370          src0 = fetchSrc(0, c);
3371          src1 = fetchSrc(1, c);
3372          mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], srcTy, src0, src1);
3373       }
3374       break;
3375    case TGSI_OPCODE_VOTE_ALL:
3376    case TGSI_OPCODE_VOTE_ANY:
3377    case TGSI_OPCODE_VOTE_EQ:
3378       val0 = new_LValue(func, FILE_PREDICATE);
3379       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3380          mkCmp(OP_SET, CC_NE, TYPE_U32, val0, TYPE_U32, fetchSrc(0, c), zero);
3381          mkOp1(op, dstTy, val0, val0)
3382             ->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode());
3383          mkCvt(OP_CVT, TYPE_U32, dst0[c], TYPE_U8, val0);
3384       }
3385       break;
3386    case TGSI_OPCODE_BALLOT:
3387       if (!tgsi.getDst(0).isMasked(0)) {
3388          val0 = new_LValue(func, FILE_PREDICATE);
3389          mkCmp(OP_SET, CC_NE, TYPE_U32, val0, TYPE_U32, fetchSrc(0, 0), zero);
3390          mkOp1(op, TYPE_U32, dst0[0], val0)->subOp = NV50_IR_SUBOP_VOTE_ANY;
3391       }
3392       if (!tgsi.getDst(0).isMasked(1))
3393          mkMov(dst0[1], zero, TYPE_U32);
3394       break;
3395    case TGSI_OPCODE_READ_FIRST:
3396       // ReadFirstInvocationARB(src) is implemented as
3397       // ReadInvocationARB(src, findLSB(ballot(true)))
3398       val0 = getScratch();
3399       mkOp1(OP_VOTE, TYPE_U32, val0, mkImm(1))->subOp = NV50_IR_SUBOP_VOTE_ANY;
3400       mkOp2(OP_EXTBF, TYPE_U32, val0, val0, mkImm(0x2000))
3401          ->subOp = NV50_IR_SUBOP_EXTBF_REV;
3402       mkOp1(OP_BFIND, TYPE_U32, val0, val0)->subOp = NV50_IR_SUBOP_BFIND_SAMT;
3403       src1 = val0;
3404       /* fallthrough */
3405    case TGSI_OPCODE_READ_INVOC:
3406       if (tgsi.getOpcode() == TGSI_OPCODE_READ_INVOC)
3407          src1 = fetchSrc(1, 0);
3408       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3409          geni = mkOp3(op, dstTy, dst0[c], fetchSrc(0, c), src1, mkImm(0x1f));
3410          geni->subOp = NV50_IR_SUBOP_SHFL_IDX;
3411       }
3412       break;
3413    case TGSI_OPCODE_CLOCK:
3414       // Stick the 32-bit clock into the high dword of the logical result.
3415       if (!tgsi.getDst(0).isMasked(0))
3416          mkOp1(OP_MOV, TYPE_U32, dst0[0], zero);
3417       if (!tgsi.getDst(0).isMasked(1))
3418          mkOp1(OP_RDSV, TYPE_U32, dst0[1], mkSysVal(SV_CLOCK, 0))->fixed = 1;
3419       break;
3420    case TGSI_OPCODE_READ_HELPER:
3421       if (!tgsi.getDst(0).isMasked(0))
3422          mkOp1(OP_RDSV, TYPE_U32, dst0[0], mkSysVal(SV_THREAD_KILL, 0))
3423             ->fixed = 1;
3424       break;
3425    case TGSI_OPCODE_KILL_IF:
3426       val0 = new_LValue(func, FILE_PREDICATE);
3427       mask = 0;
3428       for (c = 0; c < 4; ++c) {
3429          const int s = tgsi.getSrc(0).getSwizzle(c);
3430          if (mask & (1 << s))
3431             continue;
3432          mask |= 1 << s;
3433          mkCmp(OP_SET, CC_LT, TYPE_F32, val0, TYPE_F32, fetchSrc(0, c), zero);
3434          mkOp(OP_DISCARD, TYPE_NONE, NULL)->setPredicate(CC_P, val0);
3435       }
3436       break;
3437    case TGSI_OPCODE_KILL:
3438    case TGSI_OPCODE_DEMOTE:
3439       // TODO: Should we make KILL exit that invocation? Some old shaders
3440       // don't like that.
3441       mkOp(OP_DISCARD, TYPE_NONE, NULL);
3442       break;
3443    case TGSI_OPCODE_TEX:
3444    case TGSI_OPCODE_TEX_LZ:
3445    case TGSI_OPCODE_TXB:
3446    case TGSI_OPCODE_TXL:
3447    case TGSI_OPCODE_TXP:
3448    case TGSI_OPCODE_LODQ:
3449       //              R  S     L     C    Dx    Dy
3450       handleTEX(dst0, 1, 1, 0x03, 0x0f, 0x00, 0x00);
3451       break;
3452    case TGSI_OPCODE_TXD:
3453       handleTEX(dst0, 3, 3, 0x03, 0x0f, 0x10, 0x20);
3454       break;
3455    case TGSI_OPCODE_TG4:
3456       handleTEX(dst0, 2, 2, 0x03, 0x0f, 0x00, 0x00);
3457       break;
3458    case TGSI_OPCODE_TEX2:
3459       handleTEX(dst0, 2, 2, 0x03, 0x10, 0x00, 0x00);
3460       break;
3461    case TGSI_OPCODE_TXB2:
3462    case TGSI_OPCODE_TXL2:
3463       handleTEX(dst0, 2, 2, 0x10, 0x0f, 0x00, 0x00);
3464       break;
3465    case TGSI_OPCODE_SAMPLE:
3466    case TGSI_OPCODE_SAMPLE_B:
3467    case TGSI_OPCODE_SAMPLE_D:
3468    case TGSI_OPCODE_SAMPLE_L:
3469    case TGSI_OPCODE_SAMPLE_C:
3470    case TGSI_OPCODE_SAMPLE_C_LZ:
3471       handleTEX(dst0, 1, 2, 0x30, 0x30, 0x30, 0x40);
3472       break;
3473    case TGSI_OPCODE_TXF_LZ:
3474    case TGSI_OPCODE_TXF:
3475       handleTXF(dst0, 1, 0x03);
3476       break;
3477    case TGSI_OPCODE_SAMPLE_I:
3478       handleTXF(dst0, 1, 0x03);
3479       break;
3480    case TGSI_OPCODE_SAMPLE_I_MS:
3481       handleTXF(dst0, 1, 0x20);
3482       break;
3483    case TGSI_OPCODE_TXQ:
3484    case TGSI_OPCODE_SVIEWINFO:
3485       handleTXQ(dst0, TXQ_DIMS, 1);
3486       break;
3487    case TGSI_OPCODE_TXQS:
3488       // The TXQ_TYPE query returns samples in its 3rd arg, but we need it to
3489       // be in .x
3490       dst0[1] = dst0[2] = dst0[3] = NULL;
3491       std::swap(dst0[0], dst0[2]);
3492       handleTXQ(dst0, TXQ_TYPE, 0);
3493       std::swap(dst0[0], dst0[2]);
3494       break;
3495    case TGSI_OPCODE_FBFETCH:
3496       handleFBFETCH(dst0);
3497       break;
3498    case TGSI_OPCODE_F2I:
3499    case TGSI_OPCODE_F2U:
3500       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3501          mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c))->rnd = ROUND_Z;
3502       break;
3503    case TGSI_OPCODE_I2F:
3504    case TGSI_OPCODE_U2F:
3505       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3506          mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c));
3507       break;
3508    case TGSI_OPCODE_PK2H:
3509       val0 = getScratch();
3510       val1 = getScratch();
3511       mkCvt(OP_CVT, TYPE_F16, val0, TYPE_F32, fetchSrc(0, 0));
3512       mkCvt(OP_CVT, TYPE_F16, val1, TYPE_F32, fetchSrc(0, 1));
3513       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3514          mkOp3(OP_INSBF, TYPE_U32, dst0[c], val1, mkImm(0x1010), val0);
3515       break;
3516    case TGSI_OPCODE_UP2H:
3517       src0 = fetchSrc(0, 0);
3518       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3519          geni = mkCvt(OP_CVT, TYPE_F32, dst0[c], TYPE_F16, src0);
3520          geni->subOp = c & 1;
3521       }
3522       break;
3523    case TGSI_OPCODE_EMIT:
3524       /* export the saved viewport index */
3525       if (viewport != NULL) {
3526          Symbol *vpSym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_U32,
3527                                   info->out[info->io.viewportId].slot[0] * 4);
3528          mkStore(OP_EXPORT, TYPE_U32, vpSym, NULL, viewport);
3529       }
3530       /* handle user clip planes for each emitted vertex */
3531       if (info->io.genUserClip > 0)
3532          handleUserClipPlanes();
3533       /* fallthrough */
3534    case TGSI_OPCODE_ENDPRIM:
3535    {
3536       // get vertex stream (must be immediate)
3537       unsigned int stream = tgsi.getSrc(0).getValueU32(0, info);
3538       if (stream && op == OP_RESTART)
3539          break;
3540       if (info->prop.gp.maxVertices == 0)
3541          break;
3542       src0 = mkImm(stream);
3543       mkOp1(op, TYPE_U32, NULL, src0)->fixed = 1;
3544       break;
3545    }
3546    case TGSI_OPCODE_IF:
3547    case TGSI_OPCODE_UIF:
3548    {
3549       BasicBlock *ifBB = new BasicBlock(func);
3550
3551       bb->cfg.attach(&ifBB->cfg, Graph::Edge::TREE);
3552       condBBs.push(bb);
3553       joinBBs.push(bb);
3554
3555       mkFlow(OP_BRA, NULL, CC_NOT_P, fetchSrc(0, 0))->setType(srcTy);
3556
3557       setPosition(ifBB, true);
3558    }
3559       break;
3560    case TGSI_OPCODE_ELSE:
3561    {
3562       BasicBlock *elseBB = new BasicBlock(func);
3563       BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
3564
3565       forkBB->cfg.attach(&elseBB->cfg, Graph::Edge::TREE);
3566       condBBs.push(bb);
3567
3568       forkBB->getExit()->asFlow()->target.bb = elseBB;
3569       if (!bb->isTerminated())
3570          mkFlow(OP_BRA, NULL, CC_ALWAYS, NULL);
3571
3572       setPosition(elseBB, true);
3573    }
3574       break;
3575    case TGSI_OPCODE_ENDIF:
3576    {
3577       BasicBlock *convBB = new BasicBlock(func);
3578       BasicBlock *prevBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
3579       BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(joinBBs.pop().u.p);
3580
3581       if (!bb->isTerminated()) {
3582          // we only want join if none of the clauses ended with CONT/BREAK/RET
3583          if (prevBB->getExit()->op == OP_BRA && joinBBs.getSize() < 6)
3584             insertConvergenceOps(convBB, forkBB);
3585          mkFlow(OP_BRA, convBB, CC_ALWAYS, NULL);
3586          bb->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
3587       }
3588
3589       if (prevBB->getExit()->op == OP_BRA) {
3590          prevBB->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
3591          prevBB->getExit()->asFlow()->target.bb = convBB;
3592       }
3593       setPosition(convBB, true);
3594    }
3595       break;
3596    case TGSI_OPCODE_BGNLOOP:
3597    {
3598       BasicBlock *lbgnBB = new BasicBlock(func);
3599       BasicBlock *lbrkBB = new BasicBlock(func);
3600
3601       loopBBs.push(lbgnBB);
3602       breakBBs.push(lbrkBB);
3603       if (loopBBs.getSize() > func->loopNestingBound)
3604          func->loopNestingBound++;
3605
3606       mkFlow(OP_PREBREAK, lbrkBB, CC_ALWAYS, NULL);
3607
3608       bb->cfg.attach(&lbgnBB->cfg, Graph::Edge::TREE);
3609       setPosition(lbgnBB, true);
3610       mkFlow(OP_PRECONT, lbgnBB, CC_ALWAYS, NULL);
3611    }
3612       break;
3613    case TGSI_OPCODE_ENDLOOP:
3614    {
3615       BasicBlock *loopBB = reinterpret_cast<BasicBlock *>(loopBBs.pop().u.p);
3616
3617       if (!bb->isTerminated()) {
3618          mkFlow(OP_CONT, loopBB, CC_ALWAYS, NULL);
3619          bb->cfg.attach(&loopBB->cfg, Graph::Edge::BACK);
3620       }
3621       setPosition(reinterpret_cast<BasicBlock *>(breakBBs.pop().u.p), true);
3622
3623       // If the loop never breaks (e.g. only has RET's inside), then there
3624       // will be no way to get to the break bb. However BGNLOOP will have
3625       // already made a PREBREAK to it, so it must be in the CFG.
3626       if (getBB()->cfg.incidentCount() == 0)
3627          loopBB->cfg.attach(&getBB()->cfg, Graph::Edge::TREE);
3628    }
3629       break;
3630    case TGSI_OPCODE_BRK:
3631    {
3632       if (bb->isTerminated())
3633          break;
3634       BasicBlock *brkBB = reinterpret_cast<BasicBlock *>(breakBBs.peek().u.p);
3635       mkFlow(OP_BREAK, brkBB, CC_ALWAYS, NULL);
3636       bb->cfg.attach(&brkBB->cfg, Graph::Edge::CROSS);
3637    }
3638       break;
3639    case TGSI_OPCODE_CONT:
3640    {
3641       if (bb->isTerminated())
3642          break;
3643       BasicBlock *contBB = reinterpret_cast<BasicBlock *>(loopBBs.peek().u.p);
3644       mkFlow(OP_CONT, contBB, CC_ALWAYS, NULL);
3645       contBB->explicitCont = true;
3646       bb->cfg.attach(&contBB->cfg, Graph::Edge::BACK);
3647    }
3648       break;
3649    case TGSI_OPCODE_BGNSUB:
3650    {
3651       Subroutine *s = getSubroutine(ip);
3652       BasicBlock *entry = new BasicBlock(s->f);
3653       BasicBlock *leave = new BasicBlock(s->f);
3654
3655       // multiple entrypoints possible, keep the graph connected
3656       if (prog->getType() == Program::TYPE_COMPUTE)
3657          prog->main->call.attach(&s->f->call, Graph::Edge::TREE);
3658
3659       sub.cur = s;
3660       s->f->setEntry(entry);
3661       s->f->setExit(leave);
3662       setPosition(entry, true);
3663       return true;
3664    }
3665    case TGSI_OPCODE_ENDSUB:
3666    {
3667       sub.cur = getSubroutine(prog->main);
3668       setPosition(BasicBlock::get(sub.cur->f->cfg.getRoot()), true);
3669       return true;
3670    }
3671    case TGSI_OPCODE_CAL:
3672    {
3673       Subroutine *s = getSubroutine(tgsi.getLabel());
3674       mkFlow(OP_CALL, s->f, CC_ALWAYS, NULL);
3675       func->call.attach(&s->f->call, Graph::Edge::TREE);
3676       return true;
3677    }
3678    case TGSI_OPCODE_RET:
3679    {
3680       if (bb->isTerminated())
3681          return true;
3682       BasicBlock *leave = BasicBlock::get(func->cfgExit);
3683
3684       if (!isEndOfSubroutine(ip + 1)) {
3685          // insert a PRERET at the entry if this is an early return
3686          // (only needed for sharing code in the epilogue)
3687          BasicBlock *root = BasicBlock::get(func->cfg.getRoot());
3688          if (root->getEntry() == NULL || root->getEntry()->op != OP_PRERET) {
3689             BasicBlock *pos = getBB();
3690             setPosition(root, false);
3691             mkFlow(OP_PRERET, leave, CC_ALWAYS, NULL)->fixed = 1;
3692             setPosition(pos, true);
3693          }
3694       }
3695       mkFlow(OP_RET, NULL, CC_ALWAYS, NULL)->fixed = 1;
3696       bb->cfg.attach(&leave->cfg, Graph::Edge::CROSS);
3697    }
3698       break;
3699    case TGSI_OPCODE_END:
3700    {
3701       // attach and generate epilogue code
3702       BasicBlock *epilogue = BasicBlock::get(func->cfgExit);
3703       bb->cfg.attach(&epilogue->cfg, Graph::Edge::TREE);
3704       setPosition(epilogue, true);
3705       if (prog->getType() == Program::TYPE_FRAGMENT)
3706          exportOutputs();
3707       if ((prog->getType() == Program::TYPE_VERTEX ||
3708            prog->getType() == Program::TYPE_TESSELLATION_EVAL
3709           ) && info->io.genUserClip > 0)
3710          handleUserClipPlanes();
3711       mkOp(OP_EXIT, TYPE_NONE, NULL)->terminator = 1;
3712    }
3713       break;
3714    case TGSI_OPCODE_SWITCH:
3715    case TGSI_OPCODE_CASE:
3716       ERROR("switch/case opcode encountered, should have been lowered\n");
3717       abort();
3718       break;
3719    case TGSI_OPCODE_LOAD:
3720       handleLOAD(dst0);
3721       break;
3722    case TGSI_OPCODE_STORE:
3723       handleSTORE();
3724       break;
3725    case TGSI_OPCODE_BARRIER:
3726       geni = mkOp2(OP_BAR, TYPE_U32, NULL, mkImm(0), mkImm(0));
3727       geni->fixed = 1;
3728       geni->subOp = NV50_IR_SUBOP_BAR_SYNC;
3729       break;
3730    case TGSI_OPCODE_MEMBAR:
3731    {
3732       uint32_t level = tgsi.getSrc(0).getValueU32(0, info);
3733       geni = mkOp(OP_MEMBAR, TYPE_NONE, NULL);
3734       geni->fixed = 1;
3735       if (!(level & ~(TGSI_MEMBAR_THREAD_GROUP | TGSI_MEMBAR_SHARED)))
3736          geni->subOp = NV50_IR_SUBOP_MEMBAR(M, CTA);
3737       else
3738          geni->subOp = NV50_IR_SUBOP_MEMBAR(M, GL);
3739    }
3740       break;
3741    case TGSI_OPCODE_ATOMUADD:
3742    case TGSI_OPCODE_ATOMXCHG:
3743    case TGSI_OPCODE_ATOMCAS:
3744    case TGSI_OPCODE_ATOMAND:
3745    case TGSI_OPCODE_ATOMOR:
3746    case TGSI_OPCODE_ATOMXOR:
3747    case TGSI_OPCODE_ATOMUMIN:
3748    case TGSI_OPCODE_ATOMIMIN:
3749    case TGSI_OPCODE_ATOMUMAX:
3750    case TGSI_OPCODE_ATOMIMAX:
3751    case TGSI_OPCODE_ATOMFADD:
3752    case TGSI_OPCODE_ATOMDEC_WRAP:
3753    case TGSI_OPCODE_ATOMINC_WRAP:
3754       handleATOM(dst0, dstTy, tgsi::opcodeToSubOp(tgsi.getOpcode()));
3755       break;
3756    case TGSI_OPCODE_RESQ:
3757       if (tgsi.getSrc(0).getFile() == TGSI_FILE_BUFFER) {
3758          Value *ind = NULL;
3759          if (tgsi.getSrc(0).isIndirect(0))
3760             ind = fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, 0);
3761          geni = mkOp1(OP_BUFQ, TYPE_U32, dst0[0],
3762                       makeSym(tgsi.getSrc(0).getFile(),
3763                               tgsi.getSrc(0).getIndex(0), -1, 0, 0));
3764          if (ind)
3765             geni->setIndirect(0, 1, ind);
3766       } else {
3767          TexInstruction *texi = new_TexInstruction(func, OP_SUQ);
3768          for (int c = 0, d = 0; c < 4; ++c) {
3769             if (dst0[c]) {
3770                texi->setDef(d++, dst0[c]);
3771                texi->tex.mask |= 1 << c;
3772             }
3773          }
3774          if (tgsi.getSrc(0).getFile() == TGSI_FILE_IMAGE) {
3775             texi->tex.r = tgsi.getSrc(0).getIndex(0);
3776             if (tgsi.getSrc(0).isIndirect(0))
3777                texi->setIndirectR(fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, NULL));
3778          } else {
3779             texi->tex.bindless = true;
3780             texi->setIndirectR(fetchSrc(0, 0));
3781          }
3782          texi->tex.target = tgsi.getImageTarget();
3783
3784          bb->insertTail(texi);
3785       }
3786       break;
3787    case TGSI_OPCODE_IBFE:
3788    case TGSI_OPCODE_UBFE:
3789       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3790          src0 = fetchSrc(0, c);
3791          val0 = getScratch();
3792          if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE &&
3793              tgsi.getSrc(2).getFile() == TGSI_FILE_IMMEDIATE) {
3794             loadImm(val0, (tgsi.getSrc(2).getValueU32(c, info) << 8) |
3795                     tgsi.getSrc(1).getValueU32(c, info));
3796          } else {
3797             src1 = fetchSrc(1, c);
3798             src2 = fetchSrc(2, c);
3799             mkOp3(OP_INSBF, TYPE_U32, val0, src2, mkImm(0x808), src1);
3800          }
3801          mkOp2(OP_EXTBF, dstTy, dst0[c], src0, val0);
3802       }
3803       break;
3804    case TGSI_OPCODE_BFI:
3805       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3806          src0 = fetchSrc(0, c);
3807          src1 = fetchSrc(1, c);
3808          src2 = fetchSrc(2, c);
3809          src3 = fetchSrc(3, c);
3810          val0 = getScratch();
3811          mkOp3(OP_INSBF, TYPE_U32, val0, src3, mkImm(0x808), src2);
3812          mkOp3(OP_INSBF, TYPE_U32, dst0[c], src1, val0, src0);
3813       }
3814       break;
3815    case TGSI_OPCODE_LSB:
3816       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3817          src0 = fetchSrc(0, c);
3818          val0 = getScratch();
3819          geni = mkOp2(OP_EXTBF, TYPE_U32, val0, src0, mkImm(0x2000));
3820          geni->subOp = NV50_IR_SUBOP_EXTBF_REV;
3821          geni = mkOp1(OP_BFIND, TYPE_U32, dst0[c], val0);
3822          geni->subOp = NV50_IR_SUBOP_BFIND_SAMT;
3823       }
3824       break;
3825    case TGSI_OPCODE_IMSB:
3826    case TGSI_OPCODE_UMSB:
3827       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3828          src0 = fetchSrc(0, c);
3829          mkOp1(OP_BFIND, srcTy, dst0[c], src0);
3830       }
3831       break;
3832    case TGSI_OPCODE_BREV:
3833       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3834          src0 = fetchSrc(0, c);
3835          geni = mkOp2(OP_EXTBF, TYPE_U32, dst0[c], src0, mkImm(0x2000));
3836          geni->subOp = NV50_IR_SUBOP_EXTBF_REV;
3837       }
3838       break;
3839    case TGSI_OPCODE_POPC:
3840       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3841          src0 = fetchSrc(0, c);
3842          mkOp2(OP_POPCNT, TYPE_U32, dst0[c], src0, src0);
3843       }
3844       break;
3845    case TGSI_OPCODE_INTERP_CENTROID:
3846    case TGSI_OPCODE_INTERP_SAMPLE:
3847    case TGSI_OPCODE_INTERP_OFFSET:
3848       handleINTERP(dst0);
3849       break;
3850    case TGSI_OPCODE_I642F:
3851    case TGSI_OPCODE_U642F:
3852    case TGSI_OPCODE_D2I:
3853    case TGSI_OPCODE_D2U:
3854    case TGSI_OPCODE_D2F: {
3855       int pos = 0;
3856       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3857          Value *dreg = getSSA(8);
3858          src0 = fetchSrc(0, pos);
3859          src1 = fetchSrc(0, pos + 1);
3860          mkOp2(OP_MERGE, TYPE_U64, dreg, src0, src1);
3861          Instruction *cvt = mkCvt(OP_CVT, dstTy, dst0[c], srcTy, dreg);
3862          if (!isFloatType(dstTy))
3863             cvt->rnd = ROUND_Z;
3864          pos += 2;
3865       }
3866       break;
3867    }
3868    case TGSI_OPCODE_I2I64:
3869       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3870          dst0[c] = fetchSrc(0, c / 2);
3871          mkOp2(OP_SHR, TYPE_S32, dst0[c + 1], dst0[c], loadImm(NULL, 31));
3872          c++;
3873       }
3874       break;
3875    case TGSI_OPCODE_U2I64:
3876       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3877          dst0[c] = fetchSrc(0, c / 2);
3878          dst0[c + 1] = zero;
3879          c++;
3880       }
3881       break;
3882    case TGSI_OPCODE_F2I64:
3883    case TGSI_OPCODE_F2U64:
3884    case TGSI_OPCODE_I2D:
3885    case TGSI_OPCODE_U2D:
3886    case TGSI_OPCODE_F2D:
3887       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3888          Value *dreg = getSSA(8);
3889          Instruction *cvt = mkCvt(OP_CVT, dstTy, dreg, srcTy, fetchSrc(0, c / 2));
3890          if (!isFloatType(dstTy))
3891             cvt->rnd = ROUND_Z;
3892          mkSplit(&dst0[c], 4, dreg);
3893          c++;
3894       }
3895       break;
3896    case TGSI_OPCODE_D2I64:
3897    case TGSI_OPCODE_D2U64:
3898    case TGSI_OPCODE_I642D:
3899    case TGSI_OPCODE_U642D:
3900       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3901          src0 = getSSA(8);
3902          Value *dst = getSSA(8), *tmp[2];
3903          tmp[0] = fetchSrc(0, c);
3904          tmp[1] = fetchSrc(0, c + 1);
3905          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3906          Instruction *cvt = mkCvt(OP_CVT, dstTy, dst, srcTy, src0);
3907          if (!isFloatType(dstTy))
3908             cvt->rnd = ROUND_Z;
3909          mkSplit(&dst0[c], 4, dst);
3910          c++;
3911       }
3912       break;
3913    case TGSI_OPCODE_I64NEG:
3914       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3915          src0 = getSSA(8);
3916          Value *dst = getSSA(8), *tmp[2];
3917          tmp[0] = fetchSrc(0, c);
3918          tmp[1] = fetchSrc(0, c + 1);
3919          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3920          mkOp2(OP_SUB, dstTy, dst, zero, src0);
3921          mkSplit(&dst0[c], 4, dst);
3922          c++;
3923       }
3924       break;
3925    case TGSI_OPCODE_I64ABS:
3926       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3927          src0 = getSSA(8);
3928          Value *neg = getSSA(8), *srcComp[2], *negComp[2];
3929          srcComp[0] = fetchSrc(0, c);
3930          srcComp[1] = fetchSrc(0, c + 1);
3931          mkOp2(OP_MERGE, TYPE_U64, src0, srcComp[0], srcComp[1]);
3932          mkOp2(OP_SUB, dstTy, neg, zero, src0);
3933          mkSplit(negComp, 4, neg);
3934          mkCmp(OP_SLCT, CC_LT, TYPE_S32, dst0[c], TYPE_S32,
3935                negComp[0], srcComp[0], srcComp[1]);
3936          mkCmp(OP_SLCT, CC_LT, TYPE_S32, dst0[c + 1], TYPE_S32,
3937                negComp[1], srcComp[1], srcComp[1]);
3938          c++;
3939       }
3940       break;
3941    case TGSI_OPCODE_DABS:
3942    case TGSI_OPCODE_DNEG:
3943    case TGSI_OPCODE_DRCP:
3944    case TGSI_OPCODE_DSQRT:
3945    case TGSI_OPCODE_DRSQ:
3946    case TGSI_OPCODE_DTRUNC:
3947    case TGSI_OPCODE_DCEIL:
3948    case TGSI_OPCODE_DFLR:
3949       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3950          src0 = getSSA(8);
3951          Value *dst = getSSA(8), *tmp[2];
3952          tmp[0] = fetchSrc(0, c);
3953          tmp[1] = fetchSrc(0, c + 1);
3954          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3955          mkOp1(op, dstTy, dst, src0);
3956          mkSplit(&dst0[c], 4, dst);
3957          c++;
3958       }
3959       break;
3960    case TGSI_OPCODE_DFRAC:
3961       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3962          src0 = getSSA(8);
3963          Value *dst = getSSA(8), *tmp[2];
3964          tmp[0] = fetchSrc(0, c);
3965          tmp[1] = fetchSrc(0, c + 1);
3966          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3967          mkOp1(OP_FLOOR, TYPE_F64, dst, src0);
3968          mkOp2(OP_SUB, TYPE_F64, dst, src0, dst);
3969          mkSplit(&dst0[c], 4, dst);
3970          c++;
3971       }
3972       break;
3973    case TGSI_OPCODE_U64SEQ:
3974    case TGSI_OPCODE_U64SNE:
3975    case TGSI_OPCODE_U64SLT:
3976    case TGSI_OPCODE_U64SGE:
3977    case TGSI_OPCODE_I64SLT:
3978    case TGSI_OPCODE_I64SGE:
3979    case TGSI_OPCODE_DSLT:
3980    case TGSI_OPCODE_DSGE:
3981    case TGSI_OPCODE_DSEQ:
3982    case TGSI_OPCODE_DSNE: {
3983       int pos = 0;
3984       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3985          Value *tmp[2];
3986
3987          src0 = getSSA(8);
3988          src1 = getSSA(8);
3989          tmp[0] = fetchSrc(0, pos);
3990          tmp[1] = fetchSrc(0, pos + 1);
3991          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3992          tmp[0] = fetchSrc(1, pos);
3993          tmp[1] = fetchSrc(1, pos + 1);
3994          mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
3995          mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], srcTy, src0, src1);
3996          pos += 2;
3997       }
3998       break;
3999    }
4000    case TGSI_OPCODE_U64MIN:
4001    case TGSI_OPCODE_U64MAX:
4002    case TGSI_OPCODE_I64MIN:
4003    case TGSI_OPCODE_I64MAX: {
4004       dstTy = isSignedIntType(dstTy) ? TYPE_S32 : TYPE_U32;
4005       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
4006          Value *flag = getSSA(1, FILE_FLAGS);
4007          src0 = fetchSrc(0, c + 1);
4008          src1 = fetchSrc(1, c + 1);
4009          geni = mkOp2(op, dstTy, dst0[c + 1], src0, src1);
4010          geni->subOp = NV50_IR_SUBOP_MINMAX_HIGH;
4011          geni->setFlagsDef(1, flag);
4012
4013          src0 = fetchSrc(0, c);
4014          src1 = fetchSrc(1, c);
4015          geni = mkOp2(op, TYPE_U32, dst0[c], src0, src1);
4016          geni->subOp = NV50_IR_SUBOP_MINMAX_LOW;
4017          geni->setFlagsSrc(2, flag);
4018
4019          c++;
4020       }
4021       break;
4022    }
4023    case TGSI_OPCODE_U64SHL:
4024    case TGSI_OPCODE_I64SHR:
4025    case TGSI_OPCODE_U64SHR:
4026       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
4027          src0 = getSSA(8);
4028          Value *dst = getSSA(8), *tmp[2];
4029          tmp[0] = fetchSrc(0, c);
4030          tmp[1] = fetchSrc(0, c + 1);
4031          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
4032          // Theoretically src1 is a 64-bit value but in practice only the low
4033          // bits matter. The IR expects this to be a 32-bit value.
4034          src1 = fetchSrc(1, c);
4035          mkOp2(op, dstTy, dst, src0, src1);
4036          mkSplit(&dst0[c], 4, dst);
4037          c++;
4038       }
4039       break;
4040    case TGSI_OPCODE_U64ADD:
4041    case TGSI_OPCODE_U64MUL:
4042    case TGSI_OPCODE_DADD:
4043    case TGSI_OPCODE_DMUL:
4044    case TGSI_OPCODE_DDIV:
4045    case TGSI_OPCODE_DMAX:
4046    case TGSI_OPCODE_DMIN:
4047       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
4048          src0 = getSSA(8);
4049          src1 = getSSA(8);
4050          Value *dst = getSSA(8), *tmp[2];
4051          tmp[0] = fetchSrc(0, c);
4052          tmp[1] = fetchSrc(0, c + 1);
4053          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
4054          tmp[0] = fetchSrc(1, c);
4055          tmp[1] = fetchSrc(1, c + 1);
4056          mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
4057          mkOp2(op, dstTy, dst, src0, src1);
4058          mkSplit(&dst0[c], 4, dst);
4059          c++;
4060       }
4061       break;
4062    case TGSI_OPCODE_DMAD:
4063    case TGSI_OPCODE_DFMA:
4064       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
4065          src0 = getSSA(8);
4066          src1 = getSSA(8);
4067          src2 = getSSA(8);
4068          Value *dst = getSSA(8), *tmp[2];
4069          tmp[0] = fetchSrc(0, c);
4070          tmp[1] = fetchSrc(0, c + 1);
4071          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
4072          tmp[0] = fetchSrc(1, c);
4073          tmp[1] = fetchSrc(1, c + 1);
4074          mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
4075          tmp[0] = fetchSrc(2, c);
4076          tmp[1] = fetchSrc(2, c + 1);
4077          mkOp2(OP_MERGE, TYPE_U64, src2, tmp[0], tmp[1]);
4078          mkOp3(op, dstTy, dst, src0, src1, src2);
4079          mkSplit(&dst0[c], 4, dst);
4080          c++;
4081       }
4082       break;
4083    case TGSI_OPCODE_DROUND:
4084       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
4085          src0 = getSSA(8);
4086          Value *dst = getSSA(8), *tmp[2];
4087          tmp[0] = fetchSrc(0, c);
4088          tmp[1] = fetchSrc(0, c + 1);
4089          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
4090          mkCvt(OP_CVT, TYPE_F64, dst, TYPE_F64, src0)
4091          ->rnd = ROUND_NI;
4092          mkSplit(&dst0[c], 4, dst);
4093          c++;
4094       }
4095       break;
4096    case TGSI_OPCODE_DSSG:
4097       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
4098          src0 = getSSA(8);
4099          Value *dst = getSSA(8), *dstF32 = getSSA(), *tmp[2];
4100          tmp[0] = fetchSrc(0, c);
4101          tmp[1] = fetchSrc(0, c + 1);
4102          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
4103
4104          val0 = getScratch();
4105          val1 = getScratch();
4106          // The zero is wrong here since it's only 32-bit, but it works out in
4107          // the end since it gets replaced with $r63.
4108          mkCmp(OP_SET, CC_GT, TYPE_F32, val0, TYPE_F64, src0, zero);
4109          mkCmp(OP_SET, CC_LT, TYPE_F32, val1, TYPE_F64, src0, zero);
4110          mkOp2(OP_SUB, TYPE_F32, dstF32, val0, val1);
4111          mkCvt(OP_CVT, TYPE_F64, dst, TYPE_F32, dstF32);
4112          mkSplit(&dst0[c], 4, dst);
4113          c++;
4114       }
4115       break;
4116    case TGSI_OPCODE_I64SSG:
4117       FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
4118          src0 = getSSA(8);
4119          Value *tmp[2];
4120          tmp[0] = fetchSrc(0, c);
4121          tmp[1] = fetchSrc(0, c + 1);
4122          mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
4123
4124          val0 = getScratch();
4125          val1 = getScratch();
4126          mkCmp(OP_SET, CC_GT, TYPE_U32, val0, TYPE_S64, src0, zero);
4127          mkCmp(OP_SET, CC_LT, TYPE_U32, val1, TYPE_S64, src0, zero);
4128          mkOp2(OP_SUB, TYPE_S32, dst0[c], val1, val0);
4129          mkOp2(OP_SHR, TYPE_S32, dst0[c + 1], dst0[c], loadImm(0, 31));
4130          c++;
4131       }
4132       break;
4133    default:
4134       ERROR("unhandled TGSI opcode: %u\n", tgsi.getOpcode());
4135       assert(0);
4136       break;
4137    }
4138
4139    if (tgsi.dstCount() && tgsi.getOpcode() != TGSI_OPCODE_STORE) {
4140       for (c = 0; c < 4; ++c) {
4141          if (!dst0[c])
4142             continue;
4143          if (dst0[c] != rDst0[c])
4144             mkMov(rDst0[c], dst0[c]);
4145          storeDst(0, c, rDst0[c]);
4146       }
4147    }
4148    vtxBaseValid = 0;
4149
4150    return true;
4151 }
4152
4153 void
4154 Converter::exportOutputs()
4155 {
4156    if (info->io.alphaRefBase) {
4157       for (unsigned int i = 0; i < info->numOutputs; ++i) {
4158          if (info->out[i].sn != TGSI_SEMANTIC_COLOR ||
4159              info->out[i].si != 0)
4160             continue;
4161          const unsigned int c = 3;
4162          if (!oData.exists(sub.cur->values, i, c))
4163             continue;
4164          Value *val = oData.load(sub.cur->values, i, c, NULL);
4165          if (!val)
4166             continue;
4167
4168          Symbol *ref = mkSymbol(FILE_MEMORY_CONST, info->io.auxCBSlot,
4169                                 TYPE_U32, info->io.alphaRefBase);
4170          Value *pred = new_LValue(func, FILE_PREDICATE);
4171          mkCmp(OP_SET, CC_TR, TYPE_U32, pred, TYPE_F32, val,
4172                mkLoadv(TYPE_U32, ref, NULL))
4173             ->subOp = 1;
4174          mkOp(OP_DISCARD, TYPE_NONE, NULL)->setPredicate(CC_NOT_P, pred);
4175       }
4176    }
4177
4178    for (unsigned int i = 0; i < info->numOutputs; ++i) {
4179       for (unsigned int c = 0; c < 4; ++c) {
4180          if (!oData.exists(sub.cur->values, i, c))
4181             continue;
4182          Symbol *sym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32,
4183                                 info->out[i].slot[c] * 4);
4184          Value *val = oData.load(sub.cur->values, i, c, NULL);
4185          if (val) {
4186             if (info->out[i].sn == TGSI_SEMANTIC_POSITION)
4187                mkOp1(OP_SAT, TYPE_F32, val, val);
4188             mkStore(OP_EXPORT, TYPE_F32, sym, NULL, val);
4189          }
4190       }
4191    }
4192 }
4193
4194 Converter::Converter(Program *ir, const tgsi::Source *code) : ConverterCommon(ir, code->info),
4195      code(code),
4196      tgsi(NULL),
4197      tData(this), lData(this), aData(this), oData(this)
4198 {
4199    const unsigned tSize = code->fileSize(TGSI_FILE_TEMPORARY);
4200    const unsigned aSize = code->fileSize(TGSI_FILE_ADDRESS);
4201    const unsigned oSize = code->fileSize(TGSI_FILE_OUTPUT);
4202
4203    tData.setup(TGSI_FILE_TEMPORARY, 0, 0, tSize, 4, 4, FILE_GPR, 0);
4204    lData.setup(TGSI_FILE_TEMPORARY, 1, 0, tSize, 4, 4, FILE_MEMORY_LOCAL, 0);
4205    aData.setup(TGSI_FILE_ADDRESS, 0, 0, aSize, 4, 4, FILE_GPR, 0);
4206    oData.setup(TGSI_FILE_OUTPUT, 0, 0, oSize, 4, 4, FILE_GPR, 0);
4207
4208    zero = mkImm((uint32_t)0);
4209
4210    vtxBaseValid = 0;
4211 }
4212
4213 Converter::~Converter()
4214 {
4215 }
4216
4217 inline const Converter::Location *
4218 Converter::BindArgumentsPass::getValueLocation(Subroutine *s, Value *v)
4219 {
4220    ValueMap::l_iterator it = s->values.l.find(v);
4221    return it == s->values.l.end() ? NULL : &it->second;
4222 }
4223
4224 template<typename T> inline void
4225 Converter::BindArgumentsPass::updateCallArgs(
4226    Instruction *i, void (Instruction::*setArg)(int, Value *),
4227    T (Function::*proto))
4228 {
4229    Function *g = i->asFlow()->target.fn;
4230    Subroutine *subg = conv.getSubroutine(g);
4231
4232    for (unsigned a = 0; a < (g->*proto).size(); ++a) {
4233       Value *v = (g->*proto)[a].get();
4234       const Converter::Location &l = *getValueLocation(subg, v);
4235       Converter::DataArray *array = conv.getArrayForFile(l.array, l.arrayIdx);
4236
4237       (i->*setArg)(a, array->acquire(sub->values, l.i, l.c));
4238    }
4239 }
4240
4241 template<typename T> inline void
4242 Converter::BindArgumentsPass::updatePrototype(
4243    BitSet *set, void (Function::*updateSet)(), T (Function::*proto))
4244 {
4245    (func->*updateSet)();
4246
4247    for (unsigned i = 0; i < set->getSize(); ++i) {
4248       Value *v = func->getLValue(i);
4249       const Converter::Location *l = getValueLocation(sub, v);
4250
4251       // only include values with a matching TGSI register
4252       if (set->test(i) && l && !conv.code->locals.count(*l))
4253          (func->*proto).push_back(v);
4254    }
4255 }
4256
4257 bool
4258 Converter::BindArgumentsPass::visit(Function *f)
4259 {
4260    sub = conv.getSubroutine(f);
4261
4262    for (ArrayList::Iterator bi = f->allBBlocks.iterator();
4263         !bi.end(); bi.next()) {
4264       for (Instruction *i = BasicBlock::get(bi)->getFirst();
4265            i; i = i->next) {
4266          if (i->op == OP_CALL && !i->asFlow()->builtin) {
4267             updateCallArgs(i, &Instruction::setSrc, &Function::ins);
4268             updateCallArgs(i, &Instruction::setDef, &Function::outs);
4269          }
4270       }
4271    }
4272
4273    if (func == prog->main /* && prog->getType() != Program::TYPE_COMPUTE */)
4274       return true;
4275    updatePrototype(&BasicBlock::get(f->cfg.getRoot())->liveSet,
4276                    &Function::buildLiveSets, &Function::ins);
4277    updatePrototype(&BasicBlock::get(f->cfgExit)->defSet,
4278                    &Function::buildDefSets, &Function::outs);
4279
4280    return true;
4281 }
4282
4283 bool
4284 Converter::run()
4285 {
4286    BasicBlock *entry = new BasicBlock(prog->main);
4287    BasicBlock *leave = new BasicBlock(prog->main);
4288
4289    prog->main->setEntry(entry);
4290    prog->main->setExit(leave);
4291
4292    setPosition(entry, true);
4293    sub.cur = getSubroutine(prog->main);
4294
4295    if (info->io.genUserClip > 0) {
4296       for (int c = 0; c < 4; ++c)
4297          clipVtx[c] = getScratch();
4298    }
4299
4300    switch (prog->getType()) {
4301    case Program::TYPE_TESSELLATION_CONTROL:
4302       outBase = mkOp2v(
4303          OP_SUB, TYPE_U32, getSSA(),
4304          mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_LANEID, 0)),
4305          mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_INVOCATION_ID, 0)));
4306       break;
4307    case Program::TYPE_FRAGMENT: {
4308       Symbol *sv = mkSysVal(SV_POSITION, 3);
4309       fragCoord[3] = mkOp1v(OP_RDSV, TYPE_F32, getSSA(), sv);
4310       mkOp1(OP_RCP, TYPE_F32, fragCoord[3], fragCoord[3]);
4311       break;
4312    }
4313    default:
4314       break;
4315    }
4316
4317    if (info->io.viewportId >= 0)
4318       viewport = getScratch();
4319    else
4320       viewport = NULL;
4321
4322    for (ip = 0; ip < code->scan.num_instructions; ++ip) {
4323       if (!handleInstruction(&code->insns[ip]))
4324          return false;
4325    }
4326
4327    if (!BindArgumentsPass(*this).run(prog))
4328       return false;
4329
4330    return true;
4331 }
4332
4333 } // unnamed namespace
4334
4335 namespace nv50_ir {
4336
4337 bool
4338 Program::makeFromTGSI(struct nv50_ir_prog_info *info)
4339 {
4340    tgsi::Source src(info);
4341    if (!src.scanSource())
4342       return false;
4343    tlsSize = info->bin.tlsSpace;
4344
4345    Converter builder(this, &src);
4346    return builder.run();
4347 }
4348
4349 } // namespace nv50_ir