src/gallium/drivers/nv50/codegen/nv50_ir.h

   1 /*
   2  * Copyright 2011 Christoph Bumiller
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  18  * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  19  * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  20  * SOFTWARE.
  21  */
  22
  23 #ifndef __NV50_IR_H__
  24 #define __NV50_IR_H__
  25
  26 #include <stdio.h>
  27 #include <stdlib.h>
  28 #include <stdint.h>
  29 #include <deque>
  30 #include <list>
  31 #include <vector>
  32
  33 #include "nv50_ir_util.h"
  34 #include "nv50_ir_graph.h"
  35
  36 #include "nv50_ir_driver.h"
  37
  38 namespace nv50_ir {
  39
  40 enum operation
  41 {
  42    OP_NOP = 0,
  43    OP_PHI,
  44    OP_UNION, // unify a new definition and several source values
  45    OP_SPLIT, // $r0d -> { $r0, $r1 } ($r0d and $r0/$r1 will be coalesced)
  46    OP_MERGE, // opposite of split, e.g. combine 2 32 bit into a 64 bit value
  47    OP_CONSTRAINT, // copy values into consecutive registers
  48    OP_MOV, // simple copy, no modifiers allowed
  49    OP_LOAD,
  50    OP_STORE,
  51    OP_ADD,
  52    OP_SUB,
  53    OP_MUL,
  54    OP_DIV,
  55    OP_MOD,
  56    OP_MAD,
  57    OP_FMA,
  58    OP_SAD, // abs(src0 - src1) + src2
  59    OP_ABS,
  60    OP_NEG,
  61    OP_NOT,
  62    OP_AND,
  63    OP_OR,
  64    OP_XOR,
  65    OP_SHL,
  66    OP_SHR,
  67    OP_MAX,
  68    OP_MIN,
  69    OP_SAT,  // CLAMP(f32, 0.0, 1.0)
  70    OP_CEIL,
  71    OP_FLOOR,
  72    OP_TRUNC,
  73    OP_CVT,
  74    OP_SET_AND, // dst = (src0 CMP src1) & src2
  75    OP_SET_OR,
  76    OP_SET_XOR,
  77    OP_SET,
  78    OP_SELP, // dst = src2 ? src0 : src1
  79    OP_SLCT, // dst = (src2 CMP 0) ? src0 : src1
  80    OP_RCP,
  81    OP_RSQ,
  82    OP_LG2,
  83    OP_SIN,
  84    OP_COS,
  85    OP_EX2,
  86    OP_EXP, // exponential (base M_E)
  87    OP_LOG, // natural logarithm
  88    OP_PRESIN,
  89    OP_PREEX2,
  90    OP_SQRT,
  91    OP_POW,
  92    OP_BRA,
  93    OP_CALL,
  94    OP_RET,
  95    OP_CONT,
  96    OP_BREAK,
  97    OP_PRERET,
  98    OP_PRECONT,
  99    OP_PREBREAK,
 100    OP_BRKPT,     // breakpoint (not related to loops)
 101    OP_JOINAT,    // push control flow convergence point
 102    OP_JOIN,      // converge
 103    OP_DISCARD,
 104    OP_EXIT,
 105    OP_MEMBAR,
 106    OP_VFETCH, // indirection 0 in attribute space, indirection 1 is vertex base
 107    OP_PFETCH, // fetch base address of vertex src0 (immediate) [+ src1]
 108    OP_EXPORT,
 109    OP_LINTERP,
 110    OP_PINTERP,
 111    OP_EMIT,    // emit vertex
 112    OP_RESTART, // restart primitive
 113    OP_TEX,
 114    OP_TXB, // texture bias
 115    OP_TXL, // texure lod
 116    OP_TXF, // texel fetch
 117    OP_TXQ, // texture size query
 118    OP_TXD, // texture derivatives
 119    OP_TXG, // texture gather
 120    OP_TEXCSAA,
 121    OP_SULD, // surface load
 122    OP_SUST, // surface store
 123    OP_DFDX,
 124    OP_DFDY,
 125    OP_RDSV, // read system value
 126    OP_WRSV, // write system value
 127    OP_TEXPREP, // turn cube map array into 2d array coordinates, TODO: move
 128    OP_QUADOP,
 129    OP_QUADON,
 130    OP_QUADPOP,
 131    OP_POPCNT, // bitcount(src0 & src1)
 132    OP_INSBF,  // insert first src1[8:15] bits of src0 into src2 at src1[0:7]
 133    OP_EXTBF,
 134    OP_TEXBAR,
 135    OP_LAST
 136 };
 137
 138 // various instruction-specific modifier definitions Instruction::subOp
 139 // MOV_FINAL marks a MOV originating from an EXPORT (used for placing TEXBARs)
 140 #define NV50_IR_SUBOP_MUL_HIGH     1
 141 #define NV50_IR_SUBOP_EMIT_RESTART 1
 142 #define NV50_IR_SUBOP_LDC_IL       1
 143 #define NV50_IR_SUBOP_LDC_IS       2
 144 #define NV50_IR_SUBOP_LDC_ISL      3
 145 #define NV50_IR_SUBOP_SHIFT_WRAP   1
 146 #define NV50_IR_SUBOP_EMU_PRERET   1
 147 #define NV50_IR_SUBOP_TEXBAR(n)    n
 148 #define NV50_IR_SUBOP_MOV_FINAL    1
 149
 150 enum DataType
 151 {
 152    TYPE_NONE,
 153    TYPE_U8,
 154    TYPE_S8,
 155    TYPE_U16,
 156    TYPE_S16,
 157    TYPE_U32,
 158    TYPE_S32,
 159    TYPE_U64, // 64 bit operations are only lowered after register allocation
 160    TYPE_S64,
 161    TYPE_F16,
 162    TYPE_F32,
 163    TYPE_F64,
 164    TYPE_B96,
 165    TYPE_B128
 166 };
 167
 168 enum CondCode
 169 {
 170    CC_FL = 0,
 171    CC_NEVER = CC_FL, // when used with FILE_FLAGS
 172    CC_LT = 1,
 173    CC_EQ = 2,
 174    CC_NOT_P = CC_EQ, // when used with FILE_PREDICATE
 175    CC_LE = 3,
 176    CC_GT = 4,
 177    CC_NE = 5,
 178    CC_P  = CC_NE,
 179    CC_GE = 6,
 180    CC_TR = 7,
 181    CC_ALWAYS = CC_TR,
 182    CC_U  = 8,
 183    CC_LTU = 9,
 184    CC_EQU = 10,
 185    CC_LEU = 11,
 186    CC_GTU = 12,
 187    CC_NEU = 13,
 188    CC_GEU = 14,
 189    CC_NO = 0x10,
 190    CC_NC = 0x11,
 191    CC_NS = 0x12,
 192    CC_NA = 0x13,
 193    CC_A  = 0x14,
 194    CC_S  = 0x15,
 195    CC_C  = 0x16,
 196    CC_O  = 0x17
 197 };
 198
 199 enum RoundMode
 200 {
 201    ROUND_N, // nearest
 202    ROUND_M, // towards -inf
 203    ROUND_Z, // towards 0
 204    ROUND_P, // towards +inf
 205    ROUND_NI, // nearest integer
 206    ROUND_MI, // to integer towards -inf
 207    ROUND_ZI, // to integer towards 0
 208    ROUND_PI, // to integer towards +inf
 209 };
 210
 211 enum CacheMode
 212 {
 213    CACHE_CA,            // cache at all levels
 214    CACHE_WB = CACHE_CA, // cache write back
 215    CACHE_CG,            // cache at global level
 216    CACHE_CS,            // cache streaming
 217    CACHE_CV,            // cache as volatile
 218    CACHE_WT = CACHE_CV  // cache write-through
 219 };
 220
 221 enum DataFile
 222 {
 223    FILE_NULL = 0,
 224    FILE_GPR,
 225    FILE_PREDICATE,       // boolean predicate
 226    FILE_FLAGS,           // zero/sign/carry/overflow bits
 227    FILE_ADDRESS,
 228    LAST_REGISTER_FILE = FILE_ADDRESS,
 229    FILE_IMMEDIATE,
 230    FILE_MEMORY_CONST,
 231    FILE_SHADER_INPUT,
 232    FILE_SHADER_OUTPUT,
 233    FILE_MEMORY_GLOBAL,
 234    FILE_MEMORY_SHARED,
 235    FILE_MEMORY_LOCAL,
 236    FILE_SYSTEM_VALUE,
 237    DATA_FILE_COUNT
 238 };
 239
 240 enum TexTarget
 241 {
 242    TEX_TARGET_1D,
 243    TEX_TARGET_2D,
 244    TEX_TARGET_2D_MS,
 245    TEX_TARGET_3D,
 246    TEX_TARGET_CUBE,
 247    TEX_TARGET_1D_SHADOW,
 248    TEX_TARGET_2D_SHADOW,
 249    TEX_TARGET_CUBE_SHADOW,
 250    TEX_TARGET_1D_ARRAY,
 251    TEX_TARGET_2D_ARRAY,
 252    TEX_TARGET_2D_MS_ARRAY,
 253    TEX_TARGET_CUBE_ARRAY,
 254    TEX_TARGET_1D_ARRAY_SHADOW,
 255    TEX_TARGET_2D_ARRAY_SHADOW,
 256    TEX_TARGET_RECT,
 257    TEX_TARGET_RECT_SHADOW,
 258    TEX_TARGET_CUBE_ARRAY_SHADOW,
 259    TEX_TARGET_BUFFER,
 260    TEX_TARGET_COUNT
 261 };
 262
 263 enum SVSemantic
 264 {
 265    SV_POSITION, // WPOS
 266    SV_VERTEX_ID,
 267    SV_INSTANCE_ID,
 268    SV_INVOCATION_ID,
 269    SV_PRIMITIVE_ID,
 270    SV_VERTEX_COUNT, // gl_PatchVerticesIn
 271    SV_LAYER,
 272    SV_VIEWPORT_INDEX,
 273    SV_YDIR,
 274    SV_FACE,
 275    SV_POINT_SIZE,
 276    SV_POINT_COORD,
 277    SV_CLIP_DISTANCE,
 278    SV_SAMPLE_INDEX,
 279    SV_TESS_FACTOR,
 280    SV_TESS_COORD,
 281    SV_TID,
 282    SV_CTAID,
 283    SV_NTID,
 284    SV_GRIDID,
 285    SV_NCTAID,
 286    SV_LANEID,
 287    SV_PHYSID,
 288    SV_NPHYSID,
 289    SV_CLOCK,
 290    SV_LBASE,
 291    SV_SBASE,
 292    SV_UNDEFINED,
 293    SV_LAST
 294 };
 295
 296 class Program;
 297 class Function;
 298 class BasicBlock;
 299
 300 class Target;
 301
 302 class Instruction;
 303 class CmpInstruction;
 304 class TexInstruction;
 305 class FlowInstruction;
 306
 307 class Value;
 308 class LValue;
 309 class Symbol;
 310 class ImmediateValue;
 311
 312 struct Storage
 313 {
 314    DataFile file;
 315    int8_t fileIndex; // signed, may be indirect for CONST[]
 316    uint8_t size; // this should match the Instruction type's size
 317    DataType type; // mainly for pretty printing
 318    union {
 319       uint64_t u64;    // immediate values
 320       uint32_t u32;
 321       uint16_t u16;
 322       uint8_t u8;
 323       int64_t s64;
 324       int32_t s32;
 325       int16_t s16;
 326       int8_t s8;
 327       float f32;
 328       double f64;
 329       int32_t offset; // offset from 0 (base of address space)
 330       int32_t id;     // register id (< 0 if virtual/unassigned, in units <= 4)
 331       struct {
 332          SVSemantic sv;
 333          int index;
 334       } sv;
 335    } data;
 336 };
 337
 338 // precedence: NOT after SAT after NEG after ABS
 339 #define NV50_IR_MOD_ABS (1 << 0)
 340 #define NV50_IR_MOD_NEG (1 << 1)
 341 #define NV50_IR_MOD_SAT (1 << 2)
 342 #define NV50_IR_MOD_NOT (1 << 3)
 343 #define NV50_IR_MOD_NEG_ABS (NV50_IR_MOD_NEG | NV50_IR_MOD_ABS)
 344
 345 #define NV50_IR_INTERP_MODE_MASK   0x3
 346 #define NV50_IR_INTERP_LINEAR      (0 << 0)
 347 #define NV50_IR_INTERP_PERSPECTIVE (1 << 0)
 348 #define NV50_IR_INTERP_FLAT        (2 << 0)
 349 #define NV50_IR_INTERP_SC          (3 << 0) // what exactly is that ?
 350 #define NV50_IR_INTERP_SAMPLE_MASK 0xc
 351 #define NV50_IR_INTERP_DEFAULT     (0 << 2)
 352 #define NV50_IR_INTERP_CENTROID    (1 << 2)
 353 #define NV50_IR_INTERP_OFFSET      (2 << 2)
 354 #define NV50_IR_INTERP_SAMPLEID    (3 << 2)
 355
 356 // do we really want this to be a class ?
 357 class Modifier
 358 {
 359 public:
 360    Modifier() : bits(0) { }
 361    Modifier(unsigned int m) : bits(m) { }
 362    Modifier(operation op);
 363
 364    // @return new Modifier applying a after b (asserts if unrepresentable)
 365    Modifier operator*(const Modifier) const;
 366    Modifier operator*=(const Modifier m) { *this = *this * m; return *this; }
 367    Modifier operator==(const Modifier m) const { return m.bits == bits; }
 368    Modifier operator!=(const Modifier m) const { return m.bits != bits; }
 369
 370    inline Modifier operator&(const Modifier m) const { return bits & m.bits; }
 371    inline Modifier operator|(const Modifier m) const { return bits | m.bits; }
 372    inline Modifier operator^(const Modifier m) const { return bits ^ m.bits; }
 373
 374    operation getOp() const;
 375
 376    inline int neg() const { return (bits & NV50_IR_MOD_NEG) ? 1 : 0; }
 377    inline int abs() const { return (bits & NV50_IR_MOD_ABS) ? 1 : 0; }
 378
 379    inline operator bool() const { return bits ? true : false; }
 380
 381    void applyTo(ImmediateValue &imm) const;
 382
 383    int print(char *buf, size_t size) const;
 384
 385 private:
 386    uint8_t bits;
 387 };
 388
 389 class ValueRef
 390 {
 391 public:
 392    ValueRef(Value * = NULL);
 393    ValueRef(const ValueRef&);
 394    ~ValueRef();
 395
 396    inline bool exists() const { return value != NULL; }
 397
 398    void set(Value *);
 399    void set(const ValueRef&);
 400    inline Value *get() const { return value; }
 401    inline Value *rep() const;
 402
 403    inline Instruction *getInsn() const { return insn; }
 404    inline void setInsn(Instruction *inst) { insn = inst; }
 405
 406    inline bool isIndirect(int dim) const { return indirect[dim] >= 0; }
 407    inline const ValueRef *getIndirect(int dim) const;
 408
 409    inline DataFile getFile() const;
 410    inline unsigned getSize() const;
 411
 412    // SSA: return eventual (traverse MOVs) literal value, if it exists
 413    bool getImmediate(ImmediateValue&) const;
 414
 415 public:
 416    Modifier mod;
 417    int8_t indirect[2]; // >= 0 if relative to lvalue in insn->src(indirect[i])
 418    uint8_t swizzle;
 419
 420    bool usedAsPtr; // for printing
 421
 422 private:
 423    Value *value;
 424    Instruction *insn;
 425 };
 426
 427 class ValueDef
 428 {
 429 public:
 430    ValueDef(Value * = NULL);
 431    ValueDef(const ValueDef&);
 432    ~ValueDef();
 433
 434    inline bool exists() const { return value != NULL; }
 435
 436    inline Value *get() const { return value; }
 437    inline Value *rep() const;
 438    void set(Value *);
 439    bool mayReplace(const ValueRef &);
 440    void replace(const ValueRef &, bool doSet); // replace all uses of the old value
 441
 442    inline Instruction *getInsn() const { return insn; }
 443    inline void setInsn(Instruction *inst) { insn = inst; }
 444
 445    inline DataFile getFile() const;
 446    inline unsigned getSize() const;
 447
 448    inline void setSSA(LValue *);
 449    inline const LValue *preSSA() const;
 450
 451 private:
 452    Value *value;   // should make this LValue * ...
 453    LValue *origin; // pre SSA value
 454    Instruction *insn;
 455 };
 456
 457 class Value
 458 {
 459 public:
 460    Value();
 461    virtual ~Value() { }
 462
 463    virtual Value *clone(ClonePolicy<Function>&) const = 0;
 464
 465    virtual int print(char *, size_t, DataType ty = TYPE_NONE) const = 0;
 466
 467    virtual bool equals(const Value *, bool strict = false) const;
 468    virtual bool interfers(const Value *) const;
 469    virtual bool isUniform() const { return true; }
 470
 471    inline Value *rep() const { return join; }
 472
 473    inline Instruction *getUniqueInsn() const;
 474    inline Instruction *getInsn() const; // use when uniqueness is certain
 475
 476    inline int refCount() { return uses.size(); }
 477
 478    inline LValue *asLValue();
 479    inline Symbol *asSym();
 480    inline ImmediateValue *asImm();
 481    inline const Symbol *asSym() const;
 482    inline const ImmediateValue *asImm() const;
 483
 484    inline bool inFile(DataFile f) { return reg.file == f; }
 485
 486    static inline Value *get(Iterator&);
 487
 488    std::list<ValueRef *> uses;
 489    std::list<ValueDef *> defs;
 490    typedef std::list<ValueRef *>::iterator UseIterator;
 491    typedef std::list<ValueRef *>::const_iterator UseCIterator;
 492    typedef std::list<ValueDef *>::iterator DefIterator;
 493    typedef std::list<ValueDef *>::const_iterator DefCIterator;
 494
 495    int id;
 496    Storage reg;
 497
 498    // TODO: these should be in LValue:
 499    Interval livei;
 500    Value *join;
 501 };
 502
 503 class LValue : public Value
 504 {
 505 public:
 506    LValue(Function *, DataFile file);
 507    LValue(Function *, LValue *);
 508    ~LValue() { }
 509
 510    virtual bool isUniform() const;
 511
 512    virtual LValue *clone(ClonePolicy<Function>&) const;
 513
 514    virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
 515
 516 public:
 517    unsigned compMask : 8; // compound/component mask
 518    unsigned compound : 1; // used by RA, value involved in split/merge
 519    unsigned ssa      : 1;
 520    unsigned fixedReg : 1; // set & used by RA, earlier just use (id < 0)
 521    unsigned noSpill  : 1; // do not spill (e.g. if spill temporary already)
 522 };
 523
 524 class Symbol : public Value
 525 {
 526 public:
 527    Symbol(Program *, DataFile file = FILE_MEMORY_CONST, ubyte fileIdx = 0);
 528    ~Symbol() { }
 529
 530    virtual Symbol *clone(ClonePolicy<Function>&) const;
 531
 532    virtual bool equals(const Value *that, bool strict) const;
 533
 534    virtual bool isUniform() const;
 535
 536    virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
 537
 538    // print with indirect values
 539    int print(char *, size_t, Value *, Value *, DataType ty = TYPE_NONE) const;
 540
 541    inline void setFile(DataFile file, ubyte fileIndex = 0)
 542    {
 543       reg.file = file;
 544       reg.fileIndex = fileIndex;
 545    }
 546
 547    inline void setOffset(int32_t offset);
 548    inline void setAddress(Symbol *base, int32_t offset);
 549    inline void setSV(SVSemantic sv, uint32_t idx = 0);
 550
 551    inline const Symbol *getBase() const { return baseSym; }
 552
 553 private:
 554    Symbol *baseSym; // array base for Symbols representing array elements
 555 };
 556
 557 class ImmediateValue : public Value
 558 {
 559 public:
 560    ImmediateValue() { }
 561    ImmediateValue(Program *, uint32_t);
 562    ImmediateValue(Program *, float);
 563    ImmediateValue(Program *, double);
 564    // NOTE: not added to program with
 565    ImmediateValue(const ImmediateValue *, DataType ty);
 566    ~ImmediateValue() { };
 567
 568    virtual ImmediateValue *clone(ClonePolicy<Function>&) const;
 569
 570    virtual bool equals(const Value *that, bool strict) const;
 571
 572    // these only work if 'type' is valid (we mostly use untyped literals):
 573    bool isInteger(const int ival) const; // ival is cast to this' type
 574    bool isNegative() const;
 575    bool isPow2() const;
 576
 577    void applyLog2();
 578
 579    // for constant folding:
 580    ImmediateValue operator+(const ImmediateValue&) const;
 581    ImmediateValue operator-(const ImmediateValue&) const;
 582    ImmediateValue operator*(const ImmediateValue&) const;
 583    ImmediateValue operator/(const ImmediateValue&) const;
 584
 585    ImmediateValue& operator=(const ImmediateValue&); // only sets value !
 586
 587    bool compare(CondCode cc, float fval) const;
 588
 589    virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
 590 };
 591
 592 class Instruction
 593 {
 594 public:
 595    Instruction();
 596    Instruction(Function *, operation, DataType);
 597    virtual ~Instruction();
 598
 599    virtual Instruction *clone(ClonePolicy<Function>&,
 600                               Instruction * = NULL) const;
 601
 602    void setDef(int i, Value *);
 603    void setSrc(int s, Value *);
 604    void setSrc(int s, const ValueRef&);
 605    void swapSources(int a, int b);
 606    void moveSources(int s, int delta); // NOTE: only delta > 0 implemented
 607    bool setIndirect(int s, int dim, Value *);
 608
 609    inline ValueRef& src(int s) { return srcs[s]; }
 610    inline ValueDef& def(int s) { return defs[s]; }
 611    inline const ValueRef& src(int s) const { return srcs[s]; }
 612    inline const ValueDef& def(int s) const { return defs[s]; }
 613
 614    inline Value *getDef(int d) const { return defs[d].get(); }
 615    inline Value *getSrc(int s) const { return srcs[s].get(); }
 616    inline Value *getIndirect(int s, int dim) const;
 617
 618    inline bool defExists(unsigned d) const
 619    {
 620       return d < defs.size() && defs[d].exists();
 621    }
 622    inline bool srcExists(unsigned s) const
 623    {
 624       return s < srcs.size() && srcs[s].exists();
 625    }
 626
 627    inline bool constrainedDefs() const;
 628
 629    bool setPredicate(CondCode ccode, Value *);
 630    inline Value *getPredicate() const;
 631    bool writesPredicate() const;
 632    inline bool isPredicated() const { return predSrc >= 0; }
 633
 634    inline void setFlagsSrc(int s, Value *);
 635    inline void setFlagsDef(int d, Value *);
 636
 637    unsigned int defCount() const { return defs.size(); };
 638    unsigned int defCount(unsigned int mask, bool singleFile = false) const;
 639    unsigned int srcCount() const { return srcs.size(); };
 640    unsigned int srcCount(unsigned int mask, bool singleFile = false) const;
 641
 642    // save & remove / set indirect[0,1] and predicate source
 643    void takeExtraSources(int s, Value *[3]);
 644    void putExtraSources(int s, Value *[3]);
 645
 646    inline void setType(DataType type) { dType = sType = type; }
 647
 648    inline void setType(DataType dtype, DataType stype)
 649    {
 650       dType = dtype;
 651       sType = stype;
 652    }
 653
 654    inline bool isPseudo() const { return op < OP_MOV; }
 655    bool isDead() const;
 656    bool isNop() const;
 657    bool isCommutationLegal(const Instruction *) const; // must be adjacent !
 658    bool isActionEqual(const Instruction *) const;
 659    bool isResultEqual(const Instruction *) const;
 660
 661    void print() const;
 662
 663    inline CmpInstruction *asCmp();
 664    inline TexInstruction *asTex();
 665    inline FlowInstruction *asFlow();
 666    inline const TexInstruction *asTex() const;
 667    inline const CmpInstruction *asCmp() const;
 668    inline const FlowInstruction *asFlow() const;
 669
 670 public:
 671    Instruction *next;
 672    Instruction *prev;
 673    int id;
 674    int serial; // CFG order
 675
 676    operation op;
 677    DataType dType; // destination or defining type
 678    DataType sType; // source or secondary type
 679    CondCode cc;
 680    RoundMode rnd;
 681    CacheMode cache;
 682
 683    uint8_t subOp; // quadop, 1 for mul-high, etc.
 684
 685    uint8_t sched; // scheduling data (NOTE: maybe move to separate storage)
 686
 687    unsigned encSize    : 4; // encoding size in bytes
 688    unsigned saturate   : 1; // to [0.0f, 1.0f]
 689    unsigned join       : 1; // converge control flow (use OP_JOIN until end)
 690    unsigned fixed      : 1; // prevent dead code elimination
 691    unsigned terminator : 1; // end of basic block
 692    unsigned atomic     : 1;
 693    unsigned ftz        : 1; // flush denormal to zero
 694    unsigned dnz        : 1; // denormals, NaN are zero
 695    unsigned ipa        : 4; // interpolation mode
 696    unsigned lanes      : 4;
 697    unsigned perPatch   : 1;
 698    unsigned exit       : 1; // terminate program after insn
 699
 700    int8_t postFactor; // MUL/DIV(if < 0) by 1 << postFactor
 701
 702    int8_t predSrc;
 703    int8_t flagsDef;
 704    int8_t flagsSrc;
 705
 706    BasicBlock *bb;
 707
 708 protected:
 709    std::deque<ValueDef> defs; // no gaps !
 710    std::deque<ValueRef> srcs; // no gaps !
 711
 712    // instruction specific methods:
 713    // (don't want to subclass, would need more constructors and memory pools)
 714 public:
 715    inline void setInterpolate(unsigned int mode) { ipa = mode; }
 716
 717    unsigned int getInterpMode() const { return ipa & 0x3; }
 718    unsigned int getSampleMode() const { return ipa & 0xc; }
 719
 720 private:
 721    void init();
 722 };
 723
 724 enum TexQuery
 725 {
 726    TXQ_DIMS,
 727    TXQ_TYPE,
 728    TXQ_SAMPLE_POSITION,
 729    TXQ_FILTER,
 730    TXQ_LOD,
 731    TXQ_WRAP,
 732    TXQ_BORDER_COLOUR
 733 };
 734
 735 class TexInstruction : public Instruction
 736 {
 737 public:
 738    class Target
 739    {
 740    public:
 741       Target(TexTarget targ = TEX_TARGET_2D) : target(targ) { }
 742
 743       const char *getName() const { return descTable[target].name; }
 744       unsigned int getArgCount() const { return descTable[target].argc; }
 745       unsigned int getDim() const { return descTable[target].dim; }
 746       int isArray() const { return descTable[target].array ? 1 : 0; }
 747       int isCube() const { return descTable[target].cube ? 1 : 0; }
 748       int isShadow() const { return descTable[target].shadow ? 1 : 0; }
 749
 750       Target& operator=(TexTarget targ)
 751       {
 752          assert(targ < TEX_TARGET_COUNT);
 753          target = targ;
 754          return *this;
 755       }
 756
 757       inline bool operator==(TexTarget targ) const { return target == targ; }
 758
 759    private:
 760       struct Desc
 761       {
 762          char name[19];
 763          uint8_t dim;
 764          uint8_t argc;
 765          bool array;
 766          bool cube;
 767          bool shadow;
 768       };
 769
 770       static const struct Desc descTable[TEX_TARGET_COUNT];
 771
 772    private:
 773       enum TexTarget target;
 774    };
 775
 776 public:
 777    TexInstruction(Function *, operation);
 778    virtual ~TexInstruction();
 779
 780    virtual TexInstruction *clone(ClonePolicy<Function>&,
 781                                  Instruction * = NULL) const;
 782
 783    inline void setTexture(Target targ, uint8_t r, uint8_t s)
 784    {
 785       tex.r = r;
 786       tex.s = s;
 787       tex.target = targ;
 788    }
 789
 790    inline Value *getIndirectR() const;
 791    inline Value *getIndirectS() const;
 792
 793 public:
 794    struct {
 795       Target target;
 796
 797       uint8_t r;
 798       int8_t rIndirectSrc;
 799       uint8_t s;
 800       int8_t sIndirectSrc;
 801
 802       uint8_t mask;
 803       uint8_t gatherComp;
 804
 805       bool liveOnly; // only execute on live pixels of a quad (optimization)
 806       bool levelZero;
 807       bool derivAll;
 808
 809       int8_t useOffsets; // 0, 1, or 4 for textureGatherOffsets
 810       int8_t offset[4][3];
 811
 812       enum TexQuery query;
 813    } tex;
 814
 815    ValueRef dPdx[3];
 816    ValueRef dPdy[3];
 817 };
 818
 819 class CmpInstruction : public Instruction
 820 {
 821 public:
 822    CmpInstruction(Function *, operation);
 823
 824    virtual CmpInstruction *clone(ClonePolicy<Function>&,
 825                                  Instruction * = NULL) const;
 826
 827    void setCondition(CondCode cond) { setCond = cond; }
 828    CondCode getCondition() const { return setCond; }
 829
 830 public:
 831    CondCode setCond;
 832 };
 833
 834 class FlowInstruction : public Instruction
 835 {
 836 public:
 837    FlowInstruction(Function *, operation, void *target);
 838
 839    virtual FlowInstruction *clone(ClonePolicy<Function>&,
 840                                   Instruction * = NULL) const;
 841
 842 public:
 843    unsigned allWarp  : 1;
 844    unsigned absolute : 1;
 845    unsigned limit    : 1;
 846    unsigned builtin  : 1; // true for calls to emulation code
 847
 848    union {
 849       BasicBlock *bb;
 850       int builtin;
 851       Function *fn;
 852    } target;
 853 };
 854
 855 class BasicBlock
 856 {
 857 public:
 858    BasicBlock(Function *);
 859    ~BasicBlock();
 860
 861    BasicBlock *clone(ClonePolicy<Function>&) const;
 862
 863    inline int getId() const { return id; }
 864    inline unsigned int getInsnCount() const { return numInsns; }
 865    inline bool isTerminated() const { return exit && exit->terminator; }
 866
 867    bool dominatedBy(BasicBlock *bb);
 868    inline bool reachableBy(const BasicBlock *by, const BasicBlock *term);
 869
 870    // returns mask of conditional out blocks
 871    // e.g. 3 for IF { .. } ELSE { .. } ENDIF, 1 for IF { .. } ENDIF
 872    unsigned int initiatesSimpleConditional() const;
 873
 874 public:
 875    Function *getFunction() const { return func; }
 876    Program *getProgram() const { return program; }
 877
 878    Instruction *getEntry() const { return entry; } // first non-phi instruction
 879    Instruction *getPhi() const { return phi; }
 880    Instruction *getFirst() const { return phi ? phi : entry; }
 881    Instruction *getExit() const { return exit; }
 882
 883    void insertHead(Instruction *);
 884    void insertTail(Instruction *);
 885    void insertBefore(Instruction *, Instruction *);
 886    void insertAfter(Instruction *, Instruction *);
 887    void remove(Instruction *);
 888    void permuteAdjacent(Instruction *, Instruction *);
 889
 890    BasicBlock *idom() const;
 891
 892    // NOTE: currently does not rebuild the dominator tree
 893    BasicBlock *splitBefore(Instruction *, bool attach = true);
 894    BasicBlock *splitAfter(Instruction *, bool attach = true);
 895
 896    DLList& getDF() { return df; }
 897    DLList::Iterator iterDF() { return df.iterator(); }
 898
 899    static inline BasicBlock *get(Iterator&);
 900    static inline BasicBlock *get(Graph::Node *);
 901
 902 public:
 903    Graph::Node cfg; // first edge is branch *taken* (the ELSE branch)
 904    Graph::Node dom;
 905
 906    BitSet liveSet;
 907    BitSet defSet;
 908
 909    uint32_t binPos;
 910    uint32_t binSize;
 911
 912    Instruction *joinAt; // for quick reference
 913
 914    bool explicitCont; // loop headers: true if loop contains continue stmts
 915
 916 private:
 917    int id;
 918    DLList df;
 919
 920    Instruction *phi;
 921    Instruction *entry;
 922    Instruction *exit;
 923
 924    unsigned int numInsns;
 925
 926 private:
 927    Function *func;
 928    Program *program;
 929
 930    void splitCommon(Instruction *, BasicBlock *, bool attach);
 931 };
 932
 933 class Function
 934 {
 935 public:
 936    Function(Program *, const char *name, uint32_t label);
 937    ~Function();
 938
 939    static inline Function *get(Graph::Node *node);
 940
 941    inline Program *getProgram() const { return prog; }
 942    inline const char *getName() const { return name; }
 943    inline int getId() const { return id; }
 944    inline uint32_t getLabel() const { return label; }
 945
 946    void print();
 947    void printLiveIntervals() const;
 948    void printCFGraph(const char *filePath);
 949
 950    bool setEntry(BasicBlock *);
 951    bool setExit(BasicBlock *);
 952
 953    unsigned int orderInstructions(ArrayList&);
 954
 955    inline void add(BasicBlock *bb, int& id) { allBBlocks.insert(bb, id); }
 956    inline void add(Instruction *insn, int& id) { allInsns.insert(insn, id); }
 957    inline void add(LValue *lval, int& id) { allLValues.insert(lval, id); }
 958
 959    inline LValue *getLValue(int id);
 960
 961    void buildLiveSets();
 962    void buildDefSets();
 963    bool convertToSSA();
 964
 965 public:
 966    std::deque<ValueDef> ins;
 967    std::deque<ValueRef> outs;
 968    std::deque<Value *> clobbers;
 969
 970    Graph cfg;
 971    Graph::Node *cfgExit;
 972    Graph *domTree;
 973    Graph::Node call; // node in the call graph
 974
 975    BasicBlock **bbArray; // BBs in emission order
 976    int bbCount;
 977
 978    unsigned int loopNestingBound;
 979    int regClobberMax;
 980
 981    uint32_t binPos;
 982    uint32_t binSize;
 983
 984    Value *stackPtr;
 985
 986    uint32_t tlsBase; // base address for l[] space (if no stack pointer is used)
 987    uint32_t tlsSize;
 988
 989    ArrayList allBBlocks;
 990    ArrayList allInsns;
 991    ArrayList allLValues;
 992
 993 private:
 994    void buildLiveSetsPreSSA(BasicBlock *, const int sequence);
 995    void buildDefSetsPreSSA(BasicBlock *bb, const int seq);
 996
 997 private:
 998    uint32_t label;
 999    int id;
1000    const char *const name;
1001    Program *prog;
1002 };
1003
1004 enum CGStage
1005 {
1006    CG_STAGE_PRE_SSA,
1007    CG_STAGE_SSA, // expected directly before register allocation
1008    CG_STAGE_POST_RA
1009 };
1010
1011 class Program
1012 {
1013 public:
1014    enum Type
1015    {
1016       TYPE_VERTEX,
1017       TYPE_TESSELLATION_CONTROL,
1018       TYPE_TESSELLATION_EVAL,
1019       TYPE_GEOMETRY,
1020       TYPE_FRAGMENT,
1021       TYPE_COMPUTE
1022    };
1023
1024    Program(Type type, Target *targ);
1025    ~Program();
1026
1027    void print();
1028
1029    Type getType() const { return progType; }
1030
1031    inline void add(Function *fn, int& id) { allFuncs.insert(fn, id); }
1032    inline void del(Function *fn, int& id) { allFuncs.remove(id); }
1033    inline void add(Value *rval, int& id) { allRValues.insert(rval, id); }
1034
1035    bool makeFromTGSI(struct nv50_ir_prog_info *);
1036    bool makeFromSM4(struct nv50_ir_prog_info *);
1037    bool convertToSSA();
1038    bool optimizeSSA(int level);
1039    bool optimizePostRA(int level);
1040    bool registerAllocation();
1041    bool emitBinary(struct nv50_ir_prog_info *);
1042
1043    const Target *getTarget() const { return target; }
1044
1045 private:
1046    void emitSymbolTable(struct nv50_ir_prog_info *);
1047
1048    Type progType;
1049    Target *target;
1050
1051 public:
1052    Function *main;
1053    Graph calls;
1054
1055    ArrayList allFuncs;
1056    ArrayList allRValues;
1057
1058    uint32_t *code;
1059    uint32_t binSize;
1060    uint32_t tlsSize; // size required for FILE_MEMORY_LOCAL
1061
1062    int maxGPR;
1063
1064    MemoryPool mem_Instruction;
1065    MemoryPool mem_CmpInstruction;
1066    MemoryPool mem_TexInstruction;
1067    MemoryPool mem_FlowInstruction;
1068    MemoryPool mem_LValue;
1069    MemoryPool mem_Symbol;
1070    MemoryPool mem_ImmediateValue;
1071
1072    uint32_t dbgFlags;
1073    uint8_t  optLevel;
1074
1075    void *targetPriv; // e.g. to carry information between passes
1076
1077    void releaseInstruction(Instruction *);
1078    void releaseValue(Value *);
1079 };
1080
1081 // TODO: add const version
1082 class Pass
1083 {
1084 public:
1085    bool run(Program *, bool ordered = false, bool skipPhi = false);
1086    bool run(Function *, bool ordered = false, bool skipPhi = false);
1087
1088 private:
1089    // return false to continue with next entity on next higher level
1090    virtual bool visit(Function *) { return true; }
1091    virtual bool visit(BasicBlock *) { return true; }
1092    virtual bool visit(Instruction *) { return false; }
1093
1094    bool doRun(Program *, bool ordered, bool skipPhi);
1095    bool doRun(Function *, bool ordered, bool skipPhi);
1096
1097 protected:
1098    bool err;
1099    Function *func;
1100    Program *prog;
1101 };
1102
1103 // =============================================================================
1104
1105 #include "nv50_ir_inlines.h"
1106
1107 } // namespace nv50_ir
1108
1109 #endif // __NV50_IR_H__