src/gallium/drivers/nv50/codegen/nv50_ir.h

   1 /*
   2  * Copyright 2011 Christoph Bumiller
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  18  * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  19  * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  20  * SOFTWARE.
  21  */
  22
  23 #ifndef __NV50_IR_H__
  24 #define __NV50_IR_H__
  25
  26 #include <stdio.h>
  27 #include <stdlib.h>
  28 #include <stdint.h>
  29 #include <deque>
  30 #include <list>
  31 #include <vector>
  32
  33 #include "nv50_ir_util.h"
  34 #include "nv50_ir_graph.h"
  35
  36 #include "nv50_ir_driver.h"
  37
  38 namespace nv50_ir {
  39
  40 enum operation
  41 {
  42    OP_NOP = 0,
  43    OP_PHI,
  44    OP_UNION, // unify a new definition and several source values
  45    OP_SPLIT, // $r0d -> { $r0, $r1 } ($r0d and $r0/$r1 will be coalesced)
  46    OP_MERGE, // opposite of split, e.g. combine 2 32 bit into a 64 bit value
  47    OP_CONSTRAINT, // copy values into consecutive registers
  48    OP_MOV,
  49    OP_LOAD,
  50    OP_STORE,
  51    OP_ADD,
  52    OP_SUB,
  53    OP_MUL,
  54    OP_DIV,
  55    OP_MOD,
  56    OP_MAD,
  57    OP_FMA,
  58    OP_SAD, // abs(src0 - src1) + src2
  59    OP_ABS,
  60    OP_NEG,
  61    OP_NOT,
  62    OP_AND,
  63    OP_OR,
  64    OP_XOR,
  65    OP_SHL,
  66    OP_SHR,
  67    OP_MAX,
  68    OP_MIN,
  69    OP_SAT,  // CLAMP(f32, 0.0, 1.0)
  70    OP_CEIL,
  71    OP_FLOOR,
  72    OP_TRUNC,
  73    OP_CVT,
  74    OP_SET_AND, // dst = (src0 CMP src1) & src2
  75    OP_SET_OR,
  76    OP_SET_XOR,
  77    OP_SET,
  78    OP_SELP, // dst = src2 ? src0 : src1
  79    OP_SLCT, // dst = (src2 CMP 0) ? src0 : src1
  80    OP_RCP,
  81    OP_RSQ,
  82    OP_LG2,
  83    OP_SIN,
  84    OP_COS,
  85    OP_EX2,
  86    OP_EXP, // exponential (base M_E)
  87    OP_LOG, // natural logarithm
  88    OP_PRESIN,
  89    OP_PREEX2,
  90    OP_SQRT,
  91    OP_POW,
  92    OP_BRA,
  93    OP_CALL,
  94    OP_RET,
  95    OP_CONT,
  96    OP_BREAK,
  97    OP_PRERET,
  98    OP_PRECONT,
  99    OP_PREBREAK,
 100    OP_BRKPT,     // breakpoint (not related to loops)
 101    OP_JOINAT,    // push control flow convergence point
 102    OP_JOIN,      // converge
 103    OP_DISCARD,
 104    OP_EXIT,
 105    OP_MEMBAR,
 106    OP_VFETCH, // indirection 0 in attribute space, indirection 1 is vertex base
 107    OP_PFETCH, // fetch base address of vertex src0 (immediate) [+ src1]
 108    OP_EXPORT,
 109    OP_LINTERP,
 110    OP_PINTERP,
 111    OP_EMIT,    // emit vertex
 112    OP_RESTART, // restart primitive
 113    OP_TEX,
 114    OP_TXB, // texture bias
 115    OP_TXL, // texure lod
 116    OP_TXF, // texel fetch
 117    OP_TXQ, // texture size query
 118    OP_TXD, // texture derivatives
 119    OP_TXG, // texture gather
 120    OP_TEXCSAA,
 121    OP_SULD, // surface load
 122    OP_SUST, // surface store
 123    OP_DFDX,
 124    OP_DFDY,
 125    OP_RDSV, // read system value
 126    OP_WRSV, // write system value
 127    OP_PIXLD,
 128    OP_QUADOP,
 129    OP_QUADON,
 130    OP_QUADPOP,
 131    OP_POPCNT, // bitcount(src0 & src1)
 132    OP_INSBF,  // insert first src1[8:15] bits of src0 into src2 at src1[0:7]
 133    OP_EXTBF,
 134    OP_LAST
 135 };
 136
 137 #define NV50_IR_SUBOP_MUL_HIGH     1
 138 #define NV50_IR_SUBOP_EMIT_RESTART 1
 139 #define NV50_IR_SUBOP_LDC_IL       1
 140 #define NV50_IR_SUBOP_LDC_IS       2
 141 #define NV50_IR_SUBOP_LDC_ISL      3
 142 #define NV50_IR_SUBOP_SHIFT_WRAP   1
 143
 144 enum DataType
 145 {
 146    TYPE_NONE,
 147    TYPE_U8,
 148    TYPE_S8,
 149    TYPE_U16,
 150    TYPE_S16,
 151    TYPE_U32,
 152    TYPE_S32,
 153    TYPE_U64, // 64 bit operations are only lowered after register allocation
 154    TYPE_S64,
 155    TYPE_F16,
 156    TYPE_F32,
 157    TYPE_F64,
 158    TYPE_B96,
 159    TYPE_B128
 160 };
 161
 162 enum CondCode
 163 {
 164    CC_FL = 0,
 165    CC_NEVER = CC_FL, // when used with FILE_FLAGS
 166    CC_LT = 1,
 167    CC_EQ = 2,
 168    CC_NOT_P = CC_EQ, // when used with FILE_PREDICATE
 169    CC_LE = 3,
 170    CC_GT = 4,
 171    CC_NE = 5,
 172    CC_P  = CC_NE,
 173    CC_GE = 6,
 174    CC_TR = 7,
 175    CC_ALWAYS = CC_TR,
 176    CC_U  = 8,
 177    CC_LTU = 9,
 178    CC_EQU = 10,
 179    CC_LEU = 11,
 180    CC_GTU = 12,
 181    CC_NEU = 13,
 182    CC_GEU = 14,
 183    CC_NO = 0x10,
 184    CC_NC = 0x11,
 185    CC_NS = 0x12,
 186    CC_NA = 0x13,
 187    CC_A  = 0x14,
 188    CC_S  = 0x15,
 189    CC_C  = 0x16,
 190    CC_O  = 0x17
 191 };
 192
 193 enum RoundMode
 194 {
 195    ROUND_N, // nearest
 196    ROUND_M, // towards -inf
 197    ROUND_Z, // towards 0
 198    ROUND_P, // towards +inf
 199    ROUND_NI, // nearest integer
 200    ROUND_MI, // to integer towards -inf
 201    ROUND_ZI, // to integer towards 0
 202    ROUND_PI, // to integer towards +inf
 203 };
 204
 205 enum CacheMode
 206 {
 207    CACHE_CA,            // cache at all levels
 208    CACHE_WB = CACHE_CA, // cache write back
 209    CACHE_CG,            // cache at global level
 210    CACHE_CS,            // cache streaming
 211    CACHE_CV,            // cache as volatile
 212    CACHE_WT = CACHE_CV  // cache write-through
 213 };
 214
 215 enum DataFile
 216 {
 217    FILE_NULL = 0,
 218    FILE_GPR,
 219    FILE_PREDICATE,       // boolean predicate
 220    FILE_FLAGS,           // zero/sign/carry/overflow bits
 221    FILE_ADDRESS,
 222    FILE_IMMEDIATE,
 223    FILE_MEMORY_CONST,
 224    FILE_SHADER_INPUT,
 225    FILE_SHADER_OUTPUT,
 226    FILE_MEMORY_GLOBAL,
 227    FILE_MEMORY_SHARED,
 228    FILE_MEMORY_LOCAL,
 229    FILE_SYSTEM_VALUE,
 230    DATA_FILE_COUNT
 231 };
 232
 233 enum TexTarget
 234 {
 235    TEX_TARGET_1D,
 236    TEX_TARGET_2D,
 237    TEX_TARGET_2D_MS,
 238    TEX_TARGET_3D,
 239    TEX_TARGET_CUBE,
 240    TEX_TARGET_1D_SHADOW,
 241    TEX_TARGET_2D_SHADOW,
 242    TEX_TARGET_CUBE_SHADOW,
 243    TEX_TARGET_1D_ARRAY,
 244    TEX_TARGET_2D_ARRAY,
 245    TEX_TARGET_2D_MS_ARRAY,
 246    TEX_TARGET_CUBE_ARRAY,
 247    TEX_TARGET_1D_ARRAY_SHADOW,
 248    TEX_TARGET_2D_ARRAY_SHADOW,
 249    TEX_TARGET_RECT,
 250    TEX_TARGET_RECT_SHADOW,
 251    TEX_TARGET_CUBE_ARRAY_SHADOW,
 252    TEX_TARGET_BUFFER,
 253    TEX_TARGET_COUNT
 254 };
 255
 256 enum SVSemantic
 257 {
 258    SV_POSITION, // WPOS
 259    SV_VERTEX_ID,
 260    SV_INSTANCE_ID,
 261    SV_INVOCATION_ID,
 262    SV_PRIMITIVE_ID,
 263    SV_VERTEX_COUNT, // gl_PatchVerticesIn
 264    SV_LAYER,
 265    SV_VIEWPORT_INDEX,
 266    SV_YDIR,
 267    SV_FACE,
 268    SV_POINT_SIZE,
 269    SV_POINT_COORD,
 270    SV_CLIP_DISTANCE,
 271    SV_SAMPLE_INDEX,
 272    SV_TESS_FACTOR,
 273    SV_TESS_COORD,
 274    SV_TID,
 275    SV_CTAID,
 276    SV_NTID,
 277    SV_GRIDID,
 278    SV_NCTAID,
 279    SV_LANEID,
 280    SV_PHYSID,
 281    SV_NPHYSID,
 282    SV_CLOCK,
 283    SV_LBASE,
 284    SV_SBASE,
 285    SV_UNDEFINED,
 286    SV_LAST
 287 };
 288
 289 class Program;
 290 class Function;
 291 class BasicBlock;
 292
 293 class Target;
 294
 295 class Instruction;
 296 class CmpInstruction;
 297 class TexInstruction;
 298 class FlowInstruction;
 299
 300 class Value;
 301 class LValue;
 302 class Symbol;
 303 class ImmediateValue;
 304
 305 struct Storage
 306 {
 307    DataFile file;
 308    int8_t fileIndex; // signed, may be indirect for CONST[]
 309    uint8_t size; // this should match the Instruction type's size
 310    DataType type; // mainly for pretty printing
 311    union {
 312       uint64_t u64;    // immediate values
 313       uint32_t u32;
 314       uint16_t u16;
 315       uint8_t u8;
 316       int64_t s64;
 317       int32_t s32;
 318       int16_t s16;
 319       int8_t s8;
 320       float f32;
 321       double f64;
 322       int32_t offset; // offset from 0 (base of address space)
 323       int32_t id;     // register id (< 0 if virtual/unassigned)
 324       struct {
 325          SVSemantic sv;
 326          int index;
 327       } sv;
 328    } data;
 329 };
 330
 331 // precedence: NOT after SAT after NEG after ABS
 332 #define NV50_IR_MOD_ABS (1 << 0)
 333 #define NV50_IR_MOD_NEG (1 << 1)
 334 #define NV50_IR_MOD_SAT (1 << 2)
 335 #define NV50_IR_MOD_NOT (1 << 3)
 336 #define NV50_IR_MOD_NEG_ABS (NV50_IR_MOD_NEG | NV50_IR_MOD_ABS)
 337
 338 #define NV50_IR_INTERP_MODE_MASK   0x3
 339 #define NV50_IR_INTERP_LINEAR      (0 << 0)
 340 #define NV50_IR_INTERP_PERSPECTIVE (1 << 0)
 341 #define NV50_IR_INTERP_FLAT        (2 << 0)
 342 #define NV50_IR_INTERP_SC          (3 << 0) // what exactly is that ?
 343 #define NV50_IR_INTERP_SAMPLE_MASK 0xc
 344 #define NV50_IR_INTERP_DEFAULT     (0 << 2)
 345 #define NV50_IR_INTERP_CENTROID    (1 << 2)
 346 #define NV50_IR_INTERP_OFFSET      (2 << 2)
 347 #define NV50_IR_INTERP_SAMPLEID    (3 << 2)
 348
 349 // do we really want this to be a class ?
 350 class Modifier
 351 {
 352 public:
 353    Modifier() : bits(0) { }
 354    Modifier(unsigned int m) : bits(m) { }
 355    Modifier(operation op);
 356
 357    // @return new Modifier applying a after b (asserts if unrepresentable)
 358    Modifier operator*(const Modifier) const;
 359    Modifier operator==(const Modifier m) const { return m.bits == bits; }
 360    Modifier operator!=(const Modifier m) const { return m.bits != bits; }
 361
 362    inline Modifier operator&(const Modifier m) const { return bits & m.bits; }
 363    inline Modifier operator|(const Modifier m) const { return bits | m.bits; }
 364    inline Modifier operator^(const Modifier m) const { return bits ^ m.bits; }
 365
 366    operation getOp() const;
 367
 368    inline int neg() const { return (bits & NV50_IR_MOD_NEG) ? 1 : 0; }
 369    inline int abs() const { return (bits & NV50_IR_MOD_ABS) ? 1 : 0; }
 370
 371    inline operator bool() { return bits ? true : false; }
 372
 373    void applyTo(ImmediateValue &imm) const;
 374
 375    int print(char *buf, size_t size) const;
 376
 377 private:
 378    uint8_t bits;
 379 };
 380
 381 class ValueRef
 382 {
 383 public:
 384    ValueRef();
 385    ValueRef(const ValueRef&);
 386    ~ValueRef();
 387
 388    inline bool exists() const { return value != NULL; }
 389
 390    void set(Value *);
 391    void set(const ValueRef&);
 392    inline Value *get() const { return value; }
 393    inline Value *rep() const;
 394
 395    inline Instruction *getInsn() const { return insn; }
 396    inline void setInsn(Instruction *inst) { insn = inst; }
 397
 398    inline bool isIndirect(int dim) const { return indirect[dim] >= 0; }
 399    inline const ValueRef *getIndirect(int dim) const;
 400
 401    inline DataFile getFile() const;
 402    inline unsigned getSize() const;
 403
 404    // SSA: return eventual (traverse MOVs) literal value, if it exists
 405    ImmediateValue *getImmediate() const;
 406
 407 public:
 408    Modifier mod;
 409    int8_t indirect[2]; // >= 0 if relative to lvalue in insn->src(indirect[i])
 410    uint8_t swizzle;
 411
 412    bool usedAsPtr; // for printing
 413
 414 private:
 415    Value *value;
 416    Instruction *insn;
 417 };
 418
 419 class ValueDef
 420 {
 421 public:
 422    ValueDef();
 423    ValueDef(const ValueDef&);
 424    ~ValueDef();
 425
 426    inline bool exists() const { return value != NULL; }
 427
 428    inline Value *get() const { return value; }
 429    inline Value *rep() const;
 430    void set(Value *);
 431    void replace(Value *, bool doSet); // replace all uses of the old value
 432
 433    inline Instruction *getInsn() const { return insn; }
 434    inline void setInsn(Instruction *inst) { insn = inst; }
 435
 436    inline DataFile getFile() const;
 437    inline unsigned getSize() const;
 438
 439    inline void setSSA(LValue *);
 440    inline const LValue *preSSA() const;
 441
 442 private:
 443    Value *value;   // should make this LValue * ...
 444    LValue *origin; // pre SSA value
 445    Instruction *insn;
 446 };
 447
 448 class Value
 449 {
 450 public:
 451    Value();
 452    virtual ~Value() { }
 453
 454    virtual Value *clone(ClonePolicy<Function>&) const = 0;
 455
 456    virtual int print(char *, size_t, DataType ty = TYPE_NONE) const = 0;
 457
 458    virtual bool equals(const Value *, bool strict = false) const;
 459    virtual bool interfers(const Value *) const;
 460
 461    inline Value *rep() const { return join; }
 462
 463    inline Instruction *getUniqueInsn() const;
 464    inline Instruction *getInsn() const; // use when uniqueness is certain
 465
 466    inline int refCount() { return uses.size(); }
 467
 468    inline LValue *asLValue();
 469    inline Symbol *asSym();
 470    inline ImmediateValue *asImm();
 471    inline const Symbol *asSym() const;
 472    inline const ImmediateValue *asImm() const;
 473
 474    bool coalesce(Value *, bool force = false);
 475
 476    inline bool inFile(DataFile f) { return reg.file == f; }
 477
 478    static inline Value *get(Iterator&);
 479
 480    std::list<ValueRef *> uses;
 481    std::list<ValueDef *> defs;
 482    typedef std::list<ValueRef *>::iterator UseIterator;
 483    typedef std::list<ValueRef *>::const_iterator UseCIterator;
 484    typedef std::list<ValueDef *>::iterator DefIterator;
 485    typedef std::list<ValueDef *>::const_iterator DefCIterator;
 486
 487    int id;
 488    Storage reg;
 489
 490    // TODO: these should be in LValue:
 491    Interval livei;
 492    Value *join;
 493 };
 494
 495 class LValue : public Value
 496 {
 497 public:
 498    LValue(Function *, DataFile file);
 499    LValue(Function *, LValue *);
 500    ~LValue() { }
 501
 502    virtual LValue *clone(ClonePolicy<Function>&) const;
 503
 504    virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
 505
 506 public:
 507    unsigned ssa : 1;
 508
 509    int affinity;
 510 };
 511
 512 class Symbol : public Value
 513 {
 514 public:
 515    Symbol(Program *, DataFile file = FILE_MEMORY_CONST, ubyte fileIdx = 0);
 516    ~Symbol() { }
 517
 518    virtual Symbol *clone(ClonePolicy<Function>&) const;
 519
 520    virtual bool equals(const Value *that, bool strict) const;
 521
 522    virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
 523
 524    // print with indirect values
 525    int print(char *, size_t, Value *, Value *, DataType ty = TYPE_NONE) const;
 526
 527    inline void setFile(DataFile file, ubyte fileIndex = 0)
 528    {
 529       reg.file = file;
 530       reg.fileIndex = fileIndex;
 531    }
 532
 533    inline void setOffset(int32_t offset);
 534    inline void setAddress(Symbol *base, int32_t offset);
 535    inline void setSV(SVSemantic sv, uint32_t idx = 0);
 536
 537    inline const Symbol *getBase() const { return baseSym; }
 538
 539 private:
 540    Symbol *baseSym; // array base for Symbols representing array elements
 541 };
 542
 543 class ImmediateValue : public Value
 544 {
 545 public:
 546    ImmediateValue(Program *, uint32_t);
 547    ImmediateValue(Program *, float);
 548    ImmediateValue(Program *, double);
 549    // NOTE: not added to program with
 550    ImmediateValue(const ImmediateValue *, DataType ty);
 551    ~ImmediateValue() { };
 552
 553    virtual ImmediateValue *clone(ClonePolicy<Function>&) const;
 554
 555    virtual bool equals(const Value *that, bool strict) const;
 556
 557    // these only work if 'type' is valid (we mostly use untyped literals):
 558    bool isInteger(const int ival) const; // ival is cast to this' type
 559    bool isNegative() const;
 560    bool isPow2() const;
 561
 562    void applyLog2();
 563
 564    // for constant folding:
 565    ImmediateValue operator+(const ImmediateValue&) const;
 566    ImmediateValue operator-(const ImmediateValue&) const;
 567    ImmediateValue operator*(const ImmediateValue&) const;
 568    ImmediateValue operator/(const ImmediateValue&) const;
 569
 570    bool compare(CondCode cc, float fval) const;
 571
 572    virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
 573 };
 574
 575 class Instruction
 576 {
 577 public:
 578    Instruction();
 579    Instruction(Function *, operation, DataType);
 580    virtual ~Instruction();
 581
 582    virtual Instruction *clone(ClonePolicy<Function>&,
 583                               Instruction * = NULL) const;
 584
 585    void setDef(int i, Value *);
 586    void setSrc(int s, Value *);
 587    void setSrc(int s, const ValueRef&);
 588    void swapSources(int a, int b);
 589    bool setIndirect(int s, int dim, Value *);
 590
 591    inline ValueRef& src(int s) { return srcs[s]; }
 592    inline ValueDef& def(int s) { return defs[s]; }
 593    inline const ValueRef& src(int s) const { return srcs[s]; }
 594    inline const ValueDef& def(int s) const { return defs[s]; }
 595
 596    inline Value *getDef(int d) const { return defs[d].get(); }
 597    inline Value *getSrc(int s) const { return srcs[s].get(); }
 598    inline Value *getIndirect(int s, int dim) const;
 599
 600    inline bool defExists(unsigned d) const
 601    {
 602       return d < defs.size() && defs[d].exists();
 603    }
 604    inline bool srcExists(unsigned s) const
 605    {
 606       return s < srcs.size() && srcs[s].exists();
 607    }
 608
 609    inline bool constrainedDefs() const { return defExists(1); }
 610
 611    bool setPredicate(CondCode ccode, Value *);
 612    inline Value *getPredicate() const;
 613    bool writesPredicate() const;
 614
 615    inline void setFlagsSrc(int s, Value *);
 616    inline void setFlagsDef(int d, Value *);
 617
 618    unsigned int defCount() const { return defs.size(); };
 619    unsigned int defCount(unsigned int mask) const;
 620    unsigned int srcCount() const { return srcs.size(); };
 621    unsigned int srcCount(unsigned int mask) const;
 622
 623    // save & remove / set indirect[0,1] and predicate source
 624    void takeExtraSources(int s, Value *[3]);
 625    void putExtraSources(int s, Value *[3]);
 626
 627    inline void setType(DataType type) { dType = sType = type; }
 628
 629    inline void setType(DataType dtype, DataType stype)
 630    {
 631       dType = dtype;
 632       sType = stype;
 633    }
 634
 635    inline bool isPseudo() const { return op < OP_MOV; }
 636    bool isDead() const;
 637    bool isNop() const;
 638    bool isCommutationLegal(const Instruction *) const; // must be adjacent !
 639    bool isActionEqual(const Instruction *) const;
 640    bool isResultEqual(const Instruction *) const;
 641
 642    void print() const;
 643
 644    inline CmpInstruction *asCmp();
 645    inline TexInstruction *asTex();
 646    inline FlowInstruction *asFlow();
 647    inline const TexInstruction *asTex() const;
 648    inline const CmpInstruction *asCmp() const;
 649    inline const FlowInstruction *asFlow() const;
 650
 651 public:
 652    Instruction *next;
 653    Instruction *prev;
 654    int id;
 655    int serial; // CFG order
 656
 657    operation op;
 658    DataType dType; // destination or defining type
 659    DataType sType; // source or secondary type
 660    CondCode cc;
 661    RoundMode rnd;
 662    CacheMode cache;
 663
 664    uint8_t subOp; // quadop, 1 for mul-high, etc.
 665
 666    unsigned encSize    : 4; // encoding size in bytes
 667    unsigned saturate   : 1; // to [0.0f, 1.0f]
 668    unsigned join       : 1; // converge control flow (use OP_JOIN until end)
 669    unsigned fixed      : 1; // prevent dead code elimination
 670    unsigned terminator : 1; // end of basic block
 671    unsigned atomic     : 1;
 672    unsigned ftz        : 1; // flush denormal to zero
 673    unsigned dnz        : 1; // denormals, NaN are zero
 674    unsigned ipa        : 4; // interpolation mode
 675    unsigned lanes      : 4;
 676    unsigned perPatch   : 1;
 677    unsigned exit       : 1; // terminate program after insn
 678
 679    int8_t postFactor; // MUL/DIV(if < 0) by 1 << postFactor
 680
 681    int8_t predSrc;
 682    int8_t flagsDef;
 683    int8_t flagsSrc;
 684
 685    BasicBlock *bb;
 686
 687 protected:
 688    std::deque<ValueDef> defs; // no gaps !
 689    std::deque<ValueRef> srcs; // no gaps !
 690
 691    // instruction specific methods:
 692    // (don't want to subclass, would need more constructors and memory pools)
 693 public:
 694    inline void setInterpolate(unsigned int mode) { ipa = mode; }
 695
 696    unsigned int getInterpMode() const { return ipa & 0x3; }
 697    unsigned int getSampleMode() const { return ipa & 0xc; }
 698
 699 private:
 700    void init();
 701 };
 702
 703 enum TexQuery
 704 {
 705    TXQ_DIMS,
 706    TXQ_TYPE,
 707    TXQ_SAMPLE_POSITION,
 708    TXQ_FILTER,
 709    TXQ_LOD,
 710    TXQ_WRAP,
 711    TXQ_BORDER_COLOUR
 712 };
 713
 714 class TexInstruction : public Instruction
 715 {
 716 public:
 717    class Target
 718    {
 719    public:
 720       Target(TexTarget targ = TEX_TARGET_2D) : target(targ) { }
 721
 722       const char *getName() const { return descTable[target].name; }
 723       unsigned int getArgCount() const { return descTable[target].argc; }
 724       unsigned int getDim() const { return descTable[target].dim; }
 725       int isArray() const { return descTable[target].array ? 1 : 0; }
 726       int isCube() const { return descTable[target].cube ? 1 : 0; }
 727       int isShadow() const { return descTable[target].shadow ? 1 : 0; }
 728
 729       Target& operator=(TexTarget targ)
 730       {
 731          assert(targ < TEX_TARGET_COUNT);
 732          return *this;
 733       }
 734
 735       inline bool operator==(TexTarget targ) const { return target == targ; }
 736
 737    private:
 738       struct Desc
 739       {
 740          char name[19];
 741          uint8_t dim;
 742          uint8_t argc;
 743          bool array;
 744          bool cube;
 745          bool shadow;
 746       };
 747
 748       static const struct Desc descTable[TEX_TARGET_COUNT];
 749
 750    private:
 751       enum TexTarget target;
 752    };
 753
 754 public:
 755    TexInstruction(Function *, operation);
 756    virtual ~TexInstruction();
 757
 758    virtual TexInstruction *clone(ClonePolicy<Function>&,
 759                                  Instruction * = NULL) const;
 760
 761    inline void setTexture(Target targ, uint8_t r, uint8_t s)
 762    {
 763       tex.r = r;
 764       tex.s = s;
 765       tex.target = targ;
 766    }
 767
 768    inline Value *getIndirectR() const;
 769    inline Value *getIndirectS() const;
 770
 771 public:
 772    struct {
 773       Target target;
 774
 775       uint8_t r;
 776       int8_t rIndirectSrc;
 777       uint8_t s;
 778       int8_t sIndirectSrc;
 779
 780       uint8_t mask;
 781       uint8_t gatherComp;
 782
 783       bool liveOnly; // only execute on live pixels of a quad (optimization)
 784       bool levelZero;
 785       bool derivAll;
 786
 787       int8_t useOffsets; // 0, 1, or 4 for textureGatherOffsets
 788       int8_t offset[4][3];
 789
 790       enum TexQuery query;
 791    } tex;
 792
 793    ValueRef dPdx[3];
 794    ValueRef dPdy[3];
 795 };
 796
 797 class CmpInstruction : public Instruction
 798 {
 799 public:
 800    CmpInstruction(Function *, operation);
 801
 802    virtual CmpInstruction *clone(ClonePolicy<Function>&,
 803                                  Instruction * = NULL) const;
 804
 805    void setCondition(CondCode cond) { setCond = cond; }
 806    CondCode getCondition() const { return setCond; }
 807
 808 public:
 809    CondCode setCond;
 810 };
 811
 812 class FlowInstruction : public Instruction
 813 {
 814 public:
 815    FlowInstruction(Function *, operation, BasicBlock *target);
 816
 817    virtual FlowInstruction *clone(ClonePolicy<Function>&,
 818                                   Instruction * = NULL) const;
 819
 820 public:
 821    unsigned allWarp  : 1;
 822    unsigned absolute : 1;
 823    unsigned limit    : 1;
 824    unsigned builtin  : 1; // true for calls to emulation code
 825
 826    union {
 827       BasicBlock *bb;
 828       int builtin;
 829       Function *fn;
 830    } target;
 831 };
 832
 833 class BasicBlock
 834 {
 835 public:
 836    BasicBlock(Function *);
 837    ~BasicBlock();
 838
 839    BasicBlock *clone(ClonePolicy<Function>&) const;
 840
 841    inline int getId() const { return id; }
 842    inline unsigned int getInsnCount() const { return numInsns; }
 843    inline bool isTerminated() const { return exit && exit->terminator; }
 844
 845    bool dominatedBy(BasicBlock *bb);
 846    inline bool reachableBy(BasicBlock *by, BasicBlock *term);
 847
 848    // returns mask of conditional out blocks
 849    // e.g. 3 for IF { .. } ELSE { .. } ENDIF, 1 for IF { .. } ENDIF
 850    unsigned int initiatesSimpleConditional() const;
 851
 852 public:
 853    Function *getFunction() const { return func; }
 854    Program *getProgram() const { return program; }
 855
 856    Instruction *getEntry() const { return entry; } // first non-phi instruction
 857    Instruction *getPhi() const { return phi; }
 858    Instruction *getFirst() const { return phi ? phi : entry; }
 859    Instruction *getExit() const { return exit; }
 860
 861    void insertHead(Instruction *);
 862    void insertTail(Instruction *);
 863    void insertBefore(Instruction *, Instruction *);
 864    void insertAfter(Instruction *, Instruction *);
 865    void remove(Instruction *);
 866    void permuteAdjacent(Instruction *, Instruction *);
 867
 868    BasicBlock *idom() const;
 869
 870    // NOTE: currently does not rebuild the dominator tree
 871    BasicBlock *splitBefore(Instruction *, bool attach = true);
 872    BasicBlock *splitAfter(Instruction *, bool attach = true);
 873
 874    DLList& getDF() { return df; }
 875    DLList::Iterator iterDF() { return df.iterator(); }
 876
 877    static inline BasicBlock *get(Iterator&);
 878    static inline BasicBlock *get(Graph::Node *);
 879
 880 public:
 881    Graph::Node cfg; // first edge is branch *taken* (the ELSE branch)
 882    Graph::Node dom;
 883
 884    BitSet liveSet;
 885
 886    uint32_t binPos;
 887    uint32_t binSize;
 888
 889    Instruction *joinAt; // for quick reference
 890
 891    bool explicitCont; // loop headers: true if loop contains continue stmts
 892
 893 private:
 894    int id;
 895    DLList df;
 896
 897    Instruction *phi;
 898    Instruction *entry;
 899    Instruction *exit;
 900
 901    unsigned int numInsns;
 902
 903 private:
 904    Function *func;
 905    Program *program;
 906
 907    void splitCommon(Instruction *, BasicBlock *, bool attach);
 908 };
 909
 910 class Function
 911 {
 912 public:
 913    Function(Program *, const char *name);
 914    ~Function();
 915
 916    inline Program *getProgram() const { return prog; }
 917    inline const char *getName() const { return name; }
 918    inline int getId() const { return id; }
 919
 920    void print();
 921    void printLiveIntervals() const;
 922    void printCFGraph(const char *filePath);
 923
 924    bool setEntry(BasicBlock *);
 925    bool setExit(BasicBlock *);
 926
 927    unsigned int orderInstructions(ArrayList&);
 928
 929    inline void add(BasicBlock *bb, int& id) { allBBlocks.insert(bb, id); }
 930    inline void add(Instruction *insn, int& id) { allInsns.insert(insn, id); }
 931    inline void add(LValue *lval, int& id) { allLValues.insert(lval, id); }
 932
 933    inline LValue *getLValue(int id);
 934
 935    bool convertToSSA();
 936
 937 public:
 938    Graph cfg;
 939    Graph::Node *cfgExit;
 940    Graph *domTree;
 941    Graph::Node call; // node in the call graph
 942
 943    BasicBlock **bbArray; // BBs in emission order
 944    int bbCount;
 945
 946    unsigned int loopNestingBound;
 947    int regClobberMax;
 948
 949    uint32_t binPos;
 950    uint32_t binSize;
 951
 952    ArrayList allBBlocks;
 953    ArrayList allInsns;
 954    ArrayList allLValues;
 955
 956 private:
 957    void buildLiveSetsPreSSA(BasicBlock *, const int sequence);
 958
 959 private:
 960    int id;
 961    const char *const name;
 962    Program *prog;
 963 };
 964
 965 enum CGStage
 966 {
 967    CG_STAGE_PRE_SSA,
 968    CG_STAGE_SSA, // expected directly before register allocation
 969    CG_STAGE_POST_RA
 970 };
 971
 972 class Program
 973 {
 974 public:
 975    enum Type
 976    {
 977       TYPE_VERTEX,
 978       TYPE_TESSELLATION_CONTROL,
 979       TYPE_TESSELLATION_EVAL,
 980       TYPE_GEOMETRY,
 981       TYPE_FRAGMENT,
 982       TYPE_COMPUTE
 983    };
 984
 985    Program(Type type, Target *targ);
 986    ~Program();
 987
 988    void print();
 989
 990    Type getType() const { return progType; }
 991
 992    inline void add(Function *fn, int& id) { allFuncs.insert(fn, id); }
 993    inline void add(Value *rval, int& id) { allRValues.insert(rval, id); }
 994
 995    bool makeFromTGSI(struct nv50_ir_prog_info *);
 996    bool makeFromSM4(struct nv50_ir_prog_info *);
 997    bool convertToSSA();
 998    bool optimizeSSA(int level);
 999    bool optimizePostRA(int level);
1000    bool registerAllocation();
1001    bool emitBinary(struct nv50_ir_prog_info *);
1002
1003    const Target *getTarget() const { return target; }
1004
1005 private:
1006    Type progType;
1007    Target *target;
1008
1009 public:
1010    Function *main;
1011    Graph calls;
1012
1013    ArrayList allFuncs;
1014    ArrayList allRValues;
1015
1016    uint32_t *code;
1017    uint32_t binSize;
1018
1019    int maxGPR;
1020
1021    MemoryPool mem_Instruction;
1022    MemoryPool mem_CmpInstruction;
1023    MemoryPool mem_TexInstruction;
1024    MemoryPool mem_FlowInstruction;
1025    MemoryPool mem_LValue;
1026    MemoryPool mem_Symbol;
1027    MemoryPool mem_ImmediateValue;
1028
1029    uint32_t dbgFlags;
1030
1031    void releaseInstruction(Instruction *);
1032    void releaseValue(Value *);
1033 };
1034
1035 // TODO: add const version
1036 class Pass
1037 {
1038 public:
1039    bool run(Program *, bool ordered = false, bool skipPhi = false);
1040    bool run(Function *, bool ordered = false, bool skipPhi = false);
1041
1042 private:
1043    // return false to continue with next entity on next higher level
1044    virtual bool visit(Function *) { return true; }
1045    virtual bool visit(BasicBlock *) { return true; }
1046    virtual bool visit(Instruction *) { return false; }
1047
1048    bool doRun(Program *, bool ordered, bool skipPhi);
1049    bool doRun(Function *, bool ordered, bool skipPhi);
1050
1051 protected:
1052    bool err;
1053    Function *func;
1054    Program *prog;
1055 };
1056
1057 // =============================================================================
1058
1059 #include "nv50_ir_inlines.h"
1060
1061 } // namespace nv50_ir
1062
1063 #endif // __NV50_IR_H__