src/gallium/drivers/nv50/codegen/nv50_ir.h

   1 /*
   2  * Copyright 2011 Christoph Bumiller
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  18  * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  19  * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  20  * SOFTWARE.
  21  */
  22
  23 #ifndef __NV50_IR_H__
  24 #define __NV50_IR_H__
  25
  26 #include <stdio.h>
  27 #include <stdlib.h>
  28 #include <stdint.h>
  29 #include <deque>
  30 #include <list>
  31 #include <vector>
  32
  33 #include "nv50_ir_util.h"
  34 #include "nv50_ir_graph.h"
  35
  36 #include "nv50_ir_driver.h"
  37
  38 namespace nv50_ir {
  39
  40 enum operation
  41 {
  42    OP_NOP = 0,
  43    OP_PHI,
  44    OP_UNION, // unify a new definition and several source values
  45    OP_SPLIT, // $r0d -> { $r0, $r1 } ($r0d and $r0/$r1 will be coalesced)
  46    OP_MERGE, // opposite of split, e.g. combine 2 32 bit into a 64 bit value
  47    OP_CONSTRAINT, // copy values into consecutive registers
  48    OP_MOV, // simple copy, no modifiers allowed
  49    OP_LOAD,
  50    OP_STORE,
  51    OP_ADD, // NOTE: add u64 + u32 is legal for targets w/o 64-bit integer adds
  52    OP_SUB,
  53    OP_MUL,
  54    OP_DIV,
  55    OP_MOD,
  56    OP_MAD,
  57    OP_FMA,
  58    OP_SAD, // abs(src0 - src1) + src2
  59    OP_ABS,
  60    OP_NEG,
  61    OP_NOT,
  62    OP_AND,
  63    OP_OR,
  64    OP_XOR,
  65    OP_SHL,
  66    OP_SHR,
  67    OP_MAX,
  68    OP_MIN,
  69    OP_SAT, // CLAMP(f32, 0.0, 1.0)
  70    OP_CEIL,
  71    OP_FLOOR,
  72    OP_TRUNC,
  73    OP_CVT,
  74    OP_SET_AND, // dst = (src0 CMP src1) & src2
  75    OP_SET_OR,
  76    OP_SET_XOR,
  77    OP_SET,
  78    OP_SELP, // dst = src2 ? src0 : src1
  79    OP_SLCT, // dst = (src2 CMP 0) ? src0 : src1
  80    OP_RCP,
  81    OP_RSQ,
  82    OP_LG2,
  83    OP_SIN,
  84    OP_COS,
  85    OP_EX2,
  86    OP_EXP, // exponential (base M_E)
  87    OP_LOG, // natural logarithm
  88    OP_PRESIN,
  89    OP_PREEX2,
  90    OP_SQRT,
  91    OP_POW,
  92    OP_BRA,
  93    OP_CALL,
  94    OP_RET,
  95    OP_CONT,
  96    OP_BREAK,
  97    OP_PRERET,
  98    OP_PRECONT,
  99    OP_PREBREAK,
 100    OP_BRKPT,     // breakpoint (not related to loops)
 101    OP_JOINAT,    // push control flow convergence point
 102    OP_JOIN,      // converge
 103    OP_DISCARD,
 104    OP_EXIT,
 105    OP_MEMBAR, // memory barrier (mfence, lfence, sfence)
 106    OP_VFETCH, // indirection 0 in attribute space, indirection 1 is vertex base
 107    OP_PFETCH, // fetch base address of vertex src0 (immediate) [+ src1]
 108    OP_EXPORT,
 109    OP_LINTERP,
 110    OP_PINTERP,
 111    OP_EMIT,    // emit vertex
 112    OP_RESTART, // restart primitive
 113    OP_TEX,
 114    OP_TXB, // texture bias
 115    OP_TXL, // texure lod
 116    OP_TXF, // texel fetch
 117    OP_TXQ, // texture size query
 118    OP_TXD, // texture derivatives
 119    OP_TXG, // texture gather
 120    OP_TEXCSAA, // texture op for coverage sampling
 121    OP_TEXPREP, // turn cube map array into 2d array coordinates
 122    OP_SULDB, // surface load (raw)
 123    OP_SULDP, // surface load (formatted)
 124    OP_SUSTB, // surface store (raw)
 125    OP_SUSTP, // surface store (formatted)
 126    OP_SUREDB,
 127    OP_SUREDP, // surface reduction (atomic op)
 128    OP_SULEA,   // surface load effective address
 129    OP_SUBFM,   // surface bitfield manipulation
 130    OP_SUCLAMP, // clamp surface coordinates
 131    OP_SUEAU,   // surface effective address
 132    OP_MADSP,   // special integer multiply-add
 133    OP_TEXBAR, // texture dependency barrier
 134    OP_DFDX,
 135    OP_DFDY,
 136    OP_RDSV, // read system value
 137    OP_WRSV, // write system value
 138    OP_QUADOP,
 139    OP_QUADON,
 140    OP_QUADPOP,
 141    OP_POPCNT, // bitcount(src0 & src1)
 142    OP_INSBF,  // insert first src1[8:15] bits of src0 into src2 at src1[0:7]
 143    OP_EXTBF,  // place bits [K,K+N) of src0 into dst, src1 = 0xNNKK
 144    OP_PERMT,  // dst = bytes from src2,src0 selected by src1 (nvc0's src order)
 145    OP_ATOM,
 146    OP_BAR,    // execution barrier, sources = { id, thread count, predicate }
 147    OP_VADD,   // byte/word vector operations
 148    OP_VAVG,
 149    OP_VMIN,
 150    OP_VMAX,
 151    OP_VSAD,
 152    OP_VSET,
 153    OP_VSHR,
 154    OP_VSHL,
 155    OP_VSEL,
 156    OP_LAST
 157 };
 158
 159 // various instruction-specific modifier definitions Instruction::subOp
 160 // MOV_FINAL marks a MOV originating from an EXPORT (used for placing TEXBARs)
 161 #define NV50_IR_SUBOP_MUL_HIGH     1
 162 #define NV50_IR_SUBOP_EMIT_RESTART 1
 163 #define NV50_IR_SUBOP_LDC_IL       1
 164 #define NV50_IR_SUBOP_LDC_IS       2
 165 #define NV50_IR_SUBOP_LDC_ISL      3
 166 #define NV50_IR_SUBOP_SHIFT_WRAP   1
 167 #define NV50_IR_SUBOP_EMU_PRERET   1
 168 #define NV50_IR_SUBOP_TEXBAR(n)    n
 169 #define NV50_IR_SUBOP_MOV_FINAL    1
 170 #define NV50_IR_SUBOP_EXTBF_REV    1
 171 #define NV50_IR_SUBOP_PERMT_F4E    1
 172 #define NV50_IR_SUBOP_PERMT_B4E    2
 173 #define NV50_IR_SUBOP_PERMT_RC8    3
 174 #define NV50_IR_SUBOP_PERMT_ECL    4
 175 #define NV50_IR_SUBOP_PERMT_ECR    5
 176 #define NV50_IR_SUBOP_PERMT_RC16   6
 177 #define NV50_IR_SUBOP_BAR_SYNC     0
 178 #define NV50_IR_SUBOP_BAR_ARRIVE   1
 179 #define NV50_IR_SUBOP_BAR_RED_AND  2
 180 #define NV50_IR_SUBOP_BAR_RED_OR   3
 181 #define NV50_IR_SUBOP_BAR_RED_POPC 4
 182 #define NV50_IR_SUBOP_MEMBAR_L     1
 183 #define NV50_IR_SUBOP_MEMBAR_S     2
 184 #define NV50_IR_SUBOP_MEMBAR_M     3
 185 #define NV50_IR_SUBOP_MEMBAR_CTA  (0 << 2)
 186 #define NV50_IR_SUBOP_MEMBAR_GL   (1 << 2)
 187 #define NV50_IR_SUBOP_MEMBAR_SYS  (2 << 2)
 188 #define NV50_IR_SUBOP_MEMBAR_DIR(m)   ((m) & 0x3)
 189 #define NV50_IR_SUBOP_MEMBAR_SCOPE(m) ((m) & ~0x3)
 190 #define NV50_IR_SUBOP_MEMBAR(d,s) \
 191    (NV50_IR_SUBOP_MEMBAR_##d | NV50_IR_SUBOP_MEMBAR_##s)
 192 #define NV50_IR_SUBOP_ATOM_ADD      0
 193 #define NV50_IR_SUBOP_ATOM_MIN      1
 194 #define NV50_IR_SUBOP_ATOM_MAX      2
 195 #define NV50_IR_SUBOP_ATOM_INC      3
 196 #define NV50_IR_SUBOP_ATOM_DEC      4
 197 #define NV50_IR_SUBOP_ATOM_AND      5
 198 #define NV50_IR_SUBOP_ATOM_OR       6
 199 #define NV50_IR_SUBOP_ATOM_XOR      7
 200 #define NV50_IR_SUBOP_ATOM_CAS      8
 201 #define NV50_IR_SUBOP_ATOM_EXCH     9
 202 #define NV50_IR_SUBOP_SUST_IGN     0
 203 #define NV50_IR_SUBOP_SUST_TRAP    1
 204 #define NV50_IR_SUBOP_SUST_SDCL    3
 205 #define NV50_IR_SUBOP_SULD_ZERO    0
 206 #define NV50_IR_SUBOP_SULD_TRAP    1
 207 #define NV50_IR_SUBOP_SULD_SDCL    3
 208 #define NV50_IR_SUBOP_SUBFM_3D     1
 209 #define NV50_IR_SUBOP_SUCLAMP_2D   0x10
 210 #define NV50_IR_SUBOP_SUCLAMP_SD(r, d) (( 0 + (r)) | ((d == 2) ? 0x10 : 0))
 211 #define NV50_IR_SUBOP_SUCLAMP_PL(r, d) (( 5 + (r)) | ((d == 2) ? 0x10 : 0))
 212 #define NV50_IR_SUBOP_SUCLAMP_BL(r, d) ((10 + (r)) | ((d == 2) ? 0x10 : 0))
 213 #define NV50_IR_SUBOP_MADSP_SD     0xffff
 214 // Yes, we could represent those with DataType.
 215 // Or put the type into operation and have a couple 1000 values in that enum.
 216 // This will have to do for now.
 217 // The bitfields are supposed to correspond to nve4 ISA.
 218 #define NV50_IR_SUBOP_MADSP(a,b,c) (((c) << 8) | ((b) << 4) | (a))
 219 #define NV50_IR_SUBOP_V1(d,a,b)    (((d) << 10) | ((b) << 5) | (a) | 0x0000)
 220 #define NV50_IR_SUBOP_V2(d,a,b)    (((d) << 10) | ((b) << 5) | (a) | 0x4000)
 221 #define NV50_IR_SUBOP_V4(d,a,b)    (((d) << 10) | ((b) << 5) | (a) | 0x8000)
 222 #define NV50_IR_SUBOP_Vn(n)        ((n) >> 14)
 223
 224 enum DataType
 225 {
 226    TYPE_NONE,
 227    TYPE_U8,
 228    TYPE_S8,
 229    TYPE_U16,
 230    TYPE_S16,
 231    TYPE_U32,
 232    TYPE_S32,
 233    TYPE_U64, // 64 bit operations are only lowered after register allocation
 234    TYPE_S64,
 235    TYPE_F16,
 236    TYPE_F32,
 237    TYPE_F64,
 238    TYPE_B96,
 239    TYPE_B128
 240 };
 241
 242 enum CondCode
 243 {
 244    CC_FL = 0,
 245    CC_NEVER = CC_FL, // when used with FILE_FLAGS
 246    CC_LT = 1,
 247    CC_EQ = 2,
 248    CC_NOT_P = CC_EQ, // when used with FILE_PREDICATE
 249    CC_LE = 3,
 250    CC_GT = 4,
 251    CC_NE = 5,
 252    CC_P  = CC_NE,
 253    CC_GE = 6,
 254    CC_TR = 7,
 255    CC_ALWAYS = CC_TR,
 256    CC_U  = 8,
 257    CC_LTU = 9,
 258    CC_EQU = 10,
 259    CC_LEU = 11,
 260    CC_GTU = 12,
 261    CC_NEU = 13,
 262    CC_GEU = 14,
 263    CC_NO = 0x10,
 264    CC_NC = 0x11,
 265    CC_NS = 0x12,
 266    CC_NA = 0x13,
 267    CC_A  = 0x14,
 268    CC_S  = 0x15,
 269    CC_C  = 0x16,
 270    CC_O  = 0x17
 271 };
 272
 273 enum RoundMode
 274 {
 275    ROUND_N, // nearest
 276    ROUND_M, // towards -inf
 277    ROUND_Z, // towards 0
 278    ROUND_P, // towards +inf
 279    ROUND_NI, // nearest integer
 280    ROUND_MI, // to integer towards -inf
 281    ROUND_ZI, // to integer towards 0
 282    ROUND_PI, // to integer towards +inf
 283 };
 284
 285 enum CacheMode
 286 {
 287    CACHE_CA,            // cache at all levels
 288    CACHE_WB = CACHE_CA, // cache write back
 289    CACHE_CG,            // cache at global level
 290    CACHE_CS,            // cache streaming
 291    CACHE_CV,            // cache as volatile
 292    CACHE_WT = CACHE_CV  // cache write-through
 293 };
 294
 295 enum DataFile
 296 {
 297    FILE_NULL = 0,
 298    FILE_GPR,
 299    FILE_PREDICATE,       // boolean predicate
 300    FILE_FLAGS,           // zero/sign/carry/overflow bits
 301    FILE_ADDRESS,
 302    LAST_REGISTER_FILE = FILE_ADDRESS,
 303    FILE_IMMEDIATE,
 304    FILE_MEMORY_CONST,
 305    FILE_SHADER_INPUT,
 306    FILE_SHADER_OUTPUT,
 307    FILE_MEMORY_GLOBAL,
 308    FILE_MEMORY_SHARED,
 309    FILE_MEMORY_LOCAL,
 310    FILE_SYSTEM_VALUE,
 311    DATA_FILE_COUNT
 312 };
 313
 314 enum TexTarget
 315 {
 316    TEX_TARGET_1D,
 317    TEX_TARGET_2D,
 318    TEX_TARGET_2D_MS,
 319    TEX_TARGET_3D,
 320    TEX_TARGET_CUBE,
 321    TEX_TARGET_1D_SHADOW,
 322    TEX_TARGET_2D_SHADOW,
 323    TEX_TARGET_CUBE_SHADOW,
 324    TEX_TARGET_1D_ARRAY,
 325    TEX_TARGET_2D_ARRAY,
 326    TEX_TARGET_2D_MS_ARRAY,
 327    TEX_TARGET_CUBE_ARRAY,
 328    TEX_TARGET_1D_ARRAY_SHADOW,
 329    TEX_TARGET_2D_ARRAY_SHADOW,
 330    TEX_TARGET_RECT,
 331    TEX_TARGET_RECT_SHADOW,
 332    TEX_TARGET_CUBE_ARRAY_SHADOW,
 333    TEX_TARGET_BUFFER,
 334    TEX_TARGET_COUNT
 335 };
 336
 337 enum SVSemantic
 338 {
 339    SV_POSITION, // WPOS
 340    SV_VERTEX_ID,
 341    SV_INSTANCE_ID,
 342    SV_INVOCATION_ID,
 343    SV_PRIMITIVE_ID,
 344    SV_VERTEX_COUNT, // gl_PatchVerticesIn
 345    SV_LAYER,
 346    SV_VIEWPORT_INDEX,
 347    SV_YDIR,
 348    SV_FACE,
 349    SV_POINT_SIZE,
 350    SV_POINT_COORD,
 351    SV_CLIP_DISTANCE,
 352    SV_SAMPLE_INDEX,
 353    SV_TESS_FACTOR,
 354    SV_TESS_COORD,
 355    SV_TID,
 356    SV_CTAID,
 357    SV_NTID,
 358    SV_GRIDID,
 359    SV_NCTAID,
 360    SV_LANEID,
 361    SV_PHYSID,
 362    SV_NPHYSID,
 363    SV_CLOCK,
 364    SV_LBASE,
 365    SV_SBASE,
 366    SV_UNDEFINED,
 367    SV_LAST
 368 };
 369
 370 class Program;
 371 class Function;
 372 class BasicBlock;
 373
 374 class Target;
 375
 376 class Instruction;
 377 class CmpInstruction;
 378 class TexInstruction;
 379 class FlowInstruction;
 380
 381 class Value;
 382 class LValue;
 383 class Symbol;
 384 class ImmediateValue;
 385
 386 struct Storage
 387 {
 388    DataFile file;
 389    int8_t fileIndex; // signed, may be indirect for CONST[]
 390    uint8_t size; // this should match the Instruction type's size
 391    DataType type; // mainly for pretty printing
 392    union {
 393       uint64_t u64;    // immediate values
 394       uint32_t u32;
 395       uint16_t u16;
 396       uint8_t u8;
 397       int64_t s64;
 398       int32_t s32;
 399       int16_t s16;
 400       int8_t s8;
 401       float f32;
 402       double f64;
 403       int32_t offset; // offset from 0 (base of address space)
 404       int32_t id;     // register id (< 0 if virtual/unassigned, in units <= 4)
 405       struct {
 406          SVSemantic sv;
 407          int index;
 408       } sv;
 409    } data;
 410 };
 411
 412 // precedence: NOT after SAT after NEG after ABS
 413 #define NV50_IR_MOD_ABS (1 << 0)
 414 #define NV50_IR_MOD_NEG (1 << 1)
 415 #define NV50_IR_MOD_SAT (1 << 2)
 416 #define NV50_IR_MOD_NOT (1 << 3)
 417 #define NV50_IR_MOD_NEG_ABS (NV50_IR_MOD_NEG | NV50_IR_MOD_ABS)
 418
 419 #define NV50_IR_INTERP_MODE_MASK   0x3
 420 #define NV50_IR_INTERP_LINEAR      (0 << 0)
 421 #define NV50_IR_INTERP_PERSPECTIVE (1 << 0)
 422 #define NV50_IR_INTERP_FLAT        (2 << 0)
 423 #define NV50_IR_INTERP_SC          (3 << 0) // what exactly is that ?
 424 #define NV50_IR_INTERP_SAMPLE_MASK 0xc
 425 #define NV50_IR_INTERP_DEFAULT     (0 << 2)
 426 #define NV50_IR_INTERP_CENTROID    (1 << 2)
 427 #define NV50_IR_INTERP_OFFSET      (2 << 2)
 428 #define NV50_IR_INTERP_SAMPLEID    (3 << 2)
 429
 430 // do we really want this to be a class ?
 431 class Modifier
 432 {
 433 public:
 434    Modifier() : bits(0) { }
 435    Modifier(unsigned int m) : bits(m) { }
 436    Modifier(operation op);
 437
 438    // @return new Modifier applying a after b (asserts if unrepresentable)
 439    Modifier operator*(const Modifier) const;
 440    Modifier operator*=(const Modifier m) { *this = *this * m; return *this; }
 441    Modifier operator==(const Modifier m) const { return m.bits == bits; }
 442    Modifier operator!=(const Modifier m) const { return m.bits != bits; }
 443
 444    inline Modifier operator&(const Modifier m) const { return bits & m.bits; }
 445    inline Modifier operator|(const Modifier m) const { return bits | m.bits; }
 446    inline Modifier operator^(const Modifier m) const { return bits ^ m.bits; }
 447
 448    operation getOp() const;
 449
 450    inline int neg() const { return (bits & NV50_IR_MOD_NEG) ? 1 : 0; }
 451    inline int abs() const { return (bits & NV50_IR_MOD_ABS) ? 1 : 0; }
 452
 453    inline operator bool() const { return bits ? true : false; }
 454
 455    void applyTo(ImmediateValue &imm) const;
 456
 457    int print(char *buf, size_t size) const;
 458
 459 private:
 460    uint8_t bits;
 461 };
 462
 463 class ValueRef
 464 {
 465 public:
 466    ValueRef(Value * = NULL);
 467    ValueRef(const ValueRef&);
 468    ~ValueRef();
 469
 470    inline bool exists() const { return value != NULL; }
 471
 472    void set(Value *);
 473    void set(const ValueRef&);
 474    inline Value *get() const { return value; }
 475    inline Value *rep() const;
 476
 477    inline Instruction *getInsn() const { return insn; }
 478    inline void setInsn(Instruction *inst) { insn = inst; }
 479
 480    inline bool isIndirect(int dim) const { return indirect[dim] >= 0; }
 481    inline const ValueRef *getIndirect(int dim) const;
 482
 483    inline DataFile getFile() const;
 484    inline unsigned getSize() const;
 485
 486    // SSA: return eventual (traverse MOVs) literal value, if it exists
 487    bool getImmediate(ImmediateValue&) const;
 488
 489 public:
 490    Modifier mod;
 491    int8_t indirect[2]; // >= 0 if relative to lvalue in insn->src(indirect[i])
 492    uint8_t swizzle;
 493
 494    bool usedAsPtr; // for printing
 495
 496 private:
 497    Value *value;
 498    Instruction *insn;
 499 };
 500
 501 class ValueDef
 502 {
 503 public:
 504    ValueDef(Value * = NULL);
 505    ValueDef(const ValueDef&);
 506    ~ValueDef();
 507
 508    inline bool exists() const { return value != NULL; }
 509
 510    inline Value *get() const { return value; }
 511    inline Value *rep() const;
 512    void set(Value *);
 513    bool mayReplace(const ValueRef &);
 514    void replace(const ValueRef &, bool doSet); // replace all uses of the old value
 515
 516    inline Instruction *getInsn() const { return insn; }
 517    inline void setInsn(Instruction *inst) { insn = inst; }
 518
 519    inline DataFile getFile() const;
 520    inline unsigned getSize() const;
 521
 522    inline void setSSA(LValue *);
 523    inline const LValue *preSSA() const;
 524
 525 private:
 526    Value *value;   // should make this LValue * ...
 527    LValue *origin; // pre SSA value
 528    Instruction *insn;
 529 };
 530
 531 class Value
 532 {
 533 public:
 534    Value();
 535    virtual ~Value() { }
 536
 537    virtual Value *clone(ClonePolicy<Function>&) const = 0;
 538
 539    virtual int print(char *, size_t, DataType ty = TYPE_NONE) const = 0;
 540
 541    virtual bool equals(const Value *, bool strict = false) const;
 542    virtual bool interfers(const Value *) const;
 543    virtual bool isUniform() const { return true; }
 544
 545    inline Value *rep() const { return join; }
 546
 547    inline Instruction *getUniqueInsn() const;
 548    inline Instruction *getInsn() const; // use when uniqueness is certain
 549
 550    inline int refCount() { return uses.size(); }
 551
 552    inline LValue *asLValue();
 553    inline Symbol *asSym();
 554    inline ImmediateValue *asImm();
 555    inline const Symbol *asSym() const;
 556    inline const ImmediateValue *asImm() const;
 557
 558    inline bool inFile(DataFile f) { return reg.file == f; }
 559
 560    static inline Value *get(Iterator&);
 561
 562    std::list<ValueRef *> uses;
 563    std::list<ValueDef *> defs;
 564    typedef std::list<ValueRef *>::iterator UseIterator;
 565    typedef std::list<ValueRef *>::const_iterator UseCIterator;
 566    typedef std::list<ValueDef *>::iterator DefIterator;
 567    typedef std::list<ValueDef *>::const_iterator DefCIterator;
 568
 569    int id;
 570    Storage reg;
 571
 572    // TODO: these should be in LValue:
 573    Interval livei;
 574    Value *join;
 575 };
 576
 577 class LValue : public Value
 578 {
 579 public:
 580    LValue(Function *, DataFile file);
 581    LValue(Function *, LValue *);
 582    ~LValue() { }
 583
 584    virtual bool isUniform() const;
 585
 586    virtual LValue *clone(ClonePolicy<Function>&) const;
 587
 588    virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
 589
 590 public:
 591    unsigned compMask : 8; // compound/component mask
 592    unsigned compound : 1; // used by RA, value involved in split/merge
 593    unsigned ssa      : 1;
 594    unsigned fixedReg : 1; // set & used by RA, earlier just use (id < 0)
 595    unsigned noSpill  : 1; // do not spill (e.g. if spill temporary already)
 596 };
 597
 598 class Symbol : public Value
 599 {
 600 public:
 601    Symbol(Program *, DataFile file = FILE_MEMORY_CONST, ubyte fileIdx = 0);
 602    ~Symbol() { }
 603
 604    virtual Symbol *clone(ClonePolicy<Function>&) const;
 605
 606    virtual bool equals(const Value *that, bool strict) const;
 607
 608    virtual bool isUniform() const;
 609
 610    virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
 611
 612    // print with indirect values
 613    int print(char *, size_t, Value *, Value *, DataType ty = TYPE_NONE) const;
 614
 615    inline void setFile(DataFile file, ubyte fileIndex = 0)
 616    {
 617       reg.file = file;
 618       reg.fileIndex = fileIndex;
 619    }
 620
 621    inline void setOffset(int32_t offset);
 622    inline void setAddress(Symbol *base, int32_t offset);
 623    inline void setSV(SVSemantic sv, uint32_t idx = 0);
 624
 625    inline const Symbol *getBase() const { return baseSym; }
 626
 627 private:
 628    Symbol *baseSym; // array base for Symbols representing array elements
 629 };
 630
 631 class ImmediateValue : public Value
 632 {
 633 public:
 634    ImmediateValue() { }
 635    ImmediateValue(Program *, uint32_t);
 636    ImmediateValue(Program *, float);
 637    ImmediateValue(Program *, double);
 638    // NOTE: not added to program with
 639    ImmediateValue(const ImmediateValue *, DataType ty);
 640    ~ImmediateValue() { };
 641
 642    virtual ImmediateValue *clone(ClonePolicy<Function>&) const;
 643
 644    virtual bool equals(const Value *that, bool strict) const;
 645
 646    // these only work if 'type' is valid (we mostly use untyped literals):
 647    bool isInteger(const int ival) const; // ival is cast to this' type
 648    bool isNegative() const;
 649    bool isPow2() const;
 650
 651    void applyLog2();
 652
 653    // for constant folding:
 654    ImmediateValue operator+(const ImmediateValue&) const;
 655    ImmediateValue operator-(const ImmediateValue&) const;
 656    ImmediateValue operator*(const ImmediateValue&) const;
 657    ImmediateValue operator/(const ImmediateValue&) const;
 658
 659    ImmediateValue& operator=(const ImmediateValue&); // only sets value !
 660
 661    bool compare(CondCode cc, float fval) const;
 662
 663    virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
 664 };
 665
 666 class Instruction
 667 {
 668 public:
 669    Instruction();
 670    Instruction(Function *, operation, DataType);
 671    virtual ~Instruction();
 672
 673    virtual Instruction *clone(ClonePolicy<Function>&,
 674                               Instruction * = NULL) const;
 675
 676    void setDef(int i, Value *);
 677    void setSrc(int s, Value *);
 678    void setSrc(int s, const ValueRef&);
 679    void swapSources(int a, int b);
 680    void moveSources(int s, int delta);
 681    bool setIndirect(int s, int dim, Value *);
 682
 683    inline ValueRef& src(int s) { return srcs[s]; }
 684    inline ValueDef& def(int s) { return defs[s]; }
 685    inline const ValueRef& src(int s) const { return srcs[s]; }
 686    inline const ValueDef& def(int s) const { return defs[s]; }
 687
 688    inline Value *getDef(int d) const { return defs[d].get(); }
 689    inline Value *getSrc(int s) const { return srcs[s].get(); }
 690    inline Value *getIndirect(int s, int dim) const;
 691
 692    inline bool defExists(unsigned d) const
 693    {
 694       return d < defs.size() && defs[d].exists();
 695    }
 696    inline bool srcExists(unsigned s) const
 697    {
 698       return s < srcs.size() && srcs[s].exists();
 699    }
 700
 701    inline bool constrainedDefs() const;
 702
 703    bool setPredicate(CondCode ccode, Value *);
 704    inline Value *getPredicate() const;
 705    bool writesPredicate() const;
 706    inline bool isPredicated() const { return predSrc >= 0; }
 707
 708    inline void setFlagsSrc(int s, Value *);
 709    inline void setFlagsDef(int d, Value *);
 710    inline bool usesFlags() const { return flagsSrc >= 0; }
 711
 712    unsigned int defCount() const { return defs.size(); };
 713    unsigned int defCount(unsigned int mask, bool singleFile = false) const;
 714    unsigned int srcCount() const { return srcs.size(); };
 715    unsigned int srcCount(unsigned int mask, bool singleFile = false) const;
 716
 717    // save & remove / set indirect[0,1] and predicate source
 718    void takeExtraSources(int s, Value *[3]);
 719    void putExtraSources(int s, Value *[3]);
 720
 721    inline void setType(DataType type) { dType = sType = type; }
 722
 723    inline void setType(DataType dtype, DataType stype)
 724    {
 725       dType = dtype;
 726       sType = stype;
 727    }
 728
 729    inline bool isPseudo() const { return op < OP_MOV; }
 730    bool isDead() const;
 731    bool isNop() const;
 732    bool isCommutationLegal(const Instruction *) const; // must be adjacent !
 733    bool isActionEqual(const Instruction *) const;
 734    bool isResultEqual(const Instruction *) const;
 735
 736    void print() const;
 737
 738    inline CmpInstruction *asCmp();
 739    inline TexInstruction *asTex();
 740    inline FlowInstruction *asFlow();
 741    inline const TexInstruction *asTex() const;
 742    inline const CmpInstruction *asCmp() const;
 743    inline const FlowInstruction *asFlow() const;
 744
 745 public:
 746    Instruction *next;
 747    Instruction *prev;
 748    int id;
 749    int serial; // CFG order
 750
 751    operation op;
 752    DataType dType; // destination or defining type
 753    DataType sType; // source or secondary type
 754    CondCode cc;
 755    RoundMode rnd;
 756    CacheMode cache;
 757
 758    uint16_t subOp; // quadop, 1 for mul-high, etc.
 759
 760    unsigned encSize    : 4; // encoding size in bytes
 761    unsigned saturate   : 1; // to [0.0f, 1.0f]
 762    unsigned join       : 1; // converge control flow (use OP_JOIN until end)
 763    unsigned fixed      : 1; // prevent dead code elimination
 764    unsigned terminator : 1; // end of basic block
 765    unsigned ftz        : 1; // flush denormal to zero
 766    unsigned dnz        : 1; // denormals, NaN are zero
 767    unsigned ipa        : 4; // interpolation mode
 768    unsigned lanes      : 4;
 769    unsigned perPatch   : 1;
 770    unsigned exit       : 1; // terminate program after insn
 771    unsigned mask       : 4; // for vector ops
 772
 773    int8_t postFactor; // MUL/DIV(if < 0) by 1 << postFactor
 774
 775    int8_t predSrc;
 776    int8_t flagsDef;
 777    int8_t flagsSrc;
 778
 779    uint8_t sched; // scheduling data (NOTE: maybe move to separate storage)
 780
 781    BasicBlock *bb;
 782
 783 protected:
 784    std::deque<ValueDef> defs; // no gaps !
 785    std::deque<ValueRef> srcs; // no gaps !
 786
 787    // instruction specific methods:
 788    // (don't want to subclass, would need more constructors and memory pools)
 789 public:
 790    inline void setInterpolate(unsigned int mode) { ipa = mode; }
 791
 792    unsigned int getInterpMode() const { return ipa & 0x3; }
 793    unsigned int getSampleMode() const { return ipa & 0xc; }
 794
 795 private:
 796    void init();
 797 };
 798
 799 enum TexQuery
 800 {
 801    TXQ_DIMS,
 802    TXQ_TYPE,
 803    TXQ_SAMPLE_POSITION,
 804    TXQ_FILTER,
 805    TXQ_LOD,
 806    TXQ_WRAP,
 807    TXQ_BORDER_COLOUR
 808 };
 809
 810 class TexInstruction : public Instruction
 811 {
 812 public:
 813    class Target
 814    {
 815    public:
 816       Target(TexTarget targ = TEX_TARGET_2D) : target(targ) { }
 817
 818       const char *getName() const { return descTable[target].name; }
 819       unsigned int getArgCount() const { return descTable[target].argc; }
 820       unsigned int getDim() const { return descTable[target].dim; }
 821       int isArray() const { return descTable[target].array ? 1 : 0; }
 822       int isCube() const { return descTable[target].cube ? 1 : 0; }
 823       int isShadow() const { return descTable[target].shadow ? 1 : 0; }
 824
 825       Target& operator=(TexTarget targ)
 826       {
 827          assert(targ < TEX_TARGET_COUNT);
 828          target = targ;
 829          return *this;
 830       }
 831
 832       inline bool operator==(TexTarget targ) const { return target == targ; }
 833       inline bool operator!=(TexTarget targ) const { return target != targ; }
 834
 835       enum TexTarget getEnum() const { return target; }
 836
 837    private:
 838       struct Desc
 839       {
 840          char name[19];
 841          uint8_t dim;
 842          uint8_t argc;
 843          bool array;
 844          bool cube;
 845          bool shadow;
 846       };
 847
 848       static const struct Desc descTable[TEX_TARGET_COUNT];
 849
 850    private:
 851       enum TexTarget target;
 852    };
 853
 854 public:
 855    TexInstruction(Function *, operation);
 856    virtual ~TexInstruction();
 857
 858    virtual TexInstruction *clone(ClonePolicy<Function>&,
 859                                  Instruction * = NULL) const;
 860
 861    inline void setTexture(Target targ, uint8_t r, uint8_t s)
 862    {
 863       tex.r = r;
 864       tex.s = s;
 865       tex.target = targ;
 866    }
 867
 868    inline Value *getIndirectR() const;
 869    inline Value *getIndirectS() const;
 870
 871 public:
 872    struct {
 873       Target target;
 874
 875       uint16_t r;
 876       uint16_t s;
 877       int8_t rIndirectSrc;
 878       int8_t sIndirectSrc;
 879
 880       uint8_t mask;
 881       uint8_t gatherComp;
 882
 883       bool liveOnly; // only execute on live pixels of a quad (optimization)
 884       bool levelZero;
 885       bool derivAll;
 886
 887       int8_t useOffsets; // 0, 1, or 4 for textureGatherOffsets
 888       int8_t offset[4][3];
 889
 890       enum TexQuery query;
 891    } tex;
 892
 893    ValueRef dPdx[3];
 894    ValueRef dPdy[3];
 895 };
 896
 897 class CmpInstruction : public Instruction
 898 {
 899 public:
 900    CmpInstruction(Function *, operation);
 901
 902    virtual CmpInstruction *clone(ClonePolicy<Function>&,
 903                                  Instruction * = NULL) const;
 904
 905    void setCondition(CondCode cond) { setCond = cond; }
 906    CondCode getCondition() const { return setCond; }
 907
 908 public:
 909    CondCode setCond;
 910 };
 911
 912 class FlowInstruction : public Instruction
 913 {
 914 public:
 915    FlowInstruction(Function *, operation, void *target);
 916
 917    virtual FlowInstruction *clone(ClonePolicy<Function>&,
 918                                   Instruction * = NULL) const;
 919
 920 public:
 921    unsigned allWarp  : 1;
 922    unsigned absolute : 1;
 923    unsigned limit    : 1;
 924    unsigned builtin  : 1; // true for calls to emulation code
 925    unsigned indirect : 1; // target in src(0)
 926
 927    union {
 928       BasicBlock *bb;
 929       int builtin;
 930       Function *fn;
 931    } target;
 932 };
 933
 934 class BasicBlock
 935 {
 936 public:
 937    BasicBlock(Function *);
 938    ~BasicBlock();
 939
 940    BasicBlock *clone(ClonePolicy<Function>&) const;
 941
 942    inline int getId() const { return id; }
 943    inline unsigned int getInsnCount() const { return numInsns; }
 944    inline bool isTerminated() const { return exit && exit->terminator; }
 945
 946    bool dominatedBy(BasicBlock *bb);
 947    inline bool reachableBy(const BasicBlock *by, const BasicBlock *term);
 948
 949    // returns mask of conditional out blocks
 950    // e.g. 3 for IF { .. } ELSE { .. } ENDIF, 1 for IF { .. } ENDIF
 951    unsigned int initiatesSimpleConditional() const;
 952
 953 public:
 954    Function *getFunction() const { return func; }
 955    Program *getProgram() const { return program; }
 956
 957    Instruction *getEntry() const { return entry; } // first non-phi instruction
 958    Instruction *getPhi() const { return phi; }
 959    Instruction *getFirst() const { return phi ? phi : entry; }
 960    Instruction *getExit() const { return exit; }
 961
 962    void insertHead(Instruction *);
 963    void insertTail(Instruction *);
 964    void insertBefore(Instruction *, Instruction *);
 965    void insertAfter(Instruction *, Instruction *);
 966    void remove(Instruction *);
 967    void permuteAdjacent(Instruction *, Instruction *);
 968
 969    BasicBlock *idom() const;
 970
 971    // NOTE: currently does not rebuild the dominator tree
 972    BasicBlock *splitBefore(Instruction *, bool attach = true);
 973    BasicBlock *splitAfter(Instruction *, bool attach = true);
 974
 975    DLList& getDF() { return df; }
 976    DLList::Iterator iterDF() { return df.iterator(); }
 977
 978    static inline BasicBlock *get(Iterator&);
 979    static inline BasicBlock *get(Graph::Node *);
 980
 981 public:
 982    Graph::Node cfg; // first edge is branch *taken* (the ELSE branch)
 983    Graph::Node dom;
 984
 985    BitSet liveSet;
 986    BitSet defSet;
 987
 988    uint32_t binPos;
 989    uint32_t binSize;
 990
 991    Instruction *joinAt; // for quick reference
 992
 993    bool explicitCont; // loop headers: true if loop contains continue stmts
 994
 995 private:
 996    int id;
 997    DLList df;
 998
 999    Instruction *phi;
1000    Instruction *entry;
1001    Instruction *exit;
1002
1003    unsigned int numInsns;
1004
1005 private:
1006    Function *func;
1007    Program *program;
1008
1009    void splitCommon(Instruction *, BasicBlock *, bool attach);
1010 };
1011
1012 class Function
1013 {
1014 public:
1015    Function(Program *, const char *name, uint32_t label);
1016    ~Function();
1017
1018    static inline Function *get(Graph::Node *node);
1019
1020    inline Program *getProgram() const { return prog; }
1021    inline const char *getName() const { return name; }
1022    inline int getId() const { return id; }
1023    inline uint32_t getLabel() const { return label; }
1024
1025    void print();
1026    void printLiveIntervals() const;
1027    void printCFGraph(const char *filePath);
1028
1029    bool setEntry(BasicBlock *);
1030    bool setExit(BasicBlock *);
1031
1032    unsigned int orderInstructions(ArrayList&);
1033
1034    inline void add(BasicBlock *bb, int& id) { allBBlocks.insert(bb, id); }
1035    inline void add(Instruction *insn, int& id) { allInsns.insert(insn, id); }
1036    inline void add(LValue *lval, int& id) { allLValues.insert(lval, id); }
1037
1038    inline LValue *getLValue(int id);
1039
1040    void buildLiveSets();
1041    void buildDefSets();
1042    bool convertToSSA();
1043
1044 public:
1045    std::deque<ValueDef> ins;
1046    std::deque<ValueRef> outs;
1047    std::deque<Value *> clobbers;
1048
1049    Graph cfg;
1050    Graph::Node *cfgExit;
1051    Graph *domTree;
1052    Graph::Node call; // node in the call graph
1053
1054    BasicBlock **bbArray; // BBs in emission order
1055    int bbCount;
1056
1057    unsigned int loopNestingBound;
1058    int regClobberMax;
1059
1060    uint32_t binPos;
1061    uint32_t binSize;
1062
1063    Value *stackPtr;
1064
1065    uint32_t tlsBase; // base address for l[] space (if no stack pointer is used)
1066    uint32_t tlsSize;
1067
1068    ArrayList allBBlocks;
1069    ArrayList allInsns;
1070    ArrayList allLValues;
1071
1072 private:
1073    void buildLiveSetsPreSSA(BasicBlock *, const int sequence);
1074    void buildDefSetsPreSSA(BasicBlock *bb, const int seq);
1075
1076 private:
1077    uint32_t label;
1078    int id;
1079    const char *const name;
1080    Program *prog;
1081 };
1082
1083 enum CGStage
1084 {
1085    CG_STAGE_PRE_SSA,
1086    CG_STAGE_SSA, // expected directly before register allocation
1087    CG_STAGE_POST_RA
1088 };
1089
1090 class Program
1091 {
1092 public:
1093    enum Type
1094    {
1095       TYPE_VERTEX,
1096       TYPE_TESSELLATION_CONTROL,
1097       TYPE_TESSELLATION_EVAL,
1098       TYPE_GEOMETRY,
1099       TYPE_FRAGMENT,
1100       TYPE_COMPUTE
1101    };
1102
1103    Program(Type type, Target *targ);
1104    ~Program();
1105
1106    void print();
1107
1108    Type getType() const { return progType; }
1109
1110    inline void add(Function *fn, int& id) { allFuncs.insert(fn, id); }
1111    inline void del(Function *fn, int& id) { allFuncs.remove(id); }
1112    inline void add(Value *rval, int& id) { allRValues.insert(rval, id); }
1113
1114    bool makeFromTGSI(struct nv50_ir_prog_info *);
1115    bool makeFromSM4(struct nv50_ir_prog_info *);
1116    bool convertToSSA();
1117    bool optimizeSSA(int level);
1118    bool optimizePostRA(int level);
1119    bool registerAllocation();
1120    bool emitBinary(struct nv50_ir_prog_info *);
1121
1122    const Target *getTarget() const { return target; }
1123
1124 private:
1125    void emitSymbolTable(struct nv50_ir_prog_info *);
1126
1127    Type progType;
1128    Target *target;
1129
1130 public:
1131    Function *main;
1132    Graph calls;
1133
1134    ArrayList allFuncs;
1135    ArrayList allRValues;
1136
1137    uint32_t *code;
1138    uint32_t binSize;
1139    uint32_t tlsSize; // size required for FILE_MEMORY_LOCAL
1140
1141    int maxGPR;
1142
1143    MemoryPool mem_Instruction;
1144    MemoryPool mem_CmpInstruction;
1145    MemoryPool mem_TexInstruction;
1146    MemoryPool mem_FlowInstruction;
1147    MemoryPool mem_LValue;
1148    MemoryPool mem_Symbol;
1149    MemoryPool mem_ImmediateValue;
1150
1151    uint32_t dbgFlags;
1152    uint8_t  optLevel;
1153
1154    void *targetPriv; // e.g. to carry information between passes
1155
1156    const struct nv50_ir_prog_info *driver; // for driver configuration
1157
1158    void releaseInstruction(Instruction *);
1159    void releaseValue(Value *);
1160 };
1161
1162 // TODO: add const version
1163 class Pass
1164 {
1165 public:
1166    bool run(Program *, bool ordered = false, bool skipPhi = false);
1167    bool run(Function *, bool ordered = false, bool skipPhi = false);
1168
1169 private:
1170    // return false to continue with next entity on next higher level
1171    virtual bool visit(Function *) { return true; }
1172    virtual bool visit(BasicBlock *) { return true; }
1173    virtual bool visit(Instruction *) { return false; }
1174
1175    bool doRun(Program *, bool ordered, bool skipPhi);
1176    bool doRun(Function *, bool ordered, bool skipPhi);
1177
1178 protected:
1179    bool err;
1180    Function *func;
1181    Program *prog;
1182 };
1183
1184 // =============================================================================
1185
1186 #include "nv50_ir_inlines.h"
1187
1188 } // namespace nv50_ir
1189
1190 #endif // __NV50_IR_H__