src/gallium/drivers/nouveau/codegen/nv50_ir.h

   1 /*
   2  * Copyright 2011 Christoph Bumiller
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 #ifndef __NV50_IR_H__
  24 #define __NV50_IR_H__
  25
  26 #include <stdio.h>
  27 #include <stdlib.h>
  28 #include <stdint.h>
  29 #include <deque>
  30 #include <list>
  31 #include <vector>
  32 #include <tr1/unordered_set>
  33
  34 #include "codegen/nv50_ir_util.h"
  35 #include "codegen/nv50_ir_graph.h"
  36
  37 #include "codegen/nv50_ir_driver.h"
  38
  39 namespace nv50_ir {
  40
  41 enum operation
  42 {
  43    OP_NOP = 0,
  44    OP_PHI,
  45    OP_UNION, // unify a new definition and several source values
  46    OP_SPLIT, // $r0d -> { $r0, $r1 } ($r0d and $r0/$r1 will be coalesced)
  47    OP_MERGE, // opposite of split, e.g. combine 2 32 bit into a 64 bit value
  48    OP_CONSTRAINT, // copy values into consecutive registers
  49    OP_MOV, // simple copy, no modifiers allowed
  50    OP_LOAD,
  51    OP_STORE,
  52    OP_ADD, // NOTE: add u64 + u32 is legal for targets w/o 64-bit integer adds
  53    OP_SUB,
  54    OP_MUL,
  55    OP_DIV,
  56    OP_MOD,
  57    OP_MAD,
  58    OP_FMA,
  59    OP_SAD, // abs(src0 - src1) + src2
  60    OP_ABS,
  61    OP_NEG,
  62    OP_NOT,
  63    OP_AND,
  64    OP_OR,
  65    OP_XOR,
  66    OP_SHL,
  67    OP_SHR,
  68    OP_MAX,
  69    OP_MIN,
  70    OP_SAT, // CLAMP(f32, 0.0, 1.0)
  71    OP_CEIL,
  72    OP_FLOOR,
  73    OP_TRUNC,
  74    OP_CVT,
  75    OP_SET_AND, // dst = (src0 CMP src1) & src2
  76    OP_SET_OR,
  77    OP_SET_XOR,
  78    OP_SET,
  79    OP_SELP, // dst = src2 ? src0 : src1
  80    OP_SLCT, // dst = (src2 CMP 0) ? src0 : src1
  81    OP_RCP,
  82    OP_RSQ,
  83    OP_LG2,
  84    OP_SIN,
  85    OP_COS,
  86    OP_EX2,
  87    OP_EXP, // exponential (base M_E)
  88    OP_LOG, // natural logarithm
  89    OP_PRESIN,
  90    OP_PREEX2,
  91    OP_SQRT,
  92    OP_POW,
  93    OP_BRA,
  94    OP_CALL,
  95    OP_RET,
  96    OP_CONT,
  97    OP_BREAK,
  98    OP_PRERET,
  99    OP_PRECONT,
 100    OP_PREBREAK,
 101    OP_BRKPT,     // breakpoint (not related to loops)
 102    OP_JOINAT,    // push control flow convergence point
 103    OP_JOIN,      // converge
 104    OP_DISCARD,
 105    OP_EXIT,
 106    OP_MEMBAR, // memory barrier (mfence, lfence, sfence)
 107    OP_VFETCH, // indirection 0 in attribute space, indirection 1 is vertex base
 108    OP_PFETCH, // fetch base address of vertex src0 (immediate) [+ src1]
 109    OP_EXPORT,
 110    OP_LINTERP,
 111    OP_PINTERP,
 112    OP_EMIT,    // emit vertex
 113    OP_RESTART, // restart primitive
 114    OP_TEX,
 115    OP_TXB, // texture bias
 116    OP_TXL, // texure lod
 117    OP_TXF, // texel fetch
 118    OP_TXQ, // texture size query
 119    OP_TXD, // texture derivatives
 120    OP_TXG, // texture gather
 121    OP_TXLQ, // texture query lod
 122    OP_TEXCSAA, // texture op for coverage sampling
 123    OP_TEXPREP, // turn cube map array into 2d array coordinates
 124    OP_SULDB, // surface load (raw)
 125    OP_SULDP, // surface load (formatted)
 126    OP_SUSTB, // surface store (raw)
 127    OP_SUSTP, // surface store (formatted)
 128    OP_SUREDB,
 129    OP_SUREDP, // surface reduction (atomic op)
 130    OP_SULEA,   // surface load effective address
 131    OP_SUBFM,   // surface bitfield manipulation
 132    OP_SUCLAMP, // clamp surface coordinates
 133    OP_SUEAU,   // surface effective address
 134    OP_MADSP,   // special integer multiply-add
 135    OP_TEXBAR, // texture dependency barrier
 136    OP_DFDX,
 137    OP_DFDY,
 138    OP_RDSV, // read system value
 139    OP_WRSV, // write system value
 140    OP_PIXLD, // get info about raster object or surfaces
 141    OP_QUADOP,
 142    OP_QUADON,
 143    OP_QUADPOP,
 144    OP_POPCNT, // bitcount(src0 & src1)
 145    OP_INSBF,  // insert first src1[8:15] bits of src0 into src2 at src1[0:7]
 146    OP_EXTBF,  // place bits [K,K+N) of src0 into dst, src1 = 0xNNKK
 147    OP_BFIND,  // find highest/lowest set bit
 148    OP_PERMT,  // dst = bytes from src2,src0 selected by src1 (nvc0's src order)
 149    OP_ATOM,
 150    OP_BAR,    // execution barrier, sources = { id, thread count, predicate }
 151    OP_VADD,   // byte/word vector operations
 152    OP_VAVG,
 153    OP_VMIN,
 154    OP_VMAX,
 155    OP_VSAD,
 156    OP_VSET,
 157    OP_VSHR,
 158    OP_VSHL,
 159    OP_VSEL,
 160    OP_CCTL, // cache control
 161    OP_SHFL, // warp shuffle
 162    OP_LAST
 163 };
 164
 165 // various instruction-specific modifier definitions Instruction::subOp
 166 // MOV_FINAL marks a MOV originating from an EXPORT (used for placing TEXBARs)
 167 #define NV50_IR_SUBOP_MUL_HIGH     1
 168 #define NV50_IR_SUBOP_EMIT_RESTART 1
 169 #define NV50_IR_SUBOP_LDC_IL       1
 170 #define NV50_IR_SUBOP_LDC_IS       2
 171 #define NV50_IR_SUBOP_LDC_ISL      3
 172 #define NV50_IR_SUBOP_SHIFT_WRAP   1
 173 #define NV50_IR_SUBOP_EMU_PRERET   1
 174 #define NV50_IR_SUBOP_TEXBAR(n)    n
 175 #define NV50_IR_SUBOP_MOV_FINAL    1
 176 #define NV50_IR_SUBOP_EXTBF_REV    1
 177 #define NV50_IR_SUBOP_BFIND_SAMT   1
 178 #define NV50_IR_SUBOP_RCPRSQ_64H   1
 179 #define NV50_IR_SUBOP_PERMT_F4E    1
 180 #define NV50_IR_SUBOP_PERMT_B4E    2
 181 #define NV50_IR_SUBOP_PERMT_RC8    3
 182 #define NV50_IR_SUBOP_PERMT_ECL    4
 183 #define NV50_IR_SUBOP_PERMT_ECR    5
 184 #define NV50_IR_SUBOP_PERMT_RC16   6
 185 #define NV50_IR_SUBOP_BAR_SYNC     0
 186 #define NV50_IR_SUBOP_BAR_ARRIVE   1
 187 #define NV50_IR_SUBOP_BAR_RED_AND  2
 188 #define NV50_IR_SUBOP_BAR_RED_OR   3
 189 #define NV50_IR_SUBOP_BAR_RED_POPC 4
 190 #define NV50_IR_SUBOP_MEMBAR_L     1
 191 #define NV50_IR_SUBOP_MEMBAR_S     2
 192 #define NV50_IR_SUBOP_MEMBAR_M     3
 193 #define NV50_IR_SUBOP_MEMBAR_CTA  (0 << 2)
 194 #define NV50_IR_SUBOP_MEMBAR_GL   (1 << 2)
 195 #define NV50_IR_SUBOP_MEMBAR_SYS  (2 << 2)
 196 #define NV50_IR_SUBOP_MEMBAR_DIR(m)   ((m) & 0x3)
 197 #define NV50_IR_SUBOP_MEMBAR_SCOPE(m) ((m) & ~0x3)
 198 #define NV50_IR_SUBOP_MEMBAR(d,s) \
 199    (NV50_IR_SUBOP_MEMBAR_##d | NV50_IR_SUBOP_MEMBAR_##s)
 200 #define NV50_IR_SUBOP_ATOM_ADD      0
 201 #define NV50_IR_SUBOP_ATOM_MIN      1
 202 #define NV50_IR_SUBOP_ATOM_MAX      2
 203 #define NV50_IR_SUBOP_ATOM_INC      3
 204 #define NV50_IR_SUBOP_ATOM_DEC      4
 205 #define NV50_IR_SUBOP_ATOM_AND      5
 206 #define NV50_IR_SUBOP_ATOM_OR       6
 207 #define NV50_IR_SUBOP_ATOM_XOR      7
 208 #define NV50_IR_SUBOP_ATOM_CAS      8
 209 #define NV50_IR_SUBOP_ATOM_EXCH     9
 210 #define NV50_IR_SUBOP_CCTL_IV      5
 211 #define NV50_IR_SUBOP_CCTL_IVALL   6
 212 #define NV50_IR_SUBOP_SUST_IGN     0
 213 #define NV50_IR_SUBOP_SUST_TRAP    1
 214 #define NV50_IR_SUBOP_SUST_SDCL    3
 215 #define NV50_IR_SUBOP_SULD_ZERO    0
 216 #define NV50_IR_SUBOP_SULD_TRAP    1
 217 #define NV50_IR_SUBOP_SULD_SDCL    3
 218 #define NV50_IR_SUBOP_SUBFM_3D     1
 219 #define NV50_IR_SUBOP_SUCLAMP_2D   0x10
 220 #define NV50_IR_SUBOP_SUCLAMP_SD(r, d) (( 0 + (r)) | ((d == 2) ? 0x10 : 0))
 221 #define NV50_IR_SUBOP_SUCLAMP_PL(r, d) (( 5 + (r)) | ((d == 2) ? 0x10 : 0))
 222 #define NV50_IR_SUBOP_SUCLAMP_BL(r, d) ((10 + (r)) | ((d == 2) ? 0x10 : 0))
 223 #define NV50_IR_SUBOP_PIXLD_COUNT       0
 224 #define NV50_IR_SUBOP_PIXLD_COVMASK     1
 225 #define NV50_IR_SUBOP_PIXLD_COVERED     2
 226 #define NV50_IR_SUBOP_PIXLD_OFFSET      3
 227 #define NV50_IR_SUBOP_PIXLD_CENT_OFFSET 4
 228 #define NV50_IR_SUBOP_PIXLD_SAMPLEID    5
 229 #define NV50_IR_SUBOP_SHFL_IDX  0
 230 #define NV50_IR_SUBOP_SHFL_UP   1
 231 #define NV50_IR_SUBOP_SHFL_DOWN 2
 232 #define NV50_IR_SUBOP_SHFL_BFLY 3
 233 #define NV50_IR_SUBOP_MADSP_SD     0xffff
 234 // Yes, we could represent those with DataType.
 235 // Or put the type into operation and have a couple 1000 values in that enum.
 236 // This will have to do for now.
 237 // The bitfields are supposed to correspond to nve4 ISA.
 238 #define NV50_IR_SUBOP_MADSP(a,b,c) (((c) << 8) | ((b) << 4) | (a))
 239 #define NV50_IR_SUBOP_V1(d,a,b)    (((d) << 10) | ((b) << 5) | (a) | 0x0000)
 240 #define NV50_IR_SUBOP_V2(d,a,b)    (((d) << 10) | ((b) << 5) | (a) | 0x4000)
 241 #define NV50_IR_SUBOP_V4(d,a,b)    (((d) << 10) | ((b) << 5) | (a) | 0x8000)
 242 #define NV50_IR_SUBOP_Vn(n)        ((n) >> 14)
 243
 244 enum DataType
 245 {
 246    TYPE_NONE,
 247    TYPE_U8,
 248    TYPE_S8,
 249    TYPE_U16,
 250    TYPE_S16,
 251    TYPE_U32,
 252    TYPE_S32,
 253    TYPE_U64, // 64 bit operations are only lowered after register allocation
 254    TYPE_S64,
 255    TYPE_F16,
 256    TYPE_F32,
 257    TYPE_F64,
 258    TYPE_B96,
 259    TYPE_B128
 260 };
 261
 262 enum CondCode
 263 {
 264    CC_FL = 0,
 265    CC_NEVER = CC_FL, // when used with FILE_FLAGS
 266    CC_LT = 1,
 267    CC_EQ = 2,
 268    CC_NOT_P = CC_EQ, // when used with FILE_PREDICATE
 269    CC_LE = 3,
 270    CC_GT = 4,
 271    CC_NE = 5,
 272    CC_P  = CC_NE,
 273    CC_GE = 6,
 274    CC_TR = 7,
 275    CC_ALWAYS = CC_TR,
 276    CC_U  = 8,
 277    CC_LTU = 9,
 278    CC_EQU = 10,
 279    CC_LEU = 11,
 280    CC_GTU = 12,
 281    CC_NEU = 13,
 282    CC_GEU = 14,
 283    CC_NO = 0x10,
 284    CC_NC = 0x11,
 285    CC_NS = 0x12,
 286    CC_NA = 0x13,
 287    CC_A  = 0x14,
 288    CC_S  = 0x15,
 289    CC_C  = 0x16,
 290    CC_O  = 0x17
 291 };
 292
 293 enum RoundMode
 294 {
 295    ROUND_N, // nearest
 296    ROUND_M, // towards -inf
 297    ROUND_Z, // towards 0
 298    ROUND_P, // towards +inf
 299    ROUND_NI, // nearest integer
 300    ROUND_MI, // to integer towards -inf
 301    ROUND_ZI, // to integer towards 0
 302    ROUND_PI, // to integer towards +inf
 303 };
 304
 305 enum CacheMode
 306 {
 307    CACHE_CA,            // cache at all levels
 308    CACHE_WB = CACHE_CA, // cache write back
 309    CACHE_CG,            // cache at global level
 310    CACHE_CS,            // cache streaming
 311    CACHE_CV,            // cache as volatile
 312    CACHE_WT = CACHE_CV  // cache write-through
 313 };
 314
 315 enum DataFile
 316 {
 317    FILE_NULL = 0,
 318    FILE_GPR,
 319    FILE_PREDICATE,       // boolean predicate
 320    FILE_FLAGS,           // zero/sign/carry/overflow bits
 321    FILE_ADDRESS,
 322    LAST_REGISTER_FILE = FILE_ADDRESS,
 323    FILE_IMMEDIATE,
 324    FILE_MEMORY_CONST,
 325    FILE_SHADER_INPUT,
 326    FILE_SHADER_OUTPUT,
 327    FILE_MEMORY_GLOBAL,
 328    FILE_MEMORY_SHARED,
 329    FILE_MEMORY_LOCAL,
 330    FILE_SYSTEM_VALUE,
 331    DATA_FILE_COUNT
 332 };
 333
 334 enum TexTarget
 335 {
 336    TEX_TARGET_1D,
 337    TEX_TARGET_2D,
 338    TEX_TARGET_2D_MS,
 339    TEX_TARGET_3D,
 340    TEX_TARGET_CUBE,
 341    TEX_TARGET_1D_SHADOW,
 342    TEX_TARGET_2D_SHADOW,
 343    TEX_TARGET_CUBE_SHADOW,
 344    TEX_TARGET_1D_ARRAY,
 345    TEX_TARGET_2D_ARRAY,
 346    TEX_TARGET_2D_MS_ARRAY,
 347    TEX_TARGET_CUBE_ARRAY,
 348    TEX_TARGET_1D_ARRAY_SHADOW,
 349    TEX_TARGET_2D_ARRAY_SHADOW,
 350    TEX_TARGET_RECT,
 351    TEX_TARGET_RECT_SHADOW,
 352    TEX_TARGET_CUBE_ARRAY_SHADOW,
 353    TEX_TARGET_BUFFER,
 354    TEX_TARGET_COUNT
 355 };
 356
 357 enum SVSemantic
 358 {
 359    SV_POSITION, // WPOS
 360    SV_VERTEX_ID,
 361    SV_INSTANCE_ID,
 362    SV_INVOCATION_ID,
 363    SV_PRIMITIVE_ID,
 364    SV_VERTEX_COUNT, // gl_PatchVerticesIn
 365    SV_LAYER,
 366    SV_VIEWPORT_INDEX,
 367    SV_YDIR,
 368    SV_FACE,
 369    SV_POINT_SIZE,
 370    SV_POINT_COORD,
 371    SV_CLIP_DISTANCE,
 372    SV_SAMPLE_INDEX,
 373    SV_SAMPLE_POS,
 374    SV_SAMPLE_MASK,
 375    SV_TESS_FACTOR,
 376    SV_TESS_COORD,
 377    SV_TID,
 378    SV_CTAID,
 379    SV_NTID,
 380    SV_GRIDID,
 381    SV_NCTAID,
 382    SV_LANEID,
 383    SV_PHYSID,
 384    SV_NPHYSID,
 385    SV_CLOCK,
 386    SV_LBASE,
 387    SV_SBASE,
 388    SV_VERTEX_STRIDE,
 389    SV_INVOCATION_INFO,
 390    SV_UNDEFINED,
 391    SV_LAST
 392 };
 393
 394 class Program;
 395 class Function;
 396 class BasicBlock;
 397
 398 class Target;
 399
 400 class Instruction;
 401 class CmpInstruction;
 402 class TexInstruction;
 403 class FlowInstruction;
 404
 405 class Value;
 406 class LValue;
 407 class Symbol;
 408 class ImmediateValue;
 409
 410 struct Storage
 411 {
 412    DataFile file;
 413    int8_t fileIndex; // signed, may be indirect for CONST[]
 414    uint8_t size; // this should match the Instruction type's size
 415    DataType type; // mainly for pretty printing
 416    union {
 417       uint64_t u64;    // immediate values
 418       uint32_t u32;
 419       uint16_t u16;
 420       uint8_t u8;
 421       int64_t s64;
 422       int32_t s32;
 423       int16_t s16;
 424       int8_t s8;
 425       float f32;
 426       double f64;
 427       int32_t offset; // offset from 0 (base of address space)
 428       int32_t id;     // register id (< 0 if virtual/unassigned, in units <= 4)
 429       struct {
 430          SVSemantic sv;
 431          int index;
 432       } sv;
 433    } data;
 434 };
 435
 436 // precedence: NOT after SAT after NEG after ABS
 437 #define NV50_IR_MOD_ABS (1 << 0)
 438 #define NV50_IR_MOD_NEG (1 << 1)
 439 #define NV50_IR_MOD_SAT (1 << 2)
 440 #define NV50_IR_MOD_NOT (1 << 3)
 441 #define NV50_IR_MOD_NEG_ABS (NV50_IR_MOD_NEG | NV50_IR_MOD_ABS)
 442
 443 #define NV50_IR_INTERP_MODE_MASK   0x3
 444 #define NV50_IR_INTERP_LINEAR      (0 << 0)
 445 #define NV50_IR_INTERP_PERSPECTIVE (1 << 0)
 446 #define NV50_IR_INTERP_FLAT        (2 << 0)
 447 #define NV50_IR_INTERP_SC          (3 << 0) // what exactly is that ?
 448 #define NV50_IR_INTERP_SAMPLE_MASK 0xc
 449 #define NV50_IR_INTERP_DEFAULT     (0 << 2)
 450 #define NV50_IR_INTERP_CENTROID    (1 << 2)
 451 #define NV50_IR_INTERP_OFFSET      (2 << 2)
 452 #define NV50_IR_INTERP_SAMPLEID    (3 << 2)
 453
 454 // do we really want this to be a class ?
 455 class Modifier
 456 {
 457 public:
 458    Modifier() : bits(0) { }
 459    Modifier(unsigned int m) : bits(m) { }
 460    Modifier(operation op);
 461
 462    // @return new Modifier applying a after b (asserts if unrepresentable)
 463    Modifier operator*(const Modifier) const;
 464    Modifier operator*=(const Modifier m) { *this = *this * m; return *this; }
 465    Modifier operator==(const Modifier m) const { return m.bits == bits; }
 466    Modifier operator!=(const Modifier m) const { return m.bits != bits; }
 467
 468    inline Modifier operator&(const Modifier m) const { return bits & m.bits; }
 469    inline Modifier operator|(const Modifier m) const { return bits | m.bits; }
 470    inline Modifier operator^(const Modifier m) const { return bits ^ m.bits; }
 471
 472    operation getOp() const;
 473
 474    inline int neg() const { return (bits & NV50_IR_MOD_NEG) ? 1 : 0; }
 475    inline int abs() const { return (bits & NV50_IR_MOD_ABS) ? 1 : 0; }
 476
 477    inline operator bool() const { return bits ? true : false; }
 478
 479    void applyTo(ImmediateValue &imm) const;
 480
 481    int print(char *buf, size_t size) const;
 482
 483 private:
 484    uint8_t bits;
 485 };
 486
 487 class ValueRef
 488 {
 489 public:
 490    ValueRef(Value * = NULL);
 491    ValueRef(const ValueRef&);
 492    ~ValueRef();
 493
 494    inline bool exists() const { return value != NULL; }
 495
 496    void set(Value *);
 497    void set(const ValueRef&);
 498    inline Value *get() const { return value; }
 499    inline Value *rep() const;
 500
 501    inline Instruction *getInsn() const { return insn; }
 502    inline void setInsn(Instruction *inst) { insn = inst; }
 503
 504    inline bool isIndirect(int dim) const { return indirect[dim] >= 0; }
 505    inline const ValueRef *getIndirect(int dim) const;
 506
 507    inline DataFile getFile() const;
 508    inline unsigned getSize() const;
 509
 510    // SSA: return eventual (traverse MOVs) literal value, if it exists
 511    bool getImmediate(ImmediateValue&) const;
 512
 513 public:
 514    Modifier mod;
 515    int8_t indirect[2]; // >= 0 if relative to lvalue in insn->src(indirect[i])
 516    uint8_t swizzle;
 517
 518    bool usedAsPtr; // for printing
 519
 520 private:
 521    Value *value;
 522    Instruction *insn;
 523 };
 524
 525 class ValueDef
 526 {
 527 public:
 528    ValueDef(Value * = NULL);
 529    ValueDef(const ValueDef&);
 530    ~ValueDef();
 531
 532    inline bool exists() const { return value != NULL; }
 533
 534    inline Value *get() const { return value; }
 535    inline Value *rep() const;
 536    void set(Value *);
 537    bool mayReplace(const ValueRef &);
 538    void replace(const ValueRef &, bool doSet); // replace all uses of the old value
 539
 540    inline Instruction *getInsn() const { return insn; }
 541    inline void setInsn(Instruction *inst) { insn = inst; }
 542
 543    inline DataFile getFile() const;
 544    inline unsigned getSize() const;
 545
 546    inline void setSSA(LValue *);
 547    inline const LValue *preSSA() const;
 548
 549 private:
 550    Value *value;   // should make this LValue * ...
 551    LValue *origin; // pre SSA value
 552    Instruction *insn;
 553 };
 554
 555 class Value
 556 {
 557 public:
 558    Value();
 559    virtual ~Value() { }
 560
 561    virtual Value *clone(ClonePolicy<Function>&) const = 0;
 562
 563    virtual int print(char *, size_t, DataType ty = TYPE_NONE) const = 0;
 564
 565    virtual bool equals(const Value *, bool strict = false) const;
 566    virtual bool interfers(const Value *) const;
 567    virtual bool isUniform() const { return true; }
 568
 569    inline Value *rep() const { return join; }
 570
 571    inline Instruction *getUniqueInsn() const;
 572    inline Instruction *getInsn() const; // use when uniqueness is certain
 573
 574    inline int refCount() { return uses.size(); }
 575
 576    inline LValue *asLValue();
 577    inline Symbol *asSym();
 578    inline ImmediateValue *asImm();
 579    inline const Symbol *asSym() const;
 580    inline const ImmediateValue *asImm() const;
 581
 582    inline bool inFile(DataFile f) { return reg.file == f; }
 583
 584    static inline Value *get(Iterator&);
 585
 586    std::tr1::unordered_set<ValueRef *> uses;
 587    std::list<ValueDef *> defs;
 588    typedef std::tr1::unordered_set<ValueRef *>::iterator UseIterator;
 589    typedef std::tr1::unordered_set<ValueRef *>::const_iterator UseCIterator;
 590    typedef std::list<ValueDef *>::iterator DefIterator;
 591    typedef std::list<ValueDef *>::const_iterator DefCIterator;
 592
 593    int id;
 594    Storage reg;
 595
 596    // TODO: these should be in LValue:
 597    Interval livei;
 598    Value *join;
 599 };
 600
 601 class LValue : public Value
 602 {
 603 public:
 604    LValue(Function *, DataFile file);
 605    LValue(Function *, LValue *);
 606    ~LValue() { }
 607
 608    virtual bool isUniform() const;
 609
 610    virtual LValue *clone(ClonePolicy<Function>&) const;
 611
 612    virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
 613
 614 public:
 615    unsigned compMask : 8; // compound/component mask
 616    unsigned compound : 1; // used by RA, value involved in split/merge
 617    unsigned ssa      : 1;
 618    unsigned fixedReg : 1; // set & used by RA, earlier just use (id < 0)
 619    unsigned noSpill  : 1; // do not spill (e.g. if spill temporary already)
 620 };
 621
 622 class Symbol : public Value
 623 {
 624 public:
 625    Symbol(Program *, DataFile file = FILE_MEMORY_CONST, ubyte fileIdx = 0);
 626    ~Symbol() { }
 627
 628    virtual Symbol *clone(ClonePolicy<Function>&) const;
 629
 630    virtual bool equals(const Value *that, bool strict) const;
 631
 632    virtual bool isUniform() const;
 633
 634    virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
 635
 636    // print with indirect values
 637    int print(char *, size_t, Value *, Value *, DataType ty = TYPE_NONE) const;
 638
 639    inline void setFile(DataFile file, ubyte fileIndex = 0)
 640    {
 641       reg.file = file;
 642       reg.fileIndex = fileIndex;
 643    }
 644
 645    inline void setOffset(int32_t offset);
 646    inline void setAddress(Symbol *base, int32_t offset);
 647    inline void setSV(SVSemantic sv, uint32_t idx = 0);
 648
 649    inline const Symbol *getBase() const { return baseSym; }
 650
 651 private:
 652    Symbol *baseSym; // array base for Symbols representing array elements
 653 };
 654
 655 class ImmediateValue : public Value
 656 {
 657 public:
 658    ImmediateValue() { }
 659    ImmediateValue(Program *, uint32_t);
 660    ImmediateValue(Program *, float);
 661    ImmediateValue(Program *, double);
 662    // NOTE: not added to program with
 663    ImmediateValue(const ImmediateValue *, DataType ty);
 664    ~ImmediateValue() { };
 665
 666    virtual ImmediateValue *clone(ClonePolicy<Function>&) const;
 667
 668    virtual bool equals(const Value *that, bool strict) const;
 669
 670    // these only work if 'type' is valid (we mostly use untyped literals):
 671    bool isInteger(const int ival) const; // ival is cast to this' type
 672    bool isNegative() const;
 673    bool isPow2() const;
 674
 675    void applyLog2();
 676
 677    // for constant folding:
 678    ImmediateValue operator+(const ImmediateValue&) const;
 679    ImmediateValue operator-(const ImmediateValue&) const;
 680    ImmediateValue operator*(const ImmediateValue&) const;
 681    ImmediateValue operator/(const ImmediateValue&) const;
 682
 683    ImmediateValue& operator=(const ImmediateValue&); // only sets value !
 684
 685    bool compare(CondCode cc, float fval) const;
 686
 687    virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
 688 };
 689
 690 class Instruction
 691 {
 692 public:
 693    Instruction();
 694    Instruction(Function *, operation, DataType);
 695    virtual ~Instruction();
 696
 697    virtual Instruction *clone(ClonePolicy<Function>&,
 698                               Instruction * = NULL) const;
 699
 700    void setDef(int i, Value *);
 701    void setSrc(int s, Value *);
 702    void setSrc(int s, const ValueRef&);
 703    void swapSources(int a, int b);
 704    void moveSources(int s, int delta);
 705    bool setIndirect(int s, int dim, Value *);
 706
 707    inline ValueRef& src(int s) { return srcs[s]; }
 708    inline ValueDef& def(int s) { return defs[s]; }
 709    inline const ValueRef& src(int s) const { return srcs[s]; }
 710    inline const ValueDef& def(int s) const { return defs[s]; }
 711
 712    inline Value *getDef(int d) const { return defs[d].get(); }
 713    inline Value *getSrc(int s) const { return srcs[s].get(); }
 714    inline Value *getIndirect(int s, int dim) const;
 715
 716    inline bool defExists(unsigned d) const
 717    {
 718       return d < defs.size() && defs[d].exists();
 719    }
 720    inline bool srcExists(unsigned s) const
 721    {
 722       return s < srcs.size() && srcs[s].exists();
 723    }
 724
 725    inline bool constrainedDefs() const;
 726
 727    bool setPredicate(CondCode ccode, Value *);
 728    inline Value *getPredicate() const;
 729    bool writesPredicate() const;
 730    inline bool isPredicated() const { return predSrc >= 0; }
 731
 732    inline void setFlagsSrc(int s, Value *);
 733    inline void setFlagsDef(int d, Value *);
 734    inline bool usesFlags() const { return flagsSrc >= 0; }
 735
 736    unsigned int defCount() const { return defs.size(); };
 737    unsigned int defCount(unsigned int mask, bool singleFile = false) const;
 738    unsigned int srcCount() const { return srcs.size(); };
 739    unsigned int srcCount(unsigned int mask, bool singleFile = false) const;
 740
 741    // save & remove / set indirect[0,1] and predicate source
 742    void takeExtraSources(int s, Value *[3]);
 743    void putExtraSources(int s, Value *[3]);
 744
 745    inline void setType(DataType type) { dType = sType = type; }
 746
 747    inline void setType(DataType dtype, DataType stype)
 748    {
 749       dType = dtype;
 750       sType = stype;
 751    }
 752
 753    inline bool isPseudo() const { return op < OP_MOV; }
 754    bool isDead() const;
 755    bool isNop() const;
 756    bool isCommutationLegal(const Instruction *) const; // must be adjacent !
 757    bool isActionEqual(const Instruction *) const;
 758    bool isResultEqual(const Instruction *) const;
 759
 760    void print() const;
 761
 762    inline CmpInstruction *asCmp();
 763    inline TexInstruction *asTex();
 764    inline FlowInstruction *asFlow();
 765    inline const TexInstruction *asTex() const;
 766    inline const CmpInstruction *asCmp() const;
 767    inline const FlowInstruction *asFlow() const;
 768
 769 public:
 770    Instruction *next;
 771    Instruction *prev;
 772    int id;
 773    int serial; // CFG order
 774
 775    operation op;
 776    DataType dType; // destination or defining type
 777    DataType sType; // source or secondary type
 778    CondCode cc;
 779    RoundMode rnd;
 780    CacheMode cache;
 781
 782    uint16_t subOp; // quadop, 1 for mul-high, etc.
 783
 784    unsigned encSize    : 4; // encoding size in bytes
 785    unsigned saturate   : 1; // to [0.0f, 1.0f]
 786    unsigned join       : 1; // converge control flow (use OP_JOIN until end)
 787    unsigned fixed      : 1; // prevent dead code elimination
 788    unsigned terminator : 1; // end of basic block
 789    unsigned ftz        : 1; // flush denormal to zero
 790    unsigned dnz        : 1; // denormals, NaN are zero
 791    unsigned ipa        : 4; // interpolation mode
 792    unsigned lanes      : 4;
 793    unsigned perPatch   : 1;
 794    unsigned exit       : 1; // terminate program after insn
 795    unsigned mask       : 4; // for vector ops
 796
 797    int8_t postFactor; // MUL/DIV(if < 0) by 1 << postFactor
 798
 799    int8_t predSrc;
 800    int8_t flagsDef;
 801    int8_t flagsSrc;
 802
 803    uint32_t sched; // scheduling data (NOTE: maybe move to separate storage)
 804
 805    BasicBlock *bb;
 806
 807 protected:
 808    std::deque<ValueDef> defs; // no gaps !
 809    std::deque<ValueRef> srcs; // no gaps !
 810
 811    // instruction specific methods:
 812    // (don't want to subclass, would need more constructors and memory pools)
 813 public:
 814    inline void setInterpolate(unsigned int mode) { ipa = mode; }
 815
 816    unsigned int getInterpMode() const { return ipa & 0x3; }
 817    unsigned int getSampleMode() const { return ipa & 0xc; }
 818
 819 private:
 820    void init();
 821 };
 822
 823 enum TexQuery
 824 {
 825    TXQ_DIMS,
 826    TXQ_TYPE,
 827    TXQ_SAMPLE_POSITION,
 828    TXQ_FILTER,
 829    TXQ_LOD,
 830    TXQ_WRAP,
 831    TXQ_BORDER_COLOUR
 832 };
 833
 834 class TexInstruction : public Instruction
 835 {
 836 public:
 837    class Target
 838    {
 839    public:
 840       Target(TexTarget targ = TEX_TARGET_2D) : target(targ) { }
 841
 842       const char *getName() const { return descTable[target].name; }
 843       unsigned int getArgCount() const { return descTable[target].argc; }
 844       unsigned int getDim() const { return descTable[target].dim; }
 845       int isArray() const { return descTable[target].array ? 1 : 0; }
 846       int isCube() const { return descTable[target].cube ? 1 : 0; }
 847       int isShadow() const { return descTable[target].shadow ? 1 : 0; }
 848       int isMS() const {
 849         return target == TEX_TARGET_2D_MS || target == TEX_TARGET_2D_MS_ARRAY; }
 850       void clearMS() {
 851          if (isMS()) {
 852             if (isArray())
 853                target = TEX_TARGET_2D_ARRAY;
 854             else
 855                target = TEX_TARGET_2D;
 856          }
 857       }
 858
 859       Target& operator=(TexTarget targ)
 860       {
 861          assert(targ < TEX_TARGET_COUNT);
 862          target = targ;
 863          return *this;
 864       }
 865
 866       inline bool operator==(TexTarget targ) const { return target == targ; }
 867       inline bool operator!=(TexTarget targ) const { return target != targ; }
 868
 869       enum TexTarget getEnum() const { return target; }
 870
 871    private:
 872       struct Desc
 873       {
 874          char name[19];
 875          uint8_t dim;
 876          uint8_t argc;
 877          bool array;
 878          bool cube;
 879          bool shadow;
 880       };
 881
 882       static const struct Desc descTable[TEX_TARGET_COUNT];
 883
 884    private:
 885       enum TexTarget target;
 886    };
 887
 888 public:
 889    TexInstruction(Function *, operation);
 890    virtual ~TexInstruction();
 891
 892    virtual TexInstruction *clone(ClonePolicy<Function>&,
 893                                  Instruction * = NULL) const;
 894
 895    inline void setTexture(Target targ, uint8_t r, uint8_t s)
 896    {
 897       tex.r = r;
 898       tex.s = s;
 899       tex.target = targ;
 900    }
 901
 902    void setIndirectR(Value *);
 903    void setIndirectS(Value *);
 904    inline Value *getIndirectR() const;
 905    inline Value *getIndirectS() const;
 906
 907 public:
 908    struct {
 909       Target target;
 910
 911       uint16_t r;
 912       uint16_t s;
 913       int8_t rIndirectSrc;
 914       int8_t sIndirectSrc;
 915
 916       uint8_t mask;
 917       uint8_t gatherComp;
 918
 919       bool liveOnly; // only execute on live pixels of a quad (optimization)
 920       bool levelZero;
 921       bool derivAll;
 922
 923       int8_t useOffsets; // 0, 1, or 4 for textureGatherOffsets
 924       int8_t offset[3]; // only used on nv50
 925
 926       enum TexQuery query;
 927    } tex;
 928
 929    ValueRef dPdx[3];
 930    ValueRef dPdy[3];
 931    ValueRef offset[4][3];
 932 };
 933
 934 class CmpInstruction : public Instruction
 935 {
 936 public:
 937    CmpInstruction(Function *, operation);
 938
 939    virtual CmpInstruction *clone(ClonePolicy<Function>&,
 940                                  Instruction * = NULL) const;
 941
 942    void setCondition(CondCode cond) { setCond = cond; }
 943    CondCode getCondition() const { return setCond; }
 944
 945 public:
 946    CondCode setCond;
 947 };
 948
 949 class FlowInstruction : public Instruction
 950 {
 951 public:
 952    FlowInstruction(Function *, operation, void *target);
 953
 954    virtual FlowInstruction *clone(ClonePolicy<Function>&,
 955                                   Instruction * = NULL) const;
 956
 957 public:
 958    unsigned allWarp  : 1;
 959    unsigned absolute : 1;
 960    unsigned limit    : 1;
 961    unsigned builtin  : 1; // true for calls to emulation code
 962    unsigned indirect : 1; // target in src(0)
 963
 964    union {
 965       BasicBlock *bb;
 966       int builtin;
 967       Function *fn;
 968    } target;
 969 };
 970
 971 class BasicBlock
 972 {
 973 public:
 974    BasicBlock(Function *);
 975    ~BasicBlock();
 976
 977    BasicBlock *clone(ClonePolicy<Function>&) const;
 978
 979    inline int getId() const { return id; }
 980    inline unsigned int getInsnCount() const { return numInsns; }
 981    inline bool isTerminated() const { return exit && exit->terminator; }
 982
 983    bool dominatedBy(BasicBlock *bb);
 984    inline bool reachableBy(const BasicBlock *by, const BasicBlock *term);
 985
 986    // returns mask of conditional out blocks
 987    // e.g. 3 for IF { .. } ELSE { .. } ENDIF, 1 for IF { .. } ENDIF
 988    unsigned int initiatesSimpleConditional() const;
 989
 990 public:
 991    Function *getFunction() const { return func; }
 992    Program *getProgram() const { return program; }
 993
 994    Instruction *getEntry() const { return entry; } // first non-phi instruction
 995    Instruction *getPhi() const { return phi; }
 996    Instruction *getFirst() const { return phi ? phi : entry; }
 997    Instruction *getExit() const { return exit; }
 998
 999    void insertHead(Instruction *);
1000    void insertTail(Instruction *);
1001    void insertBefore(Instruction *, Instruction *);
1002    void insertAfter(Instruction *, Instruction *);
1003    void remove(Instruction *);
1004    void permuteAdjacent(Instruction *, Instruction *);
1005
1006    BasicBlock *idom() const;
1007
1008    // NOTE: currently does not rebuild the dominator tree
1009    BasicBlock *splitBefore(Instruction *, bool attach = true);
1010    BasicBlock *splitAfter(Instruction *, bool attach = true);
1011
1012    DLList& getDF() { return df; }
1013    DLList::Iterator iterDF() { return df.iterator(); }
1014
1015    static inline BasicBlock *get(Iterator&);
1016    static inline BasicBlock *get(Graph::Node *);
1017
1018 public:
1019    Graph::Node cfg; // first edge is branch *taken* (the ELSE branch)
1020    Graph::Node dom;
1021
1022    BitSet liveSet;
1023    BitSet defSet;
1024
1025    uint32_t binPos;
1026    uint32_t binSize;
1027
1028    Instruction *joinAt; // for quick reference
1029
1030    bool explicitCont; // loop headers: true if loop contains continue stmts
1031
1032 private:
1033    int id;
1034    DLList df;
1035
1036    Instruction *phi;
1037    Instruction *entry;
1038    Instruction *exit;
1039
1040    unsigned int numInsns;
1041
1042 private:
1043    Function *func;
1044    Program *program;
1045
1046    void splitCommon(Instruction *, BasicBlock *, bool attach);
1047 };
1048
1049 class Function
1050 {
1051 public:
1052    Function(Program *, const char *name, uint32_t label);
1053    ~Function();
1054
1055    static inline Function *get(Graph::Node *node);
1056
1057    inline Program *getProgram() const { return prog; }
1058    inline const char *getName() const { return name; }
1059    inline int getId() const { return id; }
1060    inline uint32_t getLabel() const { return label; }
1061
1062    void print();
1063    void printLiveIntervals() const;
1064    void printCFGraph(const char *filePath);
1065
1066    bool setEntry(BasicBlock *);
1067    bool setExit(BasicBlock *);
1068
1069    unsigned int orderInstructions(ArrayList&);
1070
1071    inline void add(BasicBlock *bb, int& id) { allBBlocks.insert(bb, id); }
1072    inline void add(Instruction *insn, int& id) { allInsns.insert(insn, id); }
1073    inline void add(LValue *lval, int& id) { allLValues.insert(lval, id); }
1074
1075    inline LValue *getLValue(int id);
1076
1077    void buildLiveSets();
1078    void buildDefSets();
1079    bool convertToSSA();
1080
1081 public:
1082    std::deque<ValueDef> ins;
1083    std::deque<ValueRef> outs;
1084    std::deque<Value *> clobbers;
1085
1086    Graph cfg;
1087    Graph::Node *cfgExit;
1088    Graph *domTree;
1089    Graph::Node call; // node in the call graph
1090
1091    BasicBlock **bbArray; // BBs in emission order
1092    int bbCount;
1093
1094    unsigned int loopNestingBound;
1095    int regClobberMax;
1096
1097    uint32_t binPos;
1098    uint32_t binSize;
1099
1100    Value *stackPtr;
1101
1102    uint32_t tlsBase; // base address for l[] space (if no stack pointer is used)
1103    uint32_t tlsSize;
1104
1105    ArrayList allBBlocks;
1106    ArrayList allInsns;
1107    ArrayList allLValues;
1108
1109 private:
1110    void buildLiveSetsPreSSA(BasicBlock *, const int sequence);
1111    void buildDefSetsPreSSA(BasicBlock *bb, const int seq);
1112
1113 private:
1114    uint32_t label;
1115    int id;
1116    const char *const name;
1117    Program *prog;
1118 };
1119
1120 enum CGStage
1121 {
1122    CG_STAGE_PRE_SSA,
1123    CG_STAGE_SSA, // expected directly before register allocation
1124    CG_STAGE_POST_RA
1125 };
1126
1127 class Program
1128 {
1129 public:
1130    enum Type
1131    {
1132       TYPE_VERTEX,
1133       TYPE_TESSELLATION_CONTROL,
1134       TYPE_TESSELLATION_EVAL,
1135       TYPE_GEOMETRY,
1136       TYPE_FRAGMENT,
1137       TYPE_COMPUTE
1138    };
1139
1140    Program(Type type, Target *targ);
1141    ~Program();
1142
1143    void print();
1144
1145    Type getType() const { return progType; }
1146
1147    inline void add(Function *fn, int& id) { allFuncs.insert(fn, id); }
1148    inline void del(Function *fn, int& id) { allFuncs.remove(id); }
1149    inline void add(Value *rval, int& id) { allRValues.insert(rval, id); }
1150
1151    bool makeFromTGSI(struct nv50_ir_prog_info *);
1152    bool makeFromSM4(struct nv50_ir_prog_info *);
1153    bool convertToSSA();
1154    bool optimizeSSA(int level);
1155    bool optimizePostRA(int level);
1156    bool registerAllocation();
1157    bool emitBinary(struct nv50_ir_prog_info *);
1158
1159    const Target *getTarget() const { return target; }
1160
1161 private:
1162    void emitSymbolTable(struct nv50_ir_prog_info *);
1163
1164    Type progType;
1165    Target *target;
1166
1167 public:
1168    Function *main;
1169    Graph calls;
1170
1171    ArrayList allFuncs;
1172    ArrayList allRValues;
1173
1174    uint32_t *code;
1175    uint32_t binSize;
1176    uint32_t tlsSize; // size required for FILE_MEMORY_LOCAL
1177
1178    int maxGPR;
1179
1180    MemoryPool mem_Instruction;
1181    MemoryPool mem_CmpInstruction;
1182    MemoryPool mem_TexInstruction;
1183    MemoryPool mem_FlowInstruction;
1184    MemoryPool mem_LValue;
1185    MemoryPool mem_Symbol;
1186    MemoryPool mem_ImmediateValue;
1187
1188    uint32_t dbgFlags;
1189    uint8_t  optLevel;
1190
1191    void *targetPriv; // e.g. to carry information between passes
1192
1193    const struct nv50_ir_prog_info *driver; // for driver configuration
1194
1195    void releaseInstruction(Instruction *);
1196    void releaseValue(Value *);
1197 };
1198
1199 // TODO: add const version
1200 class Pass
1201 {
1202 public:
1203    bool run(Program *, bool ordered = false, bool skipPhi = false);
1204    bool run(Function *, bool ordered = false, bool skipPhi = false);
1205
1206 private:
1207    // return false to continue with next entity on next higher level
1208    virtual bool visit(Function *) { return true; }
1209    virtual bool visit(BasicBlock *) { return true; }
1210    virtual bool visit(Instruction *) { return false; }
1211
1212    bool doRun(Program *, bool ordered, bool skipPhi);
1213    bool doRun(Function *, bool ordered, bool skipPhi);
1214
1215 protected:
1216    bool err;
1217    Function *func;
1218    Program *prog;
1219 };
1220
1221 // =============================================================================
1222
1223 #include "codegen/nv50_ir_inlines.h"
1224
1225 } // namespace nv50_ir
1226
1227 #endif // __NV50_IR_H__