2 * Copyright 2011 Christoph Bumiller
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
33 #include "codegen/unordered_set.h"
34 #include "codegen/nv50_ir_util.h"
35 #include "codegen/nv50_ir_graph.h"
37 #include "codegen/nv50_ir_driver.h"
45 OP_UNION
, // unify a new definition and several source values
46 OP_SPLIT
, // $r0d -> { $r0, $r1 } ($r0d and $r0/$r1 will be coalesced)
47 OP_MERGE
, // opposite of split, e.g. combine 2 32 bit into a 64 bit value
48 OP_CONSTRAINT
, // copy values into consecutive registers
49 OP_MOV
, // simple copy, no modifiers allowed
52 OP_ADD
, // NOTE: add u64 + u32 is legal for targets w/o 64-bit integer adds
59 OP_SAD
, // abs(src0 - src1) + src2
61 // extended multiply-add (GM107+), does a lot of things.
62 // see envytools for detailed documentation
76 OP_SAT
, // CLAMP(f32, 0.0, 1.0)
81 OP_SET_AND
, // dst = (src0 CMP src1) & src2
85 OP_SELP
, // dst = src2 ? src0 : src1
86 OP_SLCT
, // dst = (src2 CMP 0) ? src0 : src1
93 OP_EXP
, // exponential (base M_E)
94 OP_LOG
, // natural logarithm
107 OP_BRKPT
, // breakpoint (not related to loops)
108 OP_JOINAT
, // push control flow convergence point
112 OP_MEMBAR
, // memory barrier (mfence, lfence, sfence)
113 OP_VFETCH
, // indirection 0 in attribute space, indirection 1 is vertex base
114 OP_PFETCH
, // fetch base address of vertex src0 (immediate) [+ src1]
115 OP_AFETCH
, // fetch base address of shader input (a[%r1+0x10])
119 OP_EMIT
, // emit vertex
120 OP_RESTART
, // restart primitive
121 OP_FINAL
, // finish emitting primitives
123 OP_TXB
, // texture bias
124 OP_TXL
, // texure lod
125 OP_TXF
, // texel fetch
126 OP_TXQ
, // texture size query
127 OP_TXD
, // texture derivatives
128 OP_TXG
, // texture gather
129 OP_TXLQ
, // texture query lod
130 OP_TEXCSAA
, // texture op for coverage sampling
131 OP_TEXPREP
, // turn cube map array into 2d array coordinates
132 OP_SULDB
, // surface load (raw)
133 OP_SULDP
, // surface load (formatted)
134 OP_SUSTB
, // surface store (raw)
135 OP_SUSTP
, // surface store (formatted)
137 OP_SUREDP
, // surface reduction (atomic op)
138 OP_SULEA
, // surface load effective address
139 OP_SUBFM
, // surface bitfield manipulation
140 OP_SUCLAMP
, // clamp surface coordinates
141 OP_SUEAU
, // surface effective address
142 OP_SUQ
, // surface query
143 OP_MADSP
, // special integer multiply-add
144 OP_TEXBAR
, // texture dependency barrier
147 OP_RDSV
, // read system value
148 OP_WRSV
, // write system value
149 OP_PIXLD
, // get info about raster object or surfaces
153 OP_POPCNT
, // bitcount(src0 & src1)
154 OP_INSBF
, // insert first src1[8:15] bits of src0 into src2 at src1[0:7]
155 OP_EXTBF
, // place bits [K,K+N) of src0 into dst, src1 = 0xNNKK
156 OP_BFIND
, // find highest/lowest set bit
157 OP_BREV
, // bitfield reverse
158 OP_BMSK
, // bitfield mask
159 OP_PERMT
, // dst = bytes from src2,src0 selected by src1 (nvc0's src order)
162 OP_BAR
, // execution barrier, sources = { id, thread count, predicate }
163 OP_VADD
, // byte/word vector operations
172 OP_CCTL
, // cache control
173 OP_SHFL
, // warp shuffle
175 OP_BUFQ
, // buffer query
180 // various instruction-specific modifier definitions Instruction::subOp
181 // MOV_FINAL marks a MOV originating from an EXPORT (used for placing TEXBARs)
182 #define NV50_IR_SUBOP_MUL_HIGH 1
183 #define NV50_IR_SUBOP_EMIT_RESTART 1
184 #define NV50_IR_SUBOP_LDC_IL 1
185 #define NV50_IR_SUBOP_LDC_IS 2
186 #define NV50_IR_SUBOP_LDC_ISL 3
187 #define NV50_IR_SUBOP_SHIFT_WRAP 1
188 #define NV50_IR_SUBOP_SHIFT_HIGH 2
189 #define NV50_IR_SUBOP_EMU_PRERET 1
190 #define NV50_IR_SUBOP_TEXBAR(n) n
191 #define NV50_IR_SUBOP_MOV_FINAL 1
192 #define NV50_IR_SUBOP_EXTBF_REV 1
193 #define NV50_IR_SUBOP_BFIND_SAMT 1
194 #define NV50_IR_SUBOP_RCPRSQ_64H 1
195 #define NV50_IR_SUBOP_PERMT_F4E 1
196 #define NV50_IR_SUBOP_PERMT_B4E 2
197 #define NV50_IR_SUBOP_PERMT_RC8 3
198 #define NV50_IR_SUBOP_PERMT_ECL 4
199 #define NV50_IR_SUBOP_PERMT_ECR 5
200 #define NV50_IR_SUBOP_PERMT_RC16 6
201 #define NV50_IR_SUBOP_BAR_SYNC 0
202 #define NV50_IR_SUBOP_BAR_ARRIVE 1
203 #define NV50_IR_SUBOP_BAR_RED_AND 2
204 #define NV50_IR_SUBOP_BAR_RED_OR 3
205 #define NV50_IR_SUBOP_BAR_RED_POPC 4
206 #define NV50_IR_SUBOP_MEMBAR_L 1
207 #define NV50_IR_SUBOP_MEMBAR_S 2
208 #define NV50_IR_SUBOP_MEMBAR_M 3
209 #define NV50_IR_SUBOP_MEMBAR_CTA (0 << 2)
210 #define NV50_IR_SUBOP_MEMBAR_GL (1 << 2)
211 #define NV50_IR_SUBOP_MEMBAR_SYS (2 << 2)
212 #define NV50_IR_SUBOP_MEMBAR_DIR(m) ((m) & 0x3)
213 #define NV50_IR_SUBOP_MEMBAR_SCOPE(m) ((m) & ~0x3)
214 #define NV50_IR_SUBOP_MEMBAR(d,s) \
215 (NV50_IR_SUBOP_MEMBAR_##d | NV50_IR_SUBOP_MEMBAR_##s)
216 #define NV50_IR_SUBOP_ATOM_ADD 0
217 #define NV50_IR_SUBOP_ATOM_MIN 1
218 #define NV50_IR_SUBOP_ATOM_MAX 2
219 #define NV50_IR_SUBOP_ATOM_INC 3
220 #define NV50_IR_SUBOP_ATOM_DEC 4
221 #define NV50_IR_SUBOP_ATOM_AND 5
222 #define NV50_IR_SUBOP_ATOM_OR 6
223 #define NV50_IR_SUBOP_ATOM_XOR 7
224 #define NV50_IR_SUBOP_ATOM_CAS 8
225 #define NV50_IR_SUBOP_ATOM_EXCH 9
226 #define NV50_IR_SUBOP_CCTL_IV 5
227 #define NV50_IR_SUBOP_CCTL_IVALL 6
228 #define NV50_IR_SUBOP_SUST_IGN 0
229 #define NV50_IR_SUBOP_SUST_TRAP 1
230 #define NV50_IR_SUBOP_SUST_SDCL 3
231 #define NV50_IR_SUBOP_SULD_ZERO 0
232 #define NV50_IR_SUBOP_SULD_TRAP 1
233 #define NV50_IR_SUBOP_SULD_SDCL 3
234 #define NV50_IR_SUBOP_SUBFM_3D 1
235 #define NV50_IR_SUBOP_SUCLAMP_2D 0x10
236 #define NV50_IR_SUBOP_SUCLAMP_SD(r, d) (( 0 + (r)) | ((d == 2) ? 0x10 : 0))
237 #define NV50_IR_SUBOP_SUCLAMP_PL(r, d) (( 5 + (r)) | ((d == 2) ? 0x10 : 0))
238 #define NV50_IR_SUBOP_SUCLAMP_BL(r, d) ((10 + (r)) | ((d == 2) ? 0x10 : 0))
239 #define NV50_IR_SUBOP_PIXLD_COUNT 0
240 #define NV50_IR_SUBOP_PIXLD_COVMASK 1
241 #define NV50_IR_SUBOP_PIXLD_COVERED 2
242 #define NV50_IR_SUBOP_PIXLD_OFFSET 3
243 #define NV50_IR_SUBOP_PIXLD_CENT_OFFSET 4
244 #define NV50_IR_SUBOP_PIXLD_SAMPLEID 5
245 #define NV50_IR_SUBOP_SHFL_IDX 0
246 #define NV50_IR_SUBOP_SHFL_UP 1
247 #define NV50_IR_SUBOP_SHFL_DOWN 2
248 #define NV50_IR_SUBOP_SHFL_BFLY 3
249 #define NV50_IR_SUBOP_LOAD_LOCKED 1
250 #define NV50_IR_SUBOP_STORE_UNLOCKED 2
251 #define NV50_IR_SUBOP_MADSP_SD 0xffff
252 // Yes, we could represent those with DataType.
253 // Or put the type into operation and have a couple 1000 values in that enum.
254 // This will have to do for now.
255 // The bitfields are supposed to correspond to nve4 ISA.
256 #define NV50_IR_SUBOP_MADSP(a,b,c) (((c) << 8) | ((b) << 4) | (a))
257 #define NV50_IR_SUBOP_V1(d,a,b) (((d) << 10) | ((b) << 5) | (a) | 0x0000)
258 #define NV50_IR_SUBOP_V2(d,a,b) (((d) << 10) | ((b) << 5) | (a) | 0x4000)
259 #define NV50_IR_SUBOP_V4(d,a,b) (((d) << 10) | ((b) << 5) | (a) | 0x8000)
260 #define NV50_IR_SUBOP_Vn(n) ((n) >> 14)
261 #define NV50_IR_SUBOP_VOTE_ALL 0
262 #define NV50_IR_SUBOP_VOTE_ANY 1
263 #define NV50_IR_SUBOP_VOTE_UNI 2
264 #define NV50_IR_SUBOP_LOP3_LUT_SRC0 0xf0
265 #define NV50_IR_SUBOP_LOP3_LUT_SRC1 0xcc
266 #define NV50_IR_SUBOP_LOP3_LUT_SRC2 0xaa
267 #define NV50_IR_SUBOP_LOP3_LUT(exp) ({ \
268 uint8_t a = NV50_IR_SUBOP_LOP3_LUT_SRC0; \
269 uint8_t b = NV50_IR_SUBOP_LOP3_LUT_SRC1; \
270 uint8_t c = NV50_IR_SUBOP_LOP3_LUT_SRC2; \
273 #define NV50_IR_SUBOP_BMSK_C (0 << 0)
274 #define NV50_IR_SUBOP_BMSK_W (1 << 0)
276 #define NV50_IR_SUBOP_MINMAX_LOW 1
277 #define NV50_IR_SUBOP_MINMAX_MED 2
278 #define NV50_IR_SUBOP_MINMAX_HIGH 3
280 #define NV50_IR_SUBOP_SHF_L (0 << 0)
281 #define NV50_IR_SUBOP_SHF_R (1 << 0)
282 #define NV50_IR_SUBOP_SHF_LO (0 << 1)
283 #define NV50_IR_SUBOP_SHF_HI (1 << 1)
284 #define NV50_IR_SUBOP_SHF_C (0 << 2)
285 #define NV50_IR_SUBOP_SHF_W (1 << 2)
287 // xmad(src0, src1, 0) << 16 + src2
288 #define NV50_IR_SUBOP_XMAD_PSL (1 << 0)
289 // (xmad(src0, src1, src2) & 0xffff) | (src1 << 16)
290 #define NV50_IR_SUBOP_XMAD_MRG (1 << 1)
291 // xmad(src0, src1, src2.lo)
292 #define NV50_IR_SUBOP_XMAD_CLO (1 << 2)
293 // xmad(src0, src1, src2.hi)
294 #define NV50_IR_SUBOP_XMAD_CHI (2 << 2)
295 // if both operands to the multiplication are non-zero, subtract 65536 for each
297 #define NV50_IR_SUBOP_XMAD_CSFU (3 << 2)
298 // xmad(src0, src1, src2) + src1 << 16
299 #define NV50_IR_SUBOP_XMAD_CBCC (4 << 2)
300 #define NV50_IR_SUBOP_XMAD_CMODE_SHIFT 2
301 #define NV50_IR_SUBOP_XMAD_CMODE_MASK (0x7 << NV50_IR_SUBOP_XMAD_CMODE_SHIFT)
303 // use the high 16 bits instead of the low 16 bits for the multiplication.
304 // if the instruction's sType is signed, sign extend the operand from 16 bits
305 // to 32 before multiplication.
306 #define NV50_IR_SUBOP_XMAD_H1_SHIFT 5
307 #define NV50_IR_SUBOP_XMAD_H1(i) (1 << (NV50_IR_SUBOP_XMAD_H1_SHIFT + (i)))
308 #define NV50_IR_SUBOP_XMAD_H1_MASK (0x3 << NV50_IR_SUBOP_XMAD_H1_SHIFT)
319 TYPE_U64
, // 64 bit operations are only lowered after register allocation
331 CC_NEVER
= CC_FL
, // when used with FILE_FLAGS
334 CC_NOT_P
= CC_EQ
, // when used with FILE_PREDICATE
362 ROUND_M
, // towards -inf
363 ROUND_Z
, // towards 0
364 ROUND_P
, // towards +inf
365 ROUND_NI
, // nearest integer
366 ROUND_MI
, // to integer towards -inf
367 ROUND_ZI
, // to integer towards 0
368 ROUND_PI
, // to integer towards +inf
373 CACHE_CA
, // cache at all levels
374 CACHE_WB
= CACHE_CA
, // cache write back
375 CACHE_CG
, // cache at global level
376 CACHE_CS
, // cache streaming
377 CACHE_CV
, // cache as volatile
378 CACHE_WT
= CACHE_CV
// cache write-through
385 FILE_PREDICATE
, // boolean predicate
386 FILE_FLAGS
, // zero/sign/carry/overflow bits
388 LAST_REGISTER_FILE
= FILE_ADDRESS
,
408 TEX_TARGET_1D_SHADOW
,
409 TEX_TARGET_2D_SHADOW
,
410 TEX_TARGET_CUBE_SHADOW
,
413 TEX_TARGET_2D_MS_ARRAY
,
414 TEX_TARGET_CUBE_ARRAY
,
415 TEX_TARGET_1D_ARRAY_SHADOW
,
416 TEX_TARGET_2D_ARRAY_SHADOW
,
418 TEX_TARGET_RECT_SHADOW
,
419 TEX_TARGET_CUBE_ARRAY_SHADOW
,
492 SV_VERTEX_COUNT
, // gl_PatchVerticesIn
542 class CmpInstruction
;
543 class TexInstruction
;
544 class FlowInstruction
;
549 class ImmediateValue
;
554 int8_t fileIndex
; // signed, may be indirect for CONST[]
555 uint8_t size
; // this should match the Instruction type's size
556 DataType type
; // mainly for pretty printing
558 uint64_t u64
; // immediate values
568 int32_t offset
; // offset from 0 (base of address space)
569 int32_t id
; // register id (< 0 if virtual/unassigned, in units <= 4)
577 // precedence: NOT after SAT after NEG after ABS
578 #define NV50_IR_MOD_ABS (1 << 0)
579 #define NV50_IR_MOD_NEG (1 << 1)
580 #define NV50_IR_MOD_SAT (1 << 2)
581 #define NV50_IR_MOD_NOT (1 << 3)
582 #define NV50_IR_MOD_NEG_ABS (NV50_IR_MOD_NEG | NV50_IR_MOD_ABS)
584 #define NV50_IR_INTERP_MODE_MASK 0x3
585 #define NV50_IR_INTERP_LINEAR (0 << 0)
586 #define NV50_IR_INTERP_PERSPECTIVE (1 << 0)
587 #define NV50_IR_INTERP_FLAT (2 << 0)
588 #define NV50_IR_INTERP_SC (3 << 0) // what exactly is that ?
589 #define NV50_IR_INTERP_SAMPLE_MASK 0xc
590 #define NV50_IR_INTERP_DEFAULT (0 << 2)
591 #define NV50_IR_INTERP_CENTROID (1 << 2)
592 #define NV50_IR_INTERP_OFFSET (2 << 2)
593 #define NV50_IR_INTERP_SAMPLEID (3 << 2)
595 // do we really want this to be a class ?
599 Modifier() : bits(0) { }
600 Modifier(unsigned int m
) : bits(m
) { }
601 Modifier(operation op
);
603 // @return new Modifier applying a after b (asserts if unrepresentable)
604 Modifier
operator*(const Modifier
) const;
605 Modifier
operator*=(const Modifier m
) { *this = *this * m
; return *this; }
606 Modifier
operator==(const Modifier m
) const { return m
.bits
== bits
; }
607 Modifier
operator!=(const Modifier m
) const { return m
.bits
!= bits
; }
609 inline Modifier
operator&(const Modifier m
) const { return bits
& m
.bits
; }
610 inline Modifier
operator|(const Modifier m
) const { return bits
| m
.bits
; }
611 inline Modifier
operator^(const Modifier m
) const { return bits
^ m
.bits
; }
613 operation
getOp() const;
615 inline int neg() const { return (bits
& NV50_IR_MOD_NEG
) ? 1 : 0; }
616 inline int abs() const { return (bits
& NV50_IR_MOD_ABS
) ? 1 : 0; }
618 inline operator bool() const { return bits
? true : false; }
620 void applyTo(ImmediateValue
&imm
) const;
622 int print(char *buf
, size_t size
) const;
631 ValueRef(Value
* = NULL
);
632 ValueRef(const ValueRef
&);
635 inline bool exists() const { return value
!= NULL
; }
638 void set(const ValueRef
&);
639 inline Value
*get() const { return value
; }
640 inline Value
*rep() const;
642 inline Instruction
*getInsn() const { return insn
; }
643 inline void setInsn(Instruction
*inst
) { insn
= inst
; }
645 inline bool isIndirect(int dim
) const { return indirect
[dim
] >= 0; }
646 inline const ValueRef
*getIndirect(int dim
) const;
648 inline DataFile
getFile() const;
649 inline unsigned getSize() const;
651 // SSA: return eventual (traverse MOVs) literal value, if it exists
652 bool getImmediate(ImmediateValue
&) const;
656 int8_t indirect
[2]; // >= 0 if relative to lvalue in insn->src(indirect[i])
658 bool usedAsPtr
; // for printing
668 ValueDef(Value
* = NULL
);
669 ValueDef(const ValueDef
&);
672 inline bool exists() const { return value
!= NULL
; }
674 inline Value
*get() const { return value
; }
675 inline Value
*rep() const;
677 bool mayReplace(const ValueRef
&);
678 void replace(const ValueRef
&, bool doSet
); // replace all uses of the old value
680 inline Instruction
*getInsn() const { return insn
; }
681 inline void setInsn(Instruction
*inst
) { insn
= inst
; }
683 inline DataFile
getFile() const;
684 inline unsigned getSize() const;
686 inline void setSSA(LValue
*);
687 inline const LValue
*preSSA() const;
690 Value
*value
; // should make this LValue * ...
691 LValue
*origin
; // pre SSA value
701 virtual Value
*clone(ClonePolicy
<Function
>&) const = 0;
703 virtual int print(char *, size_t, DataType ty
= TYPE_NONE
) const = 0;
705 virtual bool equals(const Value
*, bool strict
= false) const;
706 virtual bool interfers(const Value
*) const;
707 virtual bool isUniform() const { return true; }
709 inline Value
*rep() const { return join
; }
711 inline Instruction
*getUniqueInsn() const;
712 inline Instruction
*getInsn() const; // use when uniqueness is certain
714 inline int refCount() { return uses
.size(); }
716 inline LValue
*asLValue();
717 inline Symbol
*asSym();
718 inline ImmediateValue
*asImm();
719 inline const Symbol
*asSym() const;
720 inline const ImmediateValue
*asImm() const;
722 inline bool inFile(DataFile f
) const { return reg
.file
== f
; }
724 static inline Value
*get(Iterator
&);
726 unordered_set
<ValueRef
*> uses
;
727 std::list
<ValueDef
*> defs
;
728 typedef unordered_set
<ValueRef
*>::iterator UseIterator
;
729 typedef unordered_set
<ValueRef
*>::const_iterator UseCIterator
;
730 typedef std::list
<ValueDef
*>::iterator DefIterator
;
731 typedef std::list
<ValueDef
*>::const_iterator DefCIterator
;
736 // TODO: these should be in LValue:
741 class LValue
: public Value
744 LValue(Function
*, DataFile file
);
745 LValue(Function
*, LValue
*);
748 virtual bool isUniform() const;
750 virtual LValue
*clone(ClonePolicy
<Function
>&) const;
752 virtual int print(char *, size_t, DataType ty
= TYPE_NONE
) const;
755 unsigned compMask
: 8; // compound/component mask
756 unsigned compound
: 1; // used by RA, value involved in split/merge
758 unsigned fixedReg
: 1; // set & used by RA, earlier just use (id < 0)
759 unsigned noSpill
: 1; // do not spill (e.g. if spill temporary already)
762 class Symbol
: public Value
765 Symbol(Program
*, DataFile file
= FILE_MEMORY_CONST
, ubyte fileIdx
= 0);
768 virtual Symbol
*clone(ClonePolicy
<Function
>&) const;
770 virtual bool equals(const Value
*that
, bool strict
) const;
772 virtual bool isUniform() const;
774 virtual int print(char *, size_t, DataType ty
= TYPE_NONE
) const;
776 // print with indirect values
777 int print(char *, size_t, Value
*, Value
*, DataType ty
= TYPE_NONE
) const;
779 inline void setFile(DataFile file
, ubyte fileIndex
= 0)
782 reg
.fileIndex
= fileIndex
;
785 inline void setOffset(int32_t offset
);
786 inline void setAddress(Symbol
*base
, int32_t offset
);
787 inline void setSV(SVSemantic sv
, uint32_t idx
= 0);
789 inline const Symbol
*getBase() const { return baseSym
; }
792 Symbol
*baseSym
; // array base for Symbols representing array elements
795 class ImmediateValue
: public Value
799 ImmediateValue(Program
*, uint32_t);
800 ImmediateValue(Program
*, float);
801 ImmediateValue(Program
*, double);
802 // NOTE: not added to program with
803 ImmediateValue(const ImmediateValue
*, DataType ty
);
804 ~ImmediateValue() { };
806 virtual ImmediateValue
*clone(ClonePolicy
<Function
>&) const;
808 virtual bool equals(const Value
*that
, bool strict
) const;
810 // these only work if 'type' is valid (we mostly use untyped literals):
811 bool isInteger(const int ival
) const; // ival is cast to this' type
812 bool isNegative() const;
817 // for constant folding:
818 ImmediateValue
operator+(const ImmediateValue
&) const;
819 ImmediateValue
operator-(const ImmediateValue
&) const;
820 ImmediateValue
operator*(const ImmediateValue
&) const;
821 ImmediateValue
operator/(const ImmediateValue
&) const;
823 ImmediateValue
& operator=(const ImmediateValue
&); // only sets value !
825 bool compare(CondCode cc
, float fval
) const;
827 virtual int print(char *, size_t, DataType ty
= TYPE_NONE
) const;
834 Instruction(Function
*, operation
, DataType
);
835 virtual ~Instruction();
837 virtual Instruction
*clone(ClonePolicy
<Function
>&,
838 Instruction
* = NULL
) const;
840 void setDef(int i
, Value
*);
841 void setSrc(int s
, Value
*);
842 void setSrc(int s
, const ValueRef
&);
843 void swapSources(int a
, int b
);
844 void moveSources(int s
, int delta
);
845 bool setIndirect(int s
, int dim
, Value
*);
847 inline ValueRef
& src(int s
) { return srcs
[s
]; }
848 inline ValueDef
& def(int s
) { return defs
[s
]; }
849 inline const ValueRef
& src(int s
) const { return srcs
[s
]; }
850 inline const ValueDef
& def(int s
) const { return defs
[s
]; }
852 inline Value
*getDef(int d
) const { return defs
[d
].get(); }
853 inline Value
*getSrc(int s
) const { return srcs
[s
].get(); }
854 inline Value
*getIndirect(int s
, int dim
) const;
856 inline bool defExists(unsigned d
) const
858 return d
< defs
.size() && defs
[d
].exists();
860 inline bool srcExists(unsigned s
) const
862 return s
< srcs
.size() && srcs
[s
].exists();
865 inline bool constrainedDefs() const;
867 bool setPredicate(CondCode ccode
, Value
*);
868 inline Value
*getPredicate() const;
869 bool writesPredicate() const;
870 inline bool isPredicated() const { return predSrc
>= 0; }
872 inline void setFlagsSrc(int s
, Value
*);
873 inline void setFlagsDef(int d
, Value
*);
874 inline bool usesFlags() const { return flagsSrc
>= 0; }
876 unsigned int defCount() const { return defs
.size(); };
877 unsigned int defCount(unsigned int mask
, bool singleFile
= false) const;
878 unsigned int srcCount() const { return srcs
.size(); };
879 unsigned int srcCount(unsigned int mask
, bool singleFile
= false) const;
881 // save & remove / set indirect[0,1] and predicate source
882 void takeExtraSources(int s
, Value
*[3]);
883 void putExtraSources(int s
, Value
*[3]);
885 inline void setType(DataType type
) { dType
= sType
= type
; }
887 inline void setType(DataType dtype
, DataType stype
)
893 inline bool isPseudo() const { return op
< OP_MOV
; }
896 bool isCommutationLegal(const Instruction
*) const; // must be adjacent !
897 bool isActionEqual(const Instruction
*) const;
898 bool isResultEqual(const Instruction
*) const;
900 // check whether the defs interfere with srcs and defs of another instruction
901 bool canCommuteDefDef(const Instruction
*) const;
902 bool canCommuteDefSrc(const Instruction
*) const;
906 inline CmpInstruction
*asCmp();
907 inline TexInstruction
*asTex();
908 inline FlowInstruction
*asFlow();
909 inline const TexInstruction
*asTex() const;
910 inline const CmpInstruction
*asCmp() const;
911 inline const FlowInstruction
*asFlow() const;
917 int serial
; // CFG order
920 DataType dType
; // destination or defining type
921 DataType sType
; // source or secondary type
926 uint16_t subOp
; // quadop, 1 for mul-high, etc.
928 unsigned encSize
: 5; // encoding size in bytes
929 unsigned saturate
: 1; // to [0.0f, 1.0f]
930 unsigned join
: 1; // converge control flow (use OP_JOIN until end)
931 unsigned fixed
: 1; // prevent dead code elimination
932 unsigned terminator
: 1; // end of basic block
933 unsigned ftz
: 1; // flush denormal to zero
934 unsigned dnz
: 1; // denormals, NaN are zero
935 unsigned ipa
: 4; // interpolation mode
937 unsigned perPatch
: 1;
938 unsigned exit
: 1; // terminate program after insn
939 unsigned mask
: 4; // for vector ops
940 // prevent algebraic optimisations that aren't bit-for-bit identical
941 unsigned precise
: 1;
943 int8_t postFactor
; // MUL/DIV(if < 0) by 1 << postFactor
949 uint32_t sched
; // scheduling data (NOTE: maybe move to separate storage)
954 std::deque
<ValueDef
> defs
; // no gaps !
955 std::deque
<ValueRef
> srcs
; // no gaps !
957 // instruction specific methods:
958 // (don't want to subclass, would need more constructors and memory pools)
960 inline void setInterpolate(unsigned int mode
) { ipa
= mode
; }
962 unsigned int getInterpMode() const { return ipa
& 0x3; }
963 unsigned int getSampleMode() const { return ipa
& 0xc; }
971 TXQ_DIMS
, /* x, y, z, levels */
972 TXQ_TYPE
, /* ?, ?, samples, ? */
980 class TexInstruction
: public Instruction
986 Target(TexTarget targ
= TEX_TARGET_1D
) : target(targ
) { }
988 const char *getName() const { return descTable
[target
].name
; }
989 unsigned int getArgCount() const { return descTable
[target
].argc
; }
990 unsigned int getDim() const { return descTable
[target
].dim
; }
991 int isArray() const { return descTable
[target
].array
? 1 : 0; }
992 int isCube() const { return descTable
[target
].cube
? 1 : 0; }
993 int isShadow() const { return descTable
[target
].shadow
? 1 : 0; }
995 return target
== TEX_TARGET_2D_MS
|| target
== TEX_TARGET_2D_MS_ARRAY
; }
999 target
= TEX_TARGET_2D_ARRAY
;
1001 target
= TEX_TARGET_2D
;
1005 Target
& operator=(TexTarget targ
)
1007 assert(targ
< TEX_TARGET_COUNT
);
1012 inline bool operator==(TexTarget targ
) const { return target
== targ
; }
1013 inline bool operator!=(TexTarget targ
) const { return target
!= targ
; }
1015 enum TexTarget
getEnum() const { return target
; }
1028 static const struct Desc descTable
[TEX_TARGET_COUNT
];
1031 enum TexTarget target
;
1035 struct ImgFormatDesc
1044 static const struct ImgFormatDesc formatTable
[IMG_FORMAT_COUNT
];
1045 static const struct ImgFormatDesc
*translateImgFormat(
1046 enum pipe_format format
);
1049 TexInstruction(Function
*, operation
);
1050 virtual ~TexInstruction();
1052 virtual TexInstruction
*clone(ClonePolicy
<Function
>&,
1053 Instruction
* = NULL
) const;
1055 inline void setTexture(Target targ
, uint8_t r
, uint8_t s
)
1062 void setIndirectR(Value
*);
1063 void setIndirectS(Value
*);
1064 inline Value
*getIndirectR() const;
1065 inline Value
*getIndirectS() const;
1073 int8_t rIndirectSrc
;
1074 int8_t sIndirectSrc
;
1079 bool liveOnly
; // only execute on live pixels of a quad (optimization)
1084 int8_t useOffsets
; // 0, 1, or 4 for textureGatherOffsets
1085 int8_t offset
[3]; // only used on nv50
1087 enum TexQuery query
;
1088 const struct ImgFormatDesc
*format
;
1090 bool scalar
; // for GM107s TEXS, TLDS, TLD4S
1095 ValueRef offset
[4][3];
1098 class CmpInstruction
: public Instruction
1101 CmpInstruction(Function
*, operation
);
1103 virtual CmpInstruction
*clone(ClonePolicy
<Function
>&,
1104 Instruction
* = NULL
) const;
1106 void setCondition(CondCode cond
) { setCond
= cond
; }
1107 CondCode
getCondition() const { return setCond
; }
1113 class FlowInstruction
: public Instruction
1116 FlowInstruction(Function
*, operation
, void *target
);
1118 virtual FlowInstruction
*clone(ClonePolicy
<Function
>&,
1119 Instruction
* = NULL
) const;
1122 unsigned allWarp
: 1;
1123 unsigned absolute
: 1;
1125 unsigned builtin
: 1; // true for calls to emulation code
1126 unsigned indirect
: 1; // target in src(0)
1138 BasicBlock(Function
*);
1141 BasicBlock
*clone(ClonePolicy
<Function
>&) const;
1143 inline int getId() const { return id
; }
1144 inline unsigned int getInsnCount() const { return numInsns
; }
1145 inline bool isTerminated() const { return exit
&& exit
->terminator
; }
1147 bool dominatedBy(BasicBlock
*bb
);
1148 inline bool reachableBy(const BasicBlock
*by
, const BasicBlock
*term
);
1150 // returns mask of conditional out blocks
1151 // e.g. 3 for IF { .. } ELSE { .. } ENDIF, 1 for IF { .. } ENDIF
1152 unsigned int initiatesSimpleConditional() const;
1155 Function
*getFunction() const { return func
; }
1156 Program
*getProgram() const { return program
; }
1158 Instruction
*getEntry() const { return entry
; } // first non-phi instruction
1159 Instruction
*getPhi() const { return phi
; }
1160 Instruction
*getFirst() const { return phi
? phi
: entry
; }
1161 Instruction
*getExit() const { return exit
; }
1163 void insertHead(Instruction
*);
1164 void insertTail(Instruction
*);
1165 void insertBefore(Instruction
*, Instruction
*);
1166 void insertAfter(Instruction
*, Instruction
*);
1167 void remove(Instruction
*);
1168 void permuteAdjacent(Instruction
*, Instruction
*);
1170 BasicBlock
*idom() const;
1172 // NOTE: currently does not rebuild the dominator tree
1173 BasicBlock
*splitBefore(Instruction
*, bool attach
= true);
1174 BasicBlock
*splitAfter(Instruction
*, bool attach
= true);
1176 DLList
& getDF() { return df
; }
1177 DLList::Iterator
iterDF() { return df
.iterator(); }
1179 static inline BasicBlock
*get(Iterator
&);
1180 static inline BasicBlock
*get(Graph::Node
*);
1183 Graph::Node cfg
; // first edge is branch *taken* (the ELSE branch)
1192 Instruction
*joinAt
; // for quick reference
1194 bool explicitCont
; // loop headers: true if loop contains continue stmts
1204 unsigned int numInsns
;
1210 void splitCommon(Instruction
*, BasicBlock
*, bool attach
);
1216 Function(Program
*, const char *name
, uint32_t label
);
1219 static inline Function
*get(Graph::Node
*node
);
1221 inline Program
*getProgram() const { return prog
; }
1222 inline const char *getName() const { return name
; }
1223 inline int getId() const { return id
; }
1224 inline uint32_t getLabel() const { return label
; }
1227 void printLiveIntervals() const;
1228 void printCFGraph(const char *filePath
);
1230 bool setEntry(BasicBlock
*);
1231 bool setExit(BasicBlock
*);
1233 unsigned int orderInstructions(ArrayList
&);
1235 inline void add(BasicBlock
*bb
, int& id
) { allBBlocks
.insert(bb
, id
); }
1236 inline void add(Instruction
*insn
, int& id
) { allInsns
.insert(insn
, id
); }
1237 inline void add(LValue
*lval
, int& id
) { allLValues
.insert(lval
, id
); }
1239 inline LValue
*getLValue(int id
);
1241 void buildLiveSets();
1242 void buildDefSets();
1243 bool convertToSSA();
1246 std::deque
<ValueDef
> ins
;
1247 std::deque
<ValueRef
> outs
;
1248 std::deque
<Value
*> clobbers
;
1251 Graph::Node
*cfgExit
;
1253 Graph::Node call
; // node in the call graph
1255 BasicBlock
**bbArray
; // BBs in emission order
1258 unsigned int loopNestingBound
;
1266 uint32_t tlsBase
; // base address for l[] space (if no stack pointer is used)
1269 ArrayList allBBlocks
;
1271 ArrayList allLValues
;
1274 void buildLiveSetsPreSSA(BasicBlock
*, const int sequence
);
1275 void buildDefSetsPreSSA(BasicBlock
*bb
, const int seq
);
1280 const char *const name
;
1287 CG_STAGE_SSA
, // expected directly before register allocation
1297 TYPE_TESSELLATION_CONTROL
,
1298 TYPE_TESSELLATION_EVAL
,
1304 Program(Type type
, Target
*targ
);
1309 Type
getType() const { return progType
; }
1311 inline void add(Function
*fn
, int& id
) { allFuncs
.insert(fn
, id
); }
1312 inline void del(Function
*fn
, int& id
) { allFuncs
.remove(id
); }
1313 inline void add(Value
*rval
, int& id
) { allRValues
.insert(rval
, id
); }
1315 bool makeFromNIR(struct nv50_ir_prog_info
*,
1316 struct nv50_ir_prog_info_out
*);
1317 bool makeFromTGSI(struct nv50_ir_prog_info
*,
1318 struct nv50_ir_prog_info_out
*);
1319 bool convertToSSA();
1320 bool optimizeSSA(int level
);
1321 bool optimizePostRA(int level
);
1322 bool registerAllocation();
1323 bool emitBinary(struct nv50_ir_prog_info_out
*);
1325 const Target
*getTarget() const { return target
; }
1336 ArrayList allRValues
;
1340 uint32_t tlsSize
; // size required for FILE_MEMORY_LOCAL
1344 bool persampleInvocation
;
1346 MemoryPool mem_Instruction
;
1347 MemoryPool mem_CmpInstruction
;
1348 MemoryPool mem_TexInstruction
;
1349 MemoryPool mem_FlowInstruction
;
1350 MemoryPool mem_LValue
;
1351 MemoryPool mem_Symbol
;
1352 MemoryPool mem_ImmediateValue
;
1357 void *targetPriv
; // e.g. to carry information between passes
1359 const struct nv50_ir_prog_info
*driver
; // for driver configuration
1360 const struct nv50_ir_prog_info_out
*driver_out
; // for driver configuration
1362 void releaseInstruction(Instruction
*);
1363 void releaseValue(Value
*);
1366 // TODO: add const version
1370 bool run(Program
*, bool ordered
= false, bool skipPhi
= false);
1371 bool run(Function
*, bool ordered
= false, bool skipPhi
= false);
1374 // return false to continue with next entity on next higher level
1375 virtual bool visit(Function
*) { return true; }
1376 virtual bool visit(BasicBlock
*) { return true; }
1377 virtual bool visit(Instruction
*) { return false; }
1379 bool doRun(Program
*, bool ordered
, bool skipPhi
);
1380 bool doRun(Function
*, bool ordered
, bool skipPhi
);
1388 // =============================================================================
1390 #include "codegen/nv50_ir_inlines.h"
1392 } // namespace nv50_ir
1394 #endif // __NV50_IR_H__