projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
freedreno/ir3: decouple regset from gpu gen
[mesa.git] / src / freedreno / ir3 / ir3.h
1 /*
2 * Copyright (c) 2013 Rob Clark <robdclark@gmail.com>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
23
24 #ifndef IR3_H_
25 #define IR3_H_
26
27 #include <stdint.h>
28 #include <stdbool.h>
29
30 #include "compiler/shader_enums.h"
31
32 #include "util/bitscan.h"
33 #include "util/list.h"
34 #include "util/set.h"
35 #include "util/u_debug.h"
36
37 #include "instr-a3xx.h"
38
39 /* low level intermediate representation of an adreno shader program */
40
41 struct ir3_compiler;
42 struct ir3;
43 struct ir3_instruction;
44 struct ir3_block;
45
46 struct ir3_info {
47 uint32_t gpu_id;
48 uint16_t sizedwords;
49 uint16_t instrs_count; /* expanded to account for rpt's */
50 uint16_t nops_count; /* # of nop instructions, including nopN */
51 uint16_t mov_count;
52 uint16_t cov_count;
53 /* NOTE: max_reg, etc, does not include registers not touched
54 * by the shader (ie. vertex fetched via VFD_DECODE but not
55 * touched by shader)
56 */
57 int8_t max_reg; /* highest GPR # used by shader */
58 int8_t max_half_reg;
59 int16_t max_const;
60
61 /* number of sync bits: */
62 uint16_t ss, sy;
63
64 /* estimate of number of cycles stalled on (ss) */
65 uint16_t sstall;
66
67 uint16_t last_baryf; /* instruction # of last varying fetch */
68 };
69
70 struct ir3_register {
71 enum {
72 IR3_REG_CONST = 0x001,
73 IR3_REG_IMMED = 0x002,
74 IR3_REG_HALF = 0x004,
75 /* high registers are used for some things in compute shaders,
76 * for example. Seems to be for things that are global to all
77 * threads in a wave, so possibly these are global/shared by
78 * all the threads in the wave?
79 */
80 IR3_REG_HIGH = 0x008,
81 IR3_REG_RELATIV= 0x010,
82 IR3_REG_R = 0x020,
83 /* Most instructions, it seems, can do float abs/neg but not
84 * integer. The CP pass needs to know what is intended (int or
85 * float) in order to do the right thing. For this reason the
86 * abs/neg flags are split out into float and int variants. In
87 * addition, .b (bitwise) operations, the negate is actually a
88 * bitwise not, so split that out into a new flag to make it
89 * more clear.
90 */
91 IR3_REG_FNEG = 0x040,
92 IR3_REG_FABS = 0x080,
93 IR3_REG_SNEG = 0x100,
94 IR3_REG_SABS = 0x200,
95 IR3_REG_BNOT = 0x400,
96 IR3_REG_EVEN = 0x800,
97 IR3_REG_POS_INF= 0x1000,
98 /* (ei) flag, end-input? Set on last bary, presumably to signal
99 * that the shader needs no more input:
100 */
101 IR3_REG_EI = 0x2000,
102 /* meta-flags, for intermediate stages of IR, ie.
103 * before register assignment is done:
104 */
105 IR3_REG_SSA = 0x4000, /* 'instr' is ptr to assigning instr */
106 IR3_REG_ARRAY = 0x8000,
107
108 } flags;
109
110 /* used for cat5 instructions, but also for internal/IR level
111 * tracking of what registers are read/written by an instruction.
112 * wrmask may be a bad name since it is used to represent both
113 * src and dst that touch multiple adjacent registers.
114 */
115 unsigned wrmask : 16; /* up to vec16 */
116
117 /* for relative addressing, 32bits for array size is too small,
118 * but otoh we don't need to deal with disjoint sets, so instead
119 * use a simple size field (number of scalar components).
120 *
121 * Note the size field isn't important for relative const (since
122 * we don't have to do register allocation for constants).
123 */
124 unsigned size : 16;
125
126 /* normal registers:
127 * the component is in the low two bits of the reg #, so
128 * rN.x becomes: (N << 2) | x
129 */
130 uint16_t num;
131 union {
132 /* immediate: */
133 int32_t iim_val;
134 uint32_t uim_val;
135 float fim_val;
136 /* relative: */
137 struct {
138 uint16_t id;
139 int16_t offset;
140 } array;
141 };
142
143 /* For IR3_REG_SSA, src registers contain ptr back to assigning
144 * instruction.
145 *
146 * For IR3_REG_ARRAY, the pointer is back to the last dependent
147 * array access (although the net effect is the same, it points
148 * back to a previous instruction that we depend on).
149 */
150 struct ir3_instruction *instr;
151 };
152
153 /*
154 * Stupid/simple growable array implementation:
155 */
156 #define DECLARE_ARRAY(type, name) \
157 unsigned name ## _count, name ## _sz; \
158 type * name;
159
160 #define array_insert(ctx, arr, val) do { \
161 if (arr ## _count == arr ## _sz) { \
162 arr ## _sz = MAX2(2 * arr ## _sz, 16); \
163 arr = reralloc_size(ctx, arr, arr ## _sz * sizeof(arr[0])); \
164 } \
165 arr[arr ##_count++] = val; \
166 } while (0)
167
168 struct ir3_instruction {
169 struct ir3_block *block;
170 opc_t opc;
171 enum {
172 /* (sy) flag is set on first instruction, and after sample
173 * instructions (probably just on RAW hazard).
174 */
175 IR3_INSTR_SY = 0x001,
176 /* (ss) flag is set on first instruction, and first instruction
177 * to depend on the result of "long" instructions (RAW hazard):
178 *
179 * rcp, rsq, log2, exp2, sin, cos, sqrt
180 *
181 * It seems to synchronize until all in-flight instructions are
182 * completed, for example:
183 *
184 * rsq hr1.w, hr1.w
185 * add.f hr2.z, (neg)hr2.z, hc0.y
186 * mul.f hr2.w, (neg)hr2.y, (neg)hr2.y
187 * rsq hr2.x, hr2.x
188 * (rpt1)nop
189 * mad.f16 hr2.w, hr2.z, hr2.z, hr2.w
190 * nop
191 * mad.f16 hr2.w, (neg)hr0.w, (neg)hr0.w, hr2.w
192 * (ss)(rpt2)mul.f hr1.x, (r)hr1.x, hr1.w
193 * (rpt2)mul.f hr0.x, (neg)(r)hr0.x, hr2.x
194 *
195 * The last mul.f does not have (ss) set, presumably because the
196 * (ss) on the previous instruction does the job.
197 *
198 * The blob driver also seems to set it on WAR hazards, although
199 * not really clear if this is needed or just blob compiler being
200 * sloppy. So far I haven't found a case where removing the (ss)
201 * causes problems for WAR hazard, but I could just be getting
202 * lucky:
203 *
204 * rcp r1.y, r3.y
205 * (ss)(rpt2)mad.f32 r3.y, (r)c9.x, r1.x, (r)r3.z
206 *
207 */
208 IR3_INSTR_SS = 0x002,
209 /* (jp) flag is set on jump targets:
210 */
211 IR3_INSTR_JP = 0x004,
212 IR3_INSTR_UL = 0x008,
213 IR3_INSTR_3D = 0x010,
214 IR3_INSTR_A = 0x020,
215 IR3_INSTR_O = 0x040,
216 IR3_INSTR_P = 0x080,
217 IR3_INSTR_S = 0x100,
218 IR3_INSTR_S2EN = 0x200,
219 IR3_INSTR_G = 0x400,
220 IR3_INSTR_SAT = 0x800,
221 /* (cat5/cat6) Bindless */
222 IR3_INSTR_B = 0x1000,
223 /* (cat5-only) Get some parts of the encoding from a1.x */
224 IR3_INSTR_A1EN = 0x2000,
225 /* meta-flags, for intermediate stages of IR, ie.
226 * before register assignment is done:
227 */
228 IR3_INSTR_MARK = 0x4000,
229 IR3_INSTR_UNUSED= 0x8000,
230 } flags;
231 uint8_t repeat;
232 uint8_t nop;
233 #ifdef DEBUG
234 unsigned regs_max;
235 #endif
236 unsigned regs_count;
237 struct ir3_register **regs;
238 union {
239 struct {
240 char inv;
241 char comp;
242 int immed;
243 struct ir3_block *target;
244 } cat0;
245 struct {
246 type_t src_type, dst_type;
247 } cat1;
248 struct {
249 enum {
250 IR3_COND_LT = 0,
251 IR3_COND_LE = 1,
252 IR3_COND_GT = 2,
253 IR3_COND_GE = 3,
254 IR3_COND_EQ = 4,
255 IR3_COND_NE = 5,
256 } condition;
257 } cat2;
258 struct {
259 unsigned samp, tex;
260 unsigned tex_base : 3;
261 type_t type;
262 } cat5;
263 struct {
264 type_t type;
265 int src_offset;
266 int dst_offset;
267 int iim_val : 3; /* for ldgb/stgb, # of components */
268 unsigned d : 3; /* for ldc, component offset */
269 bool typed : 1;
270 unsigned base : 3;
271 } cat6;
272 struct {
273 unsigned w : 1; /* write */
274 unsigned r : 1; /* read */
275 unsigned l : 1; /* local */
276 unsigned g : 1; /* global */
277 } cat7;
278 /* for meta-instructions, just used to hold extra data
279 * before instruction scheduling, etc
280 */
281 struct {
282 int off; /* component/offset */
283 } split;
284 struct {
285 /* for output collects, this maps back to the entry in the
286 * ir3_shader_variant::outputs table.
287 */
288 int outidx;
289 } collect;
290 struct {
291 unsigned samp, tex;
292 unsigned input_offset;
293 unsigned samp_base : 3;
294 unsigned tex_base : 3;
295 } prefetch;
296 struct {
297 /* maps back to entry in ir3_shader_variant::inputs table: */
298 int inidx;
299 /* for sysvals, identifies the sysval type. Mostly so we can
300 * identify the special cases where a sysval should not be DCE'd
301 * (currently, just pre-fs texture fetch)
302 */
303 gl_system_value sysval;
304 } input;
305 };
306
307 /* When we get to the RA stage, we need instruction's position/name: */
308 uint16_t ip;
309 uint16_t name;
310
311 /* used for per-pass extra instruction data.
312 *
313 * TODO we should remove the per-pass data like this and 'use_count'
314 * and do something similar to what RA does w/ ir3_ra_instr_data..
315 * ie. use the ir3_count_instructions pass, and then use instr->ip
316 * to index into a table of pass-private data.
317 */
318 void *data;
319
320 /**
321 * Valid if pass calls ir3_find_ssa_uses().. see foreach_ssa_use()
322 */
323 struct set *uses;
324
325 int use_count; /* currently just updated/used by cp */
326
327 /* Used during CP and RA stages. For collect and shader inputs/
328 * outputs where we need a sequence of consecutive registers,
329 * keep track of each src instructions left (ie 'n-1') and right
330 * (ie 'n+1') neighbor. The front-end must insert enough mov's
331 * to ensure that each instruction has at most one left and at
332 * most one right neighbor. During the copy-propagation pass,
333 * we only remove mov's when we can preserve this constraint.
334 * And during the RA stage, we use the neighbor information to
335 * allocate a block of registers in one shot.
336 *
337 * TODO: maybe just add something like:
338 * struct ir3_instruction_ref {
339 * struct ir3_instruction *instr;
340 * unsigned cnt;
341 * }
342 *
343 * Or can we get away without the refcnt stuff? It seems like
344 * it should be overkill.. the problem is if, potentially after
345 * already eliminating some mov's, if you have a single mov that
346 * needs to be grouped with it's neighbors in two different
347 * places (ex. shader output and a collect).
348 */
349 struct {
350 struct ir3_instruction *left, *right;
351 uint16_t left_cnt, right_cnt;
352 } cp;
353
354 /* an instruction can reference at most one address register amongst
355 * it's src/dst registers. Beyond that, you need to insert mov's.
356 *
357 * NOTE: do not write this directly, use ir3_instr_set_address()
358 */
359 struct ir3_instruction *address;
360
361 /* Tracking for additional dependent instructions. Used to handle
362 * barriers, WAR hazards for arrays/SSBOs/etc.
363 */
364 DECLARE_ARRAY(struct ir3_instruction *, deps);
365
366 /*
367 * From PoV of instruction scheduling, not execution (ie. ignores global/
368 * local distinction):
369 * shared image atomic SSBO everything
370 * barrier()/ - R/W R/W R/W R/W X
371 * groupMemoryBarrier()
372 * memoryBarrier() - R/W R/W
373 * (but only images declared coherent?)
374 * memoryBarrierAtomic() - R/W
375 * memoryBarrierBuffer() - R/W
376 * memoryBarrierImage() - R/W
377 * memoryBarrierShared() - R/W
378 *
379 * TODO I think for SSBO/image/shared, in cases where we can determine
380 * which variable is accessed, we don't need to care about accesses to
381 * different variables (unless declared coherent??)
382 */
383 enum {
384 IR3_BARRIER_EVERYTHING = 1 << 0,
385 IR3_BARRIER_SHARED_R = 1 << 1,
386 IR3_BARRIER_SHARED_W = 1 << 2,
387 IR3_BARRIER_IMAGE_R = 1 << 3,
388 IR3_BARRIER_IMAGE_W = 1 << 4,
389 IR3_BARRIER_BUFFER_R = 1 << 5,
390 IR3_BARRIER_BUFFER_W = 1 << 6,
391 IR3_BARRIER_ARRAY_R = 1 << 7,
392 IR3_BARRIER_ARRAY_W = 1 << 8,
393 } barrier_class, barrier_conflict;
394
395 /* Entry in ir3_block's instruction list: */
396 struct list_head node;
397
398 #ifdef DEBUG
399 uint32_t serialno;
400 #endif
401
402 // TODO only computerator/assembler:
403 int line;
404 };
405
406 static inline struct ir3_instruction *
407 ir3_neighbor_first(struct ir3_instruction *instr)
408 {
409 int cnt = 0;
410 while (instr->cp.left) {
411 instr = instr->cp.left;
412 if (++cnt > 0xffff) {
413 debug_assert(0);
414 break;
415 }
416 }
417 return instr;
418 }
419
420 static inline int ir3_neighbor_count(struct ir3_instruction *instr)
421 {
422 int num = 1;
423
424 debug_assert(!instr->cp.left);
425
426 while (instr->cp.right) {
427 num++;
428 instr = instr->cp.right;
429 if (num > 0xffff) {
430 debug_assert(0);
431 break;
432 }
433 }
434
435 return num;
436 }
437
438 struct ir3 {
439 struct ir3_compiler *compiler;
440 gl_shader_stage type;
441
442 DECLARE_ARRAY(struct ir3_instruction *, inputs);
443 DECLARE_ARRAY(struct ir3_instruction *, outputs);
444
445 /* Track bary.f (and ldlv) instructions.. this is needed in
446 * scheduling to ensure that all varying fetches happen before
447 * any potential kill instructions. The hw gets grumpy if all
448 * threads in a group are killed before the last bary.f gets
449 * a chance to signal end of input (ei).
450 */
451 DECLARE_ARRAY(struct ir3_instruction *, baryfs);
452
453 /* Track all indirect instructions (read and write). To avoid
454 * deadlock scenario where an address register gets scheduled,
455 * but other dependent src instructions cannot be scheduled due
456 * to dependency on a *different* address register value, the
457 * scheduler needs to ensure that all dependencies other than
458 * the instruction other than the address register are scheduled
459 * before the one that writes the address register. Having a
460 * convenient list of instructions that reference some address
461 * register simplifies this.
462 */
463 DECLARE_ARRAY(struct ir3_instruction *, a0_users);
464
465 /* same for a1.x: */
466 DECLARE_ARRAY(struct ir3_instruction *, a1_users);
467
468 /* and same for instructions that consume predicate register: */
469 DECLARE_ARRAY(struct ir3_instruction *, predicates);
470
471 /* Track texture sample instructions which need texture state
472 * patched in (for astc-srgb workaround):
473 */
474 DECLARE_ARRAY(struct ir3_instruction *, astc_srgb);
475
476 /* List of blocks: */
477 struct list_head block_list;
478
479 /* List of ir3_array's: */
480 struct list_head array_list;
481
482 #ifdef DEBUG
483 unsigned block_count, instr_count;
484 #endif
485 };
486
487 struct ir3_array {
488 struct list_head node;
489 unsigned length;
490 unsigned id;
491
492 struct nir_register *r;
493
494 /* To avoid array write's from getting DCE'd, keep track of the
495 * most recent write. Any array access depends on the most
496 * recent write. This way, nothing depends on writes after the
497 * last read. But all the writes that happen before that have
498 * something depending on them
499 */
500 struct ir3_instruction *last_write;
501
502 /* extra stuff used in RA pass: */
503 unsigned base; /* base vreg name */
504 unsigned reg; /* base physical reg */
505 uint16_t start_ip, end_ip;
506
507 /* Indicates if half-precision */
508 bool half;
509 };
510
511 struct ir3_array * ir3_lookup_array(struct ir3 *ir, unsigned id);
512
513 struct ir3_block {
514 struct list_head node;
515 struct ir3 *shader;
516
517 const struct nir_block *nblock;
518
519 struct list_head instr_list; /* list of ir3_instruction */
520
521 /* each block has either one or two successors.. in case of
522 * two successors, 'condition' decides which one to follow.
523 * A block preceding an if/else has two successors.
524 */
525 struct ir3_instruction *condition;
526 struct ir3_block *successors[2];
527
528 struct set *predecessors; /* set of ir3_block */
529
530 uint16_t start_ip, end_ip;
531
532 /* Track instructions which do not write a register but other-
533 * wise must not be discarded (such as kill, stg, etc)
534 */
535 DECLARE_ARRAY(struct ir3_instruction *, keeps);
536
537 /* used for per-pass extra block data. Mainly used right
538 * now in RA step to track livein/liveout.
539 */
540 void *data;
541
542 #ifdef DEBUG
543 uint32_t serialno;
544 #endif
545 };
546
547 static inline uint32_t
548 block_id(struct ir3_block *block)
549 {
550 #ifdef DEBUG
551 return block->serialno;
552 #else
553 return (uint32_t)(unsigned long)block;
554 #endif
555 }
556
557 struct ir3 * ir3_create(struct ir3_compiler *compiler, gl_shader_stage type);
558 void ir3_destroy(struct ir3 *shader);
559 void * ir3_assemble(struct ir3 *shader,
560 struct ir3_info *info, uint32_t gpu_id);
561 void * ir3_alloc(struct ir3 *shader, int sz);
562
563 struct ir3_block * ir3_block_create(struct ir3 *shader);
564
565 struct ir3_instruction * ir3_instr_create(struct ir3_block *block, opc_t opc);
566 struct ir3_instruction * ir3_instr_create2(struct ir3_block *block,
567 opc_t opc, int nreg);
568 struct ir3_instruction * ir3_instr_clone(struct ir3_instruction *instr);
569 void ir3_instr_add_dep(struct ir3_instruction *instr, struct ir3_instruction *dep);
570 const char *ir3_instr_name(struct ir3_instruction *instr);
571
572 struct ir3_register * ir3_reg_create(struct ir3_instruction *instr,
573 int num, int flags);
574 struct ir3_register * ir3_reg_clone(struct ir3 *shader,
575 struct ir3_register *reg);
576
577 void ir3_instr_set_address(struct ir3_instruction *instr,
578 struct ir3_instruction *addr);
579
580 static inline bool ir3_instr_check_mark(struct ir3_instruction *instr)
581 {
582 if (instr->flags & IR3_INSTR_MARK)
583 return true; /* already visited */
584 instr->flags |= IR3_INSTR_MARK;
585 return false;
586 }
587
588 void ir3_block_clear_mark(struct ir3_block *block);
589 void ir3_clear_mark(struct ir3 *shader);
590
591 unsigned ir3_count_instructions(struct ir3 *ir);
592 unsigned ir3_count_instructions_ra(struct ir3 *ir);
593
594 /**
595 * Move 'instr' to just before 'after'
596 */
597 static inline void
598 ir3_instr_move_before(struct ir3_instruction *instr,
599 struct ir3_instruction *after)
600 {
601 list_delinit(&instr->node);
602 list_addtail(&instr->node, &after->node);
603 }
604
605 /**
606 * Move 'instr' to just after 'before':
607 */
608 static inline void
609 ir3_instr_move_after(struct ir3_instruction *instr,
610 struct ir3_instruction *before)
611 {
612 list_delinit(&instr->node);
613 list_add(&instr->node, &before->node);
614 }
615
616 void ir3_find_ssa_uses(struct ir3 *ir, void *mem_ctx, bool falsedeps);
617
618 void ir3_set_dst_type(struct ir3_instruction *instr, bool half);
619 void ir3_fixup_src_type(struct ir3_instruction *instr);
620
621 bool ir3_valid_flags(struct ir3_instruction *instr, unsigned n, unsigned flags);
622
623 #include "util/set.h"
624 #define foreach_ssa_use(__use, __instr) \
625 for (struct ir3_instruction *__use = (void *)~0; \
626 __use && (__instr)->uses; __use = NULL) \
627 set_foreach ((__instr)->uses, __entry) \
628 if ((__use = (void *)__entry->key))
629
630 #define MAX_ARRAYS 16
631
632 /* comp:
633 * 0 - x
634 * 1 - y
635 * 2 - z
636 * 3 - w
637 */
638 static inline uint32_t regid(int num, int comp)
639 {
640 return (num << 2) | (comp & 0x3);
641 }
642
643 static inline uint32_t reg_num(struct ir3_register *reg)
644 {
645 return reg->num >> 2;
646 }
647
648 static inline uint32_t reg_comp(struct ir3_register *reg)
649 {
650 return reg->num & 0x3;
651 }
652
653 #define INVALID_REG regid(63, 0)
654 #define VALIDREG(r) ((r) != INVALID_REG)
655 #define CONDREG(r, val) COND(VALIDREG(r), (val))
656
657 static inline bool is_flow(struct ir3_instruction *instr)
658 {
659 return (opc_cat(instr->opc) == 0);
660 }
661
662 static inline bool is_kill(struct ir3_instruction *instr)
663 {
664 return instr->opc == OPC_KILL;
665 }
666
667 static inline bool is_nop(struct ir3_instruction *instr)
668 {
669 return instr->opc == OPC_NOP;
670 }
671
672 static inline bool is_same_type_reg(struct ir3_register *reg1,
673 struct ir3_register *reg2)
674 {
675 unsigned type_reg1 = (reg1->flags & (IR3_REG_HIGH | IR3_REG_HALF));
676 unsigned type_reg2 = (reg2->flags & (IR3_REG_HIGH | IR3_REG_HALF));
677
678 if (type_reg1 ^ type_reg2)
679 return false;
680 else
681 return true;
682 }
683
684 /* Is it a non-transformative (ie. not type changing) mov? This can
685 * also include absneg.s/absneg.f, which for the most part can be
686 * treated as a mov (single src argument).
687 */
688 static inline bool is_same_type_mov(struct ir3_instruction *instr)
689 {
690 struct ir3_register *dst;
691
692 switch (instr->opc) {
693 case OPC_MOV:
694 if (instr->cat1.src_type != instr->cat1.dst_type)
695 return false;
696 /* If the type of dest reg and src reg are different,
697 * it shouldn't be considered as same type mov
698 */
699 if (!is_same_type_reg(instr->regs[0], instr->regs[1]))
700 return false;
701 break;
702 case OPC_ABSNEG_F:
703 case OPC_ABSNEG_S:
704 if (instr->flags & IR3_INSTR_SAT)
705 return false;
706 /* If the type of dest reg and src reg are different,
707 * it shouldn't be considered as same type mov
708 */
709 if (!is_same_type_reg(instr->regs[0], instr->regs[1]))
710 return false;
711 break;
712 default:
713 return false;
714 }
715
716 dst = instr->regs[0];
717
718 /* mov's that write to a0 or p0.x are special: */
719 if (dst->num == regid(REG_P0, 0))
720 return false;
721 if (reg_num(dst) == REG_A0)
722 return false;
723
724 if (dst->flags & (IR3_REG_RELATIV | IR3_REG_ARRAY))
725 return false;
726
727 return true;
728 }
729
730 /* A move from const, which changes size but not type, can also be
731 * folded into dest instruction in some cases.
732 */
733 static inline bool is_const_mov(struct ir3_instruction *instr)
734 {
735 if (instr->opc != OPC_MOV)
736 return false;
737
738 if (!(instr->regs[1]->flags & IR3_REG_CONST))
739 return false;
740
741 type_t src_type = instr->cat1.src_type;
742 type_t dst_type = instr->cat1.dst_type;
743
744 return (type_float(src_type) && type_float(dst_type)) ||
745 (type_uint(src_type) && type_uint(dst_type)) ||
746 (type_sint(src_type) && type_sint(dst_type));
747 }
748
749 static inline bool is_alu(struct ir3_instruction *instr)
750 {
751 return (1 <= opc_cat(instr->opc)) && (opc_cat(instr->opc) <= 3);
752 }
753
754 static inline bool is_sfu(struct ir3_instruction *instr)
755 {
756 return (opc_cat(instr->opc) == 4);
757 }
758
759 static inline bool is_tex(struct ir3_instruction *instr)
760 {
761 return (opc_cat(instr->opc) == 5);
762 }
763
764 static inline bool is_tex_or_prefetch(struct ir3_instruction *instr)
765 {
766 return is_tex(instr) || (instr->opc == OPC_META_TEX_PREFETCH);
767 }
768
769 static inline bool is_mem(struct ir3_instruction *instr)
770 {
771 return (opc_cat(instr->opc) == 6);
772 }
773
774 static inline bool is_barrier(struct ir3_instruction *instr)
775 {
776 return (opc_cat(instr->opc) == 7);
777 }
778
779 static inline bool
780 is_half(struct ir3_instruction *instr)
781 {
782 return !!(instr->regs[0]->flags & IR3_REG_HALF);
783 }
784
785 static inline bool
786 is_high(struct ir3_instruction *instr)
787 {
788 return !!(instr->regs[0]->flags & IR3_REG_HIGH);
789 }
790
791 static inline bool
792 is_store(struct ir3_instruction *instr)
793 {
794 /* these instructions, the "destination" register is
795 * actually a source, the address to store to.
796 */
797 switch (instr->opc) {
798 case OPC_STG:
799 case OPC_STGB:
800 case OPC_STIB:
801 case OPC_STP:
802 case OPC_STL:
803 case OPC_STLW:
804 case OPC_L2G:
805 case OPC_G2L:
806 return true;
807 default:
808 return false;
809 }
810 }
811
812 static inline bool is_load(struct ir3_instruction *instr)
813 {
814 switch (instr->opc) {
815 case OPC_LDG:
816 case OPC_LDGB:
817 case OPC_LDIB:
818 case OPC_LDL:
819 case OPC_LDP:
820 case OPC_L2G:
821 case OPC_LDLW:
822 case OPC_LDC:
823 case OPC_LDLV:
824 /* probably some others too.. */
825 return true;
826 default:
827 return false;
828 }
829 }
830
831 static inline bool is_input(struct ir3_instruction *instr)
832 {
833 /* in some cases, ldlv is used to fetch varying without
834 * interpolation.. fortunately inloc is the first src
835 * register in either case
836 */
837 switch (instr->opc) {
838 case OPC_LDLV:
839 case OPC_BARY_F:
840 return true;
841 default:
842 return false;
843 }
844 }
845
846 static inline bool is_bool(struct ir3_instruction *instr)
847 {
848 switch (instr->opc) {
849 case OPC_CMPS_F:
850 case OPC_CMPS_S:
851 case OPC_CMPS_U:
852 return true;
853 default:
854 return false;
855 }
856 }
857
858 static inline opc_t
859 cat3_half_opc(opc_t opc)
860 {
861 switch (opc) {
862 case OPC_MAD_F32: return OPC_MAD_F16;
863 case OPC_SEL_B32: return OPC_SEL_B16;
864 case OPC_SEL_S32: return OPC_SEL_S16;
865 case OPC_SEL_F32: return OPC_SEL_F16;
866 case OPC_SAD_S32: return OPC_SAD_S16;
867 default: return opc;
868 }
869 }
870
871 static inline opc_t
872 cat3_full_opc(opc_t opc)
873 {
874 switch (opc) {
875 case OPC_MAD_F16: return OPC_MAD_F32;
876 case OPC_SEL_B16: return OPC_SEL_B32;
877 case OPC_SEL_S16: return OPC_SEL_S32;
878 case OPC_SEL_F16: return OPC_SEL_F32;
879 case OPC_SAD_S16: return OPC_SAD_S32;
880 default: return opc;
881 }
882 }
883
884 static inline opc_t
885 cat4_half_opc(opc_t opc)
886 {
887 switch (opc) {
888 case OPC_RSQ: return OPC_HRSQ;
889 case OPC_LOG2: return OPC_HLOG2;
890 case OPC_EXP2: return OPC_HEXP2;
891 default: return opc;
892 }
893 }
894
895 static inline opc_t
896 cat4_full_opc(opc_t opc)
897 {
898 switch (opc) {
899 case OPC_HRSQ: return OPC_RSQ;
900 case OPC_HLOG2: return OPC_LOG2;
901 case OPC_HEXP2: return OPC_EXP2;
902 default: return opc;
903 }
904 }
905
906 static inline bool is_meta(struct ir3_instruction *instr)
907 {
908 return (opc_cat(instr->opc) == -1);
909 }
910
911 static inline unsigned dest_regs(struct ir3_instruction *instr)
912 {
913 if ((instr->regs_count == 0) || is_store(instr) || is_flow(instr))
914 return 0;
915
916 return util_last_bit(instr->regs[0]->wrmask);
917 }
918
919 static inline bool
920 writes_gpr(struct ir3_instruction *instr)
921 {
922 if (dest_regs(instr) == 0)
923 return false;
924 /* is dest a normal temp register: */
925 struct ir3_register *reg = instr->regs[0];
926 debug_assert(!(reg->flags & (IR3_REG_CONST | IR3_REG_IMMED)));
927 if ((reg_num(reg) == REG_A0) ||
928 (reg->num == regid(REG_P0, 0)))
929 return false;
930 return true;
931 }
932
933 static inline bool writes_addr0(struct ir3_instruction *instr)
934 {
935 if (instr->regs_count > 0) {
936 struct ir3_register *dst = instr->regs[0];
937 return dst->num == regid(REG_A0, 0);
938 }
939 return false;
940 }
941
942 static inline bool writes_addr1(struct ir3_instruction *instr)
943 {
944 if (instr->regs_count > 0) {
945 struct ir3_register *dst = instr->regs[0];
946 return dst->num == regid(REG_A0, 1);
947 }
948 return false;
949 }
950
951 static inline bool writes_pred(struct ir3_instruction *instr)
952 {
953 if (instr->regs_count > 0) {
954 struct ir3_register *dst = instr->regs[0];
955 return reg_num(dst) == REG_P0;
956 }
957 return false;
958 }
959
960 /* returns defining instruction for reg */
961 /* TODO better name */
962 static inline struct ir3_instruction *ssa(struct ir3_register *reg)
963 {
964 if (reg->flags & (IR3_REG_SSA | IR3_REG_ARRAY)) {
965 return reg->instr;
966 }
967 return NULL;
968 }
969
970 static inline bool conflicts(struct ir3_instruction *a,
971 struct ir3_instruction *b)
972 {
973 return (a && b) && (a != b);
974 }
975
976 static inline bool reg_gpr(struct ir3_register *r)
977 {
978 if (r->flags & (IR3_REG_CONST | IR3_REG_IMMED))
979 return false;
980 if ((reg_num(r) == REG_A0) || (reg_num(r) == REG_P0))
981 return false;
982 return true;
983 }
984
985 static inline type_t half_type(type_t type)
986 {
987 switch (type) {
988 case TYPE_F32: return TYPE_F16;
989 case TYPE_U32: return TYPE_U16;
990 case TYPE_S32: return TYPE_S16;
991 case TYPE_F16:
992 case TYPE_U16:
993 case TYPE_S16:
994 return type;
995 default:
996 assert(0);
997 return ~0;
998 }
999 }
1000
1001 static inline type_t full_type(type_t type)
1002 {
1003 switch (type) {
1004 case TYPE_F16: return TYPE_F32;
1005 case TYPE_U16: return TYPE_U32;
1006 case TYPE_S16: return TYPE_S32;
1007 case TYPE_F32:
1008 case TYPE_U32:
1009 case TYPE_S32:
1010 return type;
1011 default:
1012 assert(0);
1013 return ~0;
1014 }
1015 }
1016
1017 /* some cat2 instructions (ie. those which are not float) can embed an
1018 * immediate:
1019 */
1020 static inline bool ir3_cat2_int(opc_t opc)
1021 {
1022 switch (opc) {
1023 case OPC_ADD_U:
1024 case OPC_ADD_S:
1025 case OPC_SUB_U:
1026 case OPC_SUB_S:
1027 case OPC_CMPS_U:
1028 case OPC_CMPS_S:
1029 case OPC_MIN_U:
1030 case OPC_MIN_S:
1031 case OPC_MAX_U:
1032 case OPC_MAX_S:
1033 case OPC_CMPV_U:
1034 case OPC_CMPV_S:
1035 case OPC_MUL_U24:
1036 case OPC_MUL_S24:
1037 case OPC_MULL_U:
1038 case OPC_CLZ_S:
1039 case OPC_ABSNEG_S:
1040 case OPC_AND_B:
1041 case OPC_OR_B:
1042 case OPC_NOT_B:
1043 case OPC_XOR_B:
1044 case OPC_BFREV_B:
1045 case OPC_CLZ_B:
1046 case OPC_SHL_B:
1047 case OPC_SHR_B:
1048 case OPC_ASHR_B:
1049 case OPC_MGEN_B:
1050 case OPC_GETBIT_B:
1051 case OPC_CBITS_B:
1052 case OPC_BARY_F:
1053 return true;
1054
1055 default:
1056 return false;
1057 }
1058 }
1059
1060 /* map cat2 instruction to valid abs/neg flags: */
1061 static inline unsigned ir3_cat2_absneg(opc_t opc)
1062 {
1063 switch (opc) {
1064 case OPC_ADD_F:
1065 case OPC_MIN_F:
1066 case OPC_MAX_F:
1067 case OPC_MUL_F:
1068 case OPC_SIGN_F:
1069 case OPC_CMPS_F:
1070 case OPC_ABSNEG_F:
1071 case OPC_CMPV_F:
1072 case OPC_FLOOR_F:
1073 case OPC_CEIL_F:
1074 case OPC_RNDNE_F:
1075 case OPC_RNDAZ_F:
1076 case OPC_TRUNC_F:
1077 case OPC_BARY_F:
1078 return IR3_REG_FABS | IR3_REG_FNEG;
1079
1080 case OPC_ADD_U:
1081 case OPC_ADD_S:
1082 case OPC_SUB_U:
1083 case OPC_SUB_S:
1084 case OPC_CMPS_U:
1085 case OPC_CMPS_S:
1086 case OPC_MIN_U:
1087 case OPC_MIN_S:
1088 case OPC_MAX_U:
1089 case OPC_MAX_S:
1090 case OPC_CMPV_U:
1091 case OPC_CMPV_S:
1092 case OPC_MUL_U24:
1093 case OPC_MUL_S24:
1094 case OPC_MULL_U:
1095 case OPC_CLZ_S:
1096 return 0;
1097
1098 case OPC_ABSNEG_S:
1099 return IR3_REG_SABS | IR3_REG_SNEG;
1100
1101 case OPC_AND_B:
1102 case OPC_OR_B:
1103 case OPC_NOT_B:
1104 case OPC_XOR_B:
1105 case OPC_BFREV_B:
1106 case OPC_CLZ_B:
1107 case OPC_SHL_B:
1108 case OPC_SHR_B:
1109 case OPC_ASHR_B:
1110 case OPC_MGEN_B:
1111 case OPC_GETBIT_B:
1112 case OPC_CBITS_B:
1113 return IR3_REG_BNOT;
1114
1115 default:
1116 return 0;
1117 }
1118 }
1119
1120 /* map cat3 instructions to valid abs/neg flags: */
1121 static inline unsigned ir3_cat3_absneg(opc_t opc)
1122 {
1123 switch (opc) {
1124 case OPC_MAD_F16:
1125 case OPC_MAD_F32:
1126 case OPC_SEL_F16:
1127 case OPC_SEL_F32:
1128 return IR3_REG_FNEG;
1129
1130 case OPC_MAD_U16:
1131 case OPC_MADSH_U16:
1132 case OPC_MAD_S16:
1133 case OPC_MADSH_M16:
1134 case OPC_MAD_U24:
1135 case OPC_MAD_S24:
1136 case OPC_SEL_S16:
1137 case OPC_SEL_S32:
1138 case OPC_SAD_S16:
1139 case OPC_SAD_S32:
1140 /* neg *may* work on 3rd src.. */
1141
1142 case OPC_SEL_B16:
1143 case OPC_SEL_B32:
1144
1145 default:
1146 return 0;
1147 }
1148 }
1149
1150 #define MASK(n) ((1 << (n)) - 1)
1151
1152 /* iterator for an instructions's sources (reg), also returns src #: */
1153 #define foreach_src_n(__srcreg, __n, __instr) \
1154 if ((__instr)->regs_count) \
1155 for (struct ir3_register *__srcreg = (void *)~0; __srcreg; __srcreg = NULL) \
1156 for (unsigned __cnt = (__instr)->regs_count - 1, __n = 0; __n < __cnt; __n++) \
1157 if ((__srcreg = (__instr)->regs[__n + 1]))
1158
1159 /* iterator for an instructions's sources (reg): */
1160 #define foreach_src(__srcreg, __instr) \
1161 foreach_src_n(__srcreg, __i, __instr)
1162
1163 static inline unsigned __ssa_src_cnt(struct ir3_instruction *instr)
1164 {
1165 unsigned cnt = instr->regs_count + instr->deps_count;
1166 if (instr->address)
1167 cnt++;
1168 return cnt;
1169 }
1170
1171 static inline struct ir3_instruction **
1172 __ssa_srcp_n(struct ir3_instruction *instr, unsigned n)
1173 {
1174 if (n == (instr->regs_count + instr->deps_count))
1175 return &instr->address;
1176 if (n >= instr->regs_count)
1177 return &instr->deps[n - instr->regs_count];
1178 if (ssa(instr->regs[n]))
1179 return &instr->regs[n]->instr;
1180 return NULL;
1181 }
1182
1183 static inline bool __is_false_dep(struct ir3_instruction *instr, unsigned n)
1184 {
1185 if (n == (instr->regs_count + instr->deps_count))
1186 return false;
1187 if (n >= instr->regs_count)
1188 return true;
1189 return false;
1190 }
1191
1192 #define foreach_ssa_srcp_n(__srcp, __n, __instr) \
1193 for (struct ir3_instruction **__srcp = (void *)~0; __srcp; __srcp = NULL) \
1194 for (unsigned __cnt = __ssa_src_cnt(__instr), __n = 0; __n < __cnt; __n++) \
1195 if ((__srcp = __ssa_srcp_n(__instr, __n)))
1196
1197 #define foreach_ssa_srcp(__srcp, __instr) \
1198 foreach_ssa_srcp_n(__srcp, __i, __instr)
1199
1200 /* iterator for an instruction's SSA sources (instr), also returns src #: */
1201 #define foreach_ssa_src_n(__srcinst, __n, __instr) \
1202 for (struct ir3_instruction *__srcinst = (void *)~0; __srcinst; __srcinst = NULL) \
1203 foreach_ssa_srcp_n(__srcp, __n, __instr) \
1204 if ((__srcinst = *__srcp))
1205
1206 /* iterator for an instruction's SSA sources (instr): */
1207 #define foreach_ssa_src(__srcinst, __instr) \
1208 foreach_ssa_src_n(__srcinst, __i, __instr)
1209
1210 /* iterators for shader inputs: */
1211 #define foreach_input_n(__ininstr, __cnt, __ir) \
1212 for (struct ir3_instruction *__ininstr = (void *)~0; __ininstr; __ininstr = NULL) \
1213 for (unsigned __cnt = 0; __cnt < (__ir)->inputs_count; __cnt++) \
1214 if ((__ininstr = (__ir)->inputs[__cnt]))
1215 #define foreach_input(__ininstr, __ir) \
1216 foreach_input_n(__ininstr, __i, __ir)
1217
1218 /* iterators for shader outputs: */
1219 #define foreach_output_n(__outinstr, __cnt, __ir) \
1220 for (struct ir3_instruction *__outinstr = (void *)~0; __outinstr; __outinstr = NULL) \
1221 for (unsigned __cnt = 0; __cnt < (__ir)->outputs_count; __cnt++) \
1222 if ((__outinstr = (__ir)->outputs[__cnt]))
1223 #define foreach_output(__outinstr, __ir) \
1224 foreach_output_n(__outinstr, __i, __ir)
1225
1226 /* iterators for instructions: */
1227 #define foreach_instr(__instr, __list) \
1228 list_for_each_entry(struct ir3_instruction, __instr, __list, node)
1229 #define foreach_instr_rev(__instr, __list) \
1230 list_for_each_entry_rev(struct ir3_instruction, __instr, __list, node)
1231 #define foreach_instr_safe(__instr, __list) \
1232 list_for_each_entry_safe(struct ir3_instruction, __instr, __list, node)
1233
1234 /* iterators for blocks: */
1235 #define foreach_block(__block, __list) \
1236 list_for_each_entry(struct ir3_block, __block, __list, node)
1237 #define foreach_block_safe(__block, __list) \
1238 list_for_each_entry_safe(struct ir3_block, __block, __list, node)
1239 #define foreach_block_rev(__block, __list) \
1240 list_for_each_entry_rev(struct ir3_block, __block, __list, node)
1241
1242 /* iterators for arrays: */
1243 #define foreach_array(__array, __list) \
1244 list_for_each_entry(struct ir3_array, __array, __list, node)
1245
1246 /* Check if condition is true for any src instruction.
1247 */
1248 static inline bool
1249 check_src_cond(struct ir3_instruction *instr, bool (*cond)(struct ir3_instruction *))
1250 {
1251 /* Note that this is also used post-RA so skip the ssa iterator: */
1252 foreach_src (reg, instr) {
1253 struct ir3_instruction *src = reg->instr;
1254
1255 if (!src)
1256 continue;
1257
1258 /* meta:split/collect aren't real instructions, the thing that
1259 * we actually care about is *their* srcs
1260 */
1261 if ((src->opc == OPC_META_SPLIT) || (src->opc == OPC_META_COLLECT)) {
1262 if (check_src_cond(src, cond))
1263 return true;
1264 } else {
1265 if (cond(src))
1266 return true;
1267 }
1268 }
1269
1270 return false;
1271 }
1272
1273 #define IR3_PASS(ir, pass, ...) ({ \
1274 bool progress = pass(ir, ##__VA_ARGS__); \
1275 if (progress) { \
1276 ir3_debug_print(ir, "AFTER: " #pass); \
1277 ir3_validate(ir); \
1278 } \
1279 progress; \
1280 })
1281
1282 /* validate: */
1283 void ir3_validate(struct ir3 *ir);
1284
1285 /* dump: */
1286 void ir3_print(struct ir3 *ir);
1287 void ir3_print_instr(struct ir3_instruction *instr);
1288
1289 /* delay calculation: */
1290 int ir3_delayslots(struct ir3_instruction *assigner,
1291 struct ir3_instruction *consumer, unsigned n, bool soft);
1292 unsigned ir3_delay_calc(struct ir3_block *block, struct ir3_instruction *instr,
1293 bool soft, bool pred);
1294 void ir3_remove_nops(struct ir3 *ir);
1295
1296 /* dead code elimination: */
1297 struct ir3_shader_variant;
1298 bool ir3_dce(struct ir3 *ir, struct ir3_shader_variant *so);
1299
1300 /* fp16 conversion folding */
1301 bool ir3_cf(struct ir3 *ir);
1302
1303 /* copy-propagate: */
1304 bool ir3_cp(struct ir3 *ir, struct ir3_shader_variant *so);
1305 bool ir3_cp_postsched(struct ir3 *ir);
1306
1307 /* group neighbors and insert mov's to resolve conflicts: */
1308 bool ir3_group(struct ir3 *ir);
1309
1310 /* scheduling: */
1311 bool ir3_sched_add_deps(struct ir3 *ir);
1312 int ir3_sched(struct ir3 *ir);
1313
1314 struct ir3_context;
1315 bool ir3_postsched(struct ir3 *ir);
1316
1317 bool ir3_a6xx_fixup_atomic_dests(struct ir3 *ir, struct ir3_shader_variant *so);
1318
1319 /* register assignment: */
1320 struct ir3_ra_reg_set * ir3_ra_alloc_reg_set(struct ir3_compiler *compiler, bool mergedregs);
1321 int ir3_ra(struct ir3_shader_variant *v, struct ir3_instruction **precolor, unsigned nprecolor);
1322
1323 /* legalize: */
1324 bool ir3_legalize(struct ir3 *ir, struct ir3_shader_variant *so, int *max_bary);
1325
1326 static inline bool
1327 ir3_has_latency_to_hide(struct ir3 *ir)
1328 {
1329 /* VS/GS/TCS/TESS co-exist with frag shader invocations, but we don't
1330 * know the nature of the fragment shader. Just assume it will have
1331 * latency to hide:
1332 */
1333 if (ir->type != MESA_SHADER_FRAGMENT)
1334 return true;
1335
1336 foreach_block (block, &ir->block_list) {
1337 foreach_instr (instr, &block->instr_list) {
1338 if (is_tex_or_prefetch(instr))
1339 return true;
1340
1341 if (is_load(instr)) {
1342 switch (instr->opc) {
1343 case OPC_LDLV:
1344 case OPC_LDL:
1345 case OPC_LDLW:
1346 break;
1347 default:
1348 return true;
1349 }
1350 }
1351 }
1352 }
1353
1354 return false;
1355 }
1356
1357 /* ************************************************************************* */
1358 /* instruction helpers */
1359
1360 /* creates SSA src of correct type (ie. half vs full precision) */
1361 static inline struct ir3_register * __ssa_src(struct ir3_instruction *instr,
1362 struct ir3_instruction *src, unsigned flags)
1363 {
1364 struct ir3_register *reg;
1365 if (src->regs[0]->flags & IR3_REG_HALF)
1366 flags |= IR3_REG_HALF;
1367 reg = ir3_reg_create(instr, 0, IR3_REG_SSA | flags);
1368 reg->instr = src;
1369 reg->wrmask = src->regs[0]->wrmask;
1370 return reg;
1371 }
1372
1373 static inline struct ir3_register * __ssa_dst(struct ir3_instruction *instr)
1374 {
1375 struct ir3_register *reg = ir3_reg_create(instr, 0, 0);
1376 reg->flags |= IR3_REG_SSA;
1377 return reg;
1378 }
1379
1380 static inline struct ir3_instruction *
1381 create_immed_typed(struct ir3_block *block, uint32_t val, type_t type)
1382 {
1383 struct ir3_instruction *mov;
1384 unsigned flags = (type_size(type) < 32) ? IR3_REG_HALF : 0;
1385
1386 mov = ir3_instr_create(block, OPC_MOV);
1387 mov->cat1.src_type = type;
1388 mov->cat1.dst_type = type;
1389 __ssa_dst(mov)->flags |= flags;
1390 ir3_reg_create(mov, 0, IR3_REG_IMMED | flags)->uim_val = val;
1391
1392 return mov;
1393 }
1394
1395 static inline struct ir3_instruction *
1396 create_immed(struct ir3_block *block, uint32_t val)
1397 {
1398 return create_immed_typed(block, val, TYPE_U32);
1399 }
1400
1401 static inline struct ir3_instruction *
1402 create_uniform_typed(struct ir3_block *block, unsigned n, type_t type)
1403 {
1404 struct ir3_instruction *mov;
1405 unsigned flags = (type_size(type) < 32) ? IR3_REG_HALF : 0;
1406
1407 mov = ir3_instr_create(block, OPC_MOV);
1408 mov->cat1.src_type = type;
1409 mov->cat1.dst_type = type;
1410 __ssa_dst(mov)->flags |= flags;
1411 ir3_reg_create(mov, n, IR3_REG_CONST | flags);
1412
1413 return mov;
1414 }
1415
1416 static inline struct ir3_instruction *
1417 create_uniform(struct ir3_block *block, unsigned n)
1418 {
1419 return create_uniform_typed(block, n, TYPE_F32);
1420 }
1421
1422 static inline struct ir3_instruction *
1423 create_uniform_indirect(struct ir3_block *block, int n,
1424 struct ir3_instruction *address)
1425 {
1426 struct ir3_instruction *mov;
1427
1428 mov = ir3_instr_create(block, OPC_MOV);
1429 mov->cat1.src_type = TYPE_U32;
1430 mov->cat1.dst_type = TYPE_U32;
1431 __ssa_dst(mov);
1432 ir3_reg_create(mov, 0, IR3_REG_CONST | IR3_REG_RELATIV)->array.offset = n;
1433
1434 ir3_instr_set_address(mov, address);
1435
1436 return mov;
1437 }
1438
1439 static inline struct ir3_instruction *
1440 ir3_MOV(struct ir3_block *block, struct ir3_instruction *src, type_t type)
1441 {
1442 struct ir3_instruction *instr = ir3_instr_create(block, OPC_MOV);
1443 unsigned flags = (type_size(type) < 32) ? IR3_REG_HALF : 0;
1444
1445 __ssa_dst(instr)->flags |= flags;
1446 if (src->regs[0]->flags & IR3_REG_ARRAY) {
1447 struct ir3_register *src_reg = __ssa_src(instr, src, IR3_REG_ARRAY);
1448 src_reg->array = src->regs[0]->array;
1449 } else {
1450 __ssa_src(instr, src, src->regs[0]->flags & IR3_REG_HIGH);
1451 }
1452 debug_assert(!(src->regs[0]->flags & IR3_REG_RELATIV));
1453 instr->cat1.src_type = type;
1454 instr->cat1.dst_type = type;
1455 return instr;
1456 }
1457
1458 static inline struct ir3_instruction *
1459 ir3_COV(struct ir3_block *block, struct ir3_instruction *src,
1460 type_t src_type, type_t dst_type)
1461 {
1462 struct ir3_instruction *instr = ir3_instr_create(block, OPC_MOV);
1463 unsigned dst_flags = (type_size(dst_type) < 32) ? IR3_REG_HALF : 0;
1464 unsigned src_flags = (type_size(src_type) < 32) ? IR3_REG_HALF : 0;
1465
1466 debug_assert((src->regs[0]->flags & IR3_REG_HALF) == src_flags);
1467
1468 __ssa_dst(instr)->flags |= dst_flags;
1469 __ssa_src(instr, src, 0);
1470 instr->cat1.src_type = src_type;
1471 instr->cat1.dst_type = dst_type;
1472 debug_assert(!(src->regs[0]->flags & IR3_REG_ARRAY));
1473 return instr;
1474 }
1475
1476 static inline struct ir3_instruction *
1477 ir3_NOP(struct ir3_block *block)
1478 {
1479 return ir3_instr_create(block, OPC_NOP);
1480 }
1481
1482 #define IR3_INSTR_0 0
1483
1484 #define __INSTR0(flag, name, opc) \
1485 static inline struct ir3_instruction * \
1486 ir3_##name(struct ir3_block *block) \
1487 { \
1488 struct ir3_instruction *instr = \
1489 ir3_instr_create(block, opc); \
1490 instr->flags |= flag; \
1491 return instr; \
1492 }
1493 #define INSTR0F(f, name) __INSTR0(IR3_INSTR_##f, name##_##f, OPC_##name)
1494 #define INSTR0(name) __INSTR0(0, name, OPC_##name)
1495
1496 #define __INSTR1(flag, name, opc) \
1497 static inline struct ir3_instruction * \
1498 ir3_##name(struct ir3_block *block, \
1499 struct ir3_instruction *a, unsigned aflags) \
1500 { \
1501 struct ir3_instruction *instr = \
1502 ir3_instr_create(block, opc); \
1503 __ssa_dst(instr); \
1504 __ssa_src(instr, a, aflags); \
1505 instr->flags |= flag; \
1506 return instr; \
1507 }
1508 #define INSTR1F(f, name) __INSTR1(IR3_INSTR_##f, name##_##f, OPC_##name)
1509 #define INSTR1(name) __INSTR1(0, name, OPC_##name)
1510
1511 #define __INSTR2(flag, name, opc) \
1512 static inline struct ir3_instruction * \
1513 ir3_##name(struct ir3_block *block, \
1514 struct ir3_instruction *a, unsigned aflags, \
1515 struct ir3_instruction *b, unsigned bflags) \
1516 { \
1517 struct ir3_instruction *instr = \
1518 ir3_instr_create(block, opc); \
1519 __ssa_dst(instr); \
1520 __ssa_src(instr, a, aflags); \
1521 __ssa_src(instr, b, bflags); \
1522 instr->flags |= flag; \
1523 return instr; \
1524 }
1525 #define INSTR2F(f, name) __INSTR2(IR3_INSTR_##f, name##_##f, OPC_##name)
1526 #define INSTR2(name) __INSTR2(0, name, OPC_##name)
1527
1528 #define __INSTR3(flag, name, opc) \
1529 static inline struct ir3_instruction * \
1530 ir3_##name(struct ir3_block *block, \
1531 struct ir3_instruction *a, unsigned aflags, \
1532 struct ir3_instruction *b, unsigned bflags, \
1533 struct ir3_instruction *c, unsigned cflags) \
1534 { \
1535 struct ir3_instruction *instr = \
1536 ir3_instr_create2(block, opc, 4); \
1537 __ssa_dst(instr); \
1538 __ssa_src(instr, a, aflags); \
1539 __ssa_src(instr, b, bflags); \
1540 __ssa_src(instr, c, cflags); \
1541 instr->flags |= flag; \
1542 return instr; \
1543 }
1544 #define INSTR3F(f, name) __INSTR3(IR3_INSTR_##f, name##_##f, OPC_##name)
1545 #define INSTR3(name) __INSTR3(0, name, OPC_##name)
1546
1547 #define __INSTR4(flag, name, opc) \
1548 static inline struct ir3_instruction * \
1549 ir3_##name(struct ir3_block *block, \
1550 struct ir3_instruction *a, unsigned aflags, \
1551 struct ir3_instruction *b, unsigned bflags, \
1552 struct ir3_instruction *c, unsigned cflags, \
1553 struct ir3_instruction *d, unsigned dflags) \
1554 { \
1555 struct ir3_instruction *instr = \
1556 ir3_instr_create2(block, opc, 5); \
1557 __ssa_dst(instr); \
1558 __ssa_src(instr, a, aflags); \
1559 __ssa_src(instr, b, bflags); \
1560 __ssa_src(instr, c, cflags); \
1561 __ssa_src(instr, d, dflags); \
1562 instr->flags |= flag; \
1563 return instr; \
1564 }
1565 #define INSTR4F(f, name) __INSTR4(IR3_INSTR_##f, name##_##f, OPC_##name)
1566 #define INSTR4(name) __INSTR4(0, name, OPC_##name)
1567
1568 /* cat0 instructions: */
1569 INSTR1(B)
1570 INSTR0(JUMP)
1571 INSTR1(KILL)
1572 INSTR0(END)
1573 INSTR0(CHSH)
1574 INSTR0(CHMASK)
1575 INSTR1(PREDT)
1576 INSTR0(PREDF)
1577 INSTR0(PREDE)
1578
1579 /* cat2 instructions, most 2 src but some 1 src: */
1580 INSTR2(ADD_F)
1581 INSTR2(MIN_F)
1582 INSTR2(MAX_F)
1583 INSTR2(MUL_F)
1584 INSTR1(SIGN_F)
1585 INSTR2(CMPS_F)
1586 INSTR1(ABSNEG_F)
1587 INSTR2(CMPV_F)
1588 INSTR1(FLOOR_F)
1589 INSTR1(CEIL_F)
1590 INSTR1(RNDNE_F)
1591 INSTR1(RNDAZ_F)
1592 INSTR1(TRUNC_F)
1593 INSTR2(ADD_U)
1594 INSTR2(ADD_S)
1595 INSTR2(SUB_U)
1596 INSTR2(SUB_S)
1597 INSTR2(CMPS_U)
1598 INSTR2(CMPS_S)
1599 INSTR2(MIN_U)
1600 INSTR2(MIN_S)
1601 INSTR2(MAX_U)
1602 INSTR2(MAX_S)
1603 INSTR1(ABSNEG_S)
1604 INSTR2(AND_B)
1605 INSTR2(OR_B)
1606 INSTR1(NOT_B)
1607 INSTR2(XOR_B)
1608 INSTR2(CMPV_U)
1609 INSTR2(CMPV_S)
1610 INSTR2(MUL_U24)
1611 INSTR2(MUL_S24)
1612 INSTR2(MULL_U)
1613 INSTR1(BFREV_B)
1614 INSTR1(CLZ_S)
1615 INSTR1(CLZ_B)
1616 INSTR2(SHL_B)
1617 INSTR2(SHR_B)
1618 INSTR2(ASHR_B)
1619 INSTR2(BARY_F)
1620 INSTR2(MGEN_B)
1621 INSTR2(GETBIT_B)
1622 INSTR1(SETRM)
1623 INSTR1(CBITS_B)
1624 INSTR2(SHB)
1625 INSTR2(MSAD)
1626
1627 /* cat3 instructions: */
1628 INSTR3(MAD_U16)
1629 INSTR3(MADSH_U16)
1630 INSTR3(MAD_S16)
1631 INSTR3(MADSH_M16)
1632 INSTR3(MAD_U24)
1633 INSTR3(MAD_S24)
1634 INSTR3(MAD_F16)
1635 INSTR3(MAD_F32)
1636 /* NOTE: SEL_B32 checks for zero vs nonzero */
1637 INSTR3(SEL_B16)
1638 INSTR3(SEL_B32)
1639 INSTR3(SEL_S16)
1640 INSTR3(SEL_S32)
1641 INSTR3(SEL_F16)
1642 INSTR3(SEL_F32)
1643 INSTR3(SAD_S16)
1644 INSTR3(SAD_S32)
1645
1646 /* cat4 instructions: */
1647 INSTR1(RCP)
1648 INSTR1(RSQ)
1649 INSTR1(HRSQ)
1650 INSTR1(LOG2)
1651 INSTR1(HLOG2)
1652 INSTR1(EXP2)
1653 INSTR1(HEXP2)
1654 INSTR1(SIN)
1655 INSTR1(COS)
1656 INSTR1(SQRT)
1657
1658 /* cat5 instructions: */
1659 INSTR1(DSX)
1660 INSTR1(DSXPP_1)
1661 INSTR1(DSY)
1662 INSTR1(DSYPP_1)
1663 INSTR1F(3D, DSX)
1664 INSTR1F(3D, DSY)
1665 INSTR1(RGETPOS)
1666
1667 static inline struct ir3_instruction *
1668 ir3_SAM(struct ir3_block *block, opc_t opc, type_t type,
1669 unsigned wrmask, unsigned flags, struct ir3_instruction *samp_tex,
1670 struct ir3_instruction *src0, struct ir3_instruction *src1)
1671 {
1672 struct ir3_instruction *sam;
1673
1674 sam = ir3_instr_create(block, opc);
1675 sam->flags |= flags;
1676 __ssa_dst(sam)->wrmask = wrmask;
1677 if (flags & IR3_INSTR_S2EN) {
1678 __ssa_src(sam, samp_tex, IR3_REG_HALF);
1679 }
1680 if (src0) {
1681 __ssa_src(sam, src0, 0);
1682 }
1683 if (src1) {
1684 __ssa_src(sam, src1, 0);
1685 }
1686 sam->cat5.type = type;
1687
1688 return sam;
1689 }
1690
1691 /* cat6 instructions: */
1692 INSTR2(LDLV)
1693 INSTR3(LDG)
1694 INSTR3(LDL)
1695 INSTR3(LDLW)
1696 INSTR3(STG)
1697 INSTR3(STL)
1698 INSTR3(STLW)
1699 INSTR1(RESINFO)
1700 INSTR1(RESFMT)
1701 INSTR2(ATOMIC_ADD)
1702 INSTR2(ATOMIC_SUB)
1703 INSTR2(ATOMIC_XCHG)
1704 INSTR2(ATOMIC_INC)
1705 INSTR2(ATOMIC_DEC)
1706 INSTR2(ATOMIC_CMPXCHG)
1707 INSTR2(ATOMIC_MIN)
1708 INSTR2(ATOMIC_MAX)
1709 INSTR2(ATOMIC_AND)
1710 INSTR2(ATOMIC_OR)
1711 INSTR2(ATOMIC_XOR)
1712 INSTR2(LDC)
1713 #if GPU >= 600
1714 INSTR3(STIB);
1715 INSTR2(LDIB);
1716 INSTR3F(G, ATOMIC_ADD)
1717 INSTR3F(G, ATOMIC_SUB)
1718 INSTR3F(G, ATOMIC_XCHG)
1719 INSTR3F(G, ATOMIC_INC)
1720 INSTR3F(G, ATOMIC_DEC)
1721 INSTR3F(G, ATOMIC_CMPXCHG)
1722 INSTR3F(G, ATOMIC_MIN)
1723 INSTR3F(G, ATOMIC_MAX)
1724 INSTR3F(G, ATOMIC_AND)
1725 INSTR3F(G, ATOMIC_OR)
1726 INSTR3F(G, ATOMIC_XOR)
1727 #elif GPU >= 400
1728 INSTR3(LDGB)
1729 INSTR4(STGB)
1730 INSTR4(STIB)
1731 INSTR4F(G, ATOMIC_ADD)
1732 INSTR4F(G, ATOMIC_SUB)
1733 INSTR4F(G, ATOMIC_XCHG)
1734 INSTR4F(G, ATOMIC_INC)
1735 INSTR4F(G, ATOMIC_DEC)
1736 INSTR4F(G, ATOMIC_CMPXCHG)
1737 INSTR4F(G, ATOMIC_MIN)
1738 INSTR4F(G, ATOMIC_MAX)
1739 INSTR4F(G, ATOMIC_AND)
1740 INSTR4F(G, ATOMIC_OR)
1741 INSTR4F(G, ATOMIC_XOR)
1742 #endif
1743
1744 INSTR4F(G, STG)
1745
1746 /* cat7 instructions: */
1747 INSTR0(BAR)
1748 INSTR0(FENCE)
1749
1750 /* meta instructions: */
1751 INSTR0(META_TEX_PREFETCH);
1752
1753 /* ************************************************************************* */
1754 /* split this out or find some helper to use.. like main/bitset.h.. */
1755
1756 #include <string.h>
1757 #include "util/bitset.h"
1758
1759 #define MAX_REG 256
1760
1761 typedef BITSET_DECLARE(regmaskstate_t, 2 * MAX_REG);
1762
1763 typedef struct {
1764 bool mergedregs;
1765 regmaskstate_t mask;
1766 } regmask_t;
1767
1768 static inline bool
1769 __regmask_get(regmask_t *regmask, struct ir3_register *reg, unsigned n)
1770 {
1771 if (regmask->mergedregs) {
1772 /* a6xx+ case, with merged register file, we track things in terms
1773 * of half-precision registers, with a full precisions register
1774 * using two half-precision slots:
1775 */
1776 if (reg->flags & IR3_REG_HALF) {
1777 return BITSET_TEST(regmask->mask, n);
1778 } else {
1779 n *= 2;
1780 return BITSET_TEST(regmask->mask, n) ||
1781 BITSET_TEST(regmask->mask, n+1);
1782 }
1783 } else {
1784 /* pre a6xx case, with separate register file for half and full
1785 * precision:
1786 */
1787 if (reg->flags & IR3_REG_HALF)
1788 n += MAX_REG;
1789 return BITSET_TEST(regmask->mask, n);
1790 }
1791 }
1792
1793 static inline void
1794 __regmask_set(regmask_t *regmask, struct ir3_register *reg, unsigned n)
1795 {
1796 if (regmask->mergedregs) {
1797 /* a6xx+ case, with merged register file, we track things in terms
1798 * of half-precision registers, with a full precisions register
1799 * using two half-precision slots:
1800 */
1801 if (reg->flags & IR3_REG_HALF) {
1802 BITSET_SET(regmask->mask, n);
1803 } else {
1804 n *= 2;
1805 BITSET_SET(regmask->mask, n);
1806 BITSET_SET(regmask->mask, n+1);
1807 }
1808 } else {
1809 /* pre a6xx case, with separate register file for half and full
1810 * precision:
1811 */
1812 if (reg->flags & IR3_REG_HALF)
1813 n += MAX_REG;
1814 BITSET_SET(regmask->mask, n);
1815 }
1816 }
1817
1818 static inline void regmask_init(regmask_t *regmask, bool mergedregs)
1819 {
1820 memset(&regmask->mask, 0, sizeof(regmask->mask));
1821 regmask->mergedregs = mergedregs;
1822 }
1823
1824 static inline void regmask_set(regmask_t *regmask, struct ir3_register *reg)
1825 {
1826 if (reg->flags & IR3_REG_RELATIV) {
1827 for (unsigned i = 0; i < reg->size; i++)
1828 __regmask_set(regmask, reg, reg->array.offset + i);
1829 } else {
1830 for (unsigned mask = reg->wrmask, n = reg->num; mask; mask >>= 1, n++)
1831 if (mask & 1)
1832 __regmask_set(regmask, reg, n);
1833 }
1834 }
1835
1836 static inline void regmask_or(regmask_t *dst, regmask_t *a, regmask_t *b)
1837 {
1838 assert(dst->mergedregs == a->mergedregs);
1839 assert(dst->mergedregs == b->mergedregs);
1840
1841 for (unsigned i = 0; i < ARRAY_SIZE(dst->mask); i++)
1842 dst->mask[i] = a->mask[i] | b->mask[i];
1843 }
1844
1845 static inline bool regmask_get(regmask_t *regmask,
1846 struct ir3_register *reg)
1847 {
1848 if (reg->flags & IR3_REG_RELATIV) {
1849 for (unsigned i = 0; i < reg->size; i++)
1850 if (__regmask_get(regmask, reg, reg->array.offset + i))
1851 return true;
1852 } else {
1853 for (unsigned mask = reg->wrmask, n = reg->num; mask; mask >>= 1, n++)
1854 if (mask & 1)
1855 if (__regmask_get(regmask, reg, n))
1856 return true;
1857 }
1858 return false;
1859 }
1860
1861 /* ************************************************************************* */
1862
1863 #endif /* IR3_H_ */