projects / mesa.git / blob
? search:


beb125c5e97acf9a95765de521e79994c91eb893
[mesa.git] / src / gallium / drivers / freedreno / ir3 / ir3.h
1 /*
2 * Copyright (c) 2013 Rob Clark <robdclark@gmail.com>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
23
24 #ifndef IR3_H_
25 #define IR3_H_
26
27 #include <stdint.h>
28 #include <stdbool.h>
29
30 #include "util/u_debug.h"
31 #include "util/list.h"
32
33 #include "instr-a3xx.h"
34 #include "disasm.h" /* TODO move 'enum shader_t' somewhere else.. */
35
36 /* low level intermediate representation of an adreno shader program */
37
38 struct ir3_compiler;
39 struct ir3;
40 struct ir3_instruction;
41 struct ir3_block;
42
43 struct ir3_info {
44 uint32_t gpu_id;
45 uint16_t sizedwords;
46 uint16_t instrs_count; /* expanded to account for rpt's */
47 /* NOTE: max_reg, etc, does not include registers not touched
48 * by the shader (ie. vertex fetched via VFD_DECODE but not
49 * touched by shader)
50 */
51 int8_t max_reg; /* highest GPR # used by shader */
52 int8_t max_half_reg;
53 int16_t max_const;
54 };
55
56 struct ir3_register {
57 enum {
58 IR3_REG_CONST = 0x001,
59 IR3_REG_IMMED = 0x002,
60 IR3_REG_HALF = 0x004,
61 /* high registers are used for some things in compute shaders,
62 * for example. Seems to be for things that are global to all
63 * threads in a wave, so possibly these are global/shared by
64 * all the threads in the wave?
65 */
66 IR3_REG_HIGH = 0x008,
67 IR3_REG_RELATIV= 0x010,
68 IR3_REG_R = 0x020,
69 /* Most instructions, it seems, can do float abs/neg but not
70 * integer. The CP pass needs to know what is intended (int or
71 * float) in order to do the right thing. For this reason the
72 * abs/neg flags are split out into float and int variants. In
73 * addition, .b (bitwise) operations, the negate is actually a
74 * bitwise not, so split that out into a new flag to make it
75 * more clear.
76 */
77 IR3_REG_FNEG = 0x040,
78 IR3_REG_FABS = 0x080,
79 IR3_REG_SNEG = 0x100,
80 IR3_REG_SABS = 0x200,
81 IR3_REG_BNOT = 0x400,
82 IR3_REG_EVEN = 0x800,
83 IR3_REG_POS_INF= 0x1000,
84 /* (ei) flag, end-input? Set on last bary, presumably to signal
85 * that the shader needs no more input:
86 */
87 IR3_REG_EI = 0x2000,
88 /* meta-flags, for intermediate stages of IR, ie.
89 * before register assignment is done:
90 */
91 IR3_REG_SSA = 0x4000, /* 'instr' is ptr to assigning instr */
92 IR3_REG_ARRAY = 0x8000,
93 IR3_REG_PHI_SRC= 0x10000, /* phi src, regs[0]->instr points to phi */
94
95 } flags;
96 union {
97 /* normal registers:
98 * the component is in the low two bits of the reg #, so
99 * rN.x becomes: (N << 2) | x
100 */
101 int num;
102 /* immediate: */
103 int32_t iim_val;
104 uint32_t uim_val;
105 float fim_val;
106 /* relative: */
107 struct {
108 uint16_t id;
109 int16_t offset;
110 } array;
111 };
112
113 /* For IR3_REG_SSA, src registers contain ptr back to assigning
114 * instruction.
115 *
116 * For IR3_REG_ARRAY, the pointer is back to the last dependent
117 * array access (although the net effect is the same, it points
118 * back to a previous instruction that we depend on).
119 */
120 struct ir3_instruction *instr;
121
122 union {
123 /* used for cat5 instructions, but also for internal/IR level
124 * tracking of what registers are read/written by an instruction.
125 * wrmask may be a bad name since it is used to represent both
126 * src and dst that touch multiple adjacent registers.
127 */
128 unsigned wrmask;
129 /* for relative addressing, 32bits for array size is too small,
130 * but otoh we don't need to deal with disjoint sets, so instead
131 * use a simple size field (number of scalar components).
132 */
133 unsigned size;
134 };
135 };
136
137 struct ir3_instruction {
138 struct ir3_block *block;
139 opc_t opc;
140 enum {
141 /* (sy) flag is set on first instruction, and after sample
142 * instructions (probably just on RAW hazard).
143 */
144 IR3_INSTR_SY = 0x001,
145 /* (ss) flag is set on first instruction, and first instruction
146 * to depend on the result of "long" instructions (RAW hazard):
147 *
148 * rcp, rsq, log2, exp2, sin, cos, sqrt
149 *
150 * It seems to synchronize until all in-flight instructions are
151 * completed, for example:
152 *
153 * rsq hr1.w, hr1.w
154 * add.f hr2.z, (neg)hr2.z, hc0.y
155 * mul.f hr2.w, (neg)hr2.y, (neg)hr2.y
156 * rsq hr2.x, hr2.x
157 * (rpt1)nop
158 * mad.f16 hr2.w, hr2.z, hr2.z, hr2.w
159 * nop
160 * mad.f16 hr2.w, (neg)hr0.w, (neg)hr0.w, hr2.w
161 * (ss)(rpt2)mul.f hr1.x, (r)hr1.x, hr1.w
162 * (rpt2)mul.f hr0.x, (neg)(r)hr0.x, hr2.x
163 *
164 * The last mul.f does not have (ss) set, presumably because the
165 * (ss) on the previous instruction does the job.
166 *
167 * The blob driver also seems to set it on WAR hazards, although
168 * not really clear if this is needed or just blob compiler being
169 * sloppy. So far I haven't found a case where removing the (ss)
170 * causes problems for WAR hazard, but I could just be getting
171 * lucky:
172 *
173 * rcp r1.y, r3.y
174 * (ss)(rpt2)mad.f32 r3.y, (r)c9.x, r1.x, (r)r3.z
175 *
176 */
177 IR3_INSTR_SS = 0x002,
178 /* (jp) flag is set on jump targets:
179 */
180 IR3_INSTR_JP = 0x004,
181 IR3_INSTR_UL = 0x008,
182 IR3_INSTR_3D = 0x010,
183 IR3_INSTR_A = 0x020,
184 IR3_INSTR_O = 0x040,
185 IR3_INSTR_P = 0x080,
186 IR3_INSTR_S = 0x100,
187 IR3_INSTR_S2EN = 0x200,
188 IR3_INSTR_G = 0x400,
189 /* meta-flags, for intermediate stages of IR, ie.
190 * before register assignment is done:
191 */
192 IR3_INSTR_MARK = 0x1000,
193 IR3_INSTR_UNUSED= 0x2000,
194 } flags;
195 int repeat;
196 #ifdef DEBUG
197 unsigned regs_max;
198 #endif
199 unsigned regs_count;
200 struct ir3_register **regs;
201 union {
202 struct {
203 char inv;
204 char comp;
205 int immed;
206 struct ir3_block *target;
207 } cat0;
208 struct {
209 type_t src_type, dst_type;
210 } cat1;
211 struct {
212 enum {
213 IR3_COND_LT = 0,
214 IR3_COND_LE = 1,
215 IR3_COND_GT = 2,
216 IR3_COND_GE = 3,
217 IR3_COND_EQ = 4,
218 IR3_COND_NE = 5,
219 } condition;
220 } cat2;
221 struct {
222 unsigned samp, tex;
223 type_t type;
224 } cat5;
225 struct {
226 type_t type;
227 int src_offset;
228 int dst_offset;
229 int iim_val; /* for ldgb/stgb, # of components */
230 } cat6;
231 /* for meta-instructions, just used to hold extra data
232 * before instruction scheduling, etc
233 */
234 struct {
235 int off; /* component/offset */
236 } fo;
237 struct {
238 /* used to temporarily hold reference to nir_phi_instr
239 * until we resolve the phi srcs
240 */
241 void *nphi;
242 } phi;
243 struct {
244 struct ir3_block *block;
245 } inout;
246 };
247
248 /* transient values used during various algorithms: */
249 union {
250 /* The instruction depth is the max dependency distance to output.
251 *
252 * You can also think of it as the "cost", if we did any sort of
253 * optimization for register footprint. Ie. a value that is just
254 * result of moving a const to a reg would have a low cost, so to
255 * it could make sense to duplicate the instruction at various
256 * points where the result is needed to reduce register footprint.
257 */
258 unsigned depth;
259 /* When we get to the RA stage, we no longer need depth, but
260 * we do need instruction's position/name:
261 */
262 struct {
263 uint16_t ip;
264 uint16_t name;
265 };
266 };
267
268 /* used for per-pass extra instruction data.
269 */
270 void *data;
271
272 /* Used during CP and RA stages. For fanin and shader inputs/
273 * outputs where we need a sequence of consecutive registers,
274 * keep track of each src instructions left (ie 'n-1') and right
275 * (ie 'n+1') neighbor. The front-end must insert enough mov's
276 * to ensure that each instruction has at most one left and at
277 * most one right neighbor. During the copy-propagation pass,
278 * we only remove mov's when we can preserve this constraint.
279 * And during the RA stage, we use the neighbor information to
280 * allocate a block of registers in one shot.
281 *
282 * TODO: maybe just add something like:
283 * struct ir3_instruction_ref {
284 * struct ir3_instruction *instr;
285 * unsigned cnt;
286 * }
287 *
288 * Or can we get away without the refcnt stuff? It seems like
289 * it should be overkill.. the problem is if, potentially after
290 * already eliminating some mov's, if you have a single mov that
291 * needs to be grouped with it's neighbors in two different
292 * places (ex. shader output and a fanin).
293 */
294 struct {
295 struct ir3_instruction *left, *right;
296 uint16_t left_cnt, right_cnt;
297 } cp;
298
299 /* an instruction can reference at most one address register amongst
300 * it's src/dst registers. Beyond that, you need to insert mov's.
301 *
302 * NOTE: do not write this directly, use ir3_instr_set_address()
303 */
304 struct ir3_instruction *address;
305
306 /* Entry in ir3_block's instruction list: */
307 struct list_head node;
308
309 #ifdef DEBUG
310 uint32_t serialno;
311 #endif
312 };
313
314 static inline struct ir3_instruction *
315 ir3_neighbor_first(struct ir3_instruction *instr)
316 {
317 int cnt = 0;
318 while (instr->cp.left) {
319 instr = instr->cp.left;
320 if (++cnt > 0xffff) {
321 debug_assert(0);
322 break;
323 }
324 }
325 return instr;
326 }
327
328 static inline int ir3_neighbor_count(struct ir3_instruction *instr)
329 {
330 int num = 1;
331
332 debug_assert(!instr->cp.left);
333
334 while (instr->cp.right) {
335 num++;
336 instr = instr->cp.right;
337 if (num > 0xffff) {
338 debug_assert(0);
339 break;
340 }
341 }
342
343 return num;
344 }
345
346 struct ir3 {
347 struct ir3_compiler *compiler;
348
349 unsigned ninputs, noutputs;
350 struct ir3_instruction **inputs;
351 struct ir3_instruction **outputs;
352
353 /* Track bary.f (and ldlv) instructions.. this is needed in
354 * scheduling to ensure that all varying fetches happen before
355 * any potential kill instructions. The hw gets grumpy if all
356 * threads in a group are killed before the last bary.f gets
357 * a chance to signal end of input (ei).
358 */
359 unsigned baryfs_count, baryfs_sz;
360 struct ir3_instruction **baryfs;
361
362 /* Track all indirect instructions (read and write). To avoid
363 * deadlock scenario where an address register gets scheduled,
364 * but other dependent src instructions cannot be scheduled due
365 * to dependency on a *different* address register value, the
366 * scheduler needs to ensure that all dependencies other than
367 * the instruction other than the address register are scheduled
368 * before the one that writes the address register. Having a
369 * convenient list of instructions that reference some address
370 * register simplifies this.
371 */
372 unsigned indirects_count, indirects_sz;
373 struct ir3_instruction **indirects;
374 /* and same for instructions that consume predicate register: */
375 unsigned predicates_count, predicates_sz;
376 struct ir3_instruction **predicates;
377
378 /* Track texture sample instructions which need texture state
379 * patched in (for astc-srgb workaround):
380 */
381 unsigned astc_srgb_count, astc_srgb_sz;
382 struct ir3_instruction **astc_srgb;
383
384 /* List of blocks: */
385 struct list_head block_list;
386
387 /* List of ir3_array's: */
388 struct list_head array_list;
389 };
390
391 typedef struct nir_variable nir_variable;
392
393 struct ir3_array {
394 struct list_head node;
395 unsigned length;
396 unsigned id;
397
398 nir_variable *var;
399
400 /* We track the last write and last access (read or write) to
401 * setup dependencies on instructions that read or write the
402 * array. Reads can be re-ordered wrt. other reads, but should
403 * not be re-ordered wrt. to writes. Writes cannot be reordered
404 * wrt. any other access to the array.
405 *
406 * So array reads depend on last write, and array writes depend
407 * on the last access.
408 */
409 struct ir3_instruction *last_write, *last_access;
410
411 /* extra stuff used in RA pass: */
412 unsigned base; /* base vreg name */
413 unsigned reg; /* base physical reg */
414 uint16_t start_ip, end_ip;
415 };
416
417 struct ir3_array * ir3_lookup_array(struct ir3 *ir, unsigned id);
418
419 typedef struct nir_block nir_block;
420
421 struct ir3_block {
422 struct list_head node;
423 struct ir3 *shader;
424
425 nir_block *nblock;
426
427 struct list_head instr_list; /* list of ir3_instruction */
428
429 /* each block has either one or two successors.. in case of
430 * two successors, 'condition' decides which one to follow.
431 * A block preceding an if/else has two successors.
432 */
433 struct ir3_instruction *condition;
434 struct ir3_block *successors[2];
435
436 uint16_t start_ip, end_ip;
437
438 /* Track instructions which do not write a register but other-
439 * wise must not be discarded (such as kill, stg, etc)
440 */
441 unsigned keeps_count, keeps_sz;
442 struct ir3_instruction **keeps;
443
444 /* used for per-pass extra block data. Mainly used right
445 * now in RA step to track livein/liveout.
446 */
447 void *data;
448
449 #ifdef DEBUG
450 uint32_t serialno;
451 #endif
452 };
453
454 static inline uint32_t
455 block_id(struct ir3_block *block)
456 {
457 #ifdef DEBUG
458 return block->serialno;
459 #else
460 return (uint32_t)(unsigned long)block;
461 #endif
462 }
463
464 struct ir3 * ir3_create(struct ir3_compiler *compiler,
465 unsigned nin, unsigned nout);
466 void ir3_destroy(struct ir3 *shader);
467 void * ir3_assemble(struct ir3 *shader,
468 struct ir3_info *info, uint32_t gpu_id);
469 void * ir3_alloc(struct ir3 *shader, int sz);
470
471 struct ir3_block * ir3_block_create(struct ir3 *shader);
472
473 struct ir3_instruction * ir3_instr_create(struct ir3_block *block, opc_t opc);
474 struct ir3_instruction * ir3_instr_create2(struct ir3_block *block,
475 opc_t opc, int nreg);
476 struct ir3_instruction * ir3_instr_clone(struct ir3_instruction *instr);
477 const char *ir3_instr_name(struct ir3_instruction *instr);
478
479 struct ir3_register * ir3_reg_create(struct ir3_instruction *instr,
480 int num, int flags);
481 struct ir3_register * ir3_reg_clone(struct ir3 *shader,
482 struct ir3_register *reg);
483
484 void ir3_instr_set_address(struct ir3_instruction *instr,
485 struct ir3_instruction *addr);
486
487 static inline bool ir3_instr_check_mark(struct ir3_instruction *instr)
488 {
489 if (instr->flags & IR3_INSTR_MARK)
490 return true; /* already visited */
491 instr->flags |= IR3_INSTR_MARK;
492 return false;
493 }
494
495 void ir3_block_clear_mark(struct ir3_block *block);
496 void ir3_clear_mark(struct ir3 *shader);
497
498 unsigned ir3_count_instructions(struct ir3 *ir);
499
500 static inline int ir3_instr_regno(struct ir3_instruction *instr,
501 struct ir3_register *reg)
502 {
503 unsigned i;
504 for (i = 0; i < instr->regs_count; i++)
505 if (reg == instr->regs[i])
506 return i;
507 return -1;
508 }
509
510
511 #define MAX_ARRAYS 16
512
513 /* comp:
514 * 0 - x
515 * 1 - y
516 * 2 - z
517 * 3 - w
518 */
519 static inline uint32_t regid(int num, int comp)
520 {
521 return (num << 2) | (comp & 0x3);
522 }
523
524 static inline uint32_t reg_num(struct ir3_register *reg)
525 {
526 return reg->num >> 2;
527 }
528
529 static inline uint32_t reg_comp(struct ir3_register *reg)
530 {
531 return reg->num & 0x3;
532 }
533
534 static inline bool is_flow(struct ir3_instruction *instr)
535 {
536 return (opc_cat(instr->opc) == 0);
537 }
538
539 static inline bool is_kill(struct ir3_instruction *instr)
540 {
541 return instr->opc == OPC_KILL;
542 }
543
544 static inline bool is_nop(struct ir3_instruction *instr)
545 {
546 return instr->opc == OPC_NOP;
547 }
548
549 /* Is it a non-transformative (ie. not type changing) mov? This can
550 * also include absneg.s/absneg.f, which for the most part can be
551 * treated as a mov (single src argument).
552 */
553 static inline bool is_same_type_mov(struct ir3_instruction *instr)
554 {
555 struct ir3_register *dst = instr->regs[0];
556
557 /* mov's that write to a0.x or p0.x are special: */
558 if (dst->num == regid(REG_P0, 0))
559 return false;
560 if (dst->num == regid(REG_A0, 0))
561 return false;
562
563 if (dst->flags & (IR3_REG_RELATIV | IR3_REG_ARRAY))
564 return false;
565
566 switch (instr->opc) {
567 case OPC_MOV:
568 return instr->cat1.src_type == instr->cat1.dst_type;
569 case OPC_ABSNEG_F:
570 case OPC_ABSNEG_S:
571 return true;
572 default:
573 return false;
574 }
575 }
576
577 static inline bool is_alu(struct ir3_instruction *instr)
578 {
579 return (1 <= opc_cat(instr->opc)) && (opc_cat(instr->opc) <= 3);
580 }
581
582 static inline bool is_sfu(struct ir3_instruction *instr)
583 {
584 return (opc_cat(instr->opc) == 4);
585 }
586
587 static inline bool is_tex(struct ir3_instruction *instr)
588 {
589 return (opc_cat(instr->opc) == 5);
590 }
591
592 static inline bool is_mem(struct ir3_instruction *instr)
593 {
594 return (opc_cat(instr->opc) == 6);
595 }
596
597 static inline bool
598 is_store(struct ir3_instruction *instr)
599 {
600 /* these instructions, the "destination" register is
601 * actually a source, the address to store to.
602 */
603 switch (instr->opc) {
604 case OPC_STG:
605 case OPC_STGB:
606 case OPC_STP:
607 case OPC_STL:
608 case OPC_STLW:
609 case OPC_L2G:
610 case OPC_G2L:
611 return true;
612 default:
613 return false;
614 }
615 }
616
617 static inline bool is_load(struct ir3_instruction *instr)
618 {
619 switch (instr->opc) {
620 case OPC_LDG:
621 case OPC_LDGB:
622 case OPC_LDL:
623 case OPC_LDP:
624 case OPC_L2G:
625 case OPC_LDLW:
626 case OPC_LDC:
627 case OPC_LDLV:
628 /* probably some others too.. */
629 return true;
630 default:
631 return false;
632 }
633 }
634
635 static inline bool is_input(struct ir3_instruction *instr)
636 {
637 /* in some cases, ldlv is used to fetch varying without
638 * interpolation.. fortunately inloc is the first src
639 * register in either case
640 */
641 switch (instr->opc) {
642 case OPC_LDLV:
643 case OPC_BARY_F:
644 return true;
645 default:
646 return false;
647 }
648 }
649
650 static inline bool is_bool(struct ir3_instruction *instr)
651 {
652 switch (instr->opc) {
653 case OPC_CMPS_F:
654 case OPC_CMPS_S:
655 case OPC_CMPS_U:
656 return true;
657 default:
658 return false;
659 }
660 }
661
662 static inline bool is_meta(struct ir3_instruction *instr)
663 {
664 /* TODO how should we count PHI (and maybe fan-in/out) which
665 * might actually contribute some instructions to the final
666 * result?
667 */
668 return (opc_cat(instr->opc) == -1);
669 }
670
671 static inline bool writes_addr(struct ir3_instruction *instr)
672 {
673 if (instr->regs_count > 0) {
674 struct ir3_register *dst = instr->regs[0];
675 return reg_num(dst) == REG_A0;
676 }
677 return false;
678 }
679
680 static inline bool writes_pred(struct ir3_instruction *instr)
681 {
682 if (instr->regs_count > 0) {
683 struct ir3_register *dst = instr->regs[0];
684 return reg_num(dst) == REG_P0;
685 }
686 return false;
687 }
688
689 /* returns defining instruction for reg */
690 /* TODO better name */
691 static inline struct ir3_instruction *ssa(struct ir3_register *reg)
692 {
693 if (reg->flags & (IR3_REG_SSA | IR3_REG_ARRAY)) {
694 debug_assert(!(reg->instr && (reg->instr->flags & IR3_INSTR_UNUSED)));
695 return reg->instr;
696 }
697 return NULL;
698 }
699
700 static inline bool conflicts(struct ir3_instruction *a,
701 struct ir3_instruction *b)
702 {
703 return (a && b) && (a != b);
704 }
705
706 static inline bool reg_gpr(struct ir3_register *r)
707 {
708 if (r->flags & (IR3_REG_CONST | IR3_REG_IMMED))
709 return false;
710 if ((reg_num(r) == REG_A0) || (reg_num(r) == REG_P0))
711 return false;
712 return true;
713 }
714
715 static inline type_t half_type(type_t type)
716 {
717 switch (type) {
718 case TYPE_F32: return TYPE_F16;
719 case TYPE_U32: return TYPE_U16;
720 case TYPE_S32: return TYPE_S16;
721 case TYPE_F16:
722 case TYPE_U16:
723 case TYPE_S16:
724 return type;
725 default:
726 assert(0);
727 return ~0;
728 }
729 }
730
731 /* some cat2 instructions (ie. those which are not float) can embed an
732 * immediate:
733 */
734 static inline bool ir3_cat2_int(opc_t opc)
735 {
736 switch (opc) {
737 case OPC_ADD_U:
738 case OPC_ADD_S:
739 case OPC_SUB_U:
740 case OPC_SUB_S:
741 case OPC_CMPS_U:
742 case OPC_CMPS_S:
743 case OPC_MIN_U:
744 case OPC_MIN_S:
745 case OPC_MAX_U:
746 case OPC_MAX_S:
747 case OPC_CMPV_U:
748 case OPC_CMPV_S:
749 case OPC_MUL_U:
750 case OPC_MUL_S:
751 case OPC_MULL_U:
752 case OPC_CLZ_S:
753 case OPC_ABSNEG_S:
754 case OPC_AND_B:
755 case OPC_OR_B:
756 case OPC_NOT_B:
757 case OPC_XOR_B:
758 case OPC_BFREV_B:
759 case OPC_CLZ_B:
760 case OPC_SHL_B:
761 case OPC_SHR_B:
762 case OPC_ASHR_B:
763 case OPC_MGEN_B:
764 case OPC_GETBIT_B:
765 case OPC_CBITS_B:
766 case OPC_BARY_F:
767 return true;
768
769 default:
770 return false;
771 }
772 }
773
774
775 /* map cat2 instruction to valid abs/neg flags: */
776 static inline unsigned ir3_cat2_absneg(opc_t opc)
777 {
778 switch (opc) {
779 case OPC_ADD_F:
780 case OPC_MIN_F:
781 case OPC_MAX_F:
782 case OPC_MUL_F:
783 case OPC_SIGN_F:
784 case OPC_CMPS_F:
785 case OPC_ABSNEG_F:
786 case OPC_CMPV_F:
787 case OPC_FLOOR_F:
788 case OPC_CEIL_F:
789 case OPC_RNDNE_F:
790 case OPC_RNDAZ_F:
791 case OPC_TRUNC_F:
792 case OPC_BARY_F:
793 return IR3_REG_FABS | IR3_REG_FNEG;
794
795 case OPC_ADD_U:
796 case OPC_ADD_S:
797 case OPC_SUB_U:
798 case OPC_SUB_S:
799 case OPC_CMPS_U:
800 case OPC_CMPS_S:
801 case OPC_MIN_U:
802 case OPC_MIN_S:
803 case OPC_MAX_U:
804 case OPC_MAX_S:
805 case OPC_CMPV_U:
806 case OPC_CMPV_S:
807 case OPC_MUL_U:
808 case OPC_MUL_S:
809 case OPC_MULL_U:
810 case OPC_CLZ_S:
811 return 0;
812
813 case OPC_ABSNEG_S:
814 return IR3_REG_SABS | IR3_REG_SNEG;
815
816 case OPC_AND_B:
817 case OPC_OR_B:
818 case OPC_NOT_B:
819 case OPC_XOR_B:
820 case OPC_BFREV_B:
821 case OPC_CLZ_B:
822 case OPC_SHL_B:
823 case OPC_SHR_B:
824 case OPC_ASHR_B:
825 case OPC_MGEN_B:
826 case OPC_GETBIT_B:
827 case OPC_CBITS_B:
828 return IR3_REG_BNOT;
829
830 default:
831 return 0;
832 }
833 }
834
835 /* map cat3 instructions to valid abs/neg flags: */
836 static inline unsigned ir3_cat3_absneg(opc_t opc)
837 {
838 switch (opc) {
839 case OPC_MAD_F16:
840 case OPC_MAD_F32:
841 case OPC_SEL_F16:
842 case OPC_SEL_F32:
843 return IR3_REG_FNEG;
844
845 case OPC_MAD_U16:
846 case OPC_MADSH_U16:
847 case OPC_MAD_S16:
848 case OPC_MADSH_M16:
849 case OPC_MAD_U24:
850 case OPC_MAD_S24:
851 case OPC_SEL_S16:
852 case OPC_SEL_S32:
853 case OPC_SAD_S16:
854 case OPC_SAD_S32:
855 /* neg *may* work on 3rd src.. */
856
857 case OPC_SEL_B16:
858 case OPC_SEL_B32:
859
860 default:
861 return 0;
862 }
863 }
864
865 #define array_insert(ctx, arr, val) do { \
866 if (arr ## _count == arr ## _sz) { \
867 arr ## _sz = MAX2(2 * arr ## _sz, 16); \
868 arr = reralloc_size(ctx, arr, arr ## _sz * sizeof(arr[0])); \
869 } \
870 arr[arr ##_count++] = val; \
871 } while (0)
872
873 /* iterator for an instructions's sources (reg), also returns src #: */
874 #define foreach_src_n(__srcreg, __n, __instr) \
875 if ((__instr)->regs_count) \
876 for (unsigned __cnt = (__instr)->regs_count - 1, __n = 0; __n < __cnt; __n++) \
877 if ((__srcreg = (__instr)->regs[__n + 1]))
878
879 /* iterator for an instructions's sources (reg): */
880 #define foreach_src(__srcreg, __instr) \
881 foreach_src_n(__srcreg, __i, __instr)
882
883 static inline unsigned __ssa_src_cnt(struct ir3_instruction *instr)
884 {
885 if (instr->address)
886 return instr->regs_count + 1;
887 return instr->regs_count;
888 }
889
890 static inline struct ir3_instruction * __ssa_src_n(struct ir3_instruction *instr, unsigned n)
891 {
892 if (n == (instr->regs_count + 0))
893 return instr->address;
894 return ssa(instr->regs[n]);
895 }
896
897 #define __src_cnt(__instr) ((__instr)->address ? (__instr)->regs_count : (__instr)->regs_count - 1)
898
899 /* iterator for an instruction's SSA sources (instr), also returns src #: */
900 #define foreach_ssa_src_n(__srcinst, __n, __instr) \
901 if ((__instr)->regs_count) \
902 for (unsigned __cnt = __ssa_src_cnt(__instr), __n = 0; __n < __cnt; __n++) \
903 if ((__srcinst = __ssa_src_n(__instr, __n)))
904
905 /* iterator for an instruction's SSA sources (instr): */
906 #define foreach_ssa_src(__srcinst, __instr) \
907 foreach_ssa_src_n(__srcinst, __i, __instr)
908
909
910 /* dump: */
911 void ir3_print(struct ir3 *ir);
912 void ir3_print_instr(struct ir3_instruction *instr);
913
914 /* depth calculation: */
915 int ir3_delayslots(struct ir3_instruction *assigner,
916 struct ir3_instruction *consumer, unsigned n);
917 void ir3_insert_by_depth(struct ir3_instruction *instr, struct list_head *list);
918 void ir3_depth(struct ir3 *ir);
919
920 /* copy-propagate: */
921 struct ir3_shader_variant;
922 void ir3_cp(struct ir3 *ir, struct ir3_shader_variant *so);
923
924 /* group neighbors and insert mov's to resolve conflicts: */
925 void ir3_group(struct ir3 *ir);
926
927 /* scheduling: */
928 int ir3_sched(struct ir3 *ir);
929
930 /* register assignment: */
931 struct ir3_ra_reg_set * ir3_ra_alloc_reg_set(void *memctx);
932 int ir3_ra(struct ir3 *ir3, enum shader_t type,
933 bool frag_coord, bool frag_face);
934
935 /* legalize: */
936 void ir3_legalize(struct ir3 *ir, bool *has_samp, bool *has_ssbo, int *max_bary);
937
938 /* ************************************************************************* */
939 /* instruction helpers */
940
941 static inline struct ir3_instruction *
942 ir3_MOV(struct ir3_block *block, struct ir3_instruction *src, type_t type)
943 {
944 struct ir3_instruction *instr = ir3_instr_create(block, OPC_MOV);
945 ir3_reg_create(instr, 0, 0); /* dst */
946 if (src->regs[0]->flags & IR3_REG_ARRAY) {
947 struct ir3_register *src_reg =
948 ir3_reg_create(instr, 0, IR3_REG_ARRAY);
949 src_reg->array = src->regs[0]->array;
950 src_reg->instr = src;
951 } else {
952 ir3_reg_create(instr, 0, IR3_REG_SSA)->instr = src;
953 }
954 debug_assert(!(src->regs[0]->flags & IR3_REG_RELATIV));
955 instr->cat1.src_type = type;
956 instr->cat1.dst_type = type;
957 return instr;
958 }
959
960 static inline struct ir3_instruction *
961 ir3_COV(struct ir3_block *block, struct ir3_instruction *src,
962 type_t src_type, type_t dst_type)
963 {
964 struct ir3_instruction *instr = ir3_instr_create(block, OPC_MOV);
965 ir3_reg_create(instr, 0, 0); /* dst */
966 ir3_reg_create(instr, 0, IR3_REG_SSA)->instr = src;
967 instr->cat1.src_type = src_type;
968 instr->cat1.dst_type = dst_type;
969 debug_assert(!(src->regs[0]->flags & IR3_REG_ARRAY));
970 return instr;
971 }
972
973 static inline struct ir3_instruction *
974 ir3_NOP(struct ir3_block *block)
975 {
976 return ir3_instr_create(block, OPC_NOP);
977 }
978
979 #define INSTR0(name) \
980 static inline struct ir3_instruction * \
981 ir3_##name(struct ir3_block *block) \
982 { \
983 struct ir3_instruction *instr = \
984 ir3_instr_create(block, OPC_##name); \
985 return instr; \
986 }
987
988 #define INSTR1(name) \
989 static inline struct ir3_instruction * \
990 ir3_##name(struct ir3_block *block, \
991 struct ir3_instruction *a, unsigned aflags) \
992 { \
993 struct ir3_instruction *instr = \
994 ir3_instr_create(block, OPC_##name); \
995 ir3_reg_create(instr, 0, 0); /* dst */ \
996 ir3_reg_create(instr, 0, IR3_REG_SSA | aflags)->instr = a; \
997 return instr; \
998 }
999
1000 #define INSTR2(name) \
1001 static inline struct ir3_instruction * \
1002 ir3_##name(struct ir3_block *block, \
1003 struct ir3_instruction *a, unsigned aflags, \
1004 struct ir3_instruction *b, unsigned bflags) \
1005 { \
1006 struct ir3_instruction *instr = \
1007 ir3_instr_create(block, OPC_##name); \
1008 ir3_reg_create(instr, 0, 0); /* dst */ \
1009 ir3_reg_create(instr, 0, IR3_REG_SSA | aflags)->instr = a; \
1010 ir3_reg_create(instr, 0, IR3_REG_SSA | bflags)->instr = b; \
1011 return instr; \
1012 }
1013
1014 #define INSTR3(name) \
1015 static inline struct ir3_instruction * \
1016 ir3_##name(struct ir3_block *block, \
1017 struct ir3_instruction *a, unsigned aflags, \
1018 struct ir3_instruction *b, unsigned bflags, \
1019 struct ir3_instruction *c, unsigned cflags) \
1020 { \
1021 struct ir3_instruction *instr = \
1022 ir3_instr_create(block, OPC_##name); \
1023 ir3_reg_create(instr, 0, 0); /* dst */ \
1024 ir3_reg_create(instr, 0, IR3_REG_SSA | aflags)->instr = a; \
1025 ir3_reg_create(instr, 0, IR3_REG_SSA | bflags)->instr = b; \
1026 ir3_reg_create(instr, 0, IR3_REG_SSA | cflags)->instr = c; \
1027 return instr; \
1028 }
1029
1030 #define INSTR4(name) \
1031 static inline struct ir3_instruction * \
1032 ir3_##name(struct ir3_block *block, \
1033 struct ir3_instruction *a, unsigned aflags, \
1034 struct ir3_instruction *b, unsigned bflags, \
1035 struct ir3_instruction *c, unsigned cflags, \
1036 struct ir3_instruction *d, unsigned dflags) \
1037 { \
1038 struct ir3_instruction *instr = \
1039 ir3_instr_create2(block, OPC_##name, 5); \
1040 ir3_reg_create(instr, 0, 0); /* dst */ \
1041 ir3_reg_create(instr, 0, IR3_REG_SSA | aflags)->instr = a; \
1042 ir3_reg_create(instr, 0, IR3_REG_SSA | bflags)->instr = b; \
1043 ir3_reg_create(instr, 0, IR3_REG_SSA | cflags)->instr = c; \
1044 ir3_reg_create(instr, 0, IR3_REG_SSA | dflags)->instr = d; \
1045 return instr; \
1046 }
1047
1048 /* cat0 instructions: */
1049 INSTR0(BR);
1050 INSTR0(JUMP);
1051 INSTR1(KILL);
1052 INSTR0(END);
1053
1054 /* cat2 instructions, most 2 src but some 1 src: */
1055 INSTR2(ADD_F)
1056 INSTR2(MIN_F)
1057 INSTR2(MAX_F)
1058 INSTR2(MUL_F)
1059 INSTR1(SIGN_F)
1060 INSTR2(CMPS_F)
1061 INSTR1(ABSNEG_F)
1062 INSTR2(CMPV_F)
1063 INSTR1(FLOOR_F)
1064 INSTR1(CEIL_F)
1065 INSTR1(RNDNE_F)
1066 INSTR1(RNDAZ_F)
1067 INSTR1(TRUNC_F)
1068 INSTR2(ADD_U)
1069 INSTR2(ADD_S)
1070 INSTR2(SUB_U)
1071 INSTR2(SUB_S)
1072 INSTR2(CMPS_U)
1073 INSTR2(CMPS_S)
1074 INSTR2(MIN_U)
1075 INSTR2(MIN_S)
1076 INSTR2(MAX_U)
1077 INSTR2(MAX_S)
1078 INSTR1(ABSNEG_S)
1079 INSTR2(AND_B)
1080 INSTR2(OR_B)
1081 INSTR1(NOT_B)
1082 INSTR2(XOR_B)
1083 INSTR2(CMPV_U)
1084 INSTR2(CMPV_S)
1085 INSTR2(MUL_U)
1086 INSTR2(MUL_S)
1087 INSTR2(MULL_U)
1088 INSTR1(BFREV_B)
1089 INSTR1(CLZ_S)
1090 INSTR1(CLZ_B)
1091 INSTR2(SHL_B)
1092 INSTR2(SHR_B)
1093 INSTR2(ASHR_B)
1094 INSTR2(BARY_F)
1095 INSTR2(MGEN_B)
1096 INSTR2(GETBIT_B)
1097 INSTR1(SETRM)
1098 INSTR1(CBITS_B)
1099 INSTR2(SHB)
1100 INSTR2(MSAD)
1101
1102 /* cat3 instructions: */
1103 INSTR3(MAD_U16)
1104 INSTR3(MADSH_U16)
1105 INSTR3(MAD_S16)
1106 INSTR3(MADSH_M16)
1107 INSTR3(MAD_U24)
1108 INSTR3(MAD_S24)
1109 INSTR3(MAD_F16)
1110 INSTR3(MAD_F32)
1111 INSTR3(SEL_B16)
1112 INSTR3(SEL_B32)
1113 INSTR3(SEL_S16)
1114 INSTR3(SEL_S32)
1115 INSTR3(SEL_F16)
1116 INSTR3(SEL_F32)
1117 INSTR3(SAD_S16)
1118 INSTR3(SAD_S32)
1119
1120 /* cat4 instructions: */
1121 INSTR1(RCP)
1122 INSTR1(RSQ)
1123 INSTR1(LOG2)
1124 INSTR1(EXP2)
1125 INSTR1(SIN)
1126 INSTR1(COS)
1127 INSTR1(SQRT)
1128
1129 /* cat5 instructions: */
1130 INSTR1(DSX)
1131 INSTR1(DSY)
1132
1133 static inline struct ir3_instruction *
1134 ir3_SAM(struct ir3_block *block, opc_t opc, type_t type,
1135 unsigned wrmask, unsigned flags, unsigned samp, unsigned tex,
1136 struct ir3_instruction *src0, struct ir3_instruction *src1)
1137 {
1138 struct ir3_instruction *sam;
1139 struct ir3_register *reg;
1140
1141 sam = ir3_instr_create(block, opc);
1142 sam->flags |= flags;
1143 ir3_reg_create(sam, 0, 0)->wrmask = wrmask;
1144 if (src0) {
1145 reg = ir3_reg_create(sam, 0, IR3_REG_SSA);
1146 reg->wrmask = (1 << (src0->regs_count - 1)) - 1;
1147 reg->instr = src0;
1148 }
1149 if (src1) {
1150 reg = ir3_reg_create(sam, 0, IR3_REG_SSA);
1151 reg->instr = src1;
1152 reg->wrmask = (1 << (src1->regs_count - 1)) - 1;
1153 }
1154 sam->cat5.samp = samp;
1155 sam->cat5.tex = tex;
1156 sam->cat5.type = type;
1157
1158 return sam;
1159 }
1160
1161 /* cat6 instructions: */
1162 INSTR2(LDLV)
1163 INSTR2(LDG)
1164 INSTR3(STG)
1165 INSTR3(LDGB);
1166 INSTR4(STGB);
1167 INSTR4(ATOMIC_ADD);
1168 INSTR4(ATOMIC_SUB);
1169 INSTR4(ATOMIC_XCHG);
1170 INSTR4(ATOMIC_INC);
1171 INSTR4(ATOMIC_DEC);
1172 INSTR4(ATOMIC_CMPXCHG);
1173 INSTR4(ATOMIC_MIN);
1174 INSTR4(ATOMIC_MAX);
1175 INSTR4(ATOMIC_AND);
1176 INSTR4(ATOMIC_OR);
1177 INSTR4(ATOMIC_XOR);
1178
1179 /* ************************************************************************* */
1180 /* split this out or find some helper to use.. like main/bitset.h.. */
1181
1182 #include <string.h>
1183
1184 #define MAX_REG 256
1185
1186 typedef uint8_t regmask_t[2 * MAX_REG / 8];
1187
1188 static inline unsigned regmask_idx(struct ir3_register *reg)
1189 {
1190 unsigned num = (reg->flags & IR3_REG_RELATIV) ? reg->array.offset : reg->num;
1191 debug_assert(num < MAX_REG);
1192 if (reg->flags & IR3_REG_HALF)
1193 num += MAX_REG;
1194 return num;
1195 }
1196
1197 static inline void regmask_init(regmask_t *regmask)
1198 {
1199 memset(regmask, 0, sizeof(*regmask));
1200 }
1201
1202 static inline void regmask_set(regmask_t *regmask, struct ir3_register *reg)
1203 {
1204 unsigned idx = regmask_idx(reg);
1205 if (reg->flags & IR3_REG_RELATIV) {
1206 unsigned i;
1207 for (i = 0; i < reg->size; i++, idx++)
1208 (*regmask)[idx / 8] |= 1 << (idx % 8);
1209 } else {
1210 unsigned mask;
1211 for (mask = reg->wrmask; mask; mask >>= 1, idx++)
1212 if (mask & 1)
1213 (*regmask)[idx / 8] |= 1 << (idx % 8);
1214 }
1215 }
1216
1217 static inline void regmask_or(regmask_t *dst, regmask_t *a, regmask_t *b)
1218 {
1219 unsigned i;
1220 for (i = 0; i < ARRAY_SIZE(*dst); i++)
1221 (*dst)[i] = (*a)[i] | (*b)[i];
1222 }
1223
1224 /* set bits in a if not set in b, conceptually:
1225 * a |= (reg & ~b)
1226 */
1227 static inline void regmask_set_if_not(regmask_t *a,
1228 struct ir3_register *reg, regmask_t *b)
1229 {
1230 unsigned idx = regmask_idx(reg);
1231 if (reg->flags & IR3_REG_RELATIV) {
1232 unsigned i;
1233 for (i = 0; i < reg->size; i++, idx++)
1234 if (!((*b)[idx / 8] & (1 << (idx % 8))))
1235 (*a)[idx / 8] |= 1 << (idx % 8);
1236 } else {
1237 unsigned mask;
1238 for (mask = reg->wrmask; mask; mask >>= 1, idx++)
1239 if (mask & 1)
1240 if (!((*b)[idx / 8] & (1 << (idx % 8))))
1241 (*a)[idx / 8] |= 1 << (idx % 8);
1242 }
1243 }
1244
1245 static inline bool regmask_get(regmask_t *regmask,
1246 struct ir3_register *reg)
1247 {
1248 unsigned idx = regmask_idx(reg);
1249 if (reg->flags & IR3_REG_RELATIV) {
1250 unsigned i;
1251 for (i = 0; i < reg->size; i++, idx++)
1252 if ((*regmask)[idx / 8] & (1 << (idx % 8)))
1253 return true;
1254 } else {
1255 unsigned mask;
1256 for (mask = reg->wrmask; mask; mask >>= 1, idx++)
1257 if (mask & 1)
1258 if ((*regmask)[idx / 8] & (1 << (idx % 8)))
1259 return true;
1260 }
1261 return false;
1262 }
1263
1264 /* ************************************************************************* */
1265
1266 #endif /* IR3_H_ */