src/gallium/drivers/freedreno/ir3/ir3.h

   1 /*
   2  * Copyright (c) 2013 Rob Clark <robdclark@gmail.com>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  */
  23
  24 #ifndef IR3_H_
  25 #define IR3_H_
  26
  27 #include <stdint.h>
  28 #include <stdbool.h>
  29
  30 #include "util/u_debug.h"
  31 #include "util/list.h"
  32
  33 #include "instr-a3xx.h"
  34 #include "disasm.h"  /* TODO move 'enum shader_t' somewhere else.. */
  35
  36 /* low level intermediate representation of an adreno shader program */
  37
  38 struct ir3_compiler;
  39 struct ir3;
  40 struct ir3_instruction;
  41 struct ir3_block;
  42
  43 struct ir3_info {
  44         uint32_t gpu_id;
  45         uint16_t sizedwords;
  46         uint16_t instrs_count;   /* expanded to account for rpt's */
  47         /* NOTE: max_reg, etc, does not include registers not touched
  48          * by the shader (ie. vertex fetched via VFD_DECODE but not
  49          * touched by shader)
  50          */
  51         int8_t   max_reg;   /* highest GPR # used by shader */
  52         int8_t   max_half_reg;
  53         int16_t  max_const;
  54 };
  55
  56 struct ir3_register {
  57         enum {
  58                 IR3_REG_CONST  = 0x001,
  59                 IR3_REG_IMMED  = 0x002,
  60                 IR3_REG_HALF   = 0x004,
  61                 IR3_REG_RELATIV= 0x008,
  62                 IR3_REG_R      = 0x010,
  63                 /* Most instructions, it seems, can do float abs/neg but not
  64                  * integer.  The CP pass needs to know what is intended (int or
  65                  * float) in order to do the right thing.  For this reason the
  66                  * abs/neg flags are split out into float and int variants.  In
  67                  * addition, .b (bitwise) operations, the negate is actually a
  68                  * bitwise not, so split that out into a new flag to make it
  69                  * more clear.
  70                  */
  71                 IR3_REG_FNEG   = 0x020,
  72                 IR3_REG_FABS   = 0x040,
  73                 IR3_REG_SNEG   = 0x080,
  74                 IR3_REG_SABS   = 0x100,
  75                 IR3_REG_BNOT   = 0x200,
  76                 IR3_REG_EVEN   = 0x400,
  77                 IR3_REG_POS_INF= 0x800,
  78                 /* (ei) flag, end-input?  Set on last bary, presumably to signal
  79                  * that the shader needs no more input:
  80                  */
  81                 IR3_REG_EI     = 0x1000,
  82                 /* meta-flags, for intermediate stages of IR, ie.
  83                  * before register assignment is done:
  84                  */
  85                 IR3_REG_SSA    = 0x2000,   /* 'instr' is ptr to assigning instr */
  86                 IR3_REG_ARRAY  = 0x4000,
  87                 IR3_REG_PHI_SRC= 0x8000,   /* phi src, regs[0]->instr points to phi */
  88
  89         } flags;
  90         union {
  91                 /* normal registers:
  92                  * the component is in the low two bits of the reg #, so
  93                  * rN.x becomes: (N << 2) | x
  94                  */
  95                 int   num;
  96                 /* immediate: */
  97                 int32_t  iim_val;
  98                 uint32_t uim_val;
  99                 float    fim_val;
 100                 /* relative: */
 101                 struct {
 102                         uint16_t id;
 103                         int16_t offset;
 104                 } array;
 105         };
 106
 107         /* For IR3_REG_SSA, src registers contain ptr back to assigning
 108          * instruction.
 109          *
 110          * For IR3_REG_ARRAY, the pointer is back to the last dependent
 111          * array access (although the net effect is the same, it points
 112          * back to a previous instruction that we depend on).
 113          */
 114         struct ir3_instruction *instr;
 115
 116         union {
 117                 /* used for cat5 instructions, but also for internal/IR level
 118                  * tracking of what registers are read/written by an instruction.
 119                  * wrmask may be a bad name since it is used to represent both
 120                  * src and dst that touch multiple adjacent registers.
 121                  */
 122                 unsigned wrmask;
 123                 /* for relative addressing, 32bits for array size is too small,
 124                  * but otoh we don't need to deal with disjoint sets, so instead
 125                  * use a simple size field (number of scalar components).
 126                  */
 127                 unsigned size;
 128         };
 129 };
 130
 131 struct ir3_instruction {
 132         struct ir3_block *block;
 133         opc_t opc;
 134         enum {
 135                 /* (sy) flag is set on first instruction, and after sample
 136                  * instructions (probably just on RAW hazard).
 137                  */
 138                 IR3_INSTR_SY    = 0x001,
 139                 /* (ss) flag is set on first instruction, and first instruction
 140                  * to depend on the result of "long" instructions (RAW hazard):
 141                  *
 142                  *   rcp, rsq, log2, exp2, sin, cos, sqrt
 143                  *
 144                  * It seems to synchronize until all in-flight instructions are
 145                  * completed, for example:
 146                  *
 147                  *   rsq hr1.w, hr1.w
 148                  *   add.f hr2.z, (neg)hr2.z, hc0.y
 149                  *   mul.f hr2.w, (neg)hr2.y, (neg)hr2.y
 150                  *   rsq hr2.x, hr2.x
 151                  *   (rpt1)nop
 152                  *   mad.f16 hr2.w, hr2.z, hr2.z, hr2.w
 153                  *   nop
 154                  *   mad.f16 hr2.w, (neg)hr0.w, (neg)hr0.w, hr2.w
 155                  *   (ss)(rpt2)mul.f hr1.x, (r)hr1.x, hr1.w
 156                  *   (rpt2)mul.f hr0.x, (neg)(r)hr0.x, hr2.x
 157                  *
 158                  * The last mul.f does not have (ss) set, presumably because the
 159                  * (ss) on the previous instruction does the job.
 160                  *
 161                  * The blob driver also seems to set it on WAR hazards, although
 162                  * not really clear if this is needed or just blob compiler being
 163                  * sloppy.  So far I haven't found a case where removing the (ss)
 164                  * causes problems for WAR hazard, but I could just be getting
 165                  * lucky:
 166                  *
 167                  *   rcp r1.y, r3.y
 168                  *   (ss)(rpt2)mad.f32 r3.y, (r)c9.x, r1.x, (r)r3.z
 169                  *
 170                  */
 171                 IR3_INSTR_SS    = 0x002,
 172                 /* (jp) flag is set on jump targets:
 173                  */
 174                 IR3_INSTR_JP    = 0x004,
 175                 IR3_INSTR_UL    = 0x008,
 176                 IR3_INSTR_3D    = 0x010,
 177                 IR3_INSTR_A     = 0x020,
 178                 IR3_INSTR_O     = 0x040,
 179                 IR3_INSTR_P     = 0x080,
 180                 IR3_INSTR_S     = 0x100,
 181                 IR3_INSTR_S2EN  = 0x200,
 182                 IR3_INSTR_G     = 0x400,
 183                 /* meta-flags, for intermediate stages of IR, ie.
 184                  * before register assignment is done:
 185                  */
 186                 IR3_INSTR_MARK  = 0x1000,
 187                 IR3_INSTR_UNUSED= 0x2000,
 188         } flags;
 189         int repeat;
 190 #ifdef DEBUG
 191         unsigned regs_max;
 192 #endif
 193         unsigned regs_count;
 194         struct ir3_register **regs;
 195         union {
 196                 struct {
 197                         char inv;
 198                         char comp;
 199                         int  immed;
 200                         struct ir3_block *target;
 201                 } cat0;
 202                 struct {
 203                         type_t src_type, dst_type;
 204                 } cat1;
 205                 struct {
 206                         enum {
 207                                 IR3_COND_LT = 0,
 208                                 IR3_COND_LE = 1,
 209                                 IR3_COND_GT = 2,
 210                                 IR3_COND_GE = 3,
 211                                 IR3_COND_EQ = 4,
 212                                 IR3_COND_NE = 5,
 213                         } condition;
 214                 } cat2;
 215                 struct {
 216                         unsigned samp, tex;
 217                         type_t type;
 218                 } cat5;
 219                 struct {
 220                         type_t type;
 221                         int src_offset;
 222                         int dst_offset;
 223                         int iim_val;
 224                 } cat6;
 225                 /* for meta-instructions, just used to hold extra data
 226                  * before instruction scheduling, etc
 227                  */
 228                 struct {
 229                         int off;              /* component/offset */
 230                 } fo;
 231                 struct {
 232                         /* used to temporarily hold reference to nir_phi_instr
 233                          * until we resolve the phi srcs
 234                          */
 235                         void *nphi;
 236                 } phi;
 237                 struct {
 238                         struct ir3_block *block;
 239                 } inout;
 240         };
 241
 242         /* transient values used during various algorithms: */
 243         union {
 244                 /* The instruction depth is the max dependency distance to output.
 245                  *
 246                  * You can also think of it as the "cost", if we did any sort of
 247                  * optimization for register footprint.  Ie. a value that is  just
 248                  * result of moving a const to a reg would have a low cost,  so to
 249                  * it could make sense to duplicate the instruction at various
 250                  * points where the result is needed to reduce register footprint.
 251                  */
 252                 unsigned depth;
 253                 /* When we get to the RA stage, we no longer need depth, but
 254                  * we do need instruction's position/name:
 255                  */
 256                 struct {
 257                         uint16_t ip;
 258                         uint16_t name;
 259                 };
 260         };
 261
 262         /* used for per-pass extra instruction data.
 263          */
 264         void *data;
 265
 266         /* Used during CP and RA stages.  For fanin and shader inputs/
 267          * outputs where we need a sequence of consecutive registers,
 268          * keep track of each src instructions left (ie 'n-1') and right
 269          * (ie 'n+1') neighbor.  The front-end must insert enough mov's
 270          * to ensure that each instruction has at most one left and at
 271          * most one right neighbor.  During the copy-propagation pass,
 272          * we only remove mov's when we can preserve this constraint.
 273          * And during the RA stage, we use the neighbor information to
 274          * allocate a block of registers in one shot.
 275          *
 276          * TODO: maybe just add something like:
 277          *   struct ir3_instruction_ref {
 278          *       struct ir3_instruction *instr;
 279          *       unsigned cnt;
 280          *   }
 281          *
 282          * Or can we get away without the refcnt stuff?  It seems like
 283          * it should be overkill..  the problem is if, potentially after
 284          * already eliminating some mov's, if you have a single mov that
 285          * needs to be grouped with it's neighbors in two different
 286          * places (ex. shader output and a fanin).
 287          */
 288         struct {
 289                 struct ir3_instruction *left, *right;
 290                 uint16_t left_cnt, right_cnt;
 291         } cp;
 292
 293         /* an instruction can reference at most one address register amongst
 294          * it's src/dst registers.  Beyond that, you need to insert mov's.
 295          *
 296          * NOTE: do not write this directly, use ir3_instr_set_address()
 297          */
 298         struct ir3_instruction *address;
 299
 300         /* Entry in ir3_block's instruction list: */
 301         struct list_head node;
 302
 303 #ifdef DEBUG
 304         uint32_t serialno;
 305 #endif
 306 };
 307
 308 static inline struct ir3_instruction *
 309 ir3_neighbor_first(struct ir3_instruction *instr)
 310 {
 311         while (instr->cp.left)
 312                 instr = instr->cp.left;
 313         return instr;
 314 }
 315
 316 static inline int ir3_neighbor_count(struct ir3_instruction *instr)
 317 {
 318         int num = 1;
 319
 320         debug_assert(!instr->cp.left);
 321
 322         while (instr->cp.right) {
 323                 num++;
 324                 instr = instr->cp.right;
 325         }
 326
 327         return num;
 328 }
 329
 330 struct ir3_heap_chunk;
 331
 332 struct ir3 {
 333         struct ir3_compiler *compiler;
 334
 335         unsigned ninputs, noutputs;
 336         struct ir3_instruction **inputs;
 337         struct ir3_instruction **outputs;
 338
 339         /* Track bary.f (and ldlv) instructions.. this is needed in
 340          * scheduling to ensure that all varying fetches happen before
 341          * any potential kill instructions.  The hw gets grumpy if all
 342          * threads in a group are killed before the last bary.f gets
 343          * a chance to signal end of input (ei).
 344          */
 345         unsigned baryfs_count, baryfs_sz;
 346         struct ir3_instruction **baryfs;
 347
 348         /* Track all indirect instructions (read and write).  To avoid
 349          * deadlock scenario where an address register gets scheduled,
 350          * but other dependent src instructions cannot be scheduled due
 351          * to dependency on a *different* address register value, the
 352          * scheduler needs to ensure that all dependencies other than
 353          * the instruction other than the address register are scheduled
 354          * before the one that writes the address register.  Having a
 355          * convenient list of instructions that reference some address
 356          * register simplifies this.
 357          */
 358         unsigned indirects_count, indirects_sz;
 359         struct ir3_instruction **indirects;
 360         /* and same for instructions that consume predicate register: */
 361         unsigned predicates_count, predicates_sz;
 362         struct ir3_instruction **predicates;
 363
 364         /* Track instructions which do not write a register but other-
 365          * wise must not be discarded (such as kill, stg, etc)
 366          */
 367         unsigned keeps_count, keeps_sz;
 368         struct ir3_instruction **keeps;
 369
 370         /* List of blocks: */
 371         struct list_head block_list;
 372
 373         /* List of ir3_array's: */
 374         struct list_head array_list;
 375
 376         unsigned heap_idx;
 377         struct ir3_heap_chunk *chunk;
 378 };
 379
 380 typedef struct nir_variable nir_variable;
 381
 382 struct ir3_array {
 383         struct list_head node;
 384         unsigned length;
 385         unsigned id;
 386
 387         nir_variable *var;
 388
 389         /* We track the last write and last access (read or write) to
 390          * setup dependencies on instructions that read or write the
 391          * array.  Reads can be re-ordered wrt. other reads, but should
 392          * not be re-ordered wrt. to writes.  Writes cannot be reordered
 393          * wrt. any other access to the array.
 394          *
 395          * So array reads depend on last write, and array writes depend
 396          * on the last access.
 397          */
 398         struct ir3_instruction *last_write, *last_access;
 399
 400         /* extra stuff used in RA pass: */
 401         unsigned base;      /* base vreg name */
 402         unsigned reg;       /* base physical reg */
 403         uint16_t start_ip, end_ip;
 404 };
 405
 406 struct ir3_array * ir3_lookup_array(struct ir3 *ir, unsigned id);
 407
 408 typedef struct nir_block nir_block;
 409
 410 struct ir3_block {
 411         struct list_head node;
 412         struct ir3 *shader;
 413
 414         nir_block *nblock;
 415
 416         struct list_head instr_list;  /* list of ir3_instruction */
 417
 418         /* each block has either one or two successors.. in case of
 419          * two successors, 'condition' decides which one to follow.
 420          * A block preceding an if/else has two successors.
 421          */
 422         struct ir3_instruction *condition;
 423         struct ir3_block *successors[2];
 424
 425         uint16_t start_ip, end_ip;
 426
 427         /* used for per-pass extra block data.  Mainly used right
 428          * now in RA step to track livein/liveout.
 429          */
 430         void *data;
 431
 432 #ifdef DEBUG
 433         uint32_t serialno;
 434 #endif
 435 };
 436
 437 static inline uint32_t
 438 block_id(struct ir3_block *block)
 439 {
 440 #ifdef DEBUG
 441         return block->serialno;
 442 #else
 443         return (uint32_t)(unsigned long)block;
 444 #endif
 445 }
 446
 447 struct ir3 * ir3_create(struct ir3_compiler *compiler,
 448                 unsigned nin, unsigned nout);
 449 void ir3_destroy(struct ir3 *shader);
 450 void * ir3_assemble(struct ir3 *shader,
 451                 struct ir3_info *info, uint32_t gpu_id);
 452 void * ir3_alloc(struct ir3 *shader, int sz);
 453
 454 struct ir3_block * ir3_block_create(struct ir3 *shader);
 455
 456 struct ir3_instruction * ir3_instr_create(struct ir3_block *block, opc_t opc);
 457 struct ir3_instruction * ir3_instr_create2(struct ir3_block *block,
 458                 opc_t opc, int nreg);
 459 struct ir3_instruction * ir3_instr_clone(struct ir3_instruction *instr);
 460 const char *ir3_instr_name(struct ir3_instruction *instr);
 461
 462 struct ir3_register * ir3_reg_create(struct ir3_instruction *instr,
 463                 int num, int flags);
 464 struct ir3_register * ir3_reg_clone(struct ir3 *shader,
 465                 struct ir3_register *reg);
 466
 467 void ir3_instr_set_address(struct ir3_instruction *instr,
 468                 struct ir3_instruction *addr);
 469
 470 static inline bool ir3_instr_check_mark(struct ir3_instruction *instr)
 471 {
 472         if (instr->flags & IR3_INSTR_MARK)
 473                 return true;  /* already visited */
 474         instr->flags |= IR3_INSTR_MARK;
 475         return false;
 476 }
 477
 478 void ir3_block_clear_mark(struct ir3_block *block);
 479 void ir3_clear_mark(struct ir3 *shader);
 480
 481 unsigned ir3_count_instructions(struct ir3 *ir);
 482
 483 static inline int ir3_instr_regno(struct ir3_instruction *instr,
 484                 struct ir3_register *reg)
 485 {
 486         unsigned i;
 487         for (i = 0; i < instr->regs_count; i++)
 488                 if (reg == instr->regs[i])
 489                         return i;
 490         return -1;
 491 }
 492
 493
 494 #define MAX_ARRAYS 16
 495
 496 /* comp:
 497  *   0 - x
 498  *   1 - y
 499  *   2 - z
 500  *   3 - w
 501  */
 502 static inline uint32_t regid(int num, int comp)
 503 {
 504         return (num << 2) | (comp & 0x3);
 505 }
 506
 507 static inline uint32_t reg_num(struct ir3_register *reg)
 508 {
 509         return reg->num >> 2;
 510 }
 511
 512 static inline uint32_t reg_comp(struct ir3_register *reg)
 513 {
 514         return reg->num & 0x3;
 515 }
 516
 517 static inline bool is_flow(struct ir3_instruction *instr)
 518 {
 519         return (opc_cat(instr->opc) == 0);
 520 }
 521
 522 static inline bool is_kill(struct ir3_instruction *instr)
 523 {
 524         return instr->opc == OPC_KILL;
 525 }
 526
 527 static inline bool is_nop(struct ir3_instruction *instr)
 528 {
 529         return instr->opc == OPC_NOP;
 530 }
 531
 532 /* Is it a non-transformative (ie. not type changing) mov?  This can
 533  * also include absneg.s/absneg.f, which for the most part can be
 534  * treated as a mov (single src argument).
 535  */
 536 static inline bool is_same_type_mov(struct ir3_instruction *instr)
 537 {
 538         struct ir3_register *dst = instr->regs[0];
 539
 540         /* mov's that write to a0.x or p0.x are special: */
 541         if (dst->num == regid(REG_P0, 0))
 542                 return false;
 543         if (dst->num == regid(REG_A0, 0))
 544                 return false;
 545
 546         if (dst->flags & (IR3_REG_RELATIV | IR3_REG_ARRAY))
 547                 return false;
 548
 549         switch (instr->opc) {
 550         case OPC_MOV:
 551                 return instr->cat1.src_type == instr->cat1.dst_type;
 552         case OPC_ABSNEG_F:
 553         case OPC_ABSNEG_S:
 554                 return true;
 555         default:
 556                 return false;
 557         }
 558 }
 559
 560 static inline bool is_alu(struct ir3_instruction *instr)
 561 {
 562         return (1 <= opc_cat(instr->opc)) && (opc_cat(instr->opc) <= 3);
 563 }
 564
 565 static inline bool is_sfu(struct ir3_instruction *instr)
 566 {
 567         return (opc_cat(instr->opc) == 4);
 568 }
 569
 570 static inline bool is_tex(struct ir3_instruction *instr)
 571 {
 572         return (opc_cat(instr->opc) == 5);
 573 }
 574
 575 static inline bool is_mem(struct ir3_instruction *instr)
 576 {
 577         return (opc_cat(instr->opc) == 6);
 578 }
 579
 580 static inline bool
 581 is_store(struct ir3_instruction *instr)
 582 {
 583         /* these instructions, the "destination" register is
 584          * actually a source, the address to store to.
 585          */
 586         switch (instr->opc) {
 587         case OPC_STG:
 588         case OPC_STP:
 589         case OPC_STL:
 590         case OPC_STLW:
 591         case OPC_L2G:
 592         case OPC_G2L:
 593                 return true;
 594         default:
 595                 return false;
 596         }
 597 }
 598
 599 static inline bool is_load(struct ir3_instruction *instr)
 600 {
 601         switch (instr->opc) {
 602         case OPC_LDG:
 603         case OPC_LDL:
 604         case OPC_LDP:
 605         case OPC_L2G:
 606         case OPC_LDLW:
 607         case OPC_LDC_4:
 608         case OPC_LDLV:
 609                 /* probably some others too.. */
 610                 return true;
 611         default:
 612                 return false;
 613         }
 614 }
 615
 616 static inline bool is_input(struct ir3_instruction *instr)
 617 {
 618         /* in some cases, ldlv is used to fetch varying without
 619          * interpolation.. fortunately inloc is the first src
 620          * register in either case
 621          */
 622         switch (instr->opc) {
 623         case OPC_LDLV:
 624         case OPC_BARY_F:
 625                 return true;
 626         default:
 627                 return false;
 628         }
 629 }
 630
 631 static inline bool is_meta(struct ir3_instruction *instr)
 632 {
 633         /* TODO how should we count PHI (and maybe fan-in/out) which
 634          * might actually contribute some instructions to the final
 635          * result?
 636          */
 637         return (opc_cat(instr->opc) == -1);
 638 }
 639
 640 static inline bool writes_addr(struct ir3_instruction *instr)
 641 {
 642         if (instr->regs_count > 0) {
 643                 struct ir3_register *dst = instr->regs[0];
 644                 return reg_num(dst) == REG_A0;
 645         }
 646         return false;
 647 }
 648
 649 static inline bool writes_pred(struct ir3_instruction *instr)
 650 {
 651         if (instr->regs_count > 0) {
 652                 struct ir3_register *dst = instr->regs[0];
 653                 return reg_num(dst) == REG_P0;
 654         }
 655         return false;
 656 }
 657
 658 /* returns defining instruction for reg */
 659 /* TODO better name */
 660 static inline struct ir3_instruction *ssa(struct ir3_register *reg)
 661 {
 662         if (reg->flags & (IR3_REG_SSA | IR3_REG_ARRAY)) {
 663                 debug_assert(!(reg->instr && (reg->instr->flags & IR3_INSTR_UNUSED)));
 664                 return reg->instr;
 665         }
 666         return NULL;
 667 }
 668
 669 static inline bool conflicts(struct ir3_instruction *a,
 670                 struct ir3_instruction *b)
 671 {
 672         return (a && b) && (a != b);
 673 }
 674
 675 static inline bool reg_gpr(struct ir3_register *r)
 676 {
 677         if (r->flags & (IR3_REG_CONST | IR3_REG_IMMED))
 678                 return false;
 679         if ((reg_num(r) == REG_A0) || (reg_num(r) == REG_P0))
 680                 return false;
 681         return true;
 682 }
 683
 684 static inline type_t half_type(type_t type)
 685 {
 686         switch (type) {
 687         case TYPE_F32: return TYPE_F16;
 688         case TYPE_U32: return TYPE_U16;
 689         case TYPE_S32: return TYPE_S16;
 690         case TYPE_F16:
 691         case TYPE_U16:
 692         case TYPE_S16:
 693                 return type;
 694         default:
 695                 assert(0);
 696                 return ~0;
 697         }
 698 }
 699
 700 /* some cat2 instructions (ie. those which are not float) can embed an
 701  * immediate:
 702  */
 703 static inline bool ir3_cat2_int(opc_t opc)
 704 {
 705         switch (opc) {
 706         case OPC_ADD_U:
 707         case OPC_ADD_S:
 708         case OPC_SUB_U:
 709         case OPC_SUB_S:
 710         case OPC_CMPS_U:
 711         case OPC_CMPS_S:
 712         case OPC_MIN_U:
 713         case OPC_MIN_S:
 714         case OPC_MAX_U:
 715         case OPC_MAX_S:
 716         case OPC_CMPV_U:
 717         case OPC_CMPV_S:
 718         case OPC_MUL_U:
 719         case OPC_MUL_S:
 720         case OPC_MULL_U:
 721         case OPC_CLZ_S:
 722         case OPC_ABSNEG_S:
 723         case OPC_AND_B:
 724         case OPC_OR_B:
 725         case OPC_NOT_B:
 726         case OPC_XOR_B:
 727         case OPC_BFREV_B:
 728         case OPC_CLZ_B:
 729         case OPC_SHL_B:
 730         case OPC_SHR_B:
 731         case OPC_ASHR_B:
 732         case OPC_MGEN_B:
 733         case OPC_GETBIT_B:
 734         case OPC_CBITS_B:
 735         case OPC_BARY_F:
 736                 return true;
 737
 738         default:
 739                 return false;
 740         }
 741 }
 742
 743
 744 /* map cat2 instruction to valid abs/neg flags: */
 745 static inline unsigned ir3_cat2_absneg(opc_t opc)
 746 {
 747         switch (opc) {
 748         case OPC_ADD_F:
 749         case OPC_MIN_F:
 750         case OPC_MAX_F:
 751         case OPC_MUL_F:
 752         case OPC_SIGN_F:
 753         case OPC_CMPS_F:
 754         case OPC_ABSNEG_F:
 755         case OPC_CMPV_F:
 756         case OPC_FLOOR_F:
 757         case OPC_CEIL_F:
 758         case OPC_RNDNE_F:
 759         case OPC_RNDAZ_F:
 760         case OPC_TRUNC_F:
 761         case OPC_BARY_F:
 762                 return IR3_REG_FABS | IR3_REG_FNEG;
 763
 764         case OPC_ADD_U:
 765         case OPC_ADD_S:
 766         case OPC_SUB_U:
 767         case OPC_SUB_S:
 768         case OPC_CMPS_U:
 769         case OPC_CMPS_S:
 770         case OPC_MIN_U:
 771         case OPC_MIN_S:
 772         case OPC_MAX_U:
 773         case OPC_MAX_S:
 774         case OPC_CMPV_U:
 775         case OPC_CMPV_S:
 776         case OPC_MUL_U:
 777         case OPC_MUL_S:
 778         case OPC_MULL_U:
 779         case OPC_CLZ_S:
 780                 return 0;
 781
 782         case OPC_ABSNEG_S:
 783                 return IR3_REG_SABS | IR3_REG_SNEG;
 784
 785         case OPC_AND_B:
 786         case OPC_OR_B:
 787         case OPC_NOT_B:
 788         case OPC_XOR_B:
 789         case OPC_BFREV_B:
 790         case OPC_CLZ_B:
 791         case OPC_SHL_B:
 792         case OPC_SHR_B:
 793         case OPC_ASHR_B:
 794         case OPC_MGEN_B:
 795         case OPC_GETBIT_B:
 796         case OPC_CBITS_B:
 797                 return IR3_REG_BNOT;
 798
 799         default:
 800                 return 0;
 801         }
 802 }
 803
 804 /* map cat3 instructions to valid abs/neg flags: */
 805 static inline unsigned ir3_cat3_absneg(opc_t opc)
 806 {
 807         switch (opc) {
 808         case OPC_MAD_F16:
 809         case OPC_MAD_F32:
 810         case OPC_SEL_F16:
 811         case OPC_SEL_F32:
 812                 return IR3_REG_FNEG;
 813
 814         case OPC_MAD_U16:
 815         case OPC_MADSH_U16:
 816         case OPC_MAD_S16:
 817         case OPC_MADSH_M16:
 818         case OPC_MAD_U24:
 819         case OPC_MAD_S24:
 820         case OPC_SEL_S16:
 821         case OPC_SEL_S32:
 822         case OPC_SAD_S16:
 823         case OPC_SAD_S32:
 824                 /* neg *may* work on 3rd src.. */
 825
 826         case OPC_SEL_B16:
 827         case OPC_SEL_B32:
 828
 829         default:
 830                 return 0;
 831         }
 832 }
 833
 834 #define array_insert(arr, val) do { \
 835                 if (arr ## _count == arr ## _sz) { \
 836                         arr ## _sz = MAX2(2 * arr ## _sz, 16); \
 837                         arr = realloc(arr, arr ## _sz * sizeof(arr[0])); \
 838                 } \
 839                 arr[arr ##_count++] = val; \
 840         } while (0)
 841
 842 /* iterator for an instructions's sources (reg), also returns src #: */
 843 #define foreach_src_n(__srcreg, __n, __instr) \
 844         if ((__instr)->regs_count) \
 845                 for (unsigned __cnt = (__instr)->regs_count - 1, __n = 0; __n < __cnt; __n++) \
 846                         if ((__srcreg = (__instr)->regs[__n + 1]))
 847
 848 /* iterator for an instructions's sources (reg): */
 849 #define foreach_src(__srcreg, __instr) \
 850         foreach_src_n(__srcreg, __i, __instr)
 851
 852 static inline unsigned __ssa_src_cnt(struct ir3_instruction *instr)
 853 {
 854         if (instr->address)
 855                 return instr->regs_count + 1;
 856         return instr->regs_count;
 857 }
 858
 859 static inline struct ir3_instruction * __ssa_src_n(struct ir3_instruction *instr, unsigned n)
 860 {
 861         if (n == (instr->regs_count + 0))
 862                 return instr->address;
 863         return ssa(instr->regs[n]);
 864 }
 865
 866 #define __src_cnt(__instr) ((__instr)->address ? (__instr)->regs_count : (__instr)->regs_count - 1)
 867
 868 /* iterator for an instruction's SSA sources (instr), also returns src #: */
 869 #define foreach_ssa_src_n(__srcinst, __n, __instr) \
 870         if ((__instr)->regs_count) \
 871                 for (unsigned __cnt = __ssa_src_cnt(__instr), __n = 0; __n < __cnt; __n++) \
 872                         if ((__srcinst = __ssa_src_n(__instr, __n)))
 873
 874 /* iterator for an instruction's SSA sources (instr): */
 875 #define foreach_ssa_src(__srcinst, __instr) \
 876         foreach_ssa_src_n(__srcinst, __i, __instr)
 877
 878
 879 /* dump: */
 880 void ir3_print(struct ir3 *ir);
 881 void ir3_print_instr(struct ir3_instruction *instr);
 882
 883 /* depth calculation: */
 884 int ir3_delayslots(struct ir3_instruction *assigner,
 885                 struct ir3_instruction *consumer, unsigned n);
 886 void ir3_insert_by_depth(struct ir3_instruction *instr, struct list_head *list);
 887 void ir3_depth(struct ir3 *ir);
 888
 889 /* copy-propagate: */
 890 void ir3_cp(struct ir3 *ir);
 891
 892 /* group neighbors and insert mov's to resolve conflicts: */
 893 void ir3_group(struct ir3 *ir);
 894
 895 /* scheduling: */
 896 int ir3_sched(struct ir3 *ir);
 897
 898 /* register assignment: */
 899 struct ir3_ra_reg_set * ir3_ra_alloc_reg_set(void *memctx);
 900 int ir3_ra(struct ir3 *ir3, enum shader_t type,
 901                 bool frag_coord, bool frag_face);
 902
 903 /* legalize: */
 904 void ir3_legalize(struct ir3 *ir, bool *has_samp, int *max_bary);
 905
 906 /* ************************************************************************* */
 907 /* instruction helpers */
 908
 909 static inline struct ir3_instruction *
 910 ir3_MOV(struct ir3_block *block, struct ir3_instruction *src, type_t type)
 911 {
 912         struct ir3_instruction *instr = ir3_instr_create(block, OPC_MOV);
 913         ir3_reg_create(instr, 0, 0);   /* dst */
 914         if (src->regs[0]->flags & IR3_REG_ARRAY) {
 915                 struct ir3_register *src_reg =
 916                         ir3_reg_create(instr, 0, IR3_REG_ARRAY);
 917                 src_reg->array = src->regs[0]->array;
 918                 src_reg->instr = src;
 919         } else {
 920                 ir3_reg_create(instr, 0, IR3_REG_SSA)->instr = src;
 921         }
 922         debug_assert(!(src->regs[0]->flags & IR3_REG_RELATIV));
 923         instr->cat1.src_type = type;
 924         instr->cat1.dst_type = type;
 925         return instr;
 926 }
 927
 928 static inline struct ir3_instruction *
 929 ir3_COV(struct ir3_block *block, struct ir3_instruction *src,
 930                 type_t src_type, type_t dst_type)
 931 {
 932         struct ir3_instruction *instr = ir3_instr_create(block, OPC_MOV);
 933         ir3_reg_create(instr, 0, 0);   /* dst */
 934         ir3_reg_create(instr, 0, IR3_REG_SSA)->instr = src;
 935         instr->cat1.src_type = src_type;
 936         instr->cat1.dst_type = dst_type;
 937         debug_assert(!(src->regs[0]->flags & IR3_REG_ARRAY));
 938         return instr;
 939 }
 940
 941 static inline struct ir3_instruction *
 942 ir3_NOP(struct ir3_block *block)
 943 {
 944         return ir3_instr_create(block, OPC_NOP);
 945 }
 946
 947 #define INSTR0(name)                                                     \
 948 static inline struct ir3_instruction *                                   \
 949 ir3_##name(struct ir3_block *block)                                      \
 950 {                                                                        \
 951         struct ir3_instruction *instr =                                      \
 952                 ir3_instr_create(block, OPC_##name);                             \
 953         return instr;                                                        \
 954 }
 955
 956 #define INSTR1(name)                                                     \
 957 static inline struct ir3_instruction *                                   \
 958 ir3_##name(struct ir3_block *block,                                      \
 959                 struct ir3_instruction *a, unsigned aflags)                      \
 960 {                                                                        \
 961         struct ir3_instruction *instr =                                      \
 962                 ir3_instr_create(block, OPC_##name);                             \
 963         ir3_reg_create(instr, 0, 0);   /* dst */                             \
 964         ir3_reg_create(instr, 0, IR3_REG_SSA | aflags)->instr = a;           \
 965         return instr;                                                        \
 966 }
 967
 968 #define INSTR2(name)                                                     \
 969 static inline struct ir3_instruction *                                   \
 970 ir3_##name(struct ir3_block *block,                                      \
 971                 struct ir3_instruction *a, unsigned aflags,                      \
 972                 struct ir3_instruction *b, unsigned bflags)                      \
 973 {                                                                        \
 974         struct ir3_instruction *instr =                                      \
 975                 ir3_instr_create(block, OPC_##name);                             \
 976         ir3_reg_create(instr, 0, 0);   /* dst */                             \
 977         ir3_reg_create(instr, 0, IR3_REG_SSA | aflags)->instr = a;           \
 978         ir3_reg_create(instr, 0, IR3_REG_SSA | bflags)->instr = b;           \
 979         return instr;                                                        \
 980 }
 981
 982 #define INSTR3(name)                                                     \
 983 static inline struct ir3_instruction *                                   \
 984 ir3_##name(struct ir3_block *block,                                      \
 985                 struct ir3_instruction *a, unsigned aflags,                      \
 986                 struct ir3_instruction *b, unsigned bflags,                      \
 987                 struct ir3_instruction *c, unsigned cflags)                      \
 988 {                                                                        \
 989         struct ir3_instruction *instr =                                      \
 990                 ir3_instr_create(block, OPC_##name);                             \
 991         ir3_reg_create(instr, 0, 0);   /* dst */                             \
 992         ir3_reg_create(instr, 0, IR3_REG_SSA | aflags)->instr = a;           \
 993         ir3_reg_create(instr, 0, IR3_REG_SSA | bflags)->instr = b;           \
 994         ir3_reg_create(instr, 0, IR3_REG_SSA | cflags)->instr = c;           \
 995         return instr;                                                        \
 996 }
 997
 998 /* cat0 instructions: */
 999 INSTR0(BR);
1000 INSTR0(JUMP);
1001 INSTR1(KILL);
1002 INSTR0(END);
1003
1004 /* cat2 instructions, most 2 src but some 1 src: */
1005 INSTR2(ADD_F)
1006 INSTR2(MIN_F)
1007 INSTR2(MAX_F)
1008 INSTR2(MUL_F)
1009 INSTR1(SIGN_F)
1010 INSTR2(CMPS_F)
1011 INSTR1(ABSNEG_F)
1012 INSTR2(CMPV_F)
1013 INSTR1(FLOOR_F)
1014 INSTR1(CEIL_F)
1015 INSTR1(RNDNE_F)
1016 INSTR1(RNDAZ_F)
1017 INSTR1(TRUNC_F)
1018 INSTR2(ADD_U)
1019 INSTR2(ADD_S)
1020 INSTR2(SUB_U)
1021 INSTR2(SUB_S)
1022 INSTR2(CMPS_U)
1023 INSTR2(CMPS_S)
1024 INSTR2(MIN_U)
1025 INSTR2(MIN_S)
1026 INSTR2(MAX_U)
1027 INSTR2(MAX_S)
1028 INSTR1(ABSNEG_S)
1029 INSTR2(AND_B)
1030 INSTR2(OR_B)
1031 INSTR1(NOT_B)
1032 INSTR2(XOR_B)
1033 INSTR2(CMPV_U)
1034 INSTR2(CMPV_S)
1035 INSTR2(MUL_U)
1036 INSTR2(MUL_S)
1037 INSTR2(MULL_U)
1038 INSTR1(BFREV_B)
1039 INSTR1(CLZ_S)
1040 INSTR1(CLZ_B)
1041 INSTR2(SHL_B)
1042 INSTR2(SHR_B)
1043 INSTR2(ASHR_B)
1044 INSTR2(BARY_F)
1045 INSTR2(MGEN_B)
1046 INSTR2(GETBIT_B)
1047 INSTR1(SETRM)
1048 INSTR1(CBITS_B)
1049 INSTR2(SHB)
1050 INSTR2(MSAD)
1051
1052 /* cat3 instructions: */
1053 INSTR3(MAD_U16)
1054 INSTR3(MADSH_U16)
1055 INSTR3(MAD_S16)
1056 INSTR3(MADSH_M16)
1057 INSTR3(MAD_U24)
1058 INSTR3(MAD_S24)
1059 INSTR3(MAD_F16)
1060 INSTR3(MAD_F32)
1061 INSTR3(SEL_B16)
1062 INSTR3(SEL_B32)
1063 INSTR3(SEL_S16)
1064 INSTR3(SEL_S32)
1065 INSTR3(SEL_F16)
1066 INSTR3(SEL_F32)
1067 INSTR3(SAD_S16)
1068 INSTR3(SAD_S32)
1069
1070 /* cat4 instructions: */
1071 INSTR1(RCP)
1072 INSTR1(RSQ)
1073 INSTR1(LOG2)
1074 INSTR1(EXP2)
1075 INSTR1(SIN)
1076 INSTR1(COS)
1077 INSTR1(SQRT)
1078
1079 /* cat5 instructions: */
1080 INSTR1(DSX)
1081 INSTR1(DSY)
1082
1083 static inline struct ir3_instruction *
1084 ir3_SAM(struct ir3_block *block, opc_t opc, type_t type,
1085                 unsigned wrmask, unsigned flags, unsigned samp, unsigned tex,
1086                 struct ir3_instruction *src0, struct ir3_instruction *src1)
1087 {
1088         struct ir3_instruction *sam;
1089         struct ir3_register *reg;
1090
1091         sam = ir3_instr_create(block, opc);
1092         sam->flags |= flags;
1093         ir3_reg_create(sam, 0, 0)->wrmask = wrmask;
1094         if (src0) {
1095                 reg = ir3_reg_create(sam, 0, IR3_REG_SSA);
1096                 reg->wrmask = (1 << (src0->regs_count - 1)) - 1;
1097                 reg->instr = src0;
1098         }
1099         if (src1) {
1100                 reg = ir3_reg_create(sam, 0, IR3_REG_SSA);
1101                 reg->instr = src1;
1102                 reg->wrmask = (1 << (src1->regs_count - 1)) - 1;
1103         }
1104         sam->cat5.samp = samp;
1105         sam->cat5.tex  = tex;
1106         sam->cat5.type  = type;
1107
1108         return sam;
1109 }
1110
1111 /* cat6 instructions: */
1112 INSTR2(LDLV)
1113 INSTR2(LDG)
1114 INSTR3(STG)
1115
1116 /* ************************************************************************* */
1117 /* split this out or find some helper to use.. like main/bitset.h.. */
1118
1119 #include <string.h>
1120
1121 #define MAX_REG 256
1122
1123 typedef uint8_t regmask_t[2 * MAX_REG / 8];
1124
1125 static inline unsigned regmask_idx(struct ir3_register *reg)
1126 {
1127         unsigned num = (reg->flags & IR3_REG_RELATIV) ? reg->array.offset : reg->num;
1128         debug_assert(num < MAX_REG);
1129         if (reg->flags & IR3_REG_HALF)
1130                 num += MAX_REG;
1131         return num;
1132 }
1133
1134 static inline void regmask_init(regmask_t *regmask)
1135 {
1136         memset(regmask, 0, sizeof(*regmask));
1137 }
1138
1139 static inline void regmask_set(regmask_t *regmask, struct ir3_register *reg)
1140 {
1141         unsigned idx = regmask_idx(reg);
1142         if (reg->flags & IR3_REG_RELATIV) {
1143                 unsigned i;
1144                 for (i = 0; i < reg->size; i++, idx++)
1145                         (*regmask)[idx / 8] |= 1 << (idx % 8);
1146         } else {
1147                 unsigned mask;
1148                 for (mask = reg->wrmask; mask; mask >>= 1, idx++)
1149                         if (mask & 1)
1150                                 (*regmask)[idx / 8] |= 1 << (idx % 8);
1151         }
1152 }
1153
1154 static inline void regmask_or(regmask_t *dst, regmask_t *a, regmask_t *b)
1155 {
1156         unsigned i;
1157         for (i = 0; i < ARRAY_SIZE(*dst); i++)
1158                 (*dst)[i] = (*a)[i] | (*b)[i];
1159 }
1160
1161 /* set bits in a if not set in b, conceptually:
1162  *   a |= (reg & ~b)
1163  */
1164 static inline void regmask_set_if_not(regmask_t *a,
1165                 struct ir3_register *reg, regmask_t *b)
1166 {
1167         unsigned idx = regmask_idx(reg);
1168         if (reg->flags & IR3_REG_RELATIV) {
1169                 unsigned i;
1170                 for (i = 0; i < reg->size; i++, idx++)
1171                         if (!((*b)[idx / 8] & (1 << (idx % 8))))
1172                                 (*a)[idx / 8] |= 1 << (idx % 8);
1173         } else {
1174                 unsigned mask;
1175                 for (mask = reg->wrmask; mask; mask >>= 1, idx++)
1176                         if (mask & 1)
1177                                 if (!((*b)[idx / 8] & (1 << (idx % 8))))
1178                                         (*a)[idx / 8] |= 1 << (idx % 8);
1179         }
1180 }
1181
1182 static inline bool regmask_get(regmask_t *regmask,
1183                 struct ir3_register *reg)
1184 {
1185         unsigned idx = regmask_idx(reg);
1186         if (reg->flags & IR3_REG_RELATIV) {
1187                 unsigned i;
1188                 for (i = 0; i < reg->size; i++, idx++)
1189                         if ((*regmask)[idx / 8] & (1 << (idx % 8)))
1190                                 return true;
1191         } else {
1192                 unsigned mask;
1193                 for (mask = reg->wrmask; mask; mask >>= 1, idx++)
1194                         if (mask & 1)
1195                                 if ((*regmask)[idx / 8] & (1 << (idx % 8)))
1196                                         return true;
1197         }
1198         return false;
1199 }
1200
1201 /* ************************************************************************* */
1202
1203 #endif /* IR3_H_ */