src/gallium/drivers/freedreno/ir3/ir3.h

   1 /*
   2  * Copyright (c) 2013 Rob Clark <robdclark@gmail.com>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  */
  23
  24 #ifndef IR3_H_
  25 #define IR3_H_
  26
  27 #include <stdint.h>
  28 #include <stdbool.h>
  29
  30 #include "util/u_debug.h"
  31 #include "util/list.h"
  32
  33 #include "instr-a3xx.h"
  34 #include "disasm.h"  /* TODO move 'enum shader_t' somewhere else.. */
  35
  36 /* low level intermediate representation of an adreno shader program */
  37
  38 struct ir3_compiler;
  39 struct ir3;
  40 struct ir3_instruction;
  41 struct ir3_block;
  42
  43 struct ir3_info {
  44         uint32_t gpu_id;
  45         uint16_t sizedwords;
  46         uint16_t instrs_count;   /* expanded to account for rpt's */
  47         /* NOTE: max_reg, etc, does not include registers not touched
  48          * by the shader (ie. vertex fetched via VFD_DECODE but not
  49          * touched by shader)
  50          */
  51         int8_t   max_reg;   /* highest GPR # used by shader */
  52         int8_t   max_half_reg;
  53         int16_t  max_const;
  54 };
  55
  56 struct ir3_register {
  57         enum {
  58                 IR3_REG_CONST  = 0x001,
  59                 IR3_REG_IMMED  = 0x002,
  60                 IR3_REG_HALF   = 0x004,
  61                 /* high registers are used for some things in compute shaders,
  62                  * for example.  Seems to be for things that are global to all
  63                  * threads in a wave, so possibly these are global/shared by
  64                  * all the threads in the wave?
  65                  */
  66                 IR3_REG_HIGH   = 0x008,
  67                 IR3_REG_RELATIV= 0x010,
  68                 IR3_REG_R      = 0x020,
  69                 /* Most instructions, it seems, can do float abs/neg but not
  70                  * integer.  The CP pass needs to know what is intended (int or
  71                  * float) in order to do the right thing.  For this reason the
  72                  * abs/neg flags are split out into float and int variants.  In
  73                  * addition, .b (bitwise) operations, the negate is actually a
  74                  * bitwise not, so split that out into a new flag to make it
  75                  * more clear.
  76                  */
  77                 IR3_REG_FNEG   = 0x040,
  78                 IR3_REG_FABS   = 0x080,
  79                 IR3_REG_SNEG   = 0x100,
  80                 IR3_REG_SABS   = 0x200,
  81                 IR3_REG_BNOT   = 0x400,
  82                 IR3_REG_EVEN   = 0x800,
  83                 IR3_REG_POS_INF= 0x1000,
  84                 /* (ei) flag, end-input?  Set on last bary, presumably to signal
  85                  * that the shader needs no more input:
  86                  */
  87                 IR3_REG_EI     = 0x2000,
  88                 /* meta-flags, for intermediate stages of IR, ie.
  89                  * before register assignment is done:
  90                  */
  91                 IR3_REG_SSA    = 0x4000,   /* 'instr' is ptr to assigning instr */
  92                 IR3_REG_ARRAY  = 0x8000,
  93                 IR3_REG_PHI_SRC= 0x10000,  /* phi src, regs[0]->instr points to phi */
  94
  95         } flags;
  96         union {
  97                 /* normal registers:
  98                  * the component is in the low two bits of the reg #, so
  99                  * rN.x becomes: (N << 2) | x
 100                  */
 101                 int   num;
 102                 /* immediate: */
 103                 int32_t  iim_val;
 104                 uint32_t uim_val;
 105                 float    fim_val;
 106                 /* relative: */
 107                 struct {
 108                         uint16_t id;
 109                         int16_t offset;
 110                 } array;
 111         };
 112
 113         /* For IR3_REG_SSA, src registers contain ptr back to assigning
 114          * instruction.
 115          *
 116          * For IR3_REG_ARRAY, the pointer is back to the last dependent
 117          * array access (although the net effect is the same, it points
 118          * back to a previous instruction that we depend on).
 119          */
 120         struct ir3_instruction *instr;
 121
 122         union {
 123                 /* used for cat5 instructions, but also for internal/IR level
 124                  * tracking of what registers are read/written by an instruction.
 125                  * wrmask may be a bad name since it is used to represent both
 126                  * src and dst that touch multiple adjacent registers.
 127                  */
 128                 unsigned wrmask;
 129                 /* for relative addressing, 32bits for array size is too small,
 130                  * but otoh we don't need to deal with disjoint sets, so instead
 131                  * use a simple size field (number of scalar components).
 132                  */
 133                 unsigned size;
 134         };
 135 };
 136
 137 struct ir3_instruction {
 138         struct ir3_block *block;
 139         opc_t opc;
 140         enum {
 141                 /* (sy) flag is set on first instruction, and after sample
 142                  * instructions (probably just on RAW hazard).
 143                  */
 144                 IR3_INSTR_SY    = 0x001,
 145                 /* (ss) flag is set on first instruction, and first instruction
 146                  * to depend on the result of "long" instructions (RAW hazard):
 147                  *
 148                  *   rcp, rsq, log2, exp2, sin, cos, sqrt
 149                  *
 150                  * It seems to synchronize until all in-flight instructions are
 151                  * completed, for example:
 152                  *
 153                  *   rsq hr1.w, hr1.w
 154                  *   add.f hr2.z, (neg)hr2.z, hc0.y
 155                  *   mul.f hr2.w, (neg)hr2.y, (neg)hr2.y
 156                  *   rsq hr2.x, hr2.x
 157                  *   (rpt1)nop
 158                  *   mad.f16 hr2.w, hr2.z, hr2.z, hr2.w
 159                  *   nop
 160                  *   mad.f16 hr2.w, (neg)hr0.w, (neg)hr0.w, hr2.w
 161                  *   (ss)(rpt2)mul.f hr1.x, (r)hr1.x, hr1.w
 162                  *   (rpt2)mul.f hr0.x, (neg)(r)hr0.x, hr2.x
 163                  *
 164                  * The last mul.f does not have (ss) set, presumably because the
 165                  * (ss) on the previous instruction does the job.
 166                  *
 167                  * The blob driver also seems to set it on WAR hazards, although
 168                  * not really clear if this is needed or just blob compiler being
 169                  * sloppy.  So far I haven't found a case where removing the (ss)
 170                  * causes problems for WAR hazard, but I could just be getting
 171                  * lucky:
 172                  *
 173                  *   rcp r1.y, r3.y
 174                  *   (ss)(rpt2)mad.f32 r3.y, (r)c9.x, r1.x, (r)r3.z
 175                  *
 176                  */
 177                 IR3_INSTR_SS    = 0x002,
 178                 /* (jp) flag is set on jump targets:
 179                  */
 180                 IR3_INSTR_JP    = 0x004,
 181                 IR3_INSTR_UL    = 0x008,
 182                 IR3_INSTR_3D    = 0x010,
 183                 IR3_INSTR_A     = 0x020,
 184                 IR3_INSTR_O     = 0x040,
 185                 IR3_INSTR_P     = 0x080,
 186                 IR3_INSTR_S     = 0x100,
 187                 IR3_INSTR_S2EN  = 0x200,
 188                 IR3_INSTR_G     = 0x400,
 189                 /* meta-flags, for intermediate stages of IR, ie.
 190                  * before register assignment is done:
 191                  */
 192                 IR3_INSTR_MARK  = 0x1000,
 193                 IR3_INSTR_UNUSED= 0x2000,
 194         } flags;
 195         int repeat;
 196 #ifdef DEBUG
 197         unsigned regs_max;
 198 #endif
 199         unsigned regs_count;
 200         struct ir3_register **regs;
 201         union {
 202                 struct {
 203                         char inv;
 204                         char comp;
 205                         int  immed;
 206                         struct ir3_block *target;
 207                 } cat0;
 208                 struct {
 209                         type_t src_type, dst_type;
 210                 } cat1;
 211                 struct {
 212                         enum {
 213                                 IR3_COND_LT = 0,
 214                                 IR3_COND_LE = 1,
 215                                 IR3_COND_GT = 2,
 216                                 IR3_COND_GE = 3,
 217                                 IR3_COND_EQ = 4,
 218                                 IR3_COND_NE = 5,
 219                         } condition;
 220                 } cat2;
 221                 struct {
 222                         unsigned samp, tex;
 223                         type_t type;
 224                 } cat5;
 225                 struct {
 226                         type_t type;
 227                         int src_offset;
 228                         int dst_offset;
 229                         int iim_val;          /* for ldgb/stgb, # of components */
 230                 } cat6;
 231                 /* for meta-instructions, just used to hold extra data
 232                  * before instruction scheduling, etc
 233                  */
 234                 struct {
 235                         int off;              /* component/offset */
 236                 } fo;
 237                 struct {
 238                         /* used to temporarily hold reference to nir_phi_instr
 239                          * until we resolve the phi srcs
 240                          */
 241                         void *nphi;
 242                 } phi;
 243                 struct {
 244                         struct ir3_block *block;
 245                 } inout;
 246         };
 247
 248         /* transient values used during various algorithms: */
 249         union {
 250                 /* The instruction depth is the max dependency distance to output.
 251                  *
 252                  * You can also think of it as the "cost", if we did any sort of
 253                  * optimization for register footprint.  Ie. a value that is  just
 254                  * result of moving a const to a reg would have a low cost,  so to
 255                  * it could make sense to duplicate the instruction at various
 256                  * points where the result is needed to reduce register footprint.
 257                  */
 258                 unsigned depth;
 259                 /* When we get to the RA stage, we no longer need depth, but
 260                  * we do need instruction's position/name:
 261                  */
 262                 struct {
 263                         uint16_t ip;
 264                         uint16_t name;
 265                 };
 266         };
 267
 268         /* used for per-pass extra instruction data.
 269          */
 270         void *data;
 271
 272         /* Used during CP and RA stages.  For fanin and shader inputs/
 273          * outputs where we need a sequence of consecutive registers,
 274          * keep track of each src instructions left (ie 'n-1') and right
 275          * (ie 'n+1') neighbor.  The front-end must insert enough mov's
 276          * to ensure that each instruction has at most one left and at
 277          * most one right neighbor.  During the copy-propagation pass,
 278          * we only remove mov's when we can preserve this constraint.
 279          * And during the RA stage, we use the neighbor information to
 280          * allocate a block of registers in one shot.
 281          *
 282          * TODO: maybe just add something like:
 283          *   struct ir3_instruction_ref {
 284          *       struct ir3_instruction *instr;
 285          *       unsigned cnt;
 286          *   }
 287          *
 288          * Or can we get away without the refcnt stuff?  It seems like
 289          * it should be overkill..  the problem is if, potentially after
 290          * already eliminating some mov's, if you have a single mov that
 291          * needs to be grouped with it's neighbors in two different
 292          * places (ex. shader output and a fanin).
 293          */
 294         struct {
 295                 struct ir3_instruction *left, *right;
 296                 uint16_t left_cnt, right_cnt;
 297         } cp;
 298
 299         /* an instruction can reference at most one address register amongst
 300          * it's src/dst registers.  Beyond that, you need to insert mov's.
 301          *
 302          * NOTE: do not write this directly, use ir3_instr_set_address()
 303          */
 304         struct ir3_instruction *address;
 305
 306         /* Entry in ir3_block's instruction list: */
 307         struct list_head node;
 308
 309 #ifdef DEBUG
 310         uint32_t serialno;
 311 #endif
 312 };
 313
 314 static inline struct ir3_instruction *
 315 ir3_neighbor_first(struct ir3_instruction *instr)
 316 {
 317         int cnt = 0;
 318         while (instr->cp.left) {
 319                 instr = instr->cp.left;
 320                 if (++cnt > 0xffff) {
 321                         debug_assert(0);
 322                         break;
 323                 }
 324         }
 325         return instr;
 326 }
 327
 328 static inline int ir3_neighbor_count(struct ir3_instruction *instr)
 329 {
 330         int num = 1;
 331
 332         debug_assert(!instr->cp.left);
 333
 334         while (instr->cp.right) {
 335                 num++;
 336                 instr = instr->cp.right;
 337                 if (num > 0xffff) {
 338                         debug_assert(0);
 339                         break;
 340                 }
 341         }
 342
 343         return num;
 344 }
 345
 346 /*
 347  * Stupid/simple growable array implementation:
 348  */
 349 #define DECLARE_ARRAY(type, name) \
 350         unsigned name ## _count, name ## _sz; \
 351         type * name;
 352
 353 #define array_insert(ctx, arr, val) do { \
 354                 if (arr ## _count == arr ## _sz) { \
 355                         arr ## _sz = MAX2(2 * arr ## _sz, 16); \
 356                         arr = reralloc_size(ctx, arr, arr ## _sz * sizeof(arr[0])); \
 357                 } \
 358                 arr[arr ##_count++] = val; \
 359         } while (0)
 360
 361 struct ir3 {
 362         struct ir3_compiler *compiler;
 363
 364         unsigned ninputs, noutputs;
 365         struct ir3_instruction **inputs;
 366         struct ir3_instruction **outputs;
 367
 368         /* Track bary.f (and ldlv) instructions.. this is needed in
 369          * scheduling to ensure that all varying fetches happen before
 370          * any potential kill instructions.  The hw gets grumpy if all
 371          * threads in a group are killed before the last bary.f gets
 372          * a chance to signal end of input (ei).
 373          */
 374         DECLARE_ARRAY(struct ir3_instruction *, baryfs);
 375
 376         /* Track all indirect instructions (read and write).  To avoid
 377          * deadlock scenario where an address register gets scheduled,
 378          * but other dependent src instructions cannot be scheduled due
 379          * to dependency on a *different* address register value, the
 380          * scheduler needs to ensure that all dependencies other than
 381          * the instruction other than the address register are scheduled
 382          * before the one that writes the address register.  Having a
 383          * convenient list of instructions that reference some address
 384          * register simplifies this.
 385          */
 386         DECLARE_ARRAY(struct ir3_instruction *, indirects);
 387
 388         /* and same for instructions that consume predicate register: */
 389         DECLARE_ARRAY(struct ir3_instruction *, predicates);
 390
 391         /* Track texture sample instructions which need texture state
 392          * patched in (for astc-srgb workaround):
 393          */
 394         DECLARE_ARRAY(struct ir3_instruction *, astc_srgb);
 395
 396         /* List of blocks: */
 397         struct list_head block_list;
 398
 399         /* List of ir3_array's: */
 400         struct list_head array_list;
 401 };
 402
 403 typedef struct nir_variable nir_variable;
 404
 405 struct ir3_array {
 406         struct list_head node;
 407         unsigned length;
 408         unsigned id;
 409
 410         nir_variable *var;
 411
 412         /* We track the last write and last access (read or write) to
 413          * setup dependencies on instructions that read or write the
 414          * array.  Reads can be re-ordered wrt. other reads, but should
 415          * not be re-ordered wrt. to writes.  Writes cannot be reordered
 416          * wrt. any other access to the array.
 417          *
 418          * So array reads depend on last write, and array writes depend
 419          * on the last access.
 420          */
 421         struct ir3_instruction *last_write, *last_access;
 422
 423         /* extra stuff used in RA pass: */
 424         unsigned base;      /* base vreg name */
 425         unsigned reg;       /* base physical reg */
 426         uint16_t start_ip, end_ip;
 427 };
 428
 429 struct ir3_array * ir3_lookup_array(struct ir3 *ir, unsigned id);
 430
 431 typedef struct nir_block nir_block;
 432
 433 struct ir3_block {
 434         struct list_head node;
 435         struct ir3 *shader;
 436
 437         nir_block *nblock;
 438
 439         struct list_head instr_list;  /* list of ir3_instruction */
 440
 441         /* each block has either one or two successors.. in case of
 442          * two successors, 'condition' decides which one to follow.
 443          * A block preceding an if/else has two successors.
 444          */
 445         struct ir3_instruction *condition;
 446         struct ir3_block *successors[2];
 447
 448         uint16_t start_ip, end_ip;
 449
 450         /* Track instructions which do not write a register but other-
 451          * wise must not be discarded (such as kill, stg, etc)
 452          */
 453         DECLARE_ARRAY(struct ir3_instruction *, keeps);
 454
 455         /* used for per-pass extra block data.  Mainly used right
 456          * now in RA step to track livein/liveout.
 457          */
 458         void *data;
 459
 460 #ifdef DEBUG
 461         uint32_t serialno;
 462 #endif
 463 };
 464
 465 static inline uint32_t
 466 block_id(struct ir3_block *block)
 467 {
 468 #ifdef DEBUG
 469         return block->serialno;
 470 #else
 471         return (uint32_t)(unsigned long)block;
 472 #endif
 473 }
 474
 475 struct ir3 * ir3_create(struct ir3_compiler *compiler,
 476                 unsigned nin, unsigned nout);
 477 void ir3_destroy(struct ir3 *shader);
 478 void * ir3_assemble(struct ir3 *shader,
 479                 struct ir3_info *info, uint32_t gpu_id);
 480 void * ir3_alloc(struct ir3 *shader, int sz);
 481
 482 struct ir3_block * ir3_block_create(struct ir3 *shader);
 483
 484 struct ir3_instruction * ir3_instr_create(struct ir3_block *block, opc_t opc);
 485 struct ir3_instruction * ir3_instr_create2(struct ir3_block *block,
 486                 opc_t opc, int nreg);
 487 struct ir3_instruction * ir3_instr_clone(struct ir3_instruction *instr);
 488 const char *ir3_instr_name(struct ir3_instruction *instr);
 489
 490 struct ir3_register * ir3_reg_create(struct ir3_instruction *instr,
 491                 int num, int flags);
 492 struct ir3_register * ir3_reg_clone(struct ir3 *shader,
 493                 struct ir3_register *reg);
 494
 495 void ir3_instr_set_address(struct ir3_instruction *instr,
 496                 struct ir3_instruction *addr);
 497
 498 static inline bool ir3_instr_check_mark(struct ir3_instruction *instr)
 499 {
 500         if (instr->flags & IR3_INSTR_MARK)
 501                 return true;  /* already visited */
 502         instr->flags |= IR3_INSTR_MARK;
 503         return false;
 504 }
 505
 506 void ir3_block_clear_mark(struct ir3_block *block);
 507 void ir3_clear_mark(struct ir3 *shader);
 508
 509 unsigned ir3_count_instructions(struct ir3 *ir);
 510
 511 static inline int ir3_instr_regno(struct ir3_instruction *instr,
 512                 struct ir3_register *reg)
 513 {
 514         unsigned i;
 515         for (i = 0; i < instr->regs_count; i++)
 516                 if (reg == instr->regs[i])
 517                         return i;
 518         return -1;
 519 }
 520
 521
 522 #define MAX_ARRAYS 16
 523
 524 /* comp:
 525  *   0 - x
 526  *   1 - y
 527  *   2 - z
 528  *   3 - w
 529  */
 530 static inline uint32_t regid(int num, int comp)
 531 {
 532         return (num << 2) | (comp & 0x3);
 533 }
 534
 535 static inline uint32_t reg_num(struct ir3_register *reg)
 536 {
 537         return reg->num >> 2;
 538 }
 539
 540 static inline uint32_t reg_comp(struct ir3_register *reg)
 541 {
 542         return reg->num & 0x3;
 543 }
 544
 545 static inline bool is_flow(struct ir3_instruction *instr)
 546 {
 547         return (opc_cat(instr->opc) == 0);
 548 }
 549
 550 static inline bool is_kill(struct ir3_instruction *instr)
 551 {
 552         return instr->opc == OPC_KILL;
 553 }
 554
 555 static inline bool is_nop(struct ir3_instruction *instr)
 556 {
 557         return instr->opc == OPC_NOP;
 558 }
 559
 560 /* Is it a non-transformative (ie. not type changing) mov?  This can
 561  * also include absneg.s/absneg.f, which for the most part can be
 562  * treated as a mov (single src argument).
 563  */
 564 static inline bool is_same_type_mov(struct ir3_instruction *instr)
 565 {
 566         struct ir3_register *dst = instr->regs[0];
 567
 568         /* mov's that write to a0.x or p0.x are special: */
 569         if (dst->num == regid(REG_P0, 0))
 570                 return false;
 571         if (dst->num == regid(REG_A0, 0))
 572                 return false;
 573
 574         if (dst->flags & (IR3_REG_RELATIV | IR3_REG_ARRAY))
 575                 return false;
 576
 577         switch (instr->opc) {
 578         case OPC_MOV:
 579                 return instr->cat1.src_type == instr->cat1.dst_type;
 580         case OPC_ABSNEG_F:
 581         case OPC_ABSNEG_S:
 582                 return true;
 583         default:
 584                 return false;
 585         }
 586 }
 587
 588 static inline bool is_alu(struct ir3_instruction *instr)
 589 {
 590         return (1 <= opc_cat(instr->opc)) && (opc_cat(instr->opc) <= 3);
 591 }
 592
 593 static inline bool is_sfu(struct ir3_instruction *instr)
 594 {
 595         return (opc_cat(instr->opc) == 4);
 596 }
 597
 598 static inline bool is_tex(struct ir3_instruction *instr)
 599 {
 600         return (opc_cat(instr->opc) == 5);
 601 }
 602
 603 static inline bool is_mem(struct ir3_instruction *instr)
 604 {
 605         return (opc_cat(instr->opc) == 6);
 606 }
 607
 608 static inline bool
 609 is_store(struct ir3_instruction *instr)
 610 {
 611         /* these instructions, the "destination" register is
 612          * actually a source, the address to store to.
 613          */
 614         switch (instr->opc) {
 615         case OPC_STG:
 616         case OPC_STGB:
 617         case OPC_STP:
 618         case OPC_STL:
 619         case OPC_STLW:
 620         case OPC_L2G:
 621         case OPC_G2L:
 622                 return true;
 623         default:
 624                 return false;
 625         }
 626 }
 627
 628 static inline bool is_load(struct ir3_instruction *instr)
 629 {
 630         switch (instr->opc) {
 631         case OPC_LDG:
 632         case OPC_LDGB:
 633         case OPC_LDL:
 634         case OPC_LDP:
 635         case OPC_L2G:
 636         case OPC_LDLW:
 637         case OPC_LDC:
 638         case OPC_LDLV:
 639                 /* probably some others too.. */
 640                 return true;
 641         default:
 642                 return false;
 643         }
 644 }
 645
 646 static inline bool is_input(struct ir3_instruction *instr)
 647 {
 648         /* in some cases, ldlv is used to fetch varying without
 649          * interpolation.. fortunately inloc is the first src
 650          * register in either case
 651          */
 652         switch (instr->opc) {
 653         case OPC_LDLV:
 654         case OPC_BARY_F:
 655                 return true;
 656         default:
 657                 return false;
 658         }
 659 }
 660
 661 static inline bool is_bool(struct ir3_instruction *instr)
 662 {
 663         switch (instr->opc) {
 664         case OPC_CMPS_F:
 665         case OPC_CMPS_S:
 666         case OPC_CMPS_U:
 667                 return true;
 668         default:
 669                 return false;
 670         }
 671 }
 672
 673 static inline bool is_meta(struct ir3_instruction *instr)
 674 {
 675         /* TODO how should we count PHI (and maybe fan-in/out) which
 676          * might actually contribute some instructions to the final
 677          * result?
 678          */
 679         return (opc_cat(instr->opc) == -1);
 680 }
 681
 682 static inline bool writes_addr(struct ir3_instruction *instr)
 683 {
 684         if (instr->regs_count > 0) {
 685                 struct ir3_register *dst = instr->regs[0];
 686                 return reg_num(dst) == REG_A0;
 687         }
 688         return false;
 689 }
 690
 691 static inline bool writes_pred(struct ir3_instruction *instr)
 692 {
 693         if (instr->regs_count > 0) {
 694                 struct ir3_register *dst = instr->regs[0];
 695                 return reg_num(dst) == REG_P0;
 696         }
 697         return false;
 698 }
 699
 700 /* returns defining instruction for reg */
 701 /* TODO better name */
 702 static inline struct ir3_instruction *ssa(struct ir3_register *reg)
 703 {
 704         if (reg->flags & (IR3_REG_SSA | IR3_REG_ARRAY)) {
 705                 debug_assert(!(reg->instr && (reg->instr->flags & IR3_INSTR_UNUSED)));
 706                 return reg->instr;
 707         }
 708         return NULL;
 709 }
 710
 711 static inline bool conflicts(struct ir3_instruction *a,
 712                 struct ir3_instruction *b)
 713 {
 714         return (a && b) && (a != b);
 715 }
 716
 717 static inline bool reg_gpr(struct ir3_register *r)
 718 {
 719         if (r->flags & (IR3_REG_CONST | IR3_REG_IMMED))
 720                 return false;
 721         if ((reg_num(r) == REG_A0) || (reg_num(r) == REG_P0))
 722                 return false;
 723         return true;
 724 }
 725
 726 static inline type_t half_type(type_t type)
 727 {
 728         switch (type) {
 729         case TYPE_F32: return TYPE_F16;
 730         case TYPE_U32: return TYPE_U16;
 731         case TYPE_S32: return TYPE_S16;
 732         case TYPE_F16:
 733         case TYPE_U16:
 734         case TYPE_S16:
 735                 return type;
 736         default:
 737                 assert(0);
 738                 return ~0;
 739         }
 740 }
 741
 742 /* some cat2 instructions (ie. those which are not float) can embed an
 743  * immediate:
 744  */
 745 static inline bool ir3_cat2_int(opc_t opc)
 746 {
 747         switch (opc) {
 748         case OPC_ADD_U:
 749         case OPC_ADD_S:
 750         case OPC_SUB_U:
 751         case OPC_SUB_S:
 752         case OPC_CMPS_U:
 753         case OPC_CMPS_S:
 754         case OPC_MIN_U:
 755         case OPC_MIN_S:
 756         case OPC_MAX_U:
 757         case OPC_MAX_S:
 758         case OPC_CMPV_U:
 759         case OPC_CMPV_S:
 760         case OPC_MUL_U:
 761         case OPC_MUL_S:
 762         case OPC_MULL_U:
 763         case OPC_CLZ_S:
 764         case OPC_ABSNEG_S:
 765         case OPC_AND_B:
 766         case OPC_OR_B:
 767         case OPC_NOT_B:
 768         case OPC_XOR_B:
 769         case OPC_BFREV_B:
 770         case OPC_CLZ_B:
 771         case OPC_SHL_B:
 772         case OPC_SHR_B:
 773         case OPC_ASHR_B:
 774         case OPC_MGEN_B:
 775         case OPC_GETBIT_B:
 776         case OPC_CBITS_B:
 777         case OPC_BARY_F:
 778                 return true;
 779
 780         default:
 781                 return false;
 782         }
 783 }
 784
 785
 786 /* map cat2 instruction to valid abs/neg flags: */
 787 static inline unsigned ir3_cat2_absneg(opc_t opc)
 788 {
 789         switch (opc) {
 790         case OPC_ADD_F:
 791         case OPC_MIN_F:
 792         case OPC_MAX_F:
 793         case OPC_MUL_F:
 794         case OPC_SIGN_F:
 795         case OPC_CMPS_F:
 796         case OPC_ABSNEG_F:
 797         case OPC_CMPV_F:
 798         case OPC_FLOOR_F:
 799         case OPC_CEIL_F:
 800         case OPC_RNDNE_F:
 801         case OPC_RNDAZ_F:
 802         case OPC_TRUNC_F:
 803         case OPC_BARY_F:
 804                 return IR3_REG_FABS | IR3_REG_FNEG;
 805
 806         case OPC_ADD_U:
 807         case OPC_ADD_S:
 808         case OPC_SUB_U:
 809         case OPC_SUB_S:
 810         case OPC_CMPS_U:
 811         case OPC_CMPS_S:
 812         case OPC_MIN_U:
 813         case OPC_MIN_S:
 814         case OPC_MAX_U:
 815         case OPC_MAX_S:
 816         case OPC_CMPV_U:
 817         case OPC_CMPV_S:
 818         case OPC_MUL_U:
 819         case OPC_MUL_S:
 820         case OPC_MULL_U:
 821         case OPC_CLZ_S:
 822                 return 0;
 823
 824         case OPC_ABSNEG_S:
 825                 return IR3_REG_SABS | IR3_REG_SNEG;
 826
 827         case OPC_AND_B:
 828         case OPC_OR_B:
 829         case OPC_NOT_B:
 830         case OPC_XOR_B:
 831         case OPC_BFREV_B:
 832         case OPC_CLZ_B:
 833         case OPC_SHL_B:
 834         case OPC_SHR_B:
 835         case OPC_ASHR_B:
 836         case OPC_MGEN_B:
 837         case OPC_GETBIT_B:
 838         case OPC_CBITS_B:
 839                 return IR3_REG_BNOT;
 840
 841         default:
 842                 return 0;
 843         }
 844 }
 845
 846 /* map cat3 instructions to valid abs/neg flags: */
 847 static inline unsigned ir3_cat3_absneg(opc_t opc)
 848 {
 849         switch (opc) {
 850         case OPC_MAD_F16:
 851         case OPC_MAD_F32:
 852         case OPC_SEL_F16:
 853         case OPC_SEL_F32:
 854                 return IR3_REG_FNEG;
 855
 856         case OPC_MAD_U16:
 857         case OPC_MADSH_U16:
 858         case OPC_MAD_S16:
 859         case OPC_MADSH_M16:
 860         case OPC_MAD_U24:
 861         case OPC_MAD_S24:
 862         case OPC_SEL_S16:
 863         case OPC_SEL_S32:
 864         case OPC_SAD_S16:
 865         case OPC_SAD_S32:
 866                 /* neg *may* work on 3rd src.. */
 867
 868         case OPC_SEL_B16:
 869         case OPC_SEL_B32:
 870
 871         default:
 872                 return 0;
 873         }
 874 }
 875
 876 /* iterator for an instructions's sources (reg), also returns src #: */
 877 #define foreach_src_n(__srcreg, __n, __instr) \
 878         if ((__instr)->regs_count) \
 879                 for (unsigned __cnt = (__instr)->regs_count - 1, __n = 0; __n < __cnt; __n++) \
 880                         if ((__srcreg = (__instr)->regs[__n + 1]))
 881
 882 /* iterator for an instructions's sources (reg): */
 883 #define foreach_src(__srcreg, __instr) \
 884         foreach_src_n(__srcreg, __i, __instr)
 885
 886 static inline unsigned __ssa_src_cnt(struct ir3_instruction *instr)
 887 {
 888         if (instr->address)
 889                 return instr->regs_count + 1;
 890         return instr->regs_count;
 891 }
 892
 893 static inline struct ir3_instruction * __ssa_src_n(struct ir3_instruction *instr, unsigned n)
 894 {
 895         if (n == (instr->regs_count + 0))
 896                 return instr->address;
 897         return ssa(instr->regs[n]);
 898 }
 899
 900 #define __src_cnt(__instr) ((__instr)->address ? (__instr)->regs_count : (__instr)->regs_count - 1)
 901
 902 /* iterator for an instruction's SSA sources (instr), also returns src #: */
 903 #define foreach_ssa_src_n(__srcinst, __n, __instr) \
 904         if ((__instr)->regs_count) \
 905                 for (unsigned __cnt = __ssa_src_cnt(__instr), __n = 0; __n < __cnt; __n++) \
 906                         if ((__srcinst = __ssa_src_n(__instr, __n)))
 907
 908 /* iterator for an instruction's SSA sources (instr): */
 909 #define foreach_ssa_src(__srcinst, __instr) \
 910         foreach_ssa_src_n(__srcinst, __i, __instr)
 911
 912
 913 /* dump: */
 914 void ir3_print(struct ir3 *ir);
 915 void ir3_print_instr(struct ir3_instruction *instr);
 916
 917 /* depth calculation: */
 918 int ir3_delayslots(struct ir3_instruction *assigner,
 919                 struct ir3_instruction *consumer, unsigned n);
 920 void ir3_insert_by_depth(struct ir3_instruction *instr, struct list_head *list);
 921 void ir3_depth(struct ir3 *ir);
 922
 923 /* copy-propagate: */
 924 struct ir3_shader_variant;
 925 void ir3_cp(struct ir3 *ir, struct ir3_shader_variant *so);
 926
 927 /* group neighbors and insert mov's to resolve conflicts: */
 928 void ir3_group(struct ir3 *ir);
 929
 930 /* scheduling: */
 931 int ir3_sched(struct ir3 *ir);
 932
 933 /* register assignment: */
 934 struct ir3_ra_reg_set * ir3_ra_alloc_reg_set(void *memctx);
 935 int ir3_ra(struct ir3 *ir3, enum shader_t type,
 936                 bool frag_coord, bool frag_face);
 937
 938 /* legalize: */
 939 void ir3_legalize(struct ir3 *ir, bool *has_samp, bool *has_ssbo, int *max_bary);
 940
 941 /* ************************************************************************* */
 942 /* instruction helpers */
 943
 944 static inline struct ir3_instruction *
 945 ir3_MOV(struct ir3_block *block, struct ir3_instruction *src, type_t type)
 946 {
 947         struct ir3_instruction *instr = ir3_instr_create(block, OPC_MOV);
 948         ir3_reg_create(instr, 0, 0);   /* dst */
 949         if (src->regs[0]->flags & IR3_REG_ARRAY) {
 950                 struct ir3_register *src_reg =
 951                         ir3_reg_create(instr, 0, IR3_REG_ARRAY);
 952                 src_reg->array = src->regs[0]->array;
 953                 src_reg->instr = src;
 954         } else {
 955                 ir3_reg_create(instr, 0, IR3_REG_SSA)->instr = src;
 956         }
 957         debug_assert(!(src->regs[0]->flags & IR3_REG_RELATIV));
 958         instr->cat1.src_type = type;
 959         instr->cat1.dst_type = type;
 960         return instr;
 961 }
 962
 963 static inline struct ir3_instruction *
 964 ir3_COV(struct ir3_block *block, struct ir3_instruction *src,
 965                 type_t src_type, type_t dst_type)
 966 {
 967         struct ir3_instruction *instr = ir3_instr_create(block, OPC_MOV);
 968         ir3_reg_create(instr, 0, 0);   /* dst */
 969         ir3_reg_create(instr, 0, IR3_REG_SSA)->instr = src;
 970         instr->cat1.src_type = src_type;
 971         instr->cat1.dst_type = dst_type;
 972         debug_assert(!(src->regs[0]->flags & IR3_REG_ARRAY));
 973         return instr;
 974 }
 975
 976 static inline struct ir3_instruction *
 977 ir3_NOP(struct ir3_block *block)
 978 {
 979         return ir3_instr_create(block, OPC_NOP);
 980 }
 981
 982 #define INSTR0(name)                                                     \
 983 static inline struct ir3_instruction *                                   \
 984 ir3_##name(struct ir3_block *block)                                      \
 985 {                                                                        \
 986         struct ir3_instruction *instr =                                      \
 987                 ir3_instr_create(block, OPC_##name);                             \
 988         return instr;                                                        \
 989 }
 990
 991 #define INSTR1(name)                                                     \
 992 static inline struct ir3_instruction *                                   \
 993 ir3_##name(struct ir3_block *block,                                      \
 994                 struct ir3_instruction *a, unsigned aflags)                      \
 995 {                                                                        \
 996         struct ir3_instruction *instr =                                      \
 997                 ir3_instr_create(block, OPC_##name);                             \
 998         ir3_reg_create(instr, 0, 0);   /* dst */                             \
 999         ir3_reg_create(instr, 0, IR3_REG_SSA | aflags)->instr = a;           \
1000         return instr;                                                        \
1001 }
1002
1003 #define INSTR2(name)                                                     \
1004 static inline struct ir3_instruction *                                   \
1005 ir3_##name(struct ir3_block *block,                                      \
1006                 struct ir3_instruction *a, unsigned aflags,                      \
1007                 struct ir3_instruction *b, unsigned bflags)                      \
1008 {                                                                        \
1009         struct ir3_instruction *instr =                                      \
1010                 ir3_instr_create(block, OPC_##name);                             \
1011         ir3_reg_create(instr, 0, 0);   /* dst */                             \
1012         ir3_reg_create(instr, 0, IR3_REG_SSA | aflags)->instr = a;           \
1013         ir3_reg_create(instr, 0, IR3_REG_SSA | bflags)->instr = b;           \
1014         return instr;                                                        \
1015 }
1016
1017 #define INSTR3(name)                                                     \
1018 static inline struct ir3_instruction *                                   \
1019 ir3_##name(struct ir3_block *block,                                      \
1020                 struct ir3_instruction *a, unsigned aflags,                      \
1021                 struct ir3_instruction *b, unsigned bflags,                      \
1022                 struct ir3_instruction *c, unsigned cflags)                      \
1023 {                                                                        \
1024         struct ir3_instruction *instr =                                      \
1025                 ir3_instr_create(block, OPC_##name);                             \
1026         ir3_reg_create(instr, 0, 0);   /* dst */                             \
1027         ir3_reg_create(instr, 0, IR3_REG_SSA | aflags)->instr = a;           \
1028         ir3_reg_create(instr, 0, IR3_REG_SSA | bflags)->instr = b;           \
1029         ir3_reg_create(instr, 0, IR3_REG_SSA | cflags)->instr = c;           \
1030         return instr;                                                        \
1031 }
1032
1033 #define INSTR4(name)                                                     \
1034 static inline struct ir3_instruction *                                   \
1035 ir3_##name(struct ir3_block *block,                                      \
1036                 struct ir3_instruction *a, unsigned aflags,                      \
1037                 struct ir3_instruction *b, unsigned bflags,                      \
1038                 struct ir3_instruction *c, unsigned cflags,                      \
1039                 struct ir3_instruction *d, unsigned dflags)                      \
1040 {                                                                        \
1041         struct ir3_instruction *instr =                                      \
1042                 ir3_instr_create2(block, OPC_##name, 5);                         \
1043         ir3_reg_create(instr, 0, 0);   /* dst */                             \
1044         ir3_reg_create(instr, 0, IR3_REG_SSA | aflags)->instr = a;           \
1045         ir3_reg_create(instr, 0, IR3_REG_SSA | bflags)->instr = b;           \
1046         ir3_reg_create(instr, 0, IR3_REG_SSA | cflags)->instr = c;           \
1047         ir3_reg_create(instr, 0, IR3_REG_SSA | dflags)->instr = d;           \
1048         return instr;                                                        \
1049 }
1050
1051 /* cat0 instructions: */
1052 INSTR0(BR);
1053 INSTR0(JUMP);
1054 INSTR1(KILL);
1055 INSTR0(END);
1056
1057 /* cat2 instructions, most 2 src but some 1 src: */
1058 INSTR2(ADD_F)
1059 INSTR2(MIN_F)
1060 INSTR2(MAX_F)
1061 INSTR2(MUL_F)
1062 INSTR1(SIGN_F)
1063 INSTR2(CMPS_F)
1064 INSTR1(ABSNEG_F)
1065 INSTR2(CMPV_F)
1066 INSTR1(FLOOR_F)
1067 INSTR1(CEIL_F)
1068 INSTR1(RNDNE_F)
1069 INSTR1(RNDAZ_F)
1070 INSTR1(TRUNC_F)
1071 INSTR2(ADD_U)
1072 INSTR2(ADD_S)
1073 INSTR2(SUB_U)
1074 INSTR2(SUB_S)
1075 INSTR2(CMPS_U)
1076 INSTR2(CMPS_S)
1077 INSTR2(MIN_U)
1078 INSTR2(MIN_S)
1079 INSTR2(MAX_U)
1080 INSTR2(MAX_S)
1081 INSTR1(ABSNEG_S)
1082 INSTR2(AND_B)
1083 INSTR2(OR_B)
1084 INSTR1(NOT_B)
1085 INSTR2(XOR_B)
1086 INSTR2(CMPV_U)
1087 INSTR2(CMPV_S)
1088 INSTR2(MUL_U)
1089 INSTR2(MUL_S)
1090 INSTR2(MULL_U)
1091 INSTR1(BFREV_B)
1092 INSTR1(CLZ_S)
1093 INSTR1(CLZ_B)
1094 INSTR2(SHL_B)
1095 INSTR2(SHR_B)
1096 INSTR2(ASHR_B)
1097 INSTR2(BARY_F)
1098 INSTR2(MGEN_B)
1099 INSTR2(GETBIT_B)
1100 INSTR1(SETRM)
1101 INSTR1(CBITS_B)
1102 INSTR2(SHB)
1103 INSTR2(MSAD)
1104
1105 /* cat3 instructions: */
1106 INSTR3(MAD_U16)
1107 INSTR3(MADSH_U16)
1108 INSTR3(MAD_S16)
1109 INSTR3(MADSH_M16)
1110 INSTR3(MAD_U24)
1111 INSTR3(MAD_S24)
1112 INSTR3(MAD_F16)
1113 INSTR3(MAD_F32)
1114 INSTR3(SEL_B16)
1115 INSTR3(SEL_B32)
1116 INSTR3(SEL_S16)
1117 INSTR3(SEL_S32)
1118 INSTR3(SEL_F16)
1119 INSTR3(SEL_F32)
1120 INSTR3(SAD_S16)
1121 INSTR3(SAD_S32)
1122
1123 /* cat4 instructions: */
1124 INSTR1(RCP)
1125 INSTR1(RSQ)
1126 INSTR1(LOG2)
1127 INSTR1(EXP2)
1128 INSTR1(SIN)
1129 INSTR1(COS)
1130 INSTR1(SQRT)
1131
1132 /* cat5 instructions: */
1133 INSTR1(DSX)
1134 INSTR1(DSY)
1135
1136 static inline struct ir3_instruction *
1137 ir3_SAM(struct ir3_block *block, opc_t opc, type_t type,
1138                 unsigned wrmask, unsigned flags, unsigned samp, unsigned tex,
1139                 struct ir3_instruction *src0, struct ir3_instruction *src1)
1140 {
1141         struct ir3_instruction *sam;
1142         struct ir3_register *reg;
1143
1144         sam = ir3_instr_create(block, opc);
1145         sam->flags |= flags;
1146         ir3_reg_create(sam, 0, 0)->wrmask = wrmask;
1147         if (src0) {
1148                 reg = ir3_reg_create(sam, 0, IR3_REG_SSA);
1149                 reg->wrmask = (1 << (src0->regs_count - 1)) - 1;
1150                 reg->instr = src0;
1151         }
1152         if (src1) {
1153                 reg = ir3_reg_create(sam, 0, IR3_REG_SSA);
1154                 reg->instr = src1;
1155                 reg->wrmask = (1 << (src1->regs_count - 1)) - 1;
1156         }
1157         sam->cat5.samp = samp;
1158         sam->cat5.tex  = tex;
1159         sam->cat5.type  = type;
1160
1161         return sam;
1162 }
1163
1164 /* cat6 instructions: */
1165 INSTR2(LDLV)
1166 INSTR2(LDG)
1167 INSTR3(STG)
1168 INSTR3(LDGB);
1169 INSTR4(STGB);
1170 INSTR4(ATOMIC_ADD);
1171 INSTR4(ATOMIC_SUB);
1172 INSTR4(ATOMIC_XCHG);
1173 INSTR4(ATOMIC_INC);
1174 INSTR4(ATOMIC_DEC);
1175 INSTR4(ATOMIC_CMPXCHG);
1176 INSTR4(ATOMIC_MIN);
1177 INSTR4(ATOMIC_MAX);
1178 INSTR4(ATOMIC_AND);
1179 INSTR4(ATOMIC_OR);
1180 INSTR4(ATOMIC_XOR);
1181
1182 /* ************************************************************************* */
1183 /* split this out or find some helper to use.. like main/bitset.h.. */
1184
1185 #include <string.h>
1186
1187 #define MAX_REG 256
1188
1189 typedef uint8_t regmask_t[2 * MAX_REG / 8];
1190
1191 static inline unsigned regmask_idx(struct ir3_register *reg)
1192 {
1193         unsigned num = (reg->flags & IR3_REG_RELATIV) ? reg->array.offset : reg->num;
1194         debug_assert(num < MAX_REG);
1195         if (reg->flags & IR3_REG_HALF)
1196                 num += MAX_REG;
1197         return num;
1198 }
1199
1200 static inline void regmask_init(regmask_t *regmask)
1201 {
1202         memset(regmask, 0, sizeof(*regmask));
1203 }
1204
1205 static inline void regmask_set(regmask_t *regmask, struct ir3_register *reg)
1206 {
1207         unsigned idx = regmask_idx(reg);
1208         if (reg->flags & IR3_REG_RELATIV) {
1209                 unsigned i;
1210                 for (i = 0; i < reg->size; i++, idx++)
1211                         (*regmask)[idx / 8] |= 1 << (idx % 8);
1212         } else {
1213                 unsigned mask;
1214                 for (mask = reg->wrmask; mask; mask >>= 1, idx++)
1215                         if (mask & 1)
1216                                 (*regmask)[idx / 8] |= 1 << (idx % 8);
1217         }
1218 }
1219
1220 static inline void regmask_or(regmask_t *dst, regmask_t *a, regmask_t *b)
1221 {
1222         unsigned i;
1223         for (i = 0; i < ARRAY_SIZE(*dst); i++)
1224                 (*dst)[i] = (*a)[i] | (*b)[i];
1225 }
1226
1227 /* set bits in a if not set in b, conceptually:
1228  *   a |= (reg & ~b)
1229  */
1230 static inline void regmask_set_if_not(regmask_t *a,
1231                 struct ir3_register *reg, regmask_t *b)
1232 {
1233         unsigned idx = regmask_idx(reg);
1234         if (reg->flags & IR3_REG_RELATIV) {
1235                 unsigned i;
1236                 for (i = 0; i < reg->size; i++, idx++)
1237                         if (!((*b)[idx / 8] & (1 << (idx % 8))))
1238                                 (*a)[idx / 8] |= 1 << (idx % 8);
1239         } else {
1240                 unsigned mask;
1241                 for (mask = reg->wrmask; mask; mask >>= 1, idx++)
1242                         if (mask & 1)
1243                                 if (!((*b)[idx / 8] & (1 << (idx % 8))))
1244                                         (*a)[idx / 8] |= 1 << (idx % 8);
1245         }
1246 }
1247
1248 static inline bool regmask_get(regmask_t *regmask,
1249                 struct ir3_register *reg)
1250 {
1251         unsigned idx = regmask_idx(reg);
1252         if (reg->flags & IR3_REG_RELATIV) {
1253                 unsigned i;
1254                 for (i = 0; i < reg->size; i++, idx++)
1255                         if ((*regmask)[idx / 8] & (1 << (idx % 8)))
1256                                 return true;
1257         } else {
1258                 unsigned mask;
1259                 for (mask = reg->wrmask; mask; mask >>= 1, idx++)
1260                         if (mask & 1)
1261                                 if ((*regmask)[idx / 8] & (1 << (idx % 8)))
1262                                         return true;
1263         }
1264         return false;
1265 }
1266
1267 /* ************************************************************************* */
1268
1269 #endif /* IR3_H_ */