src/gallium/drivers/panfrost/midgard/midgard_compile.c

   1 /*
   2  * Copyright (C) 2018 Alyssa Rosenzweig <alyssa@rosenzweig.io>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  */
  23
  24 #include <sys/types.h>
  25 #include <sys/stat.h>
  26 #include <sys/mman.h>
  27 #include <fcntl.h>
  28 #include <stdint.h>
  29 #include <stdlib.h>
  30 #include <stdio.h>
  31 #include <err.h>
  32
  33 #include "main/mtypes.h"
  34 #include "compiler/glsl/glsl_to_nir.h"
  35 #include "compiler/nir_types.h"
  36 #include "main/imports.h"
  37 #include "compiler/nir/nir_builder.h"
  38 #include "util/half_float.h"
  39 #include "util/register_allocate.h"
  40 #include "util/u_debug.h"
  41 #include "util/u_dynarray.h"
  42 #include "util/list.h"
  43 #include "main/mtypes.h"
  44
  45 #include "midgard.h"
  46 #include "midgard_nir.h"
  47 #include "midgard_compile.h"
  48 #include "helpers.h"
  49
  50 #include "disassemble.h"
  51
  52 static const struct debug_named_value debug_options[] = {
  53         {"msgs",      MIDGARD_DBG_MSGS,         "Print debug messages"},
  54         {"shaders",   MIDGARD_DBG_SHADERS,      "Dump shaders in NIR and MIR"},
  55         DEBUG_NAMED_VALUE_END
  56 };
  57
  58 DEBUG_GET_ONCE_FLAGS_OPTION(midgard_debug, "MIDGARD_MESA_DEBUG", debug_options, 0)
  59
  60 int midgard_debug = 0;
  61
  62 #define DBG(fmt, ...) \
  63                 do { if (midgard_debug & MIDGARD_DBG_MSGS) \
  64                         fprintf(stderr, "%s:%d: "fmt, \
  65                                 __FUNCTION__, __LINE__, ##__VA_ARGS__); } while (0)
  66
  67 /* Instruction arguments represented as block-local SSA indices, rather than
  68  * registers. Negative values mean unused. */
  69
  70 typedef struct {
  71         int src0;
  72         int src1;
  73         int dest;
  74
  75         /* src1 is -not- SSA but instead a 16-bit inline constant to be smudged
  76          * in. Only valid for ALU ops. */
  77         bool inline_constant;
  78 } ssa_args;
  79
  80 /* Forward declare so midgard_branch can reference */
  81 struct midgard_block;
  82
  83 /* Target types. Defaults to TARGET_GOTO (the type corresponding directly to
  84  * the hardware), hence why that must be zero. TARGET_DISCARD signals this
  85  * instruction is actually a discard op. */
  86
  87 #define TARGET_GOTO 0
  88 #define TARGET_BREAK 1
  89 #define TARGET_CONTINUE 2
  90 #define TARGET_DISCARD 3
  91
  92 typedef struct midgard_branch {
  93         /* If conditional, the condition is specified in r31.w */
  94         bool conditional;
  95
  96         /* For conditionals, if this is true, we branch on FALSE. If false, we  branch on TRUE. */
  97         bool invert_conditional;
  98
  99         /* Branch targets: the start of a block, the start of a loop (continue), the end of a loop (break). Value is one of TARGET_ */
 100         unsigned target_type;
 101
 102         /* The actual target */
 103         union {
 104                 int target_block;
 105                 int target_break;
 106                 int target_continue;
 107         };
 108 } midgard_branch;
 109
 110 static bool
 111 midgard_is_branch_unit(unsigned unit)
 112 {
 113         return (unit == ALU_ENAB_BRANCH) || (unit == ALU_ENAB_BR_COMPACT);
 114 }
 115
 116 /* Generic in-memory data type repesenting a single logical instruction, rather
 117  * than a single instruction group. This is the preferred form for code gen.
 118  * Multiple midgard_insturctions will later be combined during scheduling,
 119  * though this is not represented in this structure.  Its format bridges
 120  * the low-level binary representation with the higher level semantic meaning.
 121  *
 122  * Notably, it allows registers to be specified as block local SSA, for code
 123  * emitted before the register allocation pass.
 124  */
 125
 126 typedef struct midgard_instruction {
 127         /* Must be first for casting */
 128         struct list_head link;
 129
 130         unsigned type; /* ALU, load/store, texture */
 131
 132         /* If the register allocator has not run yet... */
 133         ssa_args ssa_args;
 134
 135         /* Special fields for an ALU instruction */
 136         midgard_reg_info registers;
 137
 138         /* I.e. (1 << alu_bit) */
 139         int unit;
 140
 141         bool has_constants;
 142         float constants[4];
 143         uint16_t inline_constant;
 144         bool has_blend_constant;
 145
 146         bool compact_branch;
 147         bool writeout;
 148         bool prepacked_branch;
 149
 150         union {
 151                 midgard_load_store_word load_store;
 152                 midgard_vector_alu alu;
 153                 midgard_texture_word texture;
 154                 midgard_branch_extended branch_extended;
 155                 uint16_t br_compact;
 156
 157                 /* General branch, rather than packed br_compact. Higher level
 158                  * than the other components */
 159                 midgard_branch branch;
 160         };
 161 } midgard_instruction;
 162
 163 typedef struct midgard_block {
 164         /* Link to next block. Must be first for mir_get_block */
 165         struct list_head link;
 166
 167         /* List of midgard_instructions emitted for the current block */
 168         struct list_head instructions;
 169
 170         bool is_scheduled;
 171
 172         /* List of midgard_bundles emitted (after the scheduler has run) */
 173         struct util_dynarray bundles;
 174
 175         /* Number of quadwords _actually_ emitted, as determined after scheduling */
 176         unsigned quadword_count;
 177
 178         /* Successors: always one forward (the block after us), maybe
 179          * one backwards (for a backward branch). No need for a second
 180          * forward, since graph traversal would get there eventually
 181          * anyway */
 182         struct midgard_block *successors[2];
 183         unsigned nr_successors;
 184
 185         /* The successors pointer form a graph, and in the case of
 186          * complex control flow, this graph has a cycles. To aid
 187          * traversal during liveness analysis, we have a visited?
 188          * boolean for passes to use as they see fit, provided they
 189          * clean up later */
 190         bool visited;
 191 } midgard_block;
 192
 193 static void
 194 midgard_block_add_successor(midgard_block *block, midgard_block *successor)
 195 {
 196         block->successors[block->nr_successors++] = successor;
 197         assert(block->nr_successors <= ARRAY_SIZE(block->successors));
 198 }
 199
 200 /* Helpers to generate midgard_instruction's using macro magic, since every
 201  * driver seems to do it that way */
 202
 203 #define EMIT(op, ...) emit_mir_instruction(ctx, v_##op(__VA_ARGS__));
 204 #define SWIZZLE_XYZW SWIZZLE(COMPONENT_X, COMPONENT_Y, COMPONENT_Z, COMPONENT_W)
 205
 206 #define M_LOAD_STORE(name, rname, uname) \
 207         static midgard_instruction m_##name(unsigned ssa, unsigned address) { \
 208                 midgard_instruction i = { \
 209                         .type = TAG_LOAD_STORE_4, \
 210                         .ssa_args = { \
 211                                 .rname = ssa, \
 212                                 .uname = -1, \
 213                                 .src1 = -1 \
 214                         }, \
 215                         .load_store = { \
 216                                 .op = midgard_op_##name, \
 217                                 .mask = 0xF, \
 218                                 .swizzle = SWIZZLE_XYZW, \
 219                                 .address = address \
 220                         } \
 221                 }; \
 222                 \
 223                 return i; \
 224         }
 225
 226 #define M_LOAD(name) M_LOAD_STORE(name, dest, src0)
 227 #define M_STORE(name) M_LOAD_STORE(name, src0, dest)
 228
 229 const midgard_vector_alu_src blank_alu_src = {
 230         .swizzle = SWIZZLE(COMPONENT_X, COMPONENT_Y, COMPONENT_Z, COMPONENT_W),
 231 };
 232
 233 const midgard_vector_alu_src blank_alu_src_xxxx = {
 234         .swizzle = SWIZZLE(COMPONENT_X, COMPONENT_X, COMPONENT_X, COMPONENT_X),
 235 };
 236
 237 const midgard_scalar_alu_src blank_scalar_alu_src = {
 238         .full = true
 239 };
 240
 241 /* Used for encoding the unused source of 1-op instructions */
 242 const midgard_vector_alu_src zero_alu_src = { 0 };
 243
 244 /* Coerce structs to integer */
 245
 246 static unsigned
 247 vector_alu_srco_unsigned(midgard_vector_alu_src src)
 248 {
 249         unsigned u;
 250         memcpy(&u, &src, sizeof(src));
 251         return u;
 252 }
 253
 254 static midgard_vector_alu_src
 255 vector_alu_from_unsigned(unsigned u)
 256 {
 257         midgard_vector_alu_src s;
 258         memcpy(&s, &u, sizeof(s));
 259         return s;
 260 }
 261
 262 /* Inputs a NIR ALU source, with modifiers attached if necessary, and outputs
 263  * the corresponding Midgard source */
 264
 265 static midgard_vector_alu_src
 266 vector_alu_modifiers(nir_alu_src *src, bool is_int)
 267 {
 268         if (!src) return blank_alu_src;
 269
 270         midgard_vector_alu_src alu_src = {
 271                 .rep_low = 0,
 272                 .rep_high = 0,
 273                 .half = 0, /* TODO */
 274                 .swizzle = SWIZZLE_FROM_ARRAY(src->swizzle)
 275         };
 276
 277         if (is_int) {
 278                 /* TODO: sign-extend/zero-extend */
 279                 alu_src.mod = midgard_int_normal;
 280
 281                 /* These should have been lowered away */
 282                 assert(!(src->abs || src->negate));
 283         } else {
 284                 alu_src.mod = (src->abs << 0) | (src->negate << 1);
 285         }
 286
 287         return alu_src;
 288 }
 289
 290 static bool
 291 mir_nontrivial_mod(midgard_vector_alu_src src, bool is_int, unsigned mask)
 292 {
 293         /* abs or neg */
 294         if (!is_int && src.mod) return true;
 295
 296         /* swizzle */
 297         for (unsigned c = 0; c < 4; ++c) {
 298                 if (!(mask & (1 << c))) continue;
 299                 if (((src.swizzle >> (2*c)) & 3) != c) return true;
 300         }
 301
 302         return false;
 303 }
 304
 305 /* 'Intrinsic' move for misc aliasing uses independent of actual NIR ALU code */
 306
 307 static midgard_instruction
 308 v_fmov(unsigned src, midgard_vector_alu_src mod, unsigned dest)
 309 {
 310         midgard_instruction ins = {
 311                 .type = TAG_ALU_4,
 312                 .ssa_args = {
 313                         .src0 = SSA_UNUSED_1,
 314                         .src1 = src,
 315                         .dest = dest,
 316                 },
 317                 .alu = {
 318                         .op = midgard_alu_op_fmov,
 319                         .reg_mode = midgard_reg_mode_full,
 320                         .dest_override = midgard_dest_override_none,
 321                         .mask = 0xFF,
 322                         .src1 = vector_alu_srco_unsigned(zero_alu_src),
 323                         .src2 = vector_alu_srco_unsigned(mod)
 324                 },
 325         };
 326
 327         return ins;
 328 }
 329
 330 /* load/store instructions have both 32-bit and 16-bit variants, depending on
 331  * whether we are using vectors composed of highp or mediump. At the moment, we
 332  * don't support half-floats -- this requires changes in other parts of the
 333  * compiler -- therefore the 16-bit versions are commented out. */
 334
 335 //M_LOAD(load_attr_16);
 336 M_LOAD(load_attr_32);
 337 //M_LOAD(load_vary_16);
 338 M_LOAD(load_vary_32);
 339 //M_LOAD(load_uniform_16);
 340 M_LOAD(load_uniform_32);
 341 M_LOAD(load_color_buffer_8);
 342 //M_STORE(store_vary_16);
 343 M_STORE(store_vary_32);
 344 M_STORE(store_cubemap_coords);
 345
 346 static midgard_instruction
 347 v_alu_br_compact_cond(midgard_jmp_writeout_op op, unsigned tag, signed offset, unsigned cond)
 348 {
 349         midgard_branch_cond branch = {
 350                 .op = op,
 351                 .dest_tag = tag,
 352                 .offset = offset,
 353                 .cond = cond
 354         };
 355
 356         uint16_t compact;
 357         memcpy(&compact, &branch, sizeof(branch));
 358
 359         midgard_instruction ins = {
 360                 .type = TAG_ALU_4,
 361                 .unit = ALU_ENAB_BR_COMPACT,
 362                 .prepacked_branch = true,
 363                 .compact_branch = true,
 364                 .br_compact = compact
 365         };
 366
 367         if (op == midgard_jmp_writeout_op_writeout)
 368                 ins.writeout = true;
 369
 370         return ins;
 371 }
 372
 373 static midgard_instruction
 374 v_branch(bool conditional, bool invert)
 375 {
 376         midgard_instruction ins = {
 377                 .type = TAG_ALU_4,
 378                 .unit = ALU_ENAB_BRANCH,
 379                 .compact_branch = true,
 380                 .branch = {
 381                         .conditional = conditional,
 382                         .invert_conditional = invert
 383                 }
 384         };
 385
 386         return ins;
 387 }
 388
 389 static midgard_branch_extended
 390 midgard_create_branch_extended( midgard_condition cond,
 391                                 midgard_jmp_writeout_op op,
 392                                 unsigned dest_tag,
 393                                 signed quadword_offset)
 394 {
 395         /* For unclear reasons, the condition code is repeated 8 times */
 396         uint16_t duplicated_cond =
 397                 (cond << 14) |
 398                 (cond << 12) |
 399                 (cond << 10) |
 400                 (cond << 8) |
 401                 (cond << 6) |
 402                 (cond << 4) |
 403                 (cond << 2) |
 404                 (cond << 0);
 405
 406         midgard_branch_extended branch = {
 407                 .op = op,
 408                 .dest_tag = dest_tag,
 409                 .offset = quadword_offset,
 410                 .cond = duplicated_cond
 411         };
 412
 413         return branch;
 414 }
 415
 416 typedef struct midgard_bundle {
 417         /* Tag for the overall bundle */
 418         int tag;
 419
 420         /* Instructions contained by the bundle */
 421         int instruction_count;
 422         midgard_instruction instructions[5];
 423
 424         /* Bundle-wide ALU configuration */
 425         int padding;
 426         int control;
 427         bool has_embedded_constants;
 428         float constants[4];
 429         bool has_blend_constant;
 430
 431         uint16_t register_words[8];
 432         int register_words_count;
 433
 434         uint64_t body_words[8];
 435         size_t body_size[8];
 436         int body_words_count;
 437 } midgard_bundle;
 438
 439 typedef struct compiler_context {
 440         nir_shader *nir;
 441         gl_shader_stage stage;
 442
 443         /* Is internally a blend shader? Depends on stage == FRAGMENT */
 444         bool is_blend;
 445
 446         /* Tracking for blend constant patching */
 447         int blend_constant_number;
 448         int blend_constant_offset;
 449
 450         /* Current NIR function */
 451         nir_function *func;
 452
 453         /* Unordered list of midgard_blocks */
 454         int block_count;
 455         struct list_head blocks;
 456
 457         midgard_block *initial_block;
 458         midgard_block *previous_source_block;
 459         midgard_block *final_block;
 460
 461         /* List of midgard_instructions emitted for the current block */
 462         midgard_block *current_block;
 463
 464         /* The current "depth" of the loop, for disambiguating breaks/continues
 465          * when using nested loops */
 466         int current_loop_depth;
 467
 468         /* Constants which have been loaded, for later inlining */
 469         struct hash_table_u64 *ssa_constants;
 470
 471         /* SSA indices to be outputted to corresponding varying offset */
 472         struct hash_table_u64 *ssa_varyings;
 473
 474         /* SSA values / registers which have been aliased. Naively, these
 475          * demand a fmov output; instead, we alias them in a later pass to
 476          * avoid the wasted op.
 477          *
 478          * A note on encoding: to avoid dynamic memory management here, rather
 479          * than ampping to a pointer, we map to the source index; the key
 480          * itself is just the destination index. */
 481
 482         struct hash_table_u64 *ssa_to_alias;
 483         struct set *leftover_ssa_to_alias;
 484
 485         /* Actual SSA-to-register for RA */
 486         struct hash_table_u64 *ssa_to_register;
 487
 488         /* Mapping of hashes computed from NIR indices to the sequential temp indices ultimately used in MIR */
 489         struct hash_table_u64 *hash_to_temp;
 490         int temp_count;
 491         int max_hash;
 492
 493         /* Just the count of the max register used. Higher count => higher
 494          * register pressure */
 495         int work_registers;
 496
 497         /* Used for cont/last hinting. Increase when a tex op is added.
 498          * Decrease when a tex op is removed. */
 499         int texture_op_count;
 500
 501         /* Mapping of texture register -> SSA index for unaliasing */
 502         int texture_index[2];
 503
 504         /* If any path hits a discard instruction */
 505         bool can_discard;
 506
 507         /* The number of uniforms allowable for the fast path */
 508         int uniform_cutoff;
 509
 510         /* Count of instructions emitted from NIR overall, across all blocks */
 511         int instruction_count;
 512
 513         /* Alpha ref value passed in */
 514         float alpha_ref;
 515
 516         /* The index corresponding to the fragment output */
 517         unsigned fragment_output;
 518
 519         /* The mapping of sysvals to uniforms, the count, and the off-by-one inverse */
 520         unsigned sysvals[MAX_SYSVAL_COUNT];
 521         unsigned sysval_count;
 522         struct hash_table_u64 *sysval_to_id;
 523 } compiler_context;
 524
 525 /* Append instruction to end of current block */
 526
 527 static midgard_instruction *
 528 mir_upload_ins(struct midgard_instruction ins)
 529 {
 530         midgard_instruction *heap = malloc(sizeof(ins));
 531         memcpy(heap, &ins, sizeof(ins));
 532         return heap;
 533 }
 534
 535 static void
 536 emit_mir_instruction(struct compiler_context *ctx, struct midgard_instruction ins)
 537 {
 538         list_addtail(&(mir_upload_ins(ins))->link, &ctx->current_block->instructions);
 539 }
 540
 541 static void
 542 mir_insert_instruction_before(struct midgard_instruction *tag, struct midgard_instruction ins)
 543 {
 544         list_addtail(&(mir_upload_ins(ins))->link, &tag->link);
 545 }
 546
 547 static void
 548 mir_remove_instruction(struct midgard_instruction *ins)
 549 {
 550         list_del(&ins->link);
 551 }
 552
 553 static midgard_instruction*
 554 mir_prev_op(struct midgard_instruction *ins)
 555 {
 556         return list_last_entry(&(ins->link), midgard_instruction, link);
 557 }
 558
 559 static midgard_instruction*
 560 mir_next_op(struct midgard_instruction *ins)
 561 {
 562         return list_first_entry(&(ins->link), midgard_instruction, link);
 563 }
 564
 565 #define mir_foreach_block(ctx, v) list_for_each_entry(struct midgard_block, v, &ctx->blocks, link)
 566 #define mir_foreach_block_from(ctx, from, v) list_for_each_entry_from(struct midgard_block, v, from, &ctx->blocks, link)
 567
 568 #define mir_foreach_instr(ctx, v) list_for_each_entry(struct midgard_instruction, v, &ctx->current_block->instructions, link)
 569 #define mir_foreach_instr_safe(ctx, v) list_for_each_entry_safe(struct midgard_instruction, v, &ctx->current_block->instructions, link)
 570 #define mir_foreach_instr_in_block(block, v) list_for_each_entry(struct midgard_instruction, v, &block->instructions, link)
 571 #define mir_foreach_instr_in_block_safe(block, v) list_for_each_entry_safe(struct midgard_instruction, v, &block->instructions, link)
 572 #define mir_foreach_instr_in_block_safe_rev(block, v) list_for_each_entry_safe_rev(struct midgard_instruction, v, &block->instructions, link)
 573 #define mir_foreach_instr_in_block_from(block, v, from) list_for_each_entry_from(struct midgard_instruction, v, from, &block->instructions, link)
 574 #define mir_foreach_instr_in_block_from_rev(block, v, from) list_for_each_entry_from_rev(struct midgard_instruction, v, from, &block->instructions, link)
 575
 576
 577 static midgard_instruction *
 578 mir_last_in_block(struct midgard_block *block)
 579 {
 580         return list_last_entry(&block->instructions, struct midgard_instruction, link);
 581 }
 582
 583 static midgard_block *
 584 mir_get_block(compiler_context *ctx, int idx)
 585 {
 586         struct list_head *lst = &ctx->blocks;
 587
 588         while ((idx--) + 1)
 589                 lst = lst->next;
 590
 591         return (struct midgard_block *) lst;
 592 }
 593
 594 /* Pretty printer for internal Midgard IR */
 595
 596 static void
 597 print_mir_source(int source)
 598 {
 599         if (source >= SSA_FIXED_MINIMUM) {
 600                 /* Specific register */
 601                 int reg = SSA_REG_FROM_FIXED(source);
 602
 603                 /* TODO: Moving threshold */
 604                 if (reg > 16 && reg < 24)
 605                         printf("u%d", 23 - reg);
 606                 else
 607                         printf("r%d", reg);
 608         } else {
 609                 printf("%d", source);
 610         }
 611 }
 612
 613 static void
 614 print_mir_instruction(midgard_instruction *ins)
 615 {
 616         printf("\t");
 617
 618         switch (ins->type) {
 619         case TAG_ALU_4: {
 620                 midgard_alu_op op = ins->alu.op;
 621                 const char *name = alu_opcode_props[op].name;
 622
 623                 if (ins->unit)
 624                         printf("%d.", ins->unit);
 625
 626                 printf("%s", name ? name : "??");
 627                 break;
 628         }
 629
 630         case TAG_LOAD_STORE_4: {
 631                 midgard_load_store_op op = ins->load_store.op;
 632                 const char *name = load_store_opcode_names[op];
 633
 634                 assert(name);
 635                 printf("%s", name);
 636                 break;
 637         }
 638
 639         case TAG_TEXTURE_4: {
 640                 printf("texture");
 641                 break;
 642         }
 643
 644         default:
 645                 assert(0);
 646         }
 647
 648         ssa_args *args = &ins->ssa_args;
 649
 650         printf(" %d, ", args->dest);
 651
 652         print_mir_source(args->src0);
 653         printf(", ");
 654
 655         if (args->inline_constant)
 656                 printf("#%d", ins->inline_constant);
 657         else
 658                 print_mir_source(args->src1);
 659
 660         if (ins->has_constants)
 661                 printf(" <%f, %f, %f, %f>", ins->constants[0], ins->constants[1], ins->constants[2], ins->constants[3]);
 662
 663         printf("\n");
 664 }
 665
 666 static void
 667 print_mir_block(midgard_block *block)
 668 {
 669         printf("{\n");
 670
 671         mir_foreach_instr_in_block(block, ins) {
 672                 print_mir_instruction(ins);
 673         }
 674
 675         printf("}\n");
 676 }
 677
 678 static void
 679 attach_constants(compiler_context *ctx, midgard_instruction *ins, void *constants, int name)
 680 {
 681         ins->has_constants = true;
 682         memcpy(&ins->constants, constants, 16);
 683
 684         /* If this is the special blend constant, mark this instruction */
 685
 686         if (ctx->is_blend && ctx->blend_constant_number == name)
 687                 ins->has_blend_constant = true;
 688 }
 689
 690 static int
 691 glsl_type_size(const struct glsl_type *type, bool bindless)
 692 {
 693         return glsl_count_attribute_slots(type, false);
 694 }
 695
 696 /* Lower fdot2 to a vector multiplication followed by channel addition  */
 697 static void
 698 midgard_nir_lower_fdot2_body(nir_builder *b, nir_alu_instr *alu)
 699 {
 700         if (alu->op != nir_op_fdot2)
 701                 return;
 702
 703         b->cursor = nir_before_instr(&alu->instr);
 704
 705         nir_ssa_def *src0 = nir_ssa_for_alu_src(b, alu, 0);
 706         nir_ssa_def *src1 = nir_ssa_for_alu_src(b, alu, 1);
 707
 708         nir_ssa_def *product = nir_fmul(b, src0, src1);
 709
 710         nir_ssa_def *sum = nir_fadd(b,
 711                         nir_channel(b, product, 0),
 712                         nir_channel(b, product, 1));
 713
 714         /* Replace the fdot2 with this sum */
 715         nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(sum));
 716 }
 717
 718 static int
 719 midgard_nir_sysval_for_intrinsic(nir_intrinsic_instr *instr)
 720 {
 721         switch (instr->intrinsic) {
 722         case nir_intrinsic_load_viewport_scale:
 723                 return PAN_SYSVAL_VIEWPORT_SCALE;
 724         case nir_intrinsic_load_viewport_offset:
 725                 return PAN_SYSVAL_VIEWPORT_OFFSET;
 726         default:
 727                 return -1;
 728         }
 729 }
 730
 731 static void
 732 midgard_nir_assign_sysval_body(compiler_context *ctx, nir_instr *instr)
 733 {
 734         int sysval = -1;
 735
 736         if (instr->type == nir_instr_type_intrinsic) {
 737                 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
 738                 sysval = midgard_nir_sysval_for_intrinsic(intr);
 739         }
 740
 741         if (sysval < 0)
 742                 return;
 743
 744         /* We have a sysval load; check if it's already been assigned */
 745
 746         if (_mesa_hash_table_u64_search(ctx->sysval_to_id, sysval))
 747                 return;
 748
 749         /* It hasn't -- so assign it now! */
 750
 751         unsigned id = ctx->sysval_count++;
 752         _mesa_hash_table_u64_insert(ctx->sysval_to_id, sysval, (void *) ((uintptr_t) id + 1));
 753         ctx->sysvals[id] = sysval;
 754 }
 755
 756 static void
 757 midgard_nir_assign_sysvals(compiler_context *ctx, nir_shader *shader)
 758 {
 759         ctx->sysval_count = 0;
 760
 761         nir_foreach_function(function, shader) {
 762                 if (!function->impl) continue;
 763
 764                 nir_foreach_block(block, function->impl) {
 765                         nir_foreach_instr_safe(instr, block) {
 766                                 midgard_nir_assign_sysval_body(ctx, instr);
 767                         }
 768                 }
 769         }
 770 }
 771
 772 static bool
 773 midgard_nir_lower_fdot2(nir_shader *shader)
 774 {
 775         bool progress = false;
 776
 777         nir_foreach_function(function, shader) {
 778                 if (!function->impl) continue;
 779
 780                 nir_builder _b;
 781                 nir_builder *b = &_b;
 782                 nir_builder_init(b, function->impl);
 783
 784                 nir_foreach_block(block, function->impl) {
 785                         nir_foreach_instr_safe(instr, block) {
 786                                 if (instr->type != nir_instr_type_alu) continue;
 787
 788                                 nir_alu_instr *alu = nir_instr_as_alu(instr);
 789                                 midgard_nir_lower_fdot2_body(b, alu);
 790
 791                                 progress |= true;
 792                         }
 793                 }
 794
 795                 nir_metadata_preserve(function->impl, nir_metadata_block_index | nir_metadata_dominance);
 796
 797         }
 798
 799         return progress;
 800 }
 801
 802 static void
 803 optimise_nir(nir_shader *nir)
 804 {
 805         bool progress;
 806
 807         NIR_PASS(progress, nir, nir_lower_regs_to_ssa);
 808         NIR_PASS(progress, nir, midgard_nir_lower_fdot2);
 809
 810         nir_lower_tex_options lower_tex_options = {
 811                 .lower_rect = true
 812         };
 813
 814         NIR_PASS(progress, nir, nir_lower_tex, &lower_tex_options);
 815
 816         do {
 817                 progress = false;
 818
 819                 NIR_PASS(progress, nir, nir_lower_var_copies);
 820                 NIR_PASS(progress, nir, nir_lower_vars_to_ssa);
 821
 822                 NIR_PASS(progress, nir, nir_copy_prop);
 823                 NIR_PASS(progress, nir, nir_opt_dce);
 824                 NIR_PASS(progress, nir, nir_opt_dead_cf);
 825                 NIR_PASS(progress, nir, nir_opt_cse);
 826                 NIR_PASS(progress, nir, nir_opt_peephole_select, 64, false, true);
 827                 NIR_PASS(progress, nir, nir_opt_algebraic);
 828                 NIR_PASS(progress, nir, nir_opt_constant_folding);
 829                 NIR_PASS(progress, nir, nir_opt_undef);
 830                 NIR_PASS(progress, nir, nir_opt_loop_unroll,
 831                          nir_var_shader_in |
 832                          nir_var_shader_out |
 833                          nir_var_function_temp);
 834
 835                 /* TODO: Enable vectorize when merged upstream */
 836                 // NIR_PASS(progress, nir, nir_opt_vectorize);
 837         } while (progress);
 838
 839         /* Must be run at the end to prevent creation of fsin/fcos ops */
 840         NIR_PASS(progress, nir, midgard_nir_scale_trig);
 841
 842         do {
 843                 progress = false;
 844
 845                 NIR_PASS(progress, nir, nir_opt_dce);
 846                 NIR_PASS(progress, nir, nir_opt_algebraic);
 847                 NIR_PASS(progress, nir, nir_opt_constant_folding);
 848                 NIR_PASS(progress, nir, nir_copy_prop);
 849         } while (progress);
 850
 851         NIR_PASS(progress, nir, nir_opt_algebraic_late);
 852         NIR_PASS(progress, nir, midgard_nir_lower_algebraic_late);
 853
 854         /* Lower mods for float ops only. Integer ops don't support modifiers
 855          * (saturate doesn't make sense on integers, neg/abs require dedicated
 856          * instructions) */
 857
 858         NIR_PASS(progress, nir, nir_lower_to_source_mods, nir_lower_float_source_mods);
 859         NIR_PASS(progress, nir, nir_copy_prop);
 860         NIR_PASS(progress, nir, nir_opt_dce);
 861
 862         /* We implement booleans as 32-bit 0/~0 */
 863         NIR_PASS(progress, nir, nir_lower_bool_to_int32);
 864
 865         /* Take us out of SSA */
 866         NIR_PASS(progress, nir, nir_lower_locals_to_regs);
 867         NIR_PASS(progress, nir, nir_convert_from_ssa, true);
 868
 869         /* We are a vector architecture; write combine where possible */
 870         NIR_PASS(progress, nir, nir_move_vec_src_uses_to_dest);
 871         NIR_PASS(progress, nir, nir_lower_vec_to_movs);
 872
 873         NIR_PASS(progress, nir, nir_opt_dce);
 874 }
 875
 876 /* Front-half of aliasing the SSA slots, merely by inserting the flag in the
 877  * appropriate hash table. Intentional off-by-one to avoid confusing NULL with
 878  * r0. See the comments in compiler_context */
 879
 880 static void
 881 alias_ssa(compiler_context *ctx, int dest, int src)
 882 {
 883         _mesa_hash_table_u64_insert(ctx->ssa_to_alias, dest + 1, (void *) ((uintptr_t) src + 1));
 884         _mesa_set_add(ctx->leftover_ssa_to_alias, (void *) (uintptr_t) (dest + 1));
 885 }
 886
 887 /* ...or undo it, after which the original index will be used (dummy move should be emitted alongside this) */
 888
 889 static void
 890 unalias_ssa(compiler_context *ctx, int dest)
 891 {
 892         _mesa_hash_table_u64_remove(ctx->ssa_to_alias, dest + 1);
 893         /* TODO: Remove from leftover or no? */
 894 }
 895
 896 static void
 897 midgard_pin_output(compiler_context *ctx, int index, int reg)
 898 {
 899         _mesa_hash_table_u64_insert(ctx->ssa_to_register, index + 1, (void *) ((uintptr_t) reg + 1));
 900 }
 901
 902 static bool
 903 midgard_is_pinned(compiler_context *ctx, int index)
 904 {
 905         return _mesa_hash_table_u64_search(ctx->ssa_to_register, index + 1) != NULL;
 906 }
 907
 908 /* Do not actually emit a load; instead, cache the constant for inlining */
 909
 910 static void
 911 emit_load_const(compiler_context *ctx, nir_load_const_instr *instr)
 912 {
 913         nir_ssa_def def = instr->def;
 914
 915         float *v = ralloc_array(NULL, float, 4);
 916         nir_const_load_to_arr(v, instr, f32);
 917         _mesa_hash_table_u64_insert(ctx->ssa_constants, def.index + 1, v);
 918 }
 919
 920 /* Duplicate bits to convert sane 4-bit writemask to obscure 8-bit format (or
 921  * do the inverse) */
 922
 923 static unsigned
 924 expand_writemask(unsigned mask)
 925 {
 926         unsigned o = 0;
 927
 928         for (int i = 0; i < 4; ++i)
 929                 if (mask & (1 << i))
 930                         o |= (3 << (2 * i));
 931
 932         return o;
 933 }
 934
 935 static unsigned
 936 squeeze_writemask(unsigned mask)
 937 {
 938         unsigned o = 0;
 939
 940         for (int i = 0; i < 4; ++i)
 941                 if (mask & (3 << (2 * i)))
 942                         o |= (1 << i);
 943
 944         return o;
 945
 946 }
 947
 948 /* Determines effective writemask, taking quirks and expansion into account */
 949 static unsigned
 950 effective_writemask(midgard_vector_alu *alu)
 951 {
 952         /* Channel count is off-by-one to fit in two-bits (0 channel makes no
 953          * sense) */
 954
 955         unsigned channel_count = GET_CHANNEL_COUNT(alu_opcode_props[alu->op].props);
 956
 957         /* If there is a fixed channel count, construct the appropriate mask */
 958
 959         if (channel_count)
 960                 return (1 << channel_count) - 1;
 961
 962         /* Otherwise, just squeeze the existing mask */
 963         return squeeze_writemask(alu->mask);
 964 }
 965
 966 static unsigned
 967 find_or_allocate_temp(compiler_context *ctx, unsigned hash)
 968 {
 969         if ((hash < 0) || (hash >= SSA_FIXED_MINIMUM))
 970                 return hash;
 971
 972         unsigned temp = (uintptr_t) _mesa_hash_table_u64_search(ctx->hash_to_temp, hash + 1);
 973
 974         if (temp)
 975                 return temp - 1;
 976
 977         /* If no temp is find, allocate one */
 978         temp = ctx->temp_count++;
 979         ctx->max_hash = MAX2(ctx->max_hash, hash);
 980
 981         _mesa_hash_table_u64_insert(ctx->hash_to_temp, hash + 1, (void *) ((uintptr_t) temp + 1));
 982
 983         return temp;
 984 }
 985
 986 static unsigned
 987 nir_src_index(compiler_context *ctx, nir_src *src)
 988 {
 989         if (src->is_ssa)
 990                 return src->ssa->index;
 991         else {
 992                 assert(!src->reg.indirect);
 993                 return ctx->func->impl->ssa_alloc + src->reg.reg->index;
 994         }
 995 }
 996
 997 static unsigned
 998 nir_dest_index(compiler_context *ctx, nir_dest *dst)
 999 {
1000         if (dst->is_ssa)
1001                 return dst->ssa.index;
1002         else {
1003                 assert(!dst->reg.indirect);
1004                 return ctx->func->impl->ssa_alloc + dst->reg.reg->index;
1005         }
1006 }
1007
1008 static unsigned
1009 nir_alu_src_index(compiler_context *ctx, nir_alu_src *src)
1010 {
1011         return nir_src_index(ctx, &src->src);
1012 }
1013
1014 /* Midgard puts conditionals in r31.w; move an arbitrary source (the output of
1015  * a conditional test) into that register */
1016
1017 static void
1018 emit_condition(compiler_context *ctx, nir_src *src, bool for_branch, unsigned component)
1019 {
1020         int condition = nir_src_index(ctx, src);
1021
1022         /* Source to swizzle the desired component into w */
1023
1024         const midgard_vector_alu_src alu_src = {
1025                 .swizzle = SWIZZLE(component, component, component, component),
1026         };
1027
1028         /* There is no boolean move instruction. Instead, we simulate a move by
1029          * ANDing the condition with itself to get it into r31.w */
1030
1031         midgard_instruction ins = {
1032                 .type = TAG_ALU_4,
1033                 .unit = for_branch ? UNIT_SMUL : UNIT_SADD, /* TODO: DEDUCE THIS */
1034                 .ssa_args = {
1035                         .src0 = condition,
1036                         .src1 = condition,
1037                         .dest = SSA_FIXED_REGISTER(31),
1038                 },
1039                 .alu = {
1040                         .op = midgard_alu_op_iand,
1041                         .reg_mode = midgard_reg_mode_full,
1042                         .dest_override = midgard_dest_override_none,
1043                         .mask = (0x3 << 6), /* w */
1044                         .src1 = vector_alu_srco_unsigned(alu_src),
1045                         .src2 = vector_alu_srco_unsigned(alu_src)
1046                 },
1047         };
1048
1049         emit_mir_instruction(ctx, ins);
1050 }
1051
1052 /* Likewise, indirect offsets are put in r27.w. TODO: Allow componentwise
1053  * pinning to eliminate this move in all known cases */
1054
1055 static void
1056 emit_indirect_offset(compiler_context *ctx, nir_src *src)
1057 {
1058         int offset = nir_src_index(ctx, src);
1059
1060         midgard_instruction ins = {
1061                 .type = TAG_ALU_4,
1062                 .ssa_args = {
1063                         .src0 = SSA_UNUSED_1,
1064                         .src1 = offset,
1065                         .dest = SSA_FIXED_REGISTER(REGISTER_OFFSET),
1066                 },
1067                 .alu = {
1068                         .op = midgard_alu_op_imov,
1069                         .reg_mode = midgard_reg_mode_full,
1070                         .dest_override = midgard_dest_override_none,
1071                         .mask = (0x3 << 6), /* w */
1072                         .src1 = vector_alu_srco_unsigned(zero_alu_src),
1073                         .src2 = vector_alu_srco_unsigned(blank_alu_src_xxxx)
1074                 },
1075         };
1076
1077         emit_mir_instruction(ctx, ins);
1078 }
1079
1080 #define ALU_CASE(nir, _op) \
1081         case nir_op_##nir: \
1082                 op = midgard_alu_op_##_op; \
1083                 break;
1084
1085 static bool
1086 nir_is_fzero_constant(nir_src src)
1087 {
1088         if (!nir_src_is_const(src))
1089                 return false;
1090
1091         for (unsigned c = 0; c < nir_src_num_components(src); ++c) {
1092                 if (nir_src_comp_as_float(src, c) != 0.0)
1093                         return false;
1094         }
1095
1096         return true;
1097 }
1098
1099 static void
1100 emit_alu(compiler_context *ctx, nir_alu_instr *instr)
1101 {
1102         bool is_ssa = instr->dest.dest.is_ssa;
1103
1104         unsigned dest = nir_dest_index(ctx, &instr->dest.dest);
1105         unsigned nr_components = is_ssa ? instr->dest.dest.ssa.num_components : instr->dest.dest.reg.reg->num_components;
1106         unsigned nr_inputs = nir_op_infos[instr->op].num_inputs;
1107
1108         /* Most Midgard ALU ops have a 1:1 correspondance to NIR ops; these are
1109          * supported. A few do not and are commented for now. Also, there are a
1110          * number of NIR ops which Midgard does not support and need to be
1111          * lowered, also TODO. This switch block emits the opcode and calling
1112          * convention of the Midgard instruction; actual packing is done in
1113          * emit_alu below */
1114
1115         unsigned op;
1116
1117         switch (instr->op) {
1118                 ALU_CASE(fadd, fadd);
1119                 ALU_CASE(fmul, fmul);
1120                 ALU_CASE(fmin, fmin);
1121                 ALU_CASE(fmax, fmax);
1122                 ALU_CASE(imin, imin);
1123                 ALU_CASE(imax, imax);
1124                 ALU_CASE(umin, umin);
1125                 ALU_CASE(umax, umax);
1126                 ALU_CASE(fmov, fmov);
1127                 ALU_CASE(ffloor, ffloor);
1128                 ALU_CASE(fround_even, froundeven);
1129                 ALU_CASE(ftrunc, ftrunc);
1130                 ALU_CASE(fceil, fceil);
1131                 ALU_CASE(fdot3, fdot3);
1132                 ALU_CASE(fdot4, fdot4);
1133                 ALU_CASE(iadd, iadd);
1134                 ALU_CASE(isub, isub);
1135                 ALU_CASE(imul, imul);
1136                 ALU_CASE(iabs, iabs);
1137                 ALU_CASE(imov, imov);
1138
1139                 ALU_CASE(feq32, feq);
1140                 ALU_CASE(fne32, fne);
1141                 ALU_CASE(flt32, flt);
1142                 ALU_CASE(ieq32, ieq);
1143                 ALU_CASE(ine32, ine);
1144                 ALU_CASE(ilt32, ilt);
1145                 ALU_CASE(ult32, ult);
1146
1147                 /* We don't have a native b2f32 instruction. Instead, like many
1148                  * GPUs, we exploit booleans as 0/~0 for false/true, and
1149                  * correspondingly AND
1150                  * by 1.0 to do the type conversion. For the moment, prime us
1151                  * to emit:
1152                  *
1153                  * iand [whatever], #0
1154                  *
1155                  * At the end of emit_alu (as MIR), we'll fix-up the constant
1156                  */
1157
1158                 ALU_CASE(b2f32, iand);
1159                 ALU_CASE(b2i32, iand);
1160
1161                 /* Likewise, we don't have a dedicated f2b32 instruction, but
1162                  * we can do a "not equal to 0.0" test. */
1163
1164                 ALU_CASE(f2b32, fne);
1165                 ALU_CASE(i2b32, ine);
1166
1167                 ALU_CASE(frcp, frcp);
1168                 ALU_CASE(frsq, frsqrt);
1169                 ALU_CASE(fsqrt, fsqrt);
1170                 ALU_CASE(fexp2, fexp2);
1171                 ALU_CASE(flog2, flog2);
1172
1173                 ALU_CASE(f2i32, f2i);
1174                 ALU_CASE(f2u32, f2u);
1175                 ALU_CASE(i2f32, i2f);
1176                 ALU_CASE(u2f32, u2f);
1177
1178                 ALU_CASE(fsin, fsin);
1179                 ALU_CASE(fcos, fcos);
1180
1181                 ALU_CASE(iand, iand);
1182                 ALU_CASE(ior, ior);
1183                 ALU_CASE(ixor, ixor);
1184                 ALU_CASE(inot, inand);
1185                 ALU_CASE(ishl, ishl);
1186                 ALU_CASE(ishr, iasr);
1187                 ALU_CASE(ushr, ilsr);
1188
1189                 ALU_CASE(b32all_fequal2, fball_eq);
1190                 ALU_CASE(b32all_fequal3, fball_eq);
1191                 ALU_CASE(b32all_fequal4, fball_eq);
1192
1193                 ALU_CASE(b32any_fnequal2, fbany_neq);
1194                 ALU_CASE(b32any_fnequal3, fbany_neq);
1195                 ALU_CASE(b32any_fnequal4, fbany_neq);
1196
1197                 ALU_CASE(b32all_iequal2, iball_eq);
1198                 ALU_CASE(b32all_iequal3, iball_eq);
1199                 ALU_CASE(b32all_iequal4, iball_eq);
1200
1201                 ALU_CASE(b32any_inequal2, ibany_neq);
1202                 ALU_CASE(b32any_inequal3, ibany_neq);
1203                 ALU_CASE(b32any_inequal4, ibany_neq);
1204
1205         /* For greater-or-equal, we lower to less-or-equal and flip the
1206          * arguments */
1207
1208         case nir_op_fge:
1209         case nir_op_fge32:
1210         case nir_op_ige32:
1211         case nir_op_uge32: {
1212                 op =
1213                         instr->op == nir_op_fge   ? midgard_alu_op_fle :
1214                         instr->op == nir_op_fge32 ? midgard_alu_op_fle :
1215                         instr->op == nir_op_ige32 ? midgard_alu_op_ile :
1216                         instr->op == nir_op_uge32 ? midgard_alu_op_ule :
1217                         0;
1218
1219                 /* Swap via temporary */
1220                 nir_alu_src temp = instr->src[1];
1221                 instr->src[1] = instr->src[0];
1222                 instr->src[0] = temp;
1223
1224                 break;
1225         }
1226
1227         /* For a few special csel cases not handled by NIR, we can opt to
1228          * bitwise. Otherwise, we emit the condition and do a real csel */
1229
1230         case nir_op_b32csel: {
1231                 if (nir_is_fzero_constant(instr->src[2].src)) {
1232                         /* (b ? v : 0) = (b & v) */
1233                         op = midgard_alu_op_iand;
1234                         nr_inputs = 2;
1235                 } else if (nir_is_fzero_constant(instr->src[1].src)) {
1236                         /* (b ? 0 : v) = (!b ? v : 0) = (~b & v) = (v & ~b) */
1237                         op = midgard_alu_op_iandnot;
1238                         nr_inputs = 2;
1239                         instr->src[1] = instr->src[0];
1240                         instr->src[0] = instr->src[2];
1241                 } else {
1242                         op = midgard_alu_op_fcsel;
1243
1244                         /* csel works as a two-arg in Midgard, since the condition is hardcoded in r31.w */
1245                         nr_inputs = 2;
1246
1247                         /* Figure out which component the condition is in */
1248
1249                         unsigned comp = instr->src[0].swizzle[0];
1250
1251                         /* Make sure NIR isn't throwing a mixed condition at us */
1252
1253                         for (unsigned c = 1; c < nr_components; ++c)
1254                                 assert(instr->src[0].swizzle[c] == comp);
1255
1256                         /* Emit the condition into r31.w */
1257                         emit_condition(ctx, &instr->src[0].src, false, comp);
1258
1259                         /* The condition is the first argument; move the other
1260                          * arguments up one to be a binary instruction for
1261                          * Midgard */
1262
1263                         memmove(instr->src, instr->src + 1, 2 * sizeof(nir_alu_src));
1264                 }
1265                 break;
1266         }
1267
1268         default:
1269                 DBG("Unhandled ALU op %s\n", nir_op_infos[instr->op].name);
1270                 assert(0);
1271                 return;
1272         }
1273
1274         /* Midgard can perform certain modifiers on output ofa n ALU op */
1275         midgard_outmod outmod =
1276                 instr->dest.saturate ? midgard_outmod_sat : midgard_outmod_none;
1277
1278         /* fmax(a, 0.0) can turn into a .pos modifier as an optimization */
1279
1280         if (instr->op == nir_op_fmax) {
1281                 if (nir_is_fzero_constant(instr->src[0].src)) {
1282                         op = midgard_alu_op_fmov;
1283                         nr_inputs = 1;
1284                         outmod = midgard_outmod_pos;
1285                         instr->src[0] = instr->src[1];
1286                 } else if (nir_is_fzero_constant(instr->src[1].src)) {
1287                         op = midgard_alu_op_fmov;
1288                         nr_inputs = 1;
1289                         outmod = midgard_outmod_pos;
1290                 }
1291         }
1292
1293         /* Fetch unit, quirks, etc information */
1294         unsigned opcode_props = alu_opcode_props[op].props;
1295         bool quirk_flipped_r24 = opcode_props & QUIRK_FLIPPED_R24;
1296
1297         /* src0 will always exist afaik, but src1 will not for 1-argument
1298          * instructions. The latter can only be fetched if the instruction
1299          * needs it, or else we may segfault. */
1300
1301         unsigned src0 = nir_alu_src_index(ctx, &instr->src[0]);
1302         unsigned src1 = nr_inputs == 2 ? nir_alu_src_index(ctx, &instr->src[1]) : SSA_UNUSED_0;
1303
1304         /* Rather than use the instruction generation helpers, we do it
1305          * ourselves here to avoid the mess */
1306
1307         midgard_instruction ins = {
1308                 .type = TAG_ALU_4,
1309                 .ssa_args = {
1310                         .src0 = quirk_flipped_r24 ? SSA_UNUSED_1 : src0,
1311                         .src1 = quirk_flipped_r24 ? src0         : src1,
1312                         .dest = dest,
1313                 }
1314         };
1315
1316         nir_alu_src *nirmods[2] = { NULL };
1317
1318         if (nr_inputs == 2) {
1319                 nirmods[0] = &instr->src[0];
1320                 nirmods[1] = &instr->src[1];
1321         } else if (nr_inputs == 1) {
1322                 nirmods[quirk_flipped_r24] = &instr->src[0];
1323         } else {
1324                 assert(0);
1325         }
1326
1327         bool is_int = midgard_is_integer_op(op);
1328
1329         midgard_vector_alu alu = {
1330                 .op = op,
1331                 .reg_mode = midgard_reg_mode_full,
1332                 .dest_override = midgard_dest_override_none,
1333                 .outmod = outmod,
1334
1335                 /* Writemask only valid for non-SSA NIR */
1336                 .mask = expand_writemask((1 << nr_components) - 1),
1337
1338                 .src1 = vector_alu_srco_unsigned(vector_alu_modifiers(nirmods[0], is_int)),
1339                 .src2 = vector_alu_srco_unsigned(vector_alu_modifiers(nirmods[1], is_int)),
1340         };
1341
1342         /* Apply writemask if non-SSA, keeping in mind that we can't write to components that don't exist */
1343
1344         if (!is_ssa)
1345                 alu.mask &= expand_writemask(instr->dest.write_mask);
1346
1347         ins.alu = alu;
1348
1349         /* Late fixup for emulated instructions */
1350
1351         if (instr->op == nir_op_b2f32 || instr->op == nir_op_b2i32) {
1352                 /* Presently, our second argument is an inline #0 constant.
1353                  * Switch over to an embedded 1.0 constant (that can't fit
1354                  * inline, since we're 32-bit, not 16-bit like the inline
1355                  * constants) */
1356
1357                 ins.ssa_args.inline_constant = false;
1358                 ins.ssa_args.src1 = SSA_FIXED_REGISTER(REGISTER_CONSTANT);
1359                 ins.has_constants = true;
1360
1361                 if (instr->op == nir_op_b2f32) {
1362                         ins.constants[0] = 1.0f;
1363                 } else {
1364                         /* Type pun it into place */
1365                         uint32_t one = 0x1;
1366                         memcpy(&ins.constants[0], &one, sizeof(uint32_t));
1367                 }
1368
1369                 ins.alu.src2 = vector_alu_srco_unsigned(blank_alu_src_xxxx);
1370         } else if (instr->op == nir_op_f2b32 || instr->op == nir_op_i2b32) {
1371                 ins.ssa_args.inline_constant = false;
1372                 ins.ssa_args.src1 = SSA_FIXED_REGISTER(REGISTER_CONSTANT);
1373                 ins.has_constants = true;
1374                 ins.constants[0] = 0.0f;
1375                 ins.alu.src2 = vector_alu_srco_unsigned(blank_alu_src_xxxx);
1376         } else if (instr->op == nir_op_inot) {
1377                 /* ~b = ~(b & b), so duplicate the source */
1378                 ins.ssa_args.src1 = ins.ssa_args.src0;
1379                 ins.alu.src2 = ins.alu.src1;
1380         }
1381
1382         if ((opcode_props & UNITS_ALL) == UNIT_VLUT) {
1383                 /* To avoid duplicating the lookup tables (probably), true LUT
1384                  * instructions can only operate as if they were scalars. Lower
1385                  * them here by changing the component. */
1386
1387                 uint8_t original_swizzle[4];
1388                 memcpy(original_swizzle, nirmods[0]->swizzle, sizeof(nirmods[0]->swizzle));
1389
1390                 for (int i = 0; i < nr_components; ++i) {
1391                         ins.alu.mask = (0x3) << (2 * i); /* Mask the associated component */
1392
1393                         for (int j = 0; j < 4; ++j)
1394                                 nirmods[0]->swizzle[j] = original_swizzle[i]; /* Pull from the correct component */
1395
1396                         ins.alu.src1 = vector_alu_srco_unsigned(vector_alu_modifiers(nirmods[0], is_int));
1397                         emit_mir_instruction(ctx, ins);
1398                 }
1399         } else {
1400                 emit_mir_instruction(ctx, ins);
1401         }
1402 }
1403
1404 #undef ALU_CASE
1405
1406 static void
1407 emit_uniform_read(compiler_context *ctx, unsigned dest, unsigned offset, nir_src *indirect_offset)
1408 {
1409         /* TODO: half-floats */
1410
1411         if (!indirect_offset && offset < ctx->uniform_cutoff) {
1412                 /* Fast path: For the first 16 uniforms, direct accesses are
1413                  * 0-cycle, since they're just a register fetch in the usual
1414                  * case.  So, we alias the registers while we're still in
1415                  * SSA-space */
1416
1417                 int reg_slot = 23 - offset;
1418                 alias_ssa(ctx, dest, SSA_FIXED_REGISTER(reg_slot));
1419         } else {
1420                 /* Otherwise, read from the 'special' UBO to access
1421                  * higher-indexed uniforms, at a performance cost. More
1422                  * generally, we're emitting a UBO read instruction. */
1423
1424                 midgard_instruction ins = m_load_uniform_32(dest, offset);
1425
1426                 /* TODO: Don't split */
1427                 ins.load_store.varying_parameters = (offset & 7) << 7;
1428                 ins.load_store.address = offset >> 3;
1429
1430                 if (indirect_offset) {
1431                         emit_indirect_offset(ctx, indirect_offset);
1432                         ins.load_store.unknown = 0x8700; /* xxx: what is this? */
1433                 } else {
1434                         ins.load_store.unknown = 0x1E00; /* xxx: what is this? */
1435                 }
1436
1437                 emit_mir_instruction(ctx, ins);
1438         }
1439 }
1440
1441 static void
1442 emit_sysval_read(compiler_context *ctx, nir_intrinsic_instr *instr)
1443 {
1444         /* First, pull out the destination */
1445         unsigned dest = nir_dest_index(ctx, &instr->dest);
1446
1447         /* Now, figure out which uniform this is */
1448         int sysval = midgard_nir_sysval_for_intrinsic(instr);
1449         void *val = _mesa_hash_table_u64_search(ctx->sysval_to_id, sysval);
1450
1451         /* Sysvals are prefix uniforms */
1452         unsigned uniform = ((uintptr_t) val) - 1;
1453
1454         /* Emit the read itself -- this is never indirect */
1455         emit_uniform_read(ctx, dest, uniform, NULL);
1456 }
1457
1458 static void
1459 emit_intrinsic(compiler_context *ctx, nir_intrinsic_instr *instr)
1460 {
1461         unsigned offset, reg;
1462
1463         switch (instr->intrinsic) {
1464         case nir_intrinsic_discard_if:
1465                 emit_condition(ctx, &instr->src[0], true, COMPONENT_X);
1466
1467         /* fallthrough */
1468
1469         case nir_intrinsic_discard: {
1470                 bool conditional = instr->intrinsic == nir_intrinsic_discard_if;
1471                 struct midgard_instruction discard = v_branch(conditional, false);
1472                 discard.branch.target_type = TARGET_DISCARD;
1473                 emit_mir_instruction(ctx, discard);
1474
1475                 ctx->can_discard = true;
1476                 break;
1477         }
1478
1479         case nir_intrinsic_load_uniform:
1480         case nir_intrinsic_load_input:
1481                 offset = nir_intrinsic_base(instr);
1482
1483                 bool direct = nir_src_is_const(instr->src[0]);
1484
1485                 if (direct) {
1486                         offset += nir_src_as_uint(instr->src[0]);
1487                 }
1488
1489                 reg = nir_dest_index(ctx, &instr->dest);
1490
1491                 if (instr->intrinsic == nir_intrinsic_load_uniform && !ctx->is_blend) {
1492                         emit_uniform_read(ctx, reg, ctx->sysval_count + offset, !direct ? &instr->src[0] : NULL);
1493                 } else if (ctx->stage == MESA_SHADER_FRAGMENT && !ctx->is_blend) {
1494                         /* XXX: Half-floats? */
1495                         /* TODO: swizzle, mask */
1496
1497                         midgard_instruction ins = m_load_vary_32(reg, offset);
1498
1499                         midgard_varying_parameter p = {
1500                                 .is_varying = 1,
1501                                 .interpolation = midgard_interp_default,
1502                                 .flat = /*var->data.interpolation == INTERP_MODE_FLAT*/ 0
1503                         };
1504
1505                         unsigned u;
1506                         memcpy(&u, &p, sizeof(p));
1507                         ins.load_store.varying_parameters = u;
1508
1509                         if (direct) {
1510                                 /* We have the offset totally ready */
1511                                 ins.load_store.unknown = 0x1e9e; /* xxx: what is this? */
1512                         } else {
1513                                 /* We have it partially ready, but we need to
1514                                  * add in the dynamic index, moved to r27.w */
1515                                 emit_indirect_offset(ctx, &instr->src[0]);
1516                                 ins.load_store.unknown = 0x79e; /* xxx: what is this? */
1517                         }
1518
1519                         emit_mir_instruction(ctx, ins);
1520                 } else if (ctx->is_blend && instr->intrinsic == nir_intrinsic_load_uniform) {
1521                         /* Constant encoded as a pinned constant */
1522
1523                         midgard_instruction ins = v_fmov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), blank_alu_src, reg);
1524                         ins.has_constants = true;
1525                         ins.has_blend_constant = true;
1526                         emit_mir_instruction(ctx, ins);
1527                 } else if (ctx->is_blend) {
1528                         /* For blend shaders, a load might be
1529                          * translated various ways depending on what
1530                          * we're loading. Figure out how this is used */
1531
1532                         nir_variable *out = NULL;
1533
1534                         nir_foreach_variable(var, &ctx->nir->inputs) {
1535                                 int drvloc = var->data.driver_location;
1536
1537                                 if (nir_intrinsic_base(instr) == drvloc) {
1538                                         out = var;
1539                                         break;
1540                                 }
1541                         }
1542
1543                         assert(out);
1544
1545                         if (out->data.location == VARYING_SLOT_COL0) {
1546                                 /* Source color preloaded to r0 */
1547
1548                                 midgard_pin_output(ctx, reg, 0);
1549                         } else if (out->data.location == VARYING_SLOT_COL1) {
1550                                 /* Destination color must be read from framebuffer */
1551
1552                                 midgard_instruction ins = m_load_color_buffer_8(reg, 0);
1553                                 ins.load_store.swizzle = 0; /* xxxx */
1554
1555                                 /* Read each component sequentially */
1556
1557                                 for (int c = 0; c < 4; ++c) {
1558                                         ins.load_store.mask = (1 << c);
1559                                         ins.load_store.unknown = c;
1560                                         emit_mir_instruction(ctx, ins);
1561                                 }
1562
1563                                 /* vadd.u2f hr2, zext(hr2), #0 */
1564
1565                                 midgard_vector_alu_src alu_src = blank_alu_src;
1566                                 alu_src.mod = midgard_int_zero_extend;
1567                                 alu_src.half = true;
1568
1569                                 midgard_instruction u2f = {
1570                                         .type = TAG_ALU_4,
1571                                         .ssa_args = {
1572                                                 .src0 = reg,
1573                                                 .src1 = SSA_UNUSED_0,
1574                                                 .dest = reg,
1575                                                 .inline_constant = true
1576                                         },
1577                                         .alu = {
1578                                                 .op = midgard_alu_op_u2f,
1579                                                 .reg_mode = midgard_reg_mode_half,
1580                                                 .dest_override = midgard_dest_override_none,
1581                                                 .mask = 0xF,
1582                                                 .src1 = vector_alu_srco_unsigned(alu_src),
1583                                                 .src2 = vector_alu_srco_unsigned(blank_alu_src),
1584                                         }
1585                                 };
1586
1587                                 emit_mir_instruction(ctx, u2f);
1588
1589                                 /* vmul.fmul.sat r1, hr2, #0.00392151 */
1590
1591                                 alu_src.mod = 0;
1592
1593                                 midgard_instruction fmul = {
1594                                         .type = TAG_ALU_4,
1595                                         .inline_constant = _mesa_float_to_half(1.0 / 255.0),
1596                                         .ssa_args = {
1597                                                 .src0 = reg,
1598                                                 .dest = reg,
1599                                                 .src1 = SSA_UNUSED_0,
1600                                                 .inline_constant = true
1601                                         },
1602                                         .alu = {
1603                                                 .op = midgard_alu_op_fmul,
1604                                                 .reg_mode = midgard_reg_mode_full,
1605                                                 .dest_override = midgard_dest_override_none,
1606                                                 .outmod = midgard_outmod_sat,
1607                                                 .mask = 0xFF,
1608                                                 .src1 = vector_alu_srco_unsigned(alu_src),
1609                                                 .src2 = vector_alu_srco_unsigned(blank_alu_src),
1610                                         }
1611                                 };
1612
1613                                 emit_mir_instruction(ctx, fmul);
1614                         } else {
1615                                 DBG("Unknown input in blend shader\n");
1616                                 assert(0);
1617                         }
1618                 } else if (ctx->stage == MESA_SHADER_VERTEX) {
1619                         midgard_instruction ins = m_load_attr_32(reg, offset);
1620                         ins.load_store.unknown = 0x1E1E; /* XXX: What is this? */
1621                         ins.load_store.mask = (1 << instr->num_components) - 1;
1622                         emit_mir_instruction(ctx, ins);
1623                 } else {
1624                         DBG("Unknown load\n");
1625                         assert(0);
1626                 }
1627
1628                 break;
1629
1630         case nir_intrinsic_store_output:
1631                 assert(nir_src_is_const(instr->src[1]) && "no indirect outputs");
1632
1633                 offset = nir_intrinsic_base(instr) + nir_src_as_uint(instr->src[1]);
1634
1635                 reg = nir_src_index(ctx, &instr->src[0]);
1636
1637                 if (ctx->stage == MESA_SHADER_FRAGMENT) {
1638                         /* gl_FragColor is not emitted with load/store
1639                          * instructions. Instead, it gets plonked into
1640                          * r0 at the end of the shader and we do the
1641                          * framebuffer writeout dance. TODO: Defer
1642                          * writes */
1643
1644                         midgard_pin_output(ctx, reg, 0);
1645
1646                         /* Save the index we're writing to for later reference
1647                          * in the epilogue */
1648
1649                         ctx->fragment_output = reg;
1650                 } else if (ctx->stage == MESA_SHADER_VERTEX) {
1651                         /* Varyings are written into one of two special
1652                          * varying register, r26 or r27. The register itself is selected as the register
1653                          * in the st_vary instruction, minus the base of 26. E.g. write into r27 and then call st_vary(1)
1654                          *
1655                          * Normally emitting fmov's is frowned upon,
1656                          * but due to unique constraints of
1657                          * REGISTER_VARYING, fmov emission + a
1658                          * dedicated cleanup pass is the only way to
1659                          * guarantee correctness when considering some
1660                          * (common) edge cases XXX: FIXME */
1661
1662                         /* If this varying corresponds to a constant (why?!),
1663                          * emit that now since it won't get picked up by
1664                          * hoisting (since there is no corresponding move
1665                          * emitted otherwise) */
1666
1667                         void *constant_value = _mesa_hash_table_u64_search(ctx->ssa_constants, reg + 1);
1668
1669                         if (constant_value) {
1670                                 /* Special case: emit the varying write
1671                                  * directly to r26 (looks funny in asm but it's
1672                                  * fine) and emit the store _now_. Possibly
1673                                  * slightly slower, but this is a really stupid
1674                                  * special case anyway (why on earth would you
1675                                  * have a constant varying? Your own fault for
1676                                  * slightly worse perf :P) */
1677
1678                                 midgard_instruction ins = v_fmov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), blank_alu_src, SSA_FIXED_REGISTER(26));
1679                                 attach_constants(ctx, &ins, constant_value, reg + 1);
1680                                 emit_mir_instruction(ctx, ins);
1681
1682                                 midgard_instruction st = m_store_vary_32(SSA_FIXED_REGISTER(0), offset);
1683                                 st.load_store.unknown = 0x1E9E; /* XXX: What is this? */
1684                                 emit_mir_instruction(ctx, st);
1685                         } else {
1686                                 /* Do not emit the varying yet -- instead, just mark down that we need to later */
1687
1688                                 _mesa_hash_table_u64_insert(ctx->ssa_varyings, reg + 1, (void *) ((uintptr_t) (offset + 1)));
1689                         }
1690                 } else {
1691                         DBG("Unknown store\n");
1692                         assert(0);
1693                 }
1694
1695                 break;
1696
1697         case nir_intrinsic_load_alpha_ref_float:
1698                 assert(instr->dest.is_ssa);
1699
1700                 float ref_value = ctx->alpha_ref;
1701
1702                 float *v = ralloc_array(NULL, float, 4);
1703                 memcpy(v, &ref_value, sizeof(float));
1704                 _mesa_hash_table_u64_insert(ctx->ssa_constants, instr->dest.ssa.index + 1, v);
1705                 break;
1706
1707         case nir_intrinsic_load_viewport_scale:
1708         case nir_intrinsic_load_viewport_offset:
1709                 emit_sysval_read(ctx, instr);
1710                 break;
1711
1712         default:
1713                 printf ("Unhandled intrinsic\n");
1714                 assert(0);
1715                 break;
1716         }
1717 }
1718
1719 static unsigned
1720 midgard_tex_format(enum glsl_sampler_dim dim)
1721 {
1722         switch (dim) {
1723         case GLSL_SAMPLER_DIM_2D:
1724         case GLSL_SAMPLER_DIM_EXTERNAL:
1725                 return TEXTURE_2D;
1726
1727         case GLSL_SAMPLER_DIM_3D:
1728                 return TEXTURE_3D;
1729
1730         case GLSL_SAMPLER_DIM_CUBE:
1731                 return TEXTURE_CUBE;
1732
1733         default:
1734                 DBG("Unknown sampler dim type\n");
1735                 assert(0);
1736                 return 0;
1737         }
1738 }
1739
1740 static void
1741 emit_tex(compiler_context *ctx, nir_tex_instr *instr)
1742 {
1743         /* TODO */
1744         //assert (!instr->sampler);
1745         //assert (!instr->texture_array_size);
1746         assert (instr->op == nir_texop_tex);
1747
1748         /* Allocate registers via a round robin scheme to alternate between the two registers */
1749         int reg = ctx->texture_op_count & 1;
1750         int in_reg = reg, out_reg = reg;
1751
1752         /* Make room for the reg */
1753
1754         if (ctx->texture_index[reg] > -1)
1755                 unalias_ssa(ctx, ctx->texture_index[reg]);
1756
1757         int texture_index = instr->texture_index;
1758         int sampler_index = texture_index;
1759
1760         for (unsigned i = 0; i < instr->num_srcs; ++i) {
1761                 switch (instr->src[i].src_type) {
1762                 case nir_tex_src_coord: {
1763                         int index = nir_src_index(ctx, &instr->src[i].src);
1764
1765                         midgard_vector_alu_src alu_src = blank_alu_src;
1766
1767                         int reg = SSA_FIXED_REGISTER(REGISTER_TEXTURE_BASE + in_reg);
1768
1769                         if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE) {
1770                                 /* For cubemaps, we need to load coords into
1771                                  * special r27, and then use a special ld/st op
1772                                  * to copy into the texture register */
1773
1774                                 alu_src.swizzle = SWIZZLE(COMPONENT_X, COMPONENT_Y, COMPONENT_Z, COMPONENT_X);
1775
1776                                 midgard_instruction move = v_fmov(index, alu_src, SSA_FIXED_REGISTER(27));
1777                                 emit_mir_instruction(ctx, move);
1778
1779                                 midgard_instruction st = m_store_cubemap_coords(reg, 0);
1780                                 st.load_store.unknown = 0x24; /* XXX: What is this? */
1781                                 st.load_store.mask = 0x3; /* xy? */
1782                                 st.load_store.swizzle = alu_src.swizzle;
1783                                 emit_mir_instruction(ctx, st);
1784
1785                         } else {
1786                                 alu_src.swizzle = SWIZZLE(COMPONENT_X, COMPONENT_Y, COMPONENT_X, COMPONENT_X);
1787
1788                                 midgard_instruction ins = v_fmov(index, alu_src, reg);
1789                                 emit_mir_instruction(ctx, ins);
1790                         }
1791
1792                         break;
1793                 }
1794
1795                 default: {
1796                         DBG("Unknown source type\n");
1797                         //assert(0);
1798                         break;
1799                 }
1800                 }
1801         }
1802
1803         /* No helper to build texture words -- we do it all here */
1804         midgard_instruction ins = {
1805                 .type = TAG_TEXTURE_4,
1806                 .texture = {
1807                         .op = TEXTURE_OP_NORMAL,
1808                         .format = midgard_tex_format(instr->sampler_dim),
1809                         .texture_handle = texture_index,
1810                         .sampler_handle = sampler_index,
1811
1812                         /* TODO: Don't force xyzw */
1813                         .swizzle = SWIZZLE(COMPONENT_X, COMPONENT_Y, COMPONENT_Z, COMPONENT_W),
1814                         .mask = 0xF,
1815
1816                         /* TODO: half */
1817                         //.in_reg_full = 1,
1818                         .out_full = 1,
1819
1820                         .filter = 1,
1821
1822                         /* Always 1 */
1823                         .unknown7 = 1,
1824
1825                         /* Assume we can continue; hint it out later */
1826                         .cont = 1,
1827                 }
1828         };
1829
1830         /* Set registers to read and write from the same place */
1831         ins.texture.in_reg_select = in_reg;
1832         ins.texture.out_reg_select = out_reg;
1833
1834         /* TODO: Dynamic swizzle input selection, half-swizzles? */
1835         if (instr->sampler_dim == GLSL_SAMPLER_DIM_3D) {
1836                 ins.texture.in_reg_swizzle_right = COMPONENT_X;
1837                 ins.texture.in_reg_swizzle_left = COMPONENT_Y;
1838                 //ins.texture.in_reg_swizzle_third = COMPONENT_Z;
1839         } else {
1840                 ins.texture.in_reg_swizzle_left = COMPONENT_X;
1841                 ins.texture.in_reg_swizzle_right = COMPONENT_Y;
1842                 //ins.texture.in_reg_swizzle_third = COMPONENT_X;
1843         }
1844
1845         emit_mir_instruction(ctx, ins);
1846
1847         /* Simultaneously alias the destination and emit a move for it. The move will be eliminated if possible */
1848
1849         int o_reg = REGISTER_TEXTURE_BASE + out_reg, o_index = nir_dest_index(ctx, &instr->dest);
1850         alias_ssa(ctx, o_index, SSA_FIXED_REGISTER(o_reg));
1851         ctx->texture_index[reg] = o_index;
1852
1853         midgard_instruction ins2 = v_fmov(SSA_FIXED_REGISTER(o_reg), blank_alu_src, o_index);
1854         emit_mir_instruction(ctx, ins2);
1855
1856         /* Used for .cont and .last hinting */
1857         ctx->texture_op_count++;
1858 }
1859
1860 static void
1861 emit_jump(compiler_context *ctx, nir_jump_instr *instr)
1862 {
1863         switch (instr->type) {
1864                 case nir_jump_break: {
1865                         /* Emit a branch out of the loop */
1866                         struct midgard_instruction br = v_branch(false, false);
1867                         br.branch.target_type = TARGET_BREAK;
1868                         br.branch.target_break = ctx->current_loop_depth;
1869                         emit_mir_instruction(ctx, br);
1870
1871                         DBG("break..\n");
1872                         break;
1873                 }
1874
1875                 default:
1876                         DBG("Unknown jump type %d\n", instr->type);
1877                         break;
1878         }
1879 }
1880
1881 static void
1882 emit_instr(compiler_context *ctx, struct nir_instr *instr)
1883 {
1884         switch (instr->type) {
1885         case nir_instr_type_load_const:
1886                 emit_load_const(ctx, nir_instr_as_load_const(instr));
1887                 break;
1888
1889         case nir_instr_type_intrinsic:
1890                 emit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
1891                 break;
1892
1893         case nir_instr_type_alu:
1894                 emit_alu(ctx, nir_instr_as_alu(instr));
1895                 break;
1896
1897         case nir_instr_type_tex:
1898                 emit_tex(ctx, nir_instr_as_tex(instr));
1899                 break;
1900
1901         case nir_instr_type_jump:
1902                 emit_jump(ctx, nir_instr_as_jump(instr));
1903                 break;
1904
1905         case nir_instr_type_ssa_undef:
1906                 /* Spurious */
1907                 break;
1908
1909         default:
1910                 DBG("Unhandled instruction type\n");
1911                 break;
1912         }
1913 }
1914
1915 /* Determine the actual hardware from the index based on the RA results or special values */
1916
1917 static int
1918 dealias_register(compiler_context *ctx, struct ra_graph *g, int reg, int maxreg)
1919 {
1920         if (reg >= SSA_FIXED_MINIMUM)
1921                 return SSA_REG_FROM_FIXED(reg);
1922
1923         if (reg >= 0) {
1924                 assert(reg < maxreg);
1925                 int r = ra_get_node_reg(g, reg);
1926                 ctx->work_registers = MAX2(ctx->work_registers, r);
1927                 return r;
1928         }
1929
1930         switch (reg) {
1931         /* fmov style unused */
1932         case SSA_UNUSED_0:
1933                 return REGISTER_UNUSED;
1934
1935         /* lut style unused */
1936         case SSA_UNUSED_1:
1937                 return REGISTER_UNUSED;
1938
1939         default:
1940                 DBG("Unknown SSA register alias %d\n", reg);
1941                 assert(0);
1942                 return 31;
1943         }
1944 }
1945
1946 static unsigned int
1947 midgard_ra_select_callback(struct ra_graph *g, BITSET_WORD *regs, void *data)
1948 {
1949         /* Choose the first available register to minimise reported register pressure */
1950
1951         for (int i = 0; i < 16; ++i) {
1952                 if (BITSET_TEST(regs, i)) {
1953                         return i;
1954                 }
1955         }
1956
1957         assert(0);
1958         return 0;
1959 }
1960
1961 static bool
1962 midgard_is_live_in_instr(midgard_instruction *ins, int src)
1963 {
1964         if (ins->ssa_args.src0 == src) return true;
1965         if (ins->ssa_args.src1 == src) return true;
1966
1967         return false;
1968 }
1969
1970 /* Determine if a variable is live in the successors of a block */
1971 static bool
1972 is_live_after_successors(compiler_context *ctx, midgard_block *bl, int src)
1973 {
1974         for (unsigned i = 0; i < bl->nr_successors; ++i) {
1975                 midgard_block *succ = bl->successors[i];
1976
1977                 /* If we already visited, the value we're seeking
1978                  * isn't down this path (or we would have short
1979                  * circuited */
1980
1981                 if (succ->visited) continue;
1982
1983                 /* Otherwise (it's visited *now*), check the block */
1984
1985                 succ->visited = true;
1986
1987                 mir_foreach_instr_in_block(succ, ins) {
1988                         if (midgard_is_live_in_instr(ins, src))
1989                                 return true;
1990                 }
1991
1992                 /* ...and also, check *its* successors */
1993                 if (is_live_after_successors(ctx, succ, src))
1994                         return true;
1995
1996         }
1997
1998         /* Welp. We're really not live. */
1999
2000         return false;
2001 }
2002
2003 static bool
2004 is_live_after(compiler_context *ctx, midgard_block *block, midgard_instruction *start, int src)
2005 {
2006         /* Check the rest of the block for liveness */
2007
2008         mir_foreach_instr_in_block_from(block, ins, mir_next_op(start)) {
2009                 if (midgard_is_live_in_instr(ins, src))
2010                         return true;
2011         }
2012
2013         /* Check the rest of the blocks for liveness recursively */
2014
2015         bool succ = is_live_after_successors(ctx, block, src);
2016
2017         mir_foreach_block(ctx, block) {
2018                 block->visited = false;
2019         }
2020
2021         return succ;
2022 }
2023
2024 static void
2025 allocate_registers(compiler_context *ctx)
2026 {
2027         /* First, initialize the RA */
2028         struct ra_regs *regs = ra_alloc_reg_set(NULL, 32, true);
2029
2030         /* Create a primary (general purpose) class, as well as special purpose
2031          * pipeline register classes */
2032
2033         int primary_class = ra_alloc_reg_class(regs);
2034         int varying_class  = ra_alloc_reg_class(regs);
2035
2036         /* Add the full set of work registers */
2037         int work_count = 16 - MAX2((ctx->uniform_cutoff - 8), 0);
2038         for (int i = 0; i < work_count; ++i)
2039                 ra_class_add_reg(regs, primary_class, i);
2040
2041         /* Add special registers */
2042         ra_class_add_reg(regs, varying_class, REGISTER_VARYING_BASE);
2043         ra_class_add_reg(regs, varying_class, REGISTER_VARYING_BASE + 1);
2044
2045         /* We're done setting up */
2046         ra_set_finalize(regs, NULL);
2047
2048         /* Transform the MIR into squeezed index form */
2049         mir_foreach_block(ctx, block) {
2050                 mir_foreach_instr_in_block(block, ins) {
2051                         if (ins->compact_branch) continue;
2052
2053                         ins->ssa_args.src0 = find_or_allocate_temp(ctx, ins->ssa_args.src0);
2054                         ins->ssa_args.src1 = find_or_allocate_temp(ctx, ins->ssa_args.src1);
2055                         ins->ssa_args.dest = find_or_allocate_temp(ctx, ins->ssa_args.dest);
2056                 }
2057                 if (midgard_debug & MIDGARD_DBG_SHADERS)
2058                         print_mir_block(block);
2059         }
2060
2061         if (!ctx->temp_count)
2062                 return;
2063
2064         /* Let's actually do register allocation */
2065         int nodes = ctx->temp_count;
2066         struct ra_graph *g = ra_alloc_interference_graph(regs, nodes);
2067
2068         /* Set everything to the work register class, unless it has somewhere
2069          * special to go */
2070
2071         mir_foreach_block(ctx, block) {
2072                 mir_foreach_instr_in_block(block, ins) {
2073                         if (ins->compact_branch) continue;
2074
2075                         if (ins->ssa_args.dest < 0) continue;
2076
2077                         if (ins->ssa_args.dest >= SSA_FIXED_MINIMUM) continue;
2078
2079                         int class = primary_class;
2080
2081                         ra_set_node_class(g, ins->ssa_args.dest, class);
2082                 }
2083         }
2084
2085         for (int index = 0; index <= ctx->max_hash; ++index) {
2086                 unsigned temp = (uintptr_t) _mesa_hash_table_u64_search(ctx->ssa_to_register, index + 1);
2087
2088                 if (temp) {
2089                         unsigned reg = temp - 1;
2090                         int t = find_or_allocate_temp(ctx, index);
2091                         ra_set_node_reg(g, t, reg);
2092                 }
2093         }
2094
2095         /* Determine liveness */
2096
2097         int *live_start = malloc(nodes * sizeof(int));
2098         int *live_end = malloc(nodes * sizeof(int));
2099
2100         /* Initialize as non-existent */
2101
2102         for (int i = 0; i < nodes; ++i) {
2103                 live_start[i] = live_end[i] = -1;
2104         }
2105
2106         int d = 0;
2107
2108         mir_foreach_block(ctx, block) {
2109                 mir_foreach_instr_in_block(block, ins) {
2110                         if (ins->compact_branch) continue;
2111
2112                         /* Dest is < 0 for store_vary instructions, which break
2113                          * the usual SSA conventions. Liveness analysis doesn't
2114                          * make sense on these instructions, so skip them to
2115                          * avoid memory corruption */
2116
2117                         if (ins->ssa_args.dest < 0) continue;
2118
2119                         if (ins->ssa_args.dest < SSA_FIXED_MINIMUM) {
2120                                 /* If this destination is not yet live, it is now since we just wrote it */
2121
2122                                 int dest = ins->ssa_args.dest;
2123
2124                                 if (live_start[dest] == -1)
2125                                         live_start[dest] = d;
2126                         }
2127
2128                         /* Since we just used a source, the source might be
2129                          * dead now. Scan the rest of the block for
2130                          * invocations, and if there are none, the source dies
2131                          * */
2132
2133                         int sources[2] = { ins->ssa_args.src0, ins->ssa_args.src1 };
2134
2135                         for (int src = 0; src < 2; ++src) {
2136                                 int s = sources[src];
2137
2138                                 if (s < 0) continue;
2139
2140                                 if (s >= SSA_FIXED_MINIMUM) continue;
2141
2142                                 if (!is_live_after(ctx, block, ins, s)) {
2143                                         live_end[s] = d;
2144                                 }
2145                         }
2146
2147                         ++d;
2148                 }
2149         }
2150
2151         /* If a node still hasn't been killed, kill it now */
2152
2153         for (int i = 0; i < nodes; ++i) {
2154                 /* live_start == -1 most likely indicates a pinned output */
2155
2156                 if (live_end[i] == -1)
2157                         live_end[i] = d;
2158         }
2159
2160         /* Setup interference between nodes that are live at the same time */
2161
2162         for (int i = 0; i < nodes; ++i) {
2163                 for (int j = i + 1; j < nodes; ++j) {
2164                         if (!(live_start[i] >= live_end[j] || live_start[j] >= live_end[i]))
2165                                 ra_add_node_interference(g, i, j);
2166                 }
2167         }
2168
2169         ra_set_select_reg_callback(g, midgard_ra_select_callback, NULL);
2170
2171         if (!ra_allocate(g)) {
2172                 DBG("Error allocating registers\n");
2173                 assert(0);
2174         }
2175
2176         /* Cleanup */
2177         free(live_start);
2178         free(live_end);
2179
2180         mir_foreach_block(ctx, block) {
2181                 mir_foreach_instr_in_block(block, ins) {
2182                         if (ins->compact_branch) continue;
2183
2184                         ssa_args args = ins->ssa_args;
2185
2186                         switch (ins->type) {
2187                         case TAG_ALU_4:
2188                                 ins->registers.src1_reg = dealias_register(ctx, g, args.src0, nodes);
2189
2190                                 ins->registers.src2_imm = args.inline_constant;
2191
2192                                 if (args.inline_constant) {
2193                                         /* Encode inline 16-bit constant as a vector by default */
2194
2195                                         ins->registers.src2_reg = ins->inline_constant >> 11;
2196
2197                                         int lower_11 = ins->inline_constant & ((1 << 12) - 1);
2198
2199                                         uint16_t imm = ((lower_11 >> 8) & 0x7) | ((lower_11 & 0xFF) << 3);
2200                                         ins->alu.src2 = imm << 2;
2201                                 } else {
2202                                         ins->registers.src2_reg = dealias_register(ctx, g, args.src1, nodes);
2203                                 }
2204
2205                                 ins->registers.out_reg = dealias_register(ctx, g, args.dest, nodes);
2206
2207                                 break;
2208
2209                         case TAG_LOAD_STORE_4: {
2210                                 if (OP_IS_STORE_VARY(ins->load_store.op)) {
2211                                         /* TODO: use ssa_args for store_vary */
2212                                         ins->load_store.reg = 0;
2213                                 } else {
2214                                         bool has_dest = args.dest >= 0;
2215                                         int ssa_arg = has_dest ? args.dest : args.src0;
2216
2217                                         ins->load_store.reg = dealias_register(ctx, g, ssa_arg, nodes);
2218                                 }
2219
2220                                 break;
2221                         }
2222
2223                         default:
2224                                 break;
2225                         }
2226                 }
2227         }
2228 }
2229
2230 /* Midgard IR only knows vector ALU types, but we sometimes need to actually
2231  * use scalar ALU instructions, for functional or performance reasons. To do
2232  * this, we just demote vector ALU payloads to scalar. */
2233
2234 static int
2235 component_from_mask(unsigned mask)
2236 {
2237         for (int c = 0; c < 4; ++c) {
2238                 if (mask & (3 << (2 * c)))
2239                         return c;
2240         }
2241
2242         assert(0);
2243         return 0;
2244 }
2245
2246 static bool
2247 is_single_component_mask(unsigned mask)
2248 {
2249         int components = 0;
2250
2251         for (int c = 0; c < 4; ++c)
2252                 if (mask & (3 << (2 * c)))
2253                         components++;
2254
2255         return components == 1;
2256 }
2257
2258 /* Create a mask of accessed components from a swizzle to figure out vector
2259  * dependencies */
2260
2261 static unsigned
2262 swizzle_to_access_mask(unsigned swizzle)
2263 {
2264         unsigned component_mask = 0;
2265
2266         for (int i = 0; i < 4; ++i) {
2267                 unsigned c = (swizzle >> (2 * i)) & 3;
2268                 component_mask |= (1 << c);
2269         }
2270
2271         return component_mask;
2272 }
2273
2274 static unsigned
2275 vector_to_scalar_source(unsigned u, bool is_int)
2276 {
2277         midgard_vector_alu_src v;
2278         memcpy(&v, &u, sizeof(v));
2279
2280         /* TODO: Integers */
2281
2282         midgard_scalar_alu_src s = {
2283                 .full = !v.half,
2284                 .component = (v.swizzle & 3) << 1
2285         };
2286
2287         if (is_int) {
2288                 /* TODO */
2289         } else {
2290                 s.abs = v.mod & MIDGARD_FLOAT_MOD_ABS;
2291                 s.negate = v.mod & MIDGARD_FLOAT_MOD_NEG;
2292         }
2293
2294         unsigned o;
2295         memcpy(&o, &s, sizeof(s));
2296
2297         return o & ((1 << 6) - 1);
2298 }
2299
2300 static midgard_scalar_alu
2301 vector_to_scalar_alu(midgard_vector_alu v, midgard_instruction *ins)
2302 {
2303         bool is_int = midgard_is_integer_op(v.op);
2304
2305         /* The output component is from the mask */
2306         midgard_scalar_alu s = {
2307                 .op = v.op,
2308                 .src1 = vector_to_scalar_source(v.src1, is_int),
2309                 .src2 = vector_to_scalar_source(v.src2, is_int),
2310                 .unknown = 0,
2311                 .outmod = v.outmod,
2312                 .output_full = 1, /* TODO: Half */
2313                 .output_component = component_from_mask(v.mask) << 1,
2314         };
2315
2316         /* Inline constant is passed along rather than trying to extract it
2317          * from v */
2318
2319         if (ins->ssa_args.inline_constant) {
2320                 uint16_t imm = 0;
2321                 int lower_11 = ins->inline_constant & ((1 << 12) - 1);
2322                 imm |= (lower_11 >> 9) & 3;
2323                 imm |= (lower_11 >> 6) & 4;
2324                 imm |= (lower_11 >> 2) & 0x38;
2325                 imm |= (lower_11 & 63) << 6;
2326
2327                 s.src2 = imm;
2328         }
2329
2330         return s;
2331 }
2332
2333 /* Midgard prefetches instruction types, so during emission we need to
2334  * lookahead too. Unless this is the last instruction, in which we return 1. Or
2335  * if this is the second to last and the last is an ALU, then it's also 1... */
2336
2337 #define IS_ALU(tag) (tag == TAG_ALU_4 || tag == TAG_ALU_8 ||  \
2338                      tag == TAG_ALU_12 || tag == TAG_ALU_16)
2339
2340 #define EMIT_AND_COUNT(type, val) util_dynarray_append(emission, type, val); \
2341                                   bytes_emitted += sizeof(type)
2342
2343 static void
2344 emit_binary_vector_instruction(midgard_instruction *ains,
2345                                uint16_t *register_words, int *register_words_count,
2346                                uint64_t *body_words, size_t *body_size, int *body_words_count,
2347                                size_t *bytes_emitted)
2348 {
2349         memcpy(&register_words[(*register_words_count)++], &ains->registers, sizeof(ains->registers));
2350         *bytes_emitted += sizeof(midgard_reg_info);
2351
2352         body_size[*body_words_count] = sizeof(midgard_vector_alu);
2353         memcpy(&body_words[(*body_words_count)++], &ains->alu, sizeof(ains->alu));
2354         *bytes_emitted += sizeof(midgard_vector_alu);
2355 }
2356
2357 /* Checks for an SSA data hazard between two adjacent instructions, keeping in
2358  * mind that we are a vector architecture and we can write to different
2359  * components simultaneously */
2360
2361 static bool
2362 can_run_concurrent_ssa(midgard_instruction *first, midgard_instruction *second)
2363 {
2364         /* Each instruction reads some registers and writes to a register. See
2365          * where the first writes */
2366
2367         /* Figure out where exactly we wrote to */
2368         int source = first->ssa_args.dest;
2369         int source_mask = first->type == TAG_ALU_4 ? squeeze_writemask(first->alu.mask) : 0xF;
2370
2371         /* As long as the second doesn't read from the first, we're okay */
2372         if (second->ssa_args.src0 == source) {
2373                 if (first->type == TAG_ALU_4) {
2374                         /* Figure out which components we just read from */
2375
2376                         int q = second->alu.src1;
2377                         midgard_vector_alu_src *m = (midgard_vector_alu_src *) &q;
2378
2379                         /* Check if there are components in common, and fail if so */
2380                         if (swizzle_to_access_mask(m->swizzle) & source_mask)
2381                                 return false;
2382                 } else
2383                         return false;
2384
2385         }
2386
2387         if (second->ssa_args.src1 == source)
2388                 return false;
2389
2390         /* Otherwise, it's safe in that regard. Another data hazard is both
2391          * writing to the same place, of course */
2392
2393         if (second->ssa_args.dest == source) {
2394                 /* ...but only if the components overlap */
2395                 int dest_mask = second->type == TAG_ALU_4 ? squeeze_writemask(second->alu.mask) : 0xF;
2396
2397                 if (dest_mask & source_mask)
2398                         return false;
2399         }
2400
2401         /* ...That's it */
2402         return true;
2403 }
2404
2405 static bool
2406 midgard_has_hazard(
2407                 midgard_instruction **segment, unsigned segment_size,
2408                 midgard_instruction *ains)
2409 {
2410         for (int s = 0; s < segment_size; ++s)
2411                 if (!can_run_concurrent_ssa(segment[s], ains))
2412                         return true;
2413
2414         return false;
2415
2416
2417 }
2418
2419 /* Schedules, but does not emit, a single basic block. After scheduling, the
2420  * final tag and size of the block are known, which are necessary for branching
2421  * */
2422
2423 static midgard_bundle
2424 schedule_bundle(compiler_context *ctx, midgard_block *block, midgard_instruction *ins, int *skip)
2425 {
2426         int instructions_emitted = 0, instructions_consumed = -1;
2427         midgard_bundle bundle = { 0 };
2428
2429         uint8_t tag = ins->type;
2430
2431         /* Default to the instruction's tag */
2432         bundle.tag = tag;
2433
2434         switch (ins->type) {
2435         case TAG_ALU_4: {
2436                 uint32_t control = 0;
2437                 size_t bytes_emitted = sizeof(control);
2438
2439                 /* TODO: Constant combining */
2440                 int index = 0, last_unit = 0;
2441
2442                 /* Previous instructions, for the purpose of parallelism */
2443                 midgard_instruction *segment[4] = {0};
2444                 int segment_size = 0;
2445
2446                 instructions_emitted = -1;
2447                 midgard_instruction *pins = ins;
2448
2449                 for (;;) {
2450                         midgard_instruction *ains = pins;
2451
2452                         /* Advance instruction pointer */
2453                         if (index) {
2454                                 ains = mir_next_op(pins);
2455                                 pins = ains;
2456                         }
2457
2458                         /* Out-of-work condition */
2459                         if ((struct list_head *) ains == &block->instructions)
2460                                 break;
2461
2462                         /* Ensure that the chain can continue */
2463                         if (ains->type != TAG_ALU_4) break;
2464
2465                         /* According to the presentation "The ARM
2466                          * Mali-T880 Mobile GPU" from HotChips 27,
2467                          * there are two pipeline stages. Branching
2468                          * position determined experimentally. Lines
2469                          * are executed in parallel:
2470                          *
2471                          * [ VMUL ] [ SADD ]
2472                          * [ VADD ] [ SMUL ] [ LUT ] [ BRANCH ]
2473                          *
2474                          * Verify that there are no ordering dependencies here.
2475                          *
2476                          * TODO: Allow for parallelism!!!
2477                          */
2478
2479                         /* Pick a unit for it if it doesn't force a particular unit */
2480
2481                         int unit = ains->unit;
2482
2483                         if (!unit) {
2484                                 int op = ains->alu.op;
2485                                 int units = alu_opcode_props[op].props;
2486
2487                                 /* TODO: Promotion of scalars to vectors */
2488                                 int vector = ((!is_single_component_mask(ains->alu.mask)) || ((units & UNITS_SCALAR) == 0)) && (units & UNITS_ANY_VECTOR);
2489
2490                                 if (!vector)
2491                                         assert(units & UNITS_SCALAR);
2492
2493                                 if (vector) {
2494                                         if (last_unit >= UNIT_VADD) {
2495                                                 if (units & UNIT_VLUT)
2496                                                         unit = UNIT_VLUT;
2497                                                 else
2498                                                         break;
2499                                         } else {
2500                                                 if ((units & UNIT_VMUL) && !(control & UNIT_VMUL))
2501                                                         unit = UNIT_VMUL;
2502                                                 else if ((units & UNIT_VADD) && !(control & UNIT_VADD))
2503                                                         unit = UNIT_VADD;
2504                                                 else if (units & UNIT_VLUT)
2505                                                         unit = UNIT_VLUT;
2506                                                 else
2507                                                         break;
2508                                         }
2509                                 } else {
2510                                         if (last_unit >= UNIT_VADD) {
2511                                                 if ((units & UNIT_SMUL) && !(control & UNIT_SMUL))
2512                                                         unit = UNIT_SMUL;
2513                                                 else if (units & UNIT_VLUT)
2514                                                         unit = UNIT_VLUT;
2515                                                 else
2516                                                         break;
2517                                         } else {
2518                                                 if ((units & UNIT_SADD) && !(control & UNIT_SADD) && !midgard_has_hazard(segment, segment_size, ains))
2519                                                         unit = UNIT_SADD;
2520                                                 else if (units & UNIT_SMUL)
2521                                                         unit = ((units & UNIT_VMUL) && !(control & UNIT_VMUL)) ? UNIT_VMUL : UNIT_SMUL;
2522                                                 else if ((units & UNIT_VADD) && !(control & UNIT_VADD))
2523                                                         unit = UNIT_VADD;
2524                                                 else
2525                                                         break;
2526                                         }
2527                                 }
2528
2529                                 assert(unit & units);
2530                         }
2531
2532                         /* Late unit check, this time for encoding (not parallelism) */
2533                         if (unit <= last_unit) break;
2534
2535                         /* Clear the segment */
2536                         if (last_unit < UNIT_VADD && unit >= UNIT_VADD)
2537                                 segment_size = 0;
2538
2539                         if (midgard_has_hazard(segment, segment_size, ains))
2540                                 break;
2541
2542                         /* We're good to go -- emit the instruction */
2543                         ains->unit = unit;
2544
2545                         segment[segment_size++] = ains;
2546
2547                         /* Only one set of embedded constants per
2548                          * bundle possible; if we have more, we must
2549                          * break the chain early, unfortunately */
2550
2551                         if (ains->has_constants) {
2552                                 if (bundle.has_embedded_constants) {
2553                                         /* ...but if there are already
2554                                          * constants but these are the
2555                                          * *same* constants, we let it
2556                                          * through */
2557
2558                                         if (memcmp(bundle.constants, ains->constants, sizeof(bundle.constants)))
2559                                                 break;
2560                                 } else {
2561                                         bundle.has_embedded_constants = true;
2562                                         memcpy(bundle.constants, ains->constants, sizeof(bundle.constants));
2563
2564                                         /* If this is a blend shader special constant, track it for patching */
2565                                         if (ains->has_blend_constant)
2566                                                 bundle.has_blend_constant = true;
2567                                 }
2568                         }
2569
2570                         if (ains->unit & UNITS_ANY_VECTOR) {
2571                                 emit_binary_vector_instruction(ains, bundle.register_words,
2572                                                                &bundle.register_words_count, bundle.body_words,
2573                                                                bundle.body_size, &bundle.body_words_count, &bytes_emitted);
2574                         } else if (ains->compact_branch) {
2575                                 /* All of r0 has to be written out
2576                                  * along with the branch writeout.
2577                                  * (slow!) */
2578
2579                                 if (ains->writeout) {
2580                                         if (index == 0) {
2581                                                 midgard_instruction ins = v_fmov(0, blank_alu_src, SSA_FIXED_REGISTER(0));
2582                                                 ins.unit = UNIT_VMUL;
2583
2584                                                 control |= ins.unit;
2585
2586                                                 emit_binary_vector_instruction(&ins, bundle.register_words,
2587                                                                                &bundle.register_words_count, bundle.body_words,
2588                                                                                bundle.body_size, &bundle.body_words_count, &bytes_emitted);
2589                                         } else {
2590                                                 /* Analyse the group to see if r0 is written in full, on-time, without hanging dependencies*/
2591                                                 bool written_late = false;
2592                                                 bool components[4] = { 0 };
2593                                                 uint16_t register_dep_mask = 0;
2594                                                 uint16_t written_mask = 0;
2595
2596                                                 midgard_instruction *qins = ins;
2597                                                 for (int t = 0; t < index; ++t) {
2598                                                         if (qins->registers.out_reg != 0) {
2599                                                                 /* Mark down writes */
2600
2601                                                                 written_mask |= (1 << qins->registers.out_reg);
2602                                                         } else {
2603                                                                 /* Mark down the register dependencies for errata check */
2604
2605                                                                 if (qins->registers.src1_reg < 16)
2606                                                                         register_dep_mask |= (1 << qins->registers.src1_reg);
2607
2608                                                                 if (qins->registers.src2_reg < 16)
2609                                                                         register_dep_mask |= (1 << qins->registers.src2_reg);
2610
2611                                                                 int mask = qins->alu.mask;
2612
2613                                                                 for (int c = 0; c < 4; ++c)
2614                                                                         if (mask & (0x3 << (2 * c)))
2615                                                                                 components[c] = true;
2616
2617                                                                 /* ..but if the writeout is too late, we have to break up anyway... for some reason */
2618
2619                                                                 if (qins->unit == UNIT_VLUT)
2620                                                                         written_late = true;
2621                                                         }
2622
2623                                                         /* Advance instruction pointer */
2624                                                         qins = mir_next_op(qins);
2625                                                 }
2626
2627
2628                                                 /* ERRATA (?): In a bundle ending in a fragment writeout, the register dependencies of r0 cannot be written within this bundle (discovered in -bshading:shading=phong) */
2629                                                 if (register_dep_mask & written_mask) {
2630                                                         DBG("ERRATA WORKAROUND: Breakup for writeout dependency masks %X vs %X (common %X)\n", register_dep_mask, written_mask, register_dep_mask & written_mask);
2631                                                         break;
2632                                                 }
2633
2634                                                 if (written_late)
2635                                                         break;
2636
2637                                                 /* If even a single component is not written, break it up (conservative check). */
2638                                                 bool breakup = false;
2639
2640                                                 for (int c = 0; c < 4; ++c)
2641                                                         if (!components[c])
2642                                                                 breakup = true;
2643
2644                                                 if (breakup)
2645                                                         break;
2646
2647                                                 /* Otherwise, we're free to proceed */
2648                                         }
2649                                 }
2650
2651                                 if (ains->unit == ALU_ENAB_BRANCH) {
2652                                         bundle.body_size[bundle.body_words_count] = sizeof(midgard_branch_extended);
2653                                         memcpy(&bundle.body_words[bundle.body_words_count++], &ains->branch_extended, sizeof(midgard_branch_extended));
2654                                         bytes_emitted += sizeof(midgard_branch_extended);
2655                                 } else {
2656                                         bundle.body_size[bundle.body_words_count] = sizeof(ains->br_compact);
2657                                         memcpy(&bundle.body_words[bundle.body_words_count++], &ains->br_compact, sizeof(ains->br_compact));
2658                                         bytes_emitted += sizeof(ains->br_compact);
2659                                 }
2660                         } else {
2661                                 memcpy(&bundle.register_words[bundle.register_words_count++], &ains->registers, sizeof(ains->registers));
2662                                 bytes_emitted += sizeof(midgard_reg_info);
2663
2664                                 bundle.body_size[bundle.body_words_count] = sizeof(midgard_scalar_alu);
2665                                 bundle.body_words_count++;
2666                                 bytes_emitted += sizeof(midgard_scalar_alu);
2667                         }
2668
2669                         /* Defer marking until after writing to allow for break */
2670                         control |= ains->unit;
2671                         last_unit = ains->unit;
2672                         ++instructions_emitted;
2673                         ++index;
2674                 }
2675
2676                 /* Bubble up the number of instructions for skipping */
2677                 instructions_consumed = index - 1;
2678
2679                 int padding = 0;
2680
2681                 /* Pad ALU op to nearest word */
2682
2683                 if (bytes_emitted & 15) {
2684                         padding = 16 - (bytes_emitted & 15);
2685                         bytes_emitted += padding;
2686                 }
2687
2688                 /* Constants must always be quadwords */
2689                 if (bundle.has_embedded_constants)
2690                         bytes_emitted += 16;
2691
2692                 /* Size ALU instruction for tag */
2693                 bundle.tag = (TAG_ALU_4) + (bytes_emitted / 16) - 1;
2694                 bundle.padding = padding;
2695                 bundle.control = bundle.tag | control;
2696
2697                 break;
2698         }
2699
2700         case TAG_LOAD_STORE_4: {
2701                 /* Load store instructions have two words at once. If
2702                  * we only have one queued up, we need to NOP pad.
2703                  * Otherwise, we store both in succession to save space
2704                  * and cycles -- letting them go in parallel -- skip
2705                  * the next. The usefulness of this optimisation is
2706                  * greatly dependent on the quality of the instruction
2707                  * scheduler.
2708                  */
2709
2710                 midgard_instruction *next_op = mir_next_op(ins);
2711
2712                 if ((struct list_head *) next_op != &block->instructions && next_op->type == TAG_LOAD_STORE_4) {
2713                         /* As the two operate concurrently, make sure
2714                          * they are not dependent */
2715
2716                         if (can_run_concurrent_ssa(ins, next_op) || true) {
2717                                 /* Skip ahead, since it's redundant with the pair */
2718                                 instructions_consumed = 1 + (instructions_emitted++);
2719                         }
2720                 }
2721
2722                 break;
2723         }
2724
2725         default:
2726                 /* Texture ops default to single-op-per-bundle scheduling */
2727                 break;
2728         }
2729
2730         /* Copy the instructions into the bundle */
2731         bundle.instruction_count = instructions_emitted + 1;
2732
2733         int used_idx = 0;
2734
2735         midgard_instruction *uins = ins;
2736         for (int i = 0; used_idx < bundle.instruction_count; ++i) {
2737                 bundle.instructions[used_idx++] = *uins;
2738                 uins = mir_next_op(uins);
2739         }
2740
2741         *skip = (instructions_consumed == -1) ? instructions_emitted : instructions_consumed;
2742
2743         return bundle;
2744 }
2745
2746 static int
2747 quadword_size(int tag)
2748 {
2749         switch (tag) {
2750         case TAG_ALU_4:
2751                 return 1;
2752
2753         case TAG_ALU_8:
2754                 return 2;
2755
2756         case TAG_ALU_12:
2757                 return 3;
2758
2759         case TAG_ALU_16:
2760                 return 4;
2761
2762         case TAG_LOAD_STORE_4:
2763                 return 1;
2764
2765         case TAG_TEXTURE_4:
2766                 return 1;
2767
2768         default:
2769                 assert(0);
2770                 return 0;
2771         }
2772 }
2773
2774 /* Schedule a single block by iterating its instruction to create bundles.
2775  * While we go, tally about the bundle sizes to compute the block size. */
2776
2777 static void
2778 schedule_block(compiler_context *ctx, midgard_block *block)
2779 {
2780         util_dynarray_init(&block->bundles, NULL);
2781
2782         block->quadword_count = 0;
2783
2784         mir_foreach_instr_in_block(block, ins) {
2785                 int skip;
2786                 midgard_bundle bundle = schedule_bundle(ctx, block, ins, &skip);
2787                 util_dynarray_append(&block->bundles, midgard_bundle, bundle);
2788
2789                 if (bundle.has_blend_constant) {
2790                         /* TODO: Multiblock? */
2791                         int quadwords_within_block = block->quadword_count + quadword_size(bundle.tag) - 1;
2792                         ctx->blend_constant_offset = quadwords_within_block * 0x10;
2793                 }
2794
2795                 while(skip--)
2796                         ins = mir_next_op(ins);
2797
2798                 block->quadword_count += quadword_size(bundle.tag);
2799         }
2800
2801         block->is_scheduled = true;
2802 }
2803
2804 static void
2805 schedule_program(compiler_context *ctx)
2806 {
2807         allocate_registers(ctx);
2808
2809         mir_foreach_block(ctx, block) {
2810                 schedule_block(ctx, block);
2811         }
2812 }
2813
2814 /* After everything is scheduled, emit whole bundles at a time */
2815
2816 static void
2817 emit_binary_bundle(compiler_context *ctx, midgard_bundle *bundle, struct util_dynarray *emission, int next_tag)
2818 {
2819         int lookahead = next_tag << 4;
2820
2821         switch (bundle->tag) {
2822         case TAG_ALU_4:
2823         case TAG_ALU_8:
2824         case TAG_ALU_12:
2825         case TAG_ALU_16: {
2826                 /* Actually emit each component */
2827                 util_dynarray_append(emission, uint32_t, bundle->control | lookahead);
2828
2829                 for (int i = 0; i < bundle->register_words_count; ++i)
2830                         util_dynarray_append(emission, uint16_t, bundle->register_words[i]);
2831
2832                 /* Emit body words based on the instructions bundled */
2833                 for (int i = 0; i < bundle->instruction_count; ++i) {
2834                         midgard_instruction *ins = &bundle->instructions[i];
2835
2836                         if (ins->unit & UNITS_ANY_VECTOR) {
2837                                 memcpy(util_dynarray_grow(emission, sizeof(midgard_vector_alu)), &ins->alu, sizeof(midgard_vector_alu));
2838                         } else if (ins->compact_branch) {
2839                                 /* Dummy move, XXX DRY */
2840                                 if ((i == 0) && ins->writeout) {
2841                                         midgard_instruction ins = v_fmov(0, blank_alu_src, SSA_FIXED_REGISTER(0));
2842                                         memcpy(util_dynarray_grow(emission, sizeof(midgard_vector_alu)), &ins.alu, sizeof(midgard_vector_alu));
2843                                 }
2844
2845                                 if (ins->unit == ALU_ENAB_BR_COMPACT) {
2846                                         memcpy(util_dynarray_grow(emission, sizeof(ins->br_compact)), &ins->br_compact, sizeof(ins->br_compact));
2847                                 } else {
2848                                         memcpy(util_dynarray_grow(emission, sizeof(ins->branch_extended)), &ins->branch_extended, sizeof(ins->branch_extended));
2849                                 }
2850                         } else {
2851                                 /* Scalar */
2852                                 midgard_scalar_alu scalarised = vector_to_scalar_alu(ins->alu, ins);
2853                                 memcpy(util_dynarray_grow(emission, sizeof(scalarised)), &scalarised, sizeof(scalarised));
2854                         }
2855                 }
2856
2857                 /* Emit padding (all zero) */
2858                 memset(util_dynarray_grow(emission, bundle->padding), 0, bundle->padding);
2859
2860                 /* Tack on constants */
2861
2862                 if (bundle->has_embedded_constants) {
2863                         util_dynarray_append(emission, float, bundle->constants[0]);
2864                         util_dynarray_append(emission, float, bundle->constants[1]);
2865                         util_dynarray_append(emission, float, bundle->constants[2]);
2866                         util_dynarray_append(emission, float, bundle->constants[3]);
2867                 }
2868
2869                 break;
2870         }
2871
2872         case TAG_LOAD_STORE_4: {
2873                 /* One or two composing instructions */
2874
2875                 uint64_t current64, next64 = LDST_NOP;
2876
2877                 memcpy(&current64, &bundle->instructions[0].load_store, sizeof(current64));
2878
2879                 if (bundle->instruction_count == 2)
2880                         memcpy(&next64, &bundle->instructions[1].load_store, sizeof(next64));
2881
2882                 midgard_load_store instruction = {
2883                         .type = bundle->tag,
2884                         .next_type = next_tag,
2885                         .word1 = current64,
2886                         .word2 = next64
2887                 };
2888
2889                 util_dynarray_append(emission, midgard_load_store, instruction);
2890
2891                 break;
2892         }
2893
2894         case TAG_TEXTURE_4: {
2895                 /* Texture instructions are easy, since there is no
2896                  * pipelining nor VLIW to worry about. We may need to set the .last flag */
2897
2898                 midgard_instruction *ins = &bundle->instructions[0];
2899
2900                 ins->texture.type = TAG_TEXTURE_4;
2901                 ins->texture.next_type = next_tag;
2902
2903                 ctx->texture_op_count--;
2904
2905                 if (!ctx->texture_op_count) {
2906                         ins->texture.cont = 0;
2907                         ins->texture.last = 1;
2908                 }
2909
2910                 util_dynarray_append(emission, midgard_texture_word, ins->texture);
2911                 break;
2912         }
2913
2914         default:
2915                 DBG("Unknown midgard instruction type\n");
2916                 assert(0);
2917                 break;
2918         }
2919 }
2920
2921
2922 /* ALU instructions can inline or embed constants, which decreases register
2923  * pressure and saves space. */
2924
2925 #define CONDITIONAL_ATTACH(src) { \
2926         void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->ssa_args.src + 1); \
2927 \
2928         if (entry) { \
2929                 attach_constants(ctx, alu, entry, alu->ssa_args.src + 1); \
2930                 alu->ssa_args.src = SSA_FIXED_REGISTER(REGISTER_CONSTANT); \
2931         } \
2932 }
2933
2934 static void
2935 inline_alu_constants(compiler_context *ctx)
2936 {
2937         mir_foreach_instr(ctx, alu) {
2938                 /* Other instructions cannot inline constants */
2939                 if (alu->type != TAG_ALU_4) continue;
2940
2941                 /* If there is already a constant here, we can do nothing */
2942                 if (alu->has_constants) continue;
2943
2944                 /* It makes no sense to inline constants on a branch */
2945                 if (alu->compact_branch || alu->prepacked_branch) continue;
2946
2947                 CONDITIONAL_ATTACH(src0);
2948
2949                 if (!alu->has_constants) {
2950                         CONDITIONAL_ATTACH(src1)
2951                 } else if (!alu->inline_constant) {
2952                         /* Corner case: _two_ vec4 constants, for instance with a
2953                          * csel. For this case, we can only use a constant
2954                          * register for one, we'll have to emit a move for the
2955                          * other. Note, if both arguments are constants, then
2956                          * necessarily neither argument depends on the value of
2957                          * any particular register. As the destination register
2958                          * will be wiped, that means we can spill the constant
2959                          * to the destination register.
2960                          */
2961
2962                         void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->ssa_args.src1 + 1);
2963                         unsigned scratch = alu->ssa_args.dest;
2964
2965                         if (entry) {
2966                                 midgard_instruction ins = v_fmov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), blank_alu_src, scratch);
2967                                 attach_constants(ctx, &ins, entry, alu->ssa_args.src1 + 1);
2968
2969                                 /* Force a break XXX Defer r31 writes */
2970                                 ins.unit = UNIT_VLUT;
2971
2972                                 /* Set the source */
2973                                 alu->ssa_args.src1 = scratch;
2974
2975                                 /* Inject us -before- the last instruction which set r31 */
2976                                 mir_insert_instruction_before(mir_prev_op(alu), ins);
2977                         }
2978                 }
2979         }
2980 }
2981
2982 /* Midgard supports two types of constants, embedded constants (128-bit) and
2983  * inline constants (16-bit). Sometimes, especially with scalar ops, embedded
2984  * constants can be demoted to inline constants, for space savings and
2985  * sometimes a performance boost */
2986
2987 static void
2988 embedded_to_inline_constant(compiler_context *ctx)
2989 {
2990         mir_foreach_instr(ctx, ins) {
2991                 if (!ins->has_constants) continue;
2992
2993                 if (ins->ssa_args.inline_constant) continue;
2994
2995                 /* Blend constants must not be inlined by definition */
2996                 if (ins->has_blend_constant) continue;
2997
2998                 /* src1 cannot be an inline constant due to encoding
2999                  * restrictions. So, if possible we try to flip the arguments
3000                  * in that case */
3001
3002                 int op = ins->alu.op;
3003
3004                 if (ins->ssa_args.src0 == SSA_FIXED_REGISTER(REGISTER_CONSTANT)) {
3005                         switch (op) {
3006                         /* These ops require an operational change to flip
3007                          * their arguments TODO */
3008                         case midgard_alu_op_flt:
3009                         case midgard_alu_op_fle:
3010                         case midgard_alu_op_ilt:
3011                         case midgard_alu_op_ile:
3012                         case midgard_alu_op_fcsel:
3013                         case midgard_alu_op_icsel:
3014                                 DBG("Missed non-commutative flip (%s)\n", alu_opcode_props[op].name);
3015                         default:
3016                                 break;
3017                         }
3018
3019                         if (alu_opcode_props[op].props & OP_COMMUTES) {
3020                                 /* Flip the SSA numbers */
3021                                 ins->ssa_args.src0 = ins->ssa_args.src1;
3022                                 ins->ssa_args.src1 = SSA_FIXED_REGISTER(REGISTER_CONSTANT);
3023
3024                                 /* And flip the modifiers */
3025
3026                                 unsigned src_temp;
3027
3028                                 src_temp = ins->alu.src2;
3029                                 ins->alu.src2 = ins->alu.src1;
3030                                 ins->alu.src1 = src_temp;
3031                         }
3032                 }
3033
3034                 if (ins->ssa_args.src1 == SSA_FIXED_REGISTER(REGISTER_CONSTANT)) {
3035                         /* Extract the source information */
3036
3037                         midgard_vector_alu_src *src;
3038                         int q = ins->alu.src2;
3039                         midgard_vector_alu_src *m = (midgard_vector_alu_src *) &q;
3040                         src = m;
3041
3042                         /* Component is from the swizzle, e.g. r26.w -> w component. TODO: What if x is masked out? */
3043                         int component = src->swizzle & 3;
3044
3045                         /* Scale constant appropriately, if we can legally */
3046                         uint16_t scaled_constant = 0;
3047
3048                         /* XXX: Check legality */
3049                         if (midgard_is_integer_op(op)) {
3050                                 /* TODO: Inline integer */
3051                                 continue;
3052
3053                                 unsigned int *iconstants = (unsigned int *) ins->constants;
3054                                 scaled_constant = (uint16_t) iconstants[component];
3055
3056                                 /* Constant overflow after resize */
3057                                 if (scaled_constant != iconstants[component])
3058                                         continue;
3059                         } else {
3060                                 float original = (float) ins->constants[component];
3061                                 scaled_constant = _mesa_float_to_half(original);
3062
3063                                 /* Check for loss of precision. If this is
3064                                  * mediump, we don't care, but for a highp
3065                                  * shader, we need to pay attention. NIR
3066                                  * doesn't yet tell us which mode we're in!
3067                                  * Practically this prevents most constants
3068                                  * from being inlined, sadly. */
3069
3070                                 float fp32 = _mesa_half_to_float(scaled_constant);
3071
3072                                 if (fp32 != original)
3073                                         continue;
3074                         }
3075
3076                         /* We don't know how to handle these with a constant */
3077
3078                         if (src->mod || src->half || src->rep_low || src->rep_high) {
3079                                 DBG("Bailing inline constant...\n");
3080                                 continue;
3081                         }
3082
3083                         /* Make sure that the constant is not itself a
3084                          * vector by checking if all accessed values
3085                          * (by the swizzle) are the same. */
3086
3087                         uint32_t *cons = (uint32_t *) ins->constants;
3088                         uint32_t value = cons[component];
3089
3090                         bool is_vector = false;
3091                         unsigned mask = effective_writemask(&ins->alu);
3092
3093                         for (int c = 1; c < 4; ++c) {
3094                                 /* We only care if this component is actually used */
3095                                 if (!(mask & (1 << c)))
3096                                         continue;
3097
3098                                 uint32_t test = cons[(src->swizzle >> (2 * c)) & 3];
3099
3100                                 if (test != value) {
3101                                         is_vector = true;
3102                                         break;
3103                                 }
3104                         }
3105
3106                         if (is_vector)
3107                                 continue;
3108
3109                         /* Get rid of the embedded constant */
3110                         ins->has_constants = false;
3111                         ins->ssa_args.src1 = SSA_UNUSED_0;
3112                         ins->ssa_args.inline_constant = true;
3113                         ins->inline_constant = scaled_constant;
3114                 }
3115         }
3116 }
3117
3118 /* Map normal SSA sources to other SSA sources / fixed registers (like
3119  * uniforms) */
3120
3121 static void
3122 map_ssa_to_alias(compiler_context *ctx, int *ref)
3123 {
3124         unsigned int alias = (uintptr_t) _mesa_hash_table_u64_search(ctx->ssa_to_alias, *ref + 1);
3125
3126         if (alias) {
3127                 /* Remove entry in leftovers to avoid a redunant fmov */
3128
3129                 struct set_entry *leftover = _mesa_set_search(ctx->leftover_ssa_to_alias, ((void *) (uintptr_t) (*ref + 1)));
3130
3131                 if (leftover)
3132                         _mesa_set_remove(ctx->leftover_ssa_to_alias, leftover);
3133
3134                 /* Assign the alias map */
3135                 *ref = alias - 1;
3136                 return;
3137         }
3138 }
3139
3140 /* Basic dead code elimination on the MIR itself, which cleans up e.g. the
3141  * texture pipeline */
3142
3143 static bool
3144 midgard_opt_dead_code_eliminate(compiler_context *ctx, midgard_block *block)
3145 {
3146         bool progress = false;
3147
3148         mir_foreach_instr_in_block_safe(block, ins) {
3149                 if (ins->type != TAG_ALU_4) continue;
3150                 if (ins->compact_branch) continue;
3151
3152                 if (ins->ssa_args.dest >= SSA_FIXED_MINIMUM) continue;
3153                 if (midgard_is_pinned(ctx, ins->ssa_args.dest)) continue;
3154                 if (is_live_after(ctx, block, ins, ins->ssa_args.dest)) continue;
3155
3156                 mir_remove_instruction(ins);
3157                 progress = true;
3158         }
3159
3160         return progress;
3161 }
3162
3163 static bool
3164 midgard_opt_copy_prop(compiler_context *ctx, midgard_block *block)
3165 {
3166         bool progress = false;
3167
3168         mir_foreach_instr_in_block_safe(block, ins) {
3169                 if (ins->type != TAG_ALU_4) continue;
3170                 if (!OP_IS_MOVE(ins->alu.op)) continue;
3171
3172                 unsigned from = ins->ssa_args.src1;
3173                 unsigned to = ins->ssa_args.dest;
3174
3175                 /* We only work on pure SSA */
3176
3177                 if (to >= SSA_FIXED_MINIMUM) continue;
3178                 if (from >= SSA_FIXED_MINIMUM) continue;
3179                 if (to >= ctx->func->impl->ssa_alloc) continue;
3180                 if (from >= ctx->func->impl->ssa_alloc) continue;
3181
3182                 /* Also, if the move has side effects, we're helpless */
3183
3184                 midgard_vector_alu_src src =
3185                         vector_alu_from_unsigned(ins->alu.src2);
3186                 unsigned mask = squeeze_writemask(ins->alu.mask);
3187                 bool is_int = midgard_is_integer_op(ins->alu.op);
3188
3189                 if (mir_nontrivial_mod(src, is_int, mask)) continue;
3190                 if (ins->alu.outmod != midgard_outmod_none) continue;
3191
3192                 mir_foreach_instr_in_block_from(block, v, mir_next_op(ins)) {
3193                         if (v->ssa_args.src0 == to) {
3194                                 v->ssa_args.src0 = from;
3195                                 progress = true;
3196                         }
3197
3198                         if (v->ssa_args.src1 == to && !v->ssa_args.inline_constant) {
3199                                 v->ssa_args.src1 = from;
3200                                 progress = true;
3201                         }
3202                 }
3203         }
3204
3205         return progress;
3206 }
3207
3208 static bool
3209 midgard_opt_copy_prop_tex(compiler_context *ctx, midgard_block *block)
3210 {
3211         bool progress = false;
3212
3213         mir_foreach_instr_in_block_safe(block, ins) {
3214                 if (ins->type != TAG_ALU_4) continue;
3215                 if (!OP_IS_MOVE(ins->alu.op)) continue;
3216
3217                 unsigned from = ins->ssa_args.src1;
3218                 unsigned to = ins->ssa_args.dest;
3219
3220                 /* Make sure it's simple enough for us to handle */
3221
3222                 if (from >= SSA_FIXED_MINIMUM) continue;
3223                 if (from >= ctx->func->impl->ssa_alloc) continue;
3224                 if (to < SSA_FIXED_REGISTER(REGISTER_TEXTURE_BASE)) continue;
3225                 if (to > SSA_FIXED_REGISTER(REGISTER_TEXTURE_BASE + 1)) continue;
3226
3227                 bool eliminated = false;
3228
3229                 mir_foreach_instr_in_block_from_rev(block, v, mir_prev_op(ins)) {
3230                         /* The texture registers are not SSA so be careful.
3231                          * Conservatively, just stop if we hit a texture op
3232                          * (even if it may not write) to where we are */
3233
3234                         if (v->type != TAG_ALU_4)
3235                                 break;
3236
3237                         if (v->ssa_args.dest == from) {
3238                                 /* We don't want to track partial writes ... */
3239                                 if (v->alu.mask == 0xF) {
3240                                         v->ssa_args.dest = to;
3241                                         eliminated = true;
3242                                 }
3243
3244                                 break;
3245                         }
3246                 }
3247
3248                 if (eliminated)
3249                         mir_remove_instruction(ins);
3250
3251                 progress |= eliminated;
3252         }
3253
3254         return progress;
3255 }
3256
3257 /* We don't really understand the imov/fmov split, so always use fmov (but let
3258  * it be imov in the IR so we don't do unsafe floating point "optimizations"
3259  * and break things */
3260
3261 static void
3262 midgard_imov_workaround(compiler_context *ctx, midgard_block *block)
3263 {
3264         mir_foreach_instr_in_block_safe(block, ins) {
3265                 if (ins->type != TAG_ALU_4) continue;
3266                 if (ins->alu.op != midgard_alu_op_imov) continue;
3267
3268                 ins->alu.op = midgard_alu_op_fmov;
3269                 ins->alu.outmod = midgard_outmod_none;
3270
3271                 /* Remove flags that don't make sense */
3272
3273                 midgard_vector_alu_src s =
3274                         vector_alu_from_unsigned(ins->alu.src2);
3275
3276                 s.mod = 0;
3277
3278                 ins->alu.src2 = vector_alu_srco_unsigned(s);
3279         }
3280 }
3281
3282 /* The following passes reorder MIR instructions to enable better scheduling */
3283
3284 static void
3285 midgard_pair_load_store(compiler_context *ctx, midgard_block *block)
3286 {
3287         mir_foreach_instr_in_block_safe(block, ins) {
3288                 if (ins->type != TAG_LOAD_STORE_4) continue;
3289
3290                 /* We've found a load/store op. Check if next is also load/store. */
3291                 midgard_instruction *next_op = mir_next_op(ins);
3292                 if (&next_op->link != &block->instructions) {
3293                         if (next_op->type == TAG_LOAD_STORE_4) {
3294                                 /* If so, we're done since we're a pair */
3295                                 ins = mir_next_op(ins);
3296                                 continue;
3297                         }
3298
3299                         /* Maximum search distance to pair, to avoid register pressure disasters */
3300                         int search_distance = 8;
3301
3302                         /* Otherwise, we have an orphaned load/store -- search for another load */
3303                         mir_foreach_instr_in_block_from(block, c, mir_next_op(ins)) {
3304                                 /* Terminate search if necessary */
3305                                 if (!(search_distance--)) break;
3306
3307                                 if (c->type != TAG_LOAD_STORE_4) continue;
3308
3309                                 /* Stores cannot be reordered, since they have
3310                                  * dependencies. For the same reason, indirect
3311                                  * loads cannot be reordered as their index is
3312                                  * loaded in r27.w */
3313
3314                                 if (OP_IS_STORE(c->load_store.op)) continue;
3315
3316                                 /* It appears the 0x800 bit is set whenever a
3317                                  * load is direct, unset when it is indirect.
3318                                  * Skip indirect loads. */
3319
3320                                 if (!(c->load_store.unknown & 0x800)) continue;
3321
3322                                 /* We found one! Move it up to pair and remove it from the old location */
3323
3324                                 mir_insert_instruction_before(ins, *c);
3325                                 mir_remove_instruction(c);
3326
3327                                 break;
3328                         }
3329                 }
3330         }
3331 }
3332
3333 /* Emit varying stores late */
3334
3335 static void
3336 midgard_emit_store(compiler_context *ctx, midgard_block *block) {
3337         /* Iterate in reverse to get the final write, rather than the first */
3338
3339         mir_foreach_instr_in_block_safe_rev(block, ins) {
3340                 /* Check if what we just wrote needs a store */
3341                 int idx = ins->ssa_args.dest;
3342                 uintptr_t varying = ((uintptr_t) _mesa_hash_table_u64_search(ctx->ssa_varyings, idx + 1));
3343
3344                 if (!varying) continue;
3345
3346                 varying -= 1;
3347
3348                 /* We need to store to the appropriate varying, so emit the
3349                  * move/store */
3350
3351                 /* TODO: Integrate with special purpose RA (and scheduler?) */
3352                 bool high_varying_register = false;
3353
3354                 midgard_instruction mov = v_fmov(idx, blank_alu_src, SSA_FIXED_REGISTER(REGISTER_VARYING_BASE + high_varying_register));
3355
3356                 midgard_instruction st = m_store_vary_32(SSA_FIXED_REGISTER(high_varying_register), varying);
3357                 st.load_store.unknown = 0x1E9E; /* XXX: What is this? */
3358
3359                 mir_insert_instruction_before(mir_next_op(ins), st);
3360                 mir_insert_instruction_before(mir_next_op(ins), mov);
3361
3362                 /* We no longer need to store this varying */
3363                 _mesa_hash_table_u64_remove(ctx->ssa_varyings, idx + 1);
3364         }
3365 }
3366
3367 /* If there are leftovers after the below pass, emit actual fmov
3368  * instructions for the slow-but-correct path */
3369
3370 static void
3371 emit_leftover_move(compiler_context *ctx)
3372 {
3373         set_foreach(ctx->leftover_ssa_to_alias, leftover) {
3374                 int base = ((uintptr_t) leftover->key) - 1;
3375                 int mapped = base;
3376
3377                 map_ssa_to_alias(ctx, &mapped);
3378                 EMIT(fmov, mapped, blank_alu_src, base);
3379         }
3380 }
3381
3382 static void
3383 actualise_ssa_to_alias(compiler_context *ctx)
3384 {
3385         mir_foreach_instr(ctx, ins) {
3386                 map_ssa_to_alias(ctx, &ins->ssa_args.src0);
3387                 map_ssa_to_alias(ctx, &ins->ssa_args.src1);
3388         }
3389
3390         emit_leftover_move(ctx);
3391 }
3392
3393 static void
3394 emit_fragment_epilogue(compiler_context *ctx)
3395 {
3396         /* Special case: writing out constants requires us to include the move
3397          * explicitly now, so shove it into r0 */
3398
3399         void *constant_value = _mesa_hash_table_u64_search(ctx->ssa_constants, ctx->fragment_output + 1);
3400
3401         if (constant_value) {
3402                 midgard_instruction ins = v_fmov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), blank_alu_src, SSA_FIXED_REGISTER(0));
3403                 attach_constants(ctx, &ins, constant_value, ctx->fragment_output + 1);
3404                 emit_mir_instruction(ctx, ins);
3405         }
3406
3407         /* Perform the actual fragment writeout. We have two writeout/branch
3408          * instructions, forming a loop until writeout is successful as per the
3409          * docs. TODO: gl_FragDepth */
3410
3411         EMIT(alu_br_compact_cond, midgard_jmp_writeout_op_writeout, TAG_ALU_4, 0, midgard_condition_always);
3412         EMIT(alu_br_compact_cond, midgard_jmp_writeout_op_writeout, TAG_ALU_4, -1, midgard_condition_always);
3413 }
3414
3415 /* For the blend epilogue, we need to convert the blended fragment vec4 (stored
3416  * in r0) to a RGBA8888 value by scaling and type converting. We then output it
3417  * with the int8 analogue to the fragment epilogue */
3418
3419 static void
3420 emit_blend_epilogue(compiler_context *ctx)
3421 {
3422         /* vmul.fmul.none.fulllow hr48, r0, #255 */
3423
3424         midgard_instruction scale = {
3425                 .type = TAG_ALU_4,
3426                 .unit = UNIT_VMUL,
3427                 .inline_constant = _mesa_float_to_half(255.0),
3428                 .ssa_args = {
3429                         .src0 = SSA_FIXED_REGISTER(0),
3430                         .src1 = SSA_UNUSED_0,
3431                         .dest = SSA_FIXED_REGISTER(24),
3432                         .inline_constant = true
3433                 },
3434                 .alu = {
3435                         .op = midgard_alu_op_fmul,
3436                         .reg_mode = midgard_reg_mode_full,
3437                         .dest_override = midgard_dest_override_lower,
3438                         .mask = 0xFF,
3439                         .src1 = vector_alu_srco_unsigned(blank_alu_src),
3440                         .src2 = vector_alu_srco_unsigned(blank_alu_src),
3441                 }
3442         };
3443
3444         emit_mir_instruction(ctx, scale);
3445
3446         /* vadd.f2u8.pos.low hr0, hr48, #0 */
3447
3448         midgard_vector_alu_src alu_src = blank_alu_src;
3449         alu_src.half = true;
3450
3451         midgard_instruction f2u8 = {
3452                 .type = TAG_ALU_4,
3453                 .ssa_args = {
3454                         .src0 = SSA_FIXED_REGISTER(24),
3455                         .src1 = SSA_UNUSED_0,
3456                         .dest = SSA_FIXED_REGISTER(0),
3457                         .inline_constant = true
3458                 },
3459                 .alu = {
3460                         .op = midgard_alu_op_f2u8,
3461                         .reg_mode = midgard_reg_mode_half,
3462                         .dest_override = midgard_dest_override_lower,
3463                         .outmod = midgard_outmod_pos,
3464                         .mask = 0xF,
3465                         .src1 = vector_alu_srco_unsigned(alu_src),
3466                         .src2 = vector_alu_srco_unsigned(blank_alu_src),
3467                 }
3468         };
3469
3470         emit_mir_instruction(ctx, f2u8);
3471
3472         /* vmul.imov.quarter r0, r0, r0 */
3473
3474         midgard_instruction imov_8 = {
3475                 .type = TAG_ALU_4,
3476                 .ssa_args = {
3477                         .src0 = SSA_UNUSED_1,
3478                         .src1 = SSA_FIXED_REGISTER(0),
3479                         .dest = SSA_FIXED_REGISTER(0),
3480                 },
3481                 .alu = {
3482                         .op = midgard_alu_op_imov,
3483                         .reg_mode = midgard_reg_mode_quarter,
3484                         .dest_override = midgard_dest_override_none,
3485                         .mask = 0xFF,
3486                         .src1 = vector_alu_srco_unsigned(blank_alu_src),
3487                         .src2 = vector_alu_srco_unsigned(blank_alu_src),
3488                 }
3489         };
3490
3491         /* Emit branch epilogue with the 8-bit move as the source */
3492
3493         emit_mir_instruction(ctx, imov_8);
3494         EMIT(alu_br_compact_cond, midgard_jmp_writeout_op_writeout, TAG_ALU_4, 0, midgard_condition_always);
3495
3496         emit_mir_instruction(ctx, imov_8);
3497         EMIT(alu_br_compact_cond, midgard_jmp_writeout_op_writeout, TAG_ALU_4, -1, midgard_condition_always);
3498 }
3499
3500 static midgard_block *
3501 emit_block(compiler_context *ctx, nir_block *block)
3502 {
3503         midgard_block *this_block = calloc(sizeof(midgard_block), 1);
3504         list_addtail(&this_block->link, &ctx->blocks);
3505
3506         this_block->is_scheduled = false;
3507         ++ctx->block_count;
3508
3509         ctx->texture_index[0] = -1;
3510         ctx->texture_index[1] = -1;
3511
3512         /* Add us as a successor to the block we are following */
3513         if (ctx->current_block)
3514                 midgard_block_add_successor(ctx->current_block, this_block);
3515
3516         /* Set up current block */
3517         list_inithead(&this_block->instructions);
3518         ctx->current_block = this_block;
3519
3520         nir_foreach_instr(instr, block) {
3521                 emit_instr(ctx, instr);
3522                 ++ctx->instruction_count;
3523         }
3524
3525         inline_alu_constants(ctx);
3526         embedded_to_inline_constant(ctx);
3527
3528         /* Perform heavylifting for aliasing */
3529         actualise_ssa_to_alias(ctx);
3530
3531         midgard_emit_store(ctx, this_block);
3532         midgard_pair_load_store(ctx, this_block);
3533         midgard_imov_workaround(ctx, this_block);
3534
3535         /* Append fragment shader epilogue (value writeout) */
3536         if (ctx->stage == MESA_SHADER_FRAGMENT) {
3537                 if (block == nir_impl_last_block(ctx->func->impl)) {
3538                         if (ctx->is_blend)
3539                                 emit_blend_epilogue(ctx);
3540                         else
3541                                 emit_fragment_epilogue(ctx);
3542                 }
3543         }
3544
3545         if (block == nir_start_block(ctx->func->impl))
3546                 ctx->initial_block = this_block;
3547
3548         if (block == nir_impl_last_block(ctx->func->impl))
3549                 ctx->final_block = this_block;
3550
3551         /* Allow the next control flow to access us retroactively, for
3552          * branching etc */
3553         ctx->current_block = this_block;
3554
3555         /* Document the fallthrough chain */
3556         ctx->previous_source_block = this_block;
3557
3558         return this_block;
3559 }
3560
3561 static midgard_block *emit_cf_list(struct compiler_context *ctx, struct exec_list *list);
3562
3563 static void
3564 emit_if(struct compiler_context *ctx, nir_if *nif)
3565 {
3566         /* Conditional branches expect the condition in r31.w; emit a move for
3567          * that in the _previous_ block (which is the current block). */
3568         emit_condition(ctx, &nif->condition, true, COMPONENT_X);
3569
3570         /* Speculatively emit the branch, but we can't fill it in until later */
3571         EMIT(branch, true, true);
3572         midgard_instruction *then_branch = mir_last_in_block(ctx->current_block);
3573
3574         /* Emit the two subblocks */
3575         midgard_block *then_block = emit_cf_list(ctx, &nif->then_list);
3576
3577         /* Emit a jump from the end of the then block to the end of the else */
3578         EMIT(branch, false, false);
3579         midgard_instruction *then_exit = mir_last_in_block(ctx->current_block);
3580
3581         /* Emit second block, and check if it's empty */
3582
3583         int else_idx = ctx->block_count;
3584         int count_in = ctx->instruction_count;
3585         midgard_block *else_block = emit_cf_list(ctx, &nif->else_list);
3586         int after_else_idx = ctx->block_count;
3587
3588         /* Now that we have the subblocks emitted, fix up the branches */
3589
3590         assert(then_block);
3591         assert(else_block);
3592
3593         if (ctx->instruction_count == count_in) {
3594                 /* The else block is empty, so don't emit an exit jump */
3595                 mir_remove_instruction(then_exit);
3596                 then_branch->branch.target_block = after_else_idx;
3597         } else {
3598                 then_branch->branch.target_block = else_idx;
3599                 then_exit->branch.target_block = after_else_idx;
3600         }
3601 }
3602
3603 static void
3604 emit_loop(struct compiler_context *ctx, nir_loop *nloop)
3605 {
3606         /* Remember where we are */
3607         midgard_block *start_block = ctx->current_block;
3608
3609         /* Allocate a loop number, growing the current inner loop depth */
3610         int loop_idx = ++ctx->current_loop_depth;
3611
3612         /* Get index from before the body so we can loop back later */
3613         int start_idx = ctx->block_count;
3614
3615         /* Emit the body itself */
3616         emit_cf_list(ctx, &nloop->body);
3617
3618         /* Branch back to loop back */
3619         struct midgard_instruction br_back = v_branch(false, false);
3620         br_back.branch.target_block = start_idx;
3621         emit_mir_instruction(ctx, br_back);
3622
3623         /* Mark down that branch in the graph. Note that we're really branching
3624          * to the block *after* we started in. TODO: Why doesn't the branch
3625          * itself have an off-by-one then...? */
3626         midgard_block_add_successor(ctx->current_block, start_block->successors[0]);
3627
3628         /* Find the index of the block about to follow us (note: we don't add
3629          * one; blocks are 0-indexed so we get a fencepost problem) */
3630         int break_block_idx = ctx->block_count;
3631
3632         /* Fix up the break statements we emitted to point to the right place,
3633          * now that we can allocate a block number for them */
3634
3635         list_for_each_entry_from(struct midgard_block, block, start_block, &ctx->blocks, link) {
3636                 mir_foreach_instr_in_block(block, ins) {
3637                         if (ins->type != TAG_ALU_4) continue;
3638                         if (!ins->compact_branch) continue;
3639                         if (ins->prepacked_branch) continue;
3640
3641                         /* We found a branch -- check the type to see if we need to do anything */
3642                         if (ins->branch.target_type != TARGET_BREAK) continue;
3643
3644                         /* It's a break! Check if it's our break */
3645                         if (ins->branch.target_break != loop_idx) continue;
3646
3647                         /* Okay, cool, we're breaking out of this loop.
3648                          * Rewrite from a break to a goto */
3649
3650                         ins->branch.target_type = TARGET_GOTO;
3651                         ins->branch.target_block = break_block_idx;
3652                 }
3653         }
3654
3655         /* Now that we've finished emitting the loop, free up the depth again
3656          * so we play nice with recursion amid nested loops */
3657         --ctx->current_loop_depth;
3658 }
3659
3660 static midgard_block *
3661 emit_cf_list(struct compiler_context *ctx, struct exec_list *list)
3662 {
3663         midgard_block *start_block = NULL;
3664
3665         foreach_list_typed(nir_cf_node, node, node, list) {
3666                 switch (node->type) {
3667                 case nir_cf_node_block: {
3668                         midgard_block *block = emit_block(ctx, nir_cf_node_as_block(node));
3669
3670                         if (!start_block)
3671                                 start_block = block;
3672
3673                         break;
3674                 }
3675
3676                 case nir_cf_node_if:
3677                         emit_if(ctx, nir_cf_node_as_if(node));
3678                         break;
3679
3680                 case nir_cf_node_loop:
3681                         emit_loop(ctx, nir_cf_node_as_loop(node));
3682                         break;
3683
3684                 case nir_cf_node_function:
3685                         assert(0);
3686                         break;
3687                 }
3688         }
3689
3690         return start_block;
3691 }
3692
3693 /* Due to lookahead, we need to report the first tag executed in the command
3694  * stream and in branch targets. An initial block might be empty, so iterate
3695  * until we find one that 'works' */
3696
3697 static unsigned
3698 midgard_get_first_tag_from_block(compiler_context *ctx, unsigned block_idx)
3699 {
3700         midgard_block *initial_block = mir_get_block(ctx, block_idx);
3701
3702         unsigned first_tag = 0;
3703
3704         do {
3705                 midgard_bundle *initial_bundle = util_dynarray_element(&initial_block->bundles, midgard_bundle, 0);
3706
3707                 if (initial_bundle) {
3708                         first_tag = initial_bundle->tag;
3709                         break;
3710                 }
3711
3712                 /* Initial block is empty, try the next block */
3713                 initial_block = list_first_entry(&(initial_block->link), midgard_block, link);
3714         } while(initial_block != NULL);
3715
3716         assert(first_tag);
3717         return first_tag;
3718 }
3719
3720 int
3721 midgard_compile_shader_nir(nir_shader *nir, midgard_program *program, bool is_blend)
3722 {
3723         struct util_dynarray *compiled = &program->compiled;
3724
3725         midgard_debug = debug_get_option_midgard_debug();
3726
3727         compiler_context ictx = {
3728                 .nir = nir,
3729                 .stage = nir->info.stage,
3730
3731                 .is_blend = is_blend,
3732                 .blend_constant_offset = -1,
3733
3734                 .alpha_ref = program->alpha_ref
3735         };
3736
3737         compiler_context *ctx = &ictx;
3738
3739         /* TODO: Decide this at runtime */
3740         ctx->uniform_cutoff = 8;
3741
3742         /* Assign var locations early, so the epilogue can use them if necessary */
3743
3744         nir_assign_var_locations(&nir->outputs, &nir->num_outputs, glsl_type_size);
3745         nir_assign_var_locations(&nir->inputs, &nir->num_inputs, glsl_type_size);
3746         nir_assign_var_locations(&nir->uniforms, &nir->num_uniforms, glsl_type_size);
3747
3748         /* Initialize at a global (not block) level hash tables */
3749
3750         ctx->ssa_constants = _mesa_hash_table_u64_create(NULL);
3751         ctx->ssa_varyings = _mesa_hash_table_u64_create(NULL);
3752         ctx->ssa_to_alias = _mesa_hash_table_u64_create(NULL);
3753         ctx->ssa_to_register = _mesa_hash_table_u64_create(NULL);
3754         ctx->hash_to_temp = _mesa_hash_table_u64_create(NULL);
3755         ctx->sysval_to_id = _mesa_hash_table_u64_create(NULL);
3756         ctx->leftover_ssa_to_alias = _mesa_set_create(NULL, _mesa_hash_pointer, _mesa_key_pointer_equal);
3757
3758         /* Record the varying mapping for the command stream's bookkeeping */
3759
3760         struct exec_list *varyings =
3761                 ctx->stage == MESA_SHADER_VERTEX ? &nir->outputs : &nir->inputs;
3762
3763         nir_foreach_variable(var, varyings) {
3764                 unsigned loc = var->data.driver_location;
3765                 unsigned sz = glsl_type_size(var->type, FALSE);
3766
3767                 for (int c = 0; c < sz; ++c) {
3768                         program->varyings[loc + c] = var->data.location;
3769                 }
3770         }
3771
3772         /* Lower gl_Position pre-optimisation */
3773
3774         if (ctx->stage == MESA_SHADER_VERTEX)
3775                 NIR_PASS_V(nir, nir_lower_viewport_transform);
3776
3777         NIR_PASS_V(nir, nir_lower_var_copies);
3778         NIR_PASS_V(nir, nir_lower_vars_to_ssa);
3779         NIR_PASS_V(nir, nir_split_var_copies);
3780         NIR_PASS_V(nir, nir_lower_var_copies);
3781         NIR_PASS_V(nir, nir_lower_global_vars_to_local);
3782         NIR_PASS_V(nir, nir_lower_var_copies);
3783         NIR_PASS_V(nir, nir_lower_vars_to_ssa);
3784
3785         NIR_PASS_V(nir, nir_lower_io, nir_var_all, glsl_type_size, 0);
3786
3787         /* Optimisation passes */
3788
3789         optimise_nir(nir);
3790
3791         if (midgard_debug & MIDGARD_DBG_SHADERS) {
3792                 nir_print_shader(nir, stdout);
3793         }
3794
3795         /* Assign sysvals and counts, now that we're sure
3796          * (post-optimisation) */
3797
3798         midgard_nir_assign_sysvals(ctx, nir);
3799
3800         program->uniform_count = nir->num_uniforms;
3801         program->sysval_count = ctx->sysval_count;
3802         memcpy(program->sysvals, ctx->sysvals, sizeof(ctx->sysvals[0]) * ctx->sysval_count);
3803
3804         program->attribute_count = (ctx->stage == MESA_SHADER_VERTEX) ? nir->num_inputs : 0;
3805         program->varying_count = (ctx->stage == MESA_SHADER_VERTEX) ? nir->num_outputs : ((ctx->stage == MESA_SHADER_FRAGMENT) ? nir->num_inputs : 0);
3806
3807         nir_foreach_function(func, nir) {
3808                 if (!func->impl)
3809                         continue;
3810
3811                 list_inithead(&ctx->blocks);
3812                 ctx->block_count = 0;
3813                 ctx->func = func;
3814
3815                 emit_cf_list(ctx, &func->impl->body);
3816                 emit_block(ctx, func->impl->end_block);
3817
3818                 break; /* TODO: Multi-function shaders */
3819         }
3820
3821         util_dynarray_init(compiled, NULL);
3822
3823         /* MIR-level optimizations */
3824
3825         bool progress = false;
3826
3827         do {
3828                 progress = false;
3829
3830                 mir_foreach_block(ctx, block) {
3831                         progress |= midgard_opt_copy_prop(ctx, block);
3832                         progress |= midgard_opt_copy_prop_tex(ctx, block);
3833                         progress |= midgard_opt_dead_code_eliminate(ctx, block);
3834                 }
3835         } while (progress);
3836
3837         /* Schedule! */
3838         schedule_program(ctx);
3839
3840         /* Now that all the bundles are scheduled and we can calculate block
3841          * sizes, emit actual branch instructions rather than placeholders */
3842
3843         int br_block_idx = 0;
3844
3845         mir_foreach_block(ctx, block) {
3846                 util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) {
3847                         for (int c = 0; c < bundle->instruction_count; ++c) {
3848                                 midgard_instruction *ins = &bundle->instructions[c];
3849
3850                                 if (!midgard_is_branch_unit(ins->unit)) continue;
3851
3852                                 if (ins->prepacked_branch) continue;
3853
3854                                 /* Parse some basic branch info */
3855                                 bool is_compact = ins->unit == ALU_ENAB_BR_COMPACT;
3856                                 bool is_conditional = ins->branch.conditional;
3857                                 bool is_inverted = ins->branch.invert_conditional;
3858                                 bool is_discard = ins->branch.target_type == TARGET_DISCARD;
3859
3860                                 /* Determine the block we're jumping to */
3861                                 int target_number = ins->branch.target_block;
3862
3863                                 /* Report the destination tag. Discards don't need this */
3864                                 int dest_tag = is_discard ? 0 : midgard_get_first_tag_from_block(ctx, target_number);
3865
3866                                 /* Count up the number of quadwords we're jumping over. That is, the number of quadwords in each of the blocks between (br_block_idx, target_number) */
3867                                 int quadword_offset = 0;
3868
3869                                 if (is_discard) {
3870                                         /* Jump to the end of the shader. We
3871                                          * need to include not only the
3872                                          * following blocks, but also the
3873                                          * contents of our current block (since
3874                                          * discard can come in the middle of
3875                                          * the block) */
3876
3877                                         midgard_block *blk = mir_get_block(ctx, br_block_idx + 1);
3878
3879                                         for (midgard_bundle *bun = bundle + 1; bun < (midgard_bundle *)((char*) block->bundles.data + block->bundles.size); ++bun) {
3880                                                 quadword_offset += quadword_size(bun->tag);
3881                                         }
3882
3883                                         mir_foreach_block_from(ctx, blk, b) {
3884                                                 quadword_offset += b->quadword_count;
3885                                         }
3886
3887                                 } else if (target_number > br_block_idx) {
3888                                         /* Jump forward */
3889
3890                                         for (int idx = br_block_idx + 1; idx < target_number; ++idx) {
3891                                                 midgard_block *blk = mir_get_block(ctx, idx);
3892                                                 assert(blk);
3893
3894                                                 quadword_offset += blk->quadword_count;
3895                                         }
3896                                 } else {
3897                                         /* Jump backwards */
3898
3899                                         for (int idx = br_block_idx; idx >= target_number; --idx) {
3900                                                 midgard_block *blk = mir_get_block(ctx, idx);
3901                                                 assert(blk);
3902
3903                                                 quadword_offset -= blk->quadword_count;
3904                                         }
3905                                 }
3906
3907                                 /* Unconditional extended branches (far jumps)
3908                                  * have issues, so we always use a conditional
3909                                  * branch, setting the condition to always for
3910                                  * unconditional. For compact unconditional
3911                                  * branches, cond isn't used so it doesn't
3912                                  * matter what we pick. */
3913
3914                                 midgard_condition cond =
3915                                         !is_conditional ? midgard_condition_always :
3916                                         is_inverted ? midgard_condition_false :
3917                                         midgard_condition_true;
3918
3919                                 midgard_jmp_writeout_op op =
3920                                         is_discard ? midgard_jmp_writeout_op_discard :
3921                                         (is_compact && !is_conditional) ? midgard_jmp_writeout_op_branch_uncond :
3922                                         midgard_jmp_writeout_op_branch_cond;
3923
3924                                 if (!is_compact) {
3925                                         midgard_branch_extended branch =
3926                                                 midgard_create_branch_extended(
3927                                                         cond, op,
3928                                                         dest_tag,
3929                                                         quadword_offset);
3930
3931                                         memcpy(&ins->branch_extended, &branch, sizeof(branch));
3932                                 } else if (is_conditional || is_discard) {
3933                                         midgard_branch_cond branch = {
3934                                                 .op = op,
3935                                                 .dest_tag = dest_tag,
3936                                                 .offset = quadword_offset,
3937                                                 .cond = cond
3938                                         };
3939
3940                                         assert(branch.offset == quadword_offset);
3941
3942                                         memcpy(&ins->br_compact, &branch, sizeof(branch));
3943                                 } else {
3944                                         assert(op == midgard_jmp_writeout_op_branch_uncond);
3945
3946                                         midgard_branch_uncond branch = {
3947                                                 .op = op,
3948                                                 .dest_tag = dest_tag,
3949                                                 .offset = quadword_offset,
3950                                                 .unknown = 1
3951                                         };
3952
3953                                         assert(branch.offset == quadword_offset);
3954
3955                                         memcpy(&ins->br_compact, &branch, sizeof(branch));
3956                                 }
3957                         }
3958                 }
3959
3960                 ++br_block_idx;
3961         }
3962
3963         /* Emit flat binary from the instruction arrays. Iterate each block in
3964          * sequence. Save instruction boundaries such that lookahead tags can
3965          * be assigned easily */
3966
3967         /* Cache _all_ bundles in source order for lookahead across failed branches */
3968
3969         int bundle_count = 0;
3970         mir_foreach_block(ctx, block) {
3971                 bundle_count += block->bundles.size / sizeof(midgard_bundle);
3972         }
3973         midgard_bundle **source_order_bundles = malloc(sizeof(midgard_bundle *) * bundle_count);
3974         int bundle_idx = 0;
3975         mir_foreach_block(ctx, block) {
3976                 util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) {
3977                         source_order_bundles[bundle_idx++] = bundle;
3978                 }
3979         }
3980
3981         int current_bundle = 0;
3982
3983         mir_foreach_block(ctx, block) {
3984                 util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) {
3985                         int lookahead = 1;
3986
3987                         if (current_bundle + 1 < bundle_count) {
3988                                 uint8_t next = source_order_bundles[current_bundle + 1]->tag;
3989
3990                                 if (!(current_bundle + 2 < bundle_count) && IS_ALU(next)) {
3991                                         lookahead = 1;
3992                                 } else {
3993                                         lookahead = next;
3994                                 }
3995                         }
3996
3997                         emit_binary_bundle(ctx, bundle, compiled, lookahead);
3998                         ++current_bundle;
3999                 }
4000
4001                 /* TODO: Free deeper */
4002                 //util_dynarray_fini(&block->instructions);
4003         }
4004
4005         free(source_order_bundles);
4006
4007         /* Report the very first tag executed */
4008         program->first_tag = midgard_get_first_tag_from_block(ctx, 0);
4009
4010         /* Deal with off-by-one related to the fencepost problem */
4011         program->work_register_count = ctx->work_registers + 1;
4012
4013         program->can_discard = ctx->can_discard;
4014         program->uniform_cutoff = ctx->uniform_cutoff;
4015
4016         program->blend_patch_offset = ctx->blend_constant_offset;
4017
4018         if (midgard_debug & MIDGARD_DBG_SHADERS)
4019                 disassemble_midgard(program->compiled.data, program->compiled.size);
4020
4021         return 0;
4022 }