projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
panfrost/midgard: Fix RA when temp_count = 0
[mesa.git] / src / gallium / drivers / panfrost / midgard / midgard_compile.c
1 /*
2 * Copyright (C) 2018 Alyssa Rosenzweig <alyssa@rosenzweig.io>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
23
24 #include <sys/types.h>
25 #include <sys/stat.h>
26 #include <sys/mman.h>
27 #include <fcntl.h>
28 #include <stdint.h>
29 #include <stdlib.h>
30 #include <stdio.h>
31 #include <err.h>
32
33 #include "main/mtypes.h"
34 #include "compiler/glsl/glsl_to_nir.h"
35 #include "compiler/nir_types.h"
36 #include "main/imports.h"
37 #include "compiler/nir/nir_builder.h"
38 #include "util/half_float.h"
39 #include "util/register_allocate.h"
40 #include "util/u_debug.h"
41 #include "util/u_dynarray.h"
42 #include "util/list.h"
43 #include "main/mtypes.h"
44
45 #include "midgard.h"
46 #include "midgard_nir.h"
47 #include "midgard_compile.h"
48 #include "helpers.h"
49
50 #include "disassemble.h"
51
52 static const struct debug_named_value debug_options[] = {
53 {"msgs", MIDGARD_DBG_MSGS, "Print debug messages"},
54 {"shaders", MIDGARD_DBG_SHADERS, "Dump shaders in NIR and MIR"},
55 DEBUG_NAMED_VALUE_END
56 };
57
58 DEBUG_GET_ONCE_FLAGS_OPTION(midgard_debug, "MIDGARD_MESA_DEBUG", debug_options, 0)
59
60 int midgard_debug = 0;
61
62 #define DBG(fmt, ...) \
63 do { if (midgard_debug & MIDGARD_DBG_MSGS) \
64 fprintf(stderr, "%s:%d: "fmt, \
65 __FUNCTION__, __LINE__, ##__VA_ARGS__); } while (0)
66
67 /* Instruction arguments represented as block-local SSA indices, rather than
68 * registers. Negative values mean unused. */
69
70 typedef struct {
71 int src0;
72 int src1;
73 int dest;
74
75 /* src1 is -not- SSA but instead a 16-bit inline constant to be smudged
76 * in. Only valid for ALU ops. */
77 bool inline_constant;
78 } ssa_args;
79
80 /* Forward declare so midgard_branch can reference */
81 struct midgard_block;
82
83 /* Target types. Defaults to TARGET_GOTO (the type corresponding directly to
84 * the hardware), hence why that must be zero. TARGET_DISCARD signals this
85 * instruction is actually a discard op. */
86
87 #define TARGET_GOTO 0
88 #define TARGET_BREAK 1
89 #define TARGET_CONTINUE 2
90 #define TARGET_DISCARD 3
91
92 typedef struct midgard_branch {
93 /* If conditional, the condition is specified in r31.w */
94 bool conditional;
95
96 /* For conditionals, if this is true, we branch on FALSE. If false, we branch on TRUE. */
97 bool invert_conditional;
98
99 /* Branch targets: the start of a block, the start of a loop (continue), the end of a loop (break). Value is one of TARGET_ */
100 unsigned target_type;
101
102 /* The actual target */
103 union {
104 int target_block;
105 int target_break;
106 int target_continue;
107 };
108 } midgard_branch;
109
110 static bool
111 midgard_is_branch_unit(unsigned unit)
112 {
113 return (unit == ALU_ENAB_BRANCH) || (unit == ALU_ENAB_BR_COMPACT);
114 }
115
116 /* Generic in-memory data type repesenting a single logical instruction, rather
117 * than a single instruction group. This is the preferred form for code gen.
118 * Multiple midgard_insturctions will later be combined during scheduling,
119 * though this is not represented in this structure. Its format bridges
120 * the low-level binary representation with the higher level semantic meaning.
121 *
122 * Notably, it allows registers to be specified as block local SSA, for code
123 * emitted before the register allocation pass.
124 */
125
126 typedef struct midgard_instruction {
127 /* Must be first for casting */
128 struct list_head link;
129
130 unsigned type; /* ALU, load/store, texture */
131
132 /* If the register allocator has not run yet... */
133 ssa_args ssa_args;
134
135 /* Special fields for an ALU instruction */
136 midgard_reg_info registers;
137
138 /* I.e. (1 << alu_bit) */
139 int unit;
140
141 bool has_constants;
142 float constants[4];
143 uint16_t inline_constant;
144 bool has_blend_constant;
145
146 bool compact_branch;
147 bool writeout;
148 bool prepacked_branch;
149
150 union {
151 midgard_load_store_word load_store;
152 midgard_vector_alu alu;
153 midgard_texture_word texture;
154 midgard_branch_extended branch_extended;
155 uint16_t br_compact;
156
157 /* General branch, rather than packed br_compact. Higher level
158 * than the other components */
159 midgard_branch branch;
160 };
161 } midgard_instruction;
162
163 typedef struct midgard_block {
164 /* Link to next block. Must be first for mir_get_block */
165 struct list_head link;
166
167 /* List of midgard_instructions emitted for the current block */
168 struct list_head instructions;
169
170 bool is_scheduled;
171
172 /* List of midgard_bundles emitted (after the scheduler has run) */
173 struct util_dynarray bundles;
174
175 /* Number of quadwords _actually_ emitted, as determined after scheduling */
176 unsigned quadword_count;
177
178 /* Successors: always one forward (the block after us), maybe
179 * one backwards (for a backward branch). No need for a second
180 * forward, since graph traversal would get there eventually
181 * anyway */
182 struct midgard_block *successors[2];
183 unsigned nr_successors;
184
185 /* The successors pointer form a graph, and in the case of
186 * complex control flow, this graph has a cycles. To aid
187 * traversal during liveness analysis, we have a visited?
188 * boolean for passes to use as they see fit, provided they
189 * clean up later */
190 bool visited;
191 } midgard_block;
192
193 static void
194 midgard_block_add_successor(midgard_block *block, midgard_block *successor)
195 {
196 block->successors[block->nr_successors++] = successor;
197 assert(block->nr_successors <= ARRAY_SIZE(block->successors));
198 }
199
200 /* Helpers to generate midgard_instruction's using macro magic, since every
201 * driver seems to do it that way */
202
203 #define EMIT(op, ...) emit_mir_instruction(ctx, v_##op(__VA_ARGS__));
204 #define SWIZZLE_XYZW SWIZZLE(COMPONENT_X, COMPONENT_Y, COMPONENT_Z, COMPONENT_W)
205
206 #define M_LOAD_STORE(name, rname, uname) \
207 static midgard_instruction m_##name(unsigned ssa, unsigned address) { \
208 midgard_instruction i = { \
209 .type = TAG_LOAD_STORE_4, \
210 .ssa_args = { \
211 .rname = ssa, \
212 .uname = -1, \
213 .src1 = -1 \
214 }, \
215 .load_store = { \
216 .op = midgard_op_##name, \
217 .mask = 0xF, \
218 .swizzle = SWIZZLE_XYZW, \
219 .address = address \
220 } \
221 }; \
222 \
223 return i; \
224 }
225
226 #define M_LOAD(name) M_LOAD_STORE(name, dest, src0)
227 #define M_STORE(name) M_LOAD_STORE(name, src0, dest)
228
229 const midgard_vector_alu_src blank_alu_src = {
230 .swizzle = SWIZZLE(COMPONENT_X, COMPONENT_Y, COMPONENT_Z, COMPONENT_W),
231 };
232
233 const midgard_vector_alu_src blank_alu_src_xxxx = {
234 .swizzle = SWIZZLE(COMPONENT_X, COMPONENT_X, COMPONENT_X, COMPONENT_X),
235 };
236
237 const midgard_scalar_alu_src blank_scalar_alu_src = {
238 .full = true
239 };
240
241 /* Used for encoding the unused source of 1-op instructions */
242 const midgard_vector_alu_src zero_alu_src = { 0 };
243
244 /* Coerce structs to integer */
245
246 static unsigned
247 vector_alu_srco_unsigned(midgard_vector_alu_src src)
248 {
249 unsigned u;
250 memcpy(&u, &src, sizeof(src));
251 return u;
252 }
253
254 static midgard_vector_alu_src
255 vector_alu_from_unsigned(unsigned u)
256 {
257 midgard_vector_alu_src s;
258 memcpy(&s, &u, sizeof(s));
259 return s;
260 }
261
262 /* Inputs a NIR ALU source, with modifiers attached if necessary, and outputs
263 * the corresponding Midgard source */
264
265 static midgard_vector_alu_src
266 vector_alu_modifiers(nir_alu_src *src, bool is_int)
267 {
268 if (!src) return blank_alu_src;
269
270 midgard_vector_alu_src alu_src = {
271 .rep_low = 0,
272 .rep_high = 0,
273 .half = 0, /* TODO */
274 .swizzle = SWIZZLE_FROM_ARRAY(src->swizzle)
275 };
276
277 if (is_int) {
278 /* TODO: sign-extend/zero-extend */
279 alu_src.mod = midgard_int_normal;
280
281 /* These should have been lowered away */
282 assert(!(src->abs || src->negate));
283 } else {
284 alu_src.mod = (src->abs << 0) | (src->negate << 1);
285 }
286
287 return alu_src;
288 }
289
290 static bool
291 mir_nontrivial_mod(midgard_vector_alu_src src, bool is_int, unsigned mask)
292 {
293 /* abs or neg */
294 if (!is_int && src.mod) return true;
295
296 /* swizzle */
297 for (unsigned c = 0; c < 4; ++c) {
298 if (!(mask & (1 << c))) continue;
299 if (((src.swizzle >> (2*c)) & 3) != c) return true;
300 }
301
302 return false;
303 }
304
305 /* 'Intrinsic' move for misc aliasing uses independent of actual NIR ALU code */
306
307 static midgard_instruction
308 v_fmov(unsigned src, midgard_vector_alu_src mod, unsigned dest)
309 {
310 midgard_instruction ins = {
311 .type = TAG_ALU_4,
312 .ssa_args = {
313 .src0 = SSA_UNUSED_1,
314 .src1 = src,
315 .dest = dest,
316 },
317 .alu = {
318 .op = midgard_alu_op_fmov,
319 .reg_mode = midgard_reg_mode_32,
320 .dest_override = midgard_dest_override_none,
321 .mask = 0xFF,
322 .src1 = vector_alu_srco_unsigned(zero_alu_src),
323 .src2 = vector_alu_srco_unsigned(mod)
324 },
325 };
326
327 return ins;
328 }
329
330 /* load/store instructions have both 32-bit and 16-bit variants, depending on
331 * whether we are using vectors composed of highp or mediump. At the moment, we
332 * don't support half-floats -- this requires changes in other parts of the
333 * compiler -- therefore the 16-bit versions are commented out. */
334
335 //M_LOAD(load_attr_16);
336 M_LOAD(load_attr_32);
337 //M_LOAD(load_vary_16);
338 M_LOAD(load_vary_32);
339 //M_LOAD(load_uniform_16);
340 M_LOAD(load_uniform_32);
341 M_LOAD(load_color_buffer_8);
342 //M_STORE(store_vary_16);
343 M_STORE(store_vary_32);
344 M_STORE(store_cubemap_coords);
345
346 static midgard_instruction
347 v_alu_br_compact_cond(midgard_jmp_writeout_op op, unsigned tag, signed offset, unsigned cond)
348 {
349 midgard_branch_cond branch = {
350 .op = op,
351 .dest_tag = tag,
352 .offset = offset,
353 .cond = cond
354 };
355
356 uint16_t compact;
357 memcpy(&compact, &branch, sizeof(branch));
358
359 midgard_instruction ins = {
360 .type = TAG_ALU_4,
361 .unit = ALU_ENAB_BR_COMPACT,
362 .prepacked_branch = true,
363 .compact_branch = true,
364 .br_compact = compact
365 };
366
367 if (op == midgard_jmp_writeout_op_writeout)
368 ins.writeout = true;
369
370 return ins;
371 }
372
373 static midgard_instruction
374 v_branch(bool conditional, bool invert)
375 {
376 midgard_instruction ins = {
377 .type = TAG_ALU_4,
378 .unit = ALU_ENAB_BRANCH,
379 .compact_branch = true,
380 .branch = {
381 .conditional = conditional,
382 .invert_conditional = invert
383 }
384 };
385
386 return ins;
387 }
388
389 static midgard_branch_extended
390 midgard_create_branch_extended( midgard_condition cond,
391 midgard_jmp_writeout_op op,
392 unsigned dest_tag,
393 signed quadword_offset)
394 {
395 /* For unclear reasons, the condition code is repeated 8 times */
396 uint16_t duplicated_cond =
397 (cond << 14) |
398 (cond << 12) |
399 (cond << 10) |
400 (cond << 8) |
401 (cond << 6) |
402 (cond << 4) |
403 (cond << 2) |
404 (cond << 0);
405
406 midgard_branch_extended branch = {
407 .op = op,
408 .dest_tag = dest_tag,
409 .offset = quadword_offset,
410 .cond = duplicated_cond
411 };
412
413 return branch;
414 }
415
416 typedef struct midgard_bundle {
417 /* Tag for the overall bundle */
418 int tag;
419
420 /* Instructions contained by the bundle */
421 int instruction_count;
422 midgard_instruction instructions[5];
423
424 /* Bundle-wide ALU configuration */
425 int padding;
426 int control;
427 bool has_embedded_constants;
428 float constants[4];
429 bool has_blend_constant;
430
431 uint16_t register_words[8];
432 int register_words_count;
433
434 uint64_t body_words[8];
435 size_t body_size[8];
436 int body_words_count;
437 } midgard_bundle;
438
439 typedef struct compiler_context {
440 nir_shader *nir;
441 gl_shader_stage stage;
442
443 /* Is internally a blend shader? Depends on stage == FRAGMENT */
444 bool is_blend;
445
446 /* Tracking for blend constant patching */
447 int blend_constant_number;
448 int blend_constant_offset;
449
450 /* Current NIR function */
451 nir_function *func;
452
453 /* Unordered list of midgard_blocks */
454 int block_count;
455 struct list_head blocks;
456
457 midgard_block *initial_block;
458 midgard_block *previous_source_block;
459 midgard_block *final_block;
460
461 /* List of midgard_instructions emitted for the current block */
462 midgard_block *current_block;
463
464 /* The current "depth" of the loop, for disambiguating breaks/continues
465 * when using nested loops */
466 int current_loop_depth;
467
468 /* Constants which have been loaded, for later inlining */
469 struct hash_table_u64 *ssa_constants;
470
471 /* SSA indices to be outputted to corresponding varying offset */
472 struct hash_table_u64 *ssa_varyings;
473
474 /* SSA values / registers which have been aliased. Naively, these
475 * demand a fmov output; instead, we alias them in a later pass to
476 * avoid the wasted op.
477 *
478 * A note on encoding: to avoid dynamic memory management here, rather
479 * than ampping to a pointer, we map to the source index; the key
480 * itself is just the destination index. */
481
482 struct hash_table_u64 *ssa_to_alias;
483 struct set *leftover_ssa_to_alias;
484
485 /* Actual SSA-to-register for RA */
486 struct hash_table_u64 *ssa_to_register;
487
488 /* Mapping of hashes computed from NIR indices to the sequential temp indices ultimately used in MIR */
489 struct hash_table_u64 *hash_to_temp;
490 int temp_count;
491 int max_hash;
492
493 /* Just the count of the max register used. Higher count => higher
494 * register pressure */
495 int work_registers;
496
497 /* Used for cont/last hinting. Increase when a tex op is added.
498 * Decrease when a tex op is removed. */
499 int texture_op_count;
500
501 /* Mapping of texture register -> SSA index for unaliasing */
502 int texture_index[2];
503
504 /* If any path hits a discard instruction */
505 bool can_discard;
506
507 /* The number of uniforms allowable for the fast path */
508 int uniform_cutoff;
509
510 /* Count of instructions emitted from NIR overall, across all blocks */
511 int instruction_count;
512
513 /* Alpha ref value passed in */
514 float alpha_ref;
515
516 /* The index corresponding to the fragment output */
517 unsigned fragment_output;
518
519 /* The mapping of sysvals to uniforms, the count, and the off-by-one inverse */
520 unsigned sysvals[MAX_SYSVAL_COUNT];
521 unsigned sysval_count;
522 struct hash_table_u64 *sysval_to_id;
523 } compiler_context;
524
525 /* Append instruction to end of current block */
526
527 static midgard_instruction *
528 mir_upload_ins(struct midgard_instruction ins)
529 {
530 midgard_instruction *heap = malloc(sizeof(ins));
531 memcpy(heap, &ins, sizeof(ins));
532 return heap;
533 }
534
535 static void
536 emit_mir_instruction(struct compiler_context *ctx, struct midgard_instruction ins)
537 {
538 list_addtail(&(mir_upload_ins(ins))->link, &ctx->current_block->instructions);
539 }
540
541 static void
542 mir_insert_instruction_before(struct midgard_instruction *tag, struct midgard_instruction ins)
543 {
544 list_addtail(&(mir_upload_ins(ins))->link, &tag->link);
545 }
546
547 static void
548 mir_remove_instruction(struct midgard_instruction *ins)
549 {
550 list_del(&ins->link);
551 }
552
553 static midgard_instruction*
554 mir_prev_op(struct midgard_instruction *ins)
555 {
556 return list_last_entry(&(ins->link), midgard_instruction, link);
557 }
558
559 static midgard_instruction*
560 mir_next_op(struct midgard_instruction *ins)
561 {
562 return list_first_entry(&(ins->link), midgard_instruction, link);
563 }
564
565 #define mir_foreach_block(ctx, v) list_for_each_entry(struct midgard_block, v, &ctx->blocks, link)
566 #define mir_foreach_block_from(ctx, from, v) list_for_each_entry_from(struct midgard_block, v, from, &ctx->blocks, link)
567
568 #define mir_foreach_instr(ctx, v) list_for_each_entry(struct midgard_instruction, v, &ctx->current_block->instructions, link)
569 #define mir_foreach_instr_safe(ctx, v) list_for_each_entry_safe(struct midgard_instruction, v, &ctx->current_block->instructions, link)
570 #define mir_foreach_instr_in_block(block, v) list_for_each_entry(struct midgard_instruction, v, &block->instructions, link)
571 #define mir_foreach_instr_in_block_safe(block, v) list_for_each_entry_safe(struct midgard_instruction, v, &block->instructions, link)
572 #define mir_foreach_instr_in_block_safe_rev(block, v) list_for_each_entry_safe_rev(struct midgard_instruction, v, &block->instructions, link)
573 #define mir_foreach_instr_in_block_from(block, v, from) list_for_each_entry_from(struct midgard_instruction, v, from, &block->instructions, link)
574 #define mir_foreach_instr_in_block_from_rev(block, v, from) list_for_each_entry_from_rev(struct midgard_instruction, v, from, &block->instructions, link)
575
576
577 static midgard_instruction *
578 mir_last_in_block(struct midgard_block *block)
579 {
580 return list_last_entry(&block->instructions, struct midgard_instruction, link);
581 }
582
583 static midgard_block *
584 mir_get_block(compiler_context *ctx, int idx)
585 {
586 struct list_head *lst = &ctx->blocks;
587
588 while ((idx--) + 1)
589 lst = lst->next;
590
591 return (struct midgard_block *) lst;
592 }
593
594 /* Pretty printer for internal Midgard IR */
595
596 static void
597 print_mir_source(int source)
598 {
599 if (source >= SSA_FIXED_MINIMUM) {
600 /* Specific register */
601 int reg = SSA_REG_FROM_FIXED(source);
602
603 /* TODO: Moving threshold */
604 if (reg > 16 && reg < 24)
605 printf("u%d", 23 - reg);
606 else
607 printf("r%d", reg);
608 } else {
609 printf("%d", source);
610 }
611 }
612
613 static void
614 print_mir_instruction(midgard_instruction *ins)
615 {
616 printf("\t");
617
618 switch (ins->type) {
619 case TAG_ALU_4: {
620 midgard_alu_op op = ins->alu.op;
621 const char *name = alu_opcode_props[op].name;
622
623 if (ins->unit)
624 printf("%d.", ins->unit);
625
626 printf("%s", name ? name : "??");
627 break;
628 }
629
630 case TAG_LOAD_STORE_4: {
631 midgard_load_store_op op = ins->load_store.op;
632 const char *name = load_store_opcode_names[op];
633
634 assert(name);
635 printf("%s", name);
636 break;
637 }
638
639 case TAG_TEXTURE_4: {
640 printf("texture");
641 break;
642 }
643
644 default:
645 assert(0);
646 }
647
648 ssa_args *args = &ins->ssa_args;
649
650 printf(" %d, ", args->dest);
651
652 print_mir_source(args->src0);
653 printf(", ");
654
655 if (args->inline_constant)
656 printf("#%d", ins->inline_constant);
657 else
658 print_mir_source(args->src1);
659
660 if (ins->has_constants)
661 printf(" <%f, %f, %f, %f>", ins->constants[0], ins->constants[1], ins->constants[2], ins->constants[3]);
662
663 printf("\n");
664 }
665
666 static void
667 print_mir_block(midgard_block *block)
668 {
669 printf("{\n");
670
671 mir_foreach_instr_in_block(block, ins) {
672 print_mir_instruction(ins);
673 }
674
675 printf("}\n");
676 }
677
678 static void
679 attach_constants(compiler_context *ctx, midgard_instruction *ins, void *constants, int name)
680 {
681 ins->has_constants = true;
682 memcpy(&ins->constants, constants, 16);
683
684 /* If this is the special blend constant, mark this instruction */
685
686 if (ctx->is_blend && ctx->blend_constant_number == name)
687 ins->has_blend_constant = true;
688 }
689
690 static int
691 glsl_type_size(const struct glsl_type *type, bool bindless)
692 {
693 return glsl_count_attribute_slots(type, false);
694 }
695
696 /* Lower fdot2 to a vector multiplication followed by channel addition */
697 static void
698 midgard_nir_lower_fdot2_body(nir_builder *b, nir_alu_instr *alu)
699 {
700 if (alu->op != nir_op_fdot2)
701 return;
702
703 b->cursor = nir_before_instr(&alu->instr);
704
705 nir_ssa_def *src0 = nir_ssa_for_alu_src(b, alu, 0);
706 nir_ssa_def *src1 = nir_ssa_for_alu_src(b, alu, 1);
707
708 nir_ssa_def *product = nir_fmul(b, src0, src1);
709
710 nir_ssa_def *sum = nir_fadd(b,
711 nir_channel(b, product, 0),
712 nir_channel(b, product, 1));
713
714 /* Replace the fdot2 with this sum */
715 nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(sum));
716 }
717
718 static int
719 midgard_nir_sysval_for_intrinsic(nir_intrinsic_instr *instr)
720 {
721 switch (instr->intrinsic) {
722 case nir_intrinsic_load_viewport_scale:
723 return PAN_SYSVAL_VIEWPORT_SCALE;
724 case nir_intrinsic_load_viewport_offset:
725 return PAN_SYSVAL_VIEWPORT_OFFSET;
726 default:
727 return -1;
728 }
729 }
730
731 static void
732 midgard_nir_assign_sysval_body(compiler_context *ctx, nir_instr *instr)
733 {
734 int sysval = -1;
735
736 if (instr->type == nir_instr_type_intrinsic) {
737 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
738 sysval = midgard_nir_sysval_for_intrinsic(intr);
739 }
740
741 if (sysval < 0)
742 return;
743
744 /* We have a sysval load; check if it's already been assigned */
745
746 if (_mesa_hash_table_u64_search(ctx->sysval_to_id, sysval))
747 return;
748
749 /* It hasn't -- so assign it now! */
750
751 unsigned id = ctx->sysval_count++;
752 _mesa_hash_table_u64_insert(ctx->sysval_to_id, sysval, (void *) ((uintptr_t) id + 1));
753 ctx->sysvals[id] = sysval;
754 }
755
756 static void
757 midgard_nir_assign_sysvals(compiler_context *ctx, nir_shader *shader)
758 {
759 ctx->sysval_count = 0;
760
761 nir_foreach_function(function, shader) {
762 if (!function->impl) continue;
763
764 nir_foreach_block(block, function->impl) {
765 nir_foreach_instr_safe(instr, block) {
766 midgard_nir_assign_sysval_body(ctx, instr);
767 }
768 }
769 }
770 }
771
772 static bool
773 midgard_nir_lower_fdot2(nir_shader *shader)
774 {
775 bool progress = false;
776
777 nir_foreach_function(function, shader) {
778 if (!function->impl) continue;
779
780 nir_builder _b;
781 nir_builder *b = &_b;
782 nir_builder_init(b, function->impl);
783
784 nir_foreach_block(block, function->impl) {
785 nir_foreach_instr_safe(instr, block) {
786 if (instr->type != nir_instr_type_alu) continue;
787
788 nir_alu_instr *alu = nir_instr_as_alu(instr);
789 midgard_nir_lower_fdot2_body(b, alu);
790
791 progress |= true;
792 }
793 }
794
795 nir_metadata_preserve(function->impl, nir_metadata_block_index | nir_metadata_dominance);
796
797 }
798
799 return progress;
800 }
801
802 static void
803 optimise_nir(nir_shader *nir)
804 {
805 bool progress;
806
807 NIR_PASS(progress, nir, nir_lower_regs_to_ssa);
808 NIR_PASS(progress, nir, midgard_nir_lower_fdot2);
809
810 nir_lower_tex_options lower_tex_options = {
811 .lower_rect = true
812 };
813
814 NIR_PASS(progress, nir, nir_lower_tex, &lower_tex_options);
815
816 do {
817 progress = false;
818
819 NIR_PASS(progress, nir, nir_lower_var_copies);
820 NIR_PASS(progress, nir, nir_lower_vars_to_ssa);
821
822 NIR_PASS(progress, nir, nir_copy_prop);
823 NIR_PASS(progress, nir, nir_opt_dce);
824 NIR_PASS(progress, nir, nir_opt_dead_cf);
825 NIR_PASS(progress, nir, nir_opt_cse);
826 NIR_PASS(progress, nir, nir_opt_peephole_select, 64, false, true);
827 NIR_PASS(progress, nir, nir_opt_algebraic);
828 NIR_PASS(progress, nir, nir_opt_constant_folding);
829 NIR_PASS(progress, nir, nir_opt_undef);
830 NIR_PASS(progress, nir, nir_opt_loop_unroll,
831 nir_var_shader_in |
832 nir_var_shader_out |
833 nir_var_function_temp);
834
835 /* TODO: Enable vectorize when merged upstream */
836 // NIR_PASS(progress, nir, nir_opt_vectorize);
837 } while (progress);
838
839 /* Must be run at the end to prevent creation of fsin/fcos ops */
840 NIR_PASS(progress, nir, midgard_nir_scale_trig);
841
842 do {
843 progress = false;
844
845 NIR_PASS(progress, nir, nir_opt_dce);
846 NIR_PASS(progress, nir, nir_opt_algebraic);
847 NIR_PASS(progress, nir, nir_opt_constant_folding);
848 NIR_PASS(progress, nir, nir_copy_prop);
849 } while (progress);
850
851 NIR_PASS(progress, nir, nir_opt_algebraic_late);
852 NIR_PASS(progress, nir, midgard_nir_lower_algebraic_late);
853
854 /* Lower mods for float ops only. Integer ops don't support modifiers
855 * (saturate doesn't make sense on integers, neg/abs require dedicated
856 * instructions) */
857
858 NIR_PASS(progress, nir, nir_lower_to_source_mods, nir_lower_float_source_mods);
859 NIR_PASS(progress, nir, nir_copy_prop);
860 NIR_PASS(progress, nir, nir_opt_dce);
861
862 /* We implement booleans as 32-bit 0/~0 */
863 NIR_PASS(progress, nir, nir_lower_bool_to_int32);
864
865 /* Take us out of SSA */
866 NIR_PASS(progress, nir, nir_lower_locals_to_regs);
867 NIR_PASS(progress, nir, nir_convert_from_ssa, true);
868
869 /* We are a vector architecture; write combine where possible */
870 NIR_PASS(progress, nir, nir_move_vec_src_uses_to_dest);
871 NIR_PASS(progress, nir, nir_lower_vec_to_movs);
872
873 NIR_PASS(progress, nir, nir_opt_dce);
874 }
875
876 /* Front-half of aliasing the SSA slots, merely by inserting the flag in the
877 * appropriate hash table. Intentional off-by-one to avoid confusing NULL with
878 * r0. See the comments in compiler_context */
879
880 static void
881 alias_ssa(compiler_context *ctx, int dest, int src)
882 {
883 _mesa_hash_table_u64_insert(ctx->ssa_to_alias, dest + 1, (void *) ((uintptr_t) src + 1));
884 _mesa_set_add(ctx->leftover_ssa_to_alias, (void *) (uintptr_t) (dest + 1));
885 }
886
887 /* ...or undo it, after which the original index will be used (dummy move should be emitted alongside this) */
888
889 static void
890 unalias_ssa(compiler_context *ctx, int dest)
891 {
892 _mesa_hash_table_u64_remove(ctx->ssa_to_alias, dest + 1);
893 /* TODO: Remove from leftover or no? */
894 }
895
896 static void
897 midgard_pin_output(compiler_context *ctx, int index, int reg)
898 {
899 _mesa_hash_table_u64_insert(ctx->ssa_to_register, index + 1, (void *) ((uintptr_t) reg + 1));
900 }
901
902 static bool
903 midgard_is_pinned(compiler_context *ctx, int index)
904 {
905 return _mesa_hash_table_u64_search(ctx->ssa_to_register, index + 1) != NULL;
906 }
907
908 /* Do not actually emit a load; instead, cache the constant for inlining */
909
910 static void
911 emit_load_const(compiler_context *ctx, nir_load_const_instr *instr)
912 {
913 nir_ssa_def def = instr->def;
914
915 float *v = ralloc_array(NULL, float, 4);
916 nir_const_load_to_arr(v, instr, f32);
917 _mesa_hash_table_u64_insert(ctx->ssa_constants, def.index + 1, v);
918 }
919
920 /* Duplicate bits to convert sane 4-bit writemask to obscure 8-bit format (or
921 * do the inverse) */
922
923 static unsigned
924 expand_writemask(unsigned mask)
925 {
926 unsigned o = 0;
927
928 for (int i = 0; i < 4; ++i)
929 if (mask & (1 << i))
930 o |= (3 << (2 * i));
931
932 return o;
933 }
934
935 static unsigned
936 squeeze_writemask(unsigned mask)
937 {
938 unsigned o = 0;
939
940 for (int i = 0; i < 4; ++i)
941 if (mask & (3 << (2 * i)))
942 o |= (1 << i);
943
944 return o;
945
946 }
947
948 /* Determines effective writemask, taking quirks and expansion into account */
949 static unsigned
950 effective_writemask(midgard_vector_alu *alu)
951 {
952 /* Channel count is off-by-one to fit in two-bits (0 channel makes no
953 * sense) */
954
955 unsigned channel_count = GET_CHANNEL_COUNT(alu_opcode_props[alu->op].props);
956
957 /* If there is a fixed channel count, construct the appropriate mask */
958
959 if (channel_count)
960 return (1 << channel_count) - 1;
961
962 /* Otherwise, just squeeze the existing mask */
963 return squeeze_writemask(alu->mask);
964 }
965
966 static unsigned
967 find_or_allocate_temp(compiler_context *ctx, unsigned hash)
968 {
969 if ((hash < 0) || (hash >= SSA_FIXED_MINIMUM))
970 return hash;
971
972 unsigned temp = (uintptr_t) _mesa_hash_table_u64_search(ctx->hash_to_temp, hash + 1);
973
974 if (temp)
975 return temp - 1;
976
977 /* If no temp is find, allocate one */
978 temp = ctx->temp_count++;
979 ctx->max_hash = MAX2(ctx->max_hash, hash);
980
981 _mesa_hash_table_u64_insert(ctx->hash_to_temp, hash + 1, (void *) ((uintptr_t) temp + 1));
982
983 return temp;
984 }
985
986 static unsigned
987 nir_src_index(compiler_context *ctx, nir_src *src)
988 {
989 if (src->is_ssa)
990 return src->ssa->index;
991 else {
992 assert(!src->reg.indirect);
993 return ctx->func->impl->ssa_alloc + src->reg.reg->index;
994 }
995 }
996
997 static unsigned
998 nir_dest_index(compiler_context *ctx, nir_dest *dst)
999 {
1000 if (dst->is_ssa)
1001 return dst->ssa.index;
1002 else {
1003 assert(!dst->reg.indirect);
1004 return ctx->func->impl->ssa_alloc + dst->reg.reg->index;
1005 }
1006 }
1007
1008 static unsigned
1009 nir_alu_src_index(compiler_context *ctx, nir_alu_src *src)
1010 {
1011 return nir_src_index(ctx, &src->src);
1012 }
1013
1014 /* Midgard puts conditionals in r31.w; move an arbitrary source (the output of
1015 * a conditional test) into that register */
1016
1017 static void
1018 emit_condition(compiler_context *ctx, nir_src *src, bool for_branch, unsigned component)
1019 {
1020 int condition = nir_src_index(ctx, src);
1021
1022 /* Source to swizzle the desired component into w */
1023
1024 const midgard_vector_alu_src alu_src = {
1025 .swizzle = SWIZZLE(component, component, component, component),
1026 };
1027
1028 /* There is no boolean move instruction. Instead, we simulate a move by
1029 * ANDing the condition with itself to get it into r31.w */
1030
1031 midgard_instruction ins = {
1032 .type = TAG_ALU_4,
1033 .unit = for_branch ? UNIT_SMUL : UNIT_SADD, /* TODO: DEDUCE THIS */
1034 .ssa_args = {
1035 .src0 = condition,
1036 .src1 = condition,
1037 .dest = SSA_FIXED_REGISTER(31),
1038 },
1039 .alu = {
1040 .op = midgard_alu_op_iand,
1041 .reg_mode = midgard_reg_mode_32,
1042 .dest_override = midgard_dest_override_none,
1043 .mask = (0x3 << 6), /* w */
1044 .src1 = vector_alu_srco_unsigned(alu_src),
1045 .src2 = vector_alu_srco_unsigned(alu_src)
1046 },
1047 };
1048
1049 emit_mir_instruction(ctx, ins);
1050 }
1051
1052 /* Likewise, indirect offsets are put in r27.w. TODO: Allow componentwise
1053 * pinning to eliminate this move in all known cases */
1054
1055 static void
1056 emit_indirect_offset(compiler_context *ctx, nir_src *src)
1057 {
1058 int offset = nir_src_index(ctx, src);
1059
1060 midgard_instruction ins = {
1061 .type = TAG_ALU_4,
1062 .ssa_args = {
1063 .src0 = SSA_UNUSED_1,
1064 .src1 = offset,
1065 .dest = SSA_FIXED_REGISTER(REGISTER_OFFSET),
1066 },
1067 .alu = {
1068 .op = midgard_alu_op_imov,
1069 .reg_mode = midgard_reg_mode_32,
1070 .dest_override = midgard_dest_override_none,
1071 .mask = (0x3 << 6), /* w */
1072 .src1 = vector_alu_srco_unsigned(zero_alu_src),
1073 .src2 = vector_alu_srco_unsigned(blank_alu_src_xxxx)
1074 },
1075 };
1076
1077 emit_mir_instruction(ctx, ins);
1078 }
1079
1080 #define ALU_CASE(nir, _op) \
1081 case nir_op_##nir: \
1082 op = midgard_alu_op_##_op; \
1083 break;
1084
1085 static bool
1086 nir_is_fzero_constant(nir_src src)
1087 {
1088 if (!nir_src_is_const(src))
1089 return false;
1090
1091 for (unsigned c = 0; c < nir_src_num_components(src); ++c) {
1092 if (nir_src_comp_as_float(src, c) != 0.0)
1093 return false;
1094 }
1095
1096 return true;
1097 }
1098
1099 static void
1100 emit_alu(compiler_context *ctx, nir_alu_instr *instr)
1101 {
1102 bool is_ssa = instr->dest.dest.is_ssa;
1103
1104 unsigned dest = nir_dest_index(ctx, &instr->dest.dest);
1105 unsigned nr_components = is_ssa ? instr->dest.dest.ssa.num_components : instr->dest.dest.reg.reg->num_components;
1106 unsigned nr_inputs = nir_op_infos[instr->op].num_inputs;
1107
1108 /* Most Midgard ALU ops have a 1:1 correspondance to NIR ops; these are
1109 * supported. A few do not and are commented for now. Also, there are a
1110 * number of NIR ops which Midgard does not support and need to be
1111 * lowered, also TODO. This switch block emits the opcode and calling
1112 * convention of the Midgard instruction; actual packing is done in
1113 * emit_alu below */
1114
1115 unsigned op;
1116
1117 switch (instr->op) {
1118 ALU_CASE(fadd, fadd);
1119 ALU_CASE(fmul, fmul);
1120 ALU_CASE(fmin, fmin);
1121 ALU_CASE(fmax, fmax);
1122 ALU_CASE(imin, imin);
1123 ALU_CASE(imax, imax);
1124 ALU_CASE(umin, umin);
1125 ALU_CASE(umax, umax);
1126 ALU_CASE(fmov, fmov);
1127 ALU_CASE(ffloor, ffloor);
1128 ALU_CASE(fround_even, froundeven);
1129 ALU_CASE(ftrunc, ftrunc);
1130 ALU_CASE(fceil, fceil);
1131 ALU_CASE(fdot3, fdot3);
1132 ALU_CASE(fdot4, fdot4);
1133 ALU_CASE(iadd, iadd);
1134 ALU_CASE(isub, isub);
1135 ALU_CASE(imul, imul);
1136 ALU_CASE(iabs, iabs);
1137 ALU_CASE(imov, imov);
1138
1139 ALU_CASE(feq32, feq);
1140 ALU_CASE(fne32, fne);
1141 ALU_CASE(flt32, flt);
1142 ALU_CASE(ieq32, ieq);
1143 ALU_CASE(ine32, ine);
1144 ALU_CASE(ilt32, ilt);
1145 ALU_CASE(ult32, ult);
1146
1147 /* We don't have a native b2f32 instruction. Instead, like many
1148 * GPUs, we exploit booleans as 0/~0 for false/true, and
1149 * correspondingly AND
1150 * by 1.0 to do the type conversion. For the moment, prime us
1151 * to emit:
1152 *
1153 * iand [whatever], #0
1154 *
1155 * At the end of emit_alu (as MIR), we'll fix-up the constant
1156 */
1157
1158 ALU_CASE(b2f32, iand);
1159 ALU_CASE(b2i32, iand);
1160
1161 /* Likewise, we don't have a dedicated f2b32 instruction, but
1162 * we can do a "not equal to 0.0" test. */
1163
1164 ALU_CASE(f2b32, fne);
1165 ALU_CASE(i2b32, ine);
1166
1167 ALU_CASE(frcp, frcp);
1168 ALU_CASE(frsq, frsqrt);
1169 ALU_CASE(fsqrt, fsqrt);
1170 ALU_CASE(fexp2, fexp2);
1171 ALU_CASE(flog2, flog2);
1172
1173 ALU_CASE(f2i32, f2i);
1174 ALU_CASE(f2u32, f2u);
1175 ALU_CASE(i2f32, i2f);
1176 ALU_CASE(u2f32, u2f);
1177
1178 ALU_CASE(fsin, fsin);
1179 ALU_CASE(fcos, fcos);
1180
1181 ALU_CASE(iand, iand);
1182 ALU_CASE(ior, ior);
1183 ALU_CASE(ixor, ixor);
1184 ALU_CASE(inot, inand);
1185 ALU_CASE(ishl, ishl);
1186 ALU_CASE(ishr, iasr);
1187 ALU_CASE(ushr, ilsr);
1188
1189 ALU_CASE(b32all_fequal2, fball_eq);
1190 ALU_CASE(b32all_fequal3, fball_eq);
1191 ALU_CASE(b32all_fequal4, fball_eq);
1192
1193 ALU_CASE(b32any_fnequal2, fbany_neq);
1194 ALU_CASE(b32any_fnequal3, fbany_neq);
1195 ALU_CASE(b32any_fnequal4, fbany_neq);
1196
1197 ALU_CASE(b32all_iequal2, iball_eq);
1198 ALU_CASE(b32all_iequal3, iball_eq);
1199 ALU_CASE(b32all_iequal4, iball_eq);
1200
1201 ALU_CASE(b32any_inequal2, ibany_neq);
1202 ALU_CASE(b32any_inequal3, ibany_neq);
1203 ALU_CASE(b32any_inequal4, ibany_neq);
1204
1205 /* For greater-or-equal, we lower to less-or-equal and flip the
1206 * arguments */
1207
1208 case nir_op_fge:
1209 case nir_op_fge32:
1210 case nir_op_ige32:
1211 case nir_op_uge32: {
1212 op =
1213 instr->op == nir_op_fge ? midgard_alu_op_fle :
1214 instr->op == nir_op_fge32 ? midgard_alu_op_fle :
1215 instr->op == nir_op_ige32 ? midgard_alu_op_ile :
1216 instr->op == nir_op_uge32 ? midgard_alu_op_ule :
1217 0;
1218
1219 /* Swap via temporary */
1220 nir_alu_src temp = instr->src[1];
1221 instr->src[1] = instr->src[0];
1222 instr->src[0] = temp;
1223
1224 break;
1225 }
1226
1227 /* For a few special csel cases not handled by NIR, we can opt to
1228 * bitwise. Otherwise, we emit the condition and do a real csel */
1229
1230 case nir_op_b32csel: {
1231 if (nir_is_fzero_constant(instr->src[2].src)) {
1232 /* (b ? v : 0) = (b & v) */
1233 op = midgard_alu_op_iand;
1234 nr_inputs = 2;
1235 } else if (nir_is_fzero_constant(instr->src[1].src)) {
1236 /* (b ? 0 : v) = (!b ? v : 0) = (~b & v) = (v & ~b) */
1237 op = midgard_alu_op_iandnot;
1238 nr_inputs = 2;
1239 instr->src[1] = instr->src[0];
1240 instr->src[0] = instr->src[2];
1241 } else {
1242 /* Midgard features both fcsel and icsel, depending on
1243 * the type of the arguments/output. However, as long
1244 * as we're careful we can _always_ use icsel and
1245 * _never_ need fcsel, since the latter does additional
1246 * floating-point-specific processing whereas the
1247 * former just moves bits on the wire. It's not obvious
1248 * why these are separate opcodes, save for the ability
1249 * to do things like sat/pos/abs/neg for free */
1250
1251 op = midgard_alu_op_icsel;
1252
1253 /* csel works as a two-arg in Midgard, since the condition is hardcoded in r31.w */
1254 nr_inputs = 2;
1255
1256 /* Figure out which component the condition is in */
1257
1258 unsigned comp = instr->src[0].swizzle[0];
1259
1260 /* Make sure NIR isn't throwing a mixed condition at us */
1261
1262 for (unsigned c = 1; c < nr_components; ++c)
1263 assert(instr->src[0].swizzle[c] == comp);
1264
1265 /* Emit the condition into r31.w */
1266 emit_condition(ctx, &instr->src[0].src, false, comp);
1267
1268 /* The condition is the first argument; move the other
1269 * arguments up one to be a binary instruction for
1270 * Midgard */
1271
1272 memmove(instr->src, instr->src + 1, 2 * sizeof(nir_alu_src));
1273 }
1274 break;
1275 }
1276
1277 default:
1278 DBG("Unhandled ALU op %s\n", nir_op_infos[instr->op].name);
1279 assert(0);
1280 return;
1281 }
1282
1283 /* Midgard can perform certain modifiers on output ofa n ALU op */
1284 midgard_outmod outmod =
1285 instr->dest.saturate ? midgard_outmod_sat : midgard_outmod_none;
1286
1287 /* fmax(a, 0.0) can turn into a .pos modifier as an optimization */
1288
1289 if (instr->op == nir_op_fmax) {
1290 if (nir_is_fzero_constant(instr->src[0].src)) {
1291 op = midgard_alu_op_fmov;
1292 nr_inputs = 1;
1293 outmod = midgard_outmod_pos;
1294 instr->src[0] = instr->src[1];
1295 } else if (nir_is_fzero_constant(instr->src[1].src)) {
1296 op = midgard_alu_op_fmov;
1297 nr_inputs = 1;
1298 outmod = midgard_outmod_pos;
1299 }
1300 }
1301
1302 /* Fetch unit, quirks, etc information */
1303 unsigned opcode_props = alu_opcode_props[op].props;
1304 bool quirk_flipped_r24 = opcode_props & QUIRK_FLIPPED_R24;
1305
1306 /* src0 will always exist afaik, but src1 will not for 1-argument
1307 * instructions. The latter can only be fetched if the instruction
1308 * needs it, or else we may segfault. */
1309
1310 unsigned src0 = nir_alu_src_index(ctx, &instr->src[0]);
1311 unsigned src1 = nr_inputs == 2 ? nir_alu_src_index(ctx, &instr->src[1]) : SSA_UNUSED_0;
1312
1313 /* Rather than use the instruction generation helpers, we do it
1314 * ourselves here to avoid the mess */
1315
1316 midgard_instruction ins = {
1317 .type = TAG_ALU_4,
1318 .ssa_args = {
1319 .src0 = quirk_flipped_r24 ? SSA_UNUSED_1 : src0,
1320 .src1 = quirk_flipped_r24 ? src0 : src1,
1321 .dest = dest,
1322 }
1323 };
1324
1325 nir_alu_src *nirmods[2] = { NULL };
1326
1327 if (nr_inputs == 2) {
1328 nirmods[0] = &instr->src[0];
1329 nirmods[1] = &instr->src[1];
1330 } else if (nr_inputs == 1) {
1331 nirmods[quirk_flipped_r24] = &instr->src[0];
1332 } else {
1333 assert(0);
1334 }
1335
1336 bool is_int = midgard_is_integer_op(op);
1337
1338 midgard_vector_alu alu = {
1339 .op = op,
1340 .reg_mode = midgard_reg_mode_32,
1341 .dest_override = midgard_dest_override_none,
1342 .outmod = outmod,
1343
1344 /* Writemask only valid for non-SSA NIR */
1345 .mask = expand_writemask((1 << nr_components) - 1),
1346
1347 .src1 = vector_alu_srco_unsigned(vector_alu_modifiers(nirmods[0], is_int)),
1348 .src2 = vector_alu_srco_unsigned(vector_alu_modifiers(nirmods[1], is_int)),
1349 };
1350
1351 /* Apply writemask if non-SSA, keeping in mind that we can't write to components that don't exist */
1352
1353 if (!is_ssa)
1354 alu.mask &= expand_writemask(instr->dest.write_mask);
1355
1356 ins.alu = alu;
1357
1358 /* Late fixup for emulated instructions */
1359
1360 if (instr->op == nir_op_b2f32 || instr->op == nir_op_b2i32) {
1361 /* Presently, our second argument is an inline #0 constant.
1362 * Switch over to an embedded 1.0 constant (that can't fit
1363 * inline, since we're 32-bit, not 16-bit like the inline
1364 * constants) */
1365
1366 ins.ssa_args.inline_constant = false;
1367 ins.ssa_args.src1 = SSA_FIXED_REGISTER(REGISTER_CONSTANT);
1368 ins.has_constants = true;
1369
1370 if (instr->op == nir_op_b2f32) {
1371 ins.constants[0] = 1.0f;
1372 } else {
1373 /* Type pun it into place */
1374 uint32_t one = 0x1;
1375 memcpy(&ins.constants[0], &one, sizeof(uint32_t));
1376 }
1377
1378 ins.alu.src2 = vector_alu_srco_unsigned(blank_alu_src_xxxx);
1379 } else if (instr->op == nir_op_f2b32 || instr->op == nir_op_i2b32) {
1380 ins.ssa_args.inline_constant = false;
1381 ins.ssa_args.src1 = SSA_FIXED_REGISTER(REGISTER_CONSTANT);
1382 ins.has_constants = true;
1383 ins.constants[0] = 0.0f;
1384 ins.alu.src2 = vector_alu_srco_unsigned(blank_alu_src_xxxx);
1385 } else if (instr->op == nir_op_inot) {
1386 /* ~b = ~(b & b), so duplicate the source */
1387 ins.ssa_args.src1 = ins.ssa_args.src0;
1388 ins.alu.src2 = ins.alu.src1;
1389 }
1390
1391 if ((opcode_props & UNITS_ALL) == UNIT_VLUT) {
1392 /* To avoid duplicating the lookup tables (probably), true LUT
1393 * instructions can only operate as if they were scalars. Lower
1394 * them here by changing the component. */
1395
1396 uint8_t original_swizzle[4];
1397 memcpy(original_swizzle, nirmods[0]->swizzle, sizeof(nirmods[0]->swizzle));
1398
1399 for (int i = 0; i < nr_components; ++i) {
1400 ins.alu.mask = (0x3) << (2 * i); /* Mask the associated component */
1401
1402 for (int j = 0; j < 4; ++j)
1403 nirmods[0]->swizzle[j] = original_swizzle[i]; /* Pull from the correct component */
1404
1405 ins.alu.src1 = vector_alu_srco_unsigned(vector_alu_modifiers(nirmods[0], is_int));
1406 emit_mir_instruction(ctx, ins);
1407 }
1408 } else {
1409 emit_mir_instruction(ctx, ins);
1410 }
1411 }
1412
1413 #undef ALU_CASE
1414
1415 static void
1416 emit_uniform_read(compiler_context *ctx, unsigned dest, unsigned offset, nir_src *indirect_offset)
1417 {
1418 /* TODO: half-floats */
1419
1420 if (!indirect_offset && offset < ctx->uniform_cutoff) {
1421 /* Fast path: For the first 16 uniforms, direct accesses are
1422 * 0-cycle, since they're just a register fetch in the usual
1423 * case. So, we alias the registers while we're still in
1424 * SSA-space */
1425
1426 int reg_slot = 23 - offset;
1427 alias_ssa(ctx, dest, SSA_FIXED_REGISTER(reg_slot));
1428 } else {
1429 /* Otherwise, read from the 'special' UBO to access
1430 * higher-indexed uniforms, at a performance cost. More
1431 * generally, we're emitting a UBO read instruction. */
1432
1433 midgard_instruction ins = m_load_uniform_32(dest, offset);
1434
1435 /* TODO: Don't split */
1436 ins.load_store.varying_parameters = (offset & 7) << 7;
1437 ins.load_store.address = offset >> 3;
1438
1439 if (indirect_offset) {
1440 emit_indirect_offset(ctx, indirect_offset);
1441 ins.load_store.unknown = 0x8700; /* xxx: what is this? */
1442 } else {
1443 ins.load_store.unknown = 0x1E00; /* xxx: what is this? */
1444 }
1445
1446 emit_mir_instruction(ctx, ins);
1447 }
1448 }
1449
1450 static void
1451 emit_sysval_read(compiler_context *ctx, nir_intrinsic_instr *instr)
1452 {
1453 /* First, pull out the destination */
1454 unsigned dest = nir_dest_index(ctx, &instr->dest);
1455
1456 /* Now, figure out which uniform this is */
1457 int sysval = midgard_nir_sysval_for_intrinsic(instr);
1458 void *val = _mesa_hash_table_u64_search(ctx->sysval_to_id, sysval);
1459
1460 /* Sysvals are prefix uniforms */
1461 unsigned uniform = ((uintptr_t) val) - 1;
1462
1463 /* Emit the read itself -- this is never indirect */
1464 emit_uniform_read(ctx, dest, uniform, NULL);
1465 }
1466
1467 static void
1468 emit_intrinsic(compiler_context *ctx, nir_intrinsic_instr *instr)
1469 {
1470 unsigned offset, reg;
1471
1472 switch (instr->intrinsic) {
1473 case nir_intrinsic_discard_if:
1474 emit_condition(ctx, &instr->src[0], true, COMPONENT_X);
1475
1476 /* fallthrough */
1477
1478 case nir_intrinsic_discard: {
1479 bool conditional = instr->intrinsic == nir_intrinsic_discard_if;
1480 struct midgard_instruction discard = v_branch(conditional, false);
1481 discard.branch.target_type = TARGET_DISCARD;
1482 emit_mir_instruction(ctx, discard);
1483
1484 ctx->can_discard = true;
1485 break;
1486 }
1487
1488 case nir_intrinsic_load_uniform:
1489 case nir_intrinsic_load_input:
1490 offset = nir_intrinsic_base(instr);
1491
1492 bool direct = nir_src_is_const(instr->src[0]);
1493
1494 if (direct) {
1495 offset += nir_src_as_uint(instr->src[0]);
1496 }
1497
1498 reg = nir_dest_index(ctx, &instr->dest);
1499
1500 if (instr->intrinsic == nir_intrinsic_load_uniform && !ctx->is_blend) {
1501 emit_uniform_read(ctx, reg, ctx->sysval_count + offset, !direct ? &instr->src[0] : NULL);
1502 } else if (ctx->stage == MESA_SHADER_FRAGMENT && !ctx->is_blend) {
1503 /* XXX: Half-floats? */
1504 /* TODO: swizzle, mask */
1505
1506 midgard_instruction ins = m_load_vary_32(reg, offset);
1507
1508 midgard_varying_parameter p = {
1509 .is_varying = 1,
1510 .interpolation = midgard_interp_default,
1511 .flat = /*var->data.interpolation == INTERP_MODE_FLAT*/ 0
1512 };
1513
1514 unsigned u;
1515 memcpy(&u, &p, sizeof(p));
1516 ins.load_store.varying_parameters = u;
1517
1518 if (direct) {
1519 /* We have the offset totally ready */
1520 ins.load_store.unknown = 0x1e9e; /* xxx: what is this? */
1521 } else {
1522 /* We have it partially ready, but we need to
1523 * add in the dynamic index, moved to r27.w */
1524 emit_indirect_offset(ctx, &instr->src[0]);
1525 ins.load_store.unknown = 0x79e; /* xxx: what is this? */
1526 }
1527
1528 emit_mir_instruction(ctx, ins);
1529 } else if (ctx->is_blend && instr->intrinsic == nir_intrinsic_load_uniform) {
1530 /* Constant encoded as a pinned constant */
1531
1532 midgard_instruction ins = v_fmov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), blank_alu_src, reg);
1533 ins.has_constants = true;
1534 ins.has_blend_constant = true;
1535 emit_mir_instruction(ctx, ins);
1536 } else if (ctx->is_blend) {
1537 /* For blend shaders, a load might be
1538 * translated various ways depending on what
1539 * we're loading. Figure out how this is used */
1540
1541 nir_variable *out = NULL;
1542
1543 nir_foreach_variable(var, &ctx->nir->inputs) {
1544 int drvloc = var->data.driver_location;
1545
1546 if (nir_intrinsic_base(instr) == drvloc) {
1547 out = var;
1548 break;
1549 }
1550 }
1551
1552 assert(out);
1553
1554 if (out->data.location == VARYING_SLOT_COL0) {
1555 /* Source color preloaded to r0 */
1556
1557 midgard_pin_output(ctx, reg, 0);
1558 } else if (out->data.location == VARYING_SLOT_COL1) {
1559 /* Destination color must be read from framebuffer */
1560
1561 midgard_instruction ins = m_load_color_buffer_8(reg, 0);
1562 ins.load_store.swizzle = 0; /* xxxx */
1563
1564 /* Read each component sequentially */
1565
1566 for (int c = 0; c < 4; ++c) {
1567 ins.load_store.mask = (1 << c);
1568 ins.load_store.unknown = c;
1569 emit_mir_instruction(ctx, ins);
1570 }
1571
1572 /* vadd.u2f hr2, zext(hr2), #0 */
1573
1574 midgard_vector_alu_src alu_src = blank_alu_src;
1575 alu_src.mod = midgard_int_zero_extend;
1576 alu_src.half = true;
1577
1578 midgard_instruction u2f = {
1579 .type = TAG_ALU_4,
1580 .ssa_args = {
1581 .src0 = reg,
1582 .src1 = SSA_UNUSED_0,
1583 .dest = reg,
1584 .inline_constant = true
1585 },
1586 .alu = {
1587 .op = midgard_alu_op_u2f,
1588 .reg_mode = midgard_reg_mode_16,
1589 .dest_override = midgard_dest_override_none,
1590 .mask = 0xF,
1591 .src1 = vector_alu_srco_unsigned(alu_src),
1592 .src2 = vector_alu_srco_unsigned(blank_alu_src),
1593 }
1594 };
1595
1596 emit_mir_instruction(ctx, u2f);
1597
1598 /* vmul.fmul.sat r1, hr2, #0.00392151 */
1599
1600 alu_src.mod = 0;
1601
1602 midgard_instruction fmul = {
1603 .type = TAG_ALU_4,
1604 .inline_constant = _mesa_float_to_half(1.0 / 255.0),
1605 .ssa_args = {
1606 .src0 = reg,
1607 .dest = reg,
1608 .src1 = SSA_UNUSED_0,
1609 .inline_constant = true
1610 },
1611 .alu = {
1612 .op = midgard_alu_op_fmul,
1613 .reg_mode = midgard_reg_mode_32,
1614 .dest_override = midgard_dest_override_none,
1615 .outmod = midgard_outmod_sat,
1616 .mask = 0xFF,
1617 .src1 = vector_alu_srco_unsigned(alu_src),
1618 .src2 = vector_alu_srco_unsigned(blank_alu_src),
1619 }
1620 };
1621
1622 emit_mir_instruction(ctx, fmul);
1623 } else {
1624 DBG("Unknown input in blend shader\n");
1625 assert(0);
1626 }
1627 } else if (ctx->stage == MESA_SHADER_VERTEX) {
1628 midgard_instruction ins = m_load_attr_32(reg, offset);
1629 ins.load_store.unknown = 0x1E1E; /* XXX: What is this? */
1630 ins.load_store.mask = (1 << instr->num_components) - 1;
1631 emit_mir_instruction(ctx, ins);
1632 } else {
1633 DBG("Unknown load\n");
1634 assert(0);
1635 }
1636
1637 break;
1638
1639 case nir_intrinsic_store_output:
1640 assert(nir_src_is_const(instr->src[1]) && "no indirect outputs");
1641
1642 offset = nir_intrinsic_base(instr) + nir_src_as_uint(instr->src[1]);
1643
1644 reg = nir_src_index(ctx, &instr->src[0]);
1645
1646 if (ctx->stage == MESA_SHADER_FRAGMENT) {
1647 /* gl_FragColor is not emitted with load/store
1648 * instructions. Instead, it gets plonked into
1649 * r0 at the end of the shader and we do the
1650 * framebuffer writeout dance. TODO: Defer
1651 * writes */
1652
1653 midgard_pin_output(ctx, reg, 0);
1654
1655 /* Save the index we're writing to for later reference
1656 * in the epilogue */
1657
1658 ctx->fragment_output = reg;
1659 } else if (ctx->stage == MESA_SHADER_VERTEX) {
1660 /* Varyings are written into one of two special
1661 * varying register, r26 or r27. The register itself is selected as the register
1662 * in the st_vary instruction, minus the base of 26. E.g. write into r27 and then call st_vary(1)
1663 *
1664 * Normally emitting fmov's is frowned upon,
1665 * but due to unique constraints of
1666 * REGISTER_VARYING, fmov emission + a
1667 * dedicated cleanup pass is the only way to
1668 * guarantee correctness when considering some
1669 * (common) edge cases XXX: FIXME */
1670
1671 /* If this varying corresponds to a constant (why?!),
1672 * emit that now since it won't get picked up by
1673 * hoisting (since there is no corresponding move
1674 * emitted otherwise) */
1675
1676 void *constant_value = _mesa_hash_table_u64_search(ctx->ssa_constants, reg + 1);
1677
1678 if (constant_value) {
1679 /* Special case: emit the varying write
1680 * directly to r26 (looks funny in asm but it's
1681 * fine) and emit the store _now_. Possibly
1682 * slightly slower, but this is a really stupid
1683 * special case anyway (why on earth would you
1684 * have a constant varying? Your own fault for
1685 * slightly worse perf :P) */
1686
1687 midgard_instruction ins = v_fmov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), blank_alu_src, SSA_FIXED_REGISTER(26));
1688 attach_constants(ctx, &ins, constant_value, reg + 1);
1689 emit_mir_instruction(ctx, ins);
1690
1691 midgard_instruction st = m_store_vary_32(SSA_FIXED_REGISTER(0), offset);
1692 st.load_store.unknown = 0x1E9E; /* XXX: What is this? */
1693 emit_mir_instruction(ctx, st);
1694 } else {
1695 /* Do not emit the varying yet -- instead, just mark down that we need to later */
1696
1697 _mesa_hash_table_u64_insert(ctx->ssa_varyings, reg + 1, (void *) ((uintptr_t) (offset + 1)));
1698 }
1699 } else {
1700 DBG("Unknown store\n");
1701 assert(0);
1702 }
1703
1704 break;
1705
1706 case nir_intrinsic_load_alpha_ref_float:
1707 assert(instr->dest.is_ssa);
1708
1709 float ref_value = ctx->alpha_ref;
1710
1711 float *v = ralloc_array(NULL, float, 4);
1712 memcpy(v, &ref_value, sizeof(float));
1713 _mesa_hash_table_u64_insert(ctx->ssa_constants, instr->dest.ssa.index + 1, v);
1714 break;
1715
1716 case nir_intrinsic_load_viewport_scale:
1717 case nir_intrinsic_load_viewport_offset:
1718 emit_sysval_read(ctx, instr);
1719 break;
1720
1721 default:
1722 printf ("Unhandled intrinsic\n");
1723 assert(0);
1724 break;
1725 }
1726 }
1727
1728 static unsigned
1729 midgard_tex_format(enum glsl_sampler_dim dim)
1730 {
1731 switch (dim) {
1732 case GLSL_SAMPLER_DIM_2D:
1733 case GLSL_SAMPLER_DIM_EXTERNAL:
1734 return TEXTURE_2D;
1735
1736 case GLSL_SAMPLER_DIM_3D:
1737 return TEXTURE_3D;
1738
1739 case GLSL_SAMPLER_DIM_CUBE:
1740 return TEXTURE_CUBE;
1741
1742 default:
1743 DBG("Unknown sampler dim type\n");
1744 assert(0);
1745 return 0;
1746 }
1747 }
1748
1749 static void
1750 emit_tex(compiler_context *ctx, nir_tex_instr *instr)
1751 {
1752 /* TODO */
1753 //assert (!instr->sampler);
1754 //assert (!instr->texture_array_size);
1755 assert (instr->op == nir_texop_tex);
1756
1757 /* Allocate registers via a round robin scheme to alternate between the two registers */
1758 int reg = ctx->texture_op_count & 1;
1759 int in_reg = reg, out_reg = reg;
1760
1761 /* Make room for the reg */
1762
1763 if (ctx->texture_index[reg] > -1)
1764 unalias_ssa(ctx, ctx->texture_index[reg]);
1765
1766 int texture_index = instr->texture_index;
1767 int sampler_index = texture_index;
1768
1769 for (unsigned i = 0; i < instr->num_srcs; ++i) {
1770 switch (instr->src[i].src_type) {
1771 case nir_tex_src_coord: {
1772 int index = nir_src_index(ctx, &instr->src[i].src);
1773
1774 midgard_vector_alu_src alu_src = blank_alu_src;
1775
1776 int reg = SSA_FIXED_REGISTER(REGISTER_TEXTURE_BASE + in_reg);
1777
1778 if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE) {
1779 /* For cubemaps, we need to load coords into
1780 * special r27, and then use a special ld/st op
1781 * to copy into the texture register */
1782
1783 alu_src.swizzle = SWIZZLE(COMPONENT_X, COMPONENT_Y, COMPONENT_Z, COMPONENT_X);
1784
1785 midgard_instruction move = v_fmov(index, alu_src, SSA_FIXED_REGISTER(27));
1786 emit_mir_instruction(ctx, move);
1787
1788 midgard_instruction st = m_store_cubemap_coords(reg, 0);
1789 st.load_store.unknown = 0x24; /* XXX: What is this? */
1790 st.load_store.mask = 0x3; /* xy? */
1791 st.load_store.swizzle = alu_src.swizzle;
1792 emit_mir_instruction(ctx, st);
1793
1794 } else {
1795 alu_src.swizzle = SWIZZLE(COMPONENT_X, COMPONENT_Y, COMPONENT_X, COMPONENT_X);
1796
1797 midgard_instruction ins = v_fmov(index, alu_src, reg);
1798 emit_mir_instruction(ctx, ins);
1799 }
1800
1801 break;
1802 }
1803
1804 default: {
1805 DBG("Unknown source type\n");
1806 //assert(0);
1807 break;
1808 }
1809 }
1810 }
1811
1812 /* No helper to build texture words -- we do it all here */
1813 midgard_instruction ins = {
1814 .type = TAG_TEXTURE_4,
1815 .texture = {
1816 .op = TEXTURE_OP_NORMAL,
1817 .format = midgard_tex_format(instr->sampler_dim),
1818 .texture_handle = texture_index,
1819 .sampler_handle = sampler_index,
1820
1821 /* TODO: Don't force xyzw */
1822 .swizzle = SWIZZLE(COMPONENT_X, COMPONENT_Y, COMPONENT_Z, COMPONENT_W),
1823 .mask = 0xF,
1824
1825 /* TODO: half */
1826 //.in_reg_full = 1,
1827 .out_full = 1,
1828
1829 .filter = 1,
1830
1831 /* Always 1 */
1832 .unknown7 = 1,
1833
1834 /* Assume we can continue; hint it out later */
1835 .cont = 1,
1836 }
1837 };
1838
1839 /* Set registers to read and write from the same place */
1840 ins.texture.in_reg_select = in_reg;
1841 ins.texture.out_reg_select = out_reg;
1842
1843 /* TODO: Dynamic swizzle input selection, half-swizzles? */
1844 if (instr->sampler_dim == GLSL_SAMPLER_DIM_3D) {
1845 ins.texture.in_reg_swizzle_right = COMPONENT_X;
1846 ins.texture.in_reg_swizzle_left = COMPONENT_Y;
1847 //ins.texture.in_reg_swizzle_third = COMPONENT_Z;
1848 } else {
1849 ins.texture.in_reg_swizzle_left = COMPONENT_X;
1850 ins.texture.in_reg_swizzle_right = COMPONENT_Y;
1851 //ins.texture.in_reg_swizzle_third = COMPONENT_X;
1852 }
1853
1854 emit_mir_instruction(ctx, ins);
1855
1856 /* Simultaneously alias the destination and emit a move for it. The move will be eliminated if possible */
1857
1858 int o_reg = REGISTER_TEXTURE_BASE + out_reg, o_index = nir_dest_index(ctx, &instr->dest);
1859 alias_ssa(ctx, o_index, SSA_FIXED_REGISTER(o_reg));
1860 ctx->texture_index[reg] = o_index;
1861
1862 midgard_instruction ins2 = v_fmov(SSA_FIXED_REGISTER(o_reg), blank_alu_src, o_index);
1863 emit_mir_instruction(ctx, ins2);
1864
1865 /* Used for .cont and .last hinting */
1866 ctx->texture_op_count++;
1867 }
1868
1869 static void
1870 emit_jump(compiler_context *ctx, nir_jump_instr *instr)
1871 {
1872 switch (instr->type) {
1873 case nir_jump_break: {
1874 /* Emit a branch out of the loop */
1875 struct midgard_instruction br = v_branch(false, false);
1876 br.branch.target_type = TARGET_BREAK;
1877 br.branch.target_break = ctx->current_loop_depth;
1878 emit_mir_instruction(ctx, br);
1879
1880 DBG("break..\n");
1881 break;
1882 }
1883
1884 default:
1885 DBG("Unknown jump type %d\n", instr->type);
1886 break;
1887 }
1888 }
1889
1890 static void
1891 emit_instr(compiler_context *ctx, struct nir_instr *instr)
1892 {
1893 switch (instr->type) {
1894 case nir_instr_type_load_const:
1895 emit_load_const(ctx, nir_instr_as_load_const(instr));
1896 break;
1897
1898 case nir_instr_type_intrinsic:
1899 emit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
1900 break;
1901
1902 case nir_instr_type_alu:
1903 emit_alu(ctx, nir_instr_as_alu(instr));
1904 break;
1905
1906 case nir_instr_type_tex:
1907 emit_tex(ctx, nir_instr_as_tex(instr));
1908 break;
1909
1910 case nir_instr_type_jump:
1911 emit_jump(ctx, nir_instr_as_jump(instr));
1912 break;
1913
1914 case nir_instr_type_ssa_undef:
1915 /* Spurious */
1916 break;
1917
1918 default:
1919 DBG("Unhandled instruction type\n");
1920 break;
1921 }
1922 }
1923
1924 /* Determine the actual hardware from the index based on the RA results or special values */
1925
1926 static int
1927 dealias_register(compiler_context *ctx, struct ra_graph *g, int reg, int maxreg)
1928 {
1929 if (reg >= SSA_FIXED_MINIMUM)
1930 return SSA_REG_FROM_FIXED(reg);
1931
1932 if (reg >= 0) {
1933 assert(reg < maxreg);
1934 assert(g);
1935 int r = ra_get_node_reg(g, reg);
1936 ctx->work_registers = MAX2(ctx->work_registers, r);
1937 return r;
1938 }
1939
1940 switch (reg) {
1941 /* fmov style unused */
1942 case SSA_UNUSED_0:
1943 return REGISTER_UNUSED;
1944
1945 /* lut style unused */
1946 case SSA_UNUSED_1:
1947 return REGISTER_UNUSED;
1948
1949 default:
1950 DBG("Unknown SSA register alias %d\n", reg);
1951 assert(0);
1952 return 31;
1953 }
1954 }
1955
1956 static unsigned int
1957 midgard_ra_select_callback(struct ra_graph *g, BITSET_WORD *regs, void *data)
1958 {
1959 /* Choose the first available register to minimise reported register pressure */
1960
1961 for (int i = 0; i < 16; ++i) {
1962 if (BITSET_TEST(regs, i)) {
1963 return i;
1964 }
1965 }
1966
1967 assert(0);
1968 return 0;
1969 }
1970
1971 static bool
1972 midgard_is_live_in_instr(midgard_instruction *ins, int src)
1973 {
1974 if (ins->ssa_args.src0 == src) return true;
1975 if (ins->ssa_args.src1 == src) return true;
1976
1977 return false;
1978 }
1979
1980 /* Determine if a variable is live in the successors of a block */
1981 static bool
1982 is_live_after_successors(compiler_context *ctx, midgard_block *bl, int src)
1983 {
1984 for (unsigned i = 0; i < bl->nr_successors; ++i) {
1985 midgard_block *succ = bl->successors[i];
1986
1987 /* If we already visited, the value we're seeking
1988 * isn't down this path (or we would have short
1989 * circuited */
1990
1991 if (succ->visited) continue;
1992
1993 /* Otherwise (it's visited *now*), check the block */
1994
1995 succ->visited = true;
1996
1997 mir_foreach_instr_in_block(succ, ins) {
1998 if (midgard_is_live_in_instr(ins, src))
1999 return true;
2000 }
2001
2002 /* ...and also, check *its* successors */
2003 if (is_live_after_successors(ctx, succ, src))
2004 return true;
2005
2006 }
2007
2008 /* Welp. We're really not live. */
2009
2010 return false;
2011 }
2012
2013 static bool
2014 is_live_after(compiler_context *ctx, midgard_block *block, midgard_instruction *start, int src)
2015 {
2016 /* Check the rest of the block for liveness */
2017
2018 mir_foreach_instr_in_block_from(block, ins, mir_next_op(start)) {
2019 if (midgard_is_live_in_instr(ins, src))
2020 return true;
2021 }
2022
2023 /* Check the rest of the blocks for liveness recursively */
2024
2025 bool succ = is_live_after_successors(ctx, block, src);
2026
2027 mir_foreach_block(ctx, block) {
2028 block->visited = false;
2029 }
2030
2031 return succ;
2032 }
2033
2034 /* Once registers have been decided via register allocation
2035 * (allocate_registers), we need to rewrite the MIR to use registers instead of
2036 * SSA */
2037
2038 static void
2039 install_registers(compiler_context *ctx, struct ra_graph *g)
2040 {
2041 mir_foreach_block(ctx, block) {
2042 mir_foreach_instr_in_block(block, ins) {
2043 if (ins->compact_branch) continue;
2044
2045 ssa_args args = ins->ssa_args;
2046
2047 switch (ins->type) {
2048 case TAG_ALU_4:
2049 ins->registers.src1_reg = dealias_register(ctx, g, args.src0, ctx->temp_count);
2050
2051 ins->registers.src2_imm = args.inline_constant;
2052
2053 if (args.inline_constant) {
2054 /* Encode inline 16-bit constant as a vector by default */
2055
2056 ins->registers.src2_reg = ins->inline_constant >> 11;
2057
2058 int lower_11 = ins->inline_constant & ((1 << 12) - 1);
2059
2060 uint16_t imm = ((lower_11 >> 8) & 0x7) | ((lower_11 & 0xFF) << 3);
2061 ins->alu.src2 = imm << 2;
2062 } else {
2063 ins->registers.src2_reg = dealias_register(ctx, g, args.src1, ctx->temp_count);
2064 }
2065
2066 ins->registers.out_reg = dealias_register(ctx, g, args.dest, ctx->temp_count);
2067
2068 break;
2069
2070 case TAG_LOAD_STORE_4: {
2071 if (OP_IS_STORE_VARY(ins->load_store.op)) {
2072 /* TODO: use ssa_args for store_vary */
2073 ins->load_store.reg = 0;
2074 } else {
2075 bool has_dest = args.dest >= 0;
2076 int ssa_arg = has_dest ? args.dest : args.src0;
2077
2078 ins->load_store.reg = dealias_register(ctx, g, ssa_arg, ctx->temp_count);
2079 }
2080
2081 break;
2082 }
2083
2084 default:
2085 break;
2086 }
2087 }
2088 }
2089
2090 }
2091
2092 /* This routine performs the actual register allocation. It should be succeeded
2093 * by install_registers */
2094
2095 static struct ra_graph *
2096 allocate_registers(compiler_context *ctx)
2097 {
2098 /* First, initialize the RA */
2099 struct ra_regs *regs = ra_alloc_reg_set(NULL, 32, true);
2100
2101 /* Create a primary (general purpose) class, as well as special purpose
2102 * pipeline register classes */
2103
2104 int primary_class = ra_alloc_reg_class(regs);
2105 int varying_class = ra_alloc_reg_class(regs);
2106
2107 /* Add the full set of work registers */
2108 int work_count = 16 - MAX2((ctx->uniform_cutoff - 8), 0);
2109 for (int i = 0; i < work_count; ++i)
2110 ra_class_add_reg(regs, primary_class, i);
2111
2112 /* Add special registers */
2113 ra_class_add_reg(regs, varying_class, REGISTER_VARYING_BASE);
2114 ra_class_add_reg(regs, varying_class, REGISTER_VARYING_BASE + 1);
2115
2116 /* We're done setting up */
2117 ra_set_finalize(regs, NULL);
2118
2119 /* Transform the MIR into squeezed index form */
2120 mir_foreach_block(ctx, block) {
2121 mir_foreach_instr_in_block(block, ins) {
2122 if (ins->compact_branch) continue;
2123
2124 ins->ssa_args.src0 = find_or_allocate_temp(ctx, ins->ssa_args.src0);
2125 ins->ssa_args.src1 = find_or_allocate_temp(ctx, ins->ssa_args.src1);
2126 ins->ssa_args.dest = find_or_allocate_temp(ctx, ins->ssa_args.dest);
2127 }
2128 if (midgard_debug & MIDGARD_DBG_SHADERS)
2129 print_mir_block(block);
2130 }
2131
2132 /* No register allocation to do with no SSA */
2133
2134 if (!ctx->temp_count)
2135 return NULL;
2136
2137 /* Let's actually do register allocation */
2138 int nodes = ctx->temp_count;
2139 struct ra_graph *g = ra_alloc_interference_graph(regs, nodes);
2140
2141 /* Set everything to the work register class, unless it has somewhere
2142 * special to go */
2143
2144 mir_foreach_block(ctx, block) {
2145 mir_foreach_instr_in_block(block, ins) {
2146 if (ins->compact_branch) continue;
2147
2148 if (ins->ssa_args.dest < 0) continue;
2149
2150 if (ins->ssa_args.dest >= SSA_FIXED_MINIMUM) continue;
2151
2152 int class = primary_class;
2153
2154 ra_set_node_class(g, ins->ssa_args.dest, class);
2155 }
2156 }
2157
2158 for (int index = 0; index <= ctx->max_hash; ++index) {
2159 unsigned temp = (uintptr_t) _mesa_hash_table_u64_search(ctx->ssa_to_register, index + 1);
2160
2161 if (temp) {
2162 unsigned reg = temp - 1;
2163 int t = find_or_allocate_temp(ctx, index);
2164 ra_set_node_reg(g, t, reg);
2165 }
2166 }
2167
2168 /* Determine liveness */
2169
2170 int *live_start = malloc(nodes * sizeof(int));
2171 int *live_end = malloc(nodes * sizeof(int));
2172
2173 /* Initialize as non-existent */
2174
2175 for (int i = 0; i < nodes; ++i) {
2176 live_start[i] = live_end[i] = -1;
2177 }
2178
2179 int d = 0;
2180
2181 mir_foreach_block(ctx, block) {
2182 mir_foreach_instr_in_block(block, ins) {
2183 if (ins->compact_branch) continue;
2184
2185 /* Dest is < 0 for store_vary instructions, which break
2186 * the usual SSA conventions. Liveness analysis doesn't
2187 * make sense on these instructions, so skip them to
2188 * avoid memory corruption */
2189
2190 if (ins->ssa_args.dest < 0) continue;
2191
2192 if (ins->ssa_args.dest < SSA_FIXED_MINIMUM) {
2193 /* If this destination is not yet live, it is now since we just wrote it */
2194
2195 int dest = ins->ssa_args.dest;
2196
2197 if (live_start[dest] == -1)
2198 live_start[dest] = d;
2199 }
2200
2201 /* Since we just used a source, the source might be
2202 * dead now. Scan the rest of the block for
2203 * invocations, and if there are none, the source dies
2204 * */
2205
2206 int sources[2] = { ins->ssa_args.src0, ins->ssa_args.src1 };
2207
2208 for (int src = 0; src < 2; ++src) {
2209 int s = sources[src];
2210
2211 if (s < 0) continue;
2212
2213 if (s >= SSA_FIXED_MINIMUM) continue;
2214
2215 if (!is_live_after(ctx, block, ins, s)) {
2216 live_end[s] = d;
2217 }
2218 }
2219
2220 ++d;
2221 }
2222 }
2223
2224 /* If a node still hasn't been killed, kill it now */
2225
2226 for (int i = 0; i < nodes; ++i) {
2227 /* live_start == -1 most likely indicates a pinned output */
2228
2229 if (live_end[i] == -1)
2230 live_end[i] = d;
2231 }
2232
2233 /* Setup interference between nodes that are live at the same time */
2234
2235 for (int i = 0; i < nodes; ++i) {
2236 for (int j = i + 1; j < nodes; ++j) {
2237 if (!(live_start[i] >= live_end[j] || live_start[j] >= live_end[i]))
2238 ra_add_node_interference(g, i, j);
2239 }
2240 }
2241
2242 ra_set_select_reg_callback(g, midgard_ra_select_callback, NULL);
2243
2244 if (!ra_allocate(g)) {
2245 DBG("Error allocating registers\n");
2246 assert(0);
2247 }
2248
2249 /* Cleanup */
2250 free(live_start);
2251 free(live_end);
2252
2253 return g;
2254 }
2255
2256 /* Midgard IR only knows vector ALU types, but we sometimes need to actually
2257 * use scalar ALU instructions, for functional or performance reasons. To do
2258 * this, we just demote vector ALU payloads to scalar. */
2259
2260 static int
2261 component_from_mask(unsigned mask)
2262 {
2263 for (int c = 0; c < 4; ++c) {
2264 if (mask & (3 << (2 * c)))
2265 return c;
2266 }
2267
2268 assert(0);
2269 return 0;
2270 }
2271
2272 static bool
2273 is_single_component_mask(unsigned mask)
2274 {
2275 int components = 0;
2276
2277 for (int c = 0; c < 4; ++c)
2278 if (mask & (3 << (2 * c)))
2279 components++;
2280
2281 return components == 1;
2282 }
2283
2284 /* Create a mask of accessed components from a swizzle to figure out vector
2285 * dependencies */
2286
2287 static unsigned
2288 swizzle_to_access_mask(unsigned swizzle)
2289 {
2290 unsigned component_mask = 0;
2291
2292 for (int i = 0; i < 4; ++i) {
2293 unsigned c = (swizzle >> (2 * i)) & 3;
2294 component_mask |= (1 << c);
2295 }
2296
2297 return component_mask;
2298 }
2299
2300 static unsigned
2301 vector_to_scalar_source(unsigned u, bool is_int)
2302 {
2303 midgard_vector_alu_src v;
2304 memcpy(&v, &u, sizeof(v));
2305
2306 /* TODO: Integers */
2307
2308 midgard_scalar_alu_src s = {
2309 .full = !v.half,
2310 .component = (v.swizzle & 3) << 1
2311 };
2312
2313 if (is_int) {
2314 /* TODO */
2315 } else {
2316 s.abs = v.mod & MIDGARD_FLOAT_MOD_ABS;
2317 s.negate = v.mod & MIDGARD_FLOAT_MOD_NEG;
2318 }
2319
2320 unsigned o;
2321 memcpy(&o, &s, sizeof(s));
2322
2323 return o & ((1 << 6) - 1);
2324 }
2325
2326 static midgard_scalar_alu
2327 vector_to_scalar_alu(midgard_vector_alu v, midgard_instruction *ins)
2328 {
2329 bool is_int = midgard_is_integer_op(v.op);
2330
2331 /* The output component is from the mask */
2332 midgard_scalar_alu s = {
2333 .op = v.op,
2334 .src1 = vector_to_scalar_source(v.src1, is_int),
2335 .src2 = vector_to_scalar_source(v.src2, is_int),
2336 .unknown = 0,
2337 .outmod = v.outmod,
2338 .output_full = 1, /* TODO: Half */
2339 .output_component = component_from_mask(v.mask) << 1,
2340 };
2341
2342 /* Inline constant is passed along rather than trying to extract it
2343 * from v */
2344
2345 if (ins->ssa_args.inline_constant) {
2346 uint16_t imm = 0;
2347 int lower_11 = ins->inline_constant & ((1 << 12) - 1);
2348 imm |= (lower_11 >> 9) & 3;
2349 imm |= (lower_11 >> 6) & 4;
2350 imm |= (lower_11 >> 2) & 0x38;
2351 imm |= (lower_11 & 63) << 6;
2352
2353 s.src2 = imm;
2354 }
2355
2356 return s;
2357 }
2358
2359 /* Midgard prefetches instruction types, so during emission we need to
2360 * lookahead too. Unless this is the last instruction, in which we return 1. Or
2361 * if this is the second to last and the last is an ALU, then it's also 1... */
2362
2363 #define IS_ALU(tag) (tag == TAG_ALU_4 || tag == TAG_ALU_8 || \
2364 tag == TAG_ALU_12 || tag == TAG_ALU_16)
2365
2366 #define EMIT_AND_COUNT(type, val) util_dynarray_append(emission, type, val); \
2367 bytes_emitted += sizeof(type)
2368
2369 static void
2370 emit_binary_vector_instruction(midgard_instruction *ains,
2371 uint16_t *register_words, int *register_words_count,
2372 uint64_t *body_words, size_t *body_size, int *body_words_count,
2373 size_t *bytes_emitted)
2374 {
2375 memcpy(&register_words[(*register_words_count)++], &ains->registers, sizeof(ains->registers));
2376 *bytes_emitted += sizeof(midgard_reg_info);
2377
2378 body_size[*body_words_count] = sizeof(midgard_vector_alu);
2379 memcpy(&body_words[(*body_words_count)++], &ains->alu, sizeof(ains->alu));
2380 *bytes_emitted += sizeof(midgard_vector_alu);
2381 }
2382
2383 /* Checks for an SSA data hazard between two adjacent instructions, keeping in
2384 * mind that we are a vector architecture and we can write to different
2385 * components simultaneously */
2386
2387 static bool
2388 can_run_concurrent_ssa(midgard_instruction *first, midgard_instruction *second)
2389 {
2390 /* Each instruction reads some registers and writes to a register. See
2391 * where the first writes */
2392
2393 /* Figure out where exactly we wrote to */
2394 int source = first->ssa_args.dest;
2395 int source_mask = first->type == TAG_ALU_4 ? squeeze_writemask(first->alu.mask) : 0xF;
2396
2397 /* As long as the second doesn't read from the first, we're okay */
2398 if (second->ssa_args.src0 == source) {
2399 if (first->type == TAG_ALU_4) {
2400 /* Figure out which components we just read from */
2401
2402 int q = second->alu.src1;
2403 midgard_vector_alu_src *m = (midgard_vector_alu_src *) &q;
2404
2405 /* Check if there are components in common, and fail if so */
2406 if (swizzle_to_access_mask(m->swizzle) & source_mask)
2407 return false;
2408 } else
2409 return false;
2410
2411 }
2412
2413 if (second->ssa_args.src1 == source)
2414 return false;
2415
2416 /* Otherwise, it's safe in that regard. Another data hazard is both
2417 * writing to the same place, of course */
2418
2419 if (second->ssa_args.dest == source) {
2420 /* ...but only if the components overlap */
2421 int dest_mask = second->type == TAG_ALU_4 ? squeeze_writemask(second->alu.mask) : 0xF;
2422
2423 if (dest_mask & source_mask)
2424 return false;
2425 }
2426
2427 /* ...That's it */
2428 return true;
2429 }
2430
2431 static bool
2432 midgard_has_hazard(
2433 midgard_instruction **segment, unsigned segment_size,
2434 midgard_instruction *ains)
2435 {
2436 for (int s = 0; s < segment_size; ++s)
2437 if (!can_run_concurrent_ssa(segment[s], ains))
2438 return true;
2439
2440 return false;
2441
2442
2443 }
2444
2445 /* Schedules, but does not emit, a single basic block. After scheduling, the
2446 * final tag and size of the block are known, which are necessary for branching
2447 * */
2448
2449 static midgard_bundle
2450 schedule_bundle(compiler_context *ctx, midgard_block *block, midgard_instruction *ins, int *skip)
2451 {
2452 int instructions_emitted = 0, instructions_consumed = -1;
2453 midgard_bundle bundle = { 0 };
2454
2455 uint8_t tag = ins->type;
2456
2457 /* Default to the instruction's tag */
2458 bundle.tag = tag;
2459
2460 switch (ins->type) {
2461 case TAG_ALU_4: {
2462 uint32_t control = 0;
2463 size_t bytes_emitted = sizeof(control);
2464
2465 /* TODO: Constant combining */
2466 int index = 0, last_unit = 0;
2467
2468 /* Previous instructions, for the purpose of parallelism */
2469 midgard_instruction *segment[4] = {0};
2470 int segment_size = 0;
2471
2472 instructions_emitted = -1;
2473 midgard_instruction *pins = ins;
2474
2475 for (;;) {
2476 midgard_instruction *ains = pins;
2477
2478 /* Advance instruction pointer */
2479 if (index) {
2480 ains = mir_next_op(pins);
2481 pins = ains;
2482 }
2483
2484 /* Out-of-work condition */
2485 if ((struct list_head *) ains == &block->instructions)
2486 break;
2487
2488 /* Ensure that the chain can continue */
2489 if (ains->type != TAG_ALU_4) break;
2490
2491 /* According to the presentation "The ARM
2492 * Mali-T880 Mobile GPU" from HotChips 27,
2493 * there are two pipeline stages. Branching
2494 * position determined experimentally. Lines
2495 * are executed in parallel:
2496 *
2497 * [ VMUL ] [ SADD ]
2498 * [ VADD ] [ SMUL ] [ LUT ] [ BRANCH ]
2499 *
2500 * Verify that there are no ordering dependencies here.
2501 *
2502 * TODO: Allow for parallelism!!!
2503 */
2504
2505 /* Pick a unit for it if it doesn't force a particular unit */
2506
2507 int unit = ains->unit;
2508
2509 if (!unit) {
2510 int op = ains->alu.op;
2511 int units = alu_opcode_props[op].props;
2512
2513 /* TODO: Promotion of scalars to vectors */
2514 int vector = ((!is_single_component_mask(ains->alu.mask)) || ((units & UNITS_SCALAR) == 0)) && (units & UNITS_ANY_VECTOR);
2515
2516 if (!vector)
2517 assert(units & UNITS_SCALAR);
2518
2519 if (vector) {
2520 if (last_unit >= UNIT_VADD) {
2521 if (units & UNIT_VLUT)
2522 unit = UNIT_VLUT;
2523 else
2524 break;
2525 } else {
2526 if ((units & UNIT_VMUL) && !(control & UNIT_VMUL))
2527 unit = UNIT_VMUL;
2528 else if ((units & UNIT_VADD) && !(control & UNIT_VADD))
2529 unit = UNIT_VADD;
2530 else if (units & UNIT_VLUT)
2531 unit = UNIT_VLUT;
2532 else
2533 break;
2534 }
2535 } else {
2536 if (last_unit >= UNIT_VADD) {
2537 if ((units & UNIT_SMUL) && !(control & UNIT_SMUL))
2538 unit = UNIT_SMUL;
2539 else if (units & UNIT_VLUT)
2540 unit = UNIT_VLUT;
2541 else
2542 break;
2543 } else {
2544 if ((units & UNIT_SADD) && !(control & UNIT_SADD) && !midgard_has_hazard(segment, segment_size, ains))
2545 unit = UNIT_SADD;
2546 else if (units & UNIT_SMUL)
2547 unit = ((units & UNIT_VMUL) && !(control & UNIT_VMUL)) ? UNIT_VMUL : UNIT_SMUL;
2548 else if ((units & UNIT_VADD) && !(control & UNIT_VADD))
2549 unit = UNIT_VADD;
2550 else
2551 break;
2552 }
2553 }
2554
2555 assert(unit & units);
2556 }
2557
2558 /* Late unit check, this time for encoding (not parallelism) */
2559 if (unit <= last_unit) break;
2560
2561 /* Clear the segment */
2562 if (last_unit < UNIT_VADD && unit >= UNIT_VADD)
2563 segment_size = 0;
2564
2565 if (midgard_has_hazard(segment, segment_size, ains))
2566 break;
2567
2568 /* We're good to go -- emit the instruction */
2569 ains->unit = unit;
2570
2571 segment[segment_size++] = ains;
2572
2573 /* Only one set of embedded constants per
2574 * bundle possible; if we have more, we must
2575 * break the chain early, unfortunately */
2576
2577 if (ains->has_constants) {
2578 if (bundle.has_embedded_constants) {
2579 /* ...but if there are already
2580 * constants but these are the
2581 * *same* constants, we let it
2582 * through */
2583
2584 if (memcmp(bundle.constants, ains->constants, sizeof(bundle.constants)))
2585 break;
2586 } else {
2587 bundle.has_embedded_constants = true;
2588 memcpy(bundle.constants, ains->constants, sizeof(bundle.constants));
2589
2590 /* If this is a blend shader special constant, track it for patching */
2591 if (ains->has_blend_constant)
2592 bundle.has_blend_constant = true;
2593 }
2594 }
2595
2596 if (ains->unit & UNITS_ANY_VECTOR) {
2597 emit_binary_vector_instruction(ains, bundle.register_words,
2598 &bundle.register_words_count, bundle.body_words,
2599 bundle.body_size, &bundle.body_words_count, &bytes_emitted);
2600 } else if (ains->compact_branch) {
2601 /* All of r0 has to be written out
2602 * along with the branch writeout.
2603 * (slow!) */
2604
2605 if (ains->writeout) {
2606 if (index == 0) {
2607 midgard_instruction ins = v_fmov(0, blank_alu_src, SSA_FIXED_REGISTER(0));
2608 ins.unit = UNIT_VMUL;
2609
2610 control |= ins.unit;
2611
2612 emit_binary_vector_instruction(&ins, bundle.register_words,
2613 &bundle.register_words_count, bundle.body_words,
2614 bundle.body_size, &bundle.body_words_count, &bytes_emitted);
2615 } else {
2616 /* Analyse the group to see if r0 is written in full, on-time, without hanging dependencies*/
2617 bool written_late = false;
2618 bool components[4] = { 0 };
2619 uint16_t register_dep_mask = 0;
2620 uint16_t written_mask = 0;
2621
2622 midgard_instruction *qins = ins;
2623 for (int t = 0; t < index; ++t) {
2624 if (qins->registers.out_reg != 0) {
2625 /* Mark down writes */
2626
2627 written_mask |= (1 << qins->registers.out_reg);
2628 } else {
2629 /* Mark down the register dependencies for errata check */
2630
2631 if (qins->registers.src1_reg < 16)
2632 register_dep_mask |= (1 << qins->registers.src1_reg);
2633
2634 if (qins->registers.src2_reg < 16)
2635 register_dep_mask |= (1 << qins->registers.src2_reg);
2636
2637 int mask = qins->alu.mask;
2638
2639 for (int c = 0; c < 4; ++c)
2640 if (mask & (0x3 << (2 * c)))
2641 components[c] = true;
2642
2643 /* ..but if the writeout is too late, we have to break up anyway... for some reason */
2644
2645 if (qins->unit == UNIT_VLUT)
2646 written_late = true;
2647 }
2648
2649 /* Advance instruction pointer */
2650 qins = mir_next_op(qins);
2651 }
2652
2653
2654 /* ERRATA (?): In a bundle ending in a fragment writeout, the register dependencies of r0 cannot be written within this bundle (discovered in -bshading:shading=phong) */
2655 if (register_dep_mask & written_mask) {
2656 DBG("ERRATA WORKAROUND: Breakup for writeout dependency masks %X vs %X (common %X)\n", register_dep_mask, written_mask, register_dep_mask & written_mask);
2657 break;
2658 }
2659
2660 if (written_late)
2661 break;
2662
2663 /* If even a single component is not written, break it up (conservative check). */
2664 bool breakup = false;
2665
2666 for (int c = 0; c < 4; ++c)
2667 if (!components[c])
2668 breakup = true;
2669
2670 if (breakup)
2671 break;
2672
2673 /* Otherwise, we're free to proceed */
2674 }
2675 }
2676
2677 if (ains->unit == ALU_ENAB_BRANCH) {
2678 bundle.body_size[bundle.body_words_count] = sizeof(midgard_branch_extended);
2679 memcpy(&bundle.body_words[bundle.body_words_count++], &ains->branch_extended, sizeof(midgard_branch_extended));
2680 bytes_emitted += sizeof(midgard_branch_extended);
2681 } else {
2682 bundle.body_size[bundle.body_words_count] = sizeof(ains->br_compact);
2683 memcpy(&bundle.body_words[bundle.body_words_count++], &ains->br_compact, sizeof(ains->br_compact));
2684 bytes_emitted += sizeof(ains->br_compact);
2685 }
2686 } else {
2687 memcpy(&bundle.register_words[bundle.register_words_count++], &ains->registers, sizeof(ains->registers));
2688 bytes_emitted += sizeof(midgard_reg_info);
2689
2690 bundle.body_size[bundle.body_words_count] = sizeof(midgard_scalar_alu);
2691 bundle.body_words_count++;
2692 bytes_emitted += sizeof(midgard_scalar_alu);
2693 }
2694
2695 /* Defer marking until after writing to allow for break */
2696 control |= ains->unit;
2697 last_unit = ains->unit;
2698 ++instructions_emitted;
2699 ++index;
2700 }
2701
2702 /* Bubble up the number of instructions for skipping */
2703 instructions_consumed = index - 1;
2704
2705 int padding = 0;
2706
2707 /* Pad ALU op to nearest word */
2708
2709 if (bytes_emitted & 15) {
2710 padding = 16 - (bytes_emitted & 15);
2711 bytes_emitted += padding;
2712 }
2713
2714 /* Constants must always be quadwords */
2715 if (bundle.has_embedded_constants)
2716 bytes_emitted += 16;
2717
2718 /* Size ALU instruction for tag */
2719 bundle.tag = (TAG_ALU_4) + (bytes_emitted / 16) - 1;
2720 bundle.padding = padding;
2721 bundle.control = bundle.tag | control;
2722
2723 break;
2724 }
2725
2726 case TAG_LOAD_STORE_4: {
2727 /* Load store instructions have two words at once. If
2728 * we only have one queued up, we need to NOP pad.
2729 * Otherwise, we store both in succession to save space
2730 * and cycles -- letting them go in parallel -- skip
2731 * the next. The usefulness of this optimisation is
2732 * greatly dependent on the quality of the instruction
2733 * scheduler.
2734 */
2735
2736 midgard_instruction *next_op = mir_next_op(ins);
2737
2738 if ((struct list_head *) next_op != &block->instructions && next_op->type == TAG_LOAD_STORE_4) {
2739 /* As the two operate concurrently, make sure
2740 * they are not dependent */
2741
2742 if (can_run_concurrent_ssa(ins, next_op) || true) {
2743 /* Skip ahead, since it's redundant with the pair */
2744 instructions_consumed = 1 + (instructions_emitted++);
2745 }
2746 }
2747
2748 break;
2749 }
2750
2751 default:
2752 /* Texture ops default to single-op-per-bundle scheduling */
2753 break;
2754 }
2755
2756 /* Copy the instructions into the bundle */
2757 bundle.instruction_count = instructions_emitted + 1;
2758
2759 int used_idx = 0;
2760
2761 midgard_instruction *uins = ins;
2762 for (int i = 0; used_idx < bundle.instruction_count; ++i) {
2763 bundle.instructions[used_idx++] = *uins;
2764 uins = mir_next_op(uins);
2765 }
2766
2767 *skip = (instructions_consumed == -1) ? instructions_emitted : instructions_consumed;
2768
2769 return bundle;
2770 }
2771
2772 static int
2773 quadword_size(int tag)
2774 {
2775 switch (tag) {
2776 case TAG_ALU_4:
2777 return 1;
2778
2779 case TAG_ALU_8:
2780 return 2;
2781
2782 case TAG_ALU_12:
2783 return 3;
2784
2785 case TAG_ALU_16:
2786 return 4;
2787
2788 case TAG_LOAD_STORE_4:
2789 return 1;
2790
2791 case TAG_TEXTURE_4:
2792 return 1;
2793
2794 default:
2795 assert(0);
2796 return 0;
2797 }
2798 }
2799
2800 /* Schedule a single block by iterating its instruction to create bundles.
2801 * While we go, tally about the bundle sizes to compute the block size. */
2802
2803 static void
2804 schedule_block(compiler_context *ctx, midgard_block *block)
2805 {
2806 util_dynarray_init(&block->bundles, NULL);
2807
2808 block->quadword_count = 0;
2809
2810 mir_foreach_instr_in_block(block, ins) {
2811 int skip;
2812 midgard_bundle bundle = schedule_bundle(ctx, block, ins, &skip);
2813 util_dynarray_append(&block->bundles, midgard_bundle, bundle);
2814
2815 if (bundle.has_blend_constant) {
2816 /* TODO: Multiblock? */
2817 int quadwords_within_block = block->quadword_count + quadword_size(bundle.tag) - 1;
2818 ctx->blend_constant_offset = quadwords_within_block * 0x10;
2819 }
2820
2821 while(skip--)
2822 ins = mir_next_op(ins);
2823
2824 block->quadword_count += quadword_size(bundle.tag);
2825 }
2826
2827 block->is_scheduled = true;
2828 }
2829
2830 static void
2831 schedule_program(compiler_context *ctx)
2832 {
2833 /* We run RA prior to scheduling */
2834 struct ra_graph *g = allocate_registers(ctx);
2835 install_registers(ctx, g);
2836
2837 mir_foreach_block(ctx, block) {
2838 schedule_block(ctx, block);
2839 }
2840 }
2841
2842 /* After everything is scheduled, emit whole bundles at a time */
2843
2844 static void
2845 emit_binary_bundle(compiler_context *ctx, midgard_bundle *bundle, struct util_dynarray *emission, int next_tag)
2846 {
2847 int lookahead = next_tag << 4;
2848
2849 switch (bundle->tag) {
2850 case TAG_ALU_4:
2851 case TAG_ALU_8:
2852 case TAG_ALU_12:
2853 case TAG_ALU_16: {
2854 /* Actually emit each component */
2855 util_dynarray_append(emission, uint32_t, bundle->control | lookahead);
2856
2857 for (int i = 0; i < bundle->register_words_count; ++i)
2858 util_dynarray_append(emission, uint16_t, bundle->register_words[i]);
2859
2860 /* Emit body words based on the instructions bundled */
2861 for (int i = 0; i < bundle->instruction_count; ++i) {
2862 midgard_instruction *ins = &bundle->instructions[i];
2863
2864 if (ins->unit & UNITS_ANY_VECTOR) {
2865 memcpy(util_dynarray_grow(emission, sizeof(midgard_vector_alu)), &ins->alu, sizeof(midgard_vector_alu));
2866 } else if (ins->compact_branch) {
2867 /* Dummy move, XXX DRY */
2868 if ((i == 0) && ins->writeout) {
2869 midgard_instruction ins = v_fmov(0, blank_alu_src, SSA_FIXED_REGISTER(0));
2870 memcpy(util_dynarray_grow(emission, sizeof(midgard_vector_alu)), &ins.alu, sizeof(midgard_vector_alu));
2871 }
2872
2873 if (ins->unit == ALU_ENAB_BR_COMPACT) {
2874 memcpy(util_dynarray_grow(emission, sizeof(ins->br_compact)), &ins->br_compact, sizeof(ins->br_compact));
2875 } else {
2876 memcpy(util_dynarray_grow(emission, sizeof(ins->branch_extended)), &ins->branch_extended, sizeof(ins->branch_extended));
2877 }
2878 } else {
2879 /* Scalar */
2880 midgard_scalar_alu scalarised = vector_to_scalar_alu(ins->alu, ins);
2881 memcpy(util_dynarray_grow(emission, sizeof(scalarised)), &scalarised, sizeof(scalarised));
2882 }
2883 }
2884
2885 /* Emit padding (all zero) */
2886 memset(util_dynarray_grow(emission, bundle->padding), 0, bundle->padding);
2887
2888 /* Tack on constants */
2889
2890 if (bundle->has_embedded_constants) {
2891 util_dynarray_append(emission, float, bundle->constants[0]);
2892 util_dynarray_append(emission, float, bundle->constants[1]);
2893 util_dynarray_append(emission, float, bundle->constants[2]);
2894 util_dynarray_append(emission, float, bundle->constants[3]);
2895 }
2896
2897 break;
2898 }
2899
2900 case TAG_LOAD_STORE_4: {
2901 /* One or two composing instructions */
2902
2903 uint64_t current64, next64 = LDST_NOP;
2904
2905 memcpy(&current64, &bundle->instructions[0].load_store, sizeof(current64));
2906
2907 if (bundle->instruction_count == 2)
2908 memcpy(&next64, &bundle->instructions[1].load_store, sizeof(next64));
2909
2910 midgard_load_store instruction = {
2911 .type = bundle->tag,
2912 .next_type = next_tag,
2913 .word1 = current64,
2914 .word2 = next64
2915 };
2916
2917 util_dynarray_append(emission, midgard_load_store, instruction);
2918
2919 break;
2920 }
2921
2922 case TAG_TEXTURE_4: {
2923 /* Texture instructions are easy, since there is no
2924 * pipelining nor VLIW to worry about. We may need to set the .last flag */
2925
2926 midgard_instruction *ins = &bundle->instructions[0];
2927
2928 ins->texture.type = TAG_TEXTURE_4;
2929 ins->texture.next_type = next_tag;
2930
2931 ctx->texture_op_count--;
2932
2933 if (!ctx->texture_op_count) {
2934 ins->texture.cont = 0;
2935 ins->texture.last = 1;
2936 }
2937
2938 util_dynarray_append(emission, midgard_texture_word, ins->texture);
2939 break;
2940 }
2941
2942 default:
2943 DBG("Unknown midgard instruction type\n");
2944 assert(0);
2945 break;
2946 }
2947 }
2948
2949
2950 /* ALU instructions can inline or embed constants, which decreases register
2951 * pressure and saves space. */
2952
2953 #define CONDITIONAL_ATTACH(src) { \
2954 void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->ssa_args.src + 1); \
2955 \
2956 if (entry) { \
2957 attach_constants(ctx, alu, entry, alu->ssa_args.src + 1); \
2958 alu->ssa_args.src = SSA_FIXED_REGISTER(REGISTER_CONSTANT); \
2959 } \
2960 }
2961
2962 static void
2963 inline_alu_constants(compiler_context *ctx)
2964 {
2965 mir_foreach_instr(ctx, alu) {
2966 /* Other instructions cannot inline constants */
2967 if (alu->type != TAG_ALU_4) continue;
2968
2969 /* If there is already a constant here, we can do nothing */
2970 if (alu->has_constants) continue;
2971
2972 /* It makes no sense to inline constants on a branch */
2973 if (alu->compact_branch || alu->prepacked_branch) continue;
2974
2975 CONDITIONAL_ATTACH(src0);
2976
2977 if (!alu->has_constants) {
2978 CONDITIONAL_ATTACH(src1)
2979 } else if (!alu->inline_constant) {
2980 /* Corner case: _two_ vec4 constants, for instance with a
2981 * csel. For this case, we can only use a constant
2982 * register for one, we'll have to emit a move for the
2983 * other. Note, if both arguments are constants, then
2984 * necessarily neither argument depends on the value of
2985 * any particular register. As the destination register
2986 * will be wiped, that means we can spill the constant
2987 * to the destination register.
2988 */
2989
2990 void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->ssa_args.src1 + 1);
2991 unsigned scratch = alu->ssa_args.dest;
2992
2993 if (entry) {
2994 midgard_instruction ins = v_fmov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), blank_alu_src, scratch);
2995 attach_constants(ctx, &ins, entry, alu->ssa_args.src1 + 1);
2996
2997 /* Force a break XXX Defer r31 writes */
2998 ins.unit = UNIT_VLUT;
2999
3000 /* Set the source */
3001 alu->ssa_args.src1 = scratch;
3002
3003 /* Inject us -before- the last instruction which set r31 */
3004 mir_insert_instruction_before(mir_prev_op(alu), ins);
3005 }
3006 }
3007 }
3008 }
3009
3010 /* Midgard supports two types of constants, embedded constants (128-bit) and
3011 * inline constants (16-bit). Sometimes, especially with scalar ops, embedded
3012 * constants can be demoted to inline constants, for space savings and
3013 * sometimes a performance boost */
3014
3015 static void
3016 embedded_to_inline_constant(compiler_context *ctx)
3017 {
3018 mir_foreach_instr(ctx, ins) {
3019 if (!ins->has_constants) continue;
3020
3021 if (ins->ssa_args.inline_constant) continue;
3022
3023 /* Blend constants must not be inlined by definition */
3024 if (ins->has_blend_constant) continue;
3025
3026 /* src1 cannot be an inline constant due to encoding
3027 * restrictions. So, if possible we try to flip the arguments
3028 * in that case */
3029
3030 int op = ins->alu.op;
3031
3032 if (ins->ssa_args.src0 == SSA_FIXED_REGISTER(REGISTER_CONSTANT)) {
3033 switch (op) {
3034 /* These ops require an operational change to flip
3035 * their arguments TODO */
3036 case midgard_alu_op_flt:
3037 case midgard_alu_op_fle:
3038 case midgard_alu_op_ilt:
3039 case midgard_alu_op_ile:
3040 case midgard_alu_op_fcsel:
3041 case midgard_alu_op_icsel:
3042 DBG("Missed non-commutative flip (%s)\n", alu_opcode_props[op].name);
3043 default:
3044 break;
3045 }
3046
3047 if (alu_opcode_props[op].props & OP_COMMUTES) {
3048 /* Flip the SSA numbers */
3049 ins->ssa_args.src0 = ins->ssa_args.src1;
3050 ins->ssa_args.src1 = SSA_FIXED_REGISTER(REGISTER_CONSTANT);
3051
3052 /* And flip the modifiers */
3053
3054 unsigned src_temp;
3055
3056 src_temp = ins->alu.src2;
3057 ins->alu.src2 = ins->alu.src1;
3058 ins->alu.src1 = src_temp;
3059 }
3060 }
3061
3062 if (ins->ssa_args.src1 == SSA_FIXED_REGISTER(REGISTER_CONSTANT)) {
3063 /* Extract the source information */
3064
3065 midgard_vector_alu_src *src;
3066 int q = ins->alu.src2;
3067 midgard_vector_alu_src *m = (midgard_vector_alu_src *) &q;
3068 src = m;
3069
3070 /* Component is from the swizzle, e.g. r26.w -> w component. TODO: What if x is masked out? */
3071 int component = src->swizzle & 3;
3072
3073 /* Scale constant appropriately, if we can legally */
3074 uint16_t scaled_constant = 0;
3075
3076 /* XXX: Check legality */
3077 if (midgard_is_integer_op(op)) {
3078 /* TODO: Inline integer */
3079 continue;
3080
3081 unsigned int *iconstants = (unsigned int *) ins->constants;
3082 scaled_constant = (uint16_t) iconstants[component];
3083
3084 /* Constant overflow after resize */
3085 if (scaled_constant != iconstants[component])
3086 continue;
3087 } else {
3088 float original = (float) ins->constants[component];
3089 scaled_constant = _mesa_float_to_half(original);
3090
3091 /* Check for loss of precision. If this is
3092 * mediump, we don't care, but for a highp
3093 * shader, we need to pay attention. NIR
3094 * doesn't yet tell us which mode we're in!
3095 * Practically this prevents most constants
3096 * from being inlined, sadly. */
3097
3098 float fp32 = _mesa_half_to_float(scaled_constant);
3099
3100 if (fp32 != original)
3101 continue;
3102 }
3103
3104 /* We don't know how to handle these with a constant */
3105
3106 if (src->mod || src->half || src->rep_low || src->rep_high) {
3107 DBG("Bailing inline constant...\n");
3108 continue;
3109 }
3110
3111 /* Make sure that the constant is not itself a
3112 * vector by checking if all accessed values
3113 * (by the swizzle) are the same. */
3114
3115 uint32_t *cons = (uint32_t *) ins->constants;
3116 uint32_t value = cons[component];
3117
3118 bool is_vector = false;
3119 unsigned mask = effective_writemask(&ins->alu);
3120
3121 for (int c = 1; c < 4; ++c) {
3122 /* We only care if this component is actually used */
3123 if (!(mask & (1 << c)))
3124 continue;
3125
3126 uint32_t test = cons[(src->swizzle >> (2 * c)) & 3];
3127
3128 if (test != value) {
3129 is_vector = true;
3130 break;
3131 }
3132 }
3133
3134 if (is_vector)
3135 continue;
3136
3137 /* Get rid of the embedded constant */
3138 ins->has_constants = false;
3139 ins->ssa_args.src1 = SSA_UNUSED_0;
3140 ins->ssa_args.inline_constant = true;
3141 ins->inline_constant = scaled_constant;
3142 }
3143 }
3144 }
3145
3146 /* Map normal SSA sources to other SSA sources / fixed registers (like
3147 * uniforms) */
3148
3149 static void
3150 map_ssa_to_alias(compiler_context *ctx, int *ref)
3151 {
3152 unsigned int alias = (uintptr_t) _mesa_hash_table_u64_search(ctx->ssa_to_alias, *ref + 1);
3153
3154 if (alias) {
3155 /* Remove entry in leftovers to avoid a redunant fmov */
3156
3157 struct set_entry *leftover = _mesa_set_search(ctx->leftover_ssa_to_alias, ((void *) (uintptr_t) (*ref + 1)));
3158
3159 if (leftover)
3160 _mesa_set_remove(ctx->leftover_ssa_to_alias, leftover);
3161
3162 /* Assign the alias map */
3163 *ref = alias - 1;
3164 return;
3165 }
3166 }
3167
3168 /* Basic dead code elimination on the MIR itself, which cleans up e.g. the
3169 * texture pipeline */
3170
3171 static bool
3172 midgard_opt_dead_code_eliminate(compiler_context *ctx, midgard_block *block)
3173 {
3174 bool progress = false;
3175
3176 mir_foreach_instr_in_block_safe(block, ins) {
3177 if (ins->type != TAG_ALU_4) continue;
3178 if (ins->compact_branch) continue;
3179
3180 if (ins->ssa_args.dest >= SSA_FIXED_MINIMUM) continue;
3181 if (midgard_is_pinned(ctx, ins->ssa_args.dest)) continue;
3182 if (is_live_after(ctx, block, ins, ins->ssa_args.dest)) continue;
3183
3184 mir_remove_instruction(ins);
3185 progress = true;
3186 }
3187
3188 return progress;
3189 }
3190
3191 static bool
3192 midgard_opt_copy_prop(compiler_context *ctx, midgard_block *block)
3193 {
3194 bool progress = false;
3195
3196 mir_foreach_instr_in_block_safe(block, ins) {
3197 if (ins->type != TAG_ALU_4) continue;
3198 if (!OP_IS_MOVE(ins->alu.op)) continue;
3199
3200 unsigned from = ins->ssa_args.src1;
3201 unsigned to = ins->ssa_args.dest;
3202
3203 /* We only work on pure SSA */
3204
3205 if (to >= SSA_FIXED_MINIMUM) continue;
3206 if (from >= SSA_FIXED_MINIMUM) continue;
3207 if (to >= ctx->func->impl->ssa_alloc) continue;
3208 if (from >= ctx->func->impl->ssa_alloc) continue;
3209
3210 /* Also, if the move has side effects, we're helpless */
3211
3212 midgard_vector_alu_src src =
3213 vector_alu_from_unsigned(ins->alu.src2);
3214 unsigned mask = squeeze_writemask(ins->alu.mask);
3215 bool is_int = midgard_is_integer_op(ins->alu.op);
3216
3217 if (mir_nontrivial_mod(src, is_int, mask)) continue;
3218 if (ins->alu.outmod != midgard_outmod_none) continue;
3219
3220 mir_foreach_instr_in_block_from(block, v, mir_next_op(ins)) {
3221 if (v->ssa_args.src0 == to) {
3222 v->ssa_args.src0 = from;
3223 progress = true;
3224 }
3225
3226 if (v->ssa_args.src1 == to && !v->ssa_args.inline_constant) {
3227 v->ssa_args.src1 = from;
3228 progress = true;
3229 }
3230 }
3231 }
3232
3233 return progress;
3234 }
3235
3236 static bool
3237 midgard_opt_copy_prop_tex(compiler_context *ctx, midgard_block *block)
3238 {
3239 bool progress = false;
3240
3241 mir_foreach_instr_in_block_safe(block, ins) {
3242 if (ins->type != TAG_ALU_4) continue;
3243 if (!OP_IS_MOVE(ins->alu.op)) continue;
3244
3245 unsigned from = ins->ssa_args.src1;
3246 unsigned to = ins->ssa_args.dest;
3247
3248 /* Make sure it's simple enough for us to handle */
3249
3250 if (from >= SSA_FIXED_MINIMUM) continue;
3251 if (from >= ctx->func->impl->ssa_alloc) continue;
3252 if (to < SSA_FIXED_REGISTER(REGISTER_TEXTURE_BASE)) continue;
3253 if (to > SSA_FIXED_REGISTER(REGISTER_TEXTURE_BASE + 1)) continue;
3254
3255 bool eliminated = false;
3256
3257 mir_foreach_instr_in_block_from_rev(block, v, mir_prev_op(ins)) {
3258 /* The texture registers are not SSA so be careful.
3259 * Conservatively, just stop if we hit a texture op
3260 * (even if it may not write) to where we are */
3261
3262 if (v->type != TAG_ALU_4)
3263 break;
3264
3265 if (v->ssa_args.dest == from) {
3266 /* We don't want to track partial writes ... */
3267 if (v->alu.mask == 0xF) {
3268 v->ssa_args.dest = to;
3269 eliminated = true;
3270 }
3271
3272 break;
3273 }
3274 }
3275
3276 if (eliminated)
3277 mir_remove_instruction(ins);
3278
3279 progress |= eliminated;
3280 }
3281
3282 return progress;
3283 }
3284
3285 /* We don't really understand the imov/fmov split, so always use fmov (but let
3286 * it be imov in the IR so we don't do unsafe floating point "optimizations"
3287 * and break things */
3288
3289 static void
3290 midgard_imov_workaround(compiler_context *ctx, midgard_block *block)
3291 {
3292 mir_foreach_instr_in_block_safe(block, ins) {
3293 if (ins->type != TAG_ALU_4) continue;
3294 if (ins->alu.op != midgard_alu_op_imov) continue;
3295
3296 ins->alu.op = midgard_alu_op_fmov;
3297 ins->alu.outmod = midgard_outmod_none;
3298
3299 /* Remove flags that don't make sense */
3300
3301 midgard_vector_alu_src s =
3302 vector_alu_from_unsigned(ins->alu.src2);
3303
3304 s.mod = 0;
3305
3306 ins->alu.src2 = vector_alu_srco_unsigned(s);
3307 }
3308 }
3309
3310 /* The following passes reorder MIR instructions to enable better scheduling */
3311
3312 static void
3313 midgard_pair_load_store(compiler_context *ctx, midgard_block *block)
3314 {
3315 mir_foreach_instr_in_block_safe(block, ins) {
3316 if (ins->type != TAG_LOAD_STORE_4) continue;
3317
3318 /* We've found a load/store op. Check if next is also load/store. */
3319 midgard_instruction *next_op = mir_next_op(ins);
3320 if (&next_op->link != &block->instructions) {
3321 if (next_op->type == TAG_LOAD_STORE_4) {
3322 /* If so, we're done since we're a pair */
3323 ins = mir_next_op(ins);
3324 continue;
3325 }
3326
3327 /* Maximum search distance to pair, to avoid register pressure disasters */
3328 int search_distance = 8;
3329
3330 /* Otherwise, we have an orphaned load/store -- search for another load */
3331 mir_foreach_instr_in_block_from(block, c, mir_next_op(ins)) {
3332 /* Terminate search if necessary */
3333 if (!(search_distance--)) break;
3334
3335 if (c->type != TAG_LOAD_STORE_4) continue;
3336
3337 /* Stores cannot be reordered, since they have
3338 * dependencies. For the same reason, indirect
3339 * loads cannot be reordered as their index is
3340 * loaded in r27.w */
3341
3342 if (OP_IS_STORE(c->load_store.op)) continue;
3343
3344 /* It appears the 0x800 bit is set whenever a
3345 * load is direct, unset when it is indirect.
3346 * Skip indirect loads. */
3347
3348 if (!(c->load_store.unknown & 0x800)) continue;
3349
3350 /* We found one! Move it up to pair and remove it from the old location */
3351
3352 mir_insert_instruction_before(ins, *c);
3353 mir_remove_instruction(c);
3354
3355 break;
3356 }
3357 }
3358 }
3359 }
3360
3361 /* Emit varying stores late */
3362
3363 static void
3364 midgard_emit_store(compiler_context *ctx, midgard_block *block) {
3365 /* Iterate in reverse to get the final write, rather than the first */
3366
3367 mir_foreach_instr_in_block_safe_rev(block, ins) {
3368 /* Check if what we just wrote needs a store */
3369 int idx = ins->ssa_args.dest;
3370 uintptr_t varying = ((uintptr_t) _mesa_hash_table_u64_search(ctx->ssa_varyings, idx + 1));
3371
3372 if (!varying) continue;
3373
3374 varying -= 1;
3375
3376 /* We need to store to the appropriate varying, so emit the
3377 * move/store */
3378
3379 /* TODO: Integrate with special purpose RA (and scheduler?) */
3380 bool high_varying_register = false;
3381
3382 midgard_instruction mov = v_fmov(idx, blank_alu_src, SSA_FIXED_REGISTER(REGISTER_VARYING_BASE + high_varying_register));
3383
3384 midgard_instruction st = m_store_vary_32(SSA_FIXED_REGISTER(high_varying_register), varying);
3385 st.load_store.unknown = 0x1E9E; /* XXX: What is this? */
3386
3387 mir_insert_instruction_before(mir_next_op(ins), st);
3388 mir_insert_instruction_before(mir_next_op(ins), mov);
3389
3390 /* We no longer need to store this varying */
3391 _mesa_hash_table_u64_remove(ctx->ssa_varyings, idx + 1);
3392 }
3393 }
3394
3395 /* If there are leftovers after the below pass, emit actual fmov
3396 * instructions for the slow-but-correct path */
3397
3398 static void
3399 emit_leftover_move(compiler_context *ctx)
3400 {
3401 set_foreach(ctx->leftover_ssa_to_alias, leftover) {
3402 int base = ((uintptr_t) leftover->key) - 1;
3403 int mapped = base;
3404
3405 map_ssa_to_alias(ctx, &mapped);
3406 EMIT(fmov, mapped, blank_alu_src, base);
3407 }
3408 }
3409
3410 static void
3411 actualise_ssa_to_alias(compiler_context *ctx)
3412 {
3413 mir_foreach_instr(ctx, ins) {
3414 map_ssa_to_alias(ctx, &ins->ssa_args.src0);
3415 map_ssa_to_alias(ctx, &ins->ssa_args.src1);
3416 }
3417
3418 emit_leftover_move(ctx);
3419 }
3420
3421 static void
3422 emit_fragment_epilogue(compiler_context *ctx)
3423 {
3424 /* Special case: writing out constants requires us to include the move
3425 * explicitly now, so shove it into r0 */
3426
3427 void *constant_value = _mesa_hash_table_u64_search(ctx->ssa_constants, ctx->fragment_output + 1);
3428
3429 if (constant_value) {
3430 midgard_instruction ins = v_fmov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), blank_alu_src, SSA_FIXED_REGISTER(0));
3431 attach_constants(ctx, &ins, constant_value, ctx->fragment_output + 1);
3432 emit_mir_instruction(ctx, ins);
3433 }
3434
3435 /* Perform the actual fragment writeout. We have two writeout/branch
3436 * instructions, forming a loop until writeout is successful as per the
3437 * docs. TODO: gl_FragDepth */
3438
3439 EMIT(alu_br_compact_cond, midgard_jmp_writeout_op_writeout, TAG_ALU_4, 0, midgard_condition_always);
3440 EMIT(alu_br_compact_cond, midgard_jmp_writeout_op_writeout, TAG_ALU_4, -1, midgard_condition_always);
3441 }
3442
3443 /* For the blend epilogue, we need to convert the blended fragment vec4 (stored
3444 * in r0) to a RGBA8888 value by scaling and type converting. We then output it
3445 * with the int8 analogue to the fragment epilogue */
3446
3447 static void
3448 emit_blend_epilogue(compiler_context *ctx)
3449 {
3450 /* vmul.fmul.none.fulllow hr48, r0, #255 */
3451
3452 midgard_instruction scale = {
3453 .type = TAG_ALU_4,
3454 .unit = UNIT_VMUL,
3455 .inline_constant = _mesa_float_to_half(255.0),
3456 .ssa_args = {
3457 .src0 = SSA_FIXED_REGISTER(0),
3458 .src1 = SSA_UNUSED_0,
3459 .dest = SSA_FIXED_REGISTER(24),
3460 .inline_constant = true
3461 },
3462 .alu = {
3463 .op = midgard_alu_op_fmul,
3464 .reg_mode = midgard_reg_mode_32,
3465 .dest_override = midgard_dest_override_lower,
3466 .mask = 0xFF,
3467 .src1 = vector_alu_srco_unsigned(blank_alu_src),
3468 .src2 = vector_alu_srco_unsigned(blank_alu_src),
3469 }
3470 };
3471
3472 emit_mir_instruction(ctx, scale);
3473
3474 /* vadd.f2u8.pos.low hr0, hr48, #0 */
3475
3476 midgard_vector_alu_src alu_src = blank_alu_src;
3477 alu_src.half = true;
3478
3479 midgard_instruction f2u8 = {
3480 .type = TAG_ALU_4,
3481 .ssa_args = {
3482 .src0 = SSA_FIXED_REGISTER(24),
3483 .src1 = SSA_UNUSED_0,
3484 .dest = SSA_FIXED_REGISTER(0),
3485 .inline_constant = true
3486 },
3487 .alu = {
3488 .op = midgard_alu_op_f2u8,
3489 .reg_mode = midgard_reg_mode_16,
3490 .dest_override = midgard_dest_override_lower,
3491 .outmod = midgard_outmod_pos,
3492 .mask = 0xF,
3493 .src1 = vector_alu_srco_unsigned(alu_src),
3494 .src2 = vector_alu_srco_unsigned(blank_alu_src),
3495 }
3496 };
3497
3498 emit_mir_instruction(ctx, f2u8);
3499
3500 /* vmul.imov.quarter r0, r0, r0 */
3501
3502 midgard_instruction imov_8 = {
3503 .type = TAG_ALU_4,
3504 .ssa_args = {
3505 .src0 = SSA_UNUSED_1,
3506 .src1 = SSA_FIXED_REGISTER(0),
3507 .dest = SSA_FIXED_REGISTER(0),
3508 },
3509 .alu = {
3510 .op = midgard_alu_op_imov,
3511 .reg_mode = midgard_reg_mode_8,
3512 .dest_override = midgard_dest_override_none,
3513 .mask = 0xFF,
3514 .src1 = vector_alu_srco_unsigned(blank_alu_src),
3515 .src2 = vector_alu_srco_unsigned(blank_alu_src),
3516 }
3517 };
3518
3519 /* Emit branch epilogue with the 8-bit move as the source */
3520
3521 emit_mir_instruction(ctx, imov_8);
3522 EMIT(alu_br_compact_cond, midgard_jmp_writeout_op_writeout, TAG_ALU_4, 0, midgard_condition_always);
3523
3524 emit_mir_instruction(ctx, imov_8);
3525 EMIT(alu_br_compact_cond, midgard_jmp_writeout_op_writeout, TAG_ALU_4, -1, midgard_condition_always);
3526 }
3527
3528 static midgard_block *
3529 emit_block(compiler_context *ctx, nir_block *block)
3530 {
3531 midgard_block *this_block = calloc(sizeof(midgard_block), 1);
3532 list_addtail(&this_block->link, &ctx->blocks);
3533
3534 this_block->is_scheduled = false;
3535 ++ctx->block_count;
3536
3537 ctx->texture_index[0] = -1;
3538 ctx->texture_index[1] = -1;
3539
3540 /* Add us as a successor to the block we are following */
3541 if (ctx->current_block)
3542 midgard_block_add_successor(ctx->current_block, this_block);
3543
3544 /* Set up current block */
3545 list_inithead(&this_block->instructions);
3546 ctx->current_block = this_block;
3547
3548 nir_foreach_instr(instr, block) {
3549 emit_instr(ctx, instr);
3550 ++ctx->instruction_count;
3551 }
3552
3553 inline_alu_constants(ctx);
3554 embedded_to_inline_constant(ctx);
3555
3556 /* Perform heavylifting for aliasing */
3557 actualise_ssa_to_alias(ctx);
3558
3559 midgard_emit_store(ctx, this_block);
3560 midgard_pair_load_store(ctx, this_block);
3561 midgard_imov_workaround(ctx, this_block);
3562
3563 /* Append fragment shader epilogue (value writeout) */
3564 if (ctx->stage == MESA_SHADER_FRAGMENT) {
3565 if (block == nir_impl_last_block(ctx->func->impl)) {
3566 if (ctx->is_blend)
3567 emit_blend_epilogue(ctx);
3568 else
3569 emit_fragment_epilogue(ctx);
3570 }
3571 }
3572
3573 if (block == nir_start_block(ctx->func->impl))
3574 ctx->initial_block = this_block;
3575
3576 if (block == nir_impl_last_block(ctx->func->impl))
3577 ctx->final_block = this_block;
3578
3579 /* Allow the next control flow to access us retroactively, for
3580 * branching etc */
3581 ctx->current_block = this_block;
3582
3583 /* Document the fallthrough chain */
3584 ctx->previous_source_block = this_block;
3585
3586 return this_block;
3587 }
3588
3589 static midgard_block *emit_cf_list(struct compiler_context *ctx, struct exec_list *list);
3590
3591 static void
3592 emit_if(struct compiler_context *ctx, nir_if *nif)
3593 {
3594 /* Conditional branches expect the condition in r31.w; emit a move for
3595 * that in the _previous_ block (which is the current block). */
3596 emit_condition(ctx, &nif->condition, true, COMPONENT_X);
3597
3598 /* Speculatively emit the branch, but we can't fill it in until later */
3599 EMIT(branch, true, true);
3600 midgard_instruction *then_branch = mir_last_in_block(ctx->current_block);
3601
3602 /* Emit the two subblocks */
3603 midgard_block *then_block = emit_cf_list(ctx, &nif->then_list);
3604
3605 /* Emit a jump from the end of the then block to the end of the else */
3606 EMIT(branch, false, false);
3607 midgard_instruction *then_exit = mir_last_in_block(ctx->current_block);
3608
3609 /* Emit second block, and check if it's empty */
3610
3611 int else_idx = ctx->block_count;
3612 int count_in = ctx->instruction_count;
3613 midgard_block *else_block = emit_cf_list(ctx, &nif->else_list);
3614 int after_else_idx = ctx->block_count;
3615
3616 /* Now that we have the subblocks emitted, fix up the branches */
3617
3618 assert(then_block);
3619 assert(else_block);
3620
3621 if (ctx->instruction_count == count_in) {
3622 /* The else block is empty, so don't emit an exit jump */
3623 mir_remove_instruction(then_exit);
3624 then_branch->branch.target_block = after_else_idx;
3625 } else {
3626 then_branch->branch.target_block = else_idx;
3627 then_exit->branch.target_block = after_else_idx;
3628 }
3629 }
3630
3631 static void
3632 emit_loop(struct compiler_context *ctx, nir_loop *nloop)
3633 {
3634 /* Remember where we are */
3635 midgard_block *start_block = ctx->current_block;
3636
3637 /* Allocate a loop number, growing the current inner loop depth */
3638 int loop_idx = ++ctx->current_loop_depth;
3639
3640 /* Get index from before the body so we can loop back later */
3641 int start_idx = ctx->block_count;
3642
3643 /* Emit the body itself */
3644 emit_cf_list(ctx, &nloop->body);
3645
3646 /* Branch back to loop back */
3647 struct midgard_instruction br_back = v_branch(false, false);
3648 br_back.branch.target_block = start_idx;
3649 emit_mir_instruction(ctx, br_back);
3650
3651 /* Mark down that branch in the graph. Note that we're really branching
3652 * to the block *after* we started in. TODO: Why doesn't the branch
3653 * itself have an off-by-one then...? */
3654 midgard_block_add_successor(ctx->current_block, start_block->successors[0]);
3655
3656 /* Find the index of the block about to follow us (note: we don't add
3657 * one; blocks are 0-indexed so we get a fencepost problem) */
3658 int break_block_idx = ctx->block_count;
3659
3660 /* Fix up the break statements we emitted to point to the right place,
3661 * now that we can allocate a block number for them */
3662
3663 list_for_each_entry_from(struct midgard_block, block, start_block, &ctx->blocks, link) {
3664 mir_foreach_instr_in_block(block, ins) {
3665 if (ins->type != TAG_ALU_4) continue;
3666 if (!ins->compact_branch) continue;
3667 if (ins->prepacked_branch) continue;
3668
3669 /* We found a branch -- check the type to see if we need to do anything */
3670 if (ins->branch.target_type != TARGET_BREAK) continue;
3671
3672 /* It's a break! Check if it's our break */
3673 if (ins->branch.target_break != loop_idx) continue;
3674
3675 /* Okay, cool, we're breaking out of this loop.
3676 * Rewrite from a break to a goto */
3677
3678 ins->branch.target_type = TARGET_GOTO;
3679 ins->branch.target_block = break_block_idx;
3680 }
3681 }
3682
3683 /* Now that we've finished emitting the loop, free up the depth again
3684 * so we play nice with recursion amid nested loops */
3685 --ctx->current_loop_depth;
3686 }
3687
3688 static midgard_block *
3689 emit_cf_list(struct compiler_context *ctx, struct exec_list *list)
3690 {
3691 midgard_block *start_block = NULL;
3692
3693 foreach_list_typed(nir_cf_node, node, node, list) {
3694 switch (node->type) {
3695 case nir_cf_node_block: {
3696 midgard_block *block = emit_block(ctx, nir_cf_node_as_block(node));
3697
3698 if (!start_block)
3699 start_block = block;
3700
3701 break;
3702 }
3703
3704 case nir_cf_node_if:
3705 emit_if(ctx, nir_cf_node_as_if(node));
3706 break;
3707
3708 case nir_cf_node_loop:
3709 emit_loop(ctx, nir_cf_node_as_loop(node));
3710 break;
3711
3712 case nir_cf_node_function:
3713 assert(0);
3714 break;
3715 }
3716 }
3717
3718 return start_block;
3719 }
3720
3721 /* Due to lookahead, we need to report the first tag executed in the command
3722 * stream and in branch targets. An initial block might be empty, so iterate
3723 * until we find one that 'works' */
3724
3725 static unsigned
3726 midgard_get_first_tag_from_block(compiler_context *ctx, unsigned block_idx)
3727 {
3728 midgard_block *initial_block = mir_get_block(ctx, block_idx);
3729
3730 unsigned first_tag = 0;
3731
3732 do {
3733 midgard_bundle *initial_bundle = util_dynarray_element(&initial_block->bundles, midgard_bundle, 0);
3734
3735 if (initial_bundle) {
3736 first_tag = initial_bundle->tag;
3737 break;
3738 }
3739
3740 /* Initial block is empty, try the next block */
3741 initial_block = list_first_entry(&(initial_block->link), midgard_block, link);
3742 } while(initial_block != NULL);
3743
3744 assert(first_tag);
3745 return first_tag;
3746 }
3747
3748 int
3749 midgard_compile_shader_nir(nir_shader *nir, midgard_program *program, bool is_blend)
3750 {
3751 struct util_dynarray *compiled = &program->compiled;
3752
3753 midgard_debug = debug_get_option_midgard_debug();
3754
3755 compiler_context ictx = {
3756 .nir = nir,
3757 .stage = nir->info.stage,
3758
3759 .is_blend = is_blend,
3760 .blend_constant_offset = -1,
3761
3762 .alpha_ref = program->alpha_ref
3763 };
3764
3765 compiler_context *ctx = &ictx;
3766
3767 /* TODO: Decide this at runtime */
3768 ctx->uniform_cutoff = 8;
3769
3770 /* Assign var locations early, so the epilogue can use them if necessary */
3771
3772 nir_assign_var_locations(&nir->outputs, &nir->num_outputs, glsl_type_size);
3773 nir_assign_var_locations(&nir->inputs, &nir->num_inputs, glsl_type_size);
3774 nir_assign_var_locations(&nir->uniforms, &nir->num_uniforms, glsl_type_size);
3775
3776 /* Initialize at a global (not block) level hash tables */
3777
3778 ctx->ssa_constants = _mesa_hash_table_u64_create(NULL);
3779 ctx->ssa_varyings = _mesa_hash_table_u64_create(NULL);
3780 ctx->ssa_to_alias = _mesa_hash_table_u64_create(NULL);
3781 ctx->ssa_to_register = _mesa_hash_table_u64_create(NULL);
3782 ctx->hash_to_temp = _mesa_hash_table_u64_create(NULL);
3783 ctx->sysval_to_id = _mesa_hash_table_u64_create(NULL);
3784 ctx->leftover_ssa_to_alias = _mesa_set_create(NULL, _mesa_hash_pointer, _mesa_key_pointer_equal);
3785
3786 /* Record the varying mapping for the command stream's bookkeeping */
3787
3788 struct exec_list *varyings =
3789 ctx->stage == MESA_SHADER_VERTEX ? &nir->outputs : &nir->inputs;
3790
3791 nir_foreach_variable(var, varyings) {
3792 unsigned loc = var->data.driver_location;
3793 unsigned sz = glsl_type_size(var->type, FALSE);
3794
3795 for (int c = 0; c < sz; ++c) {
3796 program->varyings[loc + c] = var->data.location;
3797 }
3798 }
3799
3800 /* Lower gl_Position pre-optimisation */
3801
3802 if (ctx->stage == MESA_SHADER_VERTEX)
3803 NIR_PASS_V(nir, nir_lower_viewport_transform);
3804
3805 NIR_PASS_V(nir, nir_lower_var_copies);
3806 NIR_PASS_V(nir, nir_lower_vars_to_ssa);
3807 NIR_PASS_V(nir, nir_split_var_copies);
3808 NIR_PASS_V(nir, nir_lower_var_copies);
3809 NIR_PASS_V(nir, nir_lower_global_vars_to_local);
3810 NIR_PASS_V(nir, nir_lower_var_copies);
3811 NIR_PASS_V(nir, nir_lower_vars_to_ssa);
3812
3813 NIR_PASS_V(nir, nir_lower_io, nir_var_all, glsl_type_size, 0);
3814
3815 /* Optimisation passes */
3816
3817 optimise_nir(nir);
3818
3819 if (midgard_debug & MIDGARD_DBG_SHADERS) {
3820 nir_print_shader(nir, stdout);
3821 }
3822
3823 /* Assign sysvals and counts, now that we're sure
3824 * (post-optimisation) */
3825
3826 midgard_nir_assign_sysvals(ctx, nir);
3827
3828 program->uniform_count = nir->num_uniforms;
3829 program->sysval_count = ctx->sysval_count;
3830 memcpy(program->sysvals, ctx->sysvals, sizeof(ctx->sysvals[0]) * ctx->sysval_count);
3831
3832 program->attribute_count = (ctx->stage == MESA_SHADER_VERTEX) ? nir->num_inputs : 0;
3833 program->varying_count = (ctx->stage == MESA_SHADER_VERTEX) ? nir->num_outputs : ((ctx->stage == MESA_SHADER_FRAGMENT) ? nir->num_inputs : 0);
3834
3835 nir_foreach_function(func, nir) {
3836 if (!func->impl)
3837 continue;
3838
3839 list_inithead(&ctx->blocks);
3840 ctx->block_count = 0;
3841 ctx->func = func;
3842
3843 emit_cf_list(ctx, &func->impl->body);
3844 emit_block(ctx, func->impl->end_block);
3845
3846 break; /* TODO: Multi-function shaders */
3847 }
3848
3849 util_dynarray_init(compiled, NULL);
3850
3851 /* MIR-level optimizations */
3852
3853 bool progress = false;
3854
3855 do {
3856 progress = false;
3857
3858 mir_foreach_block(ctx, block) {
3859 progress |= midgard_opt_copy_prop(ctx, block);
3860 progress |= midgard_opt_copy_prop_tex(ctx, block);
3861 progress |= midgard_opt_dead_code_eliminate(ctx, block);
3862 }
3863 } while (progress);
3864
3865 /* Schedule! */
3866 schedule_program(ctx);
3867
3868 /* Now that all the bundles are scheduled and we can calculate block
3869 * sizes, emit actual branch instructions rather than placeholders */
3870
3871 int br_block_idx = 0;
3872
3873 mir_foreach_block(ctx, block) {
3874 util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) {
3875 for (int c = 0; c < bundle->instruction_count; ++c) {
3876 midgard_instruction *ins = &bundle->instructions[c];
3877
3878 if (!midgard_is_branch_unit(ins->unit)) continue;
3879
3880 if (ins->prepacked_branch) continue;
3881
3882 /* Parse some basic branch info */
3883 bool is_compact = ins->unit == ALU_ENAB_BR_COMPACT;
3884 bool is_conditional = ins->branch.conditional;
3885 bool is_inverted = ins->branch.invert_conditional;
3886 bool is_discard = ins->branch.target_type == TARGET_DISCARD;
3887
3888 /* Determine the block we're jumping to */
3889 int target_number = ins->branch.target_block;
3890
3891 /* Report the destination tag. Discards don't need this */
3892 int dest_tag = is_discard ? 0 : midgard_get_first_tag_from_block(ctx, target_number);
3893
3894 /* Count up the number of quadwords we're jumping over. That is, the number of quadwords in each of the blocks between (br_block_idx, target_number) */
3895 int quadword_offset = 0;
3896
3897 if (is_discard) {
3898 /* Jump to the end of the shader. We
3899 * need to include not only the
3900 * following blocks, but also the
3901 * contents of our current block (since
3902 * discard can come in the middle of
3903 * the block) */
3904
3905 midgard_block *blk = mir_get_block(ctx, br_block_idx + 1);
3906
3907 for (midgard_bundle *bun = bundle + 1; bun < (midgard_bundle *)((char*) block->bundles.data + block->bundles.size); ++bun) {
3908 quadword_offset += quadword_size(bun->tag);
3909 }
3910
3911 mir_foreach_block_from(ctx, blk, b) {
3912 quadword_offset += b->quadword_count;
3913 }
3914
3915 } else if (target_number > br_block_idx) {
3916 /* Jump forward */
3917
3918 for (int idx = br_block_idx + 1; idx < target_number; ++idx) {
3919 midgard_block *blk = mir_get_block(ctx, idx);
3920 assert(blk);
3921
3922 quadword_offset += blk->quadword_count;
3923 }
3924 } else {
3925 /* Jump backwards */
3926
3927 for (int idx = br_block_idx; idx >= target_number; --idx) {
3928 midgard_block *blk = mir_get_block(ctx, idx);
3929 assert(blk);
3930
3931 quadword_offset -= blk->quadword_count;
3932 }
3933 }
3934
3935 /* Unconditional extended branches (far jumps)
3936 * have issues, so we always use a conditional
3937 * branch, setting the condition to always for
3938 * unconditional. For compact unconditional
3939 * branches, cond isn't used so it doesn't
3940 * matter what we pick. */
3941
3942 midgard_condition cond =
3943 !is_conditional ? midgard_condition_always :
3944 is_inverted ? midgard_condition_false :
3945 midgard_condition_true;
3946
3947 midgard_jmp_writeout_op op =
3948 is_discard ? midgard_jmp_writeout_op_discard :
3949 (is_compact && !is_conditional) ? midgard_jmp_writeout_op_branch_uncond :
3950 midgard_jmp_writeout_op_branch_cond;
3951
3952 if (!is_compact) {
3953 midgard_branch_extended branch =
3954 midgard_create_branch_extended(
3955 cond, op,
3956 dest_tag,
3957 quadword_offset);
3958
3959 memcpy(&ins->branch_extended, &branch, sizeof(branch));
3960 } else if (is_conditional || is_discard) {
3961 midgard_branch_cond branch = {
3962 .op = op,
3963 .dest_tag = dest_tag,
3964 .offset = quadword_offset,
3965 .cond = cond
3966 };
3967
3968 assert(branch.offset == quadword_offset);
3969
3970 memcpy(&ins->br_compact, &branch, sizeof(branch));
3971 } else {
3972 assert(op == midgard_jmp_writeout_op_branch_uncond);
3973
3974 midgard_branch_uncond branch = {
3975 .op = op,
3976 .dest_tag = dest_tag,
3977 .offset = quadword_offset,
3978 .unknown = 1
3979 };
3980
3981 assert(branch.offset == quadword_offset);
3982
3983 memcpy(&ins->br_compact, &branch, sizeof(branch));
3984 }
3985 }
3986 }
3987
3988 ++br_block_idx;
3989 }
3990
3991 /* Emit flat binary from the instruction arrays. Iterate each block in
3992 * sequence. Save instruction boundaries such that lookahead tags can
3993 * be assigned easily */
3994
3995 /* Cache _all_ bundles in source order for lookahead across failed branches */
3996
3997 int bundle_count = 0;
3998 mir_foreach_block(ctx, block) {
3999 bundle_count += block->bundles.size / sizeof(midgard_bundle);
4000 }
4001 midgard_bundle **source_order_bundles = malloc(sizeof(midgard_bundle *) * bundle_count);
4002 int bundle_idx = 0;
4003 mir_foreach_block(ctx, block) {
4004 util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) {
4005 source_order_bundles[bundle_idx++] = bundle;
4006 }
4007 }
4008
4009 int current_bundle = 0;
4010
4011 mir_foreach_block(ctx, block) {
4012 util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) {
4013 int lookahead = 1;
4014
4015 if (current_bundle + 1 < bundle_count) {
4016 uint8_t next = source_order_bundles[current_bundle + 1]->tag;
4017
4018 if (!(current_bundle + 2 < bundle_count) && IS_ALU(next)) {
4019 lookahead = 1;
4020 } else {
4021 lookahead = next;
4022 }
4023 }
4024
4025 emit_binary_bundle(ctx, bundle, compiled, lookahead);
4026 ++current_bundle;
4027 }
4028
4029 /* TODO: Free deeper */
4030 //util_dynarray_fini(&block->instructions);
4031 }
4032
4033 free(source_order_bundles);
4034
4035 /* Report the very first tag executed */
4036 program->first_tag = midgard_get_first_tag_from_block(ctx, 0);
4037
4038 /* Deal with off-by-one related to the fencepost problem */
4039 program->work_register_count = ctx->work_registers + 1;
4040
4041 program->can_discard = ctx->can_discard;
4042 program->uniform_cutoff = ctx->uniform_cutoff;
4043
4044 program->blend_patch_offset = ctx->blend_constant_offset;
4045
4046 if (midgard_debug & MIDGARD_DBG_SHADERS)
4047 disassemble_midgard(program->compiled.data, program->compiled.size);
4048
4049 return 0;
4050 }