projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
panfrost/midgard: Preview for data hazards
[mesa.git] / src / gallium / drivers / panfrost / midgard / midgard_compile.c
1 /*
2 * Copyright (C) 2018 Alyssa Rosenzweig <alyssa@rosenzweig.io>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
23
24 #include <sys/types.h>
25 #include <sys/stat.h>
26 #include <sys/mman.h>
27 #include <fcntl.h>
28 #include <stdint.h>
29 #include <stdlib.h>
30 #include <stdio.h>
31 #include <err.h>
32
33 #include "compiler/glsl/glsl_to_nir.h"
34 #include "compiler/nir_types.h"
35 #include "main/imports.h"
36 #include "compiler/nir/nir_builder.h"
37 #include "util/half_float.h"
38 #include "util/register_allocate.h"
39 #include "util/u_dynarray.h"
40 #include "util/list.h"
41 #include "main/mtypes.h"
42
43 #include "midgard.h"
44 #include "midgard_nir.h"
45 #include "midgard_compile.h"
46 #include "helpers.h"
47
48 #include "disassemble.h"
49
50 /* Instruction arguments represented as block-local SSA indices, rather than
51 * registers. Negative values mean unused. */
52
53 typedef struct {
54 int src0;
55 int src1;
56 int dest;
57
58 /* src1 is -not- SSA but instead a 16-bit inline constant to be smudged
59 * in. Only valid for ALU ops. */
60 bool inline_constant;
61 } ssa_args;
62
63 /* Forward declare so midgard_branch can reference */
64 struct midgard_block;
65
66 /* Target types. Defaults to TARGET_GOTO (the type corresponding directly to
67 * the hardware), hence why that must be zero. TARGET_DISCARD signals this
68 * instruction is actually a discard op. */
69
70 #define TARGET_GOTO 0
71 #define TARGET_BREAK 1
72 #define TARGET_CONTINUE 2
73 #define TARGET_DISCARD 3
74
75 typedef struct midgard_branch {
76 /* If conditional, the condition is specified in r31.w */
77 bool conditional;
78
79 /* For conditionals, if this is true, we branch on FALSE. If false, we branch on TRUE. */
80 bool invert_conditional;
81
82 /* Branch targets: the start of a block, the start of a loop (continue), the end of a loop (break). Value is one of TARGET_ */
83 unsigned target_type;
84
85 /* The actual target */
86 union {
87 int target_block;
88 int target_break;
89 int target_continue;
90 };
91 } midgard_branch;
92
93 /* Generic in-memory data type repesenting a single logical instruction, rather
94 * than a single instruction group. This is the preferred form for code gen.
95 * Multiple midgard_insturctions will later be combined during scheduling,
96 * though this is not represented in this structure. Its format bridges
97 * the low-level binary representation with the higher level semantic meaning.
98 *
99 * Notably, it allows registers to be specified as block local SSA, for code
100 * emitted before the register allocation pass.
101 */
102
103 typedef struct midgard_instruction {
104 /* Must be first for casting */
105 struct list_head link;
106
107 unsigned type; /* ALU, load/store, texture */
108
109 /* If the register allocator has not run yet... */
110 ssa_args ssa_args;
111
112 /* Special fields for an ALU instruction */
113 midgard_reg_info registers;
114
115 /* I.e. (1 << alu_bit) */
116 int unit;
117
118 bool has_constants;
119 float constants[4];
120 uint16_t inline_constant;
121 bool has_blend_constant;
122
123 bool compact_branch;
124 bool writeout;
125 bool prepacked_branch;
126
127 union {
128 midgard_load_store_word load_store;
129 midgard_vector_alu alu;
130 midgard_texture_word texture;
131 midgard_branch_extended branch_extended;
132 uint16_t br_compact;
133
134 /* General branch, rather than packed br_compact. Higher level
135 * than the other components */
136 midgard_branch branch;
137 };
138 } midgard_instruction;
139
140 typedef struct midgard_block {
141 /* Link to next block. Must be first for mir_get_block */
142 struct list_head link;
143
144 /* List of midgard_instructions emitted for the current block */
145 struct list_head instructions;
146
147 bool is_scheduled;
148
149 /* List of midgard_bundles emitted (after the scheduler has run) */
150 struct util_dynarray bundles;
151
152 /* Number of quadwords _actually_ emitted, as determined after scheduling */
153 unsigned quadword_count;
154
155 struct midgard_block *next_fallthrough;
156 } midgard_block;
157
158 /* Helpers to generate midgard_instruction's using macro magic, since every
159 * driver seems to do it that way */
160
161 #define EMIT(op, ...) emit_mir_instruction(ctx, v_##op(__VA_ARGS__));
162
163 #define M_LOAD_STORE(name, rname, uname) \
164 static midgard_instruction m_##name(unsigned ssa, unsigned address) { \
165 midgard_instruction i = { \
166 .type = TAG_LOAD_STORE_4, \
167 .ssa_args = { \
168 .rname = ssa, \
169 .uname = -1, \
170 .src1 = -1 \
171 }, \
172 .load_store = { \
173 .op = midgard_op_##name, \
174 .mask = 0xF, \
175 .swizzle = SWIZZLE(COMPONENT_X, COMPONENT_Y, COMPONENT_Z, COMPONENT_W), \
176 .address = address \
177 } \
178 }; \
179 \
180 return i; \
181 }
182
183 #define M_LOAD(name) M_LOAD_STORE(name, dest, src0)
184 #define M_STORE(name) M_LOAD_STORE(name, src0, dest)
185
186 const midgard_vector_alu_src blank_alu_src = {
187 .swizzle = SWIZZLE(COMPONENT_X, COMPONENT_Y, COMPONENT_Z, COMPONENT_W),
188 };
189
190 const midgard_scalar_alu_src blank_scalar_alu_src = {
191 .full = true
192 };
193
194 /* Used for encoding the unused source of 1-op instructions */
195 const midgard_vector_alu_src zero_alu_src = { 0 };
196
197 /* Coerce structs to integer */
198
199 static unsigned
200 vector_alu_srco_unsigned(midgard_vector_alu_src src)
201 {
202 unsigned u;
203 memcpy(&u, &src, sizeof(src));
204 return u;
205 }
206
207 /* Inputs a NIR ALU source, with modifiers attached if necessary, and outputs
208 * the corresponding Midgard source */
209
210 static midgard_vector_alu_src
211 vector_alu_modifiers(nir_alu_src *src)
212 {
213 if (!src) return blank_alu_src;
214
215 midgard_vector_alu_src alu_src = {
216 .abs = src->abs,
217 .negate = src->negate,
218 .rep_low = 0,
219 .rep_high = 0,
220 .half = 0, /* TODO */
221 .swizzle = SWIZZLE_FROM_ARRAY(src->swizzle)
222 };
223
224 return alu_src;
225 }
226
227 /* 'Intrinsic' move for misc aliasing uses independent of actual NIR ALU code */
228
229 static midgard_instruction
230 v_fmov(unsigned src, midgard_vector_alu_src mod, unsigned dest)
231 {
232 midgard_instruction ins = {
233 .type = TAG_ALU_4,
234 .ssa_args = {
235 .src0 = SSA_UNUSED_1,
236 .src1 = src,
237 .dest = dest,
238 },
239 .alu = {
240 .op = midgard_alu_op_fmov,
241 .reg_mode = midgard_reg_mode_full,
242 .dest_override = midgard_dest_override_none,
243 .mask = 0xFF,
244 .src1 = vector_alu_srco_unsigned(zero_alu_src),
245 .src2 = vector_alu_srco_unsigned(mod)
246 },
247 };
248
249 return ins;
250 }
251
252 /* load/store instructions have both 32-bit and 16-bit variants, depending on
253 * whether we are using vectors composed of highp or mediump. At the moment, we
254 * don't support half-floats -- this requires changes in other parts of the
255 * compiler -- therefore the 16-bit versions are commented out. */
256
257 //M_LOAD(load_attr_16);
258 M_LOAD(load_attr_32);
259 //M_LOAD(load_vary_16);
260 M_LOAD(load_vary_32);
261 //M_LOAD(load_uniform_16);
262 M_LOAD(load_uniform_32);
263 M_LOAD(load_color_buffer_8);
264 //M_STORE(store_vary_16);
265 M_STORE(store_vary_32);
266
267 static midgard_instruction
268 v_alu_br_compact_cond(midgard_jmp_writeout_op op, unsigned tag, signed offset, unsigned cond)
269 {
270 midgard_branch_cond branch = {
271 .op = op,
272 .dest_tag = tag,
273 .offset = offset,
274 .cond = cond
275 };
276
277 uint16_t compact;
278 memcpy(&compact, &branch, sizeof(branch));
279
280 midgard_instruction ins = {
281 .type = TAG_ALU_4,
282 .unit = ALU_ENAB_BR_COMPACT,
283 .prepacked_branch = true,
284 .compact_branch = true,
285 .br_compact = compact
286 };
287
288 if (op == midgard_jmp_writeout_op_writeout)
289 ins.writeout = true;
290
291 return ins;
292 }
293
294 static midgard_instruction
295 v_branch(bool conditional, bool invert)
296 {
297 midgard_instruction ins = {
298 .type = TAG_ALU_4,
299 .unit = ALU_ENAB_BRANCH,
300 .compact_branch = true,
301 .branch = {
302 .conditional = conditional,
303 .invert_conditional = invert
304 }
305 };
306
307 return ins;
308 }
309
310 static midgard_branch_extended
311 midgard_create_branch_extended( midgard_condition cond,
312 midgard_jmp_writeout_op op,
313 unsigned dest_tag,
314 signed quadword_offset)
315 {
316 /* For unclear reasons, the condition code is repeated 8 times */
317 uint16_t duplicated_cond =
318 (cond << 14) |
319 (cond << 12) |
320 (cond << 10) |
321 (cond << 8) |
322 (cond << 6) |
323 (cond << 4) |
324 (cond << 2) |
325 (cond << 0);
326
327 midgard_branch_extended branch = {
328 .op = op,
329 .dest_tag = dest_tag,
330 .offset = quadword_offset,
331 .cond = duplicated_cond
332 };
333
334 return branch;
335 }
336
337 typedef struct midgard_bundle {
338 /* Tag for the overall bundle */
339 int tag;
340
341 /* Instructions contained by the bundle */
342 int instruction_count;
343 midgard_instruction instructions[5];
344
345 /* Bundle-wide ALU configuration */
346 int padding;
347 int control;
348 bool has_embedded_constants;
349 float constants[4];
350 bool has_blend_constant;
351
352 uint16_t register_words[8];
353 int register_words_count;
354
355 uint64_t body_words[8];
356 size_t body_size[8];
357 int body_words_count;
358 } midgard_bundle;
359
360 typedef struct compiler_context {
361 nir_shader *nir;
362 gl_shader_stage stage;
363
364 /* Is internally a blend shader? Depends on stage == FRAGMENT */
365 bool is_blend;
366
367 /* Tracking for blend constant patching */
368 int blend_constant_number;
369 int blend_constant_offset;
370
371 /* Current NIR function */
372 nir_function *func;
373
374 /* Unordered list of midgard_blocks */
375 int block_count;
376 struct list_head blocks;
377
378 midgard_block *initial_block;
379 midgard_block *previous_source_block;
380 midgard_block *final_block;
381
382 /* List of midgard_instructions emitted for the current block */
383 midgard_block *current_block;
384
385 /* The index corresponding to the current loop, e.g. for breaks/contineus */
386 int current_loop;
387
388 /* Constants which have been loaded, for later inlining */
389 struct hash_table_u64 *ssa_constants;
390
391 /* SSA indices to be outputted to corresponding varying offset */
392 struct hash_table_u64 *ssa_varyings;
393
394 /* SSA values / registers which have been aliased. Naively, these
395 * demand a fmov output; instead, we alias them in a later pass to
396 * avoid the wasted op.
397 *
398 * A note on encoding: to avoid dynamic memory management here, rather
399 * than ampping to a pointer, we map to the source index; the key
400 * itself is just the destination index. */
401
402 struct hash_table_u64 *ssa_to_alias;
403 struct set *leftover_ssa_to_alias;
404
405 /* Actual SSA-to-register for RA */
406 struct hash_table_u64 *ssa_to_register;
407
408 /* Mapping of hashes computed from NIR indices to the sequential temp indices ultimately used in MIR */
409 struct hash_table_u64 *hash_to_temp;
410 int temp_count;
411 int max_hash;
412
413 /* Uniform IDs for mdg */
414 struct hash_table_u64 *uniform_nir_to_mdg;
415 int uniform_count;
416
417 struct hash_table_u64 *varying_nir_to_mdg;
418 int varying_count;
419
420 /* Just the count of the max register used. Higher count => higher
421 * register pressure */
422 int work_registers;
423
424 /* Used for cont/last hinting. Increase when a tex op is added.
425 * Decrease when a tex op is removed. */
426 int texture_op_count;
427
428 /* Mapping of texture register -> SSA index for unaliasing */
429 int texture_index[2];
430
431 /* Count of special uniforms (viewport, etc) in vec4 units */
432 int special_uniforms;
433
434 /* If any path hits a discard instruction */
435 bool can_discard;
436
437 /* The number of uniforms allowable for the fast path */
438 int uniform_cutoff;
439
440 /* Count of instructions emitted from NIR overall, across all blocks */
441 int instruction_count;
442
443 /* Alpha ref value passed in */
444 float alpha_ref;
445
446 /* The index corresponding to the fragment output */
447 unsigned fragment_output;
448 } compiler_context;
449
450 /* Append instruction to end of current block */
451
452 static midgard_instruction *
453 mir_upload_ins(struct midgard_instruction ins)
454 {
455 midgard_instruction *heap = malloc(sizeof(ins));
456 memcpy(heap, &ins, sizeof(ins));
457 return heap;
458 }
459
460 static void
461 emit_mir_instruction(struct compiler_context *ctx, struct midgard_instruction ins)
462 {
463 list_addtail(&(mir_upload_ins(ins))->link, &ctx->current_block->instructions);
464 }
465
466 static void
467 mir_insert_instruction_before(struct midgard_instruction *tag, struct midgard_instruction ins)
468 {
469 list_addtail(&(mir_upload_ins(ins))->link, &tag->link);
470 }
471
472 static void
473 mir_remove_instruction(struct midgard_instruction *ins)
474 {
475 list_del(&ins->link);
476 }
477
478 static midgard_instruction*
479 mir_prev_op(struct midgard_instruction *ins)
480 {
481 return list_last_entry(&(ins->link), midgard_instruction, link);
482 }
483
484 static midgard_instruction*
485 mir_next_op(struct midgard_instruction *ins)
486 {
487 return list_first_entry(&(ins->link), midgard_instruction, link);
488 }
489
490 static midgard_block *
491 mir_next_block(struct midgard_block *blk)
492 {
493 return list_first_entry(&(blk->link), midgard_block, link);
494 }
495
496
497 #define mir_foreach_block(ctx, v) list_for_each_entry(struct midgard_block, v, &ctx->blocks, link)
498 #define mir_foreach_block_from(ctx, from, v) list_for_each_entry_from(struct midgard_block, v, from, &ctx->blocks, link)
499
500 #define mir_foreach_instr(ctx, v) list_for_each_entry(struct midgard_instruction, v, &ctx->current_block->instructions, link)
501 #define mir_foreach_instr_safe(ctx, v) list_for_each_entry_safe(struct midgard_instruction, v, &ctx->current_block->instructions, link)
502 #define mir_foreach_instr_in_block(block, v) list_for_each_entry(struct midgard_instruction, v, &block->instructions, link)
503 #define mir_foreach_instr_in_block_safe(block, v) list_for_each_entry_safe(struct midgard_instruction, v, &block->instructions, link)
504 #define mir_foreach_instr_in_block_safe_rev(block, v) list_for_each_entry_safe_rev(struct midgard_instruction, v, &block->instructions, link)
505 #define mir_foreach_instr_in_block_from(block, v, from) list_for_each_entry_from(struct midgard_instruction, v, from, &block->instructions, link)
506
507
508 static midgard_instruction *
509 mir_last_in_block(struct midgard_block *block)
510 {
511 return list_last_entry(&block->instructions, struct midgard_instruction, link);
512 }
513
514 static midgard_block *
515 mir_get_block(compiler_context *ctx, int idx)
516 {
517 struct list_head *lst = &ctx->blocks;
518
519 while ((idx--) + 1)
520 lst = lst->next;
521
522 return (struct midgard_block *) lst;
523 }
524
525 /* Pretty printer for internal Midgard IR */
526
527 static void
528 print_mir_source(int source)
529 {
530 if (source >= SSA_FIXED_MINIMUM) {
531 /* Specific register */
532 int reg = SSA_REG_FROM_FIXED(source);
533
534 /* TODO: Moving threshold */
535 if (reg > 16 && reg < 24)
536 printf("u%d", 23 - reg);
537 else
538 printf("r%d", reg);
539 } else {
540 printf("%d", source);
541 }
542 }
543
544 static void
545 print_mir_instruction(midgard_instruction *ins)
546 {
547 printf("\t");
548
549 switch (ins->type) {
550 case TAG_ALU_4: {
551 midgard_alu_op op = ins->alu.op;
552 const char *name = alu_opcode_names[op];
553
554 if (ins->unit)
555 printf("%d.", ins->unit);
556
557 printf("%s", name ? name : "??");
558 break;
559 }
560
561 case TAG_LOAD_STORE_4: {
562 midgard_load_store_op op = ins->load_store.op;
563 const char *name = load_store_opcode_names[op];
564
565 assert(name);
566 printf("%s", name);
567 break;
568 }
569
570 case TAG_TEXTURE_4: {
571 printf("texture");
572 break;
573 }
574
575 default:
576 assert(0);
577 }
578
579 ssa_args *args = &ins->ssa_args;
580
581 printf(" %d, ", args->dest);
582
583 print_mir_source(args->src0);
584 printf(", ");
585
586 if (args->inline_constant)
587 printf("#%d", ins->inline_constant);
588 else
589 print_mir_source(args->src1);
590
591 if (ins->has_constants)
592 printf(" <%f, %f, %f, %f>", ins->constants[0], ins->constants[1], ins->constants[2], ins->constants[3]);
593
594 printf("\n");
595 }
596
597 static void
598 print_mir_block(midgard_block *block)
599 {
600 printf("{\n");
601
602 mir_foreach_instr_in_block(block, ins) {
603 print_mir_instruction(ins);
604 }
605
606 printf("}\n");
607 }
608
609
610
611 static void
612 attach_constants(compiler_context *ctx, midgard_instruction *ins, void *constants, int name)
613 {
614 ins->has_constants = true;
615 memcpy(&ins->constants, constants, 16);
616
617 /* If this is the special blend constant, mark this instruction */
618
619 if (ctx->is_blend && ctx->blend_constant_number == name)
620 ins->has_blend_constant = true;
621 }
622
623 static int
624 glsl_type_size(const struct glsl_type *type)
625 {
626 return glsl_count_attribute_slots(type, false);
627 }
628
629 /* Lower fdot2 to a vector multiplication followed by channel addition */
630 static void
631 midgard_nir_lower_fdot2_body(nir_builder *b, nir_alu_instr *alu)
632 {
633 if (alu->op != nir_op_fdot2)
634 return;
635
636 b->cursor = nir_before_instr(&alu->instr);
637
638 nir_ssa_def *src0 = nir_ssa_for_alu_src(b, alu, 0);
639 nir_ssa_def *src1 = nir_ssa_for_alu_src(b, alu, 1);
640
641 nir_ssa_def *product = nir_fmul(b, src0, src1);
642
643 nir_ssa_def *sum = nir_fadd(b,
644 nir_channel(b, product, 0),
645 nir_channel(b, product, 1));
646
647 /* Replace the fdot2 with this sum */
648 nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(sum));
649 }
650
651 static bool
652 midgard_nir_lower_fdot2(nir_shader *shader)
653 {
654 bool progress = false;
655
656 nir_foreach_function(function, shader) {
657 if (!function->impl) continue;
658
659 nir_builder _b;
660 nir_builder *b = &_b;
661 nir_builder_init(b, function->impl);
662
663 nir_foreach_block(block, function->impl) {
664 nir_foreach_instr_safe(instr, block) {
665 if (instr->type != nir_instr_type_alu) continue;
666
667 nir_alu_instr *alu = nir_instr_as_alu(instr);
668 midgard_nir_lower_fdot2_body(b, alu);
669
670 progress |= true;
671 }
672 }
673
674 nir_metadata_preserve(function->impl, nir_metadata_block_index | nir_metadata_dominance);
675
676 }
677
678 return progress;
679 }
680
681 static void
682 optimise_nir(nir_shader *nir)
683 {
684 bool progress;
685
686 NIR_PASS(progress, nir, nir_lower_regs_to_ssa);
687 NIR_PASS(progress, nir, midgard_nir_lower_fdot2);
688
689 nir_lower_tex_options lower_tex_options = {
690 .lower_rect = true
691 };
692
693 NIR_PASS(progress, nir, nir_lower_tex, &lower_tex_options);
694
695 do {
696 progress = false;
697
698 NIR_PASS(progress, nir, midgard_nir_lower_algebraic);
699 NIR_PASS(progress, nir, nir_lower_io, nir_var_all, glsl_type_size, 0);
700 NIR_PASS(progress, nir, nir_lower_var_copies);
701 NIR_PASS(progress, nir, nir_lower_vars_to_ssa);
702
703 NIR_PASS(progress, nir, nir_copy_prop);
704 NIR_PASS(progress, nir, nir_opt_dce);
705 NIR_PASS(progress, nir, nir_opt_dead_cf);
706 NIR_PASS(progress, nir, nir_opt_cse);
707 NIR_PASS(progress, nir, nir_opt_peephole_select, 64, false, true);
708 NIR_PASS(progress, nir, nir_opt_algebraic);
709 NIR_PASS(progress, nir, nir_opt_constant_folding);
710 NIR_PASS(progress, nir, nir_opt_undef);
711 NIR_PASS(progress, nir, nir_opt_loop_unroll,
712 nir_var_shader_in |
713 nir_var_shader_out |
714 nir_var_function_temp);
715
716 /* TODO: Enable vectorize when merged upstream */
717 // NIR_PASS(progress, nir, nir_opt_vectorize);
718 } while (progress);
719
720 /* Must be run at the end to prevent creation of fsin/fcos ops */
721 NIR_PASS(progress, nir, midgard_nir_scale_trig);
722
723 do {
724 progress = false;
725
726 NIR_PASS(progress, nir, nir_opt_dce);
727 NIR_PASS(progress, nir, nir_opt_algebraic);
728 NIR_PASS(progress, nir, nir_opt_constant_folding);
729 NIR_PASS(progress, nir, nir_copy_prop);
730 } while (progress);
731
732 NIR_PASS(progress, nir, nir_opt_algebraic_late);
733
734 /* Lower mods */
735 NIR_PASS(progress, nir, nir_lower_to_source_mods, nir_lower_all_source_mods);
736 NIR_PASS(progress, nir, nir_copy_prop);
737 NIR_PASS(progress, nir, nir_opt_dce);
738
739 /* Take us out of SSA */
740 NIR_PASS(progress, nir, nir_lower_locals_to_regs);
741 NIR_PASS(progress, nir, nir_convert_from_ssa, true);
742
743 /* We are a vector architecture; write combine where possible */
744 NIR_PASS(progress, nir, nir_move_vec_src_uses_to_dest);
745 NIR_PASS(progress, nir, nir_lower_vec_to_movs);
746
747 NIR_PASS(progress, nir, nir_opt_dce);
748 }
749
750 /* Front-half of aliasing the SSA slots, merely by inserting the flag in the
751 * appropriate hash table. Intentional off-by-one to avoid confusing NULL with
752 * r0. See the comments in compiler_context */
753
754 static void
755 alias_ssa(compiler_context *ctx, int dest, int src)
756 {
757 _mesa_hash_table_u64_insert(ctx->ssa_to_alias, dest + 1, (void *) ((uintptr_t) src + 1));
758 _mesa_set_add(ctx->leftover_ssa_to_alias, (void *) (uintptr_t) (dest + 1));
759 }
760
761 /* ...or undo it, after which the original index will be used (dummy move should be emitted alongside this) */
762
763 static void
764 unalias_ssa(compiler_context *ctx, int dest)
765 {
766 _mesa_hash_table_u64_remove(ctx->ssa_to_alias, dest + 1);
767 /* TODO: Remove from leftover or no? */
768 }
769
770 static void
771 midgard_pin_output(compiler_context *ctx, int index, int reg)
772 {
773 _mesa_hash_table_u64_insert(ctx->ssa_to_register, index + 1, (void *) ((uintptr_t) reg + 1));
774 }
775
776 static bool
777 midgard_is_pinned(compiler_context *ctx, int index)
778 {
779 return _mesa_hash_table_u64_search(ctx->ssa_to_register, index + 1) != NULL;
780 }
781
782 /* Do not actually emit a load; instead, cache the constant for inlining */
783
784 static void
785 emit_load_const(compiler_context *ctx, nir_load_const_instr *instr)
786 {
787 nir_ssa_def def = instr->def;
788
789 float *v = ralloc_array(NULL, float, 4);
790 memcpy(v, &instr->value.f32, 4 * sizeof(float));
791 _mesa_hash_table_u64_insert(ctx->ssa_constants, def.index + 1, v);
792 }
793
794 /* Duplicate bits to convert sane 4-bit writemask to obscure 8-bit format (or
795 * do the inverse) */
796
797 static unsigned
798 expand_writemask(unsigned mask)
799 {
800 unsigned o = 0;
801
802 for (int i = 0; i < 4; ++i)
803 if (mask & (1 << i))
804 o |= (3 << (2 * i));
805
806 return o;
807 }
808
809 static unsigned
810 squeeze_writemask(unsigned mask)
811 {
812 unsigned o = 0;
813
814 for (int i = 0; i < 4; ++i)
815 if (mask & (3 << (2 * i)))
816 o |= (1 << i);
817
818 return o;
819
820 }
821
822 /* Determines effective writemask, taking quirks and expansion into account */
823 static unsigned
824 effective_writemask(midgard_vector_alu *alu)
825 {
826 /* Channel count is off-by-one to fit in two-bits (0 channel makes no
827 * sense) */
828
829 unsigned channel_count = GET_CHANNEL_COUNT(alu_opcode_props[alu->op]);
830
831 /* If there is a fixed channel count, construct the appropriate mask */
832
833 if (channel_count)
834 return (1 << channel_count) - 1;
835
836 /* Otherwise, just squeeze the existing mask */
837 return squeeze_writemask(alu->mask);
838 }
839
840 static unsigned
841 find_or_allocate_temp(compiler_context *ctx, unsigned hash)
842 {
843 if ((hash < 0) || (hash >= SSA_FIXED_MINIMUM))
844 return hash;
845
846 unsigned temp = (uintptr_t) _mesa_hash_table_u64_search(ctx->hash_to_temp, hash + 1);
847
848 if (temp)
849 return temp - 1;
850
851 /* If no temp is find, allocate one */
852 temp = ctx->temp_count++;
853 ctx->max_hash = MAX2(ctx->max_hash, hash);
854
855 _mesa_hash_table_u64_insert(ctx->hash_to_temp, hash + 1, (void *) ((uintptr_t) temp + 1));
856
857 return temp;
858 }
859
860 static unsigned
861 nir_src_index(compiler_context *ctx, nir_src *src)
862 {
863 if (src->is_ssa)
864 return src->ssa->index;
865 else
866 return ctx->func->impl->ssa_alloc + src->reg.reg->index;
867 }
868
869 static unsigned
870 nir_dest_index(compiler_context *ctx, nir_dest *dst)
871 {
872 if (dst->is_ssa)
873 return dst->ssa.index;
874 else
875 return ctx->func->impl->ssa_alloc + dst->reg.reg->index;
876 }
877
878 static unsigned
879 nir_alu_src_index(compiler_context *ctx, nir_alu_src *src)
880 {
881 return nir_src_index(ctx, &src->src);
882 }
883
884 /* Midgard puts conditionals in r31.w; move an arbitrary source (the output of
885 * a conditional test) into that register */
886
887 static void
888 emit_condition(compiler_context *ctx, nir_src *src, bool for_branch)
889 {
890 /* XXX: Force component correct */
891 int condition = nir_src_index(ctx, src);
892
893 const midgard_vector_alu_src alu_src = {
894 .swizzle = SWIZZLE(COMPONENT_X, COMPONENT_X, COMPONENT_X, COMPONENT_X),
895 };
896
897 /* There is no boolean move instruction. Instead, we simulate a move by
898 * ANDing the condition with itself to get it into r31.w */
899
900 midgard_instruction ins = {
901 .type = TAG_ALU_4,
902 .unit = for_branch ? UNIT_SMUL : UNIT_SADD, /* TODO: DEDUCE THIS */
903 .ssa_args = {
904 .src0 = condition,
905 .src1 = condition,
906 .dest = SSA_FIXED_REGISTER(31),
907 },
908 .alu = {
909 .op = midgard_alu_op_iand,
910 .reg_mode = midgard_reg_mode_full,
911 .dest_override = midgard_dest_override_none,
912 .mask = (0x3 << 6), /* w */
913 .src1 = vector_alu_srco_unsigned(alu_src),
914 .src2 = vector_alu_srco_unsigned(alu_src)
915 },
916 };
917
918 emit_mir_instruction(ctx, ins);
919 }
920
921 #define ALU_CASE(nir, _op) \
922 case nir_op_##nir: \
923 op = midgard_alu_op_##_op; \
924 break;
925
926 static void
927 emit_alu(compiler_context *ctx, nir_alu_instr *instr)
928 {
929 bool is_ssa = instr->dest.dest.is_ssa;
930
931 unsigned dest = nir_dest_index(ctx, &instr->dest.dest);
932 unsigned nr_components = is_ssa ? instr->dest.dest.ssa.num_components : instr->dest.dest.reg.reg->num_components;
933 unsigned nr_inputs = nir_op_infos[instr->op].num_inputs;
934
935 /* Most Midgard ALU ops have a 1:1 correspondance to NIR ops; these are
936 * supported. A few do not and are commented for now. Also, there are a
937 * number of NIR ops which Midgard does not support and need to be
938 * lowered, also TODO. This switch block emits the opcode and calling
939 * convention of the Midgard instruction; actual packing is done in
940 * emit_alu below */
941
942 unsigned op;
943
944 switch (instr->op) {
945 ALU_CASE(fadd, fadd);
946 ALU_CASE(fmul, fmul);
947 ALU_CASE(fmin, fmin);
948 ALU_CASE(fmax, fmax);
949 ALU_CASE(imin, imin);
950 ALU_CASE(imax, imax);
951 ALU_CASE(fmov, fmov);
952 ALU_CASE(ffloor, ffloor);
953 ALU_CASE(fround_even, froundeven);
954 ALU_CASE(ftrunc, ftrunc);
955 ALU_CASE(fceil, fceil);
956 ALU_CASE(fdot3, fdot3);
957 ALU_CASE(fdot4, fdot4);
958 ALU_CASE(iadd, iadd);
959 ALU_CASE(isub, isub);
960 ALU_CASE(imul, imul);
961
962 /* XXX: Use fmov, not imov, since imov was causing major
963 * issues with texture precision? XXX research */
964 ALU_CASE(imov, fmov);
965
966 ALU_CASE(feq, feq);
967 ALU_CASE(fne, fne);
968 ALU_CASE(flt, flt);
969 ALU_CASE(ieq, ieq);
970 ALU_CASE(ine, ine);
971 ALU_CASE(ilt, ilt);
972
973 ALU_CASE(frcp, frcp);
974 ALU_CASE(frsq, frsqrt);
975 ALU_CASE(fsqrt, fsqrt);
976 ALU_CASE(fexp2, fexp2);
977 ALU_CASE(flog2, flog2);
978
979 ALU_CASE(f2i32, f2i);
980 ALU_CASE(f2u32, f2u);
981 ALU_CASE(i2f32, i2f);
982 ALU_CASE(u2f32, u2f);
983
984 ALU_CASE(fsin, fsin);
985 ALU_CASE(fcos, fcos);
986
987 ALU_CASE(iand, iand);
988 ALU_CASE(ior, ior);
989 ALU_CASE(ixor, ixor);
990 ALU_CASE(inot, inot);
991 ALU_CASE(ishl, ishl);
992 ALU_CASE(ishr, iasr);
993 ALU_CASE(ushr, ilsr);
994
995 ALU_CASE(ball_fequal4, fball_eq);
996 ALU_CASE(bany_fnequal4, fbany_neq);
997 ALU_CASE(ball_iequal4, iball_eq);
998 ALU_CASE(bany_inequal4, ibany_neq);
999
1000 /* For greater-or-equal, we use less-or-equal and flip the
1001 * arguments */
1002
1003 case nir_op_ige: {
1004 op = midgard_alu_op_ile;
1005
1006 /* Swap via temporary */
1007 nir_alu_src temp = instr->src[1];
1008 instr->src[1] = instr->src[0];
1009 instr->src[0] = temp;
1010
1011 break;
1012 }
1013
1014 case nir_op_bcsel: {
1015 op = midgard_alu_op_fcsel;
1016
1017 /* csel works as a two-arg in Midgard, since the condition is hardcoded in r31.w */
1018 nr_inputs = 2;
1019
1020 emit_condition(ctx, &instr->src[0].src, false);
1021
1022 /* The condition is the first argument; move the other
1023 * arguments up one to be a binary instruction for
1024 * Midgard */
1025
1026 memmove(instr->src, instr->src + 1, 2 * sizeof(nir_alu_src));
1027 break;
1028 }
1029
1030 /* We don't have a native b2f32 instruction. Instead, like many GPUs,
1031 * we exploit booleans as 0/~0 for false/true, and correspondingly AND
1032 * by 1.0 to do the type conversion. For the moment, prime us to emit:
1033 *
1034 * iand [whatever], #0
1035 *
1036 * At the end of emit_alu (as MIR), we'll fix-up the constant */
1037
1038 case nir_op_b2f32: {
1039 op = midgard_alu_op_iand;
1040 break;
1041 }
1042
1043 default:
1044 printf("Unhandled ALU op %s\n", nir_op_infos[instr->op].name);
1045 assert(0);
1046 return;
1047 }
1048
1049 /* Fetch unit, quirks, etc information */
1050 unsigned opcode_props = alu_opcode_props[op];
1051 bool quirk_flipped_r24 = opcode_props & QUIRK_FLIPPED_R24;
1052
1053 /* Initialise fields common between scalar/vector instructions */
1054 midgard_outmod outmod = instr->dest.saturate ? midgard_outmod_sat : midgard_outmod_none;
1055
1056 /* src0 will always exist afaik, but src1 will not for 1-argument
1057 * instructions. The latter can only be fetched if the instruction
1058 * needs it, or else we may segfault. */
1059
1060 unsigned src0 = nir_alu_src_index(ctx, &instr->src[0]);
1061 unsigned src1 = nr_inputs == 2 ? nir_alu_src_index(ctx, &instr->src[1]) : SSA_UNUSED_0;
1062
1063 /* Rather than use the instruction generation helpers, we do it
1064 * ourselves here to avoid the mess */
1065
1066 midgard_instruction ins = {
1067 .type = TAG_ALU_4,
1068 .ssa_args = {
1069 .src0 = quirk_flipped_r24 ? SSA_UNUSED_1 : src0,
1070 .src1 = quirk_flipped_r24 ? src0 : src1,
1071 .dest = dest,
1072 }
1073 };
1074
1075 nir_alu_src *nirmods[2] = { NULL };
1076
1077 if (nr_inputs == 2) {
1078 nirmods[0] = &instr->src[0];
1079 nirmods[1] = &instr->src[1];
1080 } else if (nr_inputs == 1) {
1081 nirmods[quirk_flipped_r24] = &instr->src[0];
1082 } else {
1083 assert(0);
1084 }
1085
1086 midgard_vector_alu alu = {
1087 .op = op,
1088 .reg_mode = midgard_reg_mode_full,
1089 .dest_override = midgard_dest_override_none,
1090 .outmod = outmod,
1091
1092 /* Writemask only valid for non-SSA NIR */
1093 .mask = expand_writemask((1 << nr_components) - 1),
1094
1095 .src1 = vector_alu_srco_unsigned(vector_alu_modifiers(nirmods[0])),
1096 .src2 = vector_alu_srco_unsigned(vector_alu_modifiers(nirmods[1])),
1097 };
1098
1099 /* Apply writemask if non-SSA, keeping in mind that we can't write to components that don't exist */
1100
1101 if (!is_ssa)
1102 alu.mask &= expand_writemask(instr->dest.write_mask);
1103
1104 ins.alu = alu;
1105
1106 /* Late fixup for emulated instructions */
1107
1108 if (instr->op == nir_op_b2f32) {
1109 /* Presently, our second argument is an inline #0 constant.
1110 * Switch over to an embedded 1.0 constant (that can't fit
1111 * inline, since we're 32-bit, not 16-bit like the inline
1112 * constants) */
1113
1114 ins.ssa_args.inline_constant = false;
1115 ins.ssa_args.src1 = SSA_FIXED_REGISTER(REGISTER_CONSTANT);
1116 ins.has_constants = true;
1117 ins.constants[0] = 1.0;
1118 }
1119
1120 if ((opcode_props & UNITS_ALL) == UNIT_VLUT) {
1121 /* To avoid duplicating the lookup tables (probably), true LUT
1122 * instructions can only operate as if they were scalars. Lower
1123 * them here by changing the component. */
1124
1125 uint8_t original_swizzle[4];
1126 memcpy(original_swizzle, nirmods[0]->swizzle, sizeof(nirmods[0]->swizzle));
1127
1128 for (int i = 0; i < nr_components; ++i) {
1129 ins.alu.mask = (0x3) << (2 * i); /* Mask the associated component */
1130
1131 for (int j = 0; j < 4; ++j)
1132 nirmods[0]->swizzle[j] = original_swizzle[i]; /* Pull from the correct component */
1133
1134 ins.alu.src1 = vector_alu_srco_unsigned(vector_alu_modifiers(nirmods[0]));
1135 emit_mir_instruction(ctx, ins);
1136 }
1137 } else {
1138 emit_mir_instruction(ctx, ins);
1139 }
1140 }
1141
1142 #undef ALU_CASE
1143
1144 static void
1145 emit_intrinsic(compiler_context *ctx, nir_intrinsic_instr *instr)
1146 {
1147 nir_const_value *const_offset;
1148 unsigned offset, reg;
1149
1150 switch (instr->intrinsic) {
1151 case nir_intrinsic_discard_if:
1152 emit_condition(ctx, &instr->src[0], true);
1153
1154 /* fallthrough */
1155
1156 case nir_intrinsic_discard: {
1157 bool conditional = instr->intrinsic == nir_intrinsic_discard_if;
1158 struct midgard_instruction discard = v_branch(conditional, false);
1159 discard.branch.target_type = TARGET_DISCARD;
1160 emit_mir_instruction(ctx, discard);
1161
1162 ctx->can_discard = true;
1163 break;
1164 }
1165
1166 case nir_intrinsic_load_uniform:
1167 case nir_intrinsic_load_input:
1168 const_offset = nir_src_as_const_value(instr->src[0]);
1169 assert (const_offset && "no indirect inputs");
1170
1171 offset = nir_intrinsic_base(instr) + const_offset->u32[0];
1172
1173 reg = nir_dest_index(ctx, &instr->dest);
1174
1175 if (instr->intrinsic == nir_intrinsic_load_uniform && !ctx->is_blend) {
1176 /* TODO: half-floats */
1177
1178 int uniform_offset = 0;
1179
1180 if (offset >= SPECIAL_UNIFORM_BASE) {
1181 /* XXX: Resolve which uniform */
1182 uniform_offset = 0;
1183 } else {
1184 /* Offset away from the special
1185 * uniform block */
1186
1187 void *entry = _mesa_hash_table_u64_search(ctx->uniform_nir_to_mdg, offset + 1);
1188
1189 /* XXX */
1190 if (!entry) {
1191 printf("WARNING: Unknown uniform %d\n", offset);
1192 break;
1193 }
1194
1195 uniform_offset = (uintptr_t) (entry) - 1;
1196 uniform_offset += ctx->special_uniforms;
1197 }
1198
1199 if (uniform_offset < ctx->uniform_cutoff) {
1200 /* Fast path: For the first 16 uniform,
1201 * accesses are 0-cycle, since they're
1202 * just a register fetch in the usual
1203 * case. So, we alias the registers
1204 * while we're still in SSA-space */
1205
1206 int reg_slot = 23 - uniform_offset;
1207 alias_ssa(ctx, reg, SSA_FIXED_REGISTER(reg_slot));
1208 } else {
1209 /* Otherwise, read from the 'special'
1210 * UBO to access higher-indexed
1211 * uniforms, at a performance cost */
1212
1213 midgard_instruction ins = m_load_uniform_32(reg, uniform_offset);
1214
1215 /* TODO: Don't split */
1216 ins.load_store.varying_parameters = (uniform_offset & 7) << 7;
1217 ins.load_store.address = uniform_offset >> 3;
1218
1219 ins.load_store.unknown = 0x1E00; /* xxx: what is this? */
1220 emit_mir_instruction(ctx, ins);
1221 }
1222 } else if (ctx->stage == MESA_SHADER_FRAGMENT && !ctx->is_blend) {
1223 /* XXX: Half-floats? */
1224 /* TODO: swizzle, mask */
1225
1226 midgard_instruction ins = m_load_vary_32(reg, offset);
1227
1228 midgard_varying_parameter p = {
1229 .is_varying = 1,
1230 .interpolation = midgard_interp_default,
1231 .flat = /*var->data.interpolation == INTERP_MODE_FLAT*/ 0
1232 };
1233
1234 unsigned u;
1235 memcpy(&u, &p, sizeof(p));
1236 ins.load_store.varying_parameters = u;
1237
1238 ins.load_store.unknown = 0x1e9e; /* xxx: what is this? */
1239 emit_mir_instruction(ctx, ins);
1240 } else if (ctx->is_blend && instr->intrinsic == nir_intrinsic_load_uniform) {
1241 /* Constant encoded as a pinned constant */
1242
1243 midgard_instruction ins = v_fmov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), blank_alu_src, reg);
1244 ins.has_constants = true;
1245 ins.has_blend_constant = true;
1246 emit_mir_instruction(ctx, ins);
1247 } else if (ctx->is_blend) {
1248 /* For blend shaders, a load might be
1249 * translated various ways depending on what
1250 * we're loading. Figure out how this is used */
1251
1252 nir_variable *out = NULL;
1253
1254 nir_foreach_variable(var, &ctx->nir->inputs) {
1255 int drvloc = var->data.driver_location;
1256
1257 if (nir_intrinsic_base(instr) == drvloc) {
1258 out = var;
1259 break;
1260 }
1261 }
1262
1263 assert(out);
1264
1265 if (out->data.location == VARYING_SLOT_COL0) {
1266 /* Source color preloaded to r0 */
1267
1268 midgard_pin_output(ctx, reg, 0);
1269 } else if (out->data.location == VARYING_SLOT_COL1) {
1270 /* Destination color must be read from framebuffer */
1271
1272 midgard_instruction ins = m_load_color_buffer_8(reg, 0);
1273 ins.load_store.swizzle = 0; /* xxxx */
1274
1275 /* Read each component sequentially */
1276
1277 for (int c = 0; c < 4; ++c) {
1278 ins.load_store.mask = (1 << c);
1279 ins.load_store.unknown = c;
1280 emit_mir_instruction(ctx, ins);
1281 }
1282
1283 /* vadd.u2f hr2, abs(hr2), #0 */
1284
1285 midgard_vector_alu_src alu_src = blank_alu_src;
1286 alu_src.abs = true;
1287 alu_src.half = true;
1288
1289 midgard_instruction u2f = {
1290 .type = TAG_ALU_4,
1291 .ssa_args = {
1292 .src0 = reg,
1293 .src1 = SSA_UNUSED_0,
1294 .dest = reg,
1295 .inline_constant = true
1296 },
1297 .alu = {
1298 .op = midgard_alu_op_u2f,
1299 .reg_mode = midgard_reg_mode_half,
1300 .dest_override = midgard_dest_override_none,
1301 .mask = 0xF,
1302 .src1 = vector_alu_srco_unsigned(alu_src),
1303 .src2 = vector_alu_srco_unsigned(blank_alu_src),
1304 }
1305 };
1306
1307 emit_mir_instruction(ctx, u2f);
1308
1309 /* vmul.fmul.sat r1, hr2, #0.00392151 */
1310
1311 alu_src.abs = false;
1312
1313 midgard_instruction fmul = {
1314 .type = TAG_ALU_4,
1315 .inline_constant = _mesa_float_to_half(1.0 / 255.0),
1316 .ssa_args = {
1317 .src0 = reg,
1318 .dest = reg,
1319 .src1 = SSA_UNUSED_0,
1320 .inline_constant = true
1321 },
1322 .alu = {
1323 .op = midgard_alu_op_fmul,
1324 .reg_mode = midgard_reg_mode_full,
1325 .dest_override = midgard_dest_override_none,
1326 .outmod = midgard_outmod_sat,
1327 .mask = 0xFF,
1328 .src1 = vector_alu_srco_unsigned(alu_src),
1329 .src2 = vector_alu_srco_unsigned(blank_alu_src),
1330 }
1331 };
1332
1333 emit_mir_instruction(ctx, fmul);
1334 } else {
1335 printf("Unknown input in blend shader\n");
1336 assert(0);
1337 }
1338 } else if (ctx->stage == MESA_SHADER_VERTEX) {
1339 midgard_instruction ins = m_load_attr_32(reg, offset);
1340 ins.load_store.unknown = 0x1E1E; /* XXX: What is this? */
1341 ins.load_store.mask = (1 << instr->num_components) - 1;
1342 emit_mir_instruction(ctx, ins);
1343 } else {
1344 printf("Unknown load\n");
1345 assert(0);
1346 }
1347
1348 break;
1349
1350 case nir_intrinsic_store_output:
1351 const_offset = nir_src_as_const_value(instr->src[1]);
1352 assert(const_offset && "no indirect outputs");
1353
1354 offset = nir_intrinsic_base(instr) + const_offset->u32[0];
1355
1356 reg = nir_src_index(ctx, &instr->src[0]);
1357
1358 if (ctx->stage == MESA_SHADER_FRAGMENT) {
1359 /* gl_FragColor is not emitted with load/store
1360 * instructions. Instead, it gets plonked into
1361 * r0 at the end of the shader and we do the
1362 * framebuffer writeout dance. TODO: Defer
1363 * writes */
1364
1365 midgard_pin_output(ctx, reg, 0);
1366
1367 /* Save the index we're writing to for later reference
1368 * in the epilogue */
1369
1370 ctx->fragment_output = reg;
1371 } else if (ctx->stage == MESA_SHADER_VERTEX) {
1372 /* Varyings are written into one of two special
1373 * varying register, r26 or r27. The register itself is selected as the register
1374 * in the st_vary instruction, minus the base of 26. E.g. write into r27 and then call st_vary(1)
1375 *
1376 * Normally emitting fmov's is frowned upon,
1377 * but due to unique constraints of
1378 * REGISTER_VARYING, fmov emission + a
1379 * dedicated cleanup pass is the only way to
1380 * guarantee correctness when considering some
1381 * (common) edge cases XXX: FIXME */
1382
1383 /* Look up how it was actually laid out */
1384
1385 void *entry = _mesa_hash_table_u64_search(ctx->varying_nir_to_mdg, offset + 1);
1386
1387 if (!entry) {
1388 printf("WARNING: skipping varying\n");
1389 break;
1390 }
1391
1392 offset = (uintptr_t) (entry) - 1;
1393
1394 /* If this varying corresponds to a constant (why?!),
1395 * emit that now since it won't get picked up by
1396 * hoisting (since there is no corresponding move
1397 * emitted otherwise) */
1398
1399 void *constant_value = _mesa_hash_table_u64_search(ctx->ssa_constants, reg + 1);
1400
1401 if (constant_value) {
1402 /* Special case: emit the varying write
1403 * directly to r26 (looks funny in asm but it's
1404 * fine) and emit the store _now_. Possibly
1405 * slightly slower, but this is a really stupid
1406 * special case anyway (why on earth would you
1407 * have a constant varying? Your own fault for
1408 * slightly worse perf :P) */
1409
1410 midgard_instruction ins = v_fmov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), blank_alu_src, SSA_FIXED_REGISTER(26));
1411 attach_constants(ctx, &ins, constant_value, reg + 1);
1412 emit_mir_instruction(ctx, ins);
1413
1414 midgard_instruction st = m_store_vary_32(SSA_FIXED_REGISTER(0), offset);
1415 st.load_store.unknown = 0x1E9E; /* XXX: What is this? */
1416 emit_mir_instruction(ctx, st);
1417 } else {
1418 /* Do not emit the varying yet -- instead, just mark down that we need to later */
1419
1420 _mesa_hash_table_u64_insert(ctx->ssa_varyings, reg + 1, (void *) ((uintptr_t) (offset + 1)));
1421 }
1422 } else {
1423 printf("Unknown store\n");
1424 assert(0);
1425 }
1426
1427 break;
1428
1429 case nir_intrinsic_load_alpha_ref_float:
1430 assert(instr->dest.is_ssa);
1431
1432 float ref_value = ctx->alpha_ref;
1433
1434 float *v = ralloc_array(NULL, float, 4);
1435 memcpy(v, &ref_value, sizeof(float));
1436 _mesa_hash_table_u64_insert(ctx->ssa_constants, instr->dest.ssa.index + 1, v);
1437 break;
1438
1439
1440 default:
1441 printf ("Unhandled intrinsic\n");
1442 assert(0);
1443 break;
1444 }
1445 }
1446
1447 static unsigned
1448 midgard_tex_format(enum glsl_sampler_dim dim)
1449 {
1450 switch (dim) {
1451 case GLSL_SAMPLER_DIM_2D:
1452 case GLSL_SAMPLER_DIM_EXTERNAL:
1453 return TEXTURE_2D;
1454
1455 case GLSL_SAMPLER_DIM_3D:
1456 return TEXTURE_3D;
1457
1458 case GLSL_SAMPLER_DIM_CUBE:
1459 return TEXTURE_CUBE;
1460
1461 default:
1462 printf("Unknown sampler dim type\n");
1463 assert(0);
1464 return 0;
1465 }
1466 }
1467
1468 static void
1469 emit_tex(compiler_context *ctx, nir_tex_instr *instr)
1470 {
1471 /* TODO */
1472 //assert (!instr->sampler);
1473 //assert (!instr->texture_array_size);
1474 assert (instr->op == nir_texop_tex);
1475
1476 /* Allocate registers via a round robin scheme to alternate between the two registers */
1477 int reg = ctx->texture_op_count & 1;
1478 int in_reg = reg, out_reg = reg;
1479
1480 /* Make room for the reg */
1481
1482 if (ctx->texture_index[reg] > -1)
1483 unalias_ssa(ctx, ctx->texture_index[reg]);
1484
1485 int texture_index = instr->texture_index;
1486 int sampler_index = texture_index;
1487
1488 for (unsigned i = 0; i < instr->num_srcs; ++i) {
1489 switch (instr->src[i].src_type) {
1490 case nir_tex_src_coord: {
1491 int index = nir_src_index(ctx, &instr->src[i].src);
1492
1493 midgard_vector_alu_src alu_src = blank_alu_src;
1494 alu_src.swizzle = (COMPONENT_Y << 2);
1495
1496 midgard_instruction ins = v_fmov(index, alu_src, SSA_FIXED_REGISTER(REGISTER_TEXTURE_BASE + in_reg));
1497 emit_mir_instruction(ctx, ins);
1498
1499 //midgard_pin_output(ctx, index, REGISTER_TEXTURE_BASE + in_reg);
1500
1501 break;
1502 }
1503
1504 default: {
1505 printf("Unknown source type\n");
1506 //assert(0);
1507 break;
1508 }
1509 }
1510 }
1511
1512 /* No helper to build texture words -- we do it all here */
1513 midgard_instruction ins = {
1514 .type = TAG_TEXTURE_4,
1515 .texture = {
1516 .op = TEXTURE_OP_NORMAL,
1517 .format = midgard_tex_format(instr->sampler_dim),
1518 .texture_handle = texture_index,
1519 .sampler_handle = sampler_index,
1520
1521 /* TODO: Don't force xyzw */
1522 .swizzle = SWIZZLE(COMPONENT_X, COMPONENT_Y, COMPONENT_Z, COMPONENT_W),
1523 .mask = 0xF,
1524
1525 /* TODO: half */
1526 //.in_reg_full = 1,
1527 .out_full = 1,
1528
1529 .filter = 1,
1530
1531 /* Always 1 */
1532 .unknown7 = 1,
1533
1534 /* Assume we can continue; hint it out later */
1535 .cont = 1,
1536 }
1537 };
1538
1539 /* Set registers to read and write from the same place */
1540 ins.texture.in_reg_select = in_reg;
1541 ins.texture.out_reg_select = out_reg;
1542
1543 /* TODO: Dynamic swizzle input selection, half-swizzles? */
1544 if (instr->sampler_dim == GLSL_SAMPLER_DIM_3D) {
1545 ins.texture.in_reg_swizzle_right = COMPONENT_X;
1546 ins.texture.in_reg_swizzle_left = COMPONENT_Y;
1547 //ins.texture.in_reg_swizzle_third = COMPONENT_Z;
1548 } else {
1549 ins.texture.in_reg_swizzle_left = COMPONENT_X;
1550 ins.texture.in_reg_swizzle_right = COMPONENT_Y;
1551 //ins.texture.in_reg_swizzle_third = COMPONENT_X;
1552 }
1553
1554 emit_mir_instruction(ctx, ins);
1555
1556 /* Simultaneously alias the destination and emit a move for it. The move will be eliminated if possible */
1557
1558 int o_reg = REGISTER_TEXTURE_BASE + out_reg, o_index = nir_dest_index(ctx, &instr->dest);
1559 alias_ssa(ctx, o_index, SSA_FIXED_REGISTER(o_reg));
1560 ctx->texture_index[reg] = o_index;
1561
1562 midgard_instruction ins2 = v_fmov(SSA_FIXED_REGISTER(o_reg), blank_alu_src, o_index);
1563 emit_mir_instruction(ctx, ins2);
1564
1565 /* Used for .cont and .last hinting */
1566 ctx->texture_op_count++;
1567 }
1568
1569 static void
1570 emit_jump(compiler_context *ctx, nir_jump_instr *instr)
1571 {
1572 switch (instr->type) {
1573 case nir_jump_break: {
1574 /* Emit a branch out of the loop */
1575 struct midgard_instruction br = v_branch(false, false);
1576 br.branch.target_type = TARGET_BREAK;
1577 br.branch.target_break = ctx->current_loop;
1578 emit_mir_instruction(ctx, br);
1579
1580 printf("break..\n");
1581 break;
1582 }
1583
1584 default:
1585 printf("Unknown jump type %d\n", instr->type);
1586 break;
1587 }
1588 }
1589
1590 static void
1591 emit_instr(compiler_context *ctx, struct nir_instr *instr)
1592 {
1593 switch (instr->type) {
1594 case nir_instr_type_load_const:
1595 emit_load_const(ctx, nir_instr_as_load_const(instr));
1596 break;
1597
1598 case nir_instr_type_intrinsic:
1599 emit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
1600 break;
1601
1602 case nir_instr_type_alu:
1603 emit_alu(ctx, nir_instr_as_alu(instr));
1604 break;
1605
1606 case nir_instr_type_tex:
1607 emit_tex(ctx, nir_instr_as_tex(instr));
1608 break;
1609
1610 case nir_instr_type_jump:
1611 emit_jump(ctx, nir_instr_as_jump(instr));
1612 break;
1613
1614 case nir_instr_type_ssa_undef:
1615 /* Spurious */
1616 break;
1617
1618 default:
1619 printf("Unhandled instruction type\n");
1620 break;
1621 }
1622 }
1623
1624 /* Determine the actual hardware from the index based on the RA results or special values */
1625
1626 static int
1627 dealias_register(compiler_context *ctx, struct ra_graph *g, int reg, int maxreg)
1628 {
1629 if (reg >= SSA_FIXED_MINIMUM)
1630 return SSA_REG_FROM_FIXED(reg);
1631
1632 if (reg >= 0) {
1633 assert(reg < maxreg);
1634 int r = ra_get_node_reg(g, reg);
1635 ctx->work_registers = MAX2(ctx->work_registers, r);
1636 return r;
1637 }
1638
1639 switch (reg) {
1640 /* fmov style unused */
1641 case SSA_UNUSED_0:
1642 return REGISTER_UNUSED;
1643
1644 /* lut style unused */
1645 case SSA_UNUSED_1:
1646 return REGISTER_UNUSED;
1647
1648 default:
1649 printf("Unknown SSA register alias %d\n", reg);
1650 assert(0);
1651 return 31;
1652 }
1653 }
1654
1655 static unsigned int
1656 midgard_ra_select_callback(struct ra_graph *g, BITSET_WORD *regs, void *data)
1657 {
1658 /* Choose the first available register to minimise reported register pressure */
1659
1660 for (int i = 0; i < 16; ++i) {
1661 if (BITSET_TEST(regs, i)) {
1662 return i;
1663 }
1664 }
1665
1666 assert(0);
1667 return 0;
1668 }
1669
1670 static bool
1671 midgard_is_live_in_instr(midgard_instruction *ins, int src)
1672 {
1673 if (ins->ssa_args.src0 == src) return true;
1674 if (ins->ssa_args.src1 == src) return true;
1675
1676 return false;
1677 }
1678
1679 static bool
1680 is_live_after(compiler_context *ctx, midgard_block *block, midgard_instruction *start, int src)
1681 {
1682 /* Check the rest of the block for liveness */
1683 mir_foreach_instr_in_block_from(block, ins, mir_next_op(start)) {
1684 if (midgard_is_live_in_instr(ins, src))
1685 return true;
1686 }
1687
1688 /* Check the rest of the blocks for liveness */
1689 mir_foreach_block_from(ctx, mir_next_block(block), b) {
1690 mir_foreach_instr_in_block(b, ins) {
1691 if (midgard_is_live_in_instr(ins, src))
1692 return true;
1693 }
1694 }
1695
1696 /* TODO: How does control flow interact in complex shaders? */
1697
1698 return false;
1699 }
1700
1701 static void
1702 allocate_registers(compiler_context *ctx)
1703 {
1704 /* First, initialize the RA */
1705 struct ra_regs *regs = ra_alloc_reg_set(NULL, 32, true);
1706
1707 /* Create a primary (general purpose) class, as well as special purpose
1708 * pipeline register classes */
1709
1710 int primary_class = ra_alloc_reg_class(regs);
1711 int varying_class = ra_alloc_reg_class(regs);
1712
1713 /* Add the full set of work registers */
1714 int work_count = 16 - MAX2((ctx->uniform_cutoff - 8), 0);
1715 for (int i = 0; i < work_count; ++i)
1716 ra_class_add_reg(regs, primary_class, i);
1717
1718 /* Add special registers */
1719 ra_class_add_reg(regs, varying_class, REGISTER_VARYING_BASE);
1720 ra_class_add_reg(regs, varying_class, REGISTER_VARYING_BASE + 1);
1721
1722 /* We're done setting up */
1723 ra_set_finalize(regs, NULL);
1724
1725 /* Transform the MIR into squeezed index form */
1726 mir_foreach_block(ctx, block) {
1727 mir_foreach_instr_in_block(block, ins) {
1728 if (ins->compact_branch) continue;
1729
1730 ins->ssa_args.src0 = find_or_allocate_temp(ctx, ins->ssa_args.src0);
1731 ins->ssa_args.src1 = find_or_allocate_temp(ctx, ins->ssa_args.src1);
1732 ins->ssa_args.dest = find_or_allocate_temp(ctx, ins->ssa_args.dest);
1733 }
1734
1735 print_mir_block(block);
1736 }
1737
1738 /* Let's actually do register allocation */
1739 int nodes = ctx->temp_count;
1740 struct ra_graph *g = ra_alloc_interference_graph(regs, nodes);
1741
1742 /* Set everything to the work register class, unless it has somewhere
1743 * special to go */
1744
1745 mir_foreach_block(ctx, block) {
1746 mir_foreach_instr_in_block(block, ins) {
1747 if (ins->compact_branch) continue;
1748
1749 if (ins->ssa_args.dest < 0) continue;
1750
1751 if (ins->ssa_args.dest >= SSA_FIXED_MINIMUM) continue;
1752
1753 int class = primary_class;
1754
1755 ra_set_node_class(g, ins->ssa_args.dest, class);
1756 }
1757 }
1758
1759 for (int index = 0; index <= ctx->max_hash; ++index) {
1760 unsigned temp = (uintptr_t) _mesa_hash_table_u64_search(ctx->ssa_to_register, index + 1);
1761
1762 if (temp) {
1763 unsigned reg = temp - 1;
1764 int t = find_or_allocate_temp(ctx, index);
1765 ra_set_node_reg(g, t, reg);
1766 }
1767 }
1768
1769 /* Determine liveness */
1770
1771 int *live_start = malloc(nodes * sizeof(int));
1772 int *live_end = malloc(nodes * sizeof(int));
1773
1774 /* Initialize as non-existent */
1775
1776 for (int i = 0; i < nodes; ++i) {
1777 live_start[i] = live_end[i] = -1;
1778 }
1779
1780 int d = 0;
1781
1782 mir_foreach_block(ctx, block) {
1783 mir_foreach_instr_in_block(block, ins) {
1784 if (ins->compact_branch) continue;
1785
1786 if (ins->ssa_args.dest < SSA_FIXED_MINIMUM) {
1787 /* If this destination is not yet live, it is now since we just wrote it */
1788
1789 int dest = ins->ssa_args.dest;
1790
1791 if (live_start[dest] == -1)
1792 live_start[dest] = d;
1793 }
1794
1795 /* Since we just used a source, the source might be
1796 * dead now. Scan the rest of the block for
1797 * invocations, and if there are none, the source dies
1798 * */
1799
1800 int sources[2] = { ins->ssa_args.src0, ins->ssa_args.src1 };
1801
1802 for (int src = 0; src < 2; ++src) {
1803 int s = sources[src];
1804
1805 if (s < 0) continue;
1806
1807 if (s >= SSA_FIXED_MINIMUM) continue;
1808
1809 if (!is_live_after(ctx, block, ins, s)) {
1810 live_end[s] = d;
1811 }
1812 }
1813
1814 ++d;
1815 }
1816 }
1817
1818 /* If a node still hasn't been killed, kill it now */
1819
1820 for (int i = 0; i < nodes; ++i) {
1821 /* live_start == -1 most likely indicates a pinned output */
1822
1823 if (live_end[i] == -1)
1824 live_end[i] = d;
1825 }
1826
1827 /* Setup interference between nodes that are live at the same time */
1828
1829 for (int i = 0; i < nodes; ++i) {
1830 for (int j = i + 1; j < nodes; ++j) {
1831 if (!(live_start[i] >= live_end[j] || live_start[j] >= live_end[i]))
1832 ra_add_node_interference(g, i, j);
1833 }
1834 }
1835
1836 ra_set_select_reg_callback(g, midgard_ra_select_callback, NULL);
1837
1838 if (!ra_allocate(g)) {
1839 printf("Error allocating registers\n");
1840 assert(0);
1841 }
1842
1843 /* Cleanup */
1844 free(live_start);
1845 free(live_end);
1846
1847 mir_foreach_block(ctx, block) {
1848 mir_foreach_instr_in_block(block, ins) {
1849 if (ins->compact_branch) continue;
1850
1851 ssa_args args = ins->ssa_args;
1852
1853 switch (ins->type) {
1854 case TAG_ALU_4:
1855 ins->registers.src1_reg = dealias_register(ctx, g, args.src0, nodes);
1856
1857 ins->registers.src2_imm = args.inline_constant;
1858
1859 if (args.inline_constant) {
1860 /* Encode inline 16-bit constant as a vector by default */
1861
1862 ins->registers.src2_reg = ins->inline_constant >> 11;
1863
1864 int lower_11 = ins->inline_constant & ((1 << 12) - 1);
1865
1866 uint16_t imm = ((lower_11 >> 8) & 0x7) | ((lower_11 & 0xFF) << 3);
1867 ins->alu.src2 = imm << 2;
1868 } else {
1869 ins->registers.src2_reg = dealias_register(ctx, g, args.src1, nodes);
1870 }
1871
1872 ins->registers.out_reg = dealias_register(ctx, g, args.dest, nodes);
1873
1874 break;
1875
1876 case TAG_LOAD_STORE_4: {
1877 if (OP_IS_STORE(ins->load_store.op)) {
1878 /* TODO: use ssa_args for store_vary */
1879 ins->load_store.reg = 0;
1880 } else {
1881 bool has_dest = args.dest >= 0;
1882 int ssa_arg = has_dest ? args.dest : args.src0;
1883
1884 ins->load_store.reg = dealias_register(ctx, g, ssa_arg, nodes);
1885 }
1886
1887 break;
1888 }
1889
1890 default:
1891 break;
1892 }
1893 }
1894 }
1895 }
1896
1897 /* Midgard IR only knows vector ALU types, but we sometimes need to actually
1898 * use scalar ALU instructions, for functional or performance reasons. To do
1899 * this, we just demote vector ALU payloads to scalar. */
1900
1901 static int
1902 component_from_mask(unsigned mask)
1903 {
1904 for (int c = 0; c < 4; ++c) {
1905 if (mask & (3 << (2 * c)))
1906 return c;
1907 }
1908
1909 assert(0);
1910 return 0;
1911 }
1912
1913 static bool
1914 is_single_component_mask(unsigned mask)
1915 {
1916 int components = 0;
1917
1918 for (int c = 0; c < 4; ++c)
1919 if (mask & (3 << (2 * c)))
1920 components++;
1921
1922 return components == 1;
1923 }
1924
1925 /* Create a mask of accessed components from a swizzle to figure out vector
1926 * dependencies */
1927
1928 static unsigned
1929 swizzle_to_access_mask(unsigned swizzle)
1930 {
1931 unsigned component_mask = 0;
1932
1933 for (int i = 0; i < 4; ++i) {
1934 unsigned c = (swizzle >> (2 * i)) & 3;
1935 component_mask |= (1 << c);
1936 }
1937
1938 return component_mask;
1939 }
1940
1941 static unsigned
1942 vector_to_scalar_source(unsigned u)
1943 {
1944 midgard_vector_alu_src v;
1945 memcpy(&v, &u, sizeof(v));
1946
1947 midgard_scalar_alu_src s = {
1948 .abs = v.abs,
1949 .negate = v.negate,
1950 .full = !v.half,
1951 .component = (v.swizzle & 3) << 1
1952 };
1953
1954 unsigned o;
1955 memcpy(&o, &s, sizeof(s));
1956
1957 return o & ((1 << 6) - 1);
1958 }
1959
1960 static midgard_scalar_alu
1961 vector_to_scalar_alu(midgard_vector_alu v, midgard_instruction *ins)
1962 {
1963 /* The output component is from the mask */
1964 midgard_scalar_alu s = {
1965 .op = v.op,
1966 .src1 = vector_to_scalar_source(v.src1),
1967 .src2 = vector_to_scalar_source(v.src2),
1968 .unknown = 0,
1969 .outmod = v.outmod,
1970 .output_full = 1, /* TODO: Half */
1971 .output_component = component_from_mask(v.mask) << 1,
1972 };
1973
1974 /* Inline constant is passed along rather than trying to extract it
1975 * from v */
1976
1977 if (ins->ssa_args.inline_constant) {
1978 uint16_t imm = 0;
1979 int lower_11 = ins->inline_constant & ((1 << 12) - 1);
1980 imm |= (lower_11 >> 9) & 3;
1981 imm |= (lower_11 >> 6) & 4;
1982 imm |= (lower_11 >> 2) & 0x38;
1983 imm |= (lower_11 & 63) << 6;
1984
1985 s.src2 = imm;
1986 }
1987
1988 return s;
1989 }
1990
1991 /* Midgard prefetches instruction types, so during emission we need to
1992 * lookahead too. Unless this is the last instruction, in which we return 1. Or
1993 * if this is the second to last and the last is an ALU, then it's also 1... */
1994
1995 #define IS_ALU(tag) (tag == TAG_ALU_4 || tag == TAG_ALU_8 || \
1996 tag == TAG_ALU_12 || tag == TAG_ALU_16)
1997
1998 #define EMIT_AND_COUNT(type, val) util_dynarray_append(emission, type, val); \
1999 bytes_emitted += sizeof(type)
2000
2001 static void
2002 emit_binary_vector_instruction(midgard_instruction *ains,
2003 uint16_t *register_words, int *register_words_count,
2004 uint64_t *body_words, size_t *body_size, int *body_words_count,
2005 size_t *bytes_emitted)
2006 {
2007 memcpy(&register_words[(*register_words_count)++], &ains->registers, sizeof(ains->registers));
2008 *bytes_emitted += sizeof(midgard_reg_info);
2009
2010 body_size[*body_words_count] = sizeof(midgard_vector_alu);
2011 memcpy(&body_words[(*body_words_count)++], &ains->alu, sizeof(ains->alu));
2012 *bytes_emitted += sizeof(midgard_vector_alu);
2013 }
2014
2015 /* Checks for an SSA data hazard between two adjacent instructions, keeping in
2016 * mind that we are a vector architecture and we can write to different
2017 * components simultaneously */
2018
2019 static bool
2020 can_run_concurrent_ssa(midgard_instruction *first, midgard_instruction *second)
2021 {
2022 /* Each instruction reads some registers and writes to a register. See
2023 * where the first writes */
2024
2025 /* Figure out where exactly we wrote to */
2026 int source = first->ssa_args.dest;
2027 int source_mask = first->type == TAG_ALU_4 ? squeeze_writemask(first->alu.mask) : 0xF;
2028
2029 /* As long as the second doesn't read from the first, we're okay */
2030 if (second->ssa_args.src0 == source) {
2031 if (first->type == TAG_ALU_4) {
2032 /* Figure out which components we just read from */
2033
2034 int q = second->alu.src1;
2035 midgard_vector_alu_src *m = (midgard_vector_alu_src *) &q;
2036
2037 /* Check if there are components in common, and fail if so */
2038 if (swizzle_to_access_mask(m->swizzle) & source_mask)
2039 return false;
2040 } else
2041 return false;
2042
2043 }
2044
2045 if (second->ssa_args.src1 == source)
2046 return false;
2047
2048 /* Otherwise, it's safe in that regard. Another data hazard is both
2049 * writing to the same place, of course */
2050
2051 if (second->ssa_args.dest == source) {
2052 /* ...but only if the components overlap */
2053 int dest_mask = second->type == TAG_ALU_4 ? squeeze_writemask(second->alu.mask) : 0xF;
2054
2055 if (dest_mask & source_mask)
2056 return false;
2057 }
2058
2059 /* ...That's it */
2060 return true;
2061 }
2062
2063 static bool
2064 midgard_has_hazard(
2065 midgard_instruction **segment, unsigned segment_size,
2066 midgard_instruction *ains)
2067 {
2068 for (int s = 0; s < segment_size; ++s)
2069 if (!can_run_concurrent_ssa(segment[s], ains))
2070 return true;
2071
2072 return false;
2073
2074
2075 }
2076
2077 /* Schedules, but does not emit, a single basic block. After scheduling, the
2078 * final tag and size of the block are known, which are necessary for branching
2079 * */
2080
2081 static midgard_bundle
2082 schedule_bundle(compiler_context *ctx, midgard_block *block, midgard_instruction *ins, int *skip)
2083 {
2084 int instructions_emitted = 0, instructions_consumed = -1;
2085 midgard_bundle bundle = { 0 };
2086
2087 uint8_t tag = ins->type;
2088
2089 /* Default to the instruction's tag */
2090 bundle.tag = tag;
2091
2092 switch (ins->type) {
2093 case TAG_ALU_4: {
2094 uint32_t control = 0;
2095 size_t bytes_emitted = sizeof(control);
2096
2097 /* TODO: Constant combining */
2098 int index = 0, last_unit = 0;
2099
2100 /* Previous instructions, for the purpose of parallelism */
2101 midgard_instruction *segment[4] = {0};
2102 int segment_size = 0;
2103
2104 instructions_emitted = -1;
2105 midgard_instruction *pins = ins;
2106
2107 for (;;) {
2108 midgard_instruction *ains = pins;
2109
2110 /* Advance instruction pointer */
2111 if (index) {
2112 ains = mir_next_op(pins);
2113 pins = ains;
2114 }
2115
2116 /* Out-of-work condition */
2117 if ((struct list_head *) ains == &block->instructions)
2118 break;
2119
2120 /* Ensure that the chain can continue */
2121 if (ains->type != TAG_ALU_4) break;
2122
2123 /* According to the presentation "The ARM
2124 * Mali-T880 Mobile GPU" from HotChips 27,
2125 * there are two pipeline stages. Branching
2126 * position determined experimentally. Lines
2127 * are executed in parallel:
2128 *
2129 * [ VMUL ] [ SADD ]
2130 * [ VADD ] [ SMUL ] [ LUT ] [ BRANCH ]
2131 *
2132 * Verify that there are no ordering dependencies here.
2133 *
2134 * TODO: Allow for parallelism!!!
2135 */
2136
2137 /* Pick a unit for it if it doesn't force a particular unit */
2138
2139 int unit = ains->unit;
2140
2141 if (!unit) {
2142 int op = ains->alu.op;
2143 int units = alu_opcode_props[op];
2144
2145 /* TODO: Promotion of scalars to vectors */
2146 int vector = ((!is_single_component_mask(ains->alu.mask)) || ((units & UNITS_SCALAR) == 0)) && (units & UNITS_ANY_VECTOR);
2147
2148 if (!vector)
2149 assert(units & UNITS_SCALAR);
2150
2151 if (vector) {
2152 if (last_unit >= UNIT_VADD) {
2153 if (units & UNIT_VLUT)
2154 unit = UNIT_VLUT;
2155 else
2156 break;
2157 } else {
2158 if ((units & UNIT_VMUL) && !(control & UNIT_VMUL))
2159 unit = UNIT_VMUL;
2160 else if ((units & UNIT_VADD) && !(control & UNIT_VADD))
2161 unit = UNIT_VADD;
2162 else if (units & UNIT_VLUT)
2163 unit = UNIT_VLUT;
2164 else
2165 break;
2166 }
2167 } else {
2168 if (last_unit >= UNIT_VADD) {
2169 if ((units & UNIT_SMUL) && !(control & UNIT_SMUL))
2170 unit = UNIT_SMUL;
2171 else if (units & UNIT_VLUT)
2172 unit = UNIT_VLUT;
2173 else
2174 break;
2175 } else {
2176 if ((units & UNIT_SADD) && !(control & UNIT_SADD) && !midgard_has_hazard(segment, segment_size, ains))
2177 unit = UNIT_SADD;
2178 else if (units & UNIT_SMUL)
2179 unit = ((units & UNIT_VMUL) && !(control & UNIT_VMUL)) ? UNIT_VMUL : UNIT_SMUL;
2180 else if ((units & UNIT_VADD) && !(control & UNIT_VADD))
2181 unit = UNIT_VADD;
2182 else
2183 break;
2184 }
2185 }
2186
2187 assert(unit & units);
2188 }
2189
2190 /* Late unit check, this time for encoding (not parallelism) */
2191 if (unit <= last_unit) break;
2192
2193 /* Clear the segment */
2194 if (last_unit < UNIT_VADD && unit >= UNIT_VADD)
2195 segment_size = 0;
2196
2197 if (midgard_has_hazard(segment, segment_size, ains))
2198 break;
2199
2200 /* We're good to go -- emit the instruction */
2201 ains->unit = unit;
2202
2203 segment[segment_size++] = ains;
2204
2205 /* Only one set of embedded constants per
2206 * bundle possible; if we have more, we must
2207 * break the chain early, unfortunately */
2208
2209 if (ains->has_constants) {
2210 if (bundle.has_embedded_constants) {
2211 /* ...but if there are already
2212 * constants but these are the
2213 * *same* constants, we let it
2214 * through */
2215
2216 if (memcmp(bundle.constants, ains->constants, sizeof(bundle.constants)))
2217 break;
2218 } else {
2219 bundle.has_embedded_constants = true;
2220 memcpy(bundle.constants, ains->constants, sizeof(bundle.constants));
2221
2222 /* If this is a blend shader special constant, track it for patching */
2223 if (ains->has_blend_constant)
2224 bundle.has_blend_constant = true;
2225 }
2226 }
2227
2228 if (ains->unit & UNITS_ANY_VECTOR) {
2229 emit_binary_vector_instruction(ains, bundle.register_words,
2230 &bundle.register_words_count, bundle.body_words,
2231 bundle.body_size, &bundle.body_words_count, &bytes_emitted);
2232 } else if (ains->compact_branch) {
2233 /* All of r0 has to be written out
2234 * along with the branch writeout.
2235 * (slow!) */
2236
2237 if (ains->writeout) {
2238 if (index == 0) {
2239 midgard_instruction ins = v_fmov(0, blank_alu_src, SSA_FIXED_REGISTER(0));
2240 ins.unit = UNIT_VMUL;
2241
2242 control |= ins.unit;
2243
2244 emit_binary_vector_instruction(&ins, bundle.register_words,
2245 &bundle.register_words_count, bundle.body_words,
2246 bundle.body_size, &bundle.body_words_count, &bytes_emitted);
2247 } else {
2248 /* Analyse the group to see if r0 is written in full, on-time, without hanging dependencies*/
2249 bool written_late = false;
2250 bool components[4] = { 0 };
2251 uint16_t register_dep_mask = 0;
2252 uint16_t written_mask = 0;
2253
2254 midgard_instruction *qins = ins;
2255 for (int t = 0; t < index; ++t) {
2256 if (qins->registers.out_reg != 0) {
2257 /* Mark down writes */
2258
2259 written_mask |= (1 << qins->registers.out_reg);
2260 } else {
2261 /* Mark down the register dependencies for errata check */
2262
2263 if (qins->registers.src1_reg < 16)
2264 register_dep_mask |= (1 << qins->registers.src1_reg);
2265
2266 if (qins->registers.src2_reg < 16)
2267 register_dep_mask |= (1 << qins->registers.src2_reg);
2268
2269 int mask = qins->alu.mask;
2270
2271 for (int c = 0; c < 4; ++c)
2272 if (mask & (0x3 << (2 * c)))
2273 components[c] = true;
2274
2275 /* ..but if the writeout is too late, we have to break up anyway... for some reason */
2276
2277 if (qins->unit == UNIT_VLUT)
2278 written_late = true;
2279 }
2280
2281 /* Advance instruction pointer */
2282 qins = mir_next_op(qins);
2283 }
2284
2285
2286 /* ERRATA (?): In a bundle ending in a fragment writeout, the register dependencies of r0 cannot be written within this bundle (discovered in -bshading:shading=phong) */
2287 if (register_dep_mask & written_mask) {
2288 printf("ERRATA WORKAROUND: Breakup for writeout dependency masks %X vs %X (common %X)\n", register_dep_mask, written_mask, register_dep_mask & written_mask);
2289 break;
2290 }
2291
2292 if (written_late)
2293 break;
2294
2295 /* If even a single component is not written, break it up (conservative check). */
2296 bool breakup = false;
2297
2298 for (int c = 0; c < 4; ++c)
2299 if (!components[c])
2300 breakup = true;
2301
2302 if (breakup)
2303 break;
2304
2305 /* Otherwise, we're free to proceed */
2306 }
2307 }
2308
2309 if (ains->unit == ALU_ENAB_BRANCH) {
2310 bundle.body_size[bundle.body_words_count] = sizeof(midgard_branch_extended);
2311 memcpy(&bundle.body_words[bundle.body_words_count++], &ains->branch_extended, sizeof(midgard_branch_extended));
2312 bytes_emitted += sizeof(midgard_branch_extended);
2313 } else {
2314 bundle.body_size[bundle.body_words_count] = sizeof(ains->br_compact);
2315 memcpy(&bundle.body_words[bundle.body_words_count++], &ains->br_compact, sizeof(ains->br_compact));
2316 bytes_emitted += sizeof(ains->br_compact);
2317 }
2318 } else {
2319 memcpy(&bundle.register_words[bundle.register_words_count++], &ains->registers, sizeof(ains->registers));
2320 bytes_emitted += sizeof(midgard_reg_info);
2321
2322 bundle.body_size[bundle.body_words_count] = sizeof(midgard_scalar_alu);
2323 bundle.body_words_count++;
2324 bytes_emitted += sizeof(midgard_scalar_alu);
2325 }
2326
2327 /* Defer marking until after writing to allow for break */
2328 control |= ains->unit;
2329 last_unit = ains->unit;
2330 ++instructions_emitted;
2331 ++index;
2332 }
2333
2334 /* Bubble up the number of instructions for skipping */
2335 instructions_consumed = index - 1;
2336
2337 int padding = 0;
2338
2339 /* Pad ALU op to nearest word */
2340
2341 if (bytes_emitted & 15) {
2342 padding = 16 - (bytes_emitted & 15);
2343 bytes_emitted += padding;
2344 }
2345
2346 /* Constants must always be quadwords */
2347 if (bundle.has_embedded_constants)
2348 bytes_emitted += 16;
2349
2350 /* Size ALU instruction for tag */
2351 bundle.tag = (TAG_ALU_4) + (bytes_emitted / 16) - 1;
2352 bundle.padding = padding;
2353 bundle.control = bundle.tag | control;
2354
2355 break;
2356 }
2357
2358 case TAG_LOAD_STORE_4: {
2359 /* Load store instructions have two words at once. If
2360 * we only have one queued up, we need to NOP pad.
2361 * Otherwise, we store both in succession to save space
2362 * and cycles -- letting them go in parallel -- skip
2363 * the next. The usefulness of this optimisation is
2364 * greatly dependent on the quality of the instruction
2365 * scheduler.
2366 */
2367
2368 midgard_instruction *next_op = mir_next_op(ins);
2369
2370 if ((struct list_head *) next_op != &block->instructions && next_op->type == TAG_LOAD_STORE_4) {
2371 /* As the two operate concurrently, make sure
2372 * they are not dependent */
2373
2374 if (can_run_concurrent_ssa(ins, next_op) || true) {
2375 /* Skip ahead, since it's redundant with the pair */
2376 instructions_consumed = 1 + (instructions_emitted++);
2377 }
2378 }
2379
2380 break;
2381 }
2382
2383 default:
2384 /* Texture ops default to single-op-per-bundle scheduling */
2385 break;
2386 }
2387
2388 /* Copy the instructions into the bundle */
2389 bundle.instruction_count = instructions_emitted + 1;
2390
2391 int used_idx = 0;
2392
2393 midgard_instruction *uins = ins;
2394 for (int i = 0; used_idx < bundle.instruction_count; ++i) {
2395 bundle.instructions[used_idx++] = *uins;
2396 uins = mir_next_op(uins);
2397 }
2398
2399 *skip = (instructions_consumed == -1) ? instructions_emitted : instructions_consumed;
2400
2401 return bundle;
2402 }
2403
2404 static int
2405 quadword_size(int tag)
2406 {
2407 switch (tag) {
2408 case TAG_ALU_4:
2409 return 1;
2410
2411 case TAG_ALU_8:
2412 return 2;
2413
2414 case TAG_ALU_12:
2415 return 3;
2416
2417 case TAG_ALU_16:
2418 return 4;
2419
2420 case TAG_LOAD_STORE_4:
2421 return 1;
2422
2423 case TAG_TEXTURE_4:
2424 return 1;
2425
2426 default:
2427 assert(0);
2428 return 0;
2429 }
2430 }
2431
2432 /* Schedule a single block by iterating its instruction to create bundles.
2433 * While we go, tally about the bundle sizes to compute the block size. */
2434
2435 static void
2436 schedule_block(compiler_context *ctx, midgard_block *block)
2437 {
2438 util_dynarray_init(&block->bundles, NULL);
2439
2440 block->quadword_count = 0;
2441
2442 mir_foreach_instr_in_block(block, ins) {
2443 int skip;
2444 midgard_bundle bundle = schedule_bundle(ctx, block, ins, &skip);
2445 util_dynarray_append(&block->bundles, midgard_bundle, bundle);
2446
2447 if (bundle.has_blend_constant) {
2448 /* TODO: Multiblock? */
2449 int quadwords_within_block = block->quadword_count + quadword_size(bundle.tag) - 1;
2450 ctx->blend_constant_offset = quadwords_within_block * 0x10;
2451 }
2452
2453 while(skip--)
2454 ins = mir_next_op(ins);
2455
2456 block->quadword_count += quadword_size(bundle.tag);
2457 }
2458
2459 block->is_scheduled = true;
2460 }
2461
2462 static void
2463 schedule_program(compiler_context *ctx)
2464 {
2465 allocate_registers(ctx);
2466
2467 mir_foreach_block(ctx, block) {
2468 schedule_block(ctx, block);
2469 }
2470 }
2471
2472 /* After everything is scheduled, emit whole bundles at a time */
2473
2474 static void
2475 emit_binary_bundle(compiler_context *ctx, midgard_bundle *bundle, struct util_dynarray *emission, int next_tag)
2476 {
2477 int lookahead = next_tag << 4;
2478
2479 switch (bundle->tag) {
2480 case TAG_ALU_4:
2481 case TAG_ALU_8:
2482 case TAG_ALU_12:
2483 case TAG_ALU_16: {
2484 /* Actually emit each component */
2485 util_dynarray_append(emission, uint32_t, bundle->control | lookahead);
2486
2487 for (int i = 0; i < bundle->register_words_count; ++i)
2488 util_dynarray_append(emission, uint16_t, bundle->register_words[i]);
2489
2490 /* Emit body words based on the instructions bundled */
2491 for (int i = 0; i < bundle->instruction_count; ++i) {
2492 midgard_instruction *ins = &bundle->instructions[i];
2493
2494 if (ins->unit & UNITS_ANY_VECTOR) {
2495 memcpy(util_dynarray_grow(emission, sizeof(midgard_vector_alu)), &ins->alu, sizeof(midgard_vector_alu));
2496 } else if (ins->compact_branch) {
2497 /* Dummy move, XXX DRY */
2498 if ((i == 0) && ins->writeout) {
2499 midgard_instruction ins = v_fmov(0, blank_alu_src, SSA_FIXED_REGISTER(0));
2500 memcpy(util_dynarray_grow(emission, sizeof(midgard_vector_alu)), &ins.alu, sizeof(midgard_vector_alu));
2501 }
2502
2503 if (ins->unit == ALU_ENAB_BR_COMPACT) {
2504 memcpy(util_dynarray_grow(emission, sizeof(ins->br_compact)), &ins->br_compact, sizeof(ins->br_compact));
2505 } else {
2506 memcpy(util_dynarray_grow(emission, sizeof(ins->branch_extended)), &ins->branch_extended, sizeof(ins->branch_extended));
2507 }
2508 } else {
2509 /* Scalar */
2510 midgard_scalar_alu scalarised = vector_to_scalar_alu(ins->alu, ins);
2511 memcpy(util_dynarray_grow(emission, sizeof(scalarised)), &scalarised, sizeof(scalarised));
2512 }
2513 }
2514
2515 /* Emit padding (all zero) */
2516 memset(util_dynarray_grow(emission, bundle->padding), 0, bundle->padding);
2517
2518 /* Tack on constants */
2519
2520 if (bundle->has_embedded_constants) {
2521 util_dynarray_append(emission, float, bundle->constants[0]);
2522 util_dynarray_append(emission, float, bundle->constants[1]);
2523 util_dynarray_append(emission, float, bundle->constants[2]);
2524 util_dynarray_append(emission, float, bundle->constants[3]);
2525 }
2526
2527 break;
2528 }
2529
2530 case TAG_LOAD_STORE_4: {
2531 /* One or two composing instructions */
2532
2533 uint64_t current64, next64 = LDST_NOP;
2534
2535 memcpy(&current64, &bundle->instructions[0].load_store, sizeof(current64));
2536
2537 if (bundle->instruction_count == 2)
2538 memcpy(&next64, &bundle->instructions[1].load_store, sizeof(next64));
2539
2540 midgard_load_store instruction = {
2541 .type = bundle->tag,
2542 .next_type = next_tag,
2543 .word1 = current64,
2544 .word2 = next64
2545 };
2546
2547 util_dynarray_append(emission, midgard_load_store, instruction);
2548
2549 break;
2550 }
2551
2552 case TAG_TEXTURE_4: {
2553 /* Texture instructions are easy, since there is no
2554 * pipelining nor VLIW to worry about. We may need to set the .last flag */
2555
2556 midgard_instruction *ins = &bundle->instructions[0];
2557
2558 ins->texture.type = TAG_TEXTURE_4;
2559 ins->texture.next_type = next_tag;
2560
2561 ctx->texture_op_count--;
2562
2563 if (!ctx->texture_op_count) {
2564 ins->texture.cont = 0;
2565 ins->texture.last = 1;
2566 }
2567
2568 util_dynarray_append(emission, midgard_texture_word, ins->texture);
2569 break;
2570 }
2571
2572 default:
2573 printf("Unknown midgard instruction type\n");
2574 assert(0);
2575 break;
2576 }
2577 }
2578
2579
2580 /* ALU instructions can inline or embed constants, which decreases register
2581 * pressure and saves space. */
2582
2583 #define CONDITIONAL_ATTACH(src) { \
2584 void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->ssa_args.src + 1); \
2585 \
2586 if (entry) { \
2587 attach_constants(ctx, alu, entry, alu->ssa_args.src + 1); \
2588 alu->ssa_args.src = SSA_FIXED_REGISTER(REGISTER_CONSTANT); \
2589 } \
2590 }
2591
2592 static void
2593 inline_alu_constants(compiler_context *ctx)
2594 {
2595 mir_foreach_instr(ctx, alu) {
2596 /* Other instructions cannot inline constants */
2597 if (alu->type != TAG_ALU_4) continue;
2598
2599 /* If there is already a constant here, we can do nothing */
2600 if (alu->has_constants) continue;
2601
2602 CONDITIONAL_ATTACH(src0);
2603
2604 if (!alu->has_constants) {
2605 CONDITIONAL_ATTACH(src1)
2606 } else if (!alu->inline_constant) {
2607 /* Corner case: _two_ vec4 constants, for instance with a
2608 * csel. For this case, we can only use a constant
2609 * register for one, we'll have to emit a move for the
2610 * other. Note, if both arguments are constants, then
2611 * necessarily neither argument depends on the value of
2612 * any particular register. As the destination register
2613 * will be wiped, that means we can spill the constant
2614 * to the destination register.
2615 */
2616
2617 void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->ssa_args.src1 + 1);
2618 unsigned scratch = alu->ssa_args.dest;
2619
2620 if (entry) {
2621 midgard_instruction ins = v_fmov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), blank_alu_src, scratch);
2622 attach_constants(ctx, &ins, entry, alu->ssa_args.src1 + 1);
2623
2624 /* Force a break XXX Defer r31 writes */
2625 ins.unit = UNIT_VLUT;
2626
2627 /* Set the source */
2628 alu->ssa_args.src1 = scratch;
2629
2630 /* Inject us -before- the last instruction which set r31 */
2631 mir_insert_instruction_before(mir_prev_op(alu), ins);
2632 }
2633 }
2634 }
2635 }
2636
2637 /* Midgard supports two types of constants, embedded constants (128-bit) and
2638 * inline constants (16-bit). Sometimes, especially with scalar ops, embedded
2639 * constants can be demoted to inline constants, for space savings and
2640 * sometimes a performance boost */
2641
2642 static void
2643 embedded_to_inline_constant(compiler_context *ctx)
2644 {
2645 mir_foreach_instr(ctx, ins) {
2646 if (!ins->has_constants) continue;
2647
2648 if (ins->ssa_args.inline_constant) continue;
2649
2650 /* Blend constants must not be inlined by definition */
2651 if (ins->has_blend_constant) continue;
2652
2653 /* src1 cannot be an inline constant due to encoding
2654 * restrictions. So, if possible we try to flip the arguments
2655 * in that case */
2656
2657 int op = ins->alu.op;
2658
2659 if (ins->ssa_args.src0 == SSA_FIXED_REGISTER(REGISTER_CONSTANT)) {
2660 /* Flip based on op. Fallthrough intentional */
2661
2662 switch (op) {
2663 /* These ops require an operational change to flip their arguments TODO */
2664 case midgard_alu_op_flt:
2665 case midgard_alu_op_fle:
2666 case midgard_alu_op_ilt:
2667 case midgard_alu_op_ile:
2668 case midgard_alu_op_fcsel:
2669 case midgard_alu_op_icsel:
2670 case midgard_alu_op_isub:
2671 printf("Missed non-commutative flip (%s)\n", alu_opcode_names[op]);
2672 break;
2673
2674 /* These ops are commutative and Just Flip */
2675 case midgard_alu_op_fne:
2676 case midgard_alu_op_fadd:
2677 case midgard_alu_op_fmul:
2678 case midgard_alu_op_fmin:
2679 case midgard_alu_op_fmax:
2680 case midgard_alu_op_iadd:
2681 case midgard_alu_op_imul:
2682 case midgard_alu_op_feq:
2683 case midgard_alu_op_ieq:
2684 case midgard_alu_op_ine:
2685 case midgard_alu_op_iand:
2686 case midgard_alu_op_ior:
2687 case midgard_alu_op_ixor:
2688 /* Flip the SSA numbers */
2689 ins->ssa_args.src0 = ins->ssa_args.src1;
2690 ins->ssa_args.src1 = SSA_FIXED_REGISTER(REGISTER_CONSTANT);
2691
2692 /* And flip the modifiers */
2693
2694 unsigned src_temp;
2695
2696 src_temp = ins->alu.src2;
2697 ins->alu.src2 = ins->alu.src1;
2698 ins->alu.src1 = src_temp;
2699
2700 default:
2701 break;
2702 }
2703 }
2704
2705 if (ins->ssa_args.src1 == SSA_FIXED_REGISTER(REGISTER_CONSTANT)) {
2706 /* Extract the source information */
2707
2708 midgard_vector_alu_src *src;
2709 int q = ins->alu.src2;
2710 midgard_vector_alu_src *m = (midgard_vector_alu_src *) &q;
2711 src = m;
2712
2713 /* Component is from the swizzle, e.g. r26.w -> w component. TODO: What if x is masked out? */
2714 int component = src->swizzle & 3;
2715
2716 /* Scale constant appropriately, if we can legally */
2717 uint16_t scaled_constant = 0;
2718
2719 /* XXX: Check legality */
2720 if (midgard_is_integer_op(op)) {
2721 /* TODO: Inline integer */
2722 continue;
2723
2724 unsigned int *iconstants = (unsigned int *) ins->constants;
2725 scaled_constant = (uint16_t) iconstants[component];
2726
2727 /* Constant overflow after resize */
2728 if (scaled_constant != iconstants[component])
2729 continue;
2730 } else {
2731 scaled_constant = _mesa_float_to_half((float) ins->constants[component]);
2732 }
2733
2734 /* We don't know how to handle these with a constant */
2735
2736 if (src->abs || src->negate || src->half || src->rep_low || src->rep_high) {
2737 printf("Bailing inline constant...\n");
2738 continue;
2739 }
2740
2741 /* Make sure that the constant is not itself a
2742 * vector by checking if all accessed values
2743 * (by the swizzle) are the same. */
2744
2745 uint32_t *cons = (uint32_t *) ins->constants;
2746 uint32_t value = cons[component];
2747
2748 bool is_vector = false;
2749 unsigned mask = effective_writemask(&ins->alu);
2750
2751 for (int c = 1; c < 4; ++c) {
2752 /* We only care if this component is actually used */
2753 if (!(mask & (1 << c)))
2754 continue;
2755
2756 uint32_t test = cons[(src->swizzle >> (2 * c)) & 3];
2757
2758 if (test != value) {
2759 is_vector = true;
2760 break;
2761 }
2762 }
2763
2764 if (is_vector)
2765 continue;
2766
2767 /* Get rid of the embedded constant */
2768 ins->has_constants = false;
2769 ins->ssa_args.src1 = SSA_UNUSED_0;
2770 ins->ssa_args.inline_constant = true;
2771 ins->inline_constant = scaled_constant;
2772 }
2773 }
2774 }
2775
2776 /* Map normal SSA sources to other SSA sources / fixed registers (like
2777 * uniforms) */
2778
2779 static void
2780 map_ssa_to_alias(compiler_context *ctx, int *ref)
2781 {
2782 unsigned int alias = (uintptr_t) _mesa_hash_table_u64_search(ctx->ssa_to_alias, *ref + 1);
2783
2784 if (alias) {
2785 /* Remove entry in leftovers to avoid a redunant fmov */
2786
2787 struct set_entry *leftover = _mesa_set_search(ctx->leftover_ssa_to_alias, ((void *) (uintptr_t) (*ref + 1)));
2788
2789 if (leftover)
2790 _mesa_set_remove(ctx->leftover_ssa_to_alias, leftover);
2791
2792 /* Assign the alias map */
2793 *ref = alias - 1;
2794 return;
2795 }
2796 }
2797
2798 #define AS_SRC(to, u) \
2799 int q##to = ins->alu.src2; \
2800 midgard_vector_alu_src *to = (midgard_vector_alu_src *) &q##to;
2801
2802 /* Removing unused moves is necessary to clean up the texture pipeline results.
2803 *
2804 * To do so, we find moves in the MIR. We check if their destination is live later. If it's not, the move is redundant. */
2805
2806 static void
2807 midgard_eliminate_orphan_moves(compiler_context *ctx, midgard_block *block)
2808 {
2809 mir_foreach_instr_in_block_safe(block, ins) {
2810 if (ins->type != TAG_ALU_4) continue;
2811
2812 if (ins->alu.op != midgard_alu_op_fmov) continue;
2813
2814 if (ins->ssa_args.dest >= SSA_FIXED_MINIMUM) continue;
2815
2816 if (midgard_is_pinned(ctx, ins->ssa_args.dest)) continue;
2817
2818 if (is_live_after(ctx, block, ins, ins->ssa_args.dest)) continue;
2819
2820 mir_remove_instruction(ins);
2821 }
2822 }
2823
2824 /* The following passes reorder MIR instructions to enable better scheduling */
2825
2826 static void
2827 midgard_pair_load_store(compiler_context *ctx, midgard_block *block)
2828 {
2829 mir_foreach_instr_in_block_safe(block, ins) {
2830 if (ins->type != TAG_LOAD_STORE_4) continue;
2831
2832 /* We've found a load/store op. Check if next is also load/store. */
2833 midgard_instruction *next_op = mir_next_op(ins);
2834 if (&next_op->link != &block->instructions) {
2835 if (next_op->type == TAG_LOAD_STORE_4) {
2836 /* If so, we're done since we're a pair */
2837 ins = mir_next_op(ins);
2838 continue;
2839 }
2840
2841 /* Maximum search distance to pair, to avoid register pressure disasters */
2842 int search_distance = 8;
2843
2844 /* Otherwise, we have an orphaned load/store -- search for another load */
2845 mir_foreach_instr_in_block_from(block, c, mir_next_op(ins)) {
2846 /* Terminate search if necessary */
2847 if (!(search_distance--)) break;
2848
2849 if (c->type != TAG_LOAD_STORE_4) continue;
2850
2851 if (OP_IS_STORE(c->load_store.op)) continue;
2852
2853 /* We found one! Move it up to pair and remove it from the old location */
2854
2855 mir_insert_instruction_before(ins, *c);
2856 mir_remove_instruction(c);
2857
2858 break;
2859 }
2860 }
2861 }
2862 }
2863
2864 /* Emit varying stores late */
2865
2866 static void
2867 midgard_emit_store(compiler_context *ctx, midgard_block *block) {
2868 /* Iterate in reverse to get the final write, rather than the first */
2869
2870 mir_foreach_instr_in_block_safe_rev(block, ins) {
2871 /* Check if what we just wrote needs a store */
2872 int idx = ins->ssa_args.dest;
2873 uintptr_t varying = ((uintptr_t) _mesa_hash_table_u64_search(ctx->ssa_varyings, idx + 1));
2874
2875 if (!varying) continue;
2876
2877 varying -= 1;
2878
2879 /* We need to store to the appropriate varying, so emit the
2880 * move/store */
2881
2882 /* TODO: Integrate with special purpose RA (and scheduler?) */
2883 bool high_varying_register = false;
2884
2885 midgard_instruction mov = v_fmov(idx, blank_alu_src, SSA_FIXED_REGISTER(REGISTER_VARYING_BASE + high_varying_register));
2886
2887 midgard_instruction st = m_store_vary_32(SSA_FIXED_REGISTER(high_varying_register), varying);
2888 st.load_store.unknown = 0x1E9E; /* XXX: What is this? */
2889
2890 mir_insert_instruction_before(mir_next_op(ins), st);
2891 mir_insert_instruction_before(mir_next_op(ins), mov);
2892
2893 /* We no longer need to store this varying */
2894 _mesa_hash_table_u64_remove(ctx->ssa_varyings, idx + 1);
2895 }
2896 }
2897
2898 /* If there are leftovers after the below pass, emit actual fmov
2899 * instructions for the slow-but-correct path */
2900
2901 static void
2902 emit_leftover_move(compiler_context *ctx)
2903 {
2904 set_foreach(ctx->leftover_ssa_to_alias, leftover) {
2905 int base = ((uintptr_t) leftover->key) - 1;
2906 int mapped = base;
2907
2908 map_ssa_to_alias(ctx, &mapped);
2909 EMIT(fmov, mapped, blank_alu_src, base);
2910 }
2911 }
2912
2913 static void
2914 actualise_ssa_to_alias(compiler_context *ctx)
2915 {
2916 mir_foreach_instr(ctx, ins) {
2917 map_ssa_to_alias(ctx, &ins->ssa_args.src0);
2918 map_ssa_to_alias(ctx, &ins->ssa_args.src1);
2919 }
2920
2921 emit_leftover_move(ctx);
2922 }
2923
2924 /* Vertex shaders do not write gl_Position as is; instead, they write a
2925 * transformed screen space position as a varying. See section 12.5 "Coordinate
2926 * Transformation" of the ES 3.2 full specification for details.
2927 *
2928 * This transformation occurs early on, as NIR and prior to optimisation, in
2929 * order to take advantage of NIR optimisation passes of the transform itself.
2930 * */
2931
2932 static void
2933 write_transformed_position(nir_builder *b, nir_src input_point_src, int uniform_no)
2934 {
2935 nir_ssa_def *input_point = nir_ssa_for_src(b, input_point_src, 4);
2936
2937 /* Get viewport from the uniforms */
2938 nir_intrinsic_instr *load;
2939 load = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_uniform);
2940 load->num_components = 4;
2941 load->src[0] = nir_src_for_ssa(nir_imm_int(b, uniform_no));
2942 nir_ssa_dest_init(&load->instr, &load->dest, 4, 32, NULL);
2943 nir_builder_instr_insert(b, &load->instr);
2944
2945 /* Formatted as <width, height, centerx, centery> */
2946 nir_ssa_def *viewport_vec4 = &load->dest.ssa;
2947 nir_ssa_def *viewport_width_2 = nir_channel(b, viewport_vec4, 0);
2948 nir_ssa_def *viewport_height_2 = nir_channel(b, viewport_vec4, 1);
2949 nir_ssa_def *viewport_offset = nir_channels(b, viewport_vec4, 0x8 | 0x4);
2950
2951 /* XXX: From uniforms? */
2952 nir_ssa_def *depth_near = nir_imm_float(b, 0.0);
2953 nir_ssa_def *depth_far = nir_imm_float(b, 1.0);
2954
2955 /* World space to normalised device coordinates */
2956
2957 nir_ssa_def *w_recip = nir_frcp(b, nir_channel(b, input_point, 3));
2958 nir_ssa_def *ndc_point = nir_fmul(b, nir_channels(b, input_point, 0x7), w_recip);
2959
2960 /* Normalised device coordinates to screen space */
2961
2962 nir_ssa_def *viewport_multiplier = nir_vec2(b, viewport_width_2, viewport_height_2);
2963 nir_ssa_def *viewport_xy = nir_fadd(b, nir_fmul(b, nir_channels(b, ndc_point, 0x3), viewport_multiplier), viewport_offset);
2964
2965 nir_ssa_def *depth_multiplier = nir_fmul(b, nir_fsub(b, depth_far, depth_near), nir_imm_float(b, 0.5f));
2966 nir_ssa_def *depth_offset = nir_fmul(b, nir_fadd(b, depth_far, depth_near), nir_imm_float(b, 0.5f));
2967 nir_ssa_def *screen_depth = nir_fadd(b, nir_fmul(b, nir_channel(b, ndc_point, 2), depth_multiplier), depth_offset);
2968
2969 /* gl_Position will be written out in screenspace xyz, with w set to
2970 * the reciprocal we computed earlier. The transformed w component is
2971 * then used for perspective-correct varying interpolation. The
2972 * transformed w component must preserve its original sign; this is
2973 * used in depth clipping computations */
2974
2975 nir_ssa_def *screen_space = nir_vec4(b,
2976 nir_channel(b, viewport_xy, 0),
2977 nir_channel(b, viewport_xy, 1),
2978 screen_depth,
2979 w_recip);
2980
2981 /* Finally, write out the transformed values to the varying */
2982
2983 nir_intrinsic_instr *store;
2984 store = nir_intrinsic_instr_create(b->shader, nir_intrinsic_store_output);
2985 store->num_components = 4;
2986 nir_intrinsic_set_base(store, 0);
2987 nir_intrinsic_set_write_mask(store, 0xf);
2988 store->src[0].ssa = screen_space;
2989 store->src[0].is_ssa = true;
2990 store->src[1] = nir_src_for_ssa(nir_imm_int(b, 0));
2991 nir_builder_instr_insert(b, &store->instr);
2992 }
2993
2994 static void
2995 transform_position_writes(nir_shader *shader)
2996 {
2997 nir_foreach_function(func, shader) {
2998 nir_foreach_block(block, func->impl) {
2999 nir_foreach_instr_safe(instr, block) {
3000 if (instr->type != nir_instr_type_intrinsic) continue;
3001
3002 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
3003 nir_variable *out = NULL;
3004
3005 switch (intr->intrinsic) {
3006 case nir_intrinsic_store_output:
3007 /* already had i/o lowered.. lookup the matching output var: */
3008 nir_foreach_variable(var, &shader->outputs) {
3009 int drvloc = var->data.driver_location;
3010
3011 if (nir_intrinsic_base(intr) == drvloc) {
3012 out = var;
3013 break;
3014 }
3015 }
3016
3017 break;
3018
3019 default:
3020 break;
3021 }
3022
3023 if (!out) continue;
3024
3025 if (out->data.mode != nir_var_shader_out)
3026 continue;
3027
3028 if (out->data.location != VARYING_SLOT_POS)
3029 continue;
3030
3031 nir_builder b;
3032 nir_builder_init(&b, func->impl);
3033 b.cursor = nir_before_instr(instr);
3034
3035 write_transformed_position(&b, intr->src[0], UNIFORM_VIEWPORT);
3036 nir_instr_remove(instr);
3037 }
3038 }
3039 }
3040 }
3041
3042 static void
3043 emit_fragment_epilogue(compiler_context *ctx)
3044 {
3045 /* Special case: writing out constants requires us to include the move
3046 * explicitly now, so shove it into r0 */
3047
3048 void *constant_value = _mesa_hash_table_u64_search(ctx->ssa_constants, ctx->fragment_output + 1);
3049
3050 if (constant_value) {
3051 midgard_instruction ins = v_fmov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), blank_alu_src, SSA_FIXED_REGISTER(0));
3052 attach_constants(ctx, &ins, constant_value, ctx->fragment_output + 1);
3053 emit_mir_instruction(ctx, ins);
3054 }
3055
3056 /* Perform the actual fragment writeout. We have two writeout/branch
3057 * instructions, forming a loop until writeout is successful as per the
3058 * docs. TODO: gl_FragDepth */
3059
3060 EMIT(alu_br_compact_cond, midgard_jmp_writeout_op_writeout, TAG_ALU_4, 0, midgard_condition_always);
3061 EMIT(alu_br_compact_cond, midgard_jmp_writeout_op_writeout, TAG_ALU_4, -1, midgard_condition_always);
3062 }
3063
3064 /* For the blend epilogue, we need to convert the blended fragment vec4 (stored
3065 * in r0) to a RGBA8888 value by scaling and type converting. We then output it
3066 * with the int8 analogue to the fragment epilogue */
3067
3068 static void
3069 emit_blend_epilogue(compiler_context *ctx)
3070 {
3071 /* vmul.fmul.none.fulllow hr48, r0, #255 */
3072
3073 midgard_instruction scale = {
3074 .type = TAG_ALU_4,
3075 .unit = UNIT_VMUL,
3076 .inline_constant = _mesa_float_to_half(255.0),
3077 .ssa_args = {
3078 .src0 = SSA_FIXED_REGISTER(0),
3079 .src1 = SSA_UNUSED_0,
3080 .dest = SSA_FIXED_REGISTER(24),
3081 .inline_constant = true
3082 },
3083 .alu = {
3084 .op = midgard_alu_op_fmul,
3085 .reg_mode = midgard_reg_mode_full,
3086 .dest_override = midgard_dest_override_lower,
3087 .mask = 0xFF,
3088 .src1 = vector_alu_srco_unsigned(blank_alu_src),
3089 .src2 = vector_alu_srco_unsigned(blank_alu_src),
3090 }
3091 };
3092
3093 emit_mir_instruction(ctx, scale);
3094
3095 /* vadd.f2u8.pos.low hr0, hr48, #0 */
3096
3097 midgard_vector_alu_src alu_src = blank_alu_src;
3098 alu_src.half = true;
3099
3100 midgard_instruction f2u8 = {
3101 .type = TAG_ALU_4,
3102 .ssa_args = {
3103 .src0 = SSA_FIXED_REGISTER(24),
3104 .src1 = SSA_UNUSED_0,
3105 .dest = SSA_FIXED_REGISTER(0),
3106 .inline_constant = true
3107 },
3108 .alu = {
3109 .op = midgard_alu_op_f2u8,
3110 .reg_mode = midgard_reg_mode_half,
3111 .dest_override = midgard_dest_override_lower,
3112 .outmod = midgard_outmod_pos,
3113 .mask = 0xF,
3114 .src1 = vector_alu_srco_unsigned(alu_src),
3115 .src2 = vector_alu_srco_unsigned(blank_alu_src),
3116 }
3117 };
3118
3119 emit_mir_instruction(ctx, f2u8);
3120
3121 /* vmul.imov.quarter r0, r0, r0 */
3122
3123 midgard_instruction imov_8 = {
3124 .type = TAG_ALU_4,
3125 .ssa_args = {
3126 .src0 = SSA_UNUSED_1,
3127 .src1 = SSA_FIXED_REGISTER(0),
3128 .dest = SSA_FIXED_REGISTER(0),
3129 },
3130 .alu = {
3131 .op = midgard_alu_op_imov,
3132 .reg_mode = midgard_reg_mode_quarter,
3133 .dest_override = midgard_dest_override_none,
3134 .mask = 0xFF,
3135 .src1 = vector_alu_srco_unsigned(blank_alu_src),
3136 .src2 = vector_alu_srco_unsigned(blank_alu_src),
3137 }
3138 };
3139
3140 /* Emit branch epilogue with the 8-bit move as the source */
3141
3142 emit_mir_instruction(ctx, imov_8);
3143 EMIT(alu_br_compact_cond, midgard_jmp_writeout_op_writeout, TAG_ALU_4, 0, midgard_condition_always);
3144
3145 emit_mir_instruction(ctx, imov_8);
3146 EMIT(alu_br_compact_cond, midgard_jmp_writeout_op_writeout, TAG_ALU_4, -1, midgard_condition_always);
3147 }
3148
3149 static midgard_block *
3150 emit_block(compiler_context *ctx, nir_block *block)
3151 {
3152 midgard_block *this_block = malloc(sizeof(midgard_block));
3153 list_addtail(&this_block->link, &ctx->blocks);
3154
3155 this_block->is_scheduled = false;
3156 ++ctx->block_count;
3157
3158 ctx->texture_index[0] = -1;
3159 ctx->texture_index[1] = -1;
3160
3161 /* Set up current block */
3162 list_inithead(&this_block->instructions);
3163 ctx->current_block = this_block;
3164
3165 nir_foreach_instr(instr, block) {
3166 emit_instr(ctx, instr);
3167 ++ctx->instruction_count;
3168 }
3169
3170 inline_alu_constants(ctx);
3171 embedded_to_inline_constant(ctx);
3172
3173 /* Perform heavylifting for aliasing */
3174 actualise_ssa_to_alias(ctx);
3175
3176 midgard_emit_store(ctx, this_block);
3177 midgard_eliminate_orphan_moves(ctx, this_block);
3178 midgard_pair_load_store(ctx, this_block);
3179
3180 /* Append fragment shader epilogue (value writeout) */
3181 if (ctx->stage == MESA_SHADER_FRAGMENT) {
3182 if (block == nir_impl_last_block(ctx->func->impl)) {
3183 if (ctx->is_blend)
3184 emit_blend_epilogue(ctx);
3185 else
3186 emit_fragment_epilogue(ctx);
3187 }
3188 }
3189
3190 /* Fallthrough save */
3191 this_block->next_fallthrough = ctx->previous_source_block;
3192
3193 if (block == nir_start_block(ctx->func->impl))
3194 ctx->initial_block = this_block;
3195
3196 if (block == nir_impl_last_block(ctx->func->impl))
3197 ctx->final_block = this_block;
3198
3199 /* Allow the next control flow to access us retroactively, for
3200 * branching etc */
3201 ctx->current_block = this_block;
3202
3203 /* Document the fallthrough chain */
3204 ctx->previous_source_block = this_block;
3205
3206 return this_block;
3207 }
3208
3209 static midgard_block *emit_cf_list(struct compiler_context *ctx, struct exec_list *list);
3210
3211 static void
3212 emit_if(struct compiler_context *ctx, nir_if *nif)
3213 {
3214 /* Conditional branches expect the condition in r31.w; emit a move for
3215 * that in the _previous_ block (which is the current block). */
3216 emit_condition(ctx, &nif->condition, true);
3217
3218 /* Speculatively emit the branch, but we can't fill it in until later */
3219 EMIT(branch, true, true);
3220 midgard_instruction *then_branch = mir_last_in_block(ctx->current_block);
3221
3222 /* Emit the two subblocks */
3223 midgard_block *then_block = emit_cf_list(ctx, &nif->then_list);
3224
3225 /* Emit a jump from the end of the then block to the end of the else */
3226 EMIT(branch, false, false);
3227 midgard_instruction *then_exit = mir_last_in_block(ctx->current_block);
3228
3229 /* Emit second block, and check if it's empty */
3230
3231 int else_idx = ctx->block_count;
3232 int count_in = ctx->instruction_count;
3233 midgard_block *else_block = emit_cf_list(ctx, &nif->else_list);
3234 int after_else_idx = ctx->block_count;
3235
3236 /* Now that we have the subblocks emitted, fix up the branches */
3237
3238 assert(then_block);
3239 assert(else_block);
3240
3241 if (ctx->instruction_count == count_in) {
3242 /* The else block is empty, so don't emit an exit jump */
3243 mir_remove_instruction(then_exit);
3244 then_branch->branch.target_block = after_else_idx;
3245 } else {
3246 then_branch->branch.target_block = else_idx;
3247 then_exit->branch.target_block = after_else_idx;
3248 }
3249 }
3250
3251 static void
3252 emit_loop(struct compiler_context *ctx, nir_loop *nloop)
3253 {
3254 /* Remember where we are */
3255 midgard_block *start_block = ctx->current_block;
3256
3257 /* Allocate a loop number for this. TODO: Nested loops. Instead of a
3258 * single current_loop variable, maybe we need a stack */
3259
3260 int loop_idx = ++ctx->current_loop;
3261
3262 /* Get index from before the body so we can loop back later */
3263 int start_idx = ctx->block_count;
3264
3265 /* Emit the body itself */
3266 emit_cf_list(ctx, &nloop->body);
3267
3268 /* Branch back to loop back */
3269 struct midgard_instruction br_back = v_branch(false, false);
3270 br_back.branch.target_block = start_idx;
3271 emit_mir_instruction(ctx, br_back);
3272
3273 /* Find the index of the block about to follow us (note: we don't add
3274 * one; blocks are 0-indexed so we get a fencepost problem) */
3275 int break_block_idx = ctx->block_count;
3276
3277 /* Fix up the break statements we emitted to point to the right place,
3278 * now that we can allocate a block number for them */
3279
3280 list_for_each_entry_from(struct midgard_block, block, start_block, &ctx->blocks, link) {
3281 print_mir_block(block);
3282 mir_foreach_instr_in_block(block, ins) {
3283 if (ins->type != TAG_ALU_4) continue;
3284 if (!ins->compact_branch) continue;
3285 if (ins->prepacked_branch) continue;
3286
3287 /* We found a branch -- check the type to see if we need to do anything */
3288 if (ins->branch.target_type != TARGET_BREAK) continue;
3289
3290 /* It's a break! Check if it's our break */
3291 if (ins->branch.target_break != loop_idx) continue;
3292
3293 /* Okay, cool, we're breaking out of this loop.
3294 * Rewrite from a break to a goto */
3295
3296 ins->branch.target_type = TARGET_GOTO;
3297 ins->branch.target_block = break_block_idx;
3298 }
3299 }
3300 }
3301
3302 static midgard_block *
3303 emit_cf_list(struct compiler_context *ctx, struct exec_list *list)
3304 {
3305 midgard_block *start_block = NULL;
3306
3307 foreach_list_typed(nir_cf_node, node, node, list) {
3308 switch (node->type) {
3309 case nir_cf_node_block: {
3310 midgard_block *block = emit_block(ctx, nir_cf_node_as_block(node));
3311
3312 if (!start_block)
3313 start_block = block;
3314
3315 break;
3316 }
3317
3318 case nir_cf_node_if:
3319 emit_if(ctx, nir_cf_node_as_if(node));
3320 break;
3321
3322 case nir_cf_node_loop:
3323 emit_loop(ctx, nir_cf_node_as_loop(node));
3324 break;
3325
3326 case nir_cf_node_function:
3327 assert(0);
3328 break;
3329 }
3330 }
3331
3332 return start_block;
3333 }
3334
3335 /* Due to lookahead, we need to report the first tag executed in the command
3336 * stream and in branch targets. An initial block might be empty, so iterate
3337 * until we find one that 'works' */
3338
3339 static unsigned
3340 midgard_get_first_tag_from_block(compiler_context *ctx, unsigned block_idx)
3341 {
3342 midgard_block *initial_block = mir_get_block(ctx, block_idx);
3343
3344 unsigned first_tag = 0;
3345
3346 do {
3347 midgard_bundle *initial_bundle = util_dynarray_element(&initial_block->bundles, midgard_bundle, 0);
3348
3349 if (initial_bundle) {
3350 first_tag = initial_bundle->tag;
3351 break;
3352 }
3353
3354 /* Initial block is empty, try the next block */
3355 initial_block = list_first_entry(&(initial_block->link), midgard_block, link);
3356 } while(initial_block != NULL);
3357
3358 assert(first_tag);
3359 return first_tag;
3360 }
3361
3362 int
3363 midgard_compile_shader_nir(nir_shader *nir, midgard_program *program, bool is_blend)
3364 {
3365 struct util_dynarray *compiled = &program->compiled;
3366
3367 compiler_context ictx = {
3368 .nir = nir,
3369 .stage = nir->info.stage,
3370
3371 .is_blend = is_blend,
3372 .blend_constant_offset = -1,
3373
3374 .alpha_ref = program->alpha_ref
3375 };
3376
3377 compiler_context *ctx = &ictx;
3378
3379 /* TODO: Decide this at runtime */
3380 ctx->uniform_cutoff = 8;
3381
3382 switch (ctx->stage) {
3383 case MESA_SHADER_VERTEX:
3384 ctx->special_uniforms = 1;
3385 break;
3386
3387 default:
3388 ctx->special_uniforms = 0;
3389 break;
3390 }
3391
3392 /* Append epilogue uniforms if necessary. The cmdstream depends on
3393 * these being at the -end-; see assign_var_locations. */
3394
3395 if (ctx->stage == MESA_SHADER_VERTEX) {
3396 nir_variable_create(nir, nir_var_uniform, glsl_vec4_type(), "viewport");
3397 }
3398
3399 /* Assign var locations early, so the epilogue can use them if necessary */
3400
3401 nir_assign_var_locations(&nir->outputs, &nir->num_outputs, glsl_type_size);
3402 nir_assign_var_locations(&nir->inputs, &nir->num_inputs, glsl_type_size);
3403 nir_assign_var_locations(&nir->uniforms, &nir->num_uniforms, glsl_type_size);
3404
3405 /* Initialize at a global (not block) level hash tables */
3406
3407 ctx->ssa_constants = _mesa_hash_table_u64_create(NULL);
3408 ctx->ssa_varyings = _mesa_hash_table_u64_create(NULL);
3409 ctx->ssa_to_alias = _mesa_hash_table_u64_create(NULL);
3410 ctx->ssa_to_register = _mesa_hash_table_u64_create(NULL);
3411 ctx->hash_to_temp = _mesa_hash_table_u64_create(NULL);
3412 ctx->leftover_ssa_to_alias = _mesa_set_create(NULL, _mesa_hash_pointer, _mesa_key_pointer_equal);
3413
3414 /* Assign actual uniform location, skipping over samplers */
3415
3416 ctx->uniform_nir_to_mdg = _mesa_hash_table_u64_create(NULL);
3417
3418 nir_foreach_variable(var, &nir->uniforms) {
3419 if (glsl_get_base_type(var->type) == GLSL_TYPE_SAMPLER) continue;
3420
3421 unsigned length = glsl_get_aoa_size(var->type);
3422
3423 if (!length) {
3424 length = glsl_get_length(var->type);
3425 }
3426
3427 if (!length) {
3428 length = glsl_get_matrix_columns(var->type);
3429 }
3430
3431 for (int col = 0; col < length; ++col) {
3432 int id = ctx->uniform_count++;
3433 _mesa_hash_table_u64_insert(ctx->uniform_nir_to_mdg, var->data.driver_location + col + 1, (void *) ((uintptr_t) (id + 1)));
3434 }
3435 }
3436
3437 if (ctx->stage == MESA_SHADER_VERTEX) {
3438 ctx->varying_nir_to_mdg = _mesa_hash_table_u64_create(NULL);
3439
3440 /* First, collect the special varyings */
3441 nir_foreach_variable(var, &nir->outputs) {
3442 if (var->data.location == VARYING_SLOT_POS) {
3443 /* Set position first, always. It takes up two
3444 * spots, the latter one is de facto unused (at
3445 * least from the shader's perspective), we
3446 * just need to skip over the spot*/
3447
3448 _mesa_hash_table_u64_insert(ctx->varying_nir_to_mdg, var->data.driver_location + 1, (void *) ((uintptr_t) (0 + 1)));
3449 ctx->varying_count = MAX2(ctx->varying_count, 2);
3450 } else if (var->data.location == VARYING_SLOT_PSIZ) {
3451 /* Set point size second (third, see above) */
3452 _mesa_hash_table_u64_insert(ctx->varying_nir_to_mdg, var->data.driver_location + 1, (void *) ((uintptr_t) (2 + 1)));
3453 ctx->varying_count = MAX2(ctx->varying_count, 3);
3454
3455 program->writes_point_size = true;
3456 }
3457 }
3458
3459 /* Now, collect normal varyings */
3460
3461 nir_foreach_variable(var, &nir->outputs) {
3462 if (var->data.location == VARYING_SLOT_POS || var->data.location == VARYING_SLOT_PSIZ) continue;
3463
3464 for (int col = 0; col < glsl_get_matrix_columns(var->type); ++col) {
3465 int id = ctx->varying_count++;
3466 _mesa_hash_table_u64_insert(ctx->varying_nir_to_mdg, var->data.driver_location + col + 1, (void *) ((uintptr_t) (id + 1)));
3467 }
3468 }
3469 }
3470
3471
3472
3473 /* Lower vars -- not I/O -- before epilogue */
3474
3475 NIR_PASS_V(nir, nir_lower_var_copies);
3476 NIR_PASS_V(nir, nir_lower_vars_to_ssa);
3477 NIR_PASS_V(nir, nir_split_var_copies);
3478 NIR_PASS_V(nir, nir_lower_var_copies);
3479 NIR_PASS_V(nir, nir_lower_global_vars_to_local);
3480 NIR_PASS_V(nir, nir_lower_var_copies);
3481 NIR_PASS_V(nir, nir_lower_vars_to_ssa);
3482 NIR_PASS_V(nir, nir_lower_io, nir_var_all, glsl_type_size, 0);
3483
3484 /* Append vertex epilogue before optimisation, so the epilogue itself
3485 * is optimised */
3486
3487 if (ctx->stage == MESA_SHADER_VERTEX)
3488 transform_position_writes(nir);
3489
3490 /* Optimisation passes */
3491
3492 optimise_nir(nir);
3493
3494 nir_print_shader(nir, stdout);
3495
3496 /* Assign counts, now that we're sure (post-optimisation) */
3497 program->uniform_count = nir->num_uniforms;
3498
3499 program->attribute_count = (ctx->stage == MESA_SHADER_VERTEX) ? nir->num_inputs : 0;
3500 program->varying_count = (ctx->stage == MESA_SHADER_VERTEX) ? nir->num_outputs : ((ctx->stage == MESA_SHADER_FRAGMENT) ? nir->num_inputs : 0);
3501
3502
3503 nir_foreach_function(func, nir) {
3504 if (!func->impl)
3505 continue;
3506
3507 list_inithead(&ctx->blocks);
3508 ctx->block_count = 0;
3509 ctx->func = func;
3510
3511 emit_cf_list(ctx, &func->impl->body);
3512 emit_block(ctx, func->impl->end_block);
3513
3514 break; /* TODO: Multi-function shaders */
3515 }
3516
3517 util_dynarray_init(compiled, NULL);
3518
3519 /* Schedule! */
3520 schedule_program(ctx);
3521
3522 /* Now that all the bundles are scheduled and we can calculate block
3523 * sizes, emit actual branch instructions rather than placeholders */
3524
3525 int br_block_idx = 0;
3526
3527 mir_foreach_block(ctx, block) {
3528 util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) {
3529 for (int c = 0; c < bundle->instruction_count; ++c) {
3530 midgard_instruction *ins = &bundle->instructions[c];
3531
3532 if (!midgard_is_branch_unit(ins->unit)) continue;
3533
3534 if (ins->prepacked_branch) continue;
3535
3536 /* Parse some basic branch info */
3537 bool is_compact = ins->unit == ALU_ENAB_BR_COMPACT;
3538 bool is_conditional = ins->branch.conditional;
3539 bool is_inverted = ins->branch.invert_conditional;
3540 bool is_discard = ins->branch.target_type == TARGET_DISCARD;
3541
3542 /* Determine the block we're jumping to */
3543 int target_number = ins->branch.target_block;
3544
3545 /* Report the destination tag. Discards don't need this */
3546 int dest_tag = is_discard ? 0 : midgard_get_first_tag_from_block(ctx, target_number);
3547
3548 /* Count up the number of quadwords we're jumping over. That is, the number of quadwords in each of the blocks between (br_block_idx, target_number) */
3549 int quadword_offset = 0;
3550
3551 if (is_discard) {
3552 /* Jump to the end of the shader. We
3553 * need to include not only the
3554 * following blocks, but also the
3555 * contents of our current block (since
3556 * discard can come in the middle of
3557 * the block) */
3558
3559 midgard_block *blk = mir_get_block(ctx, br_block_idx + 1);
3560
3561 for (midgard_bundle *bun = bundle + 1; bun < (midgard_bundle *)((char*) block->bundles.data + block->bundles.size); ++bun) {
3562 quadword_offset += quadword_size(bun->tag);
3563 }
3564
3565 mir_foreach_block_from(ctx, blk, b) {
3566 quadword_offset += b->quadword_count;
3567 }
3568
3569 } else if (target_number > br_block_idx) {
3570 /* Jump forward */
3571
3572 for (int idx = br_block_idx + 1; idx < target_number; ++idx) {
3573 midgard_block *blk = mir_get_block(ctx, idx);
3574 assert(blk);
3575
3576 quadword_offset += blk->quadword_count;
3577 }
3578 } else {
3579 /* Jump backwards */
3580
3581 for (int idx = br_block_idx; idx >= target_number; --idx) {
3582 midgard_block *blk = mir_get_block(ctx, idx);
3583 assert(blk);
3584
3585 quadword_offset -= blk->quadword_count;
3586 }
3587 }
3588
3589 /* Unconditional extended branches (far jumps)
3590 * have issues, so we always use a conditional
3591 * branch, setting the condition to always for
3592 * unconditional. For compact unconditional
3593 * branches, cond isn't used so it doesn't
3594 * matter what we pick. */
3595
3596 midgard_condition cond =
3597 !is_conditional ? midgard_condition_always :
3598 is_inverted ? midgard_condition_false :
3599 midgard_condition_true;
3600
3601 midgard_jmp_writeout_op op =
3602 is_discard ? midgard_jmp_writeout_op_discard :
3603 (is_compact && !is_conditional) ? midgard_jmp_writeout_op_branch_uncond :
3604 midgard_jmp_writeout_op_branch_cond;
3605
3606 if (!is_compact) {
3607 midgard_branch_extended branch =
3608 midgard_create_branch_extended(
3609 cond, op,
3610 dest_tag,
3611 quadword_offset);
3612
3613 memcpy(&ins->branch_extended, &branch, sizeof(branch));
3614 } else if (is_conditional || is_discard) {
3615 midgard_branch_cond branch = {
3616 .op = op,
3617 .dest_tag = dest_tag,
3618 .offset = quadword_offset,
3619 .cond = cond
3620 };
3621
3622 assert(branch.offset == quadword_offset);
3623
3624 memcpy(&ins->br_compact, &branch, sizeof(branch));
3625 } else {
3626 assert(op == midgard_jmp_writeout_op_branch_uncond);
3627
3628 midgard_branch_uncond branch = {
3629 .op = op,
3630 .dest_tag = dest_tag,
3631 .offset = quadword_offset,
3632 .unknown = 1
3633 };
3634
3635 assert(branch.offset == quadword_offset);
3636
3637 memcpy(&ins->br_compact, &branch, sizeof(branch));
3638 }
3639 }
3640 }
3641
3642 ++br_block_idx;
3643 }
3644
3645 /* Emit flat binary from the instruction arrays. Iterate each block in
3646 * sequence. Save instruction boundaries such that lookahead tags can
3647 * be assigned easily */
3648
3649 /* Cache _all_ bundles in source order for lookahead across failed branches */
3650
3651 int bundle_count = 0;
3652 mir_foreach_block(ctx, block) {
3653 bundle_count += block->bundles.size / sizeof(midgard_bundle);
3654 }
3655 midgard_bundle **source_order_bundles = malloc(sizeof(midgard_bundle *) * bundle_count);
3656 int bundle_idx = 0;
3657 mir_foreach_block(ctx, block) {
3658 util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) {
3659 source_order_bundles[bundle_idx++] = bundle;
3660 }
3661 }
3662
3663 int current_bundle = 0;
3664
3665 mir_foreach_block(ctx, block) {
3666 util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) {
3667 int lookahead = 1;
3668
3669 if (current_bundle + 1 < bundle_count) {
3670 uint8_t next = source_order_bundles[current_bundle + 1]->tag;
3671
3672 if (!(current_bundle + 2 < bundle_count) && IS_ALU(next)) {
3673 lookahead = 1;
3674 } else {
3675 lookahead = next;
3676 }
3677 }
3678
3679 emit_binary_bundle(ctx, bundle, compiled, lookahead);
3680 ++current_bundle;
3681 }
3682
3683 /* TODO: Free deeper */
3684 //util_dynarray_fini(&block->instructions);
3685 }
3686
3687 free(source_order_bundles);
3688
3689 /* Report the very first tag executed */
3690 program->first_tag = midgard_get_first_tag_from_block(ctx, 0);
3691
3692 /* Deal with off-by-one related to the fencepost problem */
3693 program->work_register_count = ctx->work_registers + 1;
3694
3695 program->can_discard = ctx->can_discard;
3696 program->uniform_cutoff = ctx->uniform_cutoff;
3697
3698 program->blend_patch_offset = ctx->blend_constant_offset;
3699
3700 disassemble_midgard(program->compiled.data, program->compiled.size);
3701
3702 return 0;
3703 }