pan/midgard: Index blocks for printing
[mesa.git] / src / panfrost / midgard / midgard_compile.c
1 /*
2 * Copyright (C) 2018-2019 Alyssa Rosenzweig <alyssa@rosenzweig.io>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
23
24 #include <sys/types.h>
25 #include <sys/stat.h>
26 #include <sys/mman.h>
27 #include <fcntl.h>
28 #include <stdint.h>
29 #include <stdlib.h>
30 #include <stdio.h>
31 #include <err.h>
32
33 #include "main/mtypes.h"
34 #include "compiler/glsl/glsl_to_nir.h"
35 #include "compiler/nir_types.h"
36 #include "main/imports.h"
37 #include "compiler/nir/nir_builder.h"
38 #include "util/half_float.h"
39 #include "util/u_math.h"
40 #include "util/u_debug.h"
41 #include "util/u_dynarray.h"
42 #include "util/list.h"
43 #include "main/mtypes.h"
44
45 #include "midgard.h"
46 #include "midgard_nir.h"
47 #include "midgard_compile.h"
48 #include "midgard_ops.h"
49 #include "helpers.h"
50 #include "compiler.h"
51
52 #include "disassemble.h"
53
54 static const struct debug_named_value debug_options[] = {
55 {"msgs", MIDGARD_DBG_MSGS, "Print debug messages"},
56 {"shaders", MIDGARD_DBG_SHADERS, "Dump shaders in NIR and MIR"},
57 {"shaderdb", MIDGARD_DBG_SHADERDB, "Prints shader-db statistics"},
58 DEBUG_NAMED_VALUE_END
59 };
60
61 DEBUG_GET_ONCE_FLAGS_OPTION(midgard_debug, "MIDGARD_MESA_DEBUG", debug_options, 0)
62
63 unsigned SHADER_DB_COUNT = 0;
64
65 int midgard_debug = 0;
66
67 #define DBG(fmt, ...) \
68 do { if (midgard_debug & MIDGARD_DBG_MSGS) \
69 fprintf(stderr, "%s:%d: "fmt, \
70 __FUNCTION__, __LINE__, ##__VA_ARGS__); } while (0)
71
72 static bool
73 midgard_is_branch_unit(unsigned unit)
74 {
75 return (unit == ALU_ENAB_BRANCH) || (unit == ALU_ENAB_BR_COMPACT);
76 }
77
78 static void
79 midgard_block_add_successor(midgard_block *block, midgard_block *successor)
80 {
81 assert(block);
82 assert(successor);
83
84 /* Deduplicate */
85 for (unsigned i = 0; i < block->nr_successors; ++i) {
86 if (block->successors[i] == successor)
87 return;
88 }
89
90 block->successors[block->nr_successors++] = successor;
91 assert(block->nr_successors <= ARRAY_SIZE(block->successors));
92
93 /* Note the predecessor in the other direction */
94 _mesa_set_add(successor->predecessors, block);
95 }
96
97 /* Helpers to generate midgard_instruction's using macro magic, since every
98 * driver seems to do it that way */
99
100 #define EMIT(op, ...) emit_mir_instruction(ctx, v_##op(__VA_ARGS__));
101
102 #define M_LOAD_STORE(name, store) \
103 static midgard_instruction m_##name(unsigned ssa, unsigned address) { \
104 midgard_instruction i = { \
105 .type = TAG_LOAD_STORE_4, \
106 .mask = 0xF, \
107 .ssa_args = { \
108 .dest = -1, \
109 .src = { -1, -1, -1 }, \
110 }, \
111 .load_store = { \
112 .op = midgard_op_##name, \
113 .swizzle = SWIZZLE_XYZW, \
114 .address = address \
115 } \
116 }; \
117 \
118 if (store) \
119 i.ssa_args.src[0] = ssa; \
120 else \
121 i.ssa_args.dest = ssa; \
122 \
123 return i; \
124 }
125
126 #define M_LOAD(name) M_LOAD_STORE(name, false)
127 #define M_STORE(name) M_LOAD_STORE(name, true)
128
129 /* Inputs a NIR ALU source, with modifiers attached if necessary, and outputs
130 * the corresponding Midgard source */
131
132 static midgard_vector_alu_src
133 vector_alu_modifiers(nir_alu_src *src, bool is_int, unsigned broadcast_count,
134 bool half, bool sext)
135 {
136 if (!src) return blank_alu_src;
137
138 /* Figure out how many components there are so we can adjust the
139 * swizzle. Specifically we want to broadcast the last channel so
140 * things like ball2/3 work
141 */
142
143 if (broadcast_count) {
144 uint8_t last_component = src->swizzle[broadcast_count - 1];
145
146 for (unsigned c = broadcast_count; c < NIR_MAX_VEC_COMPONENTS; ++c) {
147 src->swizzle[c] = last_component;
148 }
149 }
150
151 midgard_vector_alu_src alu_src = {
152 .rep_low = 0,
153 .rep_high = 0,
154 .half = half,
155 .swizzle = SWIZZLE_FROM_ARRAY(src->swizzle)
156 };
157
158 if (is_int) {
159 alu_src.mod = midgard_int_normal;
160
161 /* Sign/zero-extend if needed */
162
163 if (half) {
164 alu_src.mod = sext ?
165 midgard_int_sign_extend
166 : midgard_int_zero_extend;
167 }
168
169 /* These should have been lowered away */
170 assert(!(src->abs || src->negate));
171 } else {
172 alu_src.mod = (src->abs << 0) | (src->negate << 1);
173 }
174
175 return alu_src;
176 }
177
178 /* load/store instructions have both 32-bit and 16-bit variants, depending on
179 * whether we are using vectors composed of highp or mediump. At the moment, we
180 * don't support half-floats -- this requires changes in other parts of the
181 * compiler -- therefore the 16-bit versions are commented out. */
182
183 //M_LOAD(ld_attr_16);
184 M_LOAD(ld_attr_32);
185 //M_LOAD(ld_vary_16);
186 M_LOAD(ld_vary_32);
187 M_LOAD(ld_ubo_int4);
188 M_LOAD(ld_int4);
189 M_STORE(st_int4);
190 M_LOAD(ld_color_buffer_8);
191 //M_STORE(st_vary_16);
192 M_STORE(st_vary_32);
193 M_LOAD(st_cubemap_coords);
194 M_LOAD(ld_compute_id);
195
196 static midgard_instruction
197 v_alu_br_compact_cond(midgard_jmp_writeout_op op, unsigned tag, signed offset, unsigned cond)
198 {
199 midgard_branch_cond branch = {
200 .op = op,
201 .dest_tag = tag,
202 .offset = offset,
203 .cond = cond
204 };
205
206 uint16_t compact;
207 memcpy(&compact, &branch, sizeof(branch));
208
209 midgard_instruction ins = {
210 .type = TAG_ALU_4,
211 .unit = ALU_ENAB_BR_COMPACT,
212 .prepacked_branch = true,
213 .compact_branch = true,
214 .br_compact = compact,
215 .ssa_args = {
216 .dest = -1,
217 .src = { -1, -1, -1 },
218 }
219 };
220
221 if (op == midgard_jmp_writeout_op_writeout)
222 ins.writeout = true;
223
224 return ins;
225 }
226
227 static midgard_instruction
228 v_branch(bool conditional, bool invert)
229 {
230 midgard_instruction ins = {
231 .type = TAG_ALU_4,
232 .unit = ALU_ENAB_BRANCH,
233 .compact_branch = true,
234 .branch = {
235 .conditional = conditional,
236 .invert_conditional = invert
237 },
238 .ssa_args = {
239 .dest = -1,
240 .src = { -1, -1, -1 },
241 }
242 };
243
244 return ins;
245 }
246
247 static midgard_branch_extended
248 midgard_create_branch_extended( midgard_condition cond,
249 midgard_jmp_writeout_op op,
250 unsigned dest_tag,
251 signed quadword_offset)
252 {
253 /* The condition code is actually a LUT describing a function to
254 * combine multiple condition codes. However, we only support a single
255 * condition code at the moment, so we just duplicate over a bunch of
256 * times. */
257
258 uint16_t duplicated_cond =
259 (cond << 14) |
260 (cond << 12) |
261 (cond << 10) |
262 (cond << 8) |
263 (cond << 6) |
264 (cond << 4) |
265 (cond << 2) |
266 (cond << 0);
267
268 midgard_branch_extended branch = {
269 .op = op,
270 .dest_tag = dest_tag,
271 .offset = quadword_offset,
272 .cond = duplicated_cond
273 };
274
275 return branch;
276 }
277
278 static void
279 attach_constants(compiler_context *ctx, midgard_instruction *ins, void *constants, int name)
280 {
281 ins->has_constants = true;
282 memcpy(&ins->constants, constants, 16);
283 }
284
285 static int
286 glsl_type_size(const struct glsl_type *type, bool bindless)
287 {
288 return glsl_count_attribute_slots(type, false);
289 }
290
291 /* Lower fdot2 to a vector multiplication followed by channel addition */
292 static void
293 midgard_nir_lower_fdot2_body(nir_builder *b, nir_alu_instr *alu)
294 {
295 if (alu->op != nir_op_fdot2)
296 return;
297
298 b->cursor = nir_before_instr(&alu->instr);
299
300 nir_ssa_def *src0 = nir_ssa_for_alu_src(b, alu, 0);
301 nir_ssa_def *src1 = nir_ssa_for_alu_src(b, alu, 1);
302
303 nir_ssa_def *product = nir_fmul(b, src0, src1);
304
305 nir_ssa_def *sum = nir_fadd(b,
306 nir_channel(b, product, 0),
307 nir_channel(b, product, 1));
308
309 /* Replace the fdot2 with this sum */
310 nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(sum));
311 }
312
313 static int
314 midgard_sysval_for_ssbo(nir_intrinsic_instr *instr)
315 {
316 /* This is way too meta */
317 bool is_store = instr->intrinsic == nir_intrinsic_store_ssbo;
318 unsigned idx_idx = is_store ? 1 : 0;
319
320 nir_src index = instr->src[idx_idx];
321 assert(nir_src_is_const(index));
322 uint32_t uindex = nir_src_as_uint(index);
323
324 return PAN_SYSVAL(SSBO, uindex);
325 }
326
327 static int
328 midgard_nir_sysval_for_intrinsic(nir_intrinsic_instr *instr)
329 {
330 switch (instr->intrinsic) {
331 case nir_intrinsic_load_viewport_scale:
332 return PAN_SYSVAL_VIEWPORT_SCALE;
333 case nir_intrinsic_load_viewport_offset:
334 return PAN_SYSVAL_VIEWPORT_OFFSET;
335 case nir_intrinsic_load_num_work_groups:
336 return PAN_SYSVAL_NUM_WORK_GROUPS;
337 case nir_intrinsic_load_ssbo:
338 case nir_intrinsic_store_ssbo:
339 return midgard_sysval_for_ssbo(instr);
340 default:
341 return -1;
342 }
343 }
344
345 static int sysval_for_instr(compiler_context *ctx, nir_instr *instr,
346 unsigned *dest)
347 {
348 nir_intrinsic_instr *intr;
349 nir_dest *dst = NULL;
350 nir_tex_instr *tex;
351 int sysval = -1;
352
353 bool is_store = false;
354
355 switch (instr->type) {
356 case nir_instr_type_intrinsic:
357 intr = nir_instr_as_intrinsic(instr);
358 sysval = midgard_nir_sysval_for_intrinsic(intr);
359 dst = &intr->dest;
360 is_store |= intr->intrinsic == nir_intrinsic_store_ssbo;
361 break;
362 case nir_instr_type_tex:
363 tex = nir_instr_as_tex(instr);
364 if (tex->op != nir_texop_txs)
365 break;
366
367 sysval = PAN_SYSVAL(TEXTURE_SIZE,
368 PAN_TXS_SYSVAL_ID(tex->texture_index,
369 nir_tex_instr_dest_size(tex) -
370 (tex->is_array ? 1 : 0),
371 tex->is_array));
372 dst = &tex->dest;
373 break;
374 default:
375 break;
376 }
377
378 if (dest && dst && !is_store)
379 *dest = nir_dest_index(ctx, dst);
380
381 return sysval;
382 }
383
384 static void
385 midgard_nir_assign_sysval_body(compiler_context *ctx, nir_instr *instr)
386 {
387 int sysval;
388
389 sysval = sysval_for_instr(ctx, instr, NULL);
390 if (sysval < 0)
391 return;
392
393 /* We have a sysval load; check if it's already been assigned */
394
395 if (_mesa_hash_table_u64_search(ctx->sysval_to_id, sysval))
396 return;
397
398 /* It hasn't -- so assign it now! */
399
400 unsigned id = ctx->sysval_count++;
401 _mesa_hash_table_u64_insert(ctx->sysval_to_id, sysval, (void *) ((uintptr_t) id + 1));
402 ctx->sysvals[id] = sysval;
403 }
404
405 static void
406 midgard_nir_assign_sysvals(compiler_context *ctx, nir_shader *shader)
407 {
408 ctx->sysval_count = 0;
409
410 nir_foreach_function(function, shader) {
411 if (!function->impl) continue;
412
413 nir_foreach_block(block, function->impl) {
414 nir_foreach_instr_safe(instr, block) {
415 midgard_nir_assign_sysval_body(ctx, instr);
416 }
417 }
418 }
419 }
420
421 static bool
422 midgard_nir_lower_fdot2(nir_shader *shader)
423 {
424 bool progress = false;
425
426 nir_foreach_function(function, shader) {
427 if (!function->impl) continue;
428
429 nir_builder _b;
430 nir_builder *b = &_b;
431 nir_builder_init(b, function->impl);
432
433 nir_foreach_block(block, function->impl) {
434 nir_foreach_instr_safe(instr, block) {
435 if (instr->type != nir_instr_type_alu) continue;
436
437 nir_alu_instr *alu = nir_instr_as_alu(instr);
438 midgard_nir_lower_fdot2_body(b, alu);
439
440 progress |= true;
441 }
442 }
443
444 nir_metadata_preserve(function->impl, nir_metadata_block_index | nir_metadata_dominance);
445
446 }
447
448 return progress;
449 }
450
451 /* Flushes undefined values to zero */
452
453 static void
454 optimise_nir(nir_shader *nir)
455 {
456 bool progress;
457 unsigned lower_flrp =
458 (nir->options->lower_flrp16 ? 16 : 0) |
459 (nir->options->lower_flrp32 ? 32 : 0) |
460 (nir->options->lower_flrp64 ? 64 : 0);
461
462 NIR_PASS(progress, nir, nir_lower_regs_to_ssa);
463 NIR_PASS(progress, nir, midgard_nir_lower_fdot2);
464 NIR_PASS(progress, nir, nir_lower_idiv);
465
466 nir_lower_tex_options lower_tex_options = {
467 .lower_txs_lod = true,
468 .lower_txp = ~0
469 };
470
471 NIR_PASS(progress, nir, nir_lower_tex, &lower_tex_options);
472
473 do {
474 progress = false;
475
476 NIR_PASS(progress, nir, nir_lower_var_copies);
477 NIR_PASS(progress, nir, nir_lower_vars_to_ssa);
478
479 NIR_PASS(progress, nir, nir_copy_prop);
480 NIR_PASS(progress, nir, nir_opt_dce);
481 NIR_PASS(progress, nir, nir_opt_dead_cf);
482 NIR_PASS(progress, nir, nir_opt_cse);
483 NIR_PASS(progress, nir, nir_opt_peephole_select, 64, false, true);
484 NIR_PASS(progress, nir, nir_opt_algebraic);
485 NIR_PASS(progress, nir, nir_opt_constant_folding);
486
487 if (lower_flrp != 0) {
488 bool lower_flrp_progress = false;
489 NIR_PASS(lower_flrp_progress,
490 nir,
491 nir_lower_flrp,
492 lower_flrp,
493 false /* always_precise */,
494 nir->options->lower_ffma);
495 if (lower_flrp_progress) {
496 NIR_PASS(progress, nir,
497 nir_opt_constant_folding);
498 progress = true;
499 }
500
501 /* Nothing should rematerialize any flrps, so we only
502 * need to do this lowering once.
503 */
504 lower_flrp = 0;
505 }
506
507 NIR_PASS(progress, nir, nir_opt_undef);
508 NIR_PASS(progress, nir, nir_undef_to_zero);
509
510 NIR_PASS(progress, nir, nir_opt_loop_unroll,
511 nir_var_shader_in |
512 nir_var_shader_out |
513 nir_var_function_temp);
514
515 NIR_PASS(progress, nir, nir_opt_vectorize);
516 } while (progress);
517
518 /* Must be run at the end to prevent creation of fsin/fcos ops */
519 NIR_PASS(progress, nir, midgard_nir_scale_trig);
520
521 do {
522 progress = false;
523
524 NIR_PASS(progress, nir, nir_opt_dce);
525 NIR_PASS(progress, nir, nir_opt_algebraic);
526 NIR_PASS(progress, nir, nir_opt_constant_folding);
527 NIR_PASS(progress, nir, nir_copy_prop);
528 } while (progress);
529
530 NIR_PASS(progress, nir, nir_opt_algebraic_late);
531
532 /* We implement booleans as 32-bit 0/~0 */
533 NIR_PASS(progress, nir, nir_lower_bool_to_int32);
534
535 /* Now that booleans are lowered, we can run out late opts */
536 NIR_PASS(progress, nir, midgard_nir_lower_algebraic_late);
537
538 /* Lower mods for float ops only. Integer ops don't support modifiers
539 * (saturate doesn't make sense on integers, neg/abs require dedicated
540 * instructions) */
541
542 NIR_PASS(progress, nir, nir_lower_to_source_mods, nir_lower_float_source_mods);
543 NIR_PASS(progress, nir, nir_copy_prop);
544 NIR_PASS(progress, nir, nir_opt_dce);
545
546 /* Take us out of SSA */
547 NIR_PASS(progress, nir, nir_lower_locals_to_regs);
548 NIR_PASS(progress, nir, nir_convert_from_ssa, true);
549
550 /* We are a vector architecture; write combine where possible */
551 NIR_PASS(progress, nir, nir_move_vec_src_uses_to_dest);
552 NIR_PASS(progress, nir, nir_lower_vec_to_movs);
553
554 NIR_PASS(progress, nir, nir_opt_dce);
555 }
556
557 /* Do not actually emit a load; instead, cache the constant for inlining */
558
559 static void
560 emit_load_const(compiler_context *ctx, nir_load_const_instr *instr)
561 {
562 nir_ssa_def def = instr->def;
563
564 float *v = rzalloc_array(NULL, float, 4);
565 nir_const_load_to_arr(v, instr, f32);
566
567 /* Shifted for SSA, +1 for off-by-one */
568 _mesa_hash_table_u64_insert(ctx->ssa_constants, (def.index << 1) + 1, v);
569 }
570
571 /* Normally constants are embedded implicitly, but for I/O and such we have to
572 * explicitly emit a move with the constant source */
573
574 static void
575 emit_explicit_constant(compiler_context *ctx, unsigned node, unsigned to)
576 {
577 void *constant_value = _mesa_hash_table_u64_search(ctx->ssa_constants, node + 1);
578
579 if (constant_value) {
580 midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), blank_alu_src, to);
581 attach_constants(ctx, &ins, constant_value, node + 1);
582 emit_mir_instruction(ctx, ins);
583 }
584 }
585
586 static bool
587 nir_is_non_scalar_swizzle(nir_alu_src *src, unsigned nr_components)
588 {
589 unsigned comp = src->swizzle[0];
590
591 for (unsigned c = 1; c < nr_components; ++c) {
592 if (src->swizzle[c] != comp)
593 return true;
594 }
595
596 return false;
597 }
598
599 /* Midgard puts scalar conditionals in r31.w; move an arbitrary source (the
600 * output of a conditional test) into that register */
601
602 static void
603 emit_condition(compiler_context *ctx, nir_src *src, bool for_branch, unsigned component)
604 {
605 int condition = nir_src_index(ctx, src);
606
607 /* Source to swizzle the desired component into w */
608
609 const midgard_vector_alu_src alu_src = {
610 .swizzle = SWIZZLE(component, component, component, component),
611 };
612
613 /* There is no boolean move instruction. Instead, we simulate a move by
614 * ANDing the condition with itself to get it into r31.w */
615
616 midgard_instruction ins = {
617 .type = TAG_ALU_4,
618
619 /* We need to set the conditional as close as possible */
620 .precede_break = true,
621 .unit = for_branch ? UNIT_SMUL : UNIT_SADD,
622 .mask = 1 << COMPONENT_W,
623
624 .ssa_args = {
625 .src = { condition, condition, -1 },
626 .dest = SSA_FIXED_REGISTER(31),
627 },
628
629 .alu = {
630 .op = midgard_alu_op_iand,
631 .outmod = midgard_outmod_int_wrap,
632 .reg_mode = midgard_reg_mode_32,
633 .dest_override = midgard_dest_override_none,
634 .src1 = vector_alu_srco_unsigned(alu_src),
635 .src2 = vector_alu_srco_unsigned(alu_src)
636 },
637 };
638
639 emit_mir_instruction(ctx, ins);
640 }
641
642 /* Or, for mixed conditions (with csel_v), here's a vector version using all of
643 * r31 instead */
644
645 static void
646 emit_condition_mixed(compiler_context *ctx, nir_alu_src *src, unsigned nr_comp)
647 {
648 int condition = nir_src_index(ctx, &src->src);
649
650 /* Source to swizzle the desired component into w */
651
652 const midgard_vector_alu_src alu_src = {
653 .swizzle = SWIZZLE_FROM_ARRAY(src->swizzle),
654 };
655
656 /* There is no boolean move instruction. Instead, we simulate a move by
657 * ANDing the condition with itself to get it into r31.w */
658
659 midgard_instruction ins = {
660 .type = TAG_ALU_4,
661 .precede_break = true,
662 .mask = mask_of(nr_comp),
663 .ssa_args = {
664 .src = { condition, condition, -1 },
665 .dest = SSA_FIXED_REGISTER(31),
666 },
667 .alu = {
668 .op = midgard_alu_op_iand,
669 .outmod = midgard_outmod_int_wrap,
670 .reg_mode = midgard_reg_mode_32,
671 .dest_override = midgard_dest_override_none,
672 .src1 = vector_alu_srco_unsigned(alu_src),
673 .src2 = vector_alu_srco_unsigned(alu_src)
674 },
675 };
676
677 emit_mir_instruction(ctx, ins);
678 }
679
680 #define ALU_CASE(nir, _op) \
681 case nir_op_##nir: \
682 op = midgard_alu_op_##_op; \
683 assert(src_bitsize == dst_bitsize); \
684 break;
685
686 #define ALU_CASE_BCAST(nir, _op, count) \
687 case nir_op_##nir: \
688 op = midgard_alu_op_##_op; \
689 broadcast_swizzle = count; \
690 assert(src_bitsize == dst_bitsize); \
691 break;
692 static bool
693 nir_is_fzero_constant(nir_src src)
694 {
695 if (!nir_src_is_const(src))
696 return false;
697
698 for (unsigned c = 0; c < nir_src_num_components(src); ++c) {
699 if (nir_src_comp_as_float(src, c) != 0.0)
700 return false;
701 }
702
703 return true;
704 }
705
706 /* Analyze the sizes of the inputs to determine which reg mode. Ops needed
707 * special treatment override this anyway. */
708
709 static midgard_reg_mode
710 reg_mode_for_nir(nir_alu_instr *instr)
711 {
712 unsigned src_bitsize = nir_src_bit_size(instr->src[0].src);
713
714 switch (src_bitsize) {
715 case 8:
716 return midgard_reg_mode_8;
717 case 16:
718 return midgard_reg_mode_16;
719 case 32:
720 return midgard_reg_mode_32;
721 case 64:
722 return midgard_reg_mode_64;
723 default:
724 unreachable("Invalid bit size");
725 }
726 }
727
728 static void
729 emit_alu(compiler_context *ctx, nir_alu_instr *instr)
730 {
731 /* Derivatives end up emitted on the texture pipe, not the ALUs. This
732 * is handled elsewhere */
733
734 if (instr->op == nir_op_fddx || instr->op == nir_op_fddy) {
735 midgard_emit_derivatives(ctx, instr);
736 return;
737 }
738
739 bool is_ssa = instr->dest.dest.is_ssa;
740
741 unsigned dest = nir_dest_index(ctx, &instr->dest.dest);
742 unsigned nr_components = nir_dest_num_components(instr->dest.dest);
743 unsigned nr_inputs = nir_op_infos[instr->op].num_inputs;
744
745 /* Most Midgard ALU ops have a 1:1 correspondance to NIR ops; these are
746 * supported. A few do not and are commented for now. Also, there are a
747 * number of NIR ops which Midgard does not support and need to be
748 * lowered, also TODO. This switch block emits the opcode and calling
749 * convention of the Midgard instruction; actual packing is done in
750 * emit_alu below */
751
752 unsigned op;
753
754 /* Number of components valid to check for the instruction (the rest
755 * will be forced to the last), or 0 to use as-is. Relevant as
756 * ball-type instructions have a channel count in NIR but are all vec4
757 * in Midgard */
758
759 unsigned broadcast_swizzle = 0;
760
761 /* What register mode should we operate in? */
762 midgard_reg_mode reg_mode =
763 reg_mode_for_nir(instr);
764
765 /* Do we need a destination override? Used for inline
766 * type conversion */
767
768 midgard_dest_override dest_override =
769 midgard_dest_override_none;
770
771 /* Should we use a smaller respective source and sign-extend? */
772
773 bool half_1 = false, sext_1 = false;
774 bool half_2 = false, sext_2 = false;
775
776 unsigned src_bitsize = nir_src_bit_size(instr->src[0].src);
777 unsigned dst_bitsize = nir_dest_bit_size(instr->dest.dest);
778
779 switch (instr->op) {
780 ALU_CASE(fadd, fadd);
781 ALU_CASE(fmul, fmul);
782 ALU_CASE(fmin, fmin);
783 ALU_CASE(fmax, fmax);
784 ALU_CASE(imin, imin);
785 ALU_CASE(imax, imax);
786 ALU_CASE(umin, umin);
787 ALU_CASE(umax, umax);
788 ALU_CASE(ffloor, ffloor);
789 ALU_CASE(fround_even, froundeven);
790 ALU_CASE(ftrunc, ftrunc);
791 ALU_CASE(fceil, fceil);
792 ALU_CASE(fdot3, fdot3);
793 ALU_CASE(fdot4, fdot4);
794 ALU_CASE(iadd, iadd);
795 ALU_CASE(isub, isub);
796 ALU_CASE(imul, imul);
797
798 /* Zero shoved as second-arg */
799 ALU_CASE(iabs, iabsdiff);
800
801 ALU_CASE(mov, imov);
802
803 ALU_CASE(feq32, feq);
804 ALU_CASE(fne32, fne);
805 ALU_CASE(flt32, flt);
806 ALU_CASE(ieq32, ieq);
807 ALU_CASE(ine32, ine);
808 ALU_CASE(ilt32, ilt);
809 ALU_CASE(ult32, ult);
810
811 /* We don't have a native b2f32 instruction. Instead, like many
812 * GPUs, we exploit booleans as 0/~0 for false/true, and
813 * correspondingly AND
814 * by 1.0 to do the type conversion. For the moment, prime us
815 * to emit:
816 *
817 * iand [whatever], #0
818 *
819 * At the end of emit_alu (as MIR), we'll fix-up the constant
820 */
821
822 ALU_CASE(b2f32, iand);
823 ALU_CASE(b2i32, iand);
824
825 /* Likewise, we don't have a dedicated f2b32 instruction, but
826 * we can do a "not equal to 0.0" test. */
827
828 ALU_CASE(f2b32, fne);
829 ALU_CASE(i2b32, ine);
830
831 ALU_CASE(frcp, frcp);
832 ALU_CASE(frsq, frsqrt);
833 ALU_CASE(fsqrt, fsqrt);
834 ALU_CASE(fexp2, fexp2);
835 ALU_CASE(flog2, flog2);
836
837 ALU_CASE(f2i32, f2i_rtz);
838 ALU_CASE(f2u32, f2u_rtz);
839 ALU_CASE(i2f32, i2f_rtz);
840 ALU_CASE(u2f32, u2f_rtz);
841
842 ALU_CASE(f2i16, f2i_rtz);
843 ALU_CASE(f2u16, f2u_rtz);
844 ALU_CASE(i2f16, i2f_rtz);
845 ALU_CASE(u2f16, u2f_rtz);
846
847 ALU_CASE(fsin, fsin);
848 ALU_CASE(fcos, fcos);
849
850 /* We'll set invert */
851 ALU_CASE(inot, imov);
852 ALU_CASE(iand, iand);
853 ALU_CASE(ior, ior);
854 ALU_CASE(ixor, ixor);
855 ALU_CASE(ishl, ishl);
856 ALU_CASE(ishr, iasr);
857 ALU_CASE(ushr, ilsr);
858
859 ALU_CASE_BCAST(b32all_fequal2, fball_eq, 2);
860 ALU_CASE_BCAST(b32all_fequal3, fball_eq, 3);
861 ALU_CASE(b32all_fequal4, fball_eq);
862
863 ALU_CASE_BCAST(b32any_fnequal2, fbany_neq, 2);
864 ALU_CASE_BCAST(b32any_fnequal3, fbany_neq, 3);
865 ALU_CASE(b32any_fnequal4, fbany_neq);
866
867 ALU_CASE_BCAST(b32all_iequal2, iball_eq, 2);
868 ALU_CASE_BCAST(b32all_iequal3, iball_eq, 3);
869 ALU_CASE(b32all_iequal4, iball_eq);
870
871 ALU_CASE_BCAST(b32any_inequal2, ibany_neq, 2);
872 ALU_CASE_BCAST(b32any_inequal3, ibany_neq, 3);
873 ALU_CASE(b32any_inequal4, ibany_neq);
874
875 /* Source mods will be shoved in later */
876 ALU_CASE(fabs, fmov);
877 ALU_CASE(fneg, fmov);
878 ALU_CASE(fsat, fmov);
879
880 /* For size conversion, we use a move. Ideally though we would squash
881 * these ops together; maybe that has to happen after in NIR as part of
882 * propagation...? An earlier algebraic pass ensured we step down by
883 * only / exactly one size. If stepping down, we use a dest override to
884 * reduce the size; if stepping up, we use a larger-sized move with a
885 * half source and a sign/zero-extension modifier */
886
887 case nir_op_i2i8:
888 case nir_op_i2i16:
889 case nir_op_i2i32:
890 /* If we end up upscale, we'll need a sign-extend on the
891 * operand (the second argument) */
892
893 sext_2 = true;
894 case nir_op_u2u8:
895 case nir_op_u2u16:
896 case nir_op_u2u32: {
897 op = midgard_alu_op_imov;
898
899 if (dst_bitsize == (src_bitsize * 2)) {
900 /* Converting up */
901 half_2 = true;
902
903 /* Use a greater register mode */
904 reg_mode++;
905 } else if (src_bitsize == (dst_bitsize * 2)) {
906 /* Converting down */
907 dest_override = midgard_dest_override_lower;
908 }
909
910 break;
911 }
912
913 case nir_op_f2f16: {
914 assert(src_bitsize == 32);
915
916 op = midgard_alu_op_fmov;
917 dest_override = midgard_dest_override_lower;
918 break;
919 }
920
921 case nir_op_f2f32: {
922 assert(src_bitsize == 16);
923
924 op = midgard_alu_op_fmov;
925 half_2 = true;
926 reg_mode++;
927 break;
928 }
929
930
931 /* For greater-or-equal, we lower to less-or-equal and flip the
932 * arguments */
933
934 case nir_op_fge:
935 case nir_op_fge32:
936 case nir_op_ige32:
937 case nir_op_uge32: {
938 op =
939 instr->op == nir_op_fge ? midgard_alu_op_fle :
940 instr->op == nir_op_fge32 ? midgard_alu_op_fle :
941 instr->op == nir_op_ige32 ? midgard_alu_op_ile :
942 instr->op == nir_op_uge32 ? midgard_alu_op_ule :
943 0;
944
945 /* Swap via temporary */
946 nir_alu_src temp = instr->src[1];
947 instr->src[1] = instr->src[0];
948 instr->src[0] = temp;
949
950 break;
951 }
952
953 case nir_op_b32csel: {
954 /* Midgard features both fcsel and icsel, depending on
955 * the type of the arguments/output. However, as long
956 * as we're careful we can _always_ use icsel and
957 * _never_ need fcsel, since the latter does additional
958 * floating-point-specific processing whereas the
959 * former just moves bits on the wire. It's not obvious
960 * why these are separate opcodes, save for the ability
961 * to do things like sat/pos/abs/neg for free */
962
963 bool mixed = nir_is_non_scalar_swizzle(&instr->src[0], nr_components);
964 op = mixed ? midgard_alu_op_icsel_v : midgard_alu_op_icsel;
965
966 /* csel works as a two-arg in Midgard, since the condition is hardcoded in r31.w */
967 nr_inputs = 2;
968
969 /* Emit the condition into r31 */
970
971 if (mixed)
972 emit_condition_mixed(ctx, &instr->src[0], nr_components);
973 else
974 emit_condition(ctx, &instr->src[0].src, false, instr->src[0].swizzle[0]);
975
976 /* The condition is the first argument; move the other
977 * arguments up one to be a binary instruction for
978 * Midgard */
979
980 memmove(instr->src, instr->src + 1, 2 * sizeof(nir_alu_src));
981 break;
982 }
983
984 default:
985 DBG("Unhandled ALU op %s\n", nir_op_infos[instr->op].name);
986 assert(0);
987 return;
988 }
989
990 /* Midgard can perform certain modifiers on output of an ALU op */
991 unsigned outmod;
992
993 if (midgard_is_integer_out_op(op)) {
994 outmod = midgard_outmod_int_wrap;
995 } else {
996 bool sat = instr->dest.saturate || instr->op == nir_op_fsat;
997 outmod = sat ? midgard_outmod_sat : midgard_outmod_none;
998 }
999
1000 /* fmax(a, 0.0) can turn into a .pos modifier as an optimization */
1001
1002 if (instr->op == nir_op_fmax) {
1003 if (nir_is_fzero_constant(instr->src[0].src)) {
1004 op = midgard_alu_op_fmov;
1005 nr_inputs = 1;
1006 outmod = midgard_outmod_pos;
1007 instr->src[0] = instr->src[1];
1008 } else if (nir_is_fzero_constant(instr->src[1].src)) {
1009 op = midgard_alu_op_fmov;
1010 nr_inputs = 1;
1011 outmod = midgard_outmod_pos;
1012 }
1013 }
1014
1015 /* Fetch unit, quirks, etc information */
1016 unsigned opcode_props = alu_opcode_props[op].props;
1017 bool quirk_flipped_r24 = opcode_props & QUIRK_FLIPPED_R24;
1018
1019 /* src0 will always exist afaik, but src1 will not for 1-argument
1020 * instructions. The latter can only be fetched if the instruction
1021 * needs it, or else we may segfault. */
1022
1023 unsigned src0 = nir_alu_src_index(ctx, &instr->src[0]);
1024 unsigned src1 = nr_inputs == 2 ? nir_alu_src_index(ctx, &instr->src[1]) : SSA_UNUSED_0;
1025
1026 /* Rather than use the instruction generation helpers, we do it
1027 * ourselves here to avoid the mess */
1028
1029 midgard_instruction ins = {
1030 .type = TAG_ALU_4,
1031 .ssa_args = {
1032 .src = {
1033 quirk_flipped_r24 ? SSA_UNUSED_1 : src0,
1034 quirk_flipped_r24 ? src0 : src1,
1035 -1
1036 },
1037 .dest = dest,
1038 }
1039 };
1040
1041 nir_alu_src *nirmods[2] = { NULL };
1042
1043 if (nr_inputs == 2) {
1044 nirmods[0] = &instr->src[0];
1045 nirmods[1] = &instr->src[1];
1046 } else if (nr_inputs == 1) {
1047 nirmods[quirk_flipped_r24] = &instr->src[0];
1048 } else {
1049 assert(0);
1050 }
1051
1052 /* These were lowered to a move, so apply the corresponding mod */
1053
1054 if (instr->op == nir_op_fneg || instr->op == nir_op_fabs) {
1055 nir_alu_src *s = nirmods[quirk_flipped_r24];
1056
1057 if (instr->op == nir_op_fneg)
1058 s->negate = !s->negate;
1059
1060 if (instr->op == nir_op_fabs)
1061 s->abs = !s->abs;
1062 }
1063
1064 bool is_int = midgard_is_integer_op(op);
1065
1066 ins.mask = mask_of(nr_components);
1067
1068 midgard_vector_alu alu = {
1069 .op = op,
1070 .reg_mode = reg_mode,
1071 .dest_override = dest_override,
1072 .outmod = outmod,
1073
1074 .src1 = vector_alu_srco_unsigned(vector_alu_modifiers(nirmods[0], is_int, broadcast_swizzle, half_1, sext_1)),
1075 .src2 = vector_alu_srco_unsigned(vector_alu_modifiers(nirmods[1], is_int, broadcast_swizzle, half_2, sext_2)),
1076 };
1077
1078 /* Apply writemask if non-SSA, keeping in mind that we can't write to components that don't exist */
1079
1080 if (!is_ssa)
1081 ins.mask &= instr->dest.write_mask;
1082
1083 ins.alu = alu;
1084
1085 /* Late fixup for emulated instructions */
1086
1087 if (instr->op == nir_op_b2f32 || instr->op == nir_op_b2i32) {
1088 /* Presently, our second argument is an inline #0 constant.
1089 * Switch over to an embedded 1.0 constant (that can't fit
1090 * inline, since we're 32-bit, not 16-bit like the inline
1091 * constants) */
1092
1093 ins.ssa_args.inline_constant = false;
1094 ins.ssa_args.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT);
1095 ins.has_constants = true;
1096
1097 if (instr->op == nir_op_b2f32) {
1098 ins.constants[0] = 1.0f;
1099 } else {
1100 /* Type pun it into place */
1101 uint32_t one = 0x1;
1102 memcpy(&ins.constants[0], &one, sizeof(uint32_t));
1103 }
1104
1105 ins.alu.src2 = vector_alu_srco_unsigned(blank_alu_src_xxxx);
1106 } else if (nr_inputs == 1 && !quirk_flipped_r24) {
1107 /* Lots of instructions need a 0 plonked in */
1108 ins.ssa_args.inline_constant = false;
1109 ins.ssa_args.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT);
1110 ins.has_constants = true;
1111 ins.constants[0] = 0.0f;
1112 ins.alu.src2 = vector_alu_srco_unsigned(blank_alu_src_xxxx);
1113 } else if (instr->op == nir_op_inot) {
1114 ins.invert = true;
1115 }
1116
1117 if ((opcode_props & UNITS_ALL) == UNIT_VLUT) {
1118 /* To avoid duplicating the lookup tables (probably), true LUT
1119 * instructions can only operate as if they were scalars. Lower
1120 * them here by changing the component. */
1121
1122 uint8_t original_swizzle[4];
1123 memcpy(original_swizzle, nirmods[0]->swizzle, sizeof(nirmods[0]->swizzle));
1124 unsigned orig_mask = ins.mask;
1125
1126 for (int i = 0; i < nr_components; ++i) {
1127 /* Mask the associated component, dropping the
1128 * instruction if needed */
1129
1130 ins.mask = 1 << i;
1131 ins.mask &= orig_mask;
1132
1133 if (!ins.mask)
1134 continue;
1135
1136 for (int j = 0; j < 4; ++j)
1137 nirmods[0]->swizzle[j] = original_swizzle[i]; /* Pull from the correct component */
1138
1139 ins.alu.src1 = vector_alu_srco_unsigned(vector_alu_modifiers(nirmods[0], is_int, broadcast_swizzle, half_1, false));
1140 emit_mir_instruction(ctx, ins);
1141 }
1142 } else {
1143 emit_mir_instruction(ctx, ins);
1144 }
1145 }
1146
1147 #undef ALU_CASE
1148
1149 static unsigned
1150 mir_mask_for_intr(nir_instr *instr, bool is_read)
1151 {
1152 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
1153
1154 if (is_read)
1155 return mask_of(nir_intrinsic_dest_components(intr));
1156 else
1157 return nir_intrinsic_write_mask(intr);
1158 }
1159
1160 /* Uniforms and UBOs use a shared code path, as uniforms are just (slightly
1161 * optimized) versions of UBO #0 */
1162
1163 midgard_instruction *
1164 emit_ubo_read(
1165 compiler_context *ctx,
1166 nir_instr *instr,
1167 unsigned dest,
1168 unsigned offset,
1169 nir_src *indirect_offset,
1170 unsigned index)
1171 {
1172 /* TODO: half-floats */
1173
1174 midgard_instruction ins = m_ld_ubo_int4(dest, offset);
1175
1176 assert((offset & 0xF) == 0);
1177 offset /= 16;
1178
1179 /* TODO: Don't split */
1180 ins.load_store.varying_parameters = (offset & 7) << 7;
1181 ins.load_store.address = offset >> 3;
1182 ins.mask = mir_mask_for_intr(instr, true);
1183
1184 if (indirect_offset) {
1185 ins.ssa_args.src[1] = nir_src_index(ctx, indirect_offset);
1186 ins.load_store.arg_2 = 0x80;
1187 } else {
1188 ins.load_store.arg_2 = 0x1E;
1189 }
1190
1191 ins.load_store.arg_1 = index;
1192
1193 return emit_mir_instruction(ctx, ins);
1194 }
1195
1196 /* SSBO reads are like UBO reads if you squint */
1197
1198 static void
1199 emit_ssbo_access(
1200 compiler_context *ctx,
1201 nir_instr *instr,
1202 bool is_read,
1203 unsigned srcdest,
1204 unsigned offset,
1205 nir_src *indirect_offset,
1206 unsigned index)
1207 {
1208 /* TODO: types */
1209
1210 midgard_instruction ins;
1211
1212 if (is_read)
1213 ins = m_ld_int4(srcdest, offset);
1214 else
1215 ins = m_st_int4(srcdest, offset);
1216
1217 /* SSBO reads use a generic memory read interface, so we need the
1218 * address of the SSBO as the first argument. This is a sysval. */
1219
1220 unsigned addr = make_compiler_temp(ctx);
1221 emit_sysval_read(ctx, instr, addr, 2);
1222
1223 /* The source array is a bit of a leaky abstraction for SSBOs.
1224 * Nevertheless, for loads:
1225 *
1226 * src[0] = arg_1
1227 * src[1] = arg_2
1228 * src[2] = unused
1229 *
1230 * Whereas for stores:
1231 *
1232 * src[0] = value
1233 * src[1] = arg_1
1234 * src[2] = arg_2
1235 *
1236 * We would like arg_1 = the address and
1237 * arg_2 = the offset.
1238 */
1239
1240 ins.ssa_args.src[is_read ? 0 : 1] = addr;
1241
1242 /* TODO: What is this? It looks superficially like a shift << 5, but
1243 * arg_1 doesn't take a shift Should it be E0 or A0? */
1244 if (indirect_offset)
1245 ins.load_store.arg_1 |= 0xE0;
1246
1247 /* We also need to emit the indirect offset */
1248
1249 if (indirect_offset)
1250 ins.ssa_args.src[is_read ? 1 : 2] = nir_src_index(ctx, indirect_offset);
1251 else
1252 ins.load_store.arg_2 = 0x7E;
1253
1254 /* TODO: Bounds check */
1255
1256 /* Finally, we emit the direct offset */
1257
1258 ins.load_store.varying_parameters = (offset & 0x1FF) << 1;
1259 ins.load_store.address = (offset >> 9);
1260 ins.mask = mir_mask_for_intr(instr, is_read);
1261
1262 emit_mir_instruction(ctx, ins);
1263 }
1264
1265 static void
1266 emit_varying_read(
1267 compiler_context *ctx,
1268 unsigned dest, unsigned offset,
1269 unsigned nr_comp, unsigned component,
1270 nir_src *indirect_offset, nir_alu_type type)
1271 {
1272 /* XXX: Half-floats? */
1273 /* TODO: swizzle, mask */
1274
1275 midgard_instruction ins = m_ld_vary_32(dest, offset);
1276 ins.mask = mask_of(nr_comp);
1277 ins.load_store.swizzle = SWIZZLE_XYZW >> (2 * component);
1278
1279 midgard_varying_parameter p = {
1280 .is_varying = 1,
1281 .interpolation = midgard_interp_default,
1282 .flat = /*var->data.interpolation == INTERP_MODE_FLAT*/ 0
1283 };
1284
1285 unsigned u;
1286 memcpy(&u, &p, sizeof(p));
1287 ins.load_store.varying_parameters = u;
1288
1289 if (indirect_offset)
1290 ins.ssa_args.src[1] = nir_src_index(ctx, indirect_offset);
1291 else
1292 ins.load_store.arg_2 = 0x1E;
1293
1294 ins.load_store.arg_1 = 0x9E;
1295
1296 /* Use the type appropriate load */
1297 switch (type) {
1298 case nir_type_uint:
1299 case nir_type_bool:
1300 ins.load_store.op = midgard_op_ld_vary_32u;
1301 break;
1302 case nir_type_int:
1303 ins.load_store.op = midgard_op_ld_vary_32i;
1304 break;
1305 case nir_type_float:
1306 ins.load_store.op = midgard_op_ld_vary_32;
1307 break;
1308 default:
1309 unreachable("Attempted to load unknown type");
1310 break;
1311 }
1312
1313 emit_mir_instruction(ctx, ins);
1314 }
1315
1316 void
1317 emit_sysval_read(compiler_context *ctx, nir_instr *instr, signed dest_override,
1318 unsigned nr_components)
1319 {
1320 unsigned dest = 0;
1321
1322 /* Figure out which uniform this is */
1323 int sysval = sysval_for_instr(ctx, instr, &dest);
1324 void *val = _mesa_hash_table_u64_search(ctx->sysval_to_id, sysval);
1325
1326 if (dest_override >= 0)
1327 dest = dest_override;
1328
1329 /* Sysvals are prefix uniforms */
1330 unsigned uniform = ((uintptr_t) val) - 1;
1331
1332 /* Emit the read itself -- this is never indirect */
1333 midgard_instruction *ins =
1334 emit_ubo_read(ctx, instr, dest, uniform * 16, NULL, 0);
1335
1336 ins->mask = mask_of(nr_components);
1337 }
1338
1339 static unsigned
1340 compute_builtin_arg(nir_op op)
1341 {
1342 switch (op) {
1343 case nir_intrinsic_load_work_group_id:
1344 return 0x14;
1345 case nir_intrinsic_load_local_invocation_id:
1346 return 0x10;
1347 default:
1348 unreachable("Invalid compute paramater loaded");
1349 }
1350 }
1351
1352 /* Emit store for a fragment shader, which is encoded via a fancy branch. TODO:
1353 * Handle MRT here */
1354
1355 static void
1356 emit_fragment_store(compiler_context *ctx, unsigned src, unsigned rt)
1357 {
1358 /* First, move in whatever we're outputting */
1359 midgard_instruction move = v_mov(src, blank_alu_src, SSA_FIXED_REGISTER(0));
1360 if (rt != 0) {
1361 /* Force a tight schedule. TODO: Make the scheduler MRT aware */
1362 move.unit = UNIT_VMUL;
1363 move.precede_break = true;
1364 move.dont_eliminate = true;
1365 }
1366
1367 emit_mir_instruction(ctx, move);
1368
1369 /* If we're doing MRT, we need to specify the render target */
1370
1371 midgard_instruction rt_move = {
1372 .ssa_args = {
1373 .dest = -1
1374 }
1375 };
1376
1377 if (rt != 0) {
1378 /* We'll write to r1.z */
1379 rt_move = v_mov(-1, blank_alu_src, SSA_FIXED_REGISTER(1));
1380 rt_move.mask = 1 << COMPONENT_Z;
1381 rt_move.unit = UNIT_SADD;
1382
1383 /* r1.z = (rt * 0x100) */
1384 rt_move.ssa_args.inline_constant = true;
1385 rt_move.inline_constant = (rt * 0x100);
1386
1387 /* r1 */
1388 ctx->work_registers = MAX2(ctx->work_registers, 1);
1389
1390 /* Do the write */
1391 emit_mir_instruction(ctx, rt_move);
1392 }
1393
1394 /* Next, generate the branch. For R render targets in the writeout, the
1395 * i'th render target jumps to pseudo-offset [2(R-1) + i] */
1396
1397 unsigned offset = (2 * (ctx->nir->num_outputs - 1)) + rt;
1398
1399 struct midgard_instruction ins =
1400 v_alu_br_compact_cond(midgard_jmp_writeout_op_writeout, TAG_ALU_4, offset, midgard_condition_always);
1401
1402 /* Add dependencies */
1403 ins.ssa_args.src[0] = move.ssa_args.dest;
1404 ins.ssa_args.src[1] = rt_move.ssa_args.dest;
1405
1406 /* Emit the branch */
1407 emit_mir_instruction(ctx, ins);
1408 }
1409
1410 static void
1411 emit_compute_builtin(compiler_context *ctx, nir_intrinsic_instr *instr)
1412 {
1413 unsigned reg = nir_dest_index(ctx, &instr->dest);
1414 midgard_instruction ins = m_ld_compute_id(reg, 0);
1415 ins.mask = mask_of(3);
1416 ins.load_store.arg_1 = compute_builtin_arg(instr->intrinsic);
1417 emit_mir_instruction(ctx, ins);
1418 }
1419 static void
1420 emit_intrinsic(compiler_context *ctx, nir_intrinsic_instr *instr)
1421 {
1422 unsigned offset = 0, reg;
1423
1424 switch (instr->intrinsic) {
1425 case nir_intrinsic_discard_if:
1426 emit_condition(ctx, &instr->src[0], true, COMPONENT_X);
1427
1428 /* fallthrough */
1429
1430 case nir_intrinsic_discard: {
1431 bool conditional = instr->intrinsic == nir_intrinsic_discard_if;
1432 struct midgard_instruction discard = v_branch(conditional, false);
1433 discard.branch.target_type = TARGET_DISCARD;
1434 emit_mir_instruction(ctx, discard);
1435 break;
1436 }
1437
1438 case nir_intrinsic_load_uniform:
1439 case nir_intrinsic_load_ubo:
1440 case nir_intrinsic_load_ssbo:
1441 case nir_intrinsic_load_input: {
1442 bool is_uniform = instr->intrinsic == nir_intrinsic_load_uniform;
1443 bool is_ubo = instr->intrinsic == nir_intrinsic_load_ubo;
1444 bool is_ssbo = instr->intrinsic == nir_intrinsic_load_ssbo;
1445
1446 /* Get the base type of the intrinsic */
1447 /* TODO: Infer type? Does it matter? */
1448 nir_alu_type t =
1449 (is_ubo || is_ssbo) ? nir_type_uint : nir_intrinsic_type(instr);
1450 t = nir_alu_type_get_base_type(t);
1451
1452 if (!(is_ubo || is_ssbo)) {
1453 offset = nir_intrinsic_base(instr);
1454 }
1455
1456 unsigned nr_comp = nir_intrinsic_dest_components(instr);
1457
1458 nir_src *src_offset = nir_get_io_offset_src(instr);
1459
1460 bool direct = nir_src_is_const(*src_offset);
1461 nir_src *indirect_offset = direct ? NULL : src_offset;
1462
1463 if (direct)
1464 offset += nir_src_as_uint(*src_offset);
1465
1466 /* We may need to apply a fractional offset */
1467 int component = instr->intrinsic == nir_intrinsic_load_input ?
1468 nir_intrinsic_component(instr) : 0;
1469 reg = nir_dest_index(ctx, &instr->dest);
1470
1471 if (is_uniform && !ctx->is_blend) {
1472 emit_ubo_read(ctx, &instr->instr, reg, (ctx->sysval_count + offset) * 16, indirect_offset, 0);
1473 } else if (is_ubo) {
1474 nir_src index = instr->src[0];
1475
1476 /* We don't yet support indirect UBOs. For indirect
1477 * block numbers (if that's possible), we don't know
1478 * enough about the hardware yet. For indirect sources,
1479 * we know what we need but we need to add some NIR
1480 * support for lowering correctly with respect to
1481 * 128-bit reads */
1482
1483 assert(nir_src_is_const(index));
1484 assert(nir_src_is_const(*src_offset));
1485
1486 uint32_t uindex = nir_src_as_uint(index) + 1;
1487 emit_ubo_read(ctx, &instr->instr, reg, offset, NULL, uindex);
1488 } else if (is_ssbo) {
1489 nir_src index = instr->src[0];
1490 assert(nir_src_is_const(index));
1491 uint32_t uindex = nir_src_as_uint(index);
1492
1493 emit_ssbo_access(ctx, &instr->instr, true, reg, offset, indirect_offset, uindex);
1494 } else if (ctx->stage == MESA_SHADER_FRAGMENT && !ctx->is_blend) {
1495 emit_varying_read(ctx, reg, offset, nr_comp, component, !direct ? &instr->src[0] : NULL, t);
1496 } else if (ctx->is_blend) {
1497 /* For blend shaders, load the input color, which is
1498 * preloaded to r0 */
1499
1500 midgard_instruction move = v_mov(SSA_FIXED_REGISTER(0), blank_alu_src, reg);
1501 emit_mir_instruction(ctx, move);
1502 } else if (ctx->stage == MESA_SHADER_VERTEX) {
1503 midgard_instruction ins = m_ld_attr_32(reg, offset);
1504 ins.load_store.arg_1 = 0x1E;
1505 ins.load_store.arg_2 = 0x1E;
1506 ins.mask = mask_of(nr_comp);
1507
1508 /* Use the type appropriate load */
1509 switch (t) {
1510 case nir_type_uint:
1511 case nir_type_bool:
1512 ins.load_store.op = midgard_op_ld_attr_32u;
1513 break;
1514 case nir_type_int:
1515 ins.load_store.op = midgard_op_ld_attr_32i;
1516 break;
1517 case nir_type_float:
1518 ins.load_store.op = midgard_op_ld_attr_32;
1519 break;
1520 default:
1521 unreachable("Attempted to load unknown type");
1522 break;
1523 }
1524
1525 emit_mir_instruction(ctx, ins);
1526 } else {
1527 DBG("Unknown load\n");
1528 assert(0);
1529 }
1530
1531 break;
1532 }
1533
1534 /* Reads 128-bit value raw off the tilebuffer during blending, tasty */
1535
1536 case nir_intrinsic_load_raw_output_pan:
1537 reg = nir_dest_index(ctx, &instr->dest);
1538 assert(ctx->is_blend);
1539
1540 midgard_instruction ins = m_ld_color_buffer_8(reg, 0);
1541 emit_mir_instruction(ctx, ins);
1542 break;
1543
1544 case nir_intrinsic_load_blend_const_color_rgba: {
1545 assert(ctx->is_blend);
1546 reg = nir_dest_index(ctx, &instr->dest);
1547
1548 /* Blend constants are embedded directly in the shader and
1549 * patched in, so we use some magic routing */
1550
1551 midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), blank_alu_src, reg);
1552 ins.has_constants = true;
1553 ins.has_blend_constant = true;
1554 emit_mir_instruction(ctx, ins);
1555 break;
1556 }
1557
1558 case nir_intrinsic_store_output:
1559 assert(nir_src_is_const(instr->src[1]) && "no indirect outputs");
1560
1561 offset = nir_intrinsic_base(instr) + nir_src_as_uint(instr->src[1]);
1562
1563 reg = nir_src_index(ctx, &instr->src[0]);
1564
1565 if (ctx->stage == MESA_SHADER_FRAGMENT) {
1566 /* Determine number of render targets */
1567 emit_fragment_store(ctx, reg, offset);
1568 } else if (ctx->stage == MESA_SHADER_VERTEX) {
1569 /* We should have been vectorized, though we don't
1570 * currently check that st_vary is emitted only once
1571 * per slot (this is relevant, since there's not a mask
1572 * parameter available on the store [set to 0 by the
1573 * blob]). We do respect the component by adjusting the
1574 * swizzle. If this is a constant source, we'll need to
1575 * emit that explicitly. */
1576
1577 emit_explicit_constant(ctx, reg, reg);
1578
1579 unsigned component = nir_intrinsic_component(instr);
1580
1581 midgard_instruction st = m_st_vary_32(reg, offset);
1582 st.load_store.arg_1 = 0x9E;
1583 st.load_store.arg_2 = 0x1E;
1584 st.load_store.swizzle = SWIZZLE_XYZW << (2*component);
1585 emit_mir_instruction(ctx, st);
1586 } else {
1587 DBG("Unknown store\n");
1588 assert(0);
1589 }
1590
1591 break;
1592
1593 /* Special case of store_output for lowered blend shaders */
1594 case nir_intrinsic_store_raw_output_pan:
1595 assert (ctx->stage == MESA_SHADER_FRAGMENT);
1596 reg = nir_src_index(ctx, &instr->src[0]);
1597 emit_fragment_store(ctx, reg, 0);
1598
1599 break;
1600
1601 case nir_intrinsic_store_ssbo:
1602 assert(nir_src_is_const(instr->src[1]));
1603
1604 bool direct_offset = nir_src_is_const(instr->src[2]);
1605 offset = direct_offset ? nir_src_as_uint(instr->src[2]) : 0;
1606 nir_src *indirect_offset = direct_offset ? NULL : &instr->src[2];
1607 reg = nir_src_index(ctx, &instr->src[0]);
1608
1609 uint32_t uindex = nir_src_as_uint(instr->src[1]);
1610
1611 emit_explicit_constant(ctx, reg, reg);
1612 emit_ssbo_access(ctx, &instr->instr, false, reg, offset, indirect_offset, uindex);
1613 break;
1614
1615 case nir_intrinsic_load_alpha_ref_float:
1616 assert(instr->dest.is_ssa);
1617
1618 float ref_value = ctx->alpha_ref;
1619
1620 /* See emit_load_const */
1621 float *v = ralloc_array(NULL, float, 4);
1622 memcpy(v, &ref_value, sizeof(float));
1623 _mesa_hash_table_u64_insert(ctx->ssa_constants, (instr->dest.ssa.index << 1) + 1, v);
1624 break;
1625
1626 case nir_intrinsic_load_viewport_scale:
1627 case nir_intrinsic_load_viewport_offset:
1628 case nir_intrinsic_load_num_work_groups:
1629 emit_sysval_read(ctx, &instr->instr, -1, 3);
1630 break;
1631
1632 case nir_intrinsic_load_work_group_id:
1633 case nir_intrinsic_load_local_invocation_id:
1634 emit_compute_builtin(ctx, instr);
1635 break;
1636
1637 default:
1638 printf ("Unhandled intrinsic\n");
1639 assert(0);
1640 break;
1641 }
1642 }
1643
1644 static unsigned
1645 midgard_tex_format(enum glsl_sampler_dim dim)
1646 {
1647 switch (dim) {
1648 case GLSL_SAMPLER_DIM_1D:
1649 case GLSL_SAMPLER_DIM_BUF:
1650 return MALI_TEX_1D;
1651
1652 case GLSL_SAMPLER_DIM_2D:
1653 case GLSL_SAMPLER_DIM_EXTERNAL:
1654 case GLSL_SAMPLER_DIM_RECT:
1655 return MALI_TEX_2D;
1656
1657 case GLSL_SAMPLER_DIM_3D:
1658 return MALI_TEX_3D;
1659
1660 case GLSL_SAMPLER_DIM_CUBE:
1661 return MALI_TEX_CUBE;
1662
1663 default:
1664 DBG("Unknown sampler dim type\n");
1665 assert(0);
1666 return 0;
1667 }
1668 }
1669
1670 /* Tries to attach an explicit LOD / bias as a constant. Returns whether this
1671 * was successful */
1672
1673 static bool
1674 pan_attach_constant_bias(
1675 compiler_context *ctx,
1676 nir_src lod,
1677 midgard_texture_word *word)
1678 {
1679 /* To attach as constant, it has to *be* constant */
1680
1681 if (!nir_src_is_const(lod))
1682 return false;
1683
1684 float f = nir_src_as_float(lod);
1685
1686 /* Break into fixed-point */
1687 signed lod_int = f;
1688 float lod_frac = f - lod_int;
1689
1690 /* Carry over negative fractions */
1691 if (lod_frac < 0.0) {
1692 lod_int--;
1693 lod_frac += 1.0;
1694 }
1695
1696 /* Encode */
1697 word->bias = float_to_ubyte(lod_frac);
1698 word->bias_int = lod_int;
1699
1700 return true;
1701 }
1702
1703 static enum mali_sampler_type
1704 midgard_sampler_type(nir_alu_type t) {
1705 switch (nir_alu_type_get_base_type(t))
1706 {
1707 case nir_type_float:
1708 return MALI_SAMPLER_FLOAT;
1709 case nir_type_int:
1710 return MALI_SAMPLER_SIGNED;
1711 case nir_type_uint:
1712 return MALI_SAMPLER_UNSIGNED;
1713 default:
1714 unreachable("Unknown sampler type");
1715 }
1716 }
1717
1718 static void
1719 emit_texop_native(compiler_context *ctx, nir_tex_instr *instr,
1720 unsigned midgard_texop)
1721 {
1722 /* TODO */
1723 //assert (!instr->sampler);
1724 //assert (!instr->texture_array_size);
1725
1726 int texture_index = instr->texture_index;
1727 int sampler_index = texture_index;
1728
1729 /* No helper to build texture words -- we do it all here */
1730 midgard_instruction ins = {
1731 .type = TAG_TEXTURE_4,
1732 .mask = 0xF,
1733 .ssa_args = {
1734 .dest = nir_dest_index(ctx, &instr->dest),
1735 .src = { -1, -1, -1 },
1736 },
1737 .texture = {
1738 .op = midgard_texop,
1739 .format = midgard_tex_format(instr->sampler_dim),
1740 .texture_handle = texture_index,
1741 .sampler_handle = sampler_index,
1742 .swizzle = SWIZZLE_XYZW,
1743 .in_reg_swizzle = SWIZZLE_XYZW,
1744
1745 /* TODO: half */
1746 .in_reg_full = 1,
1747 .out_full = 1,
1748
1749 .sampler_type = midgard_sampler_type(instr->dest_type),
1750 }
1751 };
1752
1753 for (unsigned i = 0; i < instr->num_srcs; ++i) {
1754 int index = nir_src_index(ctx, &instr->src[i].src);
1755 midgard_vector_alu_src alu_src = blank_alu_src;
1756
1757 switch (instr->src[i].src_type) {
1758 case nir_tex_src_coord: {
1759 emit_explicit_constant(ctx, index, index);
1760
1761 /* Texelfetch coordinates uses all four elements
1762 * (xyz/index) regardless of texture dimensionality,
1763 * which means it's necessary to zero the unused
1764 * components to keep everything happy */
1765
1766 if (midgard_texop == TEXTURE_OP_TEXEL_FETCH) {
1767 unsigned old_index = index;
1768
1769 index = make_compiler_temp(ctx);
1770
1771 /* mov index, old_index */
1772 midgard_instruction mov = v_mov(old_index, blank_alu_src, index);
1773 mov.mask = 0x3;
1774 emit_mir_instruction(ctx, mov);
1775
1776 /* mov index.zw, #0 */
1777 mov = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT),
1778 blank_alu_src, index);
1779 mov.has_constants = true;
1780 mov.mask = (1 << COMPONENT_Z) | (1 << COMPONENT_W);
1781 emit_mir_instruction(ctx, mov);
1782 }
1783
1784 if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE) {
1785 /* texelFetch is undefined on samplerCube */
1786 assert(midgard_texop != TEXTURE_OP_TEXEL_FETCH);
1787
1788 /* For cubemaps, we use a special ld/st op to
1789 * select the face and copy the xy into the
1790 * texture register */
1791
1792 unsigned temp = make_compiler_temp(ctx);
1793 midgard_instruction st = m_st_cubemap_coords(temp, 0);
1794 st.ssa_args.src[0] = index;
1795 st.mask = 0x3; /* xy */
1796 st.load_store.arg_1 = 0x20;
1797 st.load_store.swizzle = alu_src.swizzle;
1798 emit_mir_instruction(ctx, st);
1799
1800 ins.ssa_args.src[0] = temp;
1801 } else {
1802 ins.ssa_args.src[0] = index;
1803 }
1804
1805 if (instr->sampler_dim == GLSL_SAMPLER_DIM_2D) {
1806 /* Array component in w but NIR wants it in z */
1807 ins.texture.in_reg_swizzle = SWIZZLE_XYZZ;
1808 }
1809
1810 break;
1811 }
1812
1813 case nir_tex_src_bias:
1814 case nir_tex_src_lod: {
1815 /* Try as a constant if we can */
1816
1817 bool is_txf = midgard_texop == TEXTURE_OP_TEXEL_FETCH;
1818 if (!is_txf && pan_attach_constant_bias(ctx, instr->src[i].src, &ins.texture))
1819 break;
1820
1821 ins.texture.lod_register = true;
1822 ins.ssa_args.src[1] = index;
1823 emit_explicit_constant(ctx, index, index);
1824
1825 break;
1826 };
1827
1828 default:
1829 unreachable("Unknown texture source type\n");
1830 }
1831 }
1832
1833 emit_mir_instruction(ctx, ins);
1834
1835 /* Used for .cont and .last hinting */
1836 ctx->texture_op_count++;
1837 }
1838
1839 static void
1840 emit_tex(compiler_context *ctx, nir_tex_instr *instr)
1841 {
1842 /* Fixup op, since only textureLod is permitted in VS but NIR can give
1843 * generic tex in some cases (which confuses the hardware) */
1844
1845 bool is_vertex = ctx->stage == MESA_SHADER_VERTEX;
1846
1847 if (is_vertex && instr->op == nir_texop_tex)
1848 instr->op = nir_texop_txl;
1849
1850 switch (instr->op) {
1851 case nir_texop_tex:
1852 case nir_texop_txb:
1853 emit_texop_native(ctx, instr, TEXTURE_OP_NORMAL);
1854 break;
1855 case nir_texop_txl:
1856 emit_texop_native(ctx, instr, TEXTURE_OP_LOD);
1857 break;
1858 case nir_texop_txf:
1859 emit_texop_native(ctx, instr, TEXTURE_OP_TEXEL_FETCH);
1860 break;
1861 case nir_texop_txs:
1862 emit_sysval_read(ctx, &instr->instr, -1, 4);
1863 break;
1864 default:
1865 unreachable("Unhanlded texture op");
1866 }
1867 }
1868
1869 static void
1870 emit_jump(compiler_context *ctx, nir_jump_instr *instr)
1871 {
1872 switch (instr->type) {
1873 case nir_jump_break: {
1874 /* Emit a branch out of the loop */
1875 struct midgard_instruction br = v_branch(false, false);
1876 br.branch.target_type = TARGET_BREAK;
1877 br.branch.target_break = ctx->current_loop_depth;
1878 emit_mir_instruction(ctx, br);
1879 break;
1880 }
1881
1882 default:
1883 DBG("Unknown jump type %d\n", instr->type);
1884 break;
1885 }
1886 }
1887
1888 static void
1889 emit_instr(compiler_context *ctx, struct nir_instr *instr)
1890 {
1891 switch (instr->type) {
1892 case nir_instr_type_load_const:
1893 emit_load_const(ctx, nir_instr_as_load_const(instr));
1894 break;
1895
1896 case nir_instr_type_intrinsic:
1897 emit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
1898 break;
1899
1900 case nir_instr_type_alu:
1901 emit_alu(ctx, nir_instr_as_alu(instr));
1902 break;
1903
1904 case nir_instr_type_tex:
1905 emit_tex(ctx, nir_instr_as_tex(instr));
1906 break;
1907
1908 case nir_instr_type_jump:
1909 emit_jump(ctx, nir_instr_as_jump(instr));
1910 break;
1911
1912 case nir_instr_type_ssa_undef:
1913 /* Spurious */
1914 break;
1915
1916 default:
1917 DBG("Unhandled instruction type\n");
1918 break;
1919 }
1920 }
1921
1922
1923 /* ALU instructions can inline or embed constants, which decreases register
1924 * pressure and saves space. */
1925
1926 #define CONDITIONAL_ATTACH(src) { \
1927 void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->ssa_args.src + 1); \
1928 \
1929 if (entry) { \
1930 attach_constants(ctx, alu, entry, alu->ssa_args.src + 1); \
1931 alu->ssa_args.src = SSA_FIXED_REGISTER(REGISTER_CONSTANT); \
1932 } \
1933 }
1934
1935 static void
1936 inline_alu_constants(compiler_context *ctx)
1937 {
1938 mir_foreach_instr(ctx, alu) {
1939 /* Other instructions cannot inline constants */
1940 if (alu->type != TAG_ALU_4) continue;
1941
1942 /* If there is already a constant here, we can do nothing */
1943 if (alu->has_constants) continue;
1944
1945 CONDITIONAL_ATTACH(src[0]);
1946
1947 if (!alu->has_constants) {
1948 CONDITIONAL_ATTACH(src[1])
1949 } else if (!alu->inline_constant) {
1950 /* Corner case: _two_ vec4 constants, for instance with a
1951 * csel. For this case, we can only use a constant
1952 * register for one, we'll have to emit a move for the
1953 * other. Note, if both arguments are constants, then
1954 * necessarily neither argument depends on the value of
1955 * any particular register. As the destination register
1956 * will be wiped, that means we can spill the constant
1957 * to the destination register.
1958 */
1959
1960 void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->ssa_args.src[1] + 1);
1961 unsigned scratch = alu->ssa_args.dest;
1962
1963 if (entry) {
1964 midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), blank_alu_src, scratch);
1965 attach_constants(ctx, &ins, entry, alu->ssa_args.src[1] + 1);
1966
1967 /* Force a break XXX Defer r31 writes */
1968 ins.unit = UNIT_VLUT;
1969
1970 /* Set the source */
1971 alu->ssa_args.src[1] = scratch;
1972
1973 /* Inject us -before- the last instruction which set r31 */
1974 mir_insert_instruction_before(mir_prev_op(alu), ins);
1975 }
1976 }
1977 }
1978 }
1979
1980 /* Being a little silly with the names, but returns the op that is the bitwise
1981 * inverse of the op with the argument switched. I.e. (f and g are
1982 * contrapositives):
1983 *
1984 * f(a, b) = ~g(b, a)
1985 *
1986 * Corollary: if g is the contrapositve of f, f is the contrapositive of g:
1987 *
1988 * f(a, b) = ~g(b, a)
1989 * ~f(a, b) = g(b, a)
1990 * ~f(a, b) = ~h(a, b) where h is the contrapositive of g
1991 * f(a, b) = h(a, b)
1992 *
1993 * Thus we define this function in pairs.
1994 */
1995
1996 static inline midgard_alu_op
1997 mir_contrapositive(midgard_alu_op op)
1998 {
1999 switch (op) {
2000 case midgard_alu_op_flt:
2001 return midgard_alu_op_fle;
2002 case midgard_alu_op_fle:
2003 return midgard_alu_op_flt;
2004
2005 case midgard_alu_op_ilt:
2006 return midgard_alu_op_ile;
2007 case midgard_alu_op_ile:
2008 return midgard_alu_op_ilt;
2009
2010 default:
2011 unreachable("No known contrapositive");
2012 }
2013 }
2014
2015 /* Midgard supports two types of constants, embedded constants (128-bit) and
2016 * inline constants (16-bit). Sometimes, especially with scalar ops, embedded
2017 * constants can be demoted to inline constants, for space savings and
2018 * sometimes a performance boost */
2019
2020 static void
2021 embedded_to_inline_constant(compiler_context *ctx)
2022 {
2023 mir_foreach_instr(ctx, ins) {
2024 if (!ins->has_constants) continue;
2025
2026 if (ins->ssa_args.inline_constant) continue;
2027
2028 /* Blend constants must not be inlined by definition */
2029 if (ins->has_blend_constant) continue;
2030
2031 /* We can inline 32-bit (sometimes) or 16-bit (usually) */
2032 bool is_16 = ins->alu.reg_mode == midgard_reg_mode_16;
2033 bool is_32 = ins->alu.reg_mode == midgard_reg_mode_32;
2034
2035 if (!(is_16 || is_32))
2036 continue;
2037
2038 /* src1 cannot be an inline constant due to encoding
2039 * restrictions. So, if possible we try to flip the arguments
2040 * in that case */
2041
2042 int op = ins->alu.op;
2043
2044 if (ins->ssa_args.src[0] == SSA_FIXED_REGISTER(REGISTER_CONSTANT)) {
2045 bool flip = alu_opcode_props[op].props & OP_COMMUTES;
2046
2047 switch (op) {
2048 /* Conditionals can be inverted */
2049 case midgard_alu_op_flt:
2050 case midgard_alu_op_ilt:
2051 case midgard_alu_op_fle:
2052 case midgard_alu_op_ile:
2053 ins->alu.op = mir_contrapositive(ins->alu.op);
2054 ins->invert = true;
2055 flip = true;
2056 break;
2057
2058 case midgard_alu_op_fcsel:
2059 case midgard_alu_op_icsel:
2060 DBG("Missed non-commutative flip (%s)\n", alu_opcode_props[op].name);
2061 default:
2062 break;
2063 }
2064
2065 if (flip) {
2066 /* Flip the SSA numbers */
2067 ins->ssa_args.src[0] = ins->ssa_args.src[1];
2068 ins->ssa_args.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT);
2069
2070 /* And flip the modifiers */
2071
2072 unsigned src_temp;
2073
2074 src_temp = ins->alu.src2;
2075 ins->alu.src2 = ins->alu.src1;
2076 ins->alu.src1 = src_temp;
2077 }
2078 }
2079
2080 if (ins->ssa_args.src[1] == SSA_FIXED_REGISTER(REGISTER_CONSTANT)) {
2081 /* Extract the source information */
2082
2083 midgard_vector_alu_src *src;
2084 int q = ins->alu.src2;
2085 midgard_vector_alu_src *m = (midgard_vector_alu_src *) &q;
2086 src = m;
2087
2088 /* Component is from the swizzle, e.g. r26.w -> w component. TODO: What if x is masked out? */
2089 int component = src->swizzle & 3;
2090
2091 /* Scale constant appropriately, if we can legally */
2092 uint16_t scaled_constant = 0;
2093
2094 if (midgard_is_integer_op(op) || is_16) {
2095 unsigned int *iconstants = (unsigned int *) ins->constants;
2096 scaled_constant = (uint16_t) iconstants[component];
2097
2098 /* Constant overflow after resize */
2099 if (scaled_constant != iconstants[component])
2100 continue;
2101 } else {
2102 float original = (float) ins->constants[component];
2103 scaled_constant = _mesa_float_to_half(original);
2104
2105 /* Check for loss of precision. If this is
2106 * mediump, we don't care, but for a highp
2107 * shader, we need to pay attention. NIR
2108 * doesn't yet tell us which mode we're in!
2109 * Practically this prevents most constants
2110 * from being inlined, sadly. */
2111
2112 float fp32 = _mesa_half_to_float(scaled_constant);
2113
2114 if (fp32 != original)
2115 continue;
2116 }
2117
2118 /* We don't know how to handle these with a constant */
2119
2120 if (mir_nontrivial_source2_mod_simple(ins) || src->rep_low || src->rep_high) {
2121 DBG("Bailing inline constant...\n");
2122 continue;
2123 }
2124
2125 /* Make sure that the constant is not itself a
2126 * vector by checking if all accessed values
2127 * (by the swizzle) are the same. */
2128
2129 uint32_t *cons = (uint32_t *) ins->constants;
2130 uint32_t value = cons[component];
2131
2132 bool is_vector = false;
2133 unsigned mask = effective_writemask(&ins->alu, ins->mask);
2134
2135 for (int c = 1; c < 4; ++c) {
2136 /* We only care if this component is actually used */
2137 if (!(mask & (1 << c)))
2138 continue;
2139
2140 uint32_t test = cons[(src->swizzle >> (2 * c)) & 3];
2141
2142 if (test != value) {
2143 is_vector = true;
2144 break;
2145 }
2146 }
2147
2148 if (is_vector)
2149 continue;
2150
2151 /* Get rid of the embedded constant */
2152 ins->has_constants = false;
2153 ins->ssa_args.src[1] = -1;
2154 ins->ssa_args.inline_constant = true;
2155 ins->inline_constant = scaled_constant;
2156 }
2157 }
2158 }
2159
2160 /* Dead code elimination for branches at the end of a block - only one branch
2161 * per block is legal semantically */
2162
2163 static void
2164 midgard_opt_cull_dead_branch(compiler_context *ctx, midgard_block *block)
2165 {
2166 bool branched = false;
2167
2168 mir_foreach_instr_in_block_safe(block, ins) {
2169 if (!midgard_is_branch_unit(ins->unit)) continue;
2170
2171 /* We ignore prepacked branches since the fragment epilogue is
2172 * just generally special */
2173 if (ins->prepacked_branch) continue;
2174
2175 /* Discards are similarly special and may not correspond to the
2176 * end of a block */
2177
2178 if (ins->branch.target_type == TARGET_DISCARD) continue;
2179
2180 if (branched) {
2181 /* We already branched, so this is dead */
2182 mir_remove_instruction(ins);
2183 }
2184
2185 branched = true;
2186 }
2187 }
2188
2189 /* fmov.pos is an idiom for fpos. Propoagate the .pos up to the source, so then
2190 * the move can be propagated away entirely */
2191
2192 static bool
2193 mir_compose_float_outmod(midgard_outmod_float *outmod, midgard_outmod_float comp)
2194 {
2195 /* Nothing to do */
2196 if (comp == midgard_outmod_none)
2197 return true;
2198
2199 if (*outmod == midgard_outmod_none) {
2200 *outmod = comp;
2201 return true;
2202 }
2203
2204 /* TODO: Compose rules */
2205 return false;
2206 }
2207
2208 static bool
2209 midgard_opt_pos_propagate(compiler_context *ctx, midgard_block *block)
2210 {
2211 bool progress = false;
2212
2213 mir_foreach_instr_in_block_safe(block, ins) {
2214 if (ins->type != TAG_ALU_4) continue;
2215 if (ins->alu.op != midgard_alu_op_fmov) continue;
2216 if (ins->alu.outmod != midgard_outmod_pos) continue;
2217
2218 /* TODO: Registers? */
2219 unsigned src = ins->ssa_args.src[1];
2220 if (src & IS_REG) continue;
2221 assert(!mir_has_multiple_writes(ctx, src));
2222
2223 /* There might be a source modifier, too */
2224 if (mir_nontrivial_source2_mod(ins)) continue;
2225
2226 /* Backpropagate the modifier */
2227 mir_foreach_instr_in_block_from_rev(block, v, mir_prev_op(ins)) {
2228 if (v->type != TAG_ALU_4) continue;
2229 if (v->ssa_args.dest != src) continue;
2230
2231 /* Can we even take a float outmod? */
2232 if (midgard_is_integer_out_op(v->alu.op)) continue;
2233
2234 midgard_outmod_float temp = v->alu.outmod;
2235 progress |= mir_compose_float_outmod(&temp, ins->alu.outmod);
2236
2237 /* Throw in the towel.. */
2238 if (!progress) break;
2239
2240 /* Otherwise, transfer the modifier */
2241 v->alu.outmod = temp;
2242 ins->alu.outmod = midgard_outmod_none;
2243
2244 break;
2245 }
2246 }
2247
2248 return progress;
2249 }
2250
2251 static void
2252 emit_fragment_epilogue(compiler_context *ctx)
2253 {
2254 /* Just emit the last chunk with the branch */
2255 EMIT(alu_br_compact_cond, midgard_jmp_writeout_op_writeout, TAG_ALU_4, -1, midgard_condition_always);
2256 }
2257
2258 static midgard_block *
2259 create_empty_block(compiler_context *ctx)
2260 {
2261 midgard_block *blk = rzalloc(ctx, midgard_block);
2262
2263 blk->predecessors = _mesa_set_create(blk,
2264 _mesa_hash_pointer,
2265 _mesa_key_pointer_equal);
2266
2267 blk->source_id = ctx->block_source_count++;
2268
2269 return blk;
2270 }
2271
2272 static midgard_block *
2273 emit_block(compiler_context *ctx, nir_block *block)
2274 {
2275 midgard_block *this_block = ctx->after_block;
2276 ctx->after_block = NULL;
2277
2278 if (!this_block)
2279 this_block = create_empty_block(ctx);
2280
2281 list_addtail(&this_block->link, &ctx->blocks);
2282
2283 this_block->is_scheduled = false;
2284 ++ctx->block_count;
2285
2286 ctx->texture_index[0] = -1;
2287 ctx->texture_index[1] = -1;
2288
2289 /* Set up current block */
2290 list_inithead(&this_block->instructions);
2291 ctx->current_block = this_block;
2292
2293 nir_foreach_instr(instr, block) {
2294 emit_instr(ctx, instr);
2295 ++ctx->instruction_count;
2296 }
2297
2298 inline_alu_constants(ctx);
2299 embedded_to_inline_constant(ctx);
2300
2301 /* Append fragment shader epilogue (value writeout) */
2302 if (ctx->stage == MESA_SHADER_FRAGMENT) {
2303 if (block == nir_impl_last_block(ctx->func->impl)) {
2304 emit_fragment_epilogue(ctx);
2305 }
2306 }
2307
2308 /* Allow the next control flow to access us retroactively, for
2309 * branching etc */
2310 ctx->current_block = this_block;
2311
2312 return this_block;
2313 }
2314
2315 static midgard_block *emit_cf_list(struct compiler_context *ctx, struct exec_list *list);
2316
2317 static void
2318 emit_if(struct compiler_context *ctx, nir_if *nif)
2319 {
2320 midgard_block *before_block = ctx->current_block;
2321
2322 /* Conditional branches expect the condition in r31.w; emit a move for
2323 * that in the _previous_ block (which is the current block). */
2324 emit_condition(ctx, &nif->condition, true, COMPONENT_X);
2325
2326 /* Speculatively emit the branch, but we can't fill it in until later */
2327 EMIT(branch, true, true);
2328 midgard_instruction *then_branch = mir_last_in_block(ctx->current_block);
2329
2330 /* Emit the two subblocks. */
2331 midgard_block *then_block = emit_cf_list(ctx, &nif->then_list);
2332 midgard_block *end_then_block = ctx->current_block;
2333
2334 /* Emit a jump from the end of the then block to the end of the else */
2335 EMIT(branch, false, false);
2336 midgard_instruction *then_exit = mir_last_in_block(ctx->current_block);
2337
2338 /* Emit second block, and check if it's empty */
2339
2340 int else_idx = ctx->block_count;
2341 int count_in = ctx->instruction_count;
2342 midgard_block *else_block = emit_cf_list(ctx, &nif->else_list);
2343 midgard_block *end_else_block = ctx->current_block;
2344 int after_else_idx = ctx->block_count;
2345
2346 /* Now that we have the subblocks emitted, fix up the branches */
2347
2348 assert(then_block);
2349 assert(else_block);
2350
2351 if (ctx->instruction_count == count_in) {
2352 /* The else block is empty, so don't emit an exit jump */
2353 mir_remove_instruction(then_exit);
2354 then_branch->branch.target_block = after_else_idx;
2355 } else {
2356 then_branch->branch.target_block = else_idx;
2357 then_exit->branch.target_block = after_else_idx;
2358 }
2359
2360 /* Wire up the successors */
2361
2362 ctx->after_block = create_empty_block(ctx);
2363
2364 midgard_block_add_successor(before_block, then_block);
2365 midgard_block_add_successor(before_block, else_block);
2366
2367 midgard_block_add_successor(end_then_block, ctx->after_block);
2368 midgard_block_add_successor(end_else_block, ctx->after_block);
2369 }
2370
2371 static void
2372 emit_loop(struct compiler_context *ctx, nir_loop *nloop)
2373 {
2374 /* Remember where we are */
2375 midgard_block *start_block = ctx->current_block;
2376
2377 /* Allocate a loop number, growing the current inner loop depth */
2378 int loop_idx = ++ctx->current_loop_depth;
2379
2380 /* Get index from before the body so we can loop back later */
2381 int start_idx = ctx->block_count;
2382
2383 /* Emit the body itself */
2384 midgard_block *loop_block = emit_cf_list(ctx, &nloop->body);
2385
2386 /* Branch back to loop back */
2387 struct midgard_instruction br_back = v_branch(false, false);
2388 br_back.branch.target_block = start_idx;
2389 emit_mir_instruction(ctx, br_back);
2390
2391 /* Mark down that branch in the graph. */
2392 midgard_block_add_successor(start_block, loop_block);
2393 midgard_block_add_successor(ctx->current_block, loop_block);
2394
2395 /* Find the index of the block about to follow us (note: we don't add
2396 * one; blocks are 0-indexed so we get a fencepost problem) */
2397 int break_block_idx = ctx->block_count;
2398
2399 /* Fix up the break statements we emitted to point to the right place,
2400 * now that we can allocate a block number for them */
2401 ctx->after_block = create_empty_block(ctx);
2402
2403 list_for_each_entry_from(struct midgard_block, block, start_block, &ctx->blocks, link) {
2404 mir_foreach_instr_in_block(block, ins) {
2405 if (ins->type != TAG_ALU_4) continue;
2406 if (!ins->compact_branch) continue;
2407 if (ins->prepacked_branch) continue;
2408
2409 /* We found a branch -- check the type to see if we need to do anything */
2410 if (ins->branch.target_type != TARGET_BREAK) continue;
2411
2412 /* It's a break! Check if it's our break */
2413 if (ins->branch.target_break != loop_idx) continue;
2414
2415 /* Okay, cool, we're breaking out of this loop.
2416 * Rewrite from a break to a goto */
2417
2418 ins->branch.target_type = TARGET_GOTO;
2419 ins->branch.target_block = break_block_idx;
2420
2421 midgard_block_add_successor(block, ctx->after_block);
2422 }
2423 }
2424
2425 /* Now that we've finished emitting the loop, free up the depth again
2426 * so we play nice with recursion amid nested loops */
2427 --ctx->current_loop_depth;
2428
2429 /* Dump loop stats */
2430 ++ctx->loop_count;
2431 }
2432
2433 static midgard_block *
2434 emit_cf_list(struct compiler_context *ctx, struct exec_list *list)
2435 {
2436 midgard_block *start_block = NULL;
2437
2438 foreach_list_typed(nir_cf_node, node, node, list) {
2439 switch (node->type) {
2440 case nir_cf_node_block: {
2441 midgard_block *block = emit_block(ctx, nir_cf_node_as_block(node));
2442
2443 if (!start_block)
2444 start_block = block;
2445
2446 break;
2447 }
2448
2449 case nir_cf_node_if:
2450 emit_if(ctx, nir_cf_node_as_if(node));
2451 break;
2452
2453 case nir_cf_node_loop:
2454 emit_loop(ctx, nir_cf_node_as_loop(node));
2455 break;
2456
2457 case nir_cf_node_function:
2458 assert(0);
2459 break;
2460 }
2461 }
2462
2463 return start_block;
2464 }
2465
2466 /* Due to lookahead, we need to report the first tag executed in the command
2467 * stream and in branch targets. An initial block might be empty, so iterate
2468 * until we find one that 'works' */
2469
2470 static unsigned
2471 midgard_get_first_tag_from_block(compiler_context *ctx, unsigned block_idx)
2472 {
2473 midgard_block *initial_block = mir_get_block(ctx, block_idx);
2474
2475 unsigned first_tag = 0;
2476
2477 mir_foreach_block_from(ctx, initial_block, v) {
2478 midgard_bundle *initial_bundle =
2479 util_dynarray_element(&v->bundles, midgard_bundle, 0);
2480
2481 if (initial_bundle) {
2482 first_tag = initial_bundle->tag;
2483 break;
2484 }
2485 }
2486
2487 return first_tag;
2488 }
2489
2490 int
2491 midgard_compile_shader_nir(struct midgard_screen *screen, nir_shader *nir, midgard_program *program, bool is_blend)
2492 {
2493 struct util_dynarray *compiled = &program->compiled;
2494
2495 midgard_debug = debug_get_option_midgard_debug();
2496
2497 /* TODO: Bound against what? */
2498 compiler_context *ctx = rzalloc(NULL, compiler_context);
2499
2500 ctx->nir = nir;
2501 ctx->screen = screen;
2502 ctx->stage = nir->info.stage;
2503 ctx->is_blend = is_blend;
2504 ctx->alpha_ref = program->alpha_ref;
2505
2506 /* Start off with a safe cutoff, allowing usage of all 16 work
2507 * registers. Later, we'll promote uniform reads to uniform registers
2508 * if we determine it is beneficial to do so */
2509 ctx->uniform_cutoff = 8;
2510
2511 /* Initialize at a global (not block) level hash tables */
2512
2513 ctx->ssa_constants = _mesa_hash_table_u64_create(NULL);
2514 ctx->hash_to_temp = _mesa_hash_table_u64_create(NULL);
2515 ctx->sysval_to_id = _mesa_hash_table_u64_create(NULL);
2516
2517 /* Record the varying mapping for the command stream's bookkeeping */
2518
2519 struct exec_list *varyings =
2520 ctx->stage == MESA_SHADER_VERTEX ? &nir->outputs : &nir->inputs;
2521
2522 unsigned max_varying = 0;
2523 nir_foreach_variable(var, varyings) {
2524 unsigned loc = var->data.driver_location;
2525 unsigned sz = glsl_type_size(var->type, FALSE);
2526
2527 for (int c = 0; c < sz; ++c) {
2528 program->varyings[loc + c] = var->data.location + c;
2529 max_varying = MAX2(max_varying, loc + c);
2530 }
2531 }
2532
2533 /* Lower gl_Position pre-optimisation, but after lowering vars to ssa
2534 * (so we don't accidentally duplicate the epilogue since mesa/st has
2535 * messed with our I/O quite a bit already) */
2536
2537 NIR_PASS_V(nir, nir_lower_vars_to_ssa);
2538
2539 if (ctx->stage == MESA_SHADER_VERTEX) {
2540 NIR_PASS_V(nir, nir_lower_viewport_transform);
2541 NIR_PASS_V(nir, nir_clamp_psiz, 1.0, 1024.0);
2542 }
2543
2544 NIR_PASS_V(nir, nir_lower_var_copies);
2545 NIR_PASS_V(nir, nir_lower_vars_to_ssa);
2546 NIR_PASS_V(nir, nir_split_var_copies);
2547 NIR_PASS_V(nir, nir_lower_var_copies);
2548 NIR_PASS_V(nir, nir_lower_global_vars_to_local);
2549 NIR_PASS_V(nir, nir_lower_var_copies);
2550 NIR_PASS_V(nir, nir_lower_vars_to_ssa);
2551
2552 NIR_PASS_V(nir, nir_lower_io, nir_var_all, glsl_type_size, 0);
2553
2554 /* Optimisation passes */
2555
2556 optimise_nir(nir);
2557
2558 if (midgard_debug & MIDGARD_DBG_SHADERS) {
2559 nir_print_shader(nir, stdout);
2560 }
2561
2562 /* Assign sysvals and counts, now that we're sure
2563 * (post-optimisation) */
2564
2565 midgard_nir_assign_sysvals(ctx, nir);
2566
2567 program->uniform_count = nir->num_uniforms;
2568 program->sysval_count = ctx->sysval_count;
2569 memcpy(program->sysvals, ctx->sysvals, sizeof(ctx->sysvals[0]) * ctx->sysval_count);
2570
2571 nir_foreach_function(func, nir) {
2572 if (!func->impl)
2573 continue;
2574
2575 list_inithead(&ctx->blocks);
2576 ctx->block_count = 0;
2577 ctx->func = func;
2578
2579 emit_cf_list(ctx, &func->impl->body);
2580
2581 /* Emit empty exit block with successor */
2582
2583 struct midgard_block *semi_end = ctx->current_block;
2584
2585 struct midgard_block *end =
2586 emit_block(ctx, func->impl->end_block);
2587
2588 midgard_block_add_successor(semi_end, end);
2589
2590 break; /* TODO: Multi-function shaders */
2591 }
2592
2593 util_dynarray_init(compiled, NULL);
2594
2595 /* MIR-level optimizations */
2596
2597 bool progress = false;
2598
2599 do {
2600 progress = false;
2601
2602 mir_foreach_block(ctx, block) {
2603 progress |= midgard_opt_pos_propagate(ctx, block);
2604 progress |= midgard_opt_copy_prop(ctx, block);
2605 progress |= midgard_opt_dead_code_eliminate(ctx, block);
2606 progress |= midgard_opt_combine_projection(ctx, block);
2607 progress |= midgard_opt_varying_projection(ctx, block);
2608 progress |= midgard_opt_not_propagate(ctx, block);
2609 progress |= midgard_opt_fuse_src_invert(ctx, block);
2610 progress |= midgard_opt_fuse_dest_invert(ctx, block);
2611 }
2612 } while (progress);
2613
2614 mir_foreach_block(ctx, block) {
2615 midgard_lower_invert(ctx, block);
2616 midgard_lower_derivatives(ctx, block);
2617 }
2618
2619 /* Nested control-flow can result in dead branches at the end of the
2620 * block. This messes with our analysis and is just dead code, so cull
2621 * them */
2622 mir_foreach_block(ctx, block) {
2623 midgard_opt_cull_dead_branch(ctx, block);
2624 }
2625
2626 /* Ensure we were lowered */
2627 mir_foreach_instr_global(ctx, ins) {
2628 assert(!ins->invert);
2629 }
2630
2631 /* Schedule! */
2632 schedule_program(ctx);
2633
2634 /* Now that all the bundles are scheduled and we can calculate block
2635 * sizes, emit actual branch instructions rather than placeholders */
2636
2637 int br_block_idx = 0;
2638
2639 mir_foreach_block(ctx, block) {
2640 util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) {
2641 for (int c = 0; c < bundle->instruction_count; ++c) {
2642 midgard_instruction *ins = bundle->instructions[c];
2643
2644 if (!midgard_is_branch_unit(ins->unit)) continue;
2645
2646 if (ins->prepacked_branch) continue;
2647
2648 /* Parse some basic branch info */
2649 bool is_compact = ins->unit == ALU_ENAB_BR_COMPACT;
2650 bool is_conditional = ins->branch.conditional;
2651 bool is_inverted = ins->branch.invert_conditional;
2652 bool is_discard = ins->branch.target_type == TARGET_DISCARD;
2653
2654 /* Determine the block we're jumping to */
2655 int target_number = ins->branch.target_block;
2656
2657 /* Report the destination tag */
2658 int dest_tag = is_discard ? 0 : midgard_get_first_tag_from_block(ctx, target_number);
2659
2660 /* Count up the number of quadwords we're
2661 * jumping over = number of quadwords until
2662 * (br_block_idx, target_number) */
2663
2664 int quadword_offset = 0;
2665
2666 if (is_discard) {
2667 /* Ignored */
2668 } else if (target_number > br_block_idx) {
2669 /* Jump forward */
2670
2671 for (int idx = br_block_idx + 1; idx < target_number; ++idx) {
2672 midgard_block *blk = mir_get_block(ctx, idx);
2673 assert(blk);
2674
2675 quadword_offset += blk->quadword_count;
2676 }
2677 } else {
2678 /* Jump backwards */
2679
2680 for (int idx = br_block_idx; idx >= target_number; --idx) {
2681 midgard_block *blk = mir_get_block(ctx, idx);
2682 assert(blk);
2683
2684 quadword_offset -= blk->quadword_count;
2685 }
2686 }
2687
2688 /* Unconditional extended branches (far jumps)
2689 * have issues, so we always use a conditional
2690 * branch, setting the condition to always for
2691 * unconditional. For compact unconditional
2692 * branches, cond isn't used so it doesn't
2693 * matter what we pick. */
2694
2695 midgard_condition cond =
2696 !is_conditional ? midgard_condition_always :
2697 is_inverted ? midgard_condition_false :
2698 midgard_condition_true;
2699
2700 midgard_jmp_writeout_op op =
2701 is_discard ? midgard_jmp_writeout_op_discard :
2702 (is_compact && !is_conditional) ? midgard_jmp_writeout_op_branch_uncond :
2703 midgard_jmp_writeout_op_branch_cond;
2704
2705 if (!is_compact) {
2706 midgard_branch_extended branch =
2707 midgard_create_branch_extended(
2708 cond, op,
2709 dest_tag,
2710 quadword_offset);
2711
2712 memcpy(&ins->branch_extended, &branch, sizeof(branch));
2713 } else if (is_conditional || is_discard) {
2714 midgard_branch_cond branch = {
2715 .op = op,
2716 .dest_tag = dest_tag,
2717 .offset = quadword_offset,
2718 .cond = cond
2719 };
2720
2721 assert(branch.offset == quadword_offset);
2722
2723 memcpy(&ins->br_compact, &branch, sizeof(branch));
2724 } else {
2725 assert(op == midgard_jmp_writeout_op_branch_uncond);
2726
2727 midgard_branch_uncond branch = {
2728 .op = op,
2729 .dest_tag = dest_tag,
2730 .offset = quadword_offset,
2731 .unknown = 1
2732 };
2733
2734 assert(branch.offset == quadword_offset);
2735
2736 memcpy(&ins->br_compact, &branch, sizeof(branch));
2737 }
2738 }
2739 }
2740
2741 ++br_block_idx;
2742 }
2743
2744 /* Emit flat binary from the instruction arrays. Iterate each block in
2745 * sequence. Save instruction boundaries such that lookahead tags can
2746 * be assigned easily */
2747
2748 /* Cache _all_ bundles in source order for lookahead across failed branches */
2749
2750 int bundle_count = 0;
2751 mir_foreach_block(ctx, block) {
2752 bundle_count += block->bundles.size / sizeof(midgard_bundle);
2753 }
2754 midgard_bundle **source_order_bundles = malloc(sizeof(midgard_bundle *) * bundle_count);
2755 int bundle_idx = 0;
2756 mir_foreach_block(ctx, block) {
2757 util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) {
2758 source_order_bundles[bundle_idx++] = bundle;
2759 }
2760 }
2761
2762 int current_bundle = 0;
2763
2764 /* Midgard prefetches instruction types, so during emission we
2765 * need to lookahead. Unless this is the last instruction, in
2766 * which we return 1. Or if this is the second to last and the
2767 * last is an ALU, then it's also 1... */
2768
2769 mir_foreach_block(ctx, block) {
2770 mir_foreach_bundle_in_block(block, bundle) {
2771 int lookahead = 1;
2772
2773 if (current_bundle + 1 < bundle_count) {
2774 uint8_t next = source_order_bundles[current_bundle + 1]->tag;
2775
2776 if (!(current_bundle + 2 < bundle_count) && IS_ALU(next)) {
2777 lookahead = 1;
2778 } else {
2779 lookahead = next;
2780 }
2781 }
2782
2783 emit_binary_bundle(ctx, bundle, compiled, lookahead);
2784 ++current_bundle;
2785 }
2786
2787 /* TODO: Free deeper */
2788 //util_dynarray_fini(&block->instructions);
2789 }
2790
2791 free(source_order_bundles);
2792
2793 /* Report the very first tag executed */
2794 program->first_tag = midgard_get_first_tag_from_block(ctx, 0);
2795
2796 /* Deal with off-by-one related to the fencepost problem */
2797 program->work_register_count = ctx->work_registers + 1;
2798 program->uniform_cutoff = ctx->uniform_cutoff;
2799
2800 program->blend_patch_offset = ctx->blend_constant_offset;
2801 program->tls_size = ctx->tls_size;
2802
2803 if (midgard_debug & MIDGARD_DBG_SHADERS)
2804 disassemble_midgard(program->compiled.data, program->compiled.size, false, 0, "");
2805
2806 if (midgard_debug & MIDGARD_DBG_SHADERDB) {
2807 unsigned nr_bundles = 0, nr_ins = 0, nr_quadwords = 0;
2808
2809 /* Count instructions and bundles */
2810
2811 mir_foreach_instr_global(ctx, ins) {
2812 nr_ins++;
2813 }
2814
2815 mir_foreach_block(ctx, block) {
2816 nr_bundles += util_dynarray_num_elements(
2817 &block->bundles, midgard_bundle);
2818
2819 nr_quadwords += block->quadword_count;
2820 }
2821
2822 /* Calculate thread count. There are certain cutoffs by
2823 * register count for thread count */
2824
2825 unsigned nr_registers = program->work_register_count;
2826
2827 unsigned nr_threads =
2828 (nr_registers <= 4) ? 4 :
2829 (nr_registers <= 8) ? 2 :
2830 1;
2831
2832 /* Dump stats */
2833
2834 fprintf(stderr, "shader%d - %s shader: "
2835 "%u inst, %u bundles, %u quadwords, "
2836 "%u registers, %u threads, %u loops, "
2837 "%d:%d spills:fills\n",
2838 SHADER_DB_COUNT++,
2839 gl_shader_stage_name(ctx->stage),
2840 nr_ins, nr_bundles, nr_quadwords,
2841 nr_registers, nr_threads,
2842 ctx->loop_count,
2843 ctx->spills, ctx->fills);
2844 }
2845
2846 ralloc_free(ctx);
2847
2848 return 0;
2849 }