projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
pan/midgard,bifrost: Expand nir_const_load_to_arr
[mesa.git] / src / panfrost / midgard / midgard_compile.c
1 /*
2 * Copyright (C) 2018-2019 Alyssa Rosenzweig <alyssa@rosenzweig.io>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
23
24 #include <sys/types.h>
25 #include <sys/stat.h>
26 #include <sys/mman.h>
27 #include <fcntl.h>
28 #include <stdint.h>
29 #include <stdlib.h>
30 #include <stdio.h>
31 #include <err.h>
32
33 #include "main/mtypes.h"
34 #include "compiler/glsl/glsl_to_nir.h"
35 #include "compiler/nir_types.h"
36 #include "main/imports.h"
37 #include "compiler/nir/nir_builder.h"
38 #include "util/half_float.h"
39 #include "util/u_math.h"
40 #include "util/u_debug.h"
41 #include "util/u_dynarray.h"
42 #include "util/list.h"
43 #include "main/mtypes.h"
44
45 #include "midgard.h"
46 #include "midgard_nir.h"
47 #include "midgard_compile.h"
48 #include "midgard_ops.h"
49 #include "helpers.h"
50 #include "compiler.h"
51
52 #include "disassemble.h"
53
54 static const struct debug_named_value debug_options[] = {
55 {"msgs", MIDGARD_DBG_MSGS, "Print debug messages"},
56 {"shaders", MIDGARD_DBG_SHADERS, "Dump shaders in NIR and MIR"},
57 {"shaderdb", MIDGARD_DBG_SHADERDB, "Prints shader-db statistics"},
58 DEBUG_NAMED_VALUE_END
59 };
60
61 DEBUG_GET_ONCE_FLAGS_OPTION(midgard_debug, "MIDGARD_MESA_DEBUG", debug_options, 0)
62
63 unsigned SHADER_DB_COUNT = 0;
64
65 int midgard_debug = 0;
66
67 #define DBG(fmt, ...) \
68 do { if (midgard_debug & MIDGARD_DBG_MSGS) \
69 fprintf(stderr, "%s:%d: "fmt, \
70 __FUNCTION__, __LINE__, ##__VA_ARGS__); } while (0)
71
72 static bool
73 midgard_is_branch_unit(unsigned unit)
74 {
75 return (unit == ALU_ENAB_BRANCH) || (unit == ALU_ENAB_BR_COMPACT);
76 }
77
78 static void
79 midgard_block_add_successor(midgard_block *block, midgard_block *successor)
80 {
81 assert(block);
82 assert(successor);
83
84 /* Deduplicate */
85 for (unsigned i = 0; i < block->nr_successors; ++i) {
86 if (block->successors[i] == successor)
87 return;
88 }
89
90 block->successors[block->nr_successors++] = successor;
91 assert(block->nr_successors <= ARRAY_SIZE(block->successors));
92
93 /* Note the predecessor in the other direction */
94 _mesa_set_add(successor->predecessors, block);
95 }
96
97 /* Helpers to generate midgard_instruction's using macro magic, since every
98 * driver seems to do it that way */
99
100 #define EMIT(op, ...) emit_mir_instruction(ctx, v_##op(__VA_ARGS__));
101
102 #define M_LOAD_STORE(name, store) \
103 static midgard_instruction m_##name(unsigned ssa, unsigned address) { \
104 midgard_instruction i = { \
105 .type = TAG_LOAD_STORE_4, \
106 .mask = 0xF, \
107 .ssa_args = { \
108 .dest = ~0, \
109 .src = { ~0, ~0, ~0 }, \
110 }, \
111 .load_store = { \
112 .op = midgard_op_##name, \
113 .swizzle = SWIZZLE_XYZW, \
114 .address = address \
115 } \
116 }; \
117 \
118 if (store) \
119 i.ssa_args.src[0] = ssa; \
120 else \
121 i.ssa_args.dest = ssa; \
122 \
123 return i; \
124 }
125
126 #define M_LOAD(name) M_LOAD_STORE(name, false)
127 #define M_STORE(name) M_LOAD_STORE(name, true)
128
129 /* Inputs a NIR ALU source, with modifiers attached if necessary, and outputs
130 * the corresponding Midgard source */
131
132 static midgard_vector_alu_src
133 vector_alu_modifiers(nir_alu_src *src, bool is_int, unsigned broadcast_count,
134 bool half, bool sext)
135 {
136 if (!src) return blank_alu_src;
137
138 /* Figure out how many components there are so we can adjust the
139 * swizzle. Specifically we want to broadcast the last channel so
140 * things like ball2/3 work
141 */
142
143 if (broadcast_count) {
144 uint8_t last_component = src->swizzle[broadcast_count - 1];
145
146 for (unsigned c = broadcast_count; c < NIR_MAX_VEC_COMPONENTS; ++c) {
147 src->swizzle[c] = last_component;
148 }
149 }
150
151 midgard_vector_alu_src alu_src = {
152 .rep_low = 0,
153 .rep_high = 0,
154 .half = half,
155 .swizzle = SWIZZLE_FROM_ARRAY(src->swizzle)
156 };
157
158 if (is_int) {
159 alu_src.mod = midgard_int_normal;
160
161 /* Sign/zero-extend if needed */
162
163 if (half) {
164 alu_src.mod = sext ?
165 midgard_int_sign_extend
166 : midgard_int_zero_extend;
167 }
168
169 /* These should have been lowered away */
170 assert(!(src->abs || src->negate));
171 } else {
172 alu_src.mod = (src->abs << 0) | (src->negate << 1);
173 }
174
175 return alu_src;
176 }
177
178 /* load/store instructions have both 32-bit and 16-bit variants, depending on
179 * whether we are using vectors composed of highp or mediump. At the moment, we
180 * don't support half-floats -- this requires changes in other parts of the
181 * compiler -- therefore the 16-bit versions are commented out. */
182
183 //M_LOAD(ld_attr_16);
184 M_LOAD(ld_attr_32);
185 //M_LOAD(ld_vary_16);
186 M_LOAD(ld_vary_32);
187 M_LOAD(ld_ubo_int4);
188 M_LOAD(ld_int4);
189 M_STORE(st_int4);
190 M_LOAD(ld_color_buffer_8);
191 //M_STORE(st_vary_16);
192 M_STORE(st_vary_32);
193 M_LOAD(ld_cubemap_coords);
194 M_LOAD(ld_compute_id);
195
196 static midgard_instruction
197 v_alu_br_compact_cond(midgard_jmp_writeout_op op, unsigned tag, signed offset, unsigned cond)
198 {
199 midgard_branch_cond branch = {
200 .op = op,
201 .dest_tag = tag,
202 .offset = offset,
203 .cond = cond
204 };
205
206 uint16_t compact;
207 memcpy(&compact, &branch, sizeof(branch));
208
209 midgard_instruction ins = {
210 .type = TAG_ALU_4,
211 .unit = ALU_ENAB_BR_COMPACT,
212 .prepacked_branch = true,
213 .compact_branch = true,
214 .br_compact = compact,
215 .ssa_args = {
216 .dest = ~0,
217 .src = { ~0, ~0, ~0 },
218 }
219 };
220
221 if (op == midgard_jmp_writeout_op_writeout)
222 ins.writeout = true;
223
224 return ins;
225 }
226
227 static midgard_instruction
228 v_branch(bool conditional, bool invert)
229 {
230 midgard_instruction ins = {
231 .type = TAG_ALU_4,
232 .unit = ALU_ENAB_BRANCH,
233 .compact_branch = true,
234 .branch = {
235 .conditional = conditional,
236 .invert_conditional = invert
237 },
238 .ssa_args = {
239 .dest = ~0,
240 .src = { ~0, ~0, ~0 },
241 }
242 };
243
244 return ins;
245 }
246
247 static midgard_branch_extended
248 midgard_create_branch_extended( midgard_condition cond,
249 midgard_jmp_writeout_op op,
250 unsigned dest_tag,
251 signed quadword_offset)
252 {
253 /* The condition code is actually a LUT describing a function to
254 * combine multiple condition codes. However, we only support a single
255 * condition code at the moment, so we just duplicate over a bunch of
256 * times. */
257
258 uint16_t duplicated_cond =
259 (cond << 14) |
260 (cond << 12) |
261 (cond << 10) |
262 (cond << 8) |
263 (cond << 6) |
264 (cond << 4) |
265 (cond << 2) |
266 (cond << 0);
267
268 midgard_branch_extended branch = {
269 .op = op,
270 .dest_tag = dest_tag,
271 .offset = quadword_offset,
272 .cond = duplicated_cond
273 };
274
275 return branch;
276 }
277
278 static void
279 attach_constants(compiler_context *ctx, midgard_instruction *ins, void *constants, int name)
280 {
281 ins->has_constants = true;
282 memcpy(&ins->constants, constants, 16);
283 }
284
285 static int
286 glsl_type_size(const struct glsl_type *type, bool bindless)
287 {
288 return glsl_count_attribute_slots(type, false);
289 }
290
291 /* Lower fdot2 to a vector multiplication followed by channel addition */
292 static void
293 midgard_nir_lower_fdot2_body(nir_builder *b, nir_alu_instr *alu)
294 {
295 if (alu->op != nir_op_fdot2)
296 return;
297
298 b->cursor = nir_before_instr(&alu->instr);
299
300 nir_ssa_def *src0 = nir_ssa_for_alu_src(b, alu, 0);
301 nir_ssa_def *src1 = nir_ssa_for_alu_src(b, alu, 1);
302
303 nir_ssa_def *product = nir_fmul(b, src0, src1);
304
305 nir_ssa_def *sum = nir_fadd(b,
306 nir_channel(b, product, 0),
307 nir_channel(b, product, 1));
308
309 /* Replace the fdot2 with this sum */
310 nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(sum));
311 }
312
313 static int
314 midgard_sysval_for_ssbo(nir_intrinsic_instr *instr)
315 {
316 /* This is way too meta */
317 bool is_store = instr->intrinsic == nir_intrinsic_store_ssbo;
318 unsigned idx_idx = is_store ? 1 : 0;
319
320 nir_src index = instr->src[idx_idx];
321 assert(nir_src_is_const(index));
322 uint32_t uindex = nir_src_as_uint(index);
323
324 return PAN_SYSVAL(SSBO, uindex);
325 }
326
327 static int
328 midgard_nir_sysval_for_intrinsic(nir_intrinsic_instr *instr)
329 {
330 switch (instr->intrinsic) {
331 case nir_intrinsic_load_viewport_scale:
332 return PAN_SYSVAL_VIEWPORT_SCALE;
333 case nir_intrinsic_load_viewport_offset:
334 return PAN_SYSVAL_VIEWPORT_OFFSET;
335 case nir_intrinsic_load_num_work_groups:
336 return PAN_SYSVAL_NUM_WORK_GROUPS;
337 case nir_intrinsic_load_ssbo:
338 case nir_intrinsic_store_ssbo:
339 return midgard_sysval_for_ssbo(instr);
340 default:
341 return ~0;
342 }
343 }
344
345 static int sysval_for_instr(compiler_context *ctx, nir_instr *instr,
346 unsigned *dest)
347 {
348 nir_intrinsic_instr *intr;
349 nir_dest *dst = NULL;
350 nir_tex_instr *tex;
351 int sysval = -1;
352
353 bool is_store = false;
354
355 switch (instr->type) {
356 case nir_instr_type_intrinsic:
357 intr = nir_instr_as_intrinsic(instr);
358 sysval = midgard_nir_sysval_for_intrinsic(intr);
359 dst = &intr->dest;
360 is_store |= intr->intrinsic == nir_intrinsic_store_ssbo;
361 break;
362 case nir_instr_type_tex:
363 tex = nir_instr_as_tex(instr);
364 if (tex->op != nir_texop_txs)
365 break;
366
367 sysval = PAN_SYSVAL(TEXTURE_SIZE,
368 PAN_TXS_SYSVAL_ID(tex->texture_index,
369 nir_tex_instr_dest_size(tex) -
370 (tex->is_array ? 1 : 0),
371 tex->is_array));
372 dst = &tex->dest;
373 break;
374 default:
375 break;
376 }
377
378 if (dest && dst && !is_store)
379 *dest = nir_dest_index(ctx, dst);
380
381 return sysval;
382 }
383
384 static void
385 midgard_nir_assign_sysval_body(compiler_context *ctx, nir_instr *instr)
386 {
387 int sysval;
388
389 sysval = sysval_for_instr(ctx, instr, NULL);
390 if (sysval < 0)
391 return;
392
393 /* We have a sysval load; check if it's already been assigned */
394
395 if (_mesa_hash_table_u64_search(ctx->sysval_to_id, sysval))
396 return;
397
398 /* It hasn't -- so assign it now! */
399
400 unsigned id = ctx->sysval_count++;
401 _mesa_hash_table_u64_insert(ctx->sysval_to_id, sysval, (void *) ((uintptr_t) id + 1));
402 ctx->sysvals[id] = sysval;
403 }
404
405 static void
406 midgard_nir_assign_sysvals(compiler_context *ctx, nir_shader *shader)
407 {
408 ctx->sysval_count = 0;
409
410 nir_foreach_function(function, shader) {
411 if (!function->impl) continue;
412
413 nir_foreach_block(block, function->impl) {
414 nir_foreach_instr_safe(instr, block) {
415 midgard_nir_assign_sysval_body(ctx, instr);
416 }
417 }
418 }
419 }
420
421 static bool
422 midgard_nir_lower_fdot2(nir_shader *shader)
423 {
424 bool progress = false;
425
426 nir_foreach_function(function, shader) {
427 if (!function->impl) continue;
428
429 nir_builder _b;
430 nir_builder *b = &_b;
431 nir_builder_init(b, function->impl);
432
433 nir_foreach_block(block, function->impl) {
434 nir_foreach_instr_safe(instr, block) {
435 if (instr->type != nir_instr_type_alu) continue;
436
437 nir_alu_instr *alu = nir_instr_as_alu(instr);
438 midgard_nir_lower_fdot2_body(b, alu);
439
440 progress |= true;
441 }
442 }
443
444 nir_metadata_preserve(function->impl, nir_metadata_block_index | nir_metadata_dominance);
445
446 }
447
448 return progress;
449 }
450
451 /* Flushes undefined values to zero */
452
453 static void
454 optimise_nir(nir_shader *nir)
455 {
456 bool progress;
457 unsigned lower_flrp =
458 (nir->options->lower_flrp16 ? 16 : 0) |
459 (nir->options->lower_flrp32 ? 32 : 0) |
460 (nir->options->lower_flrp64 ? 64 : 0);
461
462 NIR_PASS(progress, nir, nir_lower_regs_to_ssa);
463 NIR_PASS(progress, nir, midgard_nir_lower_fdot2);
464 NIR_PASS(progress, nir, nir_lower_idiv);
465
466 nir_lower_tex_options lower_tex_options = {
467 .lower_txs_lod = true,
468 .lower_txp = ~0
469 };
470
471 NIR_PASS(progress, nir, nir_lower_tex, &lower_tex_options);
472
473 do {
474 progress = false;
475
476 NIR_PASS(progress, nir, nir_lower_var_copies);
477 NIR_PASS(progress, nir, nir_lower_vars_to_ssa);
478
479 NIR_PASS(progress, nir, nir_copy_prop);
480 NIR_PASS(progress, nir, nir_opt_dce);
481 NIR_PASS(progress, nir, nir_opt_dead_cf);
482 NIR_PASS(progress, nir, nir_opt_cse);
483 NIR_PASS(progress, nir, nir_opt_peephole_select, 64, false, true);
484 NIR_PASS(progress, nir, nir_opt_algebraic);
485 NIR_PASS(progress, nir, nir_opt_constant_folding);
486
487 if (lower_flrp != 0) {
488 bool lower_flrp_progress = false;
489 NIR_PASS(lower_flrp_progress,
490 nir,
491 nir_lower_flrp,
492 lower_flrp,
493 false /* always_precise */,
494 nir->options->lower_ffma);
495 if (lower_flrp_progress) {
496 NIR_PASS(progress, nir,
497 nir_opt_constant_folding);
498 progress = true;
499 }
500
501 /* Nothing should rematerialize any flrps, so we only
502 * need to do this lowering once.
503 */
504 lower_flrp = 0;
505 }
506
507 NIR_PASS(progress, nir, nir_opt_undef);
508 NIR_PASS(progress, nir, nir_undef_to_zero);
509
510 NIR_PASS(progress, nir, nir_opt_loop_unroll,
511 nir_var_shader_in |
512 nir_var_shader_out |
513 nir_var_function_temp);
514
515 NIR_PASS(progress, nir, nir_opt_vectorize);
516 } while (progress);
517
518 /* Must be run at the end to prevent creation of fsin/fcos ops */
519 NIR_PASS(progress, nir, midgard_nir_scale_trig);
520
521 do {
522 progress = false;
523
524 NIR_PASS(progress, nir, nir_opt_dce);
525 NIR_PASS(progress, nir, nir_opt_algebraic);
526 NIR_PASS(progress, nir, nir_opt_constant_folding);
527 NIR_PASS(progress, nir, nir_copy_prop);
528 } while (progress);
529
530 NIR_PASS(progress, nir, nir_opt_algebraic_late);
531
532 /* We implement booleans as 32-bit 0/~0 */
533 NIR_PASS(progress, nir, nir_lower_bool_to_int32);
534
535 /* Now that booleans are lowered, we can run out late opts */
536 NIR_PASS(progress, nir, midgard_nir_lower_algebraic_late);
537
538 /* Lower mods for float ops only. Integer ops don't support modifiers
539 * (saturate doesn't make sense on integers, neg/abs require dedicated
540 * instructions) */
541
542 NIR_PASS(progress, nir, nir_lower_to_source_mods, nir_lower_float_source_mods);
543 NIR_PASS(progress, nir, nir_copy_prop);
544 NIR_PASS(progress, nir, nir_opt_dce);
545
546 /* Take us out of SSA */
547 NIR_PASS(progress, nir, nir_lower_locals_to_regs);
548 NIR_PASS(progress, nir, nir_convert_from_ssa, true);
549
550 /* We are a vector architecture; write combine where possible */
551 NIR_PASS(progress, nir, nir_move_vec_src_uses_to_dest);
552 NIR_PASS(progress, nir, nir_lower_vec_to_movs);
553
554 NIR_PASS(progress, nir, nir_opt_dce);
555 }
556
557 /* Do not actually emit a load; instead, cache the constant for inlining */
558
559 static void
560 emit_load_const(compiler_context *ctx, nir_load_const_instr *instr)
561 {
562 nir_ssa_def def = instr->def;
563
564 float *v = rzalloc_array(NULL, float, 4);
565 nir_const_value_to_array(v, instr->value, instr->def.num_components, f32);
566
567 /* Shifted for SSA, +1 for off-by-one */
568 _mesa_hash_table_u64_insert(ctx->ssa_constants, (def.index << 1) + 1, v);
569 }
570
571 /* Normally constants are embedded implicitly, but for I/O and such we have to
572 * explicitly emit a move with the constant source */
573
574 static void
575 emit_explicit_constant(compiler_context *ctx, unsigned node, unsigned to)
576 {
577 void *constant_value = _mesa_hash_table_u64_search(ctx->ssa_constants, node + 1);
578
579 if (constant_value) {
580 midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), blank_alu_src, to);
581 attach_constants(ctx, &ins, constant_value, node + 1);
582 emit_mir_instruction(ctx, ins);
583 }
584 }
585
586 static bool
587 nir_is_non_scalar_swizzle(nir_alu_src *src, unsigned nr_components)
588 {
589 unsigned comp = src->swizzle[0];
590
591 for (unsigned c = 1; c < nr_components; ++c) {
592 if (src->swizzle[c] != comp)
593 return true;
594 }
595
596 return false;
597 }
598
599 /* Midgard puts scalar conditionals in r31.w; move an arbitrary source (the
600 * output of a conditional test) into that register */
601
602 static void
603 emit_condition(compiler_context *ctx, nir_src *src, bool for_branch, unsigned component)
604 {
605 int condition = nir_src_index(ctx, src);
606
607 /* Source to swizzle the desired component into w */
608
609 const midgard_vector_alu_src alu_src = {
610 .swizzle = SWIZZLE(component, component, component, component),
611 };
612
613 /* There is no boolean move instruction. Instead, we simulate a move by
614 * ANDing the condition with itself to get it into r31.w */
615
616 midgard_instruction ins = {
617 .type = TAG_ALU_4,
618
619 /* We need to set the conditional as close as possible */
620 .precede_break = true,
621 .unit = for_branch ? UNIT_SMUL : UNIT_SADD,
622 .mask = 1 << COMPONENT_W,
623
624 .ssa_args = {
625 .src = { condition, condition, ~0 },
626 .dest = SSA_FIXED_REGISTER(31),
627 },
628
629 .alu = {
630 .op = midgard_alu_op_iand,
631 .outmod = midgard_outmod_int_wrap,
632 .reg_mode = midgard_reg_mode_32,
633 .dest_override = midgard_dest_override_none,
634 .src1 = vector_alu_srco_unsigned(alu_src),
635 .src2 = vector_alu_srco_unsigned(alu_src)
636 },
637 };
638
639 emit_mir_instruction(ctx, ins);
640 }
641
642 /* Or, for mixed conditions (with csel_v), here's a vector version using all of
643 * r31 instead */
644
645 static void
646 emit_condition_mixed(compiler_context *ctx, nir_alu_src *src, unsigned nr_comp)
647 {
648 int condition = nir_src_index(ctx, &src->src);
649
650 /* Source to swizzle the desired component into w */
651
652 const midgard_vector_alu_src alu_src = {
653 .swizzle = SWIZZLE_FROM_ARRAY(src->swizzle),
654 };
655
656 /* There is no boolean move instruction. Instead, we simulate a move by
657 * ANDing the condition with itself to get it into r31.w */
658
659 midgard_instruction ins = {
660 .type = TAG_ALU_4,
661 .precede_break = true,
662 .mask = mask_of(nr_comp),
663 .ssa_args = {
664 .src = { condition, condition, ~0 },
665 .dest = SSA_FIXED_REGISTER(31),
666 },
667 .alu = {
668 .op = midgard_alu_op_iand,
669 .outmod = midgard_outmod_int_wrap,
670 .reg_mode = midgard_reg_mode_32,
671 .dest_override = midgard_dest_override_none,
672 .src1 = vector_alu_srco_unsigned(alu_src),
673 .src2 = vector_alu_srco_unsigned(alu_src)
674 },
675 };
676
677 emit_mir_instruction(ctx, ins);
678 }
679
680 #define ALU_CASE(nir, _op) \
681 case nir_op_##nir: \
682 op = midgard_alu_op_##_op; \
683 assert(src_bitsize == dst_bitsize); \
684 break;
685
686 #define ALU_CASE_BCAST(nir, _op, count) \
687 case nir_op_##nir: \
688 op = midgard_alu_op_##_op; \
689 broadcast_swizzle = count; \
690 assert(src_bitsize == dst_bitsize); \
691 break;
692 static bool
693 nir_is_fzero_constant(nir_src src)
694 {
695 if (!nir_src_is_const(src))
696 return false;
697
698 for (unsigned c = 0; c < nir_src_num_components(src); ++c) {
699 if (nir_src_comp_as_float(src, c) != 0.0)
700 return false;
701 }
702
703 return true;
704 }
705
706 /* Analyze the sizes of the inputs to determine which reg mode. Ops needed
707 * special treatment override this anyway. */
708
709 static midgard_reg_mode
710 reg_mode_for_nir(nir_alu_instr *instr)
711 {
712 unsigned src_bitsize = nir_src_bit_size(instr->src[0].src);
713
714 switch (src_bitsize) {
715 case 8:
716 return midgard_reg_mode_8;
717 case 16:
718 return midgard_reg_mode_16;
719 case 32:
720 return midgard_reg_mode_32;
721 case 64:
722 return midgard_reg_mode_64;
723 default:
724 unreachable("Invalid bit size");
725 }
726 }
727
728 static void
729 emit_alu(compiler_context *ctx, nir_alu_instr *instr)
730 {
731 /* Derivatives end up emitted on the texture pipe, not the ALUs. This
732 * is handled elsewhere */
733
734 if (instr->op == nir_op_fddx || instr->op == nir_op_fddy) {
735 midgard_emit_derivatives(ctx, instr);
736 return;
737 }
738
739 bool is_ssa = instr->dest.dest.is_ssa;
740
741 unsigned dest = nir_dest_index(ctx, &instr->dest.dest);
742 unsigned nr_components = nir_dest_num_components(instr->dest.dest);
743 unsigned nr_inputs = nir_op_infos[instr->op].num_inputs;
744
745 /* Most Midgard ALU ops have a 1:1 correspondance to NIR ops; these are
746 * supported. A few do not and are commented for now. Also, there are a
747 * number of NIR ops which Midgard does not support and need to be
748 * lowered, also TODO. This switch block emits the opcode and calling
749 * convention of the Midgard instruction; actual packing is done in
750 * emit_alu below */
751
752 unsigned op;
753
754 /* Number of components valid to check for the instruction (the rest
755 * will be forced to the last), or 0 to use as-is. Relevant as
756 * ball-type instructions have a channel count in NIR but are all vec4
757 * in Midgard */
758
759 unsigned broadcast_swizzle = 0;
760
761 /* What register mode should we operate in? */
762 midgard_reg_mode reg_mode =
763 reg_mode_for_nir(instr);
764
765 /* Do we need a destination override? Used for inline
766 * type conversion */
767
768 midgard_dest_override dest_override =
769 midgard_dest_override_none;
770
771 /* Should we use a smaller respective source and sign-extend? */
772
773 bool half_1 = false, sext_1 = false;
774 bool half_2 = false, sext_2 = false;
775
776 unsigned src_bitsize = nir_src_bit_size(instr->src[0].src);
777 unsigned dst_bitsize = nir_dest_bit_size(instr->dest.dest);
778
779 switch (instr->op) {
780 ALU_CASE(fadd, fadd);
781 ALU_CASE(fmul, fmul);
782 ALU_CASE(fmin, fmin);
783 ALU_CASE(fmax, fmax);
784 ALU_CASE(imin, imin);
785 ALU_CASE(imax, imax);
786 ALU_CASE(umin, umin);
787 ALU_CASE(umax, umax);
788 ALU_CASE(ffloor, ffloor);
789 ALU_CASE(fround_even, froundeven);
790 ALU_CASE(ftrunc, ftrunc);
791 ALU_CASE(fceil, fceil);
792 ALU_CASE(fdot3, fdot3);
793 ALU_CASE(fdot4, fdot4);
794 ALU_CASE(iadd, iadd);
795 ALU_CASE(isub, isub);
796 ALU_CASE(imul, imul);
797
798 /* Zero shoved as second-arg */
799 ALU_CASE(iabs, iabsdiff);
800
801 ALU_CASE(mov, imov);
802
803 ALU_CASE(feq32, feq);
804 ALU_CASE(fne32, fne);
805 ALU_CASE(flt32, flt);
806 ALU_CASE(ieq32, ieq);
807 ALU_CASE(ine32, ine);
808 ALU_CASE(ilt32, ilt);
809 ALU_CASE(ult32, ult);
810
811 /* We don't have a native b2f32 instruction. Instead, like many
812 * GPUs, we exploit booleans as 0/~0 for false/true, and
813 * correspondingly AND
814 * by 1.0 to do the type conversion. For the moment, prime us
815 * to emit:
816 *
817 * iand [whatever], #0
818 *
819 * At the end of emit_alu (as MIR), we'll fix-up the constant
820 */
821
822 ALU_CASE(b2f32, iand);
823 ALU_CASE(b2i32, iand);
824
825 /* Likewise, we don't have a dedicated f2b32 instruction, but
826 * we can do a "not equal to 0.0" test. */
827
828 ALU_CASE(f2b32, fne);
829 ALU_CASE(i2b32, ine);
830
831 ALU_CASE(frcp, frcp);
832 ALU_CASE(frsq, frsqrt);
833 ALU_CASE(fsqrt, fsqrt);
834 ALU_CASE(fexp2, fexp2);
835 ALU_CASE(flog2, flog2);
836
837 ALU_CASE(f2i32, f2i_rtz);
838 ALU_CASE(f2u32, f2u_rtz);
839 ALU_CASE(i2f32, i2f_rtz);
840 ALU_CASE(u2f32, u2f_rtz);
841
842 ALU_CASE(f2i16, f2i_rtz);
843 ALU_CASE(f2u16, f2u_rtz);
844 ALU_CASE(i2f16, i2f_rtz);
845 ALU_CASE(u2f16, u2f_rtz);
846
847 ALU_CASE(fsin, fsin);
848 ALU_CASE(fcos, fcos);
849
850 /* We'll set invert */
851 ALU_CASE(inot, imov);
852 ALU_CASE(iand, iand);
853 ALU_CASE(ior, ior);
854 ALU_CASE(ixor, ixor);
855 ALU_CASE(ishl, ishl);
856 ALU_CASE(ishr, iasr);
857 ALU_CASE(ushr, ilsr);
858
859 ALU_CASE_BCAST(b32all_fequal2, fball_eq, 2);
860 ALU_CASE_BCAST(b32all_fequal3, fball_eq, 3);
861 ALU_CASE(b32all_fequal4, fball_eq);
862
863 ALU_CASE_BCAST(b32any_fnequal2, fbany_neq, 2);
864 ALU_CASE_BCAST(b32any_fnequal3, fbany_neq, 3);
865 ALU_CASE(b32any_fnequal4, fbany_neq);
866
867 ALU_CASE_BCAST(b32all_iequal2, iball_eq, 2);
868 ALU_CASE_BCAST(b32all_iequal3, iball_eq, 3);
869 ALU_CASE(b32all_iequal4, iball_eq);
870
871 ALU_CASE_BCAST(b32any_inequal2, ibany_neq, 2);
872 ALU_CASE_BCAST(b32any_inequal3, ibany_neq, 3);
873 ALU_CASE(b32any_inequal4, ibany_neq);
874
875 /* Source mods will be shoved in later */
876 ALU_CASE(fabs, fmov);
877 ALU_CASE(fneg, fmov);
878 ALU_CASE(fsat, fmov);
879
880 /* For size conversion, we use a move. Ideally though we would squash
881 * these ops together; maybe that has to happen after in NIR as part of
882 * propagation...? An earlier algebraic pass ensured we step down by
883 * only / exactly one size. If stepping down, we use a dest override to
884 * reduce the size; if stepping up, we use a larger-sized move with a
885 * half source and a sign/zero-extension modifier */
886
887 case nir_op_i2i8:
888 case nir_op_i2i16:
889 case nir_op_i2i32:
890 /* If we end up upscale, we'll need a sign-extend on the
891 * operand (the second argument) */
892
893 sext_2 = true;
894 /* fallthrough */
895 case nir_op_u2u8:
896 case nir_op_u2u16:
897 case nir_op_u2u32: {
898 op = midgard_alu_op_imov;
899
900 if (dst_bitsize == (src_bitsize * 2)) {
901 /* Converting up */
902 half_2 = true;
903
904 /* Use a greater register mode */
905 reg_mode++;
906 } else if (src_bitsize == (dst_bitsize * 2)) {
907 /* Converting down */
908 dest_override = midgard_dest_override_lower;
909 }
910
911 break;
912 }
913
914 case nir_op_f2f16: {
915 assert(src_bitsize == 32);
916
917 op = midgard_alu_op_fmov;
918 dest_override = midgard_dest_override_lower;
919 break;
920 }
921
922 case nir_op_f2f32: {
923 assert(src_bitsize == 16);
924
925 op = midgard_alu_op_fmov;
926 half_2 = true;
927 reg_mode++;
928 break;
929 }
930
931
932 /* For greater-or-equal, we lower to less-or-equal and flip the
933 * arguments */
934
935 case nir_op_fge:
936 case nir_op_fge32:
937 case nir_op_ige32:
938 case nir_op_uge32: {
939 op =
940 instr->op == nir_op_fge ? midgard_alu_op_fle :
941 instr->op == nir_op_fge32 ? midgard_alu_op_fle :
942 instr->op == nir_op_ige32 ? midgard_alu_op_ile :
943 instr->op == nir_op_uge32 ? midgard_alu_op_ule :
944 0;
945
946 /* Swap via temporary */
947 nir_alu_src temp = instr->src[1];
948 instr->src[1] = instr->src[0];
949 instr->src[0] = temp;
950
951 break;
952 }
953
954 case nir_op_b32csel: {
955 /* Midgard features both fcsel and icsel, depending on
956 * the type of the arguments/output. However, as long
957 * as we're careful we can _always_ use icsel and
958 * _never_ need fcsel, since the latter does additional
959 * floating-point-specific processing whereas the
960 * former just moves bits on the wire. It's not obvious
961 * why these are separate opcodes, save for the ability
962 * to do things like sat/pos/abs/neg for free */
963
964 bool mixed = nir_is_non_scalar_swizzle(&instr->src[0], nr_components);
965 op = mixed ? midgard_alu_op_icsel_v : midgard_alu_op_icsel;
966
967 /* csel works as a two-arg in Midgard, since the condition is hardcoded in r31.w */
968 nr_inputs = 2;
969
970 /* Emit the condition into r31 */
971
972 if (mixed)
973 emit_condition_mixed(ctx, &instr->src[0], nr_components);
974 else
975 emit_condition(ctx, &instr->src[0].src, false, instr->src[0].swizzle[0]);
976
977 /* The condition is the first argument; move the other
978 * arguments up one to be a binary instruction for
979 * Midgard */
980
981 memmove(instr->src, instr->src + 1, 2 * sizeof(nir_alu_src));
982 break;
983 }
984
985 default:
986 DBG("Unhandled ALU op %s\n", nir_op_infos[instr->op].name);
987 assert(0);
988 return;
989 }
990
991 /* Midgard can perform certain modifiers on output of an ALU op */
992 unsigned outmod;
993
994 if (midgard_is_integer_out_op(op)) {
995 outmod = midgard_outmod_int_wrap;
996 } else {
997 bool sat = instr->dest.saturate || instr->op == nir_op_fsat;
998 outmod = sat ? midgard_outmod_sat : midgard_outmod_none;
999 }
1000
1001 /* fmax(a, 0.0) can turn into a .pos modifier as an optimization */
1002
1003 if (instr->op == nir_op_fmax) {
1004 if (nir_is_fzero_constant(instr->src[0].src)) {
1005 op = midgard_alu_op_fmov;
1006 nr_inputs = 1;
1007 outmod = midgard_outmod_pos;
1008 instr->src[0] = instr->src[1];
1009 } else if (nir_is_fzero_constant(instr->src[1].src)) {
1010 op = midgard_alu_op_fmov;
1011 nr_inputs = 1;
1012 outmod = midgard_outmod_pos;
1013 }
1014 }
1015
1016 /* Fetch unit, quirks, etc information */
1017 unsigned opcode_props = alu_opcode_props[op].props;
1018 bool quirk_flipped_r24 = opcode_props & QUIRK_FLIPPED_R24;
1019
1020 /* src0 will always exist afaik, but src1 will not for 1-argument
1021 * instructions. The latter can only be fetched if the instruction
1022 * needs it, or else we may segfault. */
1023
1024 unsigned src0 = nir_alu_src_index(ctx, &instr->src[0]);
1025 unsigned src1 = nr_inputs == 2 ? nir_alu_src_index(ctx, &instr->src[1]) : ~0;
1026
1027 /* Rather than use the instruction generation helpers, we do it
1028 * ourselves here to avoid the mess */
1029
1030 midgard_instruction ins = {
1031 .type = TAG_ALU_4,
1032 .ssa_args = {
1033 .src = {
1034 quirk_flipped_r24 ? ~0 : src0,
1035 quirk_flipped_r24 ? src0 : src1,
1036 ~0
1037 },
1038 .dest = dest,
1039 }
1040 };
1041
1042 nir_alu_src *nirmods[2] = { NULL };
1043
1044 if (nr_inputs == 2) {
1045 nirmods[0] = &instr->src[0];
1046 nirmods[1] = &instr->src[1];
1047 } else if (nr_inputs == 1) {
1048 nirmods[quirk_flipped_r24] = &instr->src[0];
1049 } else {
1050 assert(0);
1051 }
1052
1053 /* These were lowered to a move, so apply the corresponding mod */
1054
1055 if (instr->op == nir_op_fneg || instr->op == nir_op_fabs) {
1056 nir_alu_src *s = nirmods[quirk_flipped_r24];
1057
1058 if (instr->op == nir_op_fneg)
1059 s->negate = !s->negate;
1060
1061 if (instr->op == nir_op_fabs)
1062 s->abs = !s->abs;
1063 }
1064
1065 bool is_int = midgard_is_integer_op(op);
1066
1067 ins.mask = mask_of(nr_components);
1068
1069 midgard_vector_alu alu = {
1070 .op = op,
1071 .reg_mode = reg_mode,
1072 .dest_override = dest_override,
1073 .outmod = outmod,
1074
1075 .src1 = vector_alu_srco_unsigned(vector_alu_modifiers(nirmods[0], is_int, broadcast_swizzle, half_1, sext_1)),
1076 .src2 = vector_alu_srco_unsigned(vector_alu_modifiers(nirmods[1], is_int, broadcast_swizzle, half_2, sext_2)),
1077 };
1078
1079 /* Apply writemask if non-SSA, keeping in mind that we can't write to components that don't exist */
1080
1081 if (!is_ssa)
1082 ins.mask &= instr->dest.write_mask;
1083
1084 ins.alu = alu;
1085
1086 /* Late fixup for emulated instructions */
1087
1088 if (instr->op == nir_op_b2f32 || instr->op == nir_op_b2i32) {
1089 /* Presently, our second argument is an inline #0 constant.
1090 * Switch over to an embedded 1.0 constant (that can't fit
1091 * inline, since we're 32-bit, not 16-bit like the inline
1092 * constants) */
1093
1094 ins.ssa_args.inline_constant = false;
1095 ins.ssa_args.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT);
1096 ins.has_constants = true;
1097
1098 if (instr->op == nir_op_b2f32) {
1099 ins.constants[0] = 1.0f;
1100 } else {
1101 /* Type pun it into place */
1102 uint32_t one = 0x1;
1103 memcpy(&ins.constants[0], &one, sizeof(uint32_t));
1104 }
1105
1106 ins.alu.src2 = vector_alu_srco_unsigned(blank_alu_src_xxxx);
1107 } else if (nr_inputs == 1 && !quirk_flipped_r24) {
1108 /* Lots of instructions need a 0 plonked in */
1109 ins.ssa_args.inline_constant = false;
1110 ins.ssa_args.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT);
1111 ins.has_constants = true;
1112 ins.constants[0] = 0.0f;
1113 ins.alu.src2 = vector_alu_srco_unsigned(blank_alu_src_xxxx);
1114 } else if (instr->op == nir_op_inot) {
1115 ins.invert = true;
1116 }
1117
1118 if ((opcode_props & UNITS_ALL) == UNIT_VLUT) {
1119 /* To avoid duplicating the lookup tables (probably), true LUT
1120 * instructions can only operate as if they were scalars. Lower
1121 * them here by changing the component. */
1122
1123 uint8_t original_swizzle[4];
1124 memcpy(original_swizzle, nirmods[0]->swizzle, sizeof(nirmods[0]->swizzle));
1125 unsigned orig_mask = ins.mask;
1126
1127 for (int i = 0; i < nr_components; ++i) {
1128 /* Mask the associated component, dropping the
1129 * instruction if needed */
1130
1131 ins.mask = 1 << i;
1132 ins.mask &= orig_mask;
1133
1134 if (!ins.mask)
1135 continue;
1136
1137 for (int j = 0; j < 4; ++j)
1138 nirmods[0]->swizzle[j] = original_swizzle[i]; /* Pull from the correct component */
1139
1140 ins.alu.src1 = vector_alu_srco_unsigned(vector_alu_modifiers(nirmods[0], is_int, broadcast_swizzle, half_1, false));
1141 emit_mir_instruction(ctx, ins);
1142 }
1143 } else {
1144 emit_mir_instruction(ctx, ins);
1145 }
1146 }
1147
1148 #undef ALU_CASE
1149
1150 static unsigned
1151 mir_mask_for_intr(nir_instr *instr, bool is_read)
1152 {
1153 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
1154
1155 if (is_read)
1156 return mask_of(nir_intrinsic_dest_components(intr));
1157 else
1158 return nir_intrinsic_write_mask(intr);
1159 }
1160
1161 /* Uniforms and UBOs use a shared code path, as uniforms are just (slightly
1162 * optimized) versions of UBO #0 */
1163
1164 midgard_instruction *
1165 emit_ubo_read(
1166 compiler_context *ctx,
1167 nir_instr *instr,
1168 unsigned dest,
1169 unsigned offset,
1170 nir_src *indirect_offset,
1171 unsigned index)
1172 {
1173 /* TODO: half-floats */
1174
1175 midgard_instruction ins = m_ld_ubo_int4(dest, offset);
1176
1177 assert((offset & 0xF) == 0);
1178 offset /= 16;
1179
1180 /* TODO: Don't split */
1181 ins.load_store.varying_parameters = (offset & 7) << 7;
1182 ins.load_store.address = offset >> 3;
1183 ins.mask = mir_mask_for_intr(instr, true);
1184
1185 if (indirect_offset) {
1186 ins.ssa_args.src[1] = nir_src_index(ctx, indirect_offset);
1187 ins.load_store.arg_2 = 0x80;
1188 } else {
1189 ins.load_store.arg_2 = 0x1E;
1190 }
1191
1192 ins.load_store.arg_1 = index;
1193
1194 return emit_mir_instruction(ctx, ins);
1195 }
1196
1197 /* SSBO reads are like UBO reads if you squint */
1198
1199 static void
1200 emit_ssbo_access(
1201 compiler_context *ctx,
1202 nir_instr *instr,
1203 bool is_read,
1204 unsigned srcdest,
1205 unsigned offset,
1206 nir_src *indirect_offset,
1207 unsigned index)
1208 {
1209 /* TODO: types */
1210
1211 midgard_instruction ins;
1212
1213 if (is_read)
1214 ins = m_ld_int4(srcdest, offset);
1215 else
1216 ins = m_st_int4(srcdest, offset);
1217
1218 /* SSBO reads use a generic memory read interface, so we need the
1219 * address of the SSBO as the first argument. This is a sysval. */
1220
1221 unsigned addr = make_compiler_temp(ctx);
1222 emit_sysval_read(ctx, instr, addr, 2);
1223
1224 /* The source array is a bit of a leaky abstraction for SSBOs.
1225 * Nevertheless, for loads:
1226 *
1227 * src[0] = arg_1
1228 * src[1] = arg_2
1229 * src[2] = unused
1230 *
1231 * Whereas for stores:
1232 *
1233 * src[0] = value
1234 * src[1] = arg_1
1235 * src[2] = arg_2
1236 *
1237 * We would like arg_1 = the address and
1238 * arg_2 = the offset.
1239 */
1240
1241 ins.ssa_args.src[is_read ? 0 : 1] = addr;
1242
1243 /* TODO: What is this? It looks superficially like a shift << 5, but
1244 * arg_1 doesn't take a shift Should it be E0 or A0? */
1245 if (indirect_offset)
1246 ins.load_store.arg_1 |= 0xE0;
1247
1248 /* We also need to emit the indirect offset */
1249
1250 if (indirect_offset)
1251 ins.ssa_args.src[is_read ? 1 : 2] = nir_src_index(ctx, indirect_offset);
1252 else
1253 ins.load_store.arg_2 = 0x7E;
1254
1255 /* TODO: Bounds check */
1256
1257 /* Finally, we emit the direct offset */
1258
1259 ins.load_store.varying_parameters = (offset & 0x1FF) << 1;
1260 ins.load_store.address = (offset >> 9);
1261 ins.mask = mir_mask_for_intr(instr, is_read);
1262
1263 emit_mir_instruction(ctx, ins);
1264 }
1265
1266 static void
1267 emit_varying_read(
1268 compiler_context *ctx,
1269 unsigned dest, unsigned offset,
1270 unsigned nr_comp, unsigned component,
1271 nir_src *indirect_offset, nir_alu_type type)
1272 {
1273 /* XXX: Half-floats? */
1274 /* TODO: swizzle, mask */
1275
1276 midgard_instruction ins = m_ld_vary_32(dest, offset);
1277 ins.mask = mask_of(nr_comp);
1278 ins.load_store.swizzle = SWIZZLE_XYZW >> (2 * component);
1279
1280 midgard_varying_parameter p = {
1281 .is_varying = 1,
1282 .interpolation = midgard_interp_default,
1283 .flat = /*var->data.interpolation == INTERP_MODE_FLAT*/ 0
1284 };
1285
1286 unsigned u;
1287 memcpy(&u, &p, sizeof(p));
1288 ins.load_store.varying_parameters = u;
1289
1290 if (indirect_offset)
1291 ins.ssa_args.src[1] = nir_src_index(ctx, indirect_offset);
1292 else
1293 ins.load_store.arg_2 = 0x1E;
1294
1295 ins.load_store.arg_1 = 0x9E;
1296
1297 /* Use the type appropriate load */
1298 switch (type) {
1299 case nir_type_uint:
1300 case nir_type_bool:
1301 ins.load_store.op = midgard_op_ld_vary_32u;
1302 break;
1303 case nir_type_int:
1304 ins.load_store.op = midgard_op_ld_vary_32i;
1305 break;
1306 case nir_type_float:
1307 ins.load_store.op = midgard_op_ld_vary_32;
1308 break;
1309 default:
1310 unreachable("Attempted to load unknown type");
1311 break;
1312 }
1313
1314 emit_mir_instruction(ctx, ins);
1315 }
1316
1317 void
1318 emit_sysval_read(compiler_context *ctx, nir_instr *instr, signed dest_override,
1319 unsigned nr_components)
1320 {
1321 unsigned dest = 0;
1322
1323 /* Figure out which uniform this is */
1324 int sysval = sysval_for_instr(ctx, instr, &dest);
1325 void *val = _mesa_hash_table_u64_search(ctx->sysval_to_id, sysval);
1326
1327 if (dest_override >= 0)
1328 dest = dest_override;
1329
1330 /* Sysvals are prefix uniforms */
1331 unsigned uniform = ((uintptr_t) val) - 1;
1332
1333 /* Emit the read itself -- this is never indirect */
1334 midgard_instruction *ins =
1335 emit_ubo_read(ctx, instr, dest, uniform * 16, NULL, 0);
1336
1337 ins->mask = mask_of(nr_components);
1338 }
1339
1340 static unsigned
1341 compute_builtin_arg(nir_op op)
1342 {
1343 switch (op) {
1344 case nir_intrinsic_load_work_group_id:
1345 return 0x14;
1346 case nir_intrinsic_load_local_invocation_id:
1347 return 0x10;
1348 default:
1349 unreachable("Invalid compute paramater loaded");
1350 }
1351 }
1352
1353 /* Emit store for a fragment shader, which is encoded via a fancy branch. TODO:
1354 * Handle MRT here */
1355
1356 static void
1357 emit_fragment_store(compiler_context *ctx, unsigned src, unsigned rt)
1358 {
1359 /* First, move in whatever we're outputting */
1360 midgard_instruction move = v_mov(src, blank_alu_src, SSA_FIXED_REGISTER(0));
1361 if (rt != 0) {
1362 /* Force a tight schedule. TODO: Make the scheduler MRT aware */
1363 move.unit = UNIT_VMUL;
1364 move.precede_break = true;
1365 move.dont_eliminate = true;
1366 }
1367
1368 emit_mir_instruction(ctx, move);
1369
1370 /* If we're doing MRT, we need to specify the render target */
1371
1372 midgard_instruction rt_move = {
1373 .ssa_args = {
1374 .dest = ~0
1375 }
1376 };
1377
1378 if (rt != 0) {
1379 /* We'll write to r1.z */
1380 rt_move = v_mov(~0, blank_alu_src, SSA_FIXED_REGISTER(1));
1381 rt_move.mask = 1 << COMPONENT_Z;
1382 rt_move.unit = UNIT_SADD;
1383
1384 /* r1.z = (rt * 0x100) */
1385 rt_move.ssa_args.inline_constant = true;
1386 rt_move.inline_constant = (rt * 0x100);
1387
1388 /* r1 */
1389 ctx->work_registers = MAX2(ctx->work_registers, 1);
1390
1391 /* Do the write */
1392 emit_mir_instruction(ctx, rt_move);
1393 }
1394
1395 /* Next, generate the branch. For R render targets in the writeout, the
1396 * i'th render target jumps to pseudo-offset [2(R-1) + i] */
1397
1398 unsigned offset = (2 * (ctx->nir->num_outputs - 1)) + rt;
1399
1400 struct midgard_instruction ins =
1401 v_alu_br_compact_cond(midgard_jmp_writeout_op_writeout, TAG_ALU_4, offset, midgard_condition_always);
1402
1403 /* Add dependencies */
1404 ins.ssa_args.src[0] = move.ssa_args.dest;
1405 ins.ssa_args.src[1] = rt_move.ssa_args.dest;
1406
1407 /* Emit the branch */
1408 emit_mir_instruction(ctx, ins);
1409 }
1410
1411 static void
1412 emit_compute_builtin(compiler_context *ctx, nir_intrinsic_instr *instr)
1413 {
1414 unsigned reg = nir_dest_index(ctx, &instr->dest);
1415 midgard_instruction ins = m_ld_compute_id(reg, 0);
1416 ins.mask = mask_of(3);
1417 ins.load_store.arg_1 = compute_builtin_arg(instr->intrinsic);
1418 emit_mir_instruction(ctx, ins);
1419 }
1420 static void
1421 emit_intrinsic(compiler_context *ctx, nir_intrinsic_instr *instr)
1422 {
1423 unsigned offset = 0, reg;
1424
1425 switch (instr->intrinsic) {
1426 case nir_intrinsic_discard_if:
1427 emit_condition(ctx, &instr->src[0], true, COMPONENT_X);
1428
1429 /* fallthrough */
1430
1431 case nir_intrinsic_discard: {
1432 bool conditional = instr->intrinsic == nir_intrinsic_discard_if;
1433 struct midgard_instruction discard = v_branch(conditional, false);
1434 discard.branch.target_type = TARGET_DISCARD;
1435 emit_mir_instruction(ctx, discard);
1436 break;
1437 }
1438
1439 case nir_intrinsic_load_uniform:
1440 case nir_intrinsic_load_ubo:
1441 case nir_intrinsic_load_ssbo:
1442 case nir_intrinsic_load_input: {
1443 bool is_uniform = instr->intrinsic == nir_intrinsic_load_uniform;
1444 bool is_ubo = instr->intrinsic == nir_intrinsic_load_ubo;
1445 bool is_ssbo = instr->intrinsic == nir_intrinsic_load_ssbo;
1446
1447 /* Get the base type of the intrinsic */
1448 /* TODO: Infer type? Does it matter? */
1449 nir_alu_type t =
1450 (is_ubo || is_ssbo) ? nir_type_uint : nir_intrinsic_type(instr);
1451 t = nir_alu_type_get_base_type(t);
1452
1453 if (!(is_ubo || is_ssbo)) {
1454 offset = nir_intrinsic_base(instr);
1455 }
1456
1457 unsigned nr_comp = nir_intrinsic_dest_components(instr);
1458
1459 nir_src *src_offset = nir_get_io_offset_src(instr);
1460
1461 bool direct = nir_src_is_const(*src_offset);
1462 nir_src *indirect_offset = direct ? NULL : src_offset;
1463
1464 if (direct)
1465 offset += nir_src_as_uint(*src_offset);
1466
1467 /* We may need to apply a fractional offset */
1468 int component = instr->intrinsic == nir_intrinsic_load_input ?
1469 nir_intrinsic_component(instr) : 0;
1470 reg = nir_dest_index(ctx, &instr->dest);
1471
1472 if (is_uniform && !ctx->is_blend) {
1473 emit_ubo_read(ctx, &instr->instr, reg, (ctx->sysval_count + offset) * 16, indirect_offset, 0);
1474 } else if (is_ubo) {
1475 nir_src index = instr->src[0];
1476
1477 /* We don't yet support indirect UBOs. For indirect
1478 * block numbers (if that's possible), we don't know
1479 * enough about the hardware yet. For indirect sources,
1480 * we know what we need but we need to add some NIR
1481 * support for lowering correctly with respect to
1482 * 128-bit reads */
1483
1484 assert(nir_src_is_const(index));
1485 assert(nir_src_is_const(*src_offset));
1486
1487 uint32_t uindex = nir_src_as_uint(index) + 1;
1488 emit_ubo_read(ctx, &instr->instr, reg, offset, NULL, uindex);
1489 } else if (is_ssbo) {
1490 nir_src index = instr->src[0];
1491 assert(nir_src_is_const(index));
1492 uint32_t uindex = nir_src_as_uint(index);
1493
1494 emit_ssbo_access(ctx, &instr->instr, true, reg, offset, indirect_offset, uindex);
1495 } else if (ctx->stage == MESA_SHADER_FRAGMENT && !ctx->is_blend) {
1496 emit_varying_read(ctx, reg, offset, nr_comp, component, !direct ? &instr->src[0] : NULL, t);
1497 } else if (ctx->is_blend) {
1498 /* For blend shaders, load the input color, which is
1499 * preloaded to r0 */
1500
1501 midgard_instruction move = v_mov(SSA_FIXED_REGISTER(0), blank_alu_src, reg);
1502 emit_mir_instruction(ctx, move);
1503 } else if (ctx->stage == MESA_SHADER_VERTEX) {
1504 midgard_instruction ins = m_ld_attr_32(reg, offset);
1505 ins.load_store.arg_1 = 0x1E;
1506 ins.load_store.arg_2 = 0x1E;
1507 ins.mask = mask_of(nr_comp);
1508
1509 /* Use the type appropriate load */
1510 switch (t) {
1511 case nir_type_uint:
1512 case nir_type_bool:
1513 ins.load_store.op = midgard_op_ld_attr_32u;
1514 break;
1515 case nir_type_int:
1516 ins.load_store.op = midgard_op_ld_attr_32i;
1517 break;
1518 case nir_type_float:
1519 ins.load_store.op = midgard_op_ld_attr_32;
1520 break;
1521 default:
1522 unreachable("Attempted to load unknown type");
1523 break;
1524 }
1525
1526 emit_mir_instruction(ctx, ins);
1527 } else {
1528 DBG("Unknown load\n");
1529 assert(0);
1530 }
1531
1532 break;
1533 }
1534
1535 /* Reads 128-bit value raw off the tilebuffer during blending, tasty */
1536
1537 case nir_intrinsic_load_raw_output_pan:
1538 reg = nir_dest_index(ctx, &instr->dest);
1539 assert(ctx->is_blend);
1540
1541 midgard_instruction ins = m_ld_color_buffer_8(reg, 0);
1542 emit_mir_instruction(ctx, ins);
1543 break;
1544
1545 case nir_intrinsic_load_blend_const_color_rgba: {
1546 assert(ctx->is_blend);
1547 reg = nir_dest_index(ctx, &instr->dest);
1548
1549 /* Blend constants are embedded directly in the shader and
1550 * patched in, so we use some magic routing */
1551
1552 midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), blank_alu_src, reg);
1553 ins.has_constants = true;
1554 ins.has_blend_constant = true;
1555 emit_mir_instruction(ctx, ins);
1556 break;
1557 }
1558
1559 case nir_intrinsic_store_output:
1560 assert(nir_src_is_const(instr->src[1]) && "no indirect outputs");
1561
1562 offset = nir_intrinsic_base(instr) + nir_src_as_uint(instr->src[1]);
1563
1564 reg = nir_src_index(ctx, &instr->src[0]);
1565
1566 if (ctx->stage == MESA_SHADER_FRAGMENT) {
1567 /* Determine number of render targets */
1568 emit_fragment_store(ctx, reg, offset);
1569 } else if (ctx->stage == MESA_SHADER_VERTEX) {
1570 /* We should have been vectorized, though we don't
1571 * currently check that st_vary is emitted only once
1572 * per slot (this is relevant, since there's not a mask
1573 * parameter available on the store [set to 0 by the
1574 * blob]). We do respect the component by adjusting the
1575 * swizzle. If this is a constant source, we'll need to
1576 * emit that explicitly. */
1577
1578 emit_explicit_constant(ctx, reg, reg);
1579
1580 unsigned component = nir_intrinsic_component(instr);
1581 unsigned nr_comp = nir_src_num_components(instr->src[0]);
1582
1583 midgard_instruction st = m_st_vary_32(reg, offset);
1584 st.load_store.arg_1 = 0x9E;
1585 st.load_store.arg_2 = 0x1E;
1586 st.load_store.swizzle = swizzle_of(nr_comp) << (2*component);
1587 emit_mir_instruction(ctx, st);
1588 } else {
1589 DBG("Unknown store\n");
1590 assert(0);
1591 }
1592
1593 break;
1594
1595 /* Special case of store_output for lowered blend shaders */
1596 case nir_intrinsic_store_raw_output_pan:
1597 assert (ctx->stage == MESA_SHADER_FRAGMENT);
1598 reg = nir_src_index(ctx, &instr->src[0]);
1599 emit_fragment_store(ctx, reg, 0);
1600
1601 break;
1602
1603 case nir_intrinsic_store_ssbo:
1604 assert(nir_src_is_const(instr->src[1]));
1605
1606 bool direct_offset = nir_src_is_const(instr->src[2]);
1607 offset = direct_offset ? nir_src_as_uint(instr->src[2]) : 0;
1608 nir_src *indirect_offset = direct_offset ? NULL : &instr->src[2];
1609 reg = nir_src_index(ctx, &instr->src[0]);
1610
1611 uint32_t uindex = nir_src_as_uint(instr->src[1]);
1612
1613 emit_explicit_constant(ctx, reg, reg);
1614 emit_ssbo_access(ctx, &instr->instr, false, reg, offset, indirect_offset, uindex);
1615 break;
1616
1617 case nir_intrinsic_load_alpha_ref_float:
1618 assert(instr->dest.is_ssa);
1619
1620 float ref_value = ctx->alpha_ref;
1621
1622 /* See emit_load_const */
1623 float *v = ralloc_array(NULL, float, 4);
1624 memcpy(v, &ref_value, sizeof(float));
1625 _mesa_hash_table_u64_insert(ctx->ssa_constants, (instr->dest.ssa.index << 1) + 1, v);
1626 break;
1627
1628 case nir_intrinsic_load_viewport_scale:
1629 case nir_intrinsic_load_viewport_offset:
1630 case nir_intrinsic_load_num_work_groups:
1631 emit_sysval_read(ctx, &instr->instr, ~0, 3);
1632 break;
1633
1634 case nir_intrinsic_load_work_group_id:
1635 case nir_intrinsic_load_local_invocation_id:
1636 emit_compute_builtin(ctx, instr);
1637 break;
1638
1639 default:
1640 printf ("Unhandled intrinsic\n");
1641 assert(0);
1642 break;
1643 }
1644 }
1645
1646 static unsigned
1647 midgard_tex_format(enum glsl_sampler_dim dim)
1648 {
1649 switch (dim) {
1650 case GLSL_SAMPLER_DIM_1D:
1651 case GLSL_SAMPLER_DIM_BUF:
1652 return MALI_TEX_1D;
1653
1654 case GLSL_SAMPLER_DIM_2D:
1655 case GLSL_SAMPLER_DIM_EXTERNAL:
1656 case GLSL_SAMPLER_DIM_RECT:
1657 return MALI_TEX_2D;
1658
1659 case GLSL_SAMPLER_DIM_3D:
1660 return MALI_TEX_3D;
1661
1662 case GLSL_SAMPLER_DIM_CUBE:
1663 return MALI_TEX_CUBE;
1664
1665 default:
1666 DBG("Unknown sampler dim type\n");
1667 assert(0);
1668 return 0;
1669 }
1670 }
1671
1672 /* Tries to attach an explicit LOD / bias as a constant. Returns whether this
1673 * was successful */
1674
1675 static bool
1676 pan_attach_constant_bias(
1677 compiler_context *ctx,
1678 nir_src lod,
1679 midgard_texture_word *word)
1680 {
1681 /* To attach as constant, it has to *be* constant */
1682
1683 if (!nir_src_is_const(lod))
1684 return false;
1685
1686 float f = nir_src_as_float(lod);
1687
1688 /* Break into fixed-point */
1689 signed lod_int = f;
1690 float lod_frac = f - lod_int;
1691
1692 /* Carry over negative fractions */
1693 if (lod_frac < 0.0) {
1694 lod_int--;
1695 lod_frac += 1.0;
1696 }
1697
1698 /* Encode */
1699 word->bias = float_to_ubyte(lod_frac);
1700 word->bias_int = lod_int;
1701
1702 return true;
1703 }
1704
1705 static enum mali_sampler_type
1706 midgard_sampler_type(nir_alu_type t) {
1707 switch (nir_alu_type_get_base_type(t))
1708 {
1709 case nir_type_float:
1710 return MALI_SAMPLER_FLOAT;
1711 case nir_type_int:
1712 return MALI_SAMPLER_SIGNED;
1713 case nir_type_uint:
1714 return MALI_SAMPLER_UNSIGNED;
1715 default:
1716 unreachable("Unknown sampler type");
1717 }
1718 }
1719
1720 static void
1721 emit_texop_native(compiler_context *ctx, nir_tex_instr *instr,
1722 unsigned midgard_texop)
1723 {
1724 /* TODO */
1725 //assert (!instr->sampler);
1726 //assert (!instr->texture_array_size);
1727
1728 int texture_index = instr->texture_index;
1729 int sampler_index = texture_index;
1730
1731 /* No helper to build texture words -- we do it all here */
1732 midgard_instruction ins = {
1733 .type = TAG_TEXTURE_4,
1734 .mask = 0xF,
1735 .ssa_args = {
1736 .dest = nir_dest_index(ctx, &instr->dest),
1737 .src = { ~0, ~0, ~0 },
1738 },
1739 .texture = {
1740 .op = midgard_texop,
1741 .format = midgard_tex_format(instr->sampler_dim),
1742 .texture_handle = texture_index,
1743 .sampler_handle = sampler_index,
1744 .swizzle = SWIZZLE_XYZW,
1745 .in_reg_swizzle = SWIZZLE_XYZW,
1746
1747 /* TODO: half */
1748 .in_reg_full = 1,
1749 .out_full = 1,
1750
1751 .sampler_type = midgard_sampler_type(instr->dest_type),
1752 }
1753 };
1754
1755 for (unsigned i = 0; i < instr->num_srcs; ++i) {
1756 int index = nir_src_index(ctx, &instr->src[i].src);
1757 midgard_vector_alu_src alu_src = blank_alu_src;
1758 unsigned nr_components = nir_src_num_components(instr->src[i].src);
1759
1760 switch (instr->src[i].src_type) {
1761 case nir_tex_src_coord: {
1762 emit_explicit_constant(ctx, index, index);
1763
1764 /* Texelfetch coordinates uses all four elements
1765 * (xyz/index) regardless of texture dimensionality,
1766 * which means it's necessary to zero the unused
1767 * components to keep everything happy */
1768
1769 if (midgard_texop == TEXTURE_OP_TEXEL_FETCH) {
1770 unsigned old_index = index;
1771
1772 index = make_compiler_temp(ctx);
1773
1774 /* mov index, old_index */
1775 midgard_instruction mov = v_mov(old_index, blank_alu_src, index);
1776 mov.mask = 0x3;
1777 emit_mir_instruction(ctx, mov);
1778
1779 /* mov index.zw, #0 */
1780 mov = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT),
1781 blank_alu_src, index);
1782 mov.has_constants = true;
1783 mov.mask = (1 << COMPONENT_Z) | (1 << COMPONENT_W);
1784 emit_mir_instruction(ctx, mov);
1785 }
1786
1787 if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE) {
1788 /* texelFetch is undefined on samplerCube */
1789 assert(midgard_texop != TEXTURE_OP_TEXEL_FETCH);
1790
1791 /* For cubemaps, we use a special ld/st op to
1792 * select the face and copy the xy into the
1793 * texture register */
1794
1795 unsigned temp = make_compiler_temp(ctx);
1796 midgard_instruction ld = m_ld_cubemap_coords(temp, 0);
1797 ld.ssa_args.src[0] = index;
1798 ld.mask = 0x3; /* xy */
1799 ld.load_store.arg_1 = 0x20;
1800 ld.load_store.swizzle = alu_src.swizzle;
1801 emit_mir_instruction(ctx, ld);
1802
1803 ins.ssa_args.src[0] = temp;
1804 ins.texture.in_reg_swizzle = SWIZZLE_XYXX;
1805 } else {
1806 ins.ssa_args.src[0] = index;
1807 }
1808
1809 if (instr->sampler_dim == GLSL_SAMPLER_DIM_2D) {
1810 /* Array component in w but NIR wants it in z */
1811 if (nr_components == 3)
1812 ins.texture.in_reg_swizzle = SWIZZLE_XYZZ;
1813 else if (nr_components == 2)
1814 ins.texture.in_reg_swizzle = SWIZZLE_XYXX;
1815 else
1816 unreachable("Invalid texture 2D components");
1817 }
1818
1819 break;
1820 }
1821
1822 case nir_tex_src_bias:
1823 case nir_tex_src_lod: {
1824 /* Try as a constant if we can */
1825
1826 bool is_txf = midgard_texop == TEXTURE_OP_TEXEL_FETCH;
1827 if (!is_txf && pan_attach_constant_bias(ctx, instr->src[i].src, &ins.texture))
1828 break;
1829
1830 ins.texture.lod_register = true;
1831 ins.ssa_args.src[1] = index;
1832 emit_explicit_constant(ctx, index, index);
1833
1834 break;
1835 };
1836
1837 default:
1838 unreachable("Unknown texture source type\n");
1839 }
1840 }
1841
1842 emit_mir_instruction(ctx, ins);
1843
1844 /* Used for .cont and .last hinting */
1845 ctx->texture_op_count++;
1846 }
1847
1848 static void
1849 emit_tex(compiler_context *ctx, nir_tex_instr *instr)
1850 {
1851 /* Fixup op, since only textureLod is permitted in VS but NIR can give
1852 * generic tex in some cases (which confuses the hardware) */
1853
1854 bool is_vertex = ctx->stage == MESA_SHADER_VERTEX;
1855
1856 if (is_vertex && instr->op == nir_texop_tex)
1857 instr->op = nir_texop_txl;
1858
1859 switch (instr->op) {
1860 case nir_texop_tex:
1861 case nir_texop_txb:
1862 emit_texop_native(ctx, instr, TEXTURE_OP_NORMAL);
1863 break;
1864 case nir_texop_txl:
1865 emit_texop_native(ctx, instr, TEXTURE_OP_LOD);
1866 break;
1867 case nir_texop_txf:
1868 emit_texop_native(ctx, instr, TEXTURE_OP_TEXEL_FETCH);
1869 break;
1870 case nir_texop_txs:
1871 emit_sysval_read(ctx, &instr->instr, ~0, 4);
1872 break;
1873 default:
1874 unreachable("Unhanlded texture op");
1875 }
1876 }
1877
1878 static void
1879 emit_jump(compiler_context *ctx, nir_jump_instr *instr)
1880 {
1881 switch (instr->type) {
1882 case nir_jump_break: {
1883 /* Emit a branch out of the loop */
1884 struct midgard_instruction br = v_branch(false, false);
1885 br.branch.target_type = TARGET_BREAK;
1886 br.branch.target_break = ctx->current_loop_depth;
1887 emit_mir_instruction(ctx, br);
1888 break;
1889 }
1890
1891 default:
1892 DBG("Unknown jump type %d\n", instr->type);
1893 break;
1894 }
1895 }
1896
1897 static void
1898 emit_instr(compiler_context *ctx, struct nir_instr *instr)
1899 {
1900 switch (instr->type) {
1901 case nir_instr_type_load_const:
1902 emit_load_const(ctx, nir_instr_as_load_const(instr));
1903 break;
1904
1905 case nir_instr_type_intrinsic:
1906 emit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
1907 break;
1908
1909 case nir_instr_type_alu:
1910 emit_alu(ctx, nir_instr_as_alu(instr));
1911 break;
1912
1913 case nir_instr_type_tex:
1914 emit_tex(ctx, nir_instr_as_tex(instr));
1915 break;
1916
1917 case nir_instr_type_jump:
1918 emit_jump(ctx, nir_instr_as_jump(instr));
1919 break;
1920
1921 case nir_instr_type_ssa_undef:
1922 /* Spurious */
1923 break;
1924
1925 default:
1926 DBG("Unhandled instruction type\n");
1927 break;
1928 }
1929 }
1930
1931
1932 /* ALU instructions can inline or embed constants, which decreases register
1933 * pressure and saves space. */
1934
1935 #define CONDITIONAL_ATTACH(src) { \
1936 void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->ssa_args.src + 1); \
1937 \
1938 if (entry) { \
1939 attach_constants(ctx, alu, entry, alu->ssa_args.src + 1); \
1940 alu->ssa_args.src = SSA_FIXED_REGISTER(REGISTER_CONSTANT); \
1941 } \
1942 }
1943
1944 static void
1945 inline_alu_constants(compiler_context *ctx)
1946 {
1947 mir_foreach_instr(ctx, alu) {
1948 /* Other instructions cannot inline constants */
1949 if (alu->type != TAG_ALU_4) continue;
1950
1951 /* If there is already a constant here, we can do nothing */
1952 if (alu->has_constants) continue;
1953
1954 CONDITIONAL_ATTACH(src[0]);
1955
1956 if (!alu->has_constants) {
1957 CONDITIONAL_ATTACH(src[1])
1958 } else if (!alu->inline_constant) {
1959 /* Corner case: _two_ vec4 constants, for instance with a
1960 * csel. For this case, we can only use a constant
1961 * register for one, we'll have to emit a move for the
1962 * other. Note, if both arguments are constants, then
1963 * necessarily neither argument depends on the value of
1964 * any particular register. As the destination register
1965 * will be wiped, that means we can spill the constant
1966 * to the destination register.
1967 */
1968
1969 void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->ssa_args.src[1] + 1);
1970 unsigned scratch = alu->ssa_args.dest;
1971
1972 if (entry) {
1973 midgard_instruction ins = v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), blank_alu_src, scratch);
1974 attach_constants(ctx, &ins, entry, alu->ssa_args.src[1] + 1);
1975
1976 /* Force a break XXX Defer r31 writes */
1977 ins.unit = UNIT_VLUT;
1978
1979 /* Set the source */
1980 alu->ssa_args.src[1] = scratch;
1981
1982 /* Inject us -before- the last instruction which set r31 */
1983 mir_insert_instruction_before(mir_prev_op(alu), ins);
1984 }
1985 }
1986 }
1987 }
1988
1989 /* Being a little silly with the names, but returns the op that is the bitwise
1990 * inverse of the op with the argument switched. I.e. (f and g are
1991 * contrapositives):
1992 *
1993 * f(a, b) = ~g(b, a)
1994 *
1995 * Corollary: if g is the contrapositve of f, f is the contrapositive of g:
1996 *
1997 * f(a, b) = ~g(b, a)
1998 * ~f(a, b) = g(b, a)
1999 * ~f(a, b) = ~h(a, b) where h is the contrapositive of g
2000 * f(a, b) = h(a, b)
2001 *
2002 * Thus we define this function in pairs.
2003 */
2004
2005 static inline midgard_alu_op
2006 mir_contrapositive(midgard_alu_op op)
2007 {
2008 switch (op) {
2009 case midgard_alu_op_flt:
2010 return midgard_alu_op_fle;
2011 case midgard_alu_op_fle:
2012 return midgard_alu_op_flt;
2013
2014 case midgard_alu_op_ilt:
2015 return midgard_alu_op_ile;
2016 case midgard_alu_op_ile:
2017 return midgard_alu_op_ilt;
2018
2019 default:
2020 unreachable("No known contrapositive");
2021 }
2022 }
2023
2024 /* Midgard supports two types of constants, embedded constants (128-bit) and
2025 * inline constants (16-bit). Sometimes, especially with scalar ops, embedded
2026 * constants can be demoted to inline constants, for space savings and
2027 * sometimes a performance boost */
2028
2029 static void
2030 embedded_to_inline_constant(compiler_context *ctx)
2031 {
2032 mir_foreach_instr(ctx, ins) {
2033 if (!ins->has_constants) continue;
2034
2035 if (ins->ssa_args.inline_constant) continue;
2036
2037 /* Blend constants must not be inlined by definition */
2038 if (ins->has_blend_constant) continue;
2039
2040 /* We can inline 32-bit (sometimes) or 16-bit (usually) */
2041 bool is_16 = ins->alu.reg_mode == midgard_reg_mode_16;
2042 bool is_32 = ins->alu.reg_mode == midgard_reg_mode_32;
2043
2044 if (!(is_16 || is_32))
2045 continue;
2046
2047 /* src1 cannot be an inline constant due to encoding
2048 * restrictions. So, if possible we try to flip the arguments
2049 * in that case */
2050
2051 int op = ins->alu.op;
2052
2053 if (ins->ssa_args.src[0] == SSA_FIXED_REGISTER(REGISTER_CONSTANT)) {
2054 bool flip = alu_opcode_props[op].props & OP_COMMUTES;
2055
2056 switch (op) {
2057 /* Conditionals can be inverted */
2058 case midgard_alu_op_flt:
2059 case midgard_alu_op_ilt:
2060 case midgard_alu_op_fle:
2061 case midgard_alu_op_ile:
2062 ins->alu.op = mir_contrapositive(ins->alu.op);
2063 ins->invert = true;
2064 flip = true;
2065 break;
2066
2067 case midgard_alu_op_fcsel:
2068 case midgard_alu_op_icsel:
2069 DBG("Missed non-commutative flip (%s)\n", alu_opcode_props[op].name);
2070 default:
2071 break;
2072 }
2073
2074 if (flip) {
2075 /* Flip the SSA numbers */
2076 ins->ssa_args.src[0] = ins->ssa_args.src[1];
2077 ins->ssa_args.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT);
2078
2079 /* And flip the modifiers */
2080
2081 unsigned src_temp;
2082
2083 src_temp = ins->alu.src2;
2084 ins->alu.src2 = ins->alu.src1;
2085 ins->alu.src1 = src_temp;
2086 }
2087 }
2088
2089 if (ins->ssa_args.src[1] == SSA_FIXED_REGISTER(REGISTER_CONSTANT)) {
2090 /* Extract the source information */
2091
2092 midgard_vector_alu_src *src;
2093 int q = ins->alu.src2;
2094 midgard_vector_alu_src *m = (midgard_vector_alu_src *) &q;
2095 src = m;
2096
2097 /* Component is from the swizzle, e.g. r26.w -> w component. TODO: What if x is masked out? */
2098 int component = src->swizzle & 3;
2099
2100 /* Scale constant appropriately, if we can legally */
2101 uint16_t scaled_constant = 0;
2102
2103 if (midgard_is_integer_op(op) || is_16) {
2104 unsigned int *iconstants = (unsigned int *) ins->constants;
2105 scaled_constant = (uint16_t) iconstants[component];
2106
2107 /* Constant overflow after resize */
2108 if (scaled_constant != iconstants[component])
2109 continue;
2110 } else {
2111 float original = (float) ins->constants[component];
2112 scaled_constant = _mesa_float_to_half(original);
2113
2114 /* Check for loss of precision. If this is
2115 * mediump, we don't care, but for a highp
2116 * shader, we need to pay attention. NIR
2117 * doesn't yet tell us which mode we're in!
2118 * Practically this prevents most constants
2119 * from being inlined, sadly. */
2120
2121 float fp32 = _mesa_half_to_float(scaled_constant);
2122
2123 if (fp32 != original)
2124 continue;
2125 }
2126
2127 /* We don't know how to handle these with a constant */
2128
2129 if (mir_nontrivial_source2_mod_simple(ins) || src->rep_low || src->rep_high) {
2130 DBG("Bailing inline constant...\n");
2131 continue;
2132 }
2133
2134 /* Make sure that the constant is not itself a
2135 * vector by checking if all accessed values
2136 * (by the swizzle) are the same. */
2137
2138 uint32_t *cons = (uint32_t *) ins->constants;
2139 uint32_t value = cons[component];
2140
2141 bool is_vector = false;
2142 unsigned mask = effective_writemask(&ins->alu, ins->mask);
2143
2144 for (int c = 1; c < 4; ++c) {
2145 /* We only care if this component is actually used */
2146 if (!(mask & (1 << c)))
2147 continue;
2148
2149 uint32_t test = cons[(src->swizzle >> (2 * c)) & 3];
2150
2151 if (test != value) {
2152 is_vector = true;
2153 break;
2154 }
2155 }
2156
2157 if (is_vector)
2158 continue;
2159
2160 /* Get rid of the embedded constant */
2161 ins->has_constants = false;
2162 ins->ssa_args.src[1] = ~0;
2163 ins->ssa_args.inline_constant = true;
2164 ins->inline_constant = scaled_constant;
2165 }
2166 }
2167 }
2168
2169 /* Dead code elimination for branches at the end of a block - only one branch
2170 * per block is legal semantically */
2171
2172 static void
2173 midgard_opt_cull_dead_branch(compiler_context *ctx, midgard_block *block)
2174 {
2175 bool branched = false;
2176
2177 mir_foreach_instr_in_block_safe(block, ins) {
2178 if (!midgard_is_branch_unit(ins->unit)) continue;
2179
2180 /* We ignore prepacked branches since the fragment epilogue is
2181 * just generally special */
2182 if (ins->prepacked_branch) continue;
2183
2184 /* Discards are similarly special and may not correspond to the
2185 * end of a block */
2186
2187 if (ins->branch.target_type == TARGET_DISCARD) continue;
2188
2189 if (branched) {
2190 /* We already branched, so this is dead */
2191 mir_remove_instruction(ins);
2192 }
2193
2194 branched = true;
2195 }
2196 }
2197
2198 /* fmov.pos is an idiom for fpos. Propoagate the .pos up to the source, so then
2199 * the move can be propagated away entirely */
2200
2201 static bool
2202 mir_compose_float_outmod(midgard_outmod_float *outmod, midgard_outmod_float comp)
2203 {
2204 /* Nothing to do */
2205 if (comp == midgard_outmod_none)
2206 return true;
2207
2208 if (*outmod == midgard_outmod_none) {
2209 *outmod = comp;
2210 return true;
2211 }
2212
2213 /* TODO: Compose rules */
2214 return false;
2215 }
2216
2217 static bool
2218 midgard_opt_pos_propagate(compiler_context *ctx, midgard_block *block)
2219 {
2220 bool progress = false;
2221
2222 mir_foreach_instr_in_block_safe(block, ins) {
2223 if (ins->type != TAG_ALU_4) continue;
2224 if (ins->alu.op != midgard_alu_op_fmov) continue;
2225 if (ins->alu.outmod != midgard_outmod_pos) continue;
2226
2227 /* TODO: Registers? */
2228 unsigned src = ins->ssa_args.src[1];
2229 if (src & IS_REG) continue;
2230 assert(!mir_has_multiple_writes(ctx, src));
2231
2232 /* There might be a source modifier, too */
2233 if (mir_nontrivial_source2_mod(ins)) continue;
2234
2235 /* Backpropagate the modifier */
2236 mir_foreach_instr_in_block_from_rev(block, v, mir_prev_op(ins)) {
2237 if (v->type != TAG_ALU_4) continue;
2238 if (v->ssa_args.dest != src) continue;
2239
2240 /* Can we even take a float outmod? */
2241 if (midgard_is_integer_out_op(v->alu.op)) continue;
2242
2243 midgard_outmod_float temp = v->alu.outmod;
2244 progress |= mir_compose_float_outmod(&temp, ins->alu.outmod);
2245
2246 /* Throw in the towel.. */
2247 if (!progress) break;
2248
2249 /* Otherwise, transfer the modifier */
2250 v->alu.outmod = temp;
2251 ins->alu.outmod = midgard_outmod_none;
2252
2253 break;
2254 }
2255 }
2256
2257 return progress;
2258 }
2259
2260 static void
2261 emit_fragment_epilogue(compiler_context *ctx)
2262 {
2263 /* Just emit the last chunk with the branch */
2264 EMIT(alu_br_compact_cond, midgard_jmp_writeout_op_writeout, TAG_ALU_4, ~0, midgard_condition_always);
2265 }
2266
2267 static midgard_block *
2268 create_empty_block(compiler_context *ctx)
2269 {
2270 midgard_block *blk = rzalloc(ctx, midgard_block);
2271
2272 blk->predecessors = _mesa_set_create(blk,
2273 _mesa_hash_pointer,
2274 _mesa_key_pointer_equal);
2275
2276 blk->source_id = ctx->block_source_count++;
2277
2278 return blk;
2279 }
2280
2281 static midgard_block *
2282 emit_block(compiler_context *ctx, nir_block *block)
2283 {
2284 midgard_block *this_block = ctx->after_block;
2285 ctx->after_block = NULL;
2286
2287 if (!this_block)
2288 this_block = create_empty_block(ctx);
2289
2290 list_addtail(&this_block->link, &ctx->blocks);
2291
2292 this_block->is_scheduled = false;
2293 ++ctx->block_count;
2294
2295 ctx->texture_index[0] = ~0;
2296 ctx->texture_index[1] = ~0;
2297
2298 /* Set up current block */
2299 list_inithead(&this_block->instructions);
2300 ctx->current_block = this_block;
2301
2302 nir_foreach_instr(instr, block) {
2303 emit_instr(ctx, instr);
2304 ++ctx->instruction_count;
2305 }
2306
2307 inline_alu_constants(ctx);
2308 embedded_to_inline_constant(ctx);
2309
2310 /* Append fragment shader epilogue (value writeout) */
2311 if (ctx->stage == MESA_SHADER_FRAGMENT) {
2312 if (block == nir_impl_last_block(ctx->func->impl)) {
2313 emit_fragment_epilogue(ctx);
2314 }
2315 }
2316
2317 /* Allow the next control flow to access us retroactively, for
2318 * branching etc */
2319 ctx->current_block = this_block;
2320
2321 return this_block;
2322 }
2323
2324 static midgard_block *emit_cf_list(struct compiler_context *ctx, struct exec_list *list);
2325
2326 static void
2327 emit_if(struct compiler_context *ctx, nir_if *nif)
2328 {
2329 midgard_block *before_block = ctx->current_block;
2330
2331 /* Conditional branches expect the condition in r31.w; emit a move for
2332 * that in the _previous_ block (which is the current block). */
2333 emit_condition(ctx, &nif->condition, true, COMPONENT_X);
2334
2335 /* Speculatively emit the branch, but we can't fill it in until later */
2336 EMIT(branch, true, true);
2337 midgard_instruction *then_branch = mir_last_in_block(ctx->current_block);
2338
2339 /* Emit the two subblocks. */
2340 midgard_block *then_block = emit_cf_list(ctx, &nif->then_list);
2341 midgard_block *end_then_block = ctx->current_block;
2342
2343 /* Emit a jump from the end of the then block to the end of the else */
2344 EMIT(branch, false, false);
2345 midgard_instruction *then_exit = mir_last_in_block(ctx->current_block);
2346
2347 /* Emit second block, and check if it's empty */
2348
2349 int else_idx = ctx->block_count;
2350 int count_in = ctx->instruction_count;
2351 midgard_block *else_block = emit_cf_list(ctx, &nif->else_list);
2352 midgard_block *end_else_block = ctx->current_block;
2353 int after_else_idx = ctx->block_count;
2354
2355 /* Now that we have the subblocks emitted, fix up the branches */
2356
2357 assert(then_block);
2358 assert(else_block);
2359
2360 if (ctx->instruction_count == count_in) {
2361 /* The else block is empty, so don't emit an exit jump */
2362 mir_remove_instruction(then_exit);
2363 then_branch->branch.target_block = after_else_idx;
2364 } else {
2365 then_branch->branch.target_block = else_idx;
2366 then_exit->branch.target_block = after_else_idx;
2367 }
2368
2369 /* Wire up the successors */
2370
2371 ctx->after_block = create_empty_block(ctx);
2372
2373 midgard_block_add_successor(before_block, then_block);
2374 midgard_block_add_successor(before_block, else_block);
2375
2376 midgard_block_add_successor(end_then_block, ctx->after_block);
2377 midgard_block_add_successor(end_else_block, ctx->after_block);
2378 }
2379
2380 static void
2381 emit_loop(struct compiler_context *ctx, nir_loop *nloop)
2382 {
2383 /* Remember where we are */
2384 midgard_block *start_block = ctx->current_block;
2385
2386 /* Allocate a loop number, growing the current inner loop depth */
2387 int loop_idx = ++ctx->current_loop_depth;
2388
2389 /* Get index from before the body so we can loop back later */
2390 int start_idx = ctx->block_count;
2391
2392 /* Emit the body itself */
2393 midgard_block *loop_block = emit_cf_list(ctx, &nloop->body);
2394
2395 /* Branch back to loop back */
2396 struct midgard_instruction br_back = v_branch(false, false);
2397 br_back.branch.target_block = start_idx;
2398 emit_mir_instruction(ctx, br_back);
2399
2400 /* Mark down that branch in the graph. */
2401 midgard_block_add_successor(start_block, loop_block);
2402 midgard_block_add_successor(ctx->current_block, loop_block);
2403
2404 /* Find the index of the block about to follow us (note: we don't add
2405 * one; blocks are 0-indexed so we get a fencepost problem) */
2406 int break_block_idx = ctx->block_count;
2407
2408 /* Fix up the break statements we emitted to point to the right place,
2409 * now that we can allocate a block number for them */
2410 ctx->after_block = create_empty_block(ctx);
2411
2412 list_for_each_entry_from(struct midgard_block, block, start_block, &ctx->blocks, link) {
2413 mir_foreach_instr_in_block(block, ins) {
2414 if (ins->type != TAG_ALU_4) continue;
2415 if (!ins->compact_branch) continue;
2416 if (ins->prepacked_branch) continue;
2417
2418 /* We found a branch -- check the type to see if we need to do anything */
2419 if (ins->branch.target_type != TARGET_BREAK) continue;
2420
2421 /* It's a break! Check if it's our break */
2422 if (ins->branch.target_break != loop_idx) continue;
2423
2424 /* Okay, cool, we're breaking out of this loop.
2425 * Rewrite from a break to a goto */
2426
2427 ins->branch.target_type = TARGET_GOTO;
2428 ins->branch.target_block = break_block_idx;
2429
2430 midgard_block_add_successor(block, ctx->after_block);
2431 }
2432 }
2433
2434 /* Now that we've finished emitting the loop, free up the depth again
2435 * so we play nice with recursion amid nested loops */
2436 --ctx->current_loop_depth;
2437
2438 /* Dump loop stats */
2439 ++ctx->loop_count;
2440 }
2441
2442 static midgard_block *
2443 emit_cf_list(struct compiler_context *ctx, struct exec_list *list)
2444 {
2445 midgard_block *start_block = NULL;
2446
2447 foreach_list_typed(nir_cf_node, node, node, list) {
2448 switch (node->type) {
2449 case nir_cf_node_block: {
2450 midgard_block *block = emit_block(ctx, nir_cf_node_as_block(node));
2451
2452 if (!start_block)
2453 start_block = block;
2454
2455 break;
2456 }
2457
2458 case nir_cf_node_if:
2459 emit_if(ctx, nir_cf_node_as_if(node));
2460 break;
2461
2462 case nir_cf_node_loop:
2463 emit_loop(ctx, nir_cf_node_as_loop(node));
2464 break;
2465
2466 case nir_cf_node_function:
2467 assert(0);
2468 break;
2469 }
2470 }
2471
2472 return start_block;
2473 }
2474
2475 /* Due to lookahead, we need to report the first tag executed in the command
2476 * stream and in branch targets. An initial block might be empty, so iterate
2477 * until we find one that 'works' */
2478
2479 static unsigned
2480 midgard_get_first_tag_from_block(compiler_context *ctx, unsigned block_idx)
2481 {
2482 midgard_block *initial_block = mir_get_block(ctx, block_idx);
2483
2484 unsigned first_tag = 0;
2485
2486 mir_foreach_block_from(ctx, initial_block, v) {
2487 midgard_bundle *initial_bundle =
2488 util_dynarray_element(&v->bundles, midgard_bundle, 0);
2489
2490 if (initial_bundle) {
2491 first_tag = initial_bundle->tag;
2492 break;
2493 }
2494 }
2495
2496 return first_tag;
2497 }
2498
2499 int
2500 midgard_compile_shader_nir(struct midgard_screen *screen, nir_shader *nir, midgard_program *program, bool is_blend)
2501 {
2502 struct util_dynarray *compiled = &program->compiled;
2503
2504 midgard_debug = debug_get_option_midgard_debug();
2505
2506 /* TODO: Bound against what? */
2507 compiler_context *ctx = rzalloc(NULL, compiler_context);
2508
2509 ctx->nir = nir;
2510 ctx->screen = screen;
2511 ctx->stage = nir->info.stage;
2512 ctx->is_blend = is_blend;
2513 ctx->alpha_ref = program->alpha_ref;
2514
2515 /* Start off with a safe cutoff, allowing usage of all 16 work
2516 * registers. Later, we'll promote uniform reads to uniform registers
2517 * if we determine it is beneficial to do so */
2518 ctx->uniform_cutoff = 8;
2519
2520 /* Initialize at a global (not block) level hash tables */
2521
2522 ctx->ssa_constants = _mesa_hash_table_u64_create(NULL);
2523 ctx->hash_to_temp = _mesa_hash_table_u64_create(NULL);
2524 ctx->sysval_to_id = _mesa_hash_table_u64_create(NULL);
2525
2526 /* Record the varying mapping for the command stream's bookkeeping */
2527
2528 struct exec_list *varyings =
2529 ctx->stage == MESA_SHADER_VERTEX ? &nir->outputs : &nir->inputs;
2530
2531 unsigned max_varying = 0;
2532 nir_foreach_variable(var, varyings) {
2533 unsigned loc = var->data.driver_location;
2534 unsigned sz = glsl_type_size(var->type, FALSE);
2535
2536 for (int c = 0; c < sz; ++c) {
2537 program->varyings[loc + c] = var->data.location + c;
2538 max_varying = MAX2(max_varying, loc + c);
2539 }
2540 }
2541
2542 /* Lower gl_Position pre-optimisation, but after lowering vars to ssa
2543 * (so we don't accidentally duplicate the epilogue since mesa/st has
2544 * messed with our I/O quite a bit already) */
2545
2546 NIR_PASS_V(nir, nir_lower_vars_to_ssa);
2547
2548 if (ctx->stage == MESA_SHADER_VERTEX) {
2549 NIR_PASS_V(nir, nir_lower_viewport_transform);
2550 NIR_PASS_V(nir, nir_clamp_psiz, 1.0, 1024.0);
2551 }
2552
2553 NIR_PASS_V(nir, nir_lower_var_copies);
2554 NIR_PASS_V(nir, nir_lower_vars_to_ssa);
2555 NIR_PASS_V(nir, nir_split_var_copies);
2556 NIR_PASS_V(nir, nir_lower_var_copies);
2557 NIR_PASS_V(nir, nir_lower_global_vars_to_local);
2558 NIR_PASS_V(nir, nir_lower_var_copies);
2559 NIR_PASS_V(nir, nir_lower_vars_to_ssa);
2560
2561 NIR_PASS_V(nir, nir_lower_io, nir_var_all, glsl_type_size, 0);
2562
2563 /* Optimisation passes */
2564
2565 optimise_nir(nir);
2566
2567 if (midgard_debug & MIDGARD_DBG_SHADERS) {
2568 nir_print_shader(nir, stdout);
2569 }
2570
2571 /* Assign sysvals and counts, now that we're sure
2572 * (post-optimisation) */
2573
2574 midgard_nir_assign_sysvals(ctx, nir);
2575
2576 program->uniform_count = nir->num_uniforms;
2577 program->sysval_count = ctx->sysval_count;
2578 memcpy(program->sysvals, ctx->sysvals, sizeof(ctx->sysvals[0]) * ctx->sysval_count);
2579
2580 nir_foreach_function(func, nir) {
2581 if (!func->impl)
2582 continue;
2583
2584 list_inithead(&ctx->blocks);
2585 ctx->block_count = 0;
2586 ctx->func = func;
2587
2588 emit_cf_list(ctx, &func->impl->body);
2589
2590 /* Emit empty exit block with successor */
2591
2592 struct midgard_block *semi_end = ctx->current_block;
2593
2594 struct midgard_block *end =
2595 emit_block(ctx, func->impl->end_block);
2596
2597 midgard_block_add_successor(semi_end, end);
2598
2599 break; /* TODO: Multi-function shaders */
2600 }
2601
2602 util_dynarray_init(compiled, NULL);
2603
2604 /* MIR-level optimizations */
2605
2606 bool progress = false;
2607
2608 do {
2609 progress = false;
2610
2611 mir_foreach_block(ctx, block) {
2612 progress |= midgard_opt_pos_propagate(ctx, block);
2613 progress |= midgard_opt_copy_prop(ctx, block);
2614 progress |= midgard_opt_dead_code_eliminate(ctx, block);
2615 progress |= midgard_opt_combine_projection(ctx, block);
2616 progress |= midgard_opt_varying_projection(ctx, block);
2617 progress |= midgard_opt_not_propagate(ctx, block);
2618 progress |= midgard_opt_fuse_src_invert(ctx, block);
2619 progress |= midgard_opt_fuse_dest_invert(ctx, block);
2620 }
2621 } while (progress);
2622
2623 mir_foreach_block(ctx, block) {
2624 midgard_lower_invert(ctx, block);
2625 midgard_lower_derivatives(ctx, block);
2626 }
2627
2628 /* Nested control-flow can result in dead branches at the end of the
2629 * block. This messes with our analysis and is just dead code, so cull
2630 * them */
2631 mir_foreach_block(ctx, block) {
2632 midgard_opt_cull_dead_branch(ctx, block);
2633 }
2634
2635 /* Ensure we were lowered */
2636 mir_foreach_instr_global(ctx, ins) {
2637 assert(!ins->invert);
2638 }
2639
2640 /* Schedule! */
2641 schedule_program(ctx);
2642
2643 /* Now that all the bundles are scheduled and we can calculate block
2644 * sizes, emit actual branch instructions rather than placeholders */
2645
2646 int br_block_idx = 0;
2647
2648 mir_foreach_block(ctx, block) {
2649 util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) {
2650 for (int c = 0; c < bundle->instruction_count; ++c) {
2651 midgard_instruction *ins = bundle->instructions[c];
2652
2653 if (!midgard_is_branch_unit(ins->unit)) continue;
2654
2655 if (ins->prepacked_branch) continue;
2656
2657 /* Parse some basic branch info */
2658 bool is_compact = ins->unit == ALU_ENAB_BR_COMPACT;
2659 bool is_conditional = ins->branch.conditional;
2660 bool is_inverted = ins->branch.invert_conditional;
2661 bool is_discard = ins->branch.target_type == TARGET_DISCARD;
2662
2663 /* Determine the block we're jumping to */
2664 int target_number = ins->branch.target_block;
2665
2666 /* Report the destination tag */
2667 int dest_tag = is_discard ? 0 : midgard_get_first_tag_from_block(ctx, target_number);
2668
2669 /* Count up the number of quadwords we're
2670 * jumping over = number of quadwords until
2671 * (br_block_idx, target_number) */
2672
2673 int quadword_offset = 0;
2674
2675 if (is_discard) {
2676 /* Ignored */
2677 } else if (target_number > br_block_idx) {
2678 /* Jump forward */
2679
2680 for (int idx = br_block_idx + 1; idx < target_number; ++idx) {
2681 midgard_block *blk = mir_get_block(ctx, idx);
2682 assert(blk);
2683
2684 quadword_offset += blk->quadword_count;
2685 }
2686 } else {
2687 /* Jump backwards */
2688
2689 for (int idx = br_block_idx; idx >= target_number; --idx) {
2690 midgard_block *blk = mir_get_block(ctx, idx);
2691 assert(blk);
2692
2693 quadword_offset -= blk->quadword_count;
2694 }
2695 }
2696
2697 /* Unconditional extended branches (far jumps)
2698 * have issues, so we always use a conditional
2699 * branch, setting the condition to always for
2700 * unconditional. For compact unconditional
2701 * branches, cond isn't used so it doesn't
2702 * matter what we pick. */
2703
2704 midgard_condition cond =
2705 !is_conditional ? midgard_condition_always :
2706 is_inverted ? midgard_condition_false :
2707 midgard_condition_true;
2708
2709 midgard_jmp_writeout_op op =
2710 is_discard ? midgard_jmp_writeout_op_discard :
2711 (is_compact && !is_conditional) ? midgard_jmp_writeout_op_branch_uncond :
2712 midgard_jmp_writeout_op_branch_cond;
2713
2714 if (!is_compact) {
2715 midgard_branch_extended branch =
2716 midgard_create_branch_extended(
2717 cond, op,
2718 dest_tag,
2719 quadword_offset);
2720
2721 memcpy(&ins->branch_extended, &branch, sizeof(branch));
2722 } else if (is_conditional || is_discard) {
2723 midgard_branch_cond branch = {
2724 .op = op,
2725 .dest_tag = dest_tag,
2726 .offset = quadword_offset,
2727 .cond = cond
2728 };
2729
2730 assert(branch.offset == quadword_offset);
2731
2732 memcpy(&ins->br_compact, &branch, sizeof(branch));
2733 } else {
2734 assert(op == midgard_jmp_writeout_op_branch_uncond);
2735
2736 midgard_branch_uncond branch = {
2737 .op = op,
2738 .dest_tag = dest_tag,
2739 .offset = quadword_offset,
2740 .unknown = 1
2741 };
2742
2743 assert(branch.offset == quadword_offset);
2744
2745 memcpy(&ins->br_compact, &branch, sizeof(branch));
2746 }
2747 }
2748 }
2749
2750 ++br_block_idx;
2751 }
2752
2753 /* Emit flat binary from the instruction arrays. Iterate each block in
2754 * sequence. Save instruction boundaries such that lookahead tags can
2755 * be assigned easily */
2756
2757 /* Cache _all_ bundles in source order for lookahead across failed branches */
2758
2759 int bundle_count = 0;
2760 mir_foreach_block(ctx, block) {
2761 bundle_count += block->bundles.size / sizeof(midgard_bundle);
2762 }
2763 midgard_bundle **source_order_bundles = malloc(sizeof(midgard_bundle *) * bundle_count);
2764 int bundle_idx = 0;
2765 mir_foreach_block(ctx, block) {
2766 util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) {
2767 source_order_bundles[bundle_idx++] = bundle;
2768 }
2769 }
2770
2771 int current_bundle = 0;
2772
2773 /* Midgard prefetches instruction types, so during emission we
2774 * need to lookahead. Unless this is the last instruction, in
2775 * which we return 1. Or if this is the second to last and the
2776 * last is an ALU, then it's also 1... */
2777
2778 mir_foreach_block(ctx, block) {
2779 mir_foreach_bundle_in_block(block, bundle) {
2780 int lookahead = 1;
2781
2782 if (current_bundle + 1 < bundle_count) {
2783 uint8_t next = source_order_bundles[current_bundle + 1]->tag;
2784
2785 if (!(current_bundle + 2 < bundle_count) && IS_ALU(next)) {
2786 lookahead = 1;
2787 } else {
2788 lookahead = next;
2789 }
2790 }
2791
2792 emit_binary_bundle(ctx, bundle, compiled, lookahead);
2793 ++current_bundle;
2794 }
2795
2796 /* TODO: Free deeper */
2797 //util_dynarray_fini(&block->instructions);
2798 }
2799
2800 free(source_order_bundles);
2801
2802 /* Report the very first tag executed */
2803 program->first_tag = midgard_get_first_tag_from_block(ctx, 0);
2804
2805 /* Deal with off-by-one related to the fencepost problem */
2806 program->work_register_count = ctx->work_registers + 1;
2807 program->uniform_cutoff = ctx->uniform_cutoff;
2808
2809 program->blend_patch_offset = ctx->blend_constant_offset;
2810 program->tls_size = ctx->tls_size;
2811
2812 if (midgard_debug & MIDGARD_DBG_SHADERS)
2813 disassemble_midgard(program->compiled.data, program->compiled.size, false, 0, "");
2814
2815 if (midgard_debug & MIDGARD_DBG_SHADERDB) {
2816 unsigned nr_bundles = 0, nr_ins = 0, nr_quadwords = 0;
2817
2818 /* Count instructions and bundles */
2819
2820 mir_foreach_instr_global(ctx, ins) {
2821 nr_ins++;
2822 }
2823
2824 mir_foreach_block(ctx, block) {
2825 nr_bundles += util_dynarray_num_elements(
2826 &block->bundles, midgard_bundle);
2827
2828 nr_quadwords += block->quadword_count;
2829 }
2830
2831 /* Calculate thread count. There are certain cutoffs by
2832 * register count for thread count */
2833
2834 unsigned nr_registers = program->work_register_count;
2835
2836 unsigned nr_threads =
2837 (nr_registers <= 4) ? 4 :
2838 (nr_registers <= 8) ? 2 :
2839 1;
2840
2841 /* Dump stats */
2842
2843 fprintf(stderr, "shader%d - %s shader: "
2844 "%u inst, %u bundles, %u quadwords, "
2845 "%u registers, %u threads, %u loops, "
2846 "%d:%d spills:fills\n",
2847 SHADER_DB_COUNT++,
2848 gl_shader_stage_name(ctx->stage),
2849 nr_ins, nr_bundles, nr_quadwords,
2850 nr_registers, nr_threads,
2851 ctx->loop_count,
2852 ctx->spills, ctx->fills);
2853 }
2854
2855 ralloc_free(ctx);
2856
2857 return 0;
2858 }