projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
pan/bi: Implement flog2
[mesa.git] / src / panfrost / bifrost / bifrost_compile.c
1 /*
2 * Copyright (C) 2020 Collabora Ltd.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 *
23 * Authors (Collabora):
24 * Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
25 */
26
27 #include "main/mtypes.h"
28 #include "compiler/glsl/glsl_to_nir.h"
29 #include "compiler/nir_types.h"
30 #include "util/imports.h"
31 #include "compiler/nir/nir_builder.h"
32
33 #include "disassemble.h"
34 #include "bifrost_compile.h"
35 #include "bifrost_nir.h"
36 #include "compiler.h"
37 #include "bi_quirks.h"
38 #include "bi_print.h"
39
40 static bi_block *emit_cf_list(bi_context *ctx, struct exec_list *list);
41 static bi_instruction *bi_emit_branch(bi_context *ctx);
42 static void bi_schedule_barrier(bi_context *ctx);
43
44 static void
45 emit_jump(bi_context *ctx, nir_jump_instr *instr)
46 {
47 bi_instruction *branch = bi_emit_branch(ctx);
48
49 switch (instr->type) {
50 case nir_jump_break:
51 branch->branch.target = ctx->break_block;
52 break;
53 case nir_jump_continue:
54 branch->branch.target = ctx->continue_block;
55 break;
56 default:
57 unreachable("Unhandled jump type");
58 }
59
60 pan_block_add_successor(&ctx->current_block->base, &branch->branch.target->base);
61 }
62
63 /* Gets a bytemask for a complete vecN write */
64 static unsigned
65 bi_mask_for_channels_32(unsigned i)
66 {
67 return (1 << (4 * i)) - 1;
68 }
69
70 static bi_instruction
71 bi_load(enum bi_class T, nir_intrinsic_instr *instr)
72 {
73 bi_instruction load = {
74 .type = T,
75 .writemask = bi_mask_for_channels_32(instr->num_components),
76 .src = { BIR_INDEX_CONSTANT },
77 .src_types = { nir_type_uint32 },
78 .constant = { .u64 = nir_intrinsic_base(instr) },
79 };
80
81 const nir_intrinsic_info *info = &nir_intrinsic_infos[instr->intrinsic];
82
83 if (info->has_dest)
84 load.dest = bir_dest_index(&instr->dest);
85
86 if (info->has_dest && info->index_map[NIR_INTRINSIC_TYPE] > 0)
87 load.dest_type = nir_intrinsic_type(instr);
88
89 nir_src *offset = nir_get_io_offset_src(instr);
90
91 if (nir_src_is_const(*offset))
92 load.constant.u64 += nir_src_as_uint(*offset);
93 else
94 load.src[0] = bir_src_index(offset);
95
96 return load;
97 }
98
99 static void
100 bi_emit_ld_vary(bi_context *ctx, nir_intrinsic_instr *instr)
101 {
102 bi_instruction ins = bi_load(BI_LOAD_VAR, instr);
103 ins.load_vary.interp_mode = BIFROST_INTERP_DEFAULT; /* TODO */
104 ins.load_vary.reuse = false; /* TODO */
105 ins.load_vary.flat = instr->intrinsic != nir_intrinsic_load_interpolated_input;
106 ins.dest_type = nir_type_float | nir_dest_bit_size(instr->dest);
107
108 if (nir_src_is_const(*nir_get_io_offset_src(instr))) {
109 /* Zero it out for direct */
110 ins.src[1] = BIR_INDEX_ZERO;
111 } else {
112 /* R61 contains sample mask stuff, TODO RA XXX */
113 ins.src[1] = BIR_INDEX_REGISTER | 61;
114 }
115
116 bi_emit(ctx, ins);
117 }
118
119 static void
120 bi_emit_frag_out(bi_context *ctx, nir_intrinsic_instr *instr)
121 {
122 if (!ctx->emitted_atest) {
123 bi_instruction ins = {
124 .type = BI_ATEST,
125 .src = {
126 BIR_INDEX_REGISTER | 60 /* TODO: RA */,
127 bir_src_index(&instr->src[0])
128 },
129 .src_types = {
130 nir_type_uint32,
131 nir_type_float32
132 },
133 .swizzle = {
134 { 0 },
135 { 3, 0 } /* swizzle out the alpha */
136 },
137 .dest = BIR_INDEX_REGISTER | 60 /* TODO: RA */,
138 .dest_type = nir_type_uint32,
139 .writemask = 0xF
140 };
141
142 bi_emit(ctx, ins);
143 bi_schedule_barrier(ctx);
144 ctx->emitted_atest = true;
145 }
146
147 bi_instruction blend = {
148 .type = BI_BLEND,
149 .blend_location = nir_intrinsic_base(instr),
150 .src = {
151 bir_src_index(&instr->src[0]),
152 BIR_INDEX_REGISTER | 60 /* Can this be arbitrary? */,
153 },
154 .src_types = {
155 nir_type_float32,
156 nir_type_uint32
157 },
158 .swizzle = {
159 { 0, 1, 2, 3 },
160 { 0 }
161 },
162 .dest = BIR_INDEX_REGISTER | 48 /* Looks like magic */,
163 .dest_type = nir_type_uint32,
164 .writemask = 0xF
165 };
166
167 bi_emit(ctx, blend);
168 bi_schedule_barrier(ctx);
169 }
170
171 static bi_instruction
172 bi_load_with_r61(enum bi_class T, nir_intrinsic_instr *instr)
173 {
174 bi_instruction ld = bi_load(T, instr);
175 ld.src[1] = BIR_INDEX_REGISTER | 61; /* TODO: RA */
176 ld.src[2] = BIR_INDEX_REGISTER | 62;
177 ld.src[3] = 0;
178 ld.src_types[1] = nir_type_uint32;
179 ld.src_types[2] = nir_type_uint32;
180 ld.src_types[3] = nir_intrinsic_type(instr);
181 return ld;
182 }
183
184 static void
185 bi_emit_st_vary(bi_context *ctx, nir_intrinsic_instr *instr)
186 {
187 bi_instruction address = bi_load_with_r61(BI_LOAD_VAR_ADDRESS, instr);
188 address.dest = bi_make_temp(ctx);
189 address.dest_type = nir_type_uint32;
190 address.writemask = (1 << 12) - 1;
191
192 bi_instruction st = {
193 .type = BI_STORE_VAR,
194 .src = {
195 bir_src_index(&instr->src[0]),
196 address.dest, address.dest, address.dest,
197 },
198 .src_types = {
199 nir_type_uint32,
200 nir_type_uint32, nir_type_uint32, nir_type_uint32,
201 },
202 .swizzle = {
203 { 0, 1, 2, 3 },
204 { 0 }, { 1 }, { 2}
205 },
206 .store_channels = 4, /* TODO: WRITEMASK */
207 };
208
209 bi_emit(ctx, address);
210 bi_emit(ctx, st);
211 }
212
213 static void
214 bi_emit_ld_uniform(bi_context *ctx, nir_intrinsic_instr *instr)
215 {
216 bi_instruction ld = bi_load(BI_LOAD_UNIFORM, instr);
217 ld.src[1] = BIR_INDEX_ZERO; /* TODO: UBO index */
218
219 /* TODO: Indirect access, since we need to multiply by the element
220 * size. I believe we can get this lowering automatically via
221 * nir_lower_io (as mul instructions) with the proper options, but this
222 * is TODO */
223 assert(ld.src[0] & BIR_INDEX_CONSTANT);
224 ld.constant.u64 += ctx->sysvals.sysval_count;
225 ld.constant.u64 *= 16;
226
227 bi_emit(ctx, ld);
228 }
229
230 static void
231 bi_emit_sysval(bi_context *ctx, nir_instr *instr,
232 unsigned nr_components, unsigned offset)
233 {
234 nir_dest nir_dest;
235
236 /* Figure out which uniform this is */
237 int sysval = panfrost_sysval_for_instr(instr, &nir_dest);
238 void *val = _mesa_hash_table_u64_search(ctx->sysvals.sysval_to_id, sysval);
239
240 /* Sysvals are prefix uniforms */
241 unsigned uniform = ((uintptr_t) val) - 1;
242
243 /* Emit the read itself -- this is never indirect */
244
245 bi_instruction load = {
246 .type = BI_LOAD_UNIFORM,
247 .writemask = (1 << (nr_components * 4)) - 1,
248 .src = { BIR_INDEX_CONSTANT, BIR_INDEX_ZERO },
249 .src_types = { nir_type_uint32, nir_type_uint32 },
250 .constant = { (uniform * 16) + offset },
251 .dest = bir_dest_index(&nir_dest),
252 .dest_type = nir_type_uint32, /* TODO */
253 };
254
255 bi_emit(ctx, load);
256 }
257
258 static void
259 emit_intrinsic(bi_context *ctx, nir_intrinsic_instr *instr)
260 {
261
262 switch (instr->intrinsic) {
263 case nir_intrinsic_load_barycentric_pixel:
264 /* stub */
265 break;
266 case nir_intrinsic_load_interpolated_input:
267 case nir_intrinsic_load_input:
268 if (ctx->stage == MESA_SHADER_FRAGMENT)
269 bi_emit_ld_vary(ctx, instr);
270 else if (ctx->stage == MESA_SHADER_VERTEX)
271 bi_emit(ctx, bi_load_with_r61(BI_LOAD_ATTR, instr));
272 else {
273 unreachable("Unsupported shader stage");
274 }
275 break;
276
277 case nir_intrinsic_store_output:
278 if (ctx->stage == MESA_SHADER_FRAGMENT)
279 bi_emit_frag_out(ctx, instr);
280 else if (ctx->stage == MESA_SHADER_VERTEX)
281 bi_emit_st_vary(ctx, instr);
282 else
283 unreachable("Unsupported shader stage");
284 break;
285
286 case nir_intrinsic_load_uniform:
287 bi_emit_ld_uniform(ctx, instr);
288 break;
289
290 case nir_intrinsic_load_ssbo_address:
291 bi_emit_sysval(ctx, &instr->instr, 1, 0);
292 break;
293
294 case nir_intrinsic_get_buffer_size:
295 bi_emit_sysval(ctx, &instr->instr, 1, 8);
296 break;
297
298 case nir_intrinsic_load_viewport_scale:
299 case nir_intrinsic_load_viewport_offset:
300 case nir_intrinsic_load_num_work_groups:
301 case nir_intrinsic_load_sampler_lod_parameters_pan:
302 bi_emit_sysval(ctx, &instr->instr, 3, 0);
303 break;
304
305 default:
306 /* todo */
307 break;
308 }
309 }
310
311 static void
312 emit_load_const(bi_context *ctx, nir_load_const_instr *instr)
313 {
314 /* Make sure we've been lowered */
315 assert(instr->def.num_components == 1);
316
317 bi_instruction move = {
318 .type = BI_MOV,
319 .dest = bir_ssa_index(&instr->def),
320 .dest_type = instr->def.bit_size | nir_type_uint,
321 .writemask = (1 << (instr->def.bit_size / 8)) - 1,
322 .src = {
323 BIR_INDEX_CONSTANT
324 },
325 .src_types = {
326 instr->def.bit_size | nir_type_uint,
327 },
328 .constant = {
329 .u64 = nir_const_value_as_uint(instr->value[0], instr->def.bit_size)
330 }
331 };
332
333 bi_emit(ctx, move);
334 }
335
336 #define BI_CASE_CMP(op) \
337 case op##8: \
338 case op##16: \
339 case op##32: \
340
341 static enum bi_class
342 bi_class_for_nir_alu(nir_op op)
343 {
344 switch (op) {
345 case nir_op_iadd:
346 case nir_op_fadd:
347 case nir_op_fsub:
348 return BI_ADD;
349 case nir_op_isub:
350 return BI_ISUB;
351
352 BI_CASE_CMP(nir_op_flt)
353 BI_CASE_CMP(nir_op_fge)
354 BI_CASE_CMP(nir_op_feq)
355 BI_CASE_CMP(nir_op_fne)
356 BI_CASE_CMP(nir_op_ilt)
357 BI_CASE_CMP(nir_op_ige)
358 BI_CASE_CMP(nir_op_ieq)
359 BI_CASE_CMP(nir_op_ine)
360 return BI_CMP;
361
362 case nir_op_b8csel:
363 case nir_op_b16csel:
364 case nir_op_b32csel:
365 return BI_CSEL;
366
367 case nir_op_i2i8:
368 case nir_op_i2i16:
369 case nir_op_i2i32:
370 case nir_op_i2i64:
371 case nir_op_u2u8:
372 case nir_op_u2u16:
373 case nir_op_u2u32:
374 case nir_op_u2u64:
375 case nir_op_f2i16:
376 case nir_op_f2i32:
377 case nir_op_f2i64:
378 case nir_op_f2u16:
379 case nir_op_f2u32:
380 case nir_op_f2u64:
381 case nir_op_i2f16:
382 case nir_op_i2f32:
383 case nir_op_i2f64:
384 case nir_op_u2f16:
385 case nir_op_u2f32:
386 case nir_op_u2f64:
387 case nir_op_f2f16:
388 case nir_op_f2f32:
389 case nir_op_f2f64:
390 case nir_op_f2fmp:
391 return BI_CONVERT;
392
393 case nir_op_vec2:
394 case nir_op_vec3:
395 case nir_op_vec4:
396 return BI_COMBINE;
397
398 case nir_op_vec8:
399 case nir_op_vec16:
400 unreachable("should've been lowered");
401
402 case nir_op_ffma:
403 case nir_op_fmul:
404 return BI_FMA;
405
406 case nir_op_imin:
407 case nir_op_imax:
408 case nir_op_umin:
409 case nir_op_umax:
410 case nir_op_fmin:
411 case nir_op_fmax:
412 return BI_MINMAX;
413
414 case nir_op_fsat:
415 case nir_op_fneg:
416 case nir_op_fabs:
417 return BI_FMOV;
418 case nir_op_mov:
419 return BI_MOV;
420
421 case nir_op_fround_even:
422 case nir_op_fceil:
423 case nir_op_ffloor:
424 case nir_op_ftrunc:
425 return BI_ROUND;
426
427 case nir_op_frcp:
428 case nir_op_frsq:
429 return BI_SPECIAL;
430
431 default:
432 unreachable("Unknown ALU op");
433 }
434 }
435
436 /* Gets a bi_cond for a given NIR comparison opcode. In soft mode, it will
437 * return BI_COND_ALWAYS as a sentinel if it fails to do so (when used for
438 * optimizations). Otherwise it will bail (when used for primary code
439 * generation). */
440
441 static enum bi_cond
442 bi_cond_for_nir(nir_op op, bool soft)
443 {
444 switch (op) {
445 BI_CASE_CMP(nir_op_flt)
446 BI_CASE_CMP(nir_op_ilt)
447 return BI_COND_LT;
448
449 BI_CASE_CMP(nir_op_fge)
450 BI_CASE_CMP(nir_op_ige)
451 return BI_COND_GE;
452
453 BI_CASE_CMP(nir_op_feq)
454 BI_CASE_CMP(nir_op_ieq)
455 return BI_COND_EQ;
456
457 BI_CASE_CMP(nir_op_fne)
458 BI_CASE_CMP(nir_op_ine)
459 return BI_COND_NE;
460 default:
461 if (soft)
462 return BI_COND_ALWAYS;
463 else
464 unreachable("Invalid compare");
465 }
466 }
467
468 static void
469 bi_copy_src(bi_instruction *alu, nir_alu_instr *instr, unsigned i, unsigned to,
470 unsigned *constants_left, unsigned *constant_shift)
471 {
472 unsigned bits = nir_src_bit_size(instr->src[i].src);
473 unsigned dest_bits = nir_dest_bit_size(instr->dest.dest);
474
475 alu->src_types[to] = nir_op_infos[instr->op].input_types[i]
476 | bits;
477
478 /* Try to inline a constant */
479 if (nir_src_is_const(instr->src[i].src) && *constants_left && (dest_bits == bits)) {
480 alu->constant.u64 |=
481 (nir_src_as_uint(instr->src[i].src)) << *constant_shift;
482
483 alu->src[to] = BIR_INDEX_CONSTANT | (*constant_shift);
484 --(*constants_left);
485 (*constant_shift) += dest_bits;
486 return;
487 }
488
489 alu->src[to] = bir_src_index(&instr->src[i].src);
490
491 /* We assert scalarization above */
492 alu->swizzle[to][0] = instr->src[i].swizzle[0];
493 }
494
495 static void
496 bi_fuse_csel_cond(bi_instruction *csel, nir_alu_src cond,
497 unsigned *constants_left, unsigned *constant_shift)
498 {
499 /* Bail for vector weirdness */
500 if (cond.swizzle[0] != 0)
501 return;
502
503 if (!cond.src.is_ssa)
504 return;
505
506 nir_ssa_def *def = cond.src.ssa;
507 nir_instr *parent = def->parent_instr;
508
509 if (parent->type != nir_instr_type_alu)
510 return;
511
512 nir_alu_instr *alu = nir_instr_as_alu(parent);
513
514 /* Try to match a condition */
515 enum bi_cond bcond = bi_cond_for_nir(alu->op, true);
516
517 if (bcond == BI_COND_ALWAYS)
518 return;
519
520 /* We found one, let's fuse it in */
521 csel->csel_cond = bcond;
522 bi_copy_src(csel, alu, 0, 0, constants_left, constant_shift);
523 bi_copy_src(csel, alu, 1, 1, constants_left, constant_shift);
524 }
525
526 static void
527 emit_alu(bi_context *ctx, nir_alu_instr *instr)
528 {
529 /* Try some special functions */
530 switch (instr->op) {
531 case nir_op_fexp2:
532 bi_emit_fexp2(ctx, instr);
533 return;
534 case nir_op_flog2:
535 bi_emit_flog2(ctx, instr);
536 return;
537 default:
538 break;
539 }
540
541 /* Otherwise, assume it's something we can handle normally */
542 bi_instruction alu = {
543 .type = bi_class_for_nir_alu(instr->op),
544 .dest = bir_dest_index(&instr->dest.dest),
545 .dest_type = nir_op_infos[instr->op].output_type
546 | nir_dest_bit_size(instr->dest.dest),
547 };
548
549 /* TODO: Implement lowering of special functions for older Bifrost */
550 assert((alu.type != BI_SPECIAL) || !(ctx->quirks & BIFROST_NO_FAST_OP));
551
552 if (instr->dest.dest.is_ssa) {
553 /* Construct a writemask */
554 unsigned bits_per_comp = instr->dest.dest.ssa.bit_size;
555 unsigned comps = instr->dest.dest.ssa.num_components;
556
557 if (alu.type != BI_COMBINE)
558 assert(comps == 1);
559
560 unsigned bits = bits_per_comp * comps;
561 unsigned bytes = bits / 8;
562 alu.writemask = (1 << bytes) - 1;
563 } else {
564 unsigned comp_mask = instr->dest.write_mask;
565
566 alu.writemask = pan_to_bytemask(nir_dest_bit_size(instr->dest.dest),
567 comp_mask);
568 }
569
570 /* We inline constants as we go. This tracks how many constants have
571 * been inlined, since we're limited to 64-bits of constants per
572 * instruction */
573
574 unsigned dest_bits = nir_dest_bit_size(instr->dest.dest);
575 unsigned constants_left = (64 / dest_bits);
576 unsigned constant_shift = 0;
577
578 if (alu.type == BI_COMBINE)
579 constants_left = 0;
580
581 /* Copy sources */
582
583 unsigned num_inputs = nir_op_infos[instr->op].num_inputs;
584 assert(num_inputs <= ARRAY_SIZE(alu.src));
585
586 for (unsigned i = 0; i < num_inputs; ++i) {
587 unsigned f = 0;
588
589 if (i && alu.type == BI_CSEL)
590 f++;
591
592 bi_copy_src(&alu, instr, i, i + f, &constants_left, &constant_shift);
593 }
594
595 /* Op-specific fixup */
596 switch (instr->op) {
597 case nir_op_fmul:
598 alu.src[2] = BIR_INDEX_ZERO; /* FMA */
599 alu.src_types[2] = alu.src_types[1];
600 break;
601 case nir_op_fsat:
602 alu.outmod = BIFROST_SAT; /* FMOV */
603 break;
604 case nir_op_fneg:
605 alu.src_neg[0] = true; /* FMOV */
606 break;
607 case nir_op_fabs:
608 alu.src_abs[0] = true; /* FMOV */
609 break;
610 case nir_op_fsub:
611 alu.src_neg[1] = true; /* FADD */
612 break;
613 case nir_op_fmax:
614 case nir_op_imax:
615 case nir_op_umax:
616 alu.op.minmax = BI_MINMAX_MAX; /* MINMAX */
617 break;
618 case nir_op_frcp:
619 alu.op.special = BI_SPECIAL_FRCP;
620 break;
621 case nir_op_frsq:
622 alu.op.special = BI_SPECIAL_FRSQ;
623 break;
624 BI_CASE_CMP(nir_op_flt)
625 BI_CASE_CMP(nir_op_ilt)
626 BI_CASE_CMP(nir_op_fge)
627 BI_CASE_CMP(nir_op_ige)
628 BI_CASE_CMP(nir_op_feq)
629 BI_CASE_CMP(nir_op_ieq)
630 BI_CASE_CMP(nir_op_fne)
631 BI_CASE_CMP(nir_op_ine)
632 alu.op.compare = bi_cond_for_nir(instr->op, false);
633 break;
634 case nir_op_fround_even:
635 alu.op.round = BI_ROUND_MODE;
636 alu.roundmode = BIFROST_RTE;
637 break;
638 case nir_op_fceil:
639 alu.op.round = BI_ROUND_MODE;
640 alu.roundmode = BIFROST_RTP;
641 break;
642 case nir_op_ffloor:
643 alu.op.round = BI_ROUND_MODE;
644 alu.roundmode = BIFROST_RTN;
645 break;
646 case nir_op_ftrunc:
647 alu.op.round = BI_ROUND_MODE;
648 alu.roundmode = BIFROST_RTZ;
649 break;
650 default:
651 break;
652 }
653
654 if (alu.type == BI_CSEL) {
655 /* Default to csel3 */
656 alu.csel_cond = BI_COND_NE;
657 alu.src[1] = BIR_INDEX_ZERO;
658 alu.src_types[1] = alu.src_types[0];
659
660 bi_fuse_csel_cond(&alu, instr->src[0],
661 &constants_left, &constant_shift);
662 }
663
664 bi_emit(ctx, alu);
665 }
666
667 static void
668 emit_instr(bi_context *ctx, struct nir_instr *instr)
669 {
670 switch (instr->type) {
671 case nir_instr_type_load_const:
672 emit_load_const(ctx, nir_instr_as_load_const(instr));
673 break;
674
675 case nir_instr_type_intrinsic:
676 emit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
677 break;
678
679 case nir_instr_type_alu:
680 emit_alu(ctx, nir_instr_as_alu(instr));
681 break;
682
683 #if 0
684 case nir_instr_type_tex:
685 emit_tex(ctx, nir_instr_as_tex(instr));
686 break;
687 #endif
688
689 case nir_instr_type_jump:
690 emit_jump(ctx, nir_instr_as_jump(instr));
691 break;
692
693 case nir_instr_type_ssa_undef:
694 /* Spurious */
695 break;
696
697 default:
698 //unreachable("Unhandled instruction type");
699 break;
700 }
701 }
702
703
704
705 static bi_block *
706 create_empty_block(bi_context *ctx)
707 {
708 bi_block *blk = rzalloc(ctx, bi_block);
709
710 blk->base.predecessors = _mesa_set_create(blk,
711 _mesa_hash_pointer,
712 _mesa_key_pointer_equal);
713
714 blk->base.name = ctx->block_name_count++;
715
716 return blk;
717 }
718
719 static void
720 bi_schedule_barrier(bi_context *ctx)
721 {
722 bi_block *temp = ctx->after_block;
723 ctx->after_block = create_empty_block(ctx);
724 list_addtail(&ctx->after_block->base.link, &ctx->blocks);
725 list_inithead(&ctx->after_block->base.instructions);
726 pan_block_add_successor(&ctx->current_block->base, &ctx->after_block->base);
727 ctx->current_block = ctx->after_block;
728 ctx->after_block = temp;
729 }
730
731 static bi_block *
732 emit_block(bi_context *ctx, nir_block *block)
733 {
734 if (ctx->after_block) {
735 ctx->current_block = ctx->after_block;
736 ctx->after_block = NULL;
737 } else {
738 ctx->current_block = create_empty_block(ctx);
739 }
740
741 list_addtail(&ctx->current_block->base.link, &ctx->blocks);
742 list_inithead(&ctx->current_block->base.instructions);
743
744 nir_foreach_instr(instr, block) {
745 emit_instr(ctx, instr);
746 ++ctx->instruction_count;
747 }
748
749 return ctx->current_block;
750 }
751
752 /* Emits an unconditional branch to the end of the current block, returning a
753 * pointer so the user can fill in details */
754
755 static bi_instruction *
756 bi_emit_branch(bi_context *ctx)
757 {
758 bi_instruction branch = {
759 .type = BI_BRANCH,
760 .branch = {
761 .cond = BI_COND_ALWAYS
762 }
763 };
764
765 return bi_emit(ctx, branch);
766 }
767
768 /* Sets a condition for a branch by examing the NIR condition. If we're
769 * familiar with the condition, we unwrap it to fold it into the branch
770 * instruction. Otherwise, we consume the condition directly. We
771 * generally use 1-bit booleans which allows us to use small types for
772 * the conditions.
773 */
774
775 static void
776 bi_set_branch_cond(bi_instruction *branch, nir_src *cond, bool invert)
777 {
778 /* TODO: Try to unwrap instead of always bailing */
779 branch->src[0] = bir_src_index(cond);
780 branch->src[1] = BIR_INDEX_ZERO;
781 branch->src_types[0] = branch->src_types[1] = nir_type_uint16;
782 branch->branch.cond = invert ? BI_COND_EQ : BI_COND_NE;
783 }
784
785 static void
786 emit_if(bi_context *ctx, nir_if *nif)
787 {
788 bi_block *before_block = ctx->current_block;
789
790 /* Speculatively emit the branch, but we can't fill it in until later */
791 bi_instruction *then_branch = bi_emit_branch(ctx);
792 bi_set_branch_cond(then_branch, &nif->condition, true);
793
794 /* Emit the two subblocks. */
795 bi_block *then_block = emit_cf_list(ctx, &nif->then_list);
796 bi_block *end_then_block = ctx->current_block;
797
798 /* Emit a jump from the end of the then block to the end of the else */
799 bi_instruction *then_exit = bi_emit_branch(ctx);
800
801 /* Emit second block, and check if it's empty */
802
803 int count_in = ctx->instruction_count;
804 bi_block *else_block = emit_cf_list(ctx, &nif->else_list);
805 bi_block *end_else_block = ctx->current_block;
806 ctx->after_block = create_empty_block(ctx);
807
808 /* Now that we have the subblocks emitted, fix up the branches */
809
810 assert(then_block);
811 assert(else_block);
812
813 if (ctx->instruction_count == count_in) {
814 /* The else block is empty, so don't emit an exit jump */
815 bi_remove_instruction(then_exit);
816 then_branch->branch.target = ctx->after_block;
817 } else {
818 then_branch->branch.target = else_block;
819 then_exit->branch.target = ctx->after_block;
820 pan_block_add_successor(&end_then_block->base, &then_exit->branch.target->base);
821 }
822
823 /* Wire up the successors */
824
825 pan_block_add_successor(&before_block->base, &then_branch->branch.target->base); /* then_branch */
826
827 pan_block_add_successor(&before_block->base, &then_block->base); /* fallthrough */
828 pan_block_add_successor(&end_else_block->base, &ctx->after_block->base); /* fallthrough */
829 }
830
831 static void
832 emit_loop(bi_context *ctx, nir_loop *nloop)
833 {
834 /* Remember where we are */
835 bi_block *start_block = ctx->current_block;
836
837 bi_block *saved_break = ctx->break_block;
838 bi_block *saved_continue = ctx->continue_block;
839
840 ctx->continue_block = create_empty_block(ctx);
841 ctx->break_block = create_empty_block(ctx);
842 ctx->after_block = ctx->continue_block;
843
844 /* Emit the body itself */
845 emit_cf_list(ctx, &nloop->body);
846
847 /* Branch back to loop back */
848 bi_instruction *br_back = bi_emit_branch(ctx);
849 br_back->branch.target = ctx->continue_block;
850 pan_block_add_successor(&start_block->base, &ctx->continue_block->base);
851 pan_block_add_successor(&ctx->current_block->base, &ctx->continue_block->base);
852
853 ctx->after_block = ctx->break_block;
854
855 /* Pop off */
856 ctx->break_block = saved_break;
857 ctx->continue_block = saved_continue;
858 ++ctx->loop_count;
859 }
860
861 static bi_block *
862 emit_cf_list(bi_context *ctx, struct exec_list *list)
863 {
864 bi_block *start_block = NULL;
865
866 foreach_list_typed(nir_cf_node, node, node, list) {
867 switch (node->type) {
868 case nir_cf_node_block: {
869 bi_block *block = emit_block(ctx, nir_cf_node_as_block(node));
870
871 if (!start_block)
872 start_block = block;
873
874 break;
875 }
876
877 case nir_cf_node_if:
878 emit_if(ctx, nir_cf_node_as_if(node));
879 break;
880
881 case nir_cf_node_loop:
882 emit_loop(ctx, nir_cf_node_as_loop(node));
883 break;
884
885 default:
886 unreachable("Unknown control flow");
887 }
888 }
889
890 return start_block;
891 }
892
893 static int
894 glsl_type_size(const struct glsl_type *type, bool bindless)
895 {
896 return glsl_count_attribute_slots(type, false);
897 }
898
899 static void
900 bi_optimize_nir(nir_shader *nir)
901 {
902 bool progress;
903 unsigned lower_flrp = 16 | 32 | 64;
904
905 NIR_PASS(progress, nir, nir_lower_regs_to_ssa);
906 NIR_PASS(progress, nir, nir_lower_idiv, nir_lower_idiv_fast);
907
908 nir_lower_tex_options lower_tex_options = {
909 .lower_txs_lod = true,
910 .lower_txp = ~0,
911 .lower_tex_without_implicit_lod = true,
912 .lower_txd = true,
913 };
914
915 NIR_PASS(progress, nir, nir_lower_tex, &lower_tex_options);
916 NIR_PASS(progress, nir, nir_lower_alu_to_scalar, NULL, NULL);
917 NIR_PASS(progress, nir, nir_lower_load_const_to_scalar);
918
919 do {
920 progress = false;
921
922 NIR_PASS(progress, nir, nir_lower_var_copies);
923 NIR_PASS(progress, nir, nir_lower_vars_to_ssa);
924
925 NIR_PASS(progress, nir, nir_copy_prop);
926 NIR_PASS(progress, nir, nir_opt_remove_phis);
927 NIR_PASS(progress, nir, nir_opt_dce);
928 NIR_PASS(progress, nir, nir_opt_dead_cf);
929 NIR_PASS(progress, nir, nir_opt_cse);
930 NIR_PASS(progress, nir, nir_opt_peephole_select, 64, false, true);
931 NIR_PASS(progress, nir, nir_opt_algebraic);
932 NIR_PASS(progress, nir, nir_opt_constant_folding);
933
934 if (lower_flrp != 0) {
935 bool lower_flrp_progress = false;
936 NIR_PASS(lower_flrp_progress,
937 nir,
938 nir_lower_flrp,
939 lower_flrp,
940 false /* always_precise */,
941 nir->options->lower_ffma);
942 if (lower_flrp_progress) {
943 NIR_PASS(progress, nir,
944 nir_opt_constant_folding);
945 progress = true;
946 }
947
948 /* Nothing should rematerialize any flrps, so we only
949 * need to do this lowering once.
950 */
951 lower_flrp = 0;
952 }
953
954 NIR_PASS(progress, nir, nir_opt_undef);
955 NIR_PASS(progress, nir, nir_opt_loop_unroll,
956 nir_var_shader_in |
957 nir_var_shader_out |
958 nir_var_function_temp);
959 } while (progress);
960
961 NIR_PASS(progress, nir, nir_opt_algebraic_late);
962 NIR_PASS(progress, nir, nir_lower_bool_to_int32);
963 NIR_PASS(progress, nir, bifrost_nir_lower_algebraic_late);
964 NIR_PASS(progress, nir, nir_lower_alu_to_scalar, NULL, NULL);
965 NIR_PASS(progress, nir, nir_lower_load_const_to_scalar);
966
967 /* Take us out of SSA */
968 NIR_PASS(progress, nir, nir_lower_locals_to_regs);
969 NIR_PASS(progress, nir, nir_move_vec_src_uses_to_dest);
970 NIR_PASS(progress, nir, nir_convert_from_ssa, true);
971 }
972
973 void
974 bifrost_compile_shader_nir(nir_shader *nir, panfrost_program *program, unsigned product_id)
975 {
976 bi_context *ctx = rzalloc(NULL, bi_context);
977 ctx->nir = nir;
978 ctx->stage = nir->info.stage;
979 ctx->quirks = bifrost_get_quirks(product_id);
980 list_inithead(&ctx->blocks);
981
982 /* Lower gl_Position pre-optimisation, but after lowering vars to ssa
983 * (so we don't accidentally duplicate the epilogue since mesa/st has
984 * messed with our I/O quite a bit already) */
985
986 NIR_PASS_V(nir, nir_lower_vars_to_ssa);
987
988 if (ctx->stage == MESA_SHADER_VERTEX) {
989 NIR_PASS_V(nir, nir_lower_viewport_transform);
990 NIR_PASS_V(nir, nir_lower_point_size, 1.0, 1024.0);
991 }
992
993 NIR_PASS_V(nir, nir_split_var_copies);
994 NIR_PASS_V(nir, nir_lower_global_vars_to_local);
995 NIR_PASS_V(nir, nir_lower_var_copies);
996 NIR_PASS_V(nir, nir_lower_vars_to_ssa);
997 NIR_PASS_V(nir, nir_lower_io, nir_var_all, glsl_type_size, 0);
998 NIR_PASS_V(nir, nir_lower_ssbo);
999
1000 bi_optimize_nir(nir);
1001 nir_print_shader(nir, stdout);
1002
1003 panfrost_nir_assign_sysvals(&ctx->sysvals, nir);
1004 program->sysval_count = ctx->sysvals.sysval_count;
1005 memcpy(program->sysvals, ctx->sysvals.sysvals, sizeof(ctx->sysvals.sysvals[0]) * ctx->sysvals.sysval_count);
1006
1007 nir_foreach_function(func, nir) {
1008 if (!func->impl)
1009 continue;
1010
1011 ctx->impl = func->impl;
1012 emit_cf_list(ctx, &func->impl->body);
1013 break; /* TODO: Multi-function shaders */
1014 }
1015
1016 bi_foreach_block(ctx, _block) {
1017 bi_block *block = (bi_block *) _block;
1018 bi_lower_combine(ctx, block);
1019 }
1020
1021 bool progress = false;
1022
1023 do {
1024 progress = false;
1025
1026 bi_foreach_block(ctx, _block) {
1027 bi_block *block = (bi_block *) _block;
1028 progress |= bi_opt_dead_code_eliminate(ctx, block);
1029 }
1030 } while(progress);
1031
1032 bi_print_shader(ctx, stdout);
1033 bi_schedule(ctx);
1034 bi_register_allocate(ctx);
1035 bi_print_shader(ctx, stdout);
1036 bi_pack(ctx, &program->compiled);
1037 disassemble_bifrost(stdout, program->compiled.data, program->compiled.size, true);
1038
1039 ralloc_free(ctx);
1040 }