2 * Copyright (C) 2020 Collabora Ltd.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 * Authors (Collabora):
24 * Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
27 #include "main/mtypes.h"
28 #include "compiler/glsl/glsl_to_nir.h"
29 #include "compiler/nir_types.h"
30 #include "main/imports.h"
31 #include "compiler/nir/nir_builder.h"
33 #include "disassemble.h"
34 #include "bifrost_compile.h"
35 #include "bifrost_nir.h"
37 #include "bi_quirks.h"
40 static bi_block
*emit_cf_list(bi_context
*ctx
, struct exec_list
*list
);
41 static bi_instruction
*bi_emit_branch(bi_context
*ctx
);
42 static void bi_block_add_successor(bi_block
*block
, bi_block
*successor
);
43 static void bi_schedule_barrier(bi_context
*ctx
);
46 emit_jump(bi_context
*ctx
, nir_jump_instr
*instr
)
48 bi_instruction
*branch
= bi_emit_branch(ctx
);
50 switch (instr
->type
) {
52 branch
->branch
.target
= ctx
->break_block
;
54 case nir_jump_continue
:
55 branch
->branch
.target
= ctx
->continue_block
;
58 unreachable("Unhandled jump type");
61 bi_block_add_successor(ctx
->current_block
, branch
->branch
.target
);
64 /* Gets a bytemask for a complete vecN write */
66 bi_mask_for_channels_32(unsigned i
)
68 return (1 << (4 * i
)) - 1;
72 bi_load(enum bi_class T
, nir_intrinsic_instr
*instr
)
74 bi_instruction load
= {
76 .writemask
= bi_mask_for_channels_32(instr
->num_components
),
77 .src
= { BIR_INDEX_CONSTANT
},
78 .constant
= { .u64
= nir_intrinsic_base(instr
) },
81 const nir_intrinsic_info
*info
= &nir_intrinsic_infos
[instr
->intrinsic
];
84 load
.dest
= bir_dest_index(&instr
->dest
);
86 if (info
->has_dest
&& info
->index_map
[NIR_INTRINSIC_TYPE
] > 0)
87 load
.dest_type
= nir_intrinsic_type(instr
);
89 nir_src
*offset
= nir_get_io_offset_src(instr
);
91 if (nir_src_is_const(*offset
))
92 load
.constant
.u64
+= nir_src_as_uint(*offset
);
94 load
.src
[0] = bir_src_index(offset
);
100 bi_emit_ld_vary(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
102 bi_instruction ins
= bi_load(BI_LOAD_VAR
, instr
);
103 ins
.load_vary
.interp_mode
= BIFROST_INTERP_DEFAULT
; /* TODO */
104 ins
.load_vary
.reuse
= false; /* TODO */
105 ins
.load_vary
.flat
= instr
->intrinsic
!= nir_intrinsic_load_interpolated_input
;
106 ins
.dest_type
= nir_type_float
| nir_dest_bit_size(instr
->dest
),
111 bi_emit_frag_out(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
113 if (!ctx
->emitted_atest
) {
114 bi_instruction ins
= {
119 bi_schedule_barrier(ctx
);
120 ctx
->emitted_atest
= true;
123 bi_instruction blend
= {
125 .blend_location
= nir_intrinsic_base(instr
),
127 bir_src_index(&instr
->src
[0])
135 bi_schedule_barrier(ctx
);
139 bi_emit_st_vary(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
141 bi_instruction address
= bi_load(BI_LOAD_VAR_ADDRESS
, instr
);
142 address
.dest
= bi_make_temp(ctx
);
143 address
.dest_type
= nir_type_uint64
;
144 address
.writemask
= (1 << 8) - 1;
146 bi_instruction st
= {
147 .type
= BI_STORE_VAR
,
150 bir_src_index(&instr
->src
[0])
157 bi_emit(ctx
, address
);
162 bi_emit_ld_uniform(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
164 bi_instruction ld
= bi_load(BI_LOAD_UNIFORM
, instr
);
165 ld
.src
[1] = BIR_INDEX_ZERO
; /* TODO: UBO index */
170 emit_intrinsic(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
173 switch (instr
->intrinsic
) {
174 case nir_intrinsic_load_barycentric_pixel
:
177 case nir_intrinsic_load_interpolated_input
:
178 case nir_intrinsic_load_input
:
179 if (ctx
->stage
== MESA_SHADER_FRAGMENT
)
180 bi_emit_ld_vary(ctx
, instr
);
181 else if (ctx
->stage
== MESA_SHADER_VERTEX
)
182 bi_emit(ctx
, bi_load(BI_LOAD_ATTR
, instr
));
184 unreachable("Unsupported shader stage");
188 case nir_intrinsic_store_output
:
189 if (ctx
->stage
== MESA_SHADER_FRAGMENT
)
190 bi_emit_frag_out(ctx
, instr
);
191 else if (ctx
->stage
== MESA_SHADER_VERTEX
)
192 bi_emit_st_vary(ctx
, instr
);
194 unreachable("Unsupported shader stage");
197 case nir_intrinsic_load_uniform
:
198 bi_emit_ld_uniform(ctx
, instr
);
208 emit_load_const(bi_context
*ctx
, nir_load_const_instr
*instr
)
210 /* Make sure we've been lowered */
211 assert(instr
->def
.num_components
== 1);
213 bi_instruction move
= {
215 .dest
= bir_ssa_index(&instr
->def
),
216 .dest_type
= instr
->def
.bit_size
| nir_type_uint
,
217 .writemask
= (1 << (instr
->def
.bit_size
/ 8)) - 1,
222 .u64
= nir_const_value_as_uint(instr
->value
[0], instr
->def
.bit_size
)
230 bi_class_for_nir_alu(nir_op op
)
299 unreachable("Unknown ALU op");
304 bi_cond_for_nir(nir_op op
)
320 unreachable("Invalid compare");
325 emit_alu(bi_context
*ctx
, nir_alu_instr
*instr
)
327 /* Assume it's something we can handle normally */
328 bi_instruction alu
= {
329 .type
= bi_class_for_nir_alu(instr
->op
),
330 .dest
= bir_dest_index(&instr
->dest
.dest
),
331 .dest_type
= nir_op_infos
[instr
->op
].output_type
332 | nir_dest_bit_size(instr
->dest
.dest
),
335 /* TODO: Implement lowering of special functions for older Bifrost */
336 assert((alu
.type
!= BI_SPECIAL
) || !(ctx
->quirks
& BIFROST_NO_FAST_OP
));
338 if (instr
->dest
.dest
.is_ssa
) {
339 /* Construct a writemask */
340 unsigned bits_per_comp
= instr
->dest
.dest
.ssa
.bit_size
;
341 unsigned comps
= instr
->dest
.dest
.ssa
.num_components
;
343 unsigned bits
= bits_per_comp
* comps
;
344 unsigned bytes
= MAX2(bits
/ 8, 1);
345 alu
.writemask
= (1 << bytes
) - 1;
347 unsigned comp_mask
= instr
->dest
.write_mask
;
349 alu
.writemask
= pan_to_bytemask(nir_dest_bit_size(instr
->dest
.dest
),
353 /* We inline constants as we go. This tracks how many constants have
354 * been inlined, since we're limited to 64-bits of constants per
357 unsigned dest_bits
= nir_dest_bit_size(instr
->dest
.dest
);
358 unsigned constants_left
= (64 / dest_bits
);
359 unsigned constant_shift
= 0;
363 unsigned num_inputs
= nir_op_infos
[instr
->op
].num_inputs
;
364 assert(num_inputs
<= ARRAY_SIZE(alu
.src
));
366 for (unsigned i
= 0; i
< num_inputs
; ++i
) {
367 unsigned bits
= nir_src_bit_size(instr
->src
[i
].src
);
368 alu
.src_types
[i
] = nir_op_infos
[instr
->op
].input_types
[i
]
371 /* Try to inline a constant */
372 if (nir_src_is_const(instr
->src
[i
].src
) && constants_left
&& (dest_bits
== bits
)) {
374 (nir_src_as_uint(instr
->src
[i
].src
)) << constant_shift
;
376 alu
.src
[i
] = BIR_INDEX_CONSTANT
| constant_shift
;
378 constant_shift
+= dest_bits
;
382 alu
.src
[i
] = bir_src_index(&instr
->src
[i
].src
);
384 /* We assert scalarization above */
385 alu
.swizzle
[i
][0] = instr
->src
[i
].swizzle
[0];
388 /* Op-specific fixup */
391 alu
.src
[2] = BIR_INDEX_ZERO
; /* FMA */
394 alu
.outmod
= BIFROST_SAT
; /* MOV */
397 alu
.src_neg
[0] = true; /* MOV */
400 alu
.src_abs
[0] = true; /* MOV */
403 alu
.src_neg
[1] = true; /* ADD */
408 alu
.op
.minmax
= BI_MINMAX_MAX
; /* MINMAX */
411 alu
.op
.special
= BI_SPECIAL_FRCP
;
414 alu
.op
.special
= BI_SPECIAL_FRSQ
;
417 alu
.op
.special
= BI_SPECIAL_FSIN
;
420 alu
.op
.special
= BI_SPECIAL_FCOS
;
430 alu
.op
.compare
= bi_cond_for_nir(instr
->op
);
440 emit_instr(bi_context
*ctx
, struct nir_instr
*instr
)
442 switch (instr
->type
) {
443 case nir_instr_type_load_const
:
444 emit_load_const(ctx
, nir_instr_as_load_const(instr
));
447 case nir_instr_type_intrinsic
:
448 emit_intrinsic(ctx
, nir_instr_as_intrinsic(instr
));
451 case nir_instr_type_alu
:
452 emit_alu(ctx
, nir_instr_as_alu(instr
));
456 case nir_instr_type_tex
:
457 emit_tex(ctx
, nir_instr_as_tex(instr
));
461 case nir_instr_type_jump
:
462 emit_jump(ctx
, nir_instr_as_jump(instr
));
465 case nir_instr_type_ssa_undef
:
470 //unreachable("Unhandled instruction type");
478 create_empty_block(bi_context
*ctx
)
480 bi_block
*blk
= rzalloc(ctx
, bi_block
);
482 blk
->predecessors
= _mesa_set_create(blk
,
484 _mesa_key_pointer_equal
);
486 blk
->name
= ctx
->block_name_count
++;
492 bi_block_add_successor(bi_block
*block
, bi_block
*successor
)
497 for (unsigned i
= 0; i
< ARRAY_SIZE(block
->successors
); ++i
) {
498 if (block
->successors
[i
]) {
499 if (block
->successors
[i
] == successor
)
505 block
->successors
[i
] = successor
;
506 _mesa_set_add(successor
->predecessors
, block
);
510 unreachable("Too many successors");
514 bi_schedule_barrier(bi_context
*ctx
)
516 bi_block
*temp
= ctx
->after_block
;
517 ctx
->after_block
= create_empty_block(ctx
);
518 list_addtail(&ctx
->after_block
->link
, &ctx
->blocks
);
519 list_inithead(&ctx
->after_block
->instructions
);
520 bi_block_add_successor(ctx
->current_block
, ctx
->after_block
);
521 ctx
->current_block
= ctx
->after_block
;
522 ctx
->after_block
= temp
;
526 emit_block(bi_context
*ctx
, nir_block
*block
)
528 if (ctx
->after_block
) {
529 ctx
->current_block
= ctx
->after_block
;
530 ctx
->after_block
= NULL
;
532 ctx
->current_block
= create_empty_block(ctx
);
535 list_addtail(&ctx
->current_block
->link
, &ctx
->blocks
);
536 list_inithead(&ctx
->current_block
->instructions
);
538 nir_foreach_instr(instr
, block
) {
539 emit_instr(ctx
, instr
);
540 ++ctx
->instruction_count
;
543 return ctx
->current_block
;
546 /* Emits an unconditional branch to the end of the current block, returning a
547 * pointer so the user can fill in details */
549 static bi_instruction
*
550 bi_emit_branch(bi_context
*ctx
)
552 bi_instruction branch
= {
555 .cond
= BI_COND_ALWAYS
559 return bi_emit(ctx
, branch
);
562 /* Sets a condition for a branch by examing the NIR condition. If we're
563 * familiar with the condition, we unwrap it to fold it into the branch
564 * instruction. Otherwise, we consume the condition directly. We
565 * generally use 1-bit booleans which allows us to use small types for
570 bi_set_branch_cond(bi_instruction
*branch
, nir_src
*cond
, bool invert
)
572 /* TODO: Try to unwrap instead of always bailing */
573 branch
->src
[0] = bir_src_index(cond
);
574 branch
->src
[1] = BIR_INDEX_ZERO
;
575 branch
->src_types
[0] = branch
->src_types
[1] = nir_type_uint16
;
576 branch
->branch
.cond
= invert
? BI_COND_EQ
: BI_COND_NE
;
580 emit_if(bi_context
*ctx
, nir_if
*nif
)
582 bi_block
*before_block
= ctx
->current_block
;
584 /* Speculatively emit the branch, but we can't fill it in until later */
585 bi_instruction
*then_branch
= bi_emit_branch(ctx
);
586 bi_set_branch_cond(then_branch
, &nif
->condition
, true);
588 /* Emit the two subblocks. */
589 bi_block
*then_block
= emit_cf_list(ctx
, &nif
->then_list
);
590 bi_block
*end_then_block
= ctx
->current_block
;
592 /* Emit a jump from the end of the then block to the end of the else */
593 bi_instruction
*then_exit
= bi_emit_branch(ctx
);
595 /* Emit second block, and check if it's empty */
597 int count_in
= ctx
->instruction_count
;
598 bi_block
*else_block
= emit_cf_list(ctx
, &nif
->else_list
);
599 bi_block
*end_else_block
= ctx
->current_block
;
600 ctx
->after_block
= create_empty_block(ctx
);
602 /* Now that we have the subblocks emitted, fix up the branches */
607 if (ctx
->instruction_count
== count_in
) {
608 /* The else block is empty, so don't emit an exit jump */
609 bi_remove_instruction(then_exit
);
610 then_branch
->branch
.target
= ctx
->after_block
;
612 then_branch
->branch
.target
= else_block
;
613 then_exit
->branch
.target
= ctx
->after_block
;
614 bi_block_add_successor(end_then_block
, then_exit
->branch
.target
);
617 /* Wire up the successors */
619 bi_block_add_successor(before_block
, then_branch
->branch
.target
); /* then_branch */
621 bi_block_add_successor(before_block
, then_block
); /* fallthrough */
622 bi_block_add_successor(end_else_block
, ctx
->after_block
); /* fallthrough */
626 emit_loop(bi_context
*ctx
, nir_loop
*nloop
)
628 /* Remember where we are */
629 bi_block
*start_block
= ctx
->current_block
;
631 bi_block
*saved_break
= ctx
->break_block
;
632 bi_block
*saved_continue
= ctx
->continue_block
;
634 ctx
->continue_block
= create_empty_block(ctx
);
635 ctx
->break_block
= create_empty_block(ctx
);
636 ctx
->after_block
= ctx
->continue_block
;
638 /* Emit the body itself */
639 emit_cf_list(ctx
, &nloop
->body
);
641 /* Branch back to loop back */
642 bi_instruction
*br_back
= bi_emit_branch(ctx
);
643 br_back
->branch
.target
= ctx
->continue_block
;
644 bi_block_add_successor(start_block
, ctx
->continue_block
);
645 bi_block_add_successor(ctx
->current_block
, ctx
->continue_block
);
647 ctx
->after_block
= ctx
->break_block
;
650 ctx
->break_block
= saved_break
;
651 ctx
->continue_block
= saved_continue
;
656 emit_cf_list(bi_context
*ctx
, struct exec_list
*list
)
658 bi_block
*start_block
= NULL
;
660 foreach_list_typed(nir_cf_node
, node
, node
, list
) {
661 switch (node
->type
) {
662 case nir_cf_node_block
: {
663 bi_block
*block
= emit_block(ctx
, nir_cf_node_as_block(node
));
672 emit_if(ctx
, nir_cf_node_as_if(node
));
675 case nir_cf_node_loop
:
676 emit_loop(ctx
, nir_cf_node_as_loop(node
));
680 unreachable("Unknown control flow");
688 glsl_type_size(const struct glsl_type
*type
, bool bindless
)
690 return glsl_count_attribute_slots(type
, false);
694 bi_optimize_nir(nir_shader
*nir
)
697 unsigned lower_flrp
= 16 | 32 | 64;
699 NIR_PASS(progress
, nir
, nir_lower_regs_to_ssa
);
700 NIR_PASS(progress
, nir
, nir_lower_idiv
, nir_lower_idiv_fast
);
702 nir_lower_tex_options lower_tex_options
= {
703 .lower_txs_lod
= true,
705 .lower_tex_without_implicit_lod
= true,
709 NIR_PASS(progress
, nir
, nir_lower_tex
, &lower_tex_options
);
710 NIR_PASS(progress
, nir
, nir_lower_alu_to_scalar
, NULL
, NULL
);
711 NIR_PASS(progress
, nir
, nir_lower_load_const_to_scalar
);
716 NIR_PASS(progress
, nir
, nir_lower_var_copies
);
717 NIR_PASS(progress
, nir
, nir_lower_vars_to_ssa
);
719 NIR_PASS(progress
, nir
, nir_copy_prop
);
720 NIR_PASS(progress
, nir
, nir_opt_remove_phis
);
721 NIR_PASS(progress
, nir
, nir_opt_dce
);
722 NIR_PASS(progress
, nir
, nir_opt_dead_cf
);
723 NIR_PASS(progress
, nir
, nir_opt_cse
);
724 NIR_PASS(progress
, nir
, nir_opt_peephole_select
, 64, false, true);
725 NIR_PASS(progress
, nir
, nir_opt_algebraic
);
726 NIR_PASS(progress
, nir
, nir_opt_constant_folding
);
728 if (lower_flrp
!= 0) {
729 bool lower_flrp_progress
= false;
730 NIR_PASS(lower_flrp_progress
,
734 false /* always_precise */,
735 nir
->options
->lower_ffma
);
736 if (lower_flrp_progress
) {
737 NIR_PASS(progress
, nir
,
738 nir_opt_constant_folding
);
742 /* Nothing should rematerialize any flrps, so we only
743 * need to do this lowering once.
748 NIR_PASS(progress
, nir
, nir_opt_undef
);
749 NIR_PASS(progress
, nir
, nir_opt_loop_unroll
,
752 nir_var_function_temp
);
755 NIR_PASS(progress
, nir
, nir_opt_algebraic_late
);
756 NIR_PASS(progress
, nir
, bifrost_nir_lower_algebraic_late
);
757 NIR_PASS(progress
, nir
, nir_lower_alu_to_scalar
, NULL
, NULL
);
758 NIR_PASS(progress
, nir
, nir_lower_load_const_to_scalar
);
760 /* Take us out of SSA */
761 NIR_PASS(progress
, nir
, nir_lower_locals_to_regs
);
762 NIR_PASS(progress
, nir
, nir_convert_from_ssa
, true);
764 /* We're a primary scalar architecture but there's enough vector that
765 * we use a vector IR so let's not also deal with scalar hacks on top
766 * of the vector hacks */
768 NIR_PASS(progress
, nir
, nir_move_vec_src_uses_to_dest
);
769 NIR_PASS(progress
, nir
, nir_lower_vec_to_movs
);
770 NIR_PASS(progress
, nir
, nir_opt_dce
);
774 bifrost_compile_shader_nir(nir_shader
*nir
, panfrost_program
*program
, unsigned product_id
)
776 bi_context
*ctx
= rzalloc(NULL
, bi_context
);
778 ctx
->stage
= nir
->info
.stage
;
779 ctx
->quirks
= bifrost_get_quirks(product_id
);
780 list_inithead(&ctx
->blocks
);
782 /* Lower gl_Position pre-optimisation, but after lowering vars to ssa
783 * (so we don't accidentally duplicate the epilogue since mesa/st has
784 * messed with our I/O quite a bit already) */
786 NIR_PASS_V(nir
, nir_lower_vars_to_ssa
);
788 if (ctx
->stage
== MESA_SHADER_VERTEX
) {
789 NIR_PASS_V(nir
, nir_lower_viewport_transform
);
790 NIR_PASS_V(nir
, nir_lower_point_size
, 1.0, 1024.0);
793 NIR_PASS_V(nir
, nir_split_var_copies
);
794 NIR_PASS_V(nir
, nir_lower_global_vars_to_local
);
795 NIR_PASS_V(nir
, nir_lower_var_copies
);
796 NIR_PASS_V(nir
, nir_lower_vars_to_ssa
);
797 NIR_PASS_V(nir
, nir_lower_io
, nir_var_all
, glsl_type_size
, 0);
798 NIR_PASS_V(nir
, nir_lower_ssbo
);
800 bi_optimize_nir(nir
);
801 nir_print_shader(nir
, stdout
);
803 nir_foreach_function(func
, nir
) {
807 ctx
->impl
= func
->impl
;
808 emit_cf_list(ctx
, &func
->impl
->body
);
809 break; /* TODO: Multi-function shaders */
812 bi_print_shader(ctx
, stdout
);