2 * Copyright (C) 2020 Collabora Ltd.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 * Authors (Collabora):
24 * Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
27 #include "main/mtypes.h"
28 #include "compiler/glsl/glsl_to_nir.h"
29 #include "compiler/nir_types.h"
30 #include "main/imports.h"
31 #include "compiler/nir/nir_builder.h"
33 #include "disassemble.h"
34 #include "bifrost_compile.h"
35 #include "bifrost_nir.h"
37 #include "bi_quirks.h"
40 static bi_block
*emit_cf_list(bi_context
*ctx
, struct exec_list
*list
);
41 static bi_instruction
*bi_emit_branch(bi_context
*ctx
);
42 static void bi_schedule_barrier(bi_context
*ctx
);
45 emit_jump(bi_context
*ctx
, nir_jump_instr
*instr
)
47 bi_instruction
*branch
= bi_emit_branch(ctx
);
49 switch (instr
->type
) {
51 branch
->branch
.target
= ctx
->break_block
;
53 case nir_jump_continue
:
54 branch
->branch
.target
= ctx
->continue_block
;
57 unreachable("Unhandled jump type");
60 pan_block_add_successor(&ctx
->current_block
->base
, &branch
->branch
.target
->base
);
63 /* Gets a bytemask for a complete vecN write */
65 bi_mask_for_channels_32(unsigned i
)
67 return (1 << (4 * i
)) - 1;
71 bi_load(enum bi_class T
, nir_intrinsic_instr
*instr
)
73 bi_instruction load
= {
75 .writemask
= bi_mask_for_channels_32(instr
->num_components
),
76 .src
= { BIR_INDEX_CONSTANT
},
77 .constant
= { .u64
= nir_intrinsic_base(instr
) },
80 const nir_intrinsic_info
*info
= &nir_intrinsic_infos
[instr
->intrinsic
];
83 load
.dest
= bir_dest_index(&instr
->dest
);
85 if (info
->has_dest
&& info
->index_map
[NIR_INTRINSIC_TYPE
] > 0)
86 load
.dest_type
= nir_intrinsic_type(instr
);
88 nir_src
*offset
= nir_get_io_offset_src(instr
);
90 if (nir_src_is_const(*offset
))
91 load
.constant
.u64
+= nir_src_as_uint(*offset
);
93 load
.src
[0] = bir_src_index(offset
);
99 bi_emit_ld_vary(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
101 bi_instruction ins
= bi_load(BI_LOAD_VAR
, instr
);
102 ins
.load_vary
.interp_mode
= BIFROST_INTERP_DEFAULT
; /* TODO */
103 ins
.load_vary
.reuse
= false; /* TODO */
104 ins
.load_vary
.flat
= instr
->intrinsic
!= nir_intrinsic_load_interpolated_input
;
105 ins
.dest_type
= nir_type_float
| nir_dest_bit_size(instr
->dest
);
107 if (nir_src_is_const(*nir_get_io_offset_src(instr
))) {
108 /* Zero it out for direct */
109 ins
.src
[1] = BIR_INDEX_ZERO
;
111 /* R61 contains sample mask stuff, TODO RA XXX */
112 ins
.src
[1] = BIR_INDEX_REGISTER
| 61;
119 bi_emit_frag_out(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
121 if (!ctx
->emitted_atest
) {
122 bi_instruction ins
= {
125 BIR_INDEX_REGISTER
| 60 /* TODO: RA */,
126 bir_src_index(&instr
->src
[0])
134 { 3, 0 } /* swizzle out the alpha */
136 .dest
= BIR_INDEX_REGISTER
| 60 /* TODO: RA */,
137 .dest_type
= nir_type_uint32
,
142 bi_schedule_barrier(ctx
);
143 ctx
->emitted_atest
= true;
146 bi_instruction blend
= {
148 .blend_location
= nir_intrinsic_base(instr
),
150 BIR_INDEX_REGISTER
| 60 /* Can this be arbitrary? */,
151 bir_src_index(&instr
->src
[0])
161 .dest
= BIR_INDEX_REGISTER
| 48 /* Looks like magic */,
162 .dest_type
= nir_type_uint32
,
167 bi_schedule_barrier(ctx
);
170 static bi_instruction
171 bi_load_with_r61(enum bi_class T
, nir_intrinsic_instr
*instr
)
173 bi_instruction ld
= bi_load(T
, instr
);
174 ld
.src
[1] = BIR_INDEX_REGISTER
| 61; /* TODO: RA */
175 ld
.src
[2] = BIR_INDEX_REGISTER
| 62;
177 ld
.src_types
[1] = nir_type_uint32
;
178 ld
.src_types
[2] = nir_type_uint32
;
179 ld
.src_types
[3] = nir_intrinsic_type(instr
);
184 bi_emit_st_vary(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
186 bi_instruction address
= bi_load_with_r61(BI_LOAD_VAR_ADDRESS
, instr
);
187 address
.dest
= bi_make_temp(ctx
);
188 address
.dest_type
= nir_type_uint32
;
189 address
.writemask
= (1 << 12) - 1;
191 bi_instruction st
= {
192 .type
= BI_STORE_VAR
,
194 bir_src_index(&instr
->src
[0]),
195 address
.dest
, address
.dest
, address
.dest
,
199 nir_type_uint32
, nir_type_uint32
, nir_type_uint32
,
205 .store_channels
= 4, /* TODO: WRITEMASK */
208 bi_emit(ctx
, address
);
213 bi_emit_ld_uniform(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
215 bi_instruction ld
= bi_load(BI_LOAD_UNIFORM
, instr
);
216 ld
.src
[1] = BIR_INDEX_ZERO
; /* TODO: UBO index */
218 /* TODO: Indirect access, since we need to multiply by the element
219 * size. I believe we can get this lowering automatically via
220 * nir_lower_io (as mul instructions) with the proper options, but this
222 assert(ld
.src
[0] & BIR_INDEX_CONSTANT
);
223 ld
.constant
.u64
+= ctx
->sysvals
.sysval_count
;
224 ld
.constant
.u64
*= 16;
230 bi_emit_sysval(bi_context
*ctx
, nir_instr
*instr
,
231 unsigned nr_components
, unsigned offset
)
235 /* Figure out which uniform this is */
236 int sysval
= panfrost_sysval_for_instr(instr
, &nir_dest
);
237 void *val
= _mesa_hash_table_u64_search(ctx
->sysvals
.sysval_to_id
, sysval
);
239 /* Sysvals are prefix uniforms */
240 unsigned uniform
= ((uintptr_t) val
) - 1;
242 /* Emit the read itself -- this is never indirect */
244 bi_instruction load
= {
245 .type
= BI_LOAD_UNIFORM
,
246 .writemask
= (1 << (nr_components
* 4)) - 1,
247 .src
= { BIR_INDEX_CONSTANT
, BIR_INDEX_ZERO
},
248 .constant
= { (uniform
* 16) + offset
},
249 .dest
= bir_dest_index(&nir_dest
),
250 .dest_type
= nir_type_uint32
, /* TODO */
257 emit_intrinsic(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
260 switch (instr
->intrinsic
) {
261 case nir_intrinsic_load_barycentric_pixel
:
264 case nir_intrinsic_load_interpolated_input
:
265 case nir_intrinsic_load_input
:
266 if (ctx
->stage
== MESA_SHADER_FRAGMENT
)
267 bi_emit_ld_vary(ctx
, instr
);
268 else if (ctx
->stage
== MESA_SHADER_VERTEX
)
269 bi_emit(ctx
, bi_load_with_r61(BI_LOAD_ATTR
, instr
));
271 unreachable("Unsupported shader stage");
275 case nir_intrinsic_store_output
:
276 if (ctx
->stage
== MESA_SHADER_FRAGMENT
)
277 bi_emit_frag_out(ctx
, instr
);
278 else if (ctx
->stage
== MESA_SHADER_VERTEX
)
279 bi_emit_st_vary(ctx
, instr
);
281 unreachable("Unsupported shader stage");
284 case nir_intrinsic_load_uniform
:
285 bi_emit_ld_uniform(ctx
, instr
);
288 case nir_intrinsic_load_ssbo_address
:
289 bi_emit_sysval(ctx
, &instr
->instr
, 1, 0);
292 case nir_intrinsic_get_buffer_size
:
293 bi_emit_sysval(ctx
, &instr
->instr
, 1, 8);
296 case nir_intrinsic_load_viewport_scale
:
297 case nir_intrinsic_load_viewport_offset
:
298 case nir_intrinsic_load_num_work_groups
:
299 case nir_intrinsic_load_sampler_lod_parameters_pan
:
300 bi_emit_sysval(ctx
, &instr
->instr
, 3, 0);
310 emit_load_const(bi_context
*ctx
, nir_load_const_instr
*instr
)
312 /* Make sure we've been lowered */
313 assert(instr
->def
.num_components
== 1);
315 bi_instruction move
= {
317 .dest
= bir_ssa_index(&instr
->def
),
318 .dest_type
= instr
->def
.bit_size
| nir_type_uint
,
319 .writemask
= (1 << (instr
->def
.bit_size
/ 8)) - 1,
324 .u64
= nir_const_value_as_uint(instr
->value
[0], instr
->def
.bit_size
)
331 #define BI_CASE_CMP(op) \
337 bi_class_for_nir_alu(nir_op op
)
347 BI_CASE_CMP(nir_op_flt
)
348 BI_CASE_CMP(nir_op_fge
)
349 BI_CASE_CMP(nir_op_feq
)
350 BI_CASE_CMP(nir_op_fne
)
351 BI_CASE_CMP(nir_op_ilt
)
352 BI_CASE_CMP(nir_op_ige
)
353 BI_CASE_CMP(nir_op_ieq
)
354 BI_CASE_CMP(nir_op_ine
)
410 unreachable("Unknown ALU op");
414 /* Gets a bi_cond for a given NIR comparison opcode. In soft mode, it will
415 * return BI_COND_ALWAYS as a sentinel if it fails to do so (when used for
416 * optimizations). Otherwise it will bail (when used for primary code
420 bi_cond_for_nir(nir_op op
, bool soft
)
423 BI_CASE_CMP(nir_op_flt
)
424 BI_CASE_CMP(nir_op_ilt
)
427 BI_CASE_CMP(nir_op_fge
)
428 BI_CASE_CMP(nir_op_ige
)
431 BI_CASE_CMP(nir_op_feq
)
432 BI_CASE_CMP(nir_op_ieq
)
435 BI_CASE_CMP(nir_op_fne
)
436 BI_CASE_CMP(nir_op_ine
)
440 return BI_COND_ALWAYS
;
442 unreachable("Invalid compare");
447 bi_copy_src(bi_instruction
*alu
, nir_alu_instr
*instr
, unsigned i
, unsigned to
,
448 unsigned *constants_left
, unsigned *constant_shift
)
450 unsigned bits
= nir_src_bit_size(instr
->src
[i
].src
);
451 unsigned dest_bits
= nir_dest_bit_size(instr
->dest
.dest
);
453 alu
->src_types
[to
] = nir_op_infos
[instr
->op
].input_types
[i
]
456 /* Try to inline a constant */
457 if (nir_src_is_const(instr
->src
[i
].src
) && *constants_left
&& (dest_bits
== bits
)) {
459 (nir_src_as_uint(instr
->src
[i
].src
)) << *constant_shift
;
461 alu
->src
[to
] = BIR_INDEX_CONSTANT
| (*constant_shift
);
463 (*constant_shift
) += dest_bits
;
467 alu
->src
[to
] = bir_src_index(&instr
->src
[i
].src
);
469 /* We assert scalarization above */
470 alu
->swizzle
[to
][0] = instr
->src
[i
].swizzle
[0];
474 bi_fuse_csel_cond(bi_instruction
*csel
, nir_alu_src cond
,
475 unsigned *constants_left
, unsigned *constant_shift
)
477 /* Bail for vector weirdness */
478 if (cond
.swizzle
[0] != 0)
481 if (!cond
.src
.is_ssa
)
484 nir_ssa_def
*def
= cond
.src
.ssa
;
485 nir_instr
*parent
= def
->parent_instr
;
487 if (parent
->type
!= nir_instr_type_alu
)
490 nir_alu_instr
*alu
= nir_instr_as_alu(parent
);
492 /* Try to match a condition */
493 enum bi_cond bcond
= bi_cond_for_nir(alu
->op
, true);
495 if (bcond
== BI_COND_ALWAYS
)
498 /* We found one, let's fuse it in */
499 csel
->csel_cond
= bcond
;
500 bi_copy_src(csel
, alu
, 0, 0, constants_left
, constant_shift
);
501 bi_copy_src(csel
, alu
, 1, 3, constants_left
, constant_shift
);
505 emit_alu(bi_context
*ctx
, nir_alu_instr
*instr
)
507 /* Assume it's something we can handle normally */
508 bi_instruction alu
= {
509 .type
= bi_class_for_nir_alu(instr
->op
),
510 .dest
= bir_dest_index(&instr
->dest
.dest
),
511 .dest_type
= nir_op_infos
[instr
->op
].output_type
512 | nir_dest_bit_size(instr
->dest
.dest
),
515 /* TODO: Implement lowering of special functions for older Bifrost */
516 assert((alu
.type
!= BI_SPECIAL
) || !(ctx
->quirks
& BIFROST_NO_FAST_OP
));
518 if (instr
->dest
.dest
.is_ssa
) {
519 /* Construct a writemask */
520 unsigned bits_per_comp
= instr
->dest
.dest
.ssa
.bit_size
;
521 unsigned comps
= instr
->dest
.dest
.ssa
.num_components
;
523 unsigned bits
= bits_per_comp
* comps
;
524 unsigned bytes
= bits
/ 8;
525 alu
.writemask
= (1 << bytes
) - 1;
527 unsigned comp_mask
= instr
->dest
.write_mask
;
529 alu
.writemask
= pan_to_bytemask(nir_dest_bit_size(instr
->dest
.dest
),
533 /* We inline constants as we go. This tracks how many constants have
534 * been inlined, since we're limited to 64-bits of constants per
537 unsigned dest_bits
= nir_dest_bit_size(instr
->dest
.dest
);
538 unsigned constants_left
= (64 / dest_bits
);
539 unsigned constant_shift
= 0;
543 unsigned num_inputs
= nir_op_infos
[instr
->op
].num_inputs
;
544 assert(num_inputs
<= ARRAY_SIZE(alu
.src
));
546 for (unsigned i
= 0; i
< num_inputs
; ++i
)
547 bi_copy_src(&alu
, instr
, i
, i
, &constants_left
, &constant_shift
);
549 /* Op-specific fixup */
552 alu
.src
[2] = BIR_INDEX_ZERO
; /* FMA */
555 alu
.outmod
= BIFROST_SAT
; /* FMOV */
558 alu
.src_neg
[0] = true; /* FMOV */
561 alu
.src_abs
[0] = true; /* FMOV */
564 alu
.src_neg
[1] = true; /* FADD */
569 alu
.op
.minmax
= BI_MINMAX_MAX
; /* MINMAX */
572 alu
.op
.special
= BI_SPECIAL_FRCP
;
575 alu
.op
.special
= BI_SPECIAL_FRSQ
;
578 alu
.op
.special
= BI_SPECIAL_FSIN
;
581 alu
.op
.special
= BI_SPECIAL_FCOS
;
583 BI_CASE_CMP(nir_op_flt
)
584 BI_CASE_CMP(nir_op_ilt
)
585 BI_CASE_CMP(nir_op_fge
)
586 BI_CASE_CMP(nir_op_ige
)
587 BI_CASE_CMP(nir_op_feq
)
588 BI_CASE_CMP(nir_op_ieq
)
589 BI_CASE_CMP(nir_op_fne
)
590 BI_CASE_CMP(nir_op_ine
)
591 alu
.op
.compare
= bi_cond_for_nir(instr
->op
, false);
597 if (alu
.type
== BI_CSEL
) {
598 /* Default to csel3 */
599 alu
.csel_cond
= BI_COND_NE
;
600 alu
.src
[3] = BIR_INDEX_ZERO
;
601 alu
.src_types
[3] = alu
.src_types
[0];
603 bi_fuse_csel_cond(&alu
, instr
->src
[0],
604 &constants_left
, &constant_shift
);
611 emit_instr(bi_context
*ctx
, struct nir_instr
*instr
)
613 switch (instr
->type
) {
614 case nir_instr_type_load_const
:
615 emit_load_const(ctx
, nir_instr_as_load_const(instr
));
618 case nir_instr_type_intrinsic
:
619 emit_intrinsic(ctx
, nir_instr_as_intrinsic(instr
));
622 case nir_instr_type_alu
:
623 emit_alu(ctx
, nir_instr_as_alu(instr
));
627 case nir_instr_type_tex
:
628 emit_tex(ctx
, nir_instr_as_tex(instr
));
632 case nir_instr_type_jump
:
633 emit_jump(ctx
, nir_instr_as_jump(instr
));
636 case nir_instr_type_ssa_undef
:
641 //unreachable("Unhandled instruction type");
649 create_empty_block(bi_context
*ctx
)
651 bi_block
*blk
= rzalloc(ctx
, bi_block
);
653 blk
->base
.predecessors
= _mesa_set_create(blk
,
655 _mesa_key_pointer_equal
);
657 blk
->base
.name
= ctx
->block_name_count
++;
663 bi_schedule_barrier(bi_context
*ctx
)
665 bi_block
*temp
= ctx
->after_block
;
666 ctx
->after_block
= create_empty_block(ctx
);
667 list_addtail(&ctx
->after_block
->base
.link
, &ctx
->blocks
);
668 list_inithead(&ctx
->after_block
->base
.instructions
);
669 pan_block_add_successor(&ctx
->current_block
->base
, &ctx
->after_block
->base
);
670 ctx
->current_block
= ctx
->after_block
;
671 ctx
->after_block
= temp
;
675 emit_block(bi_context
*ctx
, nir_block
*block
)
677 if (ctx
->after_block
) {
678 ctx
->current_block
= ctx
->after_block
;
679 ctx
->after_block
= NULL
;
681 ctx
->current_block
= create_empty_block(ctx
);
684 list_addtail(&ctx
->current_block
->base
.link
, &ctx
->blocks
);
685 list_inithead(&ctx
->current_block
->base
.instructions
);
687 nir_foreach_instr(instr
, block
) {
688 emit_instr(ctx
, instr
);
689 ++ctx
->instruction_count
;
692 return ctx
->current_block
;
695 /* Emits an unconditional branch to the end of the current block, returning a
696 * pointer so the user can fill in details */
698 static bi_instruction
*
699 bi_emit_branch(bi_context
*ctx
)
701 bi_instruction branch
= {
704 .cond
= BI_COND_ALWAYS
708 return bi_emit(ctx
, branch
);
711 /* Sets a condition for a branch by examing the NIR condition. If we're
712 * familiar with the condition, we unwrap it to fold it into the branch
713 * instruction. Otherwise, we consume the condition directly. We
714 * generally use 1-bit booleans which allows us to use small types for
719 bi_set_branch_cond(bi_instruction
*branch
, nir_src
*cond
, bool invert
)
721 /* TODO: Try to unwrap instead of always bailing */
722 branch
->src
[0] = bir_src_index(cond
);
723 branch
->src
[1] = BIR_INDEX_ZERO
;
724 branch
->src_types
[0] = branch
->src_types
[1] = nir_type_uint16
;
725 branch
->branch
.cond
= invert
? BI_COND_EQ
: BI_COND_NE
;
729 emit_if(bi_context
*ctx
, nir_if
*nif
)
731 bi_block
*before_block
= ctx
->current_block
;
733 /* Speculatively emit the branch, but we can't fill it in until later */
734 bi_instruction
*then_branch
= bi_emit_branch(ctx
);
735 bi_set_branch_cond(then_branch
, &nif
->condition
, true);
737 /* Emit the two subblocks. */
738 bi_block
*then_block
= emit_cf_list(ctx
, &nif
->then_list
);
739 bi_block
*end_then_block
= ctx
->current_block
;
741 /* Emit a jump from the end of the then block to the end of the else */
742 bi_instruction
*then_exit
= bi_emit_branch(ctx
);
744 /* Emit second block, and check if it's empty */
746 int count_in
= ctx
->instruction_count
;
747 bi_block
*else_block
= emit_cf_list(ctx
, &nif
->else_list
);
748 bi_block
*end_else_block
= ctx
->current_block
;
749 ctx
->after_block
= create_empty_block(ctx
);
751 /* Now that we have the subblocks emitted, fix up the branches */
756 if (ctx
->instruction_count
== count_in
) {
757 /* The else block is empty, so don't emit an exit jump */
758 bi_remove_instruction(then_exit
);
759 then_branch
->branch
.target
= ctx
->after_block
;
761 then_branch
->branch
.target
= else_block
;
762 then_exit
->branch
.target
= ctx
->after_block
;
763 pan_block_add_successor(&end_then_block
->base
, &then_exit
->branch
.target
->base
);
766 /* Wire up the successors */
768 pan_block_add_successor(&before_block
->base
, &then_branch
->branch
.target
->base
); /* then_branch */
770 pan_block_add_successor(&before_block
->base
, &then_block
->base
); /* fallthrough */
771 pan_block_add_successor(&end_else_block
->base
, &ctx
->after_block
->base
); /* fallthrough */
775 emit_loop(bi_context
*ctx
, nir_loop
*nloop
)
777 /* Remember where we are */
778 bi_block
*start_block
= ctx
->current_block
;
780 bi_block
*saved_break
= ctx
->break_block
;
781 bi_block
*saved_continue
= ctx
->continue_block
;
783 ctx
->continue_block
= create_empty_block(ctx
);
784 ctx
->break_block
= create_empty_block(ctx
);
785 ctx
->after_block
= ctx
->continue_block
;
787 /* Emit the body itself */
788 emit_cf_list(ctx
, &nloop
->body
);
790 /* Branch back to loop back */
791 bi_instruction
*br_back
= bi_emit_branch(ctx
);
792 br_back
->branch
.target
= ctx
->continue_block
;
793 pan_block_add_successor(&start_block
->base
, &ctx
->continue_block
->base
);
794 pan_block_add_successor(&ctx
->current_block
->base
, &ctx
->continue_block
->base
);
796 ctx
->after_block
= ctx
->break_block
;
799 ctx
->break_block
= saved_break
;
800 ctx
->continue_block
= saved_continue
;
805 emit_cf_list(bi_context
*ctx
, struct exec_list
*list
)
807 bi_block
*start_block
= NULL
;
809 foreach_list_typed(nir_cf_node
, node
, node
, list
) {
810 switch (node
->type
) {
811 case nir_cf_node_block
: {
812 bi_block
*block
= emit_block(ctx
, nir_cf_node_as_block(node
));
821 emit_if(ctx
, nir_cf_node_as_if(node
));
824 case nir_cf_node_loop
:
825 emit_loop(ctx
, nir_cf_node_as_loop(node
));
829 unreachable("Unknown control flow");
837 glsl_type_size(const struct glsl_type
*type
, bool bindless
)
839 return glsl_count_attribute_slots(type
, false);
843 bi_optimize_nir(nir_shader
*nir
)
846 unsigned lower_flrp
= 16 | 32 | 64;
848 NIR_PASS(progress
, nir
, nir_lower_regs_to_ssa
);
849 NIR_PASS(progress
, nir
, nir_lower_idiv
, nir_lower_idiv_fast
);
851 nir_lower_tex_options lower_tex_options
= {
852 .lower_txs_lod
= true,
854 .lower_tex_without_implicit_lod
= true,
858 NIR_PASS(progress
, nir
, nir_lower_tex
, &lower_tex_options
);
859 NIR_PASS(progress
, nir
, nir_lower_alu_to_scalar
, NULL
, NULL
);
860 NIR_PASS(progress
, nir
, nir_lower_load_const_to_scalar
);
865 NIR_PASS(progress
, nir
, nir_lower_var_copies
);
866 NIR_PASS(progress
, nir
, nir_lower_vars_to_ssa
);
868 NIR_PASS(progress
, nir
, nir_copy_prop
);
869 NIR_PASS(progress
, nir
, nir_opt_remove_phis
);
870 NIR_PASS(progress
, nir
, nir_opt_dce
);
871 NIR_PASS(progress
, nir
, nir_opt_dead_cf
);
872 NIR_PASS(progress
, nir
, nir_opt_cse
);
873 NIR_PASS(progress
, nir
, nir_opt_peephole_select
, 64, false, true);
874 NIR_PASS(progress
, nir
, nir_opt_algebraic
);
875 NIR_PASS(progress
, nir
, nir_opt_constant_folding
);
877 if (lower_flrp
!= 0) {
878 bool lower_flrp_progress
= false;
879 NIR_PASS(lower_flrp_progress
,
883 false /* always_precise */,
884 nir
->options
->lower_ffma
);
885 if (lower_flrp_progress
) {
886 NIR_PASS(progress
, nir
,
887 nir_opt_constant_folding
);
891 /* Nothing should rematerialize any flrps, so we only
892 * need to do this lowering once.
897 NIR_PASS(progress
, nir
, nir_opt_undef
);
898 NIR_PASS(progress
, nir
, nir_opt_loop_unroll
,
901 nir_var_function_temp
);
904 NIR_PASS(progress
, nir
, nir_opt_algebraic_late
);
905 NIR_PASS(progress
, nir
, nir_lower_bool_to_int32
);
906 NIR_PASS(progress
, nir
, bifrost_nir_lower_algebraic_late
);
907 NIR_PASS(progress
, nir
, nir_lower_alu_to_scalar
, NULL
, NULL
);
908 NIR_PASS(progress
, nir
, nir_lower_load_const_to_scalar
);
910 /* Take us out of SSA */
911 NIR_PASS(progress
, nir
, nir_lower_locals_to_regs
);
912 NIR_PASS(progress
, nir
, nir_convert_from_ssa
, true);
914 /* We're a primary scalar architecture but there's enough vector that
915 * we use a vector IR so let's not also deal with scalar hacks on top
916 * of the vector hacks */
918 NIR_PASS(progress
, nir
, nir_move_vec_src_uses_to_dest
);
919 NIR_PASS(progress
, nir
, nir_lower_vec_to_movs
);
920 NIR_PASS(progress
, nir
, nir_opt_dce
);
924 bi_insert_mov32(bi_context
*ctx
, bi_instruction
*parent
, unsigned comp
)
926 bi_instruction move
= {
928 .dest
= parent
->dest
,
929 .dest_type
= nir_type_uint32
,
930 .writemask
= (0xF << (4 * comp
)),
931 .src
= { parent
->src
[0] },
932 .src_types
= { nir_type_uint32
},
933 .swizzle
= { { comp
} }
936 bi_emit_before(ctx
, parent
, move
);
940 bi_lower_mov(bi_context
*ctx
, bi_block
*block
)
942 bi_foreach_instr_in_block_safe(block
, ins
) {
943 if (ins
->type
!= BI_MOV
) continue;
944 if (util_bitcount(ins
->writemask
) <= 4) continue;
946 for (unsigned i
= 0; i
< 4; ++i
) {
947 unsigned quad
= (ins
->writemask
>> (4 * i
)) & 0xF;
951 else if (quad
== 0xF)
952 bi_insert_mov32(ctx
, ins
, i
);
954 unreachable("TODO: Lowering <32bit moves");
957 bi_remove_instruction(ins
);
962 bifrost_compile_shader_nir(nir_shader
*nir
, panfrost_program
*program
, unsigned product_id
)
964 bi_context
*ctx
= rzalloc(NULL
, bi_context
);
966 ctx
->stage
= nir
->info
.stage
;
967 ctx
->quirks
= bifrost_get_quirks(product_id
);
968 list_inithead(&ctx
->blocks
);
970 /* Lower gl_Position pre-optimisation, but after lowering vars to ssa
971 * (so we don't accidentally duplicate the epilogue since mesa/st has
972 * messed with our I/O quite a bit already) */
974 NIR_PASS_V(nir
, nir_lower_vars_to_ssa
);
976 if (ctx
->stage
== MESA_SHADER_VERTEX
) {
977 NIR_PASS_V(nir
, nir_lower_viewport_transform
);
978 NIR_PASS_V(nir
, nir_lower_point_size
, 1.0, 1024.0);
981 NIR_PASS_V(nir
, nir_split_var_copies
);
982 NIR_PASS_V(nir
, nir_lower_global_vars_to_local
);
983 NIR_PASS_V(nir
, nir_lower_var_copies
);
984 NIR_PASS_V(nir
, nir_lower_vars_to_ssa
);
985 NIR_PASS_V(nir
, nir_lower_io
, nir_var_all
, glsl_type_size
, 0);
986 NIR_PASS_V(nir
, nir_lower_ssbo
);
988 bi_optimize_nir(nir
);
989 nir_print_shader(nir
, stdout
);
991 panfrost_nir_assign_sysvals(&ctx
->sysvals
, nir
);
992 program
->sysval_count
= ctx
->sysvals
.sysval_count
;
993 memcpy(program
->sysvals
, ctx
->sysvals
.sysvals
, sizeof(ctx
->sysvals
.sysvals
[0]) * ctx
->sysvals
.sysval_count
);
995 nir_foreach_function(func
, nir
) {
999 ctx
->impl
= func
->impl
;
1000 emit_cf_list(ctx
, &func
->impl
->body
);
1001 break; /* TODO: Multi-function shaders */
1004 bi_foreach_block(ctx
, _block
) {
1005 bi_block
*block
= (bi_block
*) _block
;
1006 bi_lower_mov(ctx
, block
);
1009 bool progress
= false;
1014 bi_foreach_block(ctx
, _block
) {
1015 bi_block
*block
= (bi_block
*) _block
;
1016 progress
|= bi_opt_dead_code_eliminate(ctx
, block
);
1020 bi_print_shader(ctx
, stdout
);
1022 bi_register_allocate(ctx
);
1023 bi_print_shader(ctx
, stdout
);
1024 bi_pack(ctx
, &program
->compiled
);
1025 disassemble_bifrost(stdout
, program
->compiled
.data
, program
->compiled
.size
, true);