2 * Copyright (C) 2020 Collabora Ltd.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 * Authors (Collabora):
24 * Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
27 #include "main/mtypes.h"
28 #include "compiler/glsl/glsl_to_nir.h"
29 #include "compiler/nir_types.h"
30 #include "compiler/nir/nir_builder.h"
31 #include "util/u_debug.h"
33 #include "disassemble.h"
34 #include "bifrost_compile.h"
35 #include "bifrost_nir.h"
37 #include "bi_quirks.h"
40 static const struct debug_named_value debug_options
[] = {
41 {"msgs", BIFROST_DBG_MSGS
, "Print debug messages"},
42 {"shaders", BIFROST_DBG_SHADERS
, "Dump shaders in NIR and MIR"},
46 DEBUG_GET_ONCE_FLAGS_OPTION(bifrost_debug
, "BIFROST_MESA_DEBUG", debug_options
, 0)
48 int bifrost_debug
= 0;
50 #define DBG(fmt, ...) \
51 do { if (bifrost_debug & BIFROST_DBG_MSGS) \
52 fprintf(stderr, "%s:%d: "fmt, \
53 __FUNCTION__, __LINE__, ##__VA_ARGS__); } while (0)
55 static bi_block
*emit_cf_list(bi_context
*ctx
, struct exec_list
*list
);
56 static bi_instruction
*bi_emit_branch(bi_context
*ctx
);
59 emit_jump(bi_context
*ctx
, nir_jump_instr
*instr
)
61 bi_instruction
*branch
= bi_emit_branch(ctx
);
63 switch (instr
->type
) {
65 branch
->branch_target
= ctx
->break_block
;
67 case nir_jump_continue
:
68 branch
->branch_target
= ctx
->continue_block
;
71 unreachable("Unhandled jump type");
74 pan_block_add_successor(&ctx
->current_block
->base
, &branch
->branch_target
->base
);
78 bi_load(enum bi_class T
, nir_intrinsic_instr
*instr
)
80 bi_instruction load
= {
82 .vector_channels
= instr
->num_components
,
83 .src
= { BIR_INDEX_CONSTANT
},
84 .src_types
= { nir_type_uint32
},
85 .constant
= { .u64
= nir_intrinsic_base(instr
) },
88 const nir_intrinsic_info
*info
= &nir_intrinsic_infos
[instr
->intrinsic
];
91 load
.dest
= pan_dest_index(&instr
->dest
);
93 if (info
->has_dest
&& info
->index_map
[NIR_INTRINSIC_TYPE
] > 0)
94 load
.dest_type
= nir_intrinsic_type(instr
);
96 nir_src
*offset
= nir_get_io_offset_src(instr
);
98 if (nir_src_is_const(*offset
))
99 load
.constant
.u64
+= nir_src_as_uint(*offset
);
101 load
.src
[0] = pan_src_index(offset
);
107 bi_emit_ld_vary(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
109 bi_instruction ins
= bi_load(BI_LOAD_VAR
, instr
);
110 ins
.load_vary
.interp_mode
= BIFROST_INTERP_DEFAULT
; /* TODO */
111 ins
.load_vary
.reuse
= false; /* TODO */
112 ins
.load_vary
.flat
= instr
->intrinsic
!= nir_intrinsic_load_interpolated_input
;
113 ins
.dest_type
= nir_type_float
| nir_dest_bit_size(instr
->dest
);
115 if (nir_src_is_const(*nir_get_io_offset_src(instr
))) {
116 /* Zero it out for direct */
117 ins
.src
[1] = BIR_INDEX_ZERO
;
119 /* R61 contains sample mask stuff, TODO RA XXX */
120 ins
.src
[1] = BIR_INDEX_REGISTER
| 61;
127 bi_emit_frag_out(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
129 if (!ctx
->emitted_atest
) {
130 bi_instruction ins
= {
133 BIR_INDEX_REGISTER
| 60 /* TODO: RA */,
134 pan_src_index(&instr
->src
[0])
138 nir_intrinsic_type(instr
)
142 { 3, 0 } /* swizzle out the alpha */
144 .dest
= BIR_INDEX_REGISTER
| 60 /* TODO: RA */,
145 .dest_type
= nir_type_uint32
,
149 ctx
->emitted_atest
= true;
152 bi_instruction blend
= {
154 .blend_location
= nir_intrinsic_base(instr
),
156 pan_src_index(&instr
->src
[0]),
157 BIR_INDEX_REGISTER
| 60 /* Can this be arbitrary? */,
160 nir_intrinsic_type(instr
),
167 .dest
= BIR_INDEX_REGISTER
| 48 /* Looks like magic */,
168 .dest_type
= nir_type_uint32
,
172 assert(blend
.blend_location
< 8);
173 assert(ctx
->blend_types
);
174 assert(blend
.src_types
[0]);
175 ctx
->blend_types
[blend
.blend_location
] = blend
.src_types
[0];
180 static bi_instruction
181 bi_load_with_r61(enum bi_class T
, nir_intrinsic_instr
*instr
)
183 bi_instruction ld
= bi_load(T
, instr
);
184 ld
.src
[1] = BIR_INDEX_REGISTER
| 61; /* TODO: RA */
185 ld
.src
[2] = BIR_INDEX_REGISTER
| 62;
187 ld
.src_types
[1] = nir_type_uint32
;
188 ld
.src_types
[2] = nir_type_uint32
;
189 ld
.src_types
[3] = nir_intrinsic_type(instr
);
194 bi_emit_st_vary(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
196 bi_instruction address
= bi_load_with_r61(BI_LOAD_VAR_ADDRESS
, instr
);
197 address
.dest
= bi_make_temp(ctx
);
198 address
.dest_type
= nir_type_uint32
;
199 address
.vector_channels
= 3;
201 unsigned nr
= nir_intrinsic_src_components(instr
, 0);
202 assert(nir_intrinsic_write_mask(instr
) == ((1 << nr
) - 1));
204 bi_instruction st
= {
205 .type
= BI_STORE_VAR
,
207 pan_src_index(&instr
->src
[0]),
208 address
.dest
, address
.dest
, address
.dest
,
212 nir_type_uint32
, nir_type_uint32
, nir_type_uint32
,
218 .vector_channels
= nr
,
221 for (unsigned i
= 0; i
< nr
; ++i
)
222 st
.swizzle
[0][i
] = i
;
224 bi_emit(ctx
, address
);
229 bi_emit_ld_uniform(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
231 bi_instruction ld
= bi_load(BI_LOAD_UNIFORM
, instr
);
232 ld
.src
[1] = BIR_INDEX_ZERO
; /* TODO: UBO index */
234 /* TODO: Indirect access, since we need to multiply by the element
235 * size. I believe we can get this lowering automatically via
236 * nir_lower_io (as mul instructions) with the proper options, but this
238 assert(ld
.src
[0] & BIR_INDEX_CONSTANT
);
239 ld
.constant
.u64
+= ctx
->sysvals
.sysval_count
;
240 ld
.constant
.u64
*= 16;
246 bi_emit_sysval(bi_context
*ctx
, nir_instr
*instr
,
247 unsigned nr_components
, unsigned offset
)
251 /* Figure out which uniform this is */
252 int sysval
= panfrost_sysval_for_instr(instr
, &nir_dest
);
253 void *val
= _mesa_hash_table_u64_search(ctx
->sysvals
.sysval_to_id
, sysval
);
255 /* Sysvals are prefix uniforms */
256 unsigned uniform
= ((uintptr_t) val
) - 1;
258 /* Emit the read itself -- this is never indirect */
260 bi_instruction load
= {
261 .type
= BI_LOAD_UNIFORM
,
262 .vector_channels
= nr_components
,
263 .src
= { BIR_INDEX_CONSTANT
, BIR_INDEX_ZERO
},
264 .src_types
= { nir_type_uint32
, nir_type_uint32
},
265 .constant
= { (uniform
* 16) + offset
},
266 .dest
= pan_dest_index(&nir_dest
),
267 .dest_type
= nir_type_uint32
, /* TODO */
273 /* gl_FragCoord.xy = u16_to_f32(R59.xy) + 0.5
274 * gl_FragCoord.z = ld_vary(fragz)
275 * gl_FragCoord.w = ld_vary(fragw)
279 bi_emit_ld_frag_coord(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
281 /* Future proofing for mediump fragcoord at some point.. */
282 nir_alu_type T
= nir_type_float32
;
284 /* First, sketch a combine */
285 bi_instruction combine
= {
287 .dest_type
= nir_type_uint32
,
288 .dest
= pan_dest_index(&instr
->dest
),
289 .src_types
= { T
, T
, T
, T
},
292 /* Second, handle xy */
293 for (unsigned i
= 0; i
< 2; ++i
) {
294 bi_instruction conv
= {
297 .dest
= bi_make_temp(ctx
),
300 BIR_INDEX_REGISTER
| 59
302 .src_types
= { nir_type_uint16
},
306 bi_instruction add
= {
309 .dest
= bi_make_temp(ctx
),
310 .src
= { conv
.dest
, BIR_INDEX_CONSTANT
},
311 .src_types
= { T
, T
},
315 memcpy(&add
.constant
.u32
, &half
, sizeof(float));
320 combine
.src
[i
] = add
.dest
;
324 for (unsigned i
= 0; i
< 2; ++i
) {
325 bi_instruction load
= {
328 .interp_mode
= BIFROST_INTERP_DEFAULT
,
332 .vector_channels
= 1,
333 .dest_type
= nir_type_float32
,
334 .dest
= bi_make_temp(ctx
),
335 .src
= { BIR_INDEX_CONSTANT
, BIR_INDEX_ZERO
},
336 .src_types
= { nir_type_uint32
, nir_type_uint32
},
338 .u32
= (i
== 0) ? BIFROST_FRAGZ
: BIFROST_FRAGW
344 combine
.src
[i
+ 2] = load
.dest
;
347 /* Finally, emit the combine */
348 bi_emit(ctx
, combine
);
352 bi_emit_discard(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
355 bi_instruction discard
= {
358 .src_types
= { nir_type_uint32
, nir_type_uint32
},
359 .src
= { BIR_INDEX_ZERO
, BIR_INDEX_ZERO
},
362 bi_emit(ctx
, discard
);
366 bi_fuse_cond(bi_instruction
*csel
, nir_alu_src cond
,
367 unsigned *constants_left
, unsigned *constant_shift
,
368 unsigned comps
, bool float_only
);
371 bi_emit_discard_if(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
373 nir_src cond
= instr
->src
[0];
374 nir_alu_type T
= nir_type_uint
| nir_src_bit_size(cond
);
376 bi_instruction discard
= {
379 .src_types
= { T
, T
},
381 pan_src_index(&cond
),
386 /* Try to fuse in the condition */
387 unsigned constants_left
= 1, constant_shift
= 0;
389 /* Scalar so no swizzle */
394 /* May or may not succeed but we're optimistic */
395 bi_fuse_cond(&discard
, wrap
, &constants_left
, &constant_shift
, 1, true);
397 bi_emit(ctx
, discard
);
401 emit_intrinsic(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
404 switch (instr
->intrinsic
) {
405 case nir_intrinsic_load_barycentric_pixel
:
408 case nir_intrinsic_load_interpolated_input
:
409 case nir_intrinsic_load_input
:
410 if (ctx
->stage
== MESA_SHADER_FRAGMENT
)
411 bi_emit_ld_vary(ctx
, instr
);
412 else if (ctx
->stage
== MESA_SHADER_VERTEX
)
413 bi_emit(ctx
, bi_load_with_r61(BI_LOAD_ATTR
, instr
));
415 unreachable("Unsupported shader stage");
419 case nir_intrinsic_store_output
:
420 if (ctx
->stage
== MESA_SHADER_FRAGMENT
)
421 bi_emit_frag_out(ctx
, instr
);
422 else if (ctx
->stage
== MESA_SHADER_VERTEX
)
423 bi_emit_st_vary(ctx
, instr
);
425 unreachable("Unsupported shader stage");
428 case nir_intrinsic_load_uniform
:
429 bi_emit_ld_uniform(ctx
, instr
);
432 case nir_intrinsic_load_frag_coord
:
433 bi_emit_ld_frag_coord(ctx
, instr
);
436 case nir_intrinsic_discard
:
437 bi_emit_discard(ctx
, instr
);
440 case nir_intrinsic_discard_if
:
441 bi_emit_discard_if(ctx
, instr
);
444 case nir_intrinsic_load_ssbo_address
:
445 bi_emit_sysval(ctx
, &instr
->instr
, 1, 0);
448 case nir_intrinsic_get_buffer_size
:
449 bi_emit_sysval(ctx
, &instr
->instr
, 1, 8);
452 case nir_intrinsic_load_viewport_scale
:
453 case nir_intrinsic_load_viewport_offset
:
454 case nir_intrinsic_load_num_work_groups
:
455 case nir_intrinsic_load_sampler_lod_parameters_pan
:
456 bi_emit_sysval(ctx
, &instr
->instr
, 3, 0);
460 unreachable("Unknown intrinsic");
466 emit_load_const(bi_context
*ctx
, nir_load_const_instr
*instr
)
468 /* Make sure we've been lowered */
469 assert(instr
->def
.num_components
<= (32 / instr
->def
.bit_size
));
471 /* Accumulate all the channels of the constant, as if we did an
472 * implicit SEL over them */
475 for (unsigned i
= 0; i
< instr
->def
.num_components
; ++i
) {
476 unsigned v
= nir_const_value_as_uint(instr
->value
[i
], instr
->def
.bit_size
);
477 acc
|= (v
<< (i
* instr
->def
.bit_size
));
480 bi_instruction move
= {
482 .dest
= pan_ssa_index(&instr
->def
),
483 .dest_type
= nir_type_uint32
,
498 #define BI_CASE_CMP(op) \
504 bi_class_for_nir_alu(nir_op op
)
525 BI_CASE_CMP(nir_op_flt
)
526 BI_CASE_CMP(nir_op_fge
)
527 BI_CASE_CMP(nir_op_feq
)
528 BI_CASE_CMP(nir_op_fne
)
529 BI_CASE_CMP(nir_op_ilt
)
530 BI_CASE_CMP(nir_op_ige
)
531 BI_CASE_CMP(nir_op_ieq
)
532 BI_CASE_CMP(nir_op_ine
)
533 BI_CASE_CMP(nir_op_uge
)
574 unreachable("should've been lowered");
595 case nir_op_fround_even
:
607 unreachable("Unknown ALU op");
611 /* Gets a bi_cond for a given NIR comparison opcode. In soft mode, it will
612 * return BI_COND_ALWAYS as a sentinel if it fails to do so (when used for
613 * optimizations). Otherwise it will bail (when used for primary code
617 bi_cond_for_nir(nir_op op
, bool soft
)
620 BI_CASE_CMP(nir_op_flt
)
621 BI_CASE_CMP(nir_op_ilt
)
624 BI_CASE_CMP(nir_op_fge
)
625 BI_CASE_CMP(nir_op_ige
)
626 BI_CASE_CMP(nir_op_uge
)
629 BI_CASE_CMP(nir_op_feq
)
630 BI_CASE_CMP(nir_op_ieq
)
633 BI_CASE_CMP(nir_op_fne
)
634 BI_CASE_CMP(nir_op_ine
)
638 return BI_COND_ALWAYS
;
640 unreachable("Invalid compare");
645 bi_copy_src(bi_instruction
*alu
, nir_alu_instr
*instr
, unsigned i
, unsigned to
,
646 unsigned *constants_left
, unsigned *constant_shift
, unsigned comps
)
648 unsigned bits
= nir_src_bit_size(instr
->src
[i
].src
);
649 unsigned dest_bits
= nir_dest_bit_size(instr
->dest
.dest
);
651 alu
->src_types
[to
] = nir_op_infos
[instr
->op
].input_types
[i
]
654 /* Try to inline a constant */
655 if (nir_src_is_const(instr
->src
[i
].src
) && *constants_left
&& (dest_bits
== bits
)) {
656 uint64_t mask
= (1ull << dest_bits
) - 1;
657 uint64_t cons
= nir_src_as_uint(instr
->src
[i
].src
);
659 /* Try to reuse a constant */
660 for (unsigned i
= 0; i
< (*constant_shift
); i
+= dest_bits
) {
661 if (((alu
->constant
.u64
>> i
) & mask
) == cons
) {
662 alu
->src
[to
] = BIR_INDEX_CONSTANT
| i
;
667 alu
->constant
.u64
|= cons
<< *constant_shift
;
668 alu
->src
[to
] = BIR_INDEX_CONSTANT
| (*constant_shift
);
670 (*constant_shift
) += MAX2(dest_bits
, 32); /* lo/hi */
674 alu
->src
[to
] = pan_src_index(&instr
->src
[i
].src
);
676 /* Copy swizzle for all vectored components, replicating last component
677 * to fill undersized */
679 unsigned vec
= alu
->type
== BI_COMBINE
? 1 :
680 MAX2(1, 32 / dest_bits
);
682 for (unsigned j
= 0; j
< vec
; ++j
)
683 alu
->swizzle
[to
][j
] = instr
->src
[i
].swizzle
[MIN2(j
, comps
- 1)];
687 bi_fuse_cond(bi_instruction
*csel
, nir_alu_src cond
,
688 unsigned *constants_left
, unsigned *constant_shift
,
689 unsigned comps
, bool float_only
)
691 /* Bail for vector weirdness */
692 if (cond
.swizzle
[0] != 0)
695 if (!cond
.src
.is_ssa
)
698 nir_ssa_def
*def
= cond
.src
.ssa
;
699 nir_instr
*parent
= def
->parent_instr
;
701 if (parent
->type
!= nir_instr_type_alu
)
704 nir_alu_instr
*alu
= nir_instr_as_alu(parent
);
706 /* Try to match a condition */
707 enum bi_cond bcond
= bi_cond_for_nir(alu
->op
, true);
709 if (bcond
== BI_COND_ALWAYS
)
712 /* Some instructions can't compare ints */
714 nir_alu_type T
= nir_op_infos
[alu
->op
].input_types
[0];
715 T
= nir_alu_type_get_base_type(T
);
717 if (T
!= nir_type_float
)
721 /* We found one, let's fuse it in */
723 bi_copy_src(csel
, alu
, 0, 0, constants_left
, constant_shift
, comps
);
724 bi_copy_src(csel
, alu
, 1, 1, constants_left
, constant_shift
, comps
);
728 emit_alu(bi_context
*ctx
, nir_alu_instr
*instr
)
730 /* Try some special functions */
733 bi_emit_fexp2(ctx
, instr
);
736 bi_emit_flog2(ctx
, instr
);
742 /* Otherwise, assume it's something we can handle normally */
743 bi_instruction alu
= {
744 .type
= bi_class_for_nir_alu(instr
->op
),
745 .dest
= pan_dest_index(&instr
->dest
.dest
),
746 .dest_type
= nir_op_infos
[instr
->op
].output_type
747 | nir_dest_bit_size(instr
->dest
.dest
),
750 /* TODO: Implement lowering of special functions for older Bifrost */
751 assert((alu
.type
!= BI_SPECIAL
) || !(ctx
->quirks
& BIFROST_NO_FAST_OP
));
753 unsigned comps
= nir_dest_num_components(instr
->dest
.dest
);
755 if (alu
.type
!= BI_COMBINE
)
756 assert(comps
<= MAX2(1, 32 / comps
));
758 if (!instr
->dest
.dest
.is_ssa
) {
759 for (unsigned i
= 0; i
< comps
; ++i
)
760 assert(instr
->dest
.write_mask
);
763 /* We inline constants as we go. This tracks how many constants have
764 * been inlined, since we're limited to 64-bits of constants per
767 unsigned dest_bits
= nir_dest_bit_size(instr
->dest
.dest
);
768 unsigned constants_left
= (64 / dest_bits
);
769 unsigned constant_shift
= 0;
771 if (alu
.type
== BI_COMBINE
)
776 unsigned num_inputs
= nir_op_infos
[instr
->op
].num_inputs
;
777 assert(num_inputs
<= ARRAY_SIZE(alu
.src
));
779 for (unsigned i
= 0; i
< num_inputs
; ++i
) {
782 if (i
&& alu
.type
== BI_CSEL
)
785 bi_copy_src(&alu
, instr
, i
, i
+ f
, &constants_left
, &constant_shift
, comps
);
788 /* Op-specific fixup */
791 alu
.src
[2] = BIR_INDEX_ZERO
; /* FMA */
792 alu
.src_types
[2] = alu
.src_types
[1];
795 alu
.outmod
= BIFROST_SAT
; /* FMOV */
798 alu
.src_neg
[0] = true; /* FMOV */
801 alu
.src_abs
[0] = true; /* FMOV */
804 alu
.src_neg
[1] = true; /* FADD */
807 alu
.op
.imath
= BI_IMATH_ADD
;
810 alu
.op
.imath
= BI_IMATH_SUB
;
813 alu
.op
.special
= BI_SPECIAL_IABS
;
816 /* no dedicated bitwise not, but we can invert sources. convert to ~a | 0 */
817 alu
.op
.bitwise
= BI_BITWISE_OR
;
818 alu
.bitwise
.src_invert
[0] = true;
819 alu
.src
[1] = BIR_INDEX_ZERO
;
821 alu
.src
[2] = BIR_INDEX_ZERO
;
822 alu
.src_types
[2] = alu
.src_types
[1];
825 alu
.op
.bitwise
= BI_BITWISE_OR
;
826 /* move src1 to src2 and replace with zero. underlying op is (src0 << src2) | src1 */
827 alu
.src
[2] = alu
.src
[1];
828 alu
.src_types
[2] = alu
.src_types
[1];
829 alu
.src
[1] = BIR_INDEX_ZERO
;
832 alu
.op
.imul
= BI_IMUL_IMUL
;
837 alu
.op
.minmax
= BI_MINMAX_MAX
; /* MINMAX */
840 alu
.op
.special
= BI_SPECIAL_FRCP
;
843 alu
.op
.special
= BI_SPECIAL_FRSQ
;
845 BI_CASE_CMP(nir_op_flt
)
846 BI_CASE_CMP(nir_op_ilt
)
847 BI_CASE_CMP(nir_op_fge
)
848 BI_CASE_CMP(nir_op_ige
)
849 BI_CASE_CMP(nir_op_feq
)
850 BI_CASE_CMP(nir_op_ieq
)
851 BI_CASE_CMP(nir_op_fne
)
852 BI_CASE_CMP(nir_op_ine
)
853 BI_CASE_CMP(nir_op_uge
)
854 alu
.cond
= bi_cond_for_nir(instr
->op
, false);
856 case nir_op_fround_even
:
857 alu
.roundmode
= BIFROST_RTE
;
860 alu
.roundmode
= BIFROST_RTP
;
863 alu
.roundmode
= BIFROST_RTN
;
866 alu
.roundmode
= BIFROST_RTZ
;
869 alu
.op
.bitwise
= BI_BITWISE_AND
;
871 alu
.src
[2] = BIR_INDEX_ZERO
;
872 alu
.src_types
[2] = alu
.src_types
[1];
875 alu
.op
.bitwise
= BI_BITWISE_OR
;
877 alu
.src
[2] = BIR_INDEX_ZERO
;
878 alu
.src_types
[2] = alu
.src_types
[1];
881 alu
.op
.bitwise
= BI_BITWISE_XOR
;
883 alu
.src
[2] = BIR_INDEX_ZERO
;
884 alu
.src_types
[2] = alu
.src_types
[1];
887 alu
.roundmode
= BIFROST_RTZ
;
893 if (nir_src_bit_size(instr
->src
[0].src
) != 32)
896 /* Should have been const folded */
897 assert(!nir_src_is_const(instr
->src
[0].src
));
899 alu
.src_types
[1] = alu
.src_types
[0];
900 alu
.src
[1] = alu
.src
[0];
902 unsigned last
= nir_dest_num_components(instr
->dest
.dest
) - 1;
905 alu
.swizzle
[1][0] = instr
->src
[0].swizzle
[last
];
913 if (alu
.type
== BI_CSEL
) {
914 /* Default to csel3 */
915 alu
.cond
= BI_COND_NE
;
916 alu
.src
[1] = BIR_INDEX_ZERO
;
917 alu
.src_types
[1] = alu
.src_types
[0];
919 /* TODO: Reenable cond fusing when we can split up registers
922 bi_fuse_cond(&alu
, instr
->src
[0],
923 &constants_left
, &constant_shift
, comps
, false);
930 /* TEX_COMPACT instructions assume normal 2D f32 operation but are more
931 * space-efficient and with simpler RA/scheduling requirements*/
934 emit_tex_compact(bi_context
*ctx
, nir_tex_instr
*instr
)
936 bi_instruction tex
= {
938 .op
= { .texture
= BI_TEX_COMPACT
},
940 .texture_index
= instr
->texture_index
,
941 .sampler_index
= instr
->sampler_index
,
943 .dest
= pan_dest_index(&instr
->dest
),
944 .dest_type
= instr
->dest_type
,
945 .src_types
= { nir_type_float32
, nir_type_float32
},
949 for (unsigned i
= 0; i
< instr
->num_srcs
; ++i
) {
950 int index
= pan_src_index(&instr
->src
[i
].src
);
952 /* We were checked ahead-of-time */
953 if (instr
->src
[i
].src_type
== nir_tex_src_lod
)
956 assert (instr
->src
[i
].src_type
== nir_tex_src_coord
);
960 tex
.swizzle
[0][0] = 0;
961 tex
.swizzle
[1][0] = 1;
968 emit_tex_full(bi_context
*ctx
, nir_tex_instr
*instr
)
973 /* Normal textures ops are tex for frag shaders and txl for vertex shaders with
974 * lod a constant 0. Anything else needs a full texture op. */
977 bi_is_normal_tex(gl_shader_stage stage
, nir_tex_instr
*instr
)
979 if (stage
== MESA_SHADER_FRAGMENT
)
980 return instr
->op
== nir_texop_tex
;
982 if (instr
->op
!= nir_texop_txl
)
985 for (unsigned i
= 0; i
< instr
->num_srcs
; ++i
) {
986 if (instr
->src
[i
].src_type
!= nir_tex_src_lod
)
989 nir_src src
= instr
->src
[i
].src
;
991 if (!nir_src_is_const(src
))
994 if (nir_src_as_uint(src
) != 0)
1002 emit_tex(bi_context
*ctx
, nir_tex_instr
*instr
)
1004 nir_alu_type base
= nir_alu_type_get_base_type(instr
->dest_type
);
1005 unsigned sz
= nir_dest_bit_size(instr
->dest
);
1006 instr
->dest_type
= base
| sz
;
1008 bool is_normal
= bi_is_normal_tex(ctx
->stage
, instr
);
1009 bool is_2d
= instr
->sampler_dim
== GLSL_SAMPLER_DIM_2D
||
1010 instr
->sampler_dim
== GLSL_SAMPLER_DIM_EXTERNAL
;
1011 bool is_f
= base
== nir_type_float
&& (sz
== 16 || sz
== 32);
1013 bool is_compact
= is_normal
&& is_2d
&& is_f
&& !instr
->is_shadow
;
1016 emit_tex_compact(ctx
, instr
);
1018 emit_tex_full(ctx
, instr
);
1022 emit_instr(bi_context
*ctx
, struct nir_instr
*instr
)
1024 switch (instr
->type
) {
1025 case nir_instr_type_load_const
:
1026 emit_load_const(ctx
, nir_instr_as_load_const(instr
));
1029 case nir_instr_type_intrinsic
:
1030 emit_intrinsic(ctx
, nir_instr_as_intrinsic(instr
));
1033 case nir_instr_type_alu
:
1034 emit_alu(ctx
, nir_instr_as_alu(instr
));
1037 case nir_instr_type_tex
:
1038 emit_tex(ctx
, nir_instr_as_tex(instr
));
1041 case nir_instr_type_jump
:
1042 emit_jump(ctx
, nir_instr_as_jump(instr
));
1045 case nir_instr_type_ssa_undef
:
1050 unreachable("Unhandled instruction type");
1058 create_empty_block(bi_context
*ctx
)
1060 bi_block
*blk
= rzalloc(ctx
, bi_block
);
1062 blk
->base
.predecessors
= _mesa_set_create(blk
,
1064 _mesa_key_pointer_equal
);
1070 emit_block(bi_context
*ctx
, nir_block
*block
)
1072 if (ctx
->after_block
) {
1073 ctx
->current_block
= ctx
->after_block
;
1074 ctx
->after_block
= NULL
;
1076 ctx
->current_block
= create_empty_block(ctx
);
1079 list_addtail(&ctx
->current_block
->base
.link
, &ctx
->blocks
);
1080 list_inithead(&ctx
->current_block
->base
.instructions
);
1082 nir_foreach_instr(instr
, block
) {
1083 emit_instr(ctx
, instr
);
1084 ++ctx
->instruction_count
;
1087 return ctx
->current_block
;
1090 /* Emits an unconditional branch to the end of the current block, returning a
1091 * pointer so the user can fill in details */
1093 static bi_instruction
*
1094 bi_emit_branch(bi_context
*ctx
)
1096 bi_instruction branch
= {
1098 .cond
= BI_COND_ALWAYS
1101 return bi_emit(ctx
, branch
);
1104 /* Sets a condition for a branch by examing the NIR condition. If we're
1105 * familiar with the condition, we unwrap it to fold it into the branch
1106 * instruction. Otherwise, we consume the condition directly. We
1107 * generally use 1-bit booleans which allows us to use small types for
1112 bi_set_branch_cond(bi_instruction
*branch
, nir_src
*cond
, bool invert
)
1114 /* TODO: Try to unwrap instead of always bailing */
1115 branch
->src
[0] = pan_src_index(cond
);
1116 branch
->src
[1] = BIR_INDEX_ZERO
;
1117 branch
->src_types
[0] = branch
->src_types
[1] = nir_type_uint
|
1118 nir_src_bit_size(*cond
);
1119 branch
->cond
= invert
? BI_COND_EQ
: BI_COND_NE
;
1123 emit_if(bi_context
*ctx
, nir_if
*nif
)
1125 bi_block
*before_block
= ctx
->current_block
;
1127 /* Speculatively emit the branch, but we can't fill it in until later */
1128 bi_instruction
*then_branch
= bi_emit_branch(ctx
);
1129 bi_set_branch_cond(then_branch
, &nif
->condition
, true);
1131 /* Emit the two subblocks. */
1132 bi_block
*then_block
= emit_cf_list(ctx
, &nif
->then_list
);
1133 bi_block
*end_then_block
= ctx
->current_block
;
1135 /* Emit a jump from the end of the then block to the end of the else */
1136 bi_instruction
*then_exit
= bi_emit_branch(ctx
);
1138 /* Emit second block, and check if it's empty */
1140 int count_in
= ctx
->instruction_count
;
1141 bi_block
*else_block
= emit_cf_list(ctx
, &nif
->else_list
);
1142 bi_block
*end_else_block
= ctx
->current_block
;
1143 ctx
->after_block
= create_empty_block(ctx
);
1145 /* Now that we have the subblocks emitted, fix up the branches */
1150 if (ctx
->instruction_count
== count_in
) {
1151 /* The else block is empty, so don't emit an exit jump */
1152 bi_remove_instruction(then_exit
);
1153 then_branch
->branch_target
= ctx
->after_block
;
1154 pan_block_add_successor(&end_then_block
->base
, &ctx
->after_block
->base
); /* fallthrough */
1156 then_branch
->branch_target
= else_block
;
1157 then_exit
->branch_target
= ctx
->after_block
;
1158 pan_block_add_successor(&end_then_block
->base
, &then_exit
->branch_target
->base
);
1159 pan_block_add_successor(&end_else_block
->base
, &ctx
->after_block
->base
); /* fallthrough */
1162 pan_block_add_successor(&before_block
->base
, &then_branch
->branch_target
->base
); /* then_branch */
1163 pan_block_add_successor(&before_block
->base
, &then_block
->base
); /* fallthrough */
1167 emit_loop(bi_context
*ctx
, nir_loop
*nloop
)
1169 /* Remember where we are */
1170 bi_block
*start_block
= ctx
->current_block
;
1172 bi_block
*saved_break
= ctx
->break_block
;
1173 bi_block
*saved_continue
= ctx
->continue_block
;
1175 ctx
->continue_block
= create_empty_block(ctx
);
1176 ctx
->break_block
= create_empty_block(ctx
);
1177 ctx
->after_block
= ctx
->continue_block
;
1179 /* Emit the body itself */
1180 emit_cf_list(ctx
, &nloop
->body
);
1182 /* Branch back to loop back */
1183 bi_instruction
*br_back
= bi_emit_branch(ctx
);
1184 br_back
->branch_target
= ctx
->continue_block
;
1185 pan_block_add_successor(&start_block
->base
, &ctx
->continue_block
->base
);
1186 pan_block_add_successor(&ctx
->current_block
->base
, &ctx
->continue_block
->base
);
1188 ctx
->after_block
= ctx
->break_block
;
1191 ctx
->break_block
= saved_break
;
1192 ctx
->continue_block
= saved_continue
;
1197 emit_cf_list(bi_context
*ctx
, struct exec_list
*list
)
1199 bi_block
*start_block
= NULL
;
1201 foreach_list_typed(nir_cf_node
, node
, node
, list
) {
1202 switch (node
->type
) {
1203 case nir_cf_node_block
: {
1204 bi_block
*block
= emit_block(ctx
, nir_cf_node_as_block(node
));
1207 start_block
= block
;
1212 case nir_cf_node_if
:
1213 emit_if(ctx
, nir_cf_node_as_if(node
));
1216 case nir_cf_node_loop
:
1217 emit_loop(ctx
, nir_cf_node_as_loop(node
));
1221 unreachable("Unknown control flow");
1229 glsl_type_size(const struct glsl_type
*type
, bool bindless
)
1231 return glsl_count_attribute_slots(type
, false);
1235 bi_optimize_nir(nir_shader
*nir
)
1238 unsigned lower_flrp
= 16 | 32 | 64;
1240 NIR_PASS(progress
, nir
, nir_lower_regs_to_ssa
);
1241 NIR_PASS(progress
, nir
, nir_lower_idiv
, nir_lower_idiv_fast
);
1243 nir_lower_tex_options lower_tex_options
= {
1244 .lower_txs_lod
= true,
1246 .lower_tex_without_implicit_lod
= true,
1250 NIR_PASS(progress
, nir
, nir_lower_tex
, &lower_tex_options
);
1251 NIR_PASS(progress
, nir
, nir_lower_alu_to_scalar
, NULL
, NULL
);
1252 NIR_PASS(progress
, nir
, nir_lower_load_const_to_scalar
);
1257 NIR_PASS(progress
, nir
, nir_lower_var_copies
);
1258 NIR_PASS(progress
, nir
, nir_lower_vars_to_ssa
);
1260 NIR_PASS(progress
, nir
, nir_copy_prop
);
1261 NIR_PASS(progress
, nir
, nir_opt_remove_phis
);
1262 NIR_PASS(progress
, nir
, nir_opt_dce
);
1263 NIR_PASS(progress
, nir
, nir_opt_dead_cf
);
1264 NIR_PASS(progress
, nir
, nir_opt_cse
);
1265 NIR_PASS(progress
, nir
, nir_opt_peephole_select
, 64, false, true);
1266 NIR_PASS(progress
, nir
, nir_opt_algebraic
);
1267 NIR_PASS(progress
, nir
, nir_opt_constant_folding
);
1269 if (lower_flrp
!= 0) {
1270 bool lower_flrp_progress
= false;
1271 NIR_PASS(lower_flrp_progress
,
1275 false /* always_precise */,
1276 nir
->options
->lower_ffma
);
1277 if (lower_flrp_progress
) {
1278 NIR_PASS(progress
, nir
,
1279 nir_opt_constant_folding
);
1283 /* Nothing should rematerialize any flrps, so we only
1284 * need to do this lowering once.
1289 NIR_PASS(progress
, nir
, nir_opt_undef
);
1290 NIR_PASS(progress
, nir
, nir_opt_loop_unroll
,
1292 nir_var_shader_out
|
1293 nir_var_function_temp
);
1296 NIR_PASS(progress
, nir
, nir_opt_algebraic_late
);
1297 NIR_PASS(progress
, nir
, nir_lower_bool_to_int32
);
1298 NIR_PASS(progress
, nir
, bifrost_nir_lower_algebraic_late
);
1299 NIR_PASS(progress
, nir
, nir_lower_alu_to_scalar
, NULL
, NULL
);
1300 NIR_PASS(progress
, nir
, nir_lower_load_const_to_scalar
);
1302 /* Take us out of SSA */
1303 NIR_PASS(progress
, nir
, nir_lower_locals_to_regs
);
1304 NIR_PASS(progress
, nir
, nir_move_vec_src_uses_to_dest
);
1305 NIR_PASS(progress
, nir
, nir_convert_from_ssa
, true);
1309 bifrost_compile_shader_nir(nir_shader
*nir
, panfrost_program
*program
, unsigned product_id
)
1311 bifrost_debug
= debug_get_option_bifrost_debug();
1313 bi_context
*ctx
= rzalloc(NULL
, bi_context
);
1315 ctx
->stage
= nir
->info
.stage
;
1316 ctx
->quirks
= bifrost_get_quirks(product_id
);
1317 list_inithead(&ctx
->blocks
);
1319 /* Lower gl_Position pre-optimisation, but after lowering vars to ssa
1320 * (so we don't accidentally duplicate the epilogue since mesa/st has
1321 * messed with our I/O quite a bit already) */
1323 NIR_PASS_V(nir
, nir_lower_vars_to_ssa
);
1325 if (ctx
->stage
== MESA_SHADER_VERTEX
) {
1326 NIR_PASS_V(nir
, nir_lower_viewport_transform
);
1327 NIR_PASS_V(nir
, nir_lower_point_size
, 1.0, 1024.0);
1330 NIR_PASS_V(nir
, nir_split_var_copies
);
1331 NIR_PASS_V(nir
, nir_lower_global_vars_to_local
);
1332 NIR_PASS_V(nir
, nir_lower_var_copies
);
1333 NIR_PASS_V(nir
, nir_lower_vars_to_ssa
);
1334 NIR_PASS_V(nir
, nir_lower_io
, nir_var_shader_in
| nir_var_shader_out
,
1336 NIR_PASS_V(nir
, nir_lower_ssbo
);
1337 NIR_PASS_V(nir
, nir_lower_mediump_outputs
);
1339 bi_optimize_nir(nir
);
1341 if (bifrost_debug
& BIFROST_DBG_SHADERS
) {
1342 nir_print_shader(nir
, stdout
);
1345 panfrost_nir_assign_sysvals(&ctx
->sysvals
, nir
);
1346 program
->sysval_count
= ctx
->sysvals
.sysval_count
;
1347 memcpy(program
->sysvals
, ctx
->sysvals
.sysvals
, sizeof(ctx
->sysvals
.sysvals
[0]) * ctx
->sysvals
.sysval_count
);
1348 ctx
->blend_types
= program
->blend_types
;
1350 nir_foreach_function(func
, nir
) {
1354 ctx
->impl
= func
->impl
;
1355 emit_cf_list(ctx
, &func
->impl
->body
);
1356 break; /* TODO: Multi-function shaders */
1359 unsigned block_source_count
= 0;
1361 bi_foreach_block(ctx
, _block
) {
1362 bi_block
*block
= (bi_block
*) _block
;
1364 /* Name blocks now that we're done emitting so the order is
1366 block
->base
.name
= block_source_count
++;
1368 bi_lower_combine(ctx
, block
);
1371 bool progress
= false;
1376 bi_foreach_block(ctx
, _block
) {
1377 bi_block
*block
= (bi_block
*) _block
;
1378 progress
|= bi_opt_dead_code_eliminate(ctx
, block
);
1382 if (bifrost_debug
& BIFROST_DBG_SHADERS
)
1383 bi_print_shader(ctx
, stdout
);
1385 bi_register_allocate(ctx
);
1386 if (bifrost_debug
& BIFROST_DBG_SHADERS
)
1387 bi_print_shader(ctx
, stdout
);
1388 bi_pack(ctx
, &program
->compiled
);
1390 if (bifrost_debug
& BIFROST_DBG_SHADERS
)
1391 disassemble_bifrost(stdout
, program
->compiled
.data
, program
->compiled
.size
, true);