2 * Copyright (C) 2020 Collabora Ltd.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 * Authors (Collabora):
24 * Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
27 #include "main/mtypes.h"
28 #include "compiler/glsl/glsl_to_nir.h"
29 #include "compiler/nir_types.h"
30 #include "main/imports.h"
31 #include "compiler/nir/nir_builder.h"
33 #include "disassemble.h"
34 #include "bifrost_compile.h"
36 #include "bi_quirks.h"
39 static bi_block
*emit_cf_list(bi_context
*ctx
, struct exec_list
*list
);
40 static bi_instruction
*bi_emit_branch(bi_context
*ctx
);
41 static void bi_block_add_successor(bi_block
*block
, bi_block
*successor
);
42 static void bi_schedule_barrier(bi_context
*ctx
);
45 emit_jump(bi_context
*ctx
, nir_jump_instr
*instr
)
47 bi_instruction
*branch
= bi_emit_branch(ctx
);
49 switch (instr
->type
) {
51 branch
->branch
.target
= ctx
->break_block
;
53 case nir_jump_continue
:
54 branch
->branch
.target
= ctx
->continue_block
;
57 unreachable("Unhandled jump type");
60 bi_block_add_successor(ctx
->current_block
, branch
->branch
.target
);
64 bi_emit_ld_vary(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
66 bi_instruction ins
= {
70 .location
= nir_intrinsic_base(instr
),
71 .channels
= instr
->num_components
,
73 .interp_mode
= BIFROST_INTERP_DEFAULT
, /* TODO */
74 .reuse
= false, /* TODO */
75 .flat
= instr
->intrinsic
!= nir_intrinsic_load_interpolated_input
77 .dest
= bir_dest_index(&instr
->dest
),
78 .dest_type
= nir_type_float
| nir_dest_bit_size(instr
->dest
),
81 nir_src
*offset
= nir_get_io_offset_src(instr
);
83 if (nir_src_is_const(*offset
))
84 ins
.load_vary
.load
.location
+= nir_src_as_uint(*offset
);
86 ins
.src
[0] = bir_src_index(offset
);
92 bi_emit_frag_out(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
94 if (!ctx
->emitted_atest
) {
95 bi_instruction ins
= {
100 bi_schedule_barrier(ctx
);
101 ctx
->emitted_atest
= true;
104 bi_instruction blend
= {
106 .blend_location
= nir_intrinsic_base(instr
),
108 bir_src_index(&instr
->src
[0])
113 bi_schedule_barrier(ctx
);
116 static struct bi_load
117 bi_direct_load_for_instr(nir_intrinsic_instr
*instr
)
119 nir_src
*offset
= nir_get_io_offset_src(instr
);
120 assert(nir_src_is_const(*offset
)); /* no indirects */
122 struct bi_load load
= {
123 .location
= nir_intrinsic_base(instr
) + nir_src_as_uint(*offset
),
124 .channels
= instr
->num_components
131 bi_emit_ld_attr(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
133 bi_instruction load
= {
134 .type
= BI_LOAD_ATTR
,
135 .load
= bi_direct_load_for_instr(instr
),
136 .dest
= bir_dest_index(&instr
->dest
),
137 .dest_type
= nir_intrinsic_type(instr
)
144 bi_emit_st_vary(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
146 nir_src
*offset
= nir_get_io_offset_src(instr
);
147 assert(nir_src_is_const(*offset
)); /* no indirects */
149 bi_instruction address
= {
150 .type
= BI_LOAD_VAR_ADDRESS
,
151 .load
= bi_direct_load_for_instr(instr
),
152 .dest_type
= nir_intrinsic_type(instr
),
153 .dest
= bi_make_temp(ctx
)
156 bi_instruction st
= {
157 .type
= BI_STORE_VAR
,
160 bir_src_index(&instr
->src
[0])
164 bi_emit(ctx
, address
);
169 bi_emit_ld_uniform(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
171 /* TODO: Indirect access */
173 bi_instruction ld
= {
174 .type
= BI_LOAD_UNIFORM
,
175 .load
= bi_direct_load_for_instr(instr
),
176 .dest
= bir_dest_index(&instr
->dest
),
177 .dest_type
= nir_intrinsic_type(instr
),
179 BIR_INDEX_ZERO
/* TODO: UBOs */
187 emit_intrinsic(bi_context
*ctx
, nir_intrinsic_instr
*instr
)
190 switch (instr
->intrinsic
) {
191 case nir_intrinsic_load_barycentric_pixel
:
194 case nir_intrinsic_load_interpolated_input
:
195 case nir_intrinsic_load_input
:
196 if (ctx
->stage
== MESA_SHADER_FRAGMENT
)
197 bi_emit_ld_vary(ctx
, instr
);
198 else if (ctx
->stage
== MESA_SHADER_VERTEX
)
199 bi_emit_ld_attr(ctx
, instr
);
201 unreachable("Unsupported shader stage");
205 case nir_intrinsic_store_output
:
206 if (ctx
->stage
== MESA_SHADER_FRAGMENT
)
207 bi_emit_frag_out(ctx
, instr
);
208 else if (ctx
->stage
== MESA_SHADER_VERTEX
)
209 bi_emit_st_vary(ctx
, instr
);
211 unreachable("Unsupported shader stage");
214 case nir_intrinsic_load_uniform
:
215 bi_emit_ld_uniform(ctx
, instr
);
225 emit_load_const(bi_context
*ctx
, nir_load_const_instr
*instr
)
227 /* Make sure we've been lowered */
228 assert(instr
->def
.num_components
== 1);
230 bi_instruction move
= {
232 .dest
= bir_ssa_index(&instr
->def
),
233 .dest_type
= instr
->def
.bit_size
| nir_type_uint
,
238 .u64
= nir_const_value_as_uint(instr
->value
[0], instr
->def
.bit_size
)
246 emit_instr(bi_context
*ctx
, struct nir_instr
*instr
)
248 switch (instr
->type
) {
249 case nir_instr_type_load_const
:
250 emit_load_const(ctx
, nir_instr_as_load_const(instr
));
253 case nir_instr_type_intrinsic
:
254 emit_intrinsic(ctx
, nir_instr_as_intrinsic(instr
));
258 case nir_instr_type_alu
:
259 emit_alu(ctx
, nir_instr_as_alu(instr
));
262 case nir_instr_type_tex
:
263 emit_tex(ctx
, nir_instr_as_tex(instr
));
267 case nir_instr_type_jump
:
268 emit_jump(ctx
, nir_instr_as_jump(instr
));
271 case nir_instr_type_ssa_undef
:
276 //unreachable("Unhandled instruction type");
284 create_empty_block(bi_context
*ctx
)
286 bi_block
*blk
= rzalloc(ctx
, bi_block
);
288 blk
->predecessors
= _mesa_set_create(blk
,
290 _mesa_key_pointer_equal
);
292 blk
->name
= ctx
->block_name_count
++;
298 bi_block_add_successor(bi_block
*block
, bi_block
*successor
)
303 for (unsigned i
= 0; i
< ARRAY_SIZE(block
->successors
); ++i
) {
304 if (block
->successors
[i
]) {
305 if (block
->successors
[i
] == successor
)
311 block
->successors
[i
] = successor
;
312 _mesa_set_add(successor
->predecessors
, block
);
316 unreachable("Too many successors");
320 bi_schedule_barrier(bi_context
*ctx
)
322 bi_block
*temp
= ctx
->after_block
;
323 ctx
->after_block
= create_empty_block(ctx
);
324 list_addtail(&ctx
->after_block
->link
, &ctx
->blocks
);
325 list_inithead(&ctx
->after_block
->instructions
);
326 bi_block_add_successor(ctx
->current_block
, ctx
->after_block
);
327 ctx
->current_block
= ctx
->after_block
;
328 ctx
->after_block
= temp
;
332 emit_block(bi_context
*ctx
, nir_block
*block
)
334 if (ctx
->after_block
) {
335 ctx
->current_block
= ctx
->after_block
;
336 ctx
->after_block
= NULL
;
338 ctx
->current_block
= create_empty_block(ctx
);
341 list_addtail(&ctx
->current_block
->link
, &ctx
->blocks
);
342 list_inithead(&ctx
->current_block
->instructions
);
344 nir_foreach_instr(instr
, block
) {
345 emit_instr(ctx
, instr
);
346 ++ctx
->instruction_count
;
349 return ctx
->current_block
;
352 /* Emits an unconditional branch to the end of the current block, returning a
353 * pointer so the user can fill in details */
355 static bi_instruction
*
356 bi_emit_branch(bi_context
*ctx
)
358 bi_instruction branch
= {
361 .cond
= BI_COND_ALWAYS
365 return bi_emit(ctx
, branch
);
368 /* Sets a condition for a branch by examing the NIR condition. If we're
369 * familiar with the condition, we unwrap it to fold it into the branch
370 * instruction. Otherwise, we consume the condition directly. We
371 * generally use 1-bit booleans which allows us to use small types for
376 bi_set_branch_cond(bi_instruction
*branch
, nir_src
*cond
, bool invert
)
378 /* TODO: Try to unwrap instead of always bailing */
379 branch
->src
[0] = bir_src_index(cond
);
380 branch
->src
[1] = BIR_INDEX_ZERO
;
381 branch
->src_types
[0] = branch
->src_types
[1] = nir_type_uint16
;
382 branch
->branch
.cond
= invert
? BI_COND_EQ
: BI_COND_NE
;
386 emit_if(bi_context
*ctx
, nir_if
*nif
)
388 bi_block
*before_block
= ctx
->current_block
;
390 /* Speculatively emit the branch, but we can't fill it in until later */
391 bi_instruction
*then_branch
= bi_emit_branch(ctx
);
392 bi_set_branch_cond(then_branch
, &nif
->condition
, true);
394 /* Emit the two subblocks. */
395 bi_block
*then_block
= emit_cf_list(ctx
, &nif
->then_list
);
396 bi_block
*end_then_block
= ctx
->current_block
;
398 /* Emit a jump from the end of the then block to the end of the else */
399 bi_instruction
*then_exit
= bi_emit_branch(ctx
);
401 /* Emit second block, and check if it's empty */
403 int count_in
= ctx
->instruction_count
;
404 bi_block
*else_block
= emit_cf_list(ctx
, &nif
->else_list
);
405 bi_block
*end_else_block
= ctx
->current_block
;
406 ctx
->after_block
= create_empty_block(ctx
);
408 /* Now that we have the subblocks emitted, fix up the branches */
413 if (ctx
->instruction_count
== count_in
) {
414 /* The else block is empty, so don't emit an exit jump */
415 bi_remove_instruction(then_exit
);
416 then_branch
->branch
.target
= ctx
->after_block
;
418 then_branch
->branch
.target
= else_block
;
419 then_exit
->branch
.target
= ctx
->after_block
;
420 bi_block_add_successor(end_then_block
, then_exit
->branch
.target
);
423 /* Wire up the successors */
425 bi_block_add_successor(before_block
, then_branch
->branch
.target
); /* then_branch */
427 bi_block_add_successor(before_block
, then_block
); /* fallthrough */
428 bi_block_add_successor(end_else_block
, ctx
->after_block
); /* fallthrough */
432 emit_loop(bi_context
*ctx
, nir_loop
*nloop
)
434 /* Remember where we are */
435 bi_block
*start_block
= ctx
->current_block
;
437 bi_block
*saved_break
= ctx
->break_block
;
438 bi_block
*saved_continue
= ctx
->continue_block
;
440 ctx
->continue_block
= create_empty_block(ctx
);
441 ctx
->break_block
= create_empty_block(ctx
);
442 ctx
->after_block
= ctx
->continue_block
;
444 /* Emit the body itself */
445 emit_cf_list(ctx
, &nloop
->body
);
447 /* Branch back to loop back */
448 bi_instruction
*br_back
= bi_emit_branch(ctx
);
449 br_back
->branch
.target
= ctx
->continue_block
;
450 bi_block_add_successor(start_block
, ctx
->continue_block
);
451 bi_block_add_successor(ctx
->current_block
, ctx
->continue_block
);
453 ctx
->after_block
= ctx
->break_block
;
456 ctx
->break_block
= saved_break
;
457 ctx
->continue_block
= saved_continue
;
462 emit_cf_list(bi_context
*ctx
, struct exec_list
*list
)
464 bi_block
*start_block
= NULL
;
466 foreach_list_typed(nir_cf_node
, node
, node
, list
) {
467 switch (node
->type
) {
468 case nir_cf_node_block
: {
469 bi_block
*block
= emit_block(ctx
, nir_cf_node_as_block(node
));
478 emit_if(ctx
, nir_cf_node_as_if(node
));
481 case nir_cf_node_loop
:
482 emit_loop(ctx
, nir_cf_node_as_loop(node
));
486 unreachable("Unknown control flow");
494 glsl_type_size(const struct glsl_type
*type
, bool bindless
)
496 return glsl_count_attribute_slots(type
, false);
500 bi_optimize_nir(nir_shader
*nir
)
503 unsigned lower_flrp
= 16 | 32 | 64;
505 NIR_PASS(progress
, nir
, nir_lower_regs_to_ssa
);
506 NIR_PASS(progress
, nir
, nir_lower_idiv
, nir_lower_idiv_fast
);
508 nir_lower_tex_options lower_tex_options
= {
509 .lower_txs_lod
= true,
511 .lower_tex_without_implicit_lod
= true,
515 NIR_PASS(progress
, nir
, nir_lower_tex
, &lower_tex_options
);
516 NIR_PASS(progress
, nir
, nir_lower_alu_to_scalar
, NULL
, NULL
);
517 NIR_PASS(progress
, nir
, nir_lower_load_const_to_scalar
);
522 NIR_PASS(progress
, nir
, nir_lower_var_copies
);
523 NIR_PASS(progress
, nir
, nir_lower_vars_to_ssa
);
525 NIR_PASS(progress
, nir
, nir_copy_prop
);
526 NIR_PASS(progress
, nir
, nir_opt_remove_phis
);
527 NIR_PASS(progress
, nir
, nir_opt_dce
);
528 NIR_PASS(progress
, nir
, nir_opt_dead_cf
);
529 NIR_PASS(progress
, nir
, nir_opt_cse
);
530 NIR_PASS(progress
, nir
, nir_opt_peephole_select
, 64, false, true);
531 NIR_PASS(progress
, nir
, nir_opt_algebraic
);
532 NIR_PASS(progress
, nir
, nir_opt_constant_folding
);
534 if (lower_flrp
!= 0) {
535 bool lower_flrp_progress
= false;
536 NIR_PASS(lower_flrp_progress
,
540 false /* always_precise */,
541 nir
->options
->lower_ffma
);
542 if (lower_flrp_progress
) {
543 NIR_PASS(progress
, nir
,
544 nir_opt_constant_folding
);
548 /* Nothing should rematerialize any flrps, so we only
549 * need to do this lowering once.
554 NIR_PASS(progress
, nir
, nir_opt_undef
);
555 NIR_PASS(progress
, nir
, nir_opt_loop_unroll
,
558 nir_var_function_temp
);
561 NIR_PASS(progress
, nir
, nir_opt_algebraic_late
);
562 NIR_PASS(progress
, nir
, nir_lower_alu_to_scalar
, NULL
, NULL
);
563 NIR_PASS(progress
, nir
, nir_lower_load_const_to_scalar
);
565 /* Take us out of SSA */
566 NIR_PASS(progress
, nir
, nir_lower_locals_to_regs
);
567 NIR_PASS(progress
, nir
, nir_convert_from_ssa
, true);
571 bifrost_compile_shader_nir(nir_shader
*nir
, bifrost_program
*program
, unsigned product_id
)
573 bi_context
*ctx
= rzalloc(NULL
, bi_context
);
575 ctx
->stage
= nir
->info
.stage
;
576 ctx
->quirks
= bifrost_get_quirks(product_id
);
577 list_inithead(&ctx
->blocks
);
579 /* Lower gl_Position pre-optimisation, but after lowering vars to ssa
580 * (so we don't accidentally duplicate the epilogue since mesa/st has
581 * messed with our I/O quite a bit already) */
583 NIR_PASS_V(nir
, nir_lower_vars_to_ssa
);
585 if (ctx
->stage
== MESA_SHADER_VERTEX
) {
586 NIR_PASS_V(nir
, nir_lower_viewport_transform
);
587 NIR_PASS_V(nir
, nir_lower_point_size
, 1.0, 1024.0);
590 NIR_PASS_V(nir
, nir_split_var_copies
);
591 NIR_PASS_V(nir
, nir_lower_global_vars_to_local
);
592 NIR_PASS_V(nir
, nir_lower_var_copies
);
593 NIR_PASS_V(nir
, nir_lower_vars_to_ssa
);
594 NIR_PASS_V(nir
, nir_lower_io
, nir_var_all
, glsl_type_size
, 0);
595 NIR_PASS_V(nir
, nir_lower_ssbo
);
597 bi_optimize_nir(nir
);
598 nir_print_shader(nir
, stdout
);
600 nir_foreach_function(func
, nir
) {
604 ctx
->impl
= func
->impl
;
605 emit_cf_list(ctx
, &func
->impl
->body
);
606 break; /* TODO: Multi-function shaders */
609 bi_print_shader(ctx
, stdout
);
611 bi_print_shader(ctx
, stdout
);