3 * Copyright (c) 2019 Collabora LTD
5 * Author: Gert Wollny <gert.wollny@collabora.com>
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * on the rights to use, copy, modify, merge, publish, distribute, sub
11 * license, and/or sell copies of the Software, and to permit persons to whom
12 * the Software is furnished to do so, subject to the following conditions:
14 * The above copyright notice and this permission notice (including the next
15 * paragraph) shall be included in all copies or substantial portions of the
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
28 #include "nir_builder.h"
30 #include "../r600_pipe.h"
31 #include "../r600_shader.h"
33 #include "sfn_instruction_tex.h"
35 #include "sfn_shader_vertex.h"
36 #include "sfn_shader_fragment.h"
37 #include "sfn_shader_geometry.h"
38 #include "sfn_nir_lower_fs_out_to_vector.h"
39 #include "sfn_ir_to_assembly.h"
47 ShaderFromNir::ShaderFromNir():sh(nullptr),
53 bool ShaderFromNir::lower(const nir_shader
*shader
, r600_pipe_shader
*pipe_shader
,
54 r600_pipe_shader_selector
*sel
, r600_shader_key
& key
,
55 struct r600_shader
* gs_shader
)
60 switch (shader
->info
.stage
) {
61 case MESA_SHADER_VERTEX
:
63 sfn_log
<< SfnLog::trans
<< "Start VS for GS\n";
64 impl
.reset(new VertexShaderFromNirForGS(pipe_shader
, *sel
, key
, gs_shader
));
65 } else if (key
.vs
.as_ls
) {
66 sfn_log
<< "VS: next type TCS and TES not yet supported\n";
69 sfn_log
<< SfnLog::trans
<< "Start VS for FS\n";
70 impl
.reset(new VertexShaderFromNirForFS(pipe_shader
, *sel
, key
));
73 case MESA_SHADER_GEOMETRY
:
74 sfn_log
<< SfnLog::trans
<< "Start GS\n";
75 impl
.reset(new GeometryShaderFromNir(pipe_shader
, *sel
, key
));
77 case MESA_SHADER_FRAGMENT
:
78 sfn_log
<< SfnLog::trans
<< "Start FS\n";
79 impl
.reset(new FragmentShaderFromNir(*shader
, pipe_shader
->shader
, *sel
, key
));
85 sfn_log
<< SfnLog::trans
<< "Process declarations\n";
86 if (!process_declaration())
89 // at this point all functions should be inlined
90 const nir_function
*func
= reinterpret_cast<const nir_function
*>(exec_list_get_head_const(&sh
->functions
));
92 sfn_log
<< SfnLog::trans
<< "Scan shader\n";
93 nir_foreach_block(block
, func
->impl
) {
94 nir_foreach_instr(instr
, block
) {
95 if (!impl
->scan_instruction(instr
)) {
96 fprintf(stderr
, "Unhandled sysvalue access ");
97 nir_print_instr(instr
, stderr
);
98 fprintf(stderr
, "\n");
104 sfn_log
<< SfnLog::trans
<< "Reserve registers\n";
105 if (!impl
->allocate_reserved_registers()) {
109 ValuePool::array_list arrays
;
110 sfn_log
<< SfnLog::trans
<< "Allocate local registers\n";
111 foreach_list_typed(nir_register
, reg
, node
, &func
->impl
->registers
) {
112 impl
->allocate_local_register(*reg
, arrays
);
115 sfn_log
<< SfnLog::trans
<< "Emit shader start\n";
116 impl
->allocate_arrays(arrays
);
118 impl
->emit_shader_start();
120 sfn_log
<< SfnLog::trans
<< "Process shader \n";
121 foreach_list_typed(nir_cf_node
, node
, node
, &func
->impl
->body
) {
122 if (!process_cf_node(node
))
126 // Add optimizations here
127 sfn_log
<< SfnLog::trans
<< "Finalize\n";
130 if (!sfn_log
.has_debug_flag(SfnLog::nomerge
)) {
131 sfn_log
<< SfnLog::trans
<< "Merge registers\n";
132 impl
->remap_registers();
134 sfn_log
<< SfnLog::trans
<< "Finished translating to R600 IR\n";
138 Shader
ShaderFromNir::shader() const
140 return Shader
{impl
->m_output
, impl
->get_temp_registers()};
144 bool ShaderFromNir::process_cf_node(nir_cf_node
*node
)
146 SFN_TRACE_FUNC(SfnLog::flow
, "CF");
147 switch (node
->type
) {
148 case nir_cf_node_block
:
149 return process_block(nir_cf_node_as_block(node
));
151 return process_if(nir_cf_node_as_if(node
));
152 case nir_cf_node_loop
:
153 return process_loop(nir_cf_node_as_loop(node
));
159 bool ShaderFromNir::process_if(nir_if
*if_stmt
)
161 SFN_TRACE_FUNC(SfnLog::flow
, "IF");
163 if (!impl
->emit_if_start(m_current_if_id
, if_stmt
))
166 int if_id
= m_current_if_id
++;
167 m_if_stack
.push(if_id
);
169 foreach_list_typed(nir_cf_node
, n
, node
, &if_stmt
->then_list
)
170 if (!process_cf_node(n
)) return false;
172 if (!if_stmt
->then_list
.is_empty()) {
173 if (!impl
->emit_else_start(if_id
))
176 foreach_list_typed(nir_cf_node
, n
, node
, &if_stmt
->else_list
)
177 if (!process_cf_node(n
)) return false;
180 if (!impl
->emit_ifelse_end(if_id
))
187 bool ShaderFromNir::process_loop(nir_loop
*node
)
189 SFN_TRACE_FUNC(SfnLog::flow
, "LOOP");
190 int loop_id
= m_current_loop_id
++;
192 if (!impl
->emit_loop_start(loop_id
))
195 foreach_list_typed(nir_cf_node
, n
, node
, &node
->body
)
196 if (!process_cf_node(n
)) return false;
198 if (!impl
->emit_loop_end(loop_id
))
204 bool ShaderFromNir::process_block(nir_block
*block
)
206 SFN_TRACE_FUNC(SfnLog::flow
, "BLOCK");
207 nir_foreach_instr(instr
, block
) {
208 int r
= emit_instruction(instr
);
210 sfn_log
<< SfnLog::err
<< "R600: Unsupported instruction: "
219 ShaderFromNir::~ShaderFromNir()
223 pipe_shader_type
ShaderFromNir::processor_type() const
225 return impl
->m_processor_type
;
229 bool ShaderFromNir::emit_instruction(nir_instr
*instr
)
233 sfn_log
<< SfnLog::instr
<< "Read instruction " << *instr
<< "\n";
235 switch (instr
->type
) {
236 case nir_instr_type_alu
:
237 return impl
->emit_alu_instruction(instr
);
238 case nir_instr_type_deref
:
239 return impl
->emit_deref_instruction(nir_instr_as_deref(instr
));
240 case nir_instr_type_intrinsic
:
241 return impl
->emit_intrinsic_instruction(nir_instr_as_intrinsic(instr
));
242 case nir_instr_type_load_const
:
243 return impl
->set_literal_constant(nir_instr_as_load_const(instr
));
244 case nir_instr_type_tex
:
245 return impl
->emit_tex_instruction(instr
);
246 case nir_instr_type_jump
:
247 return impl
->emit_jump_instruction(nir_instr_as_jump(instr
));
249 fprintf(stderr
, "R600: %s: ShaderFromNir Unsupported instruction: type %d:'", __func__
, instr
->type
);
250 nir_print_instr(instr
, stderr
);
251 fprintf(stderr
, "'\n");
253 case nir_instr_type_ssa_undef
:
254 return impl
->create_undef(nir_instr_as_ssa_undef(instr
));
259 bool ShaderFromNir::process_declaration()
262 nir_foreach_variable(variable
, &sh
->inputs
) {
263 if (!impl
->process_inputs(variable
)) {
264 fprintf(stderr
, "R600: error parsing input varible %s\n", variable
->name
);
270 nir_foreach_variable(variable
, &sh
->outputs
) {
271 if (!impl
->process_outputs(variable
)) {
272 fprintf(stderr
, "R600: error parsing outputs varible %s\n", variable
->name
);
278 nir_foreach_variable(variable
, &sh
->uniforms
) {
279 if (!impl
->process_uniforms(variable
)) {
280 fprintf(stderr
, "R600: error parsing outputs varible %s\n", variable
->name
);
288 const std::vector
<Instruction::Pointer
>& ShaderFromNir::shader_ir() const
291 return impl
->m_output
;
295 AssemblyFromShader::~AssemblyFromShader()
299 bool AssemblyFromShader::lower(const std::vector
<Instruction::Pointer
>& ir
)
305 r600_nir_lower_pack_unpack_2x16_impl(nir_builder
*b
, nir_instr
*instr
, void *_options
)
307 nir_alu_instr
*alu
= nir_instr_as_alu(instr
);
310 case nir_op_unpack_half_2x16
: {
311 nir_ssa_def
*packed
= nir_ssa_for_alu_src(b
, alu
, 0);
312 return nir_vec2(b
, nir_unpack_half_2x16_split_x(b
, packed
),
313 nir_unpack_half_2x16_split_y(b
, packed
));
316 case nir_op_pack_half_2x16
: {
317 nir_ssa_def
*src_vec2
= nir_ssa_for_alu_src(b
, alu
, 0);
318 return nir_pack_half_2x16_split(b
, nir_channel(b
, src_vec2
, 0),
319 nir_channel(b
, src_vec2
, 1));
326 bool r600_nir_lower_pack_unpack_2x16_filter(const nir_instr
*instr
, const void *_options
)
328 return instr
->type
== nir_instr_type_alu
;
331 bool r600_nir_lower_pack_unpack_2x16(nir_shader
*shader
)
333 return nir_shader_lower_instructions(shader
,
334 r600_nir_lower_pack_unpack_2x16_filter
,
335 r600_nir_lower_pack_unpack_2x16_impl
,
340 r600_nir_lower_scratch_address_impl(nir_builder
*b
, nir_intrinsic_instr
*instr
)
342 b
->cursor
= nir_before_instr(&instr
->instr
);
344 int address_index
= 0;
347 if (instr
->intrinsic
== nir_intrinsic_store_scratch
) {
348 align
= instr
->src
[0].ssa
->num_components
;
351 align
= instr
->dest
.ssa
.num_components
;
354 nir_ssa_def
*address
= instr
->src
[address_index
].ssa
;
355 nir_ssa_def
*new_address
= nir_ishr(b
, address
, nir_imm_int(b
, 4 * align
));
357 nir_instr_rewrite_src(&instr
->instr
, &instr
->src
[address_index
],
358 nir_src_for_ssa(new_address
));
361 bool r600_lower_scratch_addresses(nir_shader
*shader
)
363 bool progress
= false;
364 nir_foreach_function(function
, shader
) {
366 nir_builder_init(&build
, function
->impl
);
368 nir_foreach_block(block
, function
->impl
) {
369 nir_foreach_instr(instr
, block
) {
370 if (instr
->type
!= nir_instr_type_intrinsic
)
372 nir_intrinsic_instr
*op
= nir_instr_as_intrinsic(instr
);
373 if (op
->intrinsic
!= nir_intrinsic_load_scratch
&&
374 op
->intrinsic
!= nir_intrinsic_store_scratch
)
376 r600_nir_lower_scratch_address_impl(&build
, op
);
386 using r600::r600_nir_lower_int_tg4
;
387 using r600::r600_nir_lower_pack_unpack_2x16
;
388 using r600::r600_lower_scratch_addresses
;
389 using r600::r600_lower_fs_out_to_vector
;
392 r600_glsl_type_size(const struct glsl_type
*type
, bool is_bindless
)
394 return glsl_count_vec4_slots(type
, false, is_bindless
);
398 r600_get_natural_size_align_bytes(const struct glsl_type
*type
,
399 unsigned *size
, unsigned *align
)
401 if (type
->base_type
!= GLSL_TYPE_ARRAY
) {
405 unsigned elem_size
, elem_align
;
406 glsl_get_natural_size_align_bytes(type
->fields
.array
,
407 &elem_size
, &elem_align
);
409 *size
= type
->length
;
414 optimize_once(nir_shader
*shader
)
416 bool progress
= false;
417 NIR_PASS(progress
, shader
, nir_copy_prop
);
418 NIR_PASS(progress
, shader
, nir_opt_dce
);
419 NIR_PASS(progress
, shader
, nir_opt_algebraic
);
420 NIR_PASS(progress
, shader
, nir_opt_constant_folding
);
421 NIR_PASS(progress
, shader
, nir_opt_copy_prop_vars
);
422 NIR_PASS(progress
, shader
, nir_opt_vectorize
);
424 NIR_PASS(progress
, shader
, nir_opt_remove_phis
);
426 if (nir_opt_trivial_continues(shader
)) {
428 NIR_PASS(progress
, shader
, nir_copy_prop
);
429 NIR_PASS(progress
, shader
, nir_opt_dce
);
432 NIR_PASS(progress
, shader
, nir_opt_if
, false);
433 NIR_PASS(progress
, shader
, nir_opt_dead_cf
);
434 NIR_PASS(progress
, shader
, nir_opt_cse
);
435 NIR_PASS(progress
, shader
, nir_opt_peephole_select
, 200, true, true);
437 NIR_PASS(progress
, shader
, nir_opt_conditional_discard
);
438 NIR_PASS(progress
, shader
, nir_opt_dce
);
439 NIR_PASS(progress
, shader
, nir_opt_undef
);
441 NIR_PASS(progress
, shader
, nir_remove_dead_variables
, nir_var_shader_in
);
442 NIR_PASS(progress
, shader
, nir_remove_dead_variables
, nir_var_shader_out
);
446 bool has_saturate(const nir_function
*func
)
448 nir_foreach_block(block
, func
->impl
) {
449 nir_foreach_instr(instr
, block
) {
450 if (instr
->type
== nir_instr_type_alu
) {
451 auto alu
= nir_instr_as_alu(instr
);
452 if (alu
->dest
.saturate
)
460 int r600_shader_from_nir(struct r600_context
*rctx
,
461 struct r600_pipe_shader
*pipeshader
,
462 r600_shader_key
*key
)
465 struct r600_pipe_shader_selector
*sel
= pipeshader
->selector
;
467 r600::ShaderFromNir convert
;
469 if (rctx
->screen
->b
.debug_flags
& DBG_PREOPT_IR
) {
470 fprintf(stderr
, "PRE-OPT-NIR-----------.------------------------------\n");
471 nir_print_shader(sel
->nir
, stderr
);
472 fprintf(stderr
, "END PRE-OPT-NIR--------------------------------------\n\n");
475 NIR_PASS_V(sel
->nir
, nir_lower_vars_to_ssa
);
476 NIR_PASS_V(sel
->nir
, nir_lower_regs_to_ssa
);
477 NIR_PASS_V(sel
->nir
, nir_lower_phis_to_scalar
);
479 static const struct nir_lower_tex_options lower_tex_options
= {
482 NIR_PASS_V(sel
->nir
, nir_lower_tex
, &lower_tex_options
);
484 NIR_PASS_V(sel
->nir
, r600::r600_nir_lower_txl_txf_array_or_cube
);
486 NIR_PASS_V(sel
->nir
, r600_nir_lower_int_tg4
);
487 NIR_PASS_V(sel
->nir
, r600_nir_lower_pack_unpack_2x16
);
489 NIR_PASS_V(sel
->nir
, nir_lower_io
, nir_var_uniform
, r600_glsl_type_size
,
490 nir_lower_io_lower_64bit_to_32
);
492 if (sel
->nir
->info
.stage
== MESA_SHADER_VERTEX
)
493 NIR_PASS_V(sel
->nir
, r600_vectorize_vs_inputs
);
495 if (sel
->nir
->info
.stage
== MESA_SHADER_FRAGMENT
)
496 NIR_PASS_V(sel
->nir
, r600_lower_fs_out_to_vector
);
498 if (sel
->nir
->info
.stage
== MESA_SHADER_TESS_CTRL
||
499 sel
->nir
->info
.stage
== MESA_SHADER_TESS_EVAL
)
500 NIR_PASS_V(sel
->nir
, nir_lower_io
, nir_var_shader_in
, r600_glsl_type_size
,
501 nir_lower_io_lower_64bit_to_32
);
503 if (sel
->nir
->info
.stage
== MESA_SHADER_TESS_CTRL
)
504 NIR_PASS_V(sel
->nir
, nir_lower_io
, nir_var_shader_out
, r600_glsl_type_size
,
505 nir_lower_io_lower_64bit_to_32
);
507 const nir_function
*func
= reinterpret_cast<const nir_function
*>(exec_list_get_head_const(&sel
->nir
->functions
));
508 bool optimize
= func
->impl
->registers
.length() == 0 && !has_saturate(func
);
511 /* It seems the output of this optimization is cached somewhere, and
512 * when there are registers, then we can no longer copy propagate, so
513 * skip the optimization then. (There is probably a better way, but yeah)
516 while(optimize_once(sel
->nir
));
518 NIR_PASS_V(sel
->nir
, nir_lower_vars_to_scratch
,
519 nir_var_function_temp
,
521 r600_get_natural_size_align_bytes
);
523 while (optimize
&& optimize_once(sel
->nir
));
525 NIR_PASS_V(sel
->nir
, nir_lower_locals_to_regs
);
526 //NIR_PASS_V(sel->nir, nir_opt_algebraic);
527 //NIR_PASS_V(sel->nir, nir_copy_prop);
528 NIR_PASS_V(sel
->nir
, nir_lower_to_source_mods
, nir_lower_float_source_mods
);
529 NIR_PASS_V(sel
->nir
, nir_convert_from_ssa
, true);
530 NIR_PASS_V(sel
->nir
, nir_opt_dce
);
532 if ((rctx
->screen
->b
.debug_flags
& DBG_NIR
) &&
533 (rctx
->screen
->b
.debug_flags
& DBG_ALL_SHADERS
)) {
534 fprintf(stderr
, "-- NIR --------------------------------------------------------\n");
535 struct nir_function
*func
= (struct nir_function
*)exec_list_get_head(&sel
->nir
->functions
);
536 nir_index_ssa_defs(func
->impl
);
537 nir_print_shader(sel
->nir
, stderr
);
538 fprintf(stderr
, "-- END --------------------------------------------------------\n");
541 memset(&pipeshader
->shader
, 0, sizeof(r600_shader
));
542 pipeshader
->scratch_space_needed
= sel
->nir
->scratch_size
;
544 if (sel
->nir
->info
.stage
== MESA_SHADER_TESS_EVAL
||
545 sel
->nir
->info
.stage
== MESA_SHADER_VERTEX
||
546 sel
->nir
->info
.stage
== MESA_SHADER_GEOMETRY
) {
547 pipeshader
->shader
.clip_dist_write
|= ((1 << sel
->nir
->info
.clip_distance_array_size
) - 1);
548 pipeshader
->shader
.cull_dist_write
= ((1 << sel
->nir
->info
.cull_distance_array_size
) - 1)
549 << sel
->nir
->info
.clip_distance_array_size
;
550 pipeshader
->shader
.cc_dist_mask
= (1 << (sel
->nir
->info
.cull_distance_array_size
+
551 sel
->nir
->info
.clip_distance_array_size
)) - 1;
554 // For learning we print out the complete failed shader
555 // and instead of asserts we use exceptions
558 struct r600_shader
* gs_shader
= nullptr;
560 gs_shader
= &rctx
->gs_shader
->current
->shader
;
561 r
= convert
.lower(sel
->nir
, pipeshader
, sel
, *key
, gs_shader
);
563 } catch (std::logic_error
& x
) {
566 if (!r
|| rctx
->screen
->b
.debug_flags
& DBG_ALL_SHADERS
) {
569 snprintf(filename
, 4000, "nir-%s_%d.inc", sel
->nir
->info
.name
, shnr
++);
571 if (access(filename
, F_OK
) == -1) {
572 FILE *f
= fopen(filename
, "w");
575 fprintf(f
, "const char *shader_blob_%s = {\nR\"(", sel
->nir
->info
.name
);
576 nir_print_shader(sel
->nir
, f
);
577 fprintf(f
, ")\";\n");
585 auto shader
= convert
.shader();
587 r600_screen
*rscreen
= rctx
->screen
;
588 r600_bytecode_init(&pipeshader
->shader
.bc
, rscreen
->b
.chip_class
, rscreen
->b
.family
,
589 rscreen
->has_compressed_msaa_texturing
);
591 r600::sfn_log
<< r600::SfnLog::shader_info
592 << "pipeshader->shader.processor_type = "
593 << pipeshader
->shader
.processor_type
<< "\n";
595 pipeshader
->shader
.bc
.type
= pipeshader
->shader
.processor_type
;
596 pipeshader
->shader
.bc
.isa
= rctx
->isa
;
598 r600::AssemblyFromShaderLegacy
afs(&pipeshader
->shader
, key
);
599 if (!afs
.lower(shader
.m_ir
)) {
600 R600_ERR("%s: Lowering to assembly failed\n", __func__
);
604 if (sel
->nir
->info
.stage
== MESA_SHADER_GEOMETRY
) {
605 r600::sfn_log
<< r600::SfnLog::shader_info
<< "Geometry shader, create copy shader\n";
606 generate_gs_copy_shader(rctx
, pipeshader
, &sel
->so
);
607 assert(pipeshader
->gs_copy_shader
);
609 r600::sfn_log
<< r600::SfnLog::shader_info
<< "This is not a Geometry shader\n";