2 * Copyright © 2018 Valve Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
26 #include <unordered_map>
29 #include "nir_control_flow.h"
30 #include "vulkan/radv_shader.h"
31 #include "vulkan/radv_descriptor_set.h"
32 #include "vulkan/radv_shader_args.h"
34 #include "ac_exp_param.h"
35 #include "ac_shader_util.h"
37 #include "util/u_math.h"
39 #define MAX_INLINE_PUSH_CONSTS 8
43 struct shader_io_state
{
44 uint8_t mask
[VARYING_SLOT_MAX
];
45 Temp temps
[VARYING_SLOT_MAX
* 4u];
48 memset(mask
, 0, sizeof(mask
));
49 std::fill_n(temps
, VARYING_SLOT_MAX
* 4u, Temp(0, RegClass::v1
));
54 const struct radv_nir_compiler_options
*options
;
55 struct radv_shader_args
*args
;
58 uint32_t constant_data_offset
;
61 std::unique_ptr
<Temp
[]> allocated
;
62 std::unordered_map
<unsigned, std::array
<Temp
,NIR_MAX_VEC_COMPONENTS
>> allocated_vec
;
63 Stage stage
; /* Stage */
64 bool has_gfx10_wave64_bpermute
= false;
67 uint16_t loop_nest_depth
= 0;
71 bool has_divergent_continue
= false;
72 bool has_divergent_branch
= false;
75 bool is_divergent
= false;
77 bool exec_potentially_empty_discard
= false; /* set to false when loop_nest_depth==0 && parent_if.is_divergent==false */
78 uint16_t exec_potentially_empty_break_depth
= UINT16_MAX
;
79 /* Set to false when loop_nest_depth==exec_potentially_empty_break_depth
80 * and parent_if.is_divergent==false. Called _break but it's also used for
82 bool exec_potentially_empty_break
= false;
83 std::unique_ptr
<unsigned[]> nir_to_aco
; /* NIR block index to ACO block index */
86 Temp arg_temps
[AC_MAX_ARGS
];
89 Temp persp_centroid
, linear_centroid
;
94 /* gathered information */
95 uint64_t input_masks
[MESA_SHADER_COMPUTE
];
96 uint64_t output_masks
[MESA_SHADER_COMPUTE
];
98 /* VS output information */
99 bool export_clip_dists
;
100 unsigned num_clip_distances
;
101 unsigned num_cull_distances
;
103 /* tessellation information */
104 unsigned tcs_tess_lvl_out_loc
;
105 unsigned tcs_tess_lvl_in_loc
;
106 uint64_t tcs_temp_only_inputs
;
107 uint32_t tcs_num_inputs
;
108 uint32_t tcs_num_patches
;
109 bool tcs_in_out_eq
= false;
111 /* I/O information */
112 shader_io_state inputs
;
113 shader_io_state outputs
;
116 Temp
get_arg(isel_context
*ctx
, struct ac_arg arg
)
119 return ctx
->arg_temps
[arg
.arg_index
];
122 unsigned get_interp_input(nir_intrinsic_op intrin
, enum glsl_interp_mode interp
)
125 case INTERP_MODE_SMOOTH
:
126 case INTERP_MODE_NONE
:
127 if (intrin
== nir_intrinsic_load_barycentric_pixel
||
128 intrin
== nir_intrinsic_load_barycentric_at_sample
||
129 intrin
== nir_intrinsic_load_barycentric_at_offset
)
130 return S_0286CC_PERSP_CENTER_ENA(1);
131 else if (intrin
== nir_intrinsic_load_barycentric_centroid
)
132 return S_0286CC_PERSP_CENTROID_ENA(1);
133 else if (intrin
== nir_intrinsic_load_barycentric_sample
)
134 return S_0286CC_PERSP_SAMPLE_ENA(1);
136 case INTERP_MODE_NOPERSPECTIVE
:
137 if (intrin
== nir_intrinsic_load_barycentric_pixel
)
138 return S_0286CC_LINEAR_CENTER_ENA(1);
139 else if (intrin
== nir_intrinsic_load_barycentric_centroid
)
140 return S_0286CC_LINEAR_CENTROID_ENA(1);
141 else if (intrin
== nir_intrinsic_load_barycentric_sample
)
142 return S_0286CC_LINEAR_SAMPLE_ENA(1);
150 /* If one side of a divergent IF ends in a branch and the other doesn't, we
151 * might have to emit the contents of the side without the branch at the merge
152 * block instead. This is so that we can use any SGPR live-out of the side
153 * without the branch without creating a linear phi in the invert or merge block. */
155 sanitize_if(nir_function_impl
*impl
, bool *divergent
, nir_if
*nif
)
157 //TODO: skip this if the condition is uniform and there are no divergent breaks/continues?
159 nir_block
*then_block
= nir_if_last_then_block(nif
);
160 nir_block
*else_block
= nir_if_last_else_block(nif
);
161 bool then_jump
= nir_block_ends_in_jump(then_block
) || nir_block_is_unreachable(then_block
);
162 bool else_jump
= nir_block_ends_in_jump(else_block
) || nir_block_is_unreachable(else_block
);
163 if (then_jump
== else_jump
)
166 /* If the continue from block is empty then return as there is nothing to
169 if (nir_cf_list_is_empty_block(else_jump
? &nif
->then_list
: &nif
->else_list
))
172 /* Even though this if statement has a jump on one side, we may still have
173 * phis afterwards. Single-source phis can be produced by loop unrolling
174 * or dead control-flow passes and are perfectly legal. Run a quick phi
175 * removal on the block after the if to clean up any such phis.
177 nir_opt_remove_phis_block(nir_cf_node_as_block(nir_cf_node_next(&nif
->cf_node
)));
179 /* Finally, move the continue from branch after the if-statement. */
180 nir_block
*last_continue_from_blk
= else_jump
? then_block
: else_block
;
181 nir_block
*first_continue_from_blk
= else_jump
?
182 nir_if_first_then_block(nif
) : nir_if_first_else_block(nif
);
185 nir_cf_extract(&tmp
, nir_before_block(first_continue_from_blk
),
186 nir_after_block(last_continue_from_blk
));
187 nir_cf_reinsert(&tmp
, nir_after_cf_node(&nif
->cf_node
));
189 /* nir_cf_extract() invalidates dominance metadata, but it should still be
190 * correct because of the specific type of transformation we did. Block
191 * indices are not valid except for block_0's, which is all we care about for
192 * nir_block_is_unreachable(). */
193 impl
->valid_metadata
=
194 (nir_metadata
)(impl
->valid_metadata
| nir_metadata_dominance
| nir_metadata_block_index
);
200 sanitize_cf_list(nir_function_impl
*impl
, bool *divergent
, struct exec_list
*cf_list
)
202 bool progress
= false;
203 foreach_list_typed(nir_cf_node
, cf_node
, node
, cf_list
) {
204 switch (cf_node
->type
) {
205 case nir_cf_node_block
:
207 case nir_cf_node_if
: {
208 nir_if
*nif
= nir_cf_node_as_if(cf_node
);
209 progress
|= sanitize_cf_list(impl
, divergent
, &nif
->then_list
);
210 progress
|= sanitize_cf_list(impl
, divergent
, &nif
->else_list
);
211 progress
|= sanitize_if(impl
, divergent
, nif
);
214 case nir_cf_node_loop
: {
215 nir_loop
*loop
= nir_cf_node_as_loop(cf_node
);
216 progress
|= sanitize_cf_list(impl
, divergent
, &loop
->body
);
219 case nir_cf_node_function
:
220 unreachable("Invalid cf type");
227 RegClass
get_reg_class(isel_context
*ctx
, RegType type
, unsigned components
, unsigned bitsize
)
231 return RegClass(RegType::sgpr
, ctx
->program
->lane_mask
.size() * components
);
233 return type
== RegType::sgpr
? s1
: RegClass(type
, components
).as_subdword();
235 return type
== RegType::sgpr
? RegClass(type
, DIV_ROUND_UP(components
, 2)) :
236 RegClass(type
, 2 * components
).as_subdword();
238 return RegClass(type
, components
);
240 return RegClass(type
, components
* 2);
242 unreachable("Unsupported bit size");
246 void init_context(isel_context
*ctx
, nir_shader
*shader
)
248 nir_function_impl
*impl
= nir_shader_get_entrypoint(shader
);
249 unsigned lane_mask_size
= ctx
->program
->lane_mask
.size();
251 ctx
->shader
= shader
;
252 ctx
->divergent_vals
= nir_divergence_analysis(shader
, nir_divergence_view_index_uniform
);
254 /* sanitize control flow */
255 nir_metadata_require(impl
, nir_metadata_dominance
);
256 sanitize_cf_list(impl
, ctx
->divergent_vals
, &impl
->body
);
257 nir_metadata_preserve(impl
, (nir_metadata
)~nir_metadata_block_index
);
259 /* we'll need this for isel */
260 nir_metadata_require(impl
, nir_metadata_block_index
);
262 if (!(ctx
->stage
& sw_gs_copy
) && ctx
->options
->dump_preoptir
) {
263 fprintf(stderr
, "NIR shader before instruction selection:\n");
264 nir_print_shader(shader
, stderr
);
267 std::unique_ptr
<Temp
[]> allocated
{new Temp
[impl
->ssa_alloc
]()};
269 unsigned spi_ps_inputs
= 0;
271 std::unique_ptr
<unsigned[]> nir_to_aco
{new unsigned[impl
->num_blocks
]()};
276 nir_foreach_block(block
, impl
) {
277 nir_foreach_instr(instr
, block
) {
278 switch(instr
->type
) {
279 case nir_instr_type_alu
: {
280 nir_alu_instr
*alu_instr
= nir_instr_as_alu(instr
);
281 RegType type
= RegType::sgpr
;
282 switch(alu_instr
->op
) {
304 case nir_op_fround_even
:
308 case nir_op_f2f16_rtz
:
309 case nir_op_f2f16_rtne
:
316 case nir_op_pack_half_2x16
:
317 case nir_op_unpack_half_2x16_split_x
:
318 case nir_op_unpack_half_2x16_split_y
:
321 case nir_op_fddx_fine
:
322 case nir_op_fddy_fine
:
323 case nir_op_fddx_coarse
:
324 case nir_op_fddy_coarse
:
325 case nir_op_fquantize2f16
:
327 case nir_op_frexp_sig
:
328 case nir_op_frexp_exp
:
329 case nir_op_cube_face_index
:
330 case nir_op_cube_face_coord
:
331 type
= RegType::vgpr
;
343 type
= ctx
->divergent_vals
[alu_instr
->dest
.dest
.ssa
.index
] ? RegType::vgpr
: RegType::sgpr
;
346 type
= ctx
->divergent_vals
[alu_instr
->dest
.dest
.ssa
.index
] ? RegType::vgpr
: RegType::sgpr
;
349 for (unsigned i
= 0; i
< nir_op_infos
[alu_instr
->op
].num_inputs
; i
++) {
350 if (allocated
[alu_instr
->src
[i
].src
.ssa
->index
].type() == RegType::vgpr
)
351 type
= RegType::vgpr
;
356 RegClass rc
= get_reg_class(ctx
, type
, alu_instr
->dest
.dest
.ssa
.num_components
, alu_instr
->dest
.dest
.ssa
.bit_size
);
357 allocated
[alu_instr
->dest
.dest
.ssa
.index
] = Temp(0, rc
);
360 case nir_instr_type_load_const
: {
361 unsigned size
= nir_instr_as_load_const(instr
)->def
.num_components
;
362 if (nir_instr_as_load_const(instr
)->def
.bit_size
== 64)
364 else if (nir_instr_as_load_const(instr
)->def
.bit_size
== 1)
365 size
*= lane_mask_size
;
366 allocated
[nir_instr_as_load_const(instr
)->def
.index
] = Temp(0, RegClass(RegType::sgpr
, size
));
369 case nir_instr_type_intrinsic
: {
370 nir_intrinsic_instr
*intrinsic
= nir_instr_as_intrinsic(instr
);
371 if (!nir_intrinsic_infos
[intrinsic
->intrinsic
].has_dest
)
373 RegType type
= RegType::sgpr
;
374 switch(intrinsic
->intrinsic
) {
375 case nir_intrinsic_load_push_constant
:
376 case nir_intrinsic_load_work_group_id
:
377 case nir_intrinsic_load_num_work_groups
:
378 case nir_intrinsic_load_subgroup_id
:
379 case nir_intrinsic_load_num_subgroups
:
380 case nir_intrinsic_load_first_vertex
:
381 case nir_intrinsic_load_base_instance
:
382 case nir_intrinsic_get_buffer_size
:
383 case nir_intrinsic_vote_all
:
384 case nir_intrinsic_vote_any
:
385 case nir_intrinsic_read_first_invocation
:
386 case nir_intrinsic_read_invocation
:
387 case nir_intrinsic_first_invocation
:
388 case nir_intrinsic_ballot
:
389 type
= RegType::sgpr
;
391 case nir_intrinsic_load_sample_id
:
392 case nir_intrinsic_load_sample_mask_in
:
393 case nir_intrinsic_load_input
:
394 case nir_intrinsic_load_output
:
395 case nir_intrinsic_load_input_vertex
:
396 case nir_intrinsic_load_per_vertex_input
:
397 case nir_intrinsic_load_per_vertex_output
:
398 case nir_intrinsic_load_vertex_id
:
399 case nir_intrinsic_load_vertex_id_zero_base
:
400 case nir_intrinsic_load_barycentric_sample
:
401 case nir_intrinsic_load_barycentric_pixel
:
402 case nir_intrinsic_load_barycentric_model
:
403 case nir_intrinsic_load_barycentric_centroid
:
404 case nir_intrinsic_load_barycentric_at_sample
:
405 case nir_intrinsic_load_barycentric_at_offset
:
406 case nir_intrinsic_load_interpolated_input
:
407 case nir_intrinsic_load_frag_coord
:
408 case nir_intrinsic_load_sample_pos
:
409 case nir_intrinsic_load_layer_id
:
410 case nir_intrinsic_load_local_invocation_id
:
411 case nir_intrinsic_load_local_invocation_index
:
412 case nir_intrinsic_load_subgroup_invocation
:
413 case nir_intrinsic_load_tess_coord
:
414 case nir_intrinsic_write_invocation_amd
:
415 case nir_intrinsic_mbcnt_amd
:
416 case nir_intrinsic_load_instance_id
:
417 case nir_intrinsic_ssbo_atomic_add
:
418 case nir_intrinsic_ssbo_atomic_imin
:
419 case nir_intrinsic_ssbo_atomic_umin
:
420 case nir_intrinsic_ssbo_atomic_imax
:
421 case nir_intrinsic_ssbo_atomic_umax
:
422 case nir_intrinsic_ssbo_atomic_and
:
423 case nir_intrinsic_ssbo_atomic_or
:
424 case nir_intrinsic_ssbo_atomic_xor
:
425 case nir_intrinsic_ssbo_atomic_exchange
:
426 case nir_intrinsic_ssbo_atomic_comp_swap
:
427 case nir_intrinsic_global_atomic_add
:
428 case nir_intrinsic_global_atomic_imin
:
429 case nir_intrinsic_global_atomic_umin
:
430 case nir_intrinsic_global_atomic_imax
:
431 case nir_intrinsic_global_atomic_umax
:
432 case nir_intrinsic_global_atomic_and
:
433 case nir_intrinsic_global_atomic_or
:
434 case nir_intrinsic_global_atomic_xor
:
435 case nir_intrinsic_global_atomic_exchange
:
436 case nir_intrinsic_global_atomic_comp_swap
:
437 case nir_intrinsic_image_deref_atomic_add
:
438 case nir_intrinsic_image_deref_atomic_umin
:
439 case nir_intrinsic_image_deref_atomic_imin
:
440 case nir_intrinsic_image_deref_atomic_umax
:
441 case nir_intrinsic_image_deref_atomic_imax
:
442 case nir_intrinsic_image_deref_atomic_and
:
443 case nir_intrinsic_image_deref_atomic_or
:
444 case nir_intrinsic_image_deref_atomic_xor
:
445 case nir_intrinsic_image_deref_atomic_exchange
:
446 case nir_intrinsic_image_deref_atomic_comp_swap
:
447 case nir_intrinsic_image_deref_size
:
448 case nir_intrinsic_shared_atomic_add
:
449 case nir_intrinsic_shared_atomic_imin
:
450 case nir_intrinsic_shared_atomic_umin
:
451 case nir_intrinsic_shared_atomic_imax
:
452 case nir_intrinsic_shared_atomic_umax
:
453 case nir_intrinsic_shared_atomic_and
:
454 case nir_intrinsic_shared_atomic_or
:
455 case nir_intrinsic_shared_atomic_xor
:
456 case nir_intrinsic_shared_atomic_exchange
:
457 case nir_intrinsic_shared_atomic_comp_swap
:
458 case nir_intrinsic_load_scratch
:
459 case nir_intrinsic_load_invocation_id
:
460 case nir_intrinsic_load_primitive_id
:
461 type
= RegType::vgpr
;
463 case nir_intrinsic_shuffle
:
464 case nir_intrinsic_quad_broadcast
:
465 case nir_intrinsic_quad_swap_horizontal
:
466 case nir_intrinsic_quad_swap_vertical
:
467 case nir_intrinsic_quad_swap_diagonal
:
468 case nir_intrinsic_quad_swizzle_amd
:
469 case nir_intrinsic_masked_swizzle_amd
:
470 case nir_intrinsic_inclusive_scan
:
471 case nir_intrinsic_exclusive_scan
:
472 case nir_intrinsic_reduce
:
473 case nir_intrinsic_load_ubo
:
474 case nir_intrinsic_load_ssbo
:
475 case nir_intrinsic_load_global
:
476 case nir_intrinsic_vulkan_resource_index
:
477 case nir_intrinsic_load_shared
:
478 type
= ctx
->divergent_vals
[intrinsic
->dest
.ssa
.index
] ? RegType::vgpr
: RegType::sgpr
;
480 case nir_intrinsic_load_view_index
:
481 type
= ctx
->stage
== fragment_fs
? RegType::vgpr
: RegType::sgpr
;
484 for (unsigned i
= 0; i
< nir_intrinsic_infos
[intrinsic
->intrinsic
].num_srcs
; i
++) {
485 if (allocated
[intrinsic
->src
[i
].ssa
->index
].type() == RegType::vgpr
)
486 type
= RegType::vgpr
;
490 RegClass rc
= get_reg_class(ctx
, type
, intrinsic
->dest
.ssa
.num_components
, intrinsic
->dest
.ssa
.bit_size
);
491 allocated
[intrinsic
->dest
.ssa
.index
] = Temp(0, rc
);
493 switch(intrinsic
->intrinsic
) {
494 case nir_intrinsic_load_barycentric_sample
:
495 case nir_intrinsic_load_barycentric_pixel
:
496 case nir_intrinsic_load_barycentric_centroid
:
497 case nir_intrinsic_load_barycentric_at_sample
:
498 case nir_intrinsic_load_barycentric_at_offset
: {
499 glsl_interp_mode mode
= (glsl_interp_mode
)nir_intrinsic_interp_mode(intrinsic
);
500 spi_ps_inputs
|= get_interp_input(intrinsic
->intrinsic
, mode
);
503 case nir_intrinsic_load_barycentric_model
:
504 spi_ps_inputs
|= S_0286CC_PERSP_PULL_MODEL_ENA(1);
506 case nir_intrinsic_load_front_face
:
507 spi_ps_inputs
|= S_0286CC_FRONT_FACE_ENA(1);
509 case nir_intrinsic_load_frag_coord
:
510 case nir_intrinsic_load_sample_pos
: {
511 uint8_t mask
= nir_ssa_def_components_read(&intrinsic
->dest
.ssa
);
512 for (unsigned i
= 0; i
< 4; i
++) {
514 spi_ps_inputs
|= S_0286CC_POS_X_FLOAT_ENA(1) << i
;
519 case nir_intrinsic_load_sample_id
:
520 spi_ps_inputs
|= S_0286CC_ANCILLARY_ENA(1);
522 case nir_intrinsic_load_sample_mask_in
:
523 spi_ps_inputs
|= S_0286CC_ANCILLARY_ENA(1);
524 spi_ps_inputs
|= S_0286CC_SAMPLE_COVERAGE_ENA(1);
531 case nir_instr_type_tex
: {
532 nir_tex_instr
* tex
= nir_instr_as_tex(instr
);
533 unsigned size
= tex
->dest
.ssa
.num_components
;
535 if (tex
->dest
.ssa
.bit_size
== 64)
537 if (tex
->op
== nir_texop_texture_samples
)
538 assert(!ctx
->divergent_vals
[tex
->dest
.ssa
.index
]);
539 if (ctx
->divergent_vals
[tex
->dest
.ssa
.index
])
540 allocated
[tex
->dest
.ssa
.index
] = Temp(0, RegClass(RegType::vgpr
, size
));
542 allocated
[tex
->dest
.ssa
.index
] = Temp(0, RegClass(RegType::sgpr
, size
));
545 case nir_instr_type_parallel_copy
: {
546 nir_foreach_parallel_copy_entry(entry
, nir_instr_as_parallel_copy(instr
)) {
547 allocated
[entry
->dest
.ssa
.index
] = allocated
[entry
->src
.ssa
->index
];
551 case nir_instr_type_ssa_undef
: {
552 unsigned size
= nir_instr_as_ssa_undef(instr
)->def
.num_components
;
553 if (nir_instr_as_ssa_undef(instr
)->def
.bit_size
== 64)
555 else if (nir_instr_as_ssa_undef(instr
)->def
.bit_size
== 1)
556 size
*= lane_mask_size
;
557 allocated
[nir_instr_as_ssa_undef(instr
)->def
.index
] = Temp(0, RegClass(RegType::sgpr
, size
));
560 case nir_instr_type_phi
: {
561 nir_phi_instr
* phi
= nir_instr_as_phi(instr
);
563 unsigned size
= phi
->dest
.ssa
.num_components
;
565 if (phi
->dest
.ssa
.bit_size
== 1) {
566 assert(size
== 1 && "multiple components not yet supported on boolean phis.");
567 type
= RegType::sgpr
;
568 size
*= lane_mask_size
;
569 allocated
[phi
->dest
.ssa
.index
] = Temp(0, RegClass(type
, size
));
573 if (ctx
->divergent_vals
[phi
->dest
.ssa
.index
]) {
574 type
= RegType::vgpr
;
576 type
= RegType::sgpr
;
577 nir_foreach_phi_src (src
, phi
) {
578 if (allocated
[src
->src
.ssa
->index
].type() == RegType::vgpr
)
579 type
= RegType::vgpr
;
580 if (allocated
[src
->src
.ssa
->index
].type() == RegType::none
)
585 RegClass rc
= get_reg_class(ctx
, type
, phi
->dest
.ssa
.num_components
, phi
->dest
.ssa
.bit_size
);
586 if (rc
!= allocated
[phi
->dest
.ssa
.index
].regClass()) {
589 nir_foreach_phi_src(src
, phi
)
590 assert(allocated
[src
->src
.ssa
->index
].size() == rc
.size());
592 allocated
[phi
->dest
.ssa
.index
] = Temp(0, rc
);
602 if (G_0286CC_POS_W_FLOAT_ENA(spi_ps_inputs
)) {
603 /* If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be enabled too */
604 spi_ps_inputs
|= S_0286CC_PERSP_CENTER_ENA(1);
607 if (!(spi_ps_inputs
& 0x7F)) {
608 /* At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled */
609 spi_ps_inputs
|= S_0286CC_PERSP_CENTER_ENA(1);
612 ctx
->program
->config
->spi_ps_input_ena
= spi_ps_inputs
;
613 ctx
->program
->config
->spi_ps_input_addr
= spi_ps_inputs
;
615 for (unsigned i
= 0; i
< impl
->ssa_alloc
; i
++)
616 allocated
[i
] = Temp(ctx
->program
->allocateId(), allocated
[i
].regClass());
618 ctx
->allocated
.reset(allocated
.release());
619 ctx
->cf_info
.nir_to_aco
.reset(nir_to_aco
.release());
622 Pseudo_instruction
*add_startpgm(struct isel_context
*ctx
)
624 unsigned arg_count
= ctx
->args
->ac
.arg_count
;
625 if (ctx
->stage
== fragment_fs
) {
626 /* LLVM optimizes away unused FS inputs and computes spi_ps_input_addr
627 * itself and then communicates the results back via the ELF binary.
628 * Mirror what LLVM does by re-mapping the VGPR arguments here.
630 * TODO: If we made the FS input scanning code into a separate pass that
631 * could run before argument setup, then this wouldn't be necessary
634 struct ac_shader_args
*args
= &ctx
->args
->ac
;
636 for (unsigned i
= 0, vgpr_arg
= 0, vgpr_reg
= 0; i
< args
->arg_count
; i
++) {
637 if (args
->args
[i
].file
!= AC_ARG_VGPR
) {
642 if (!(ctx
->program
->config
->spi_ps_input_addr
& (1 << vgpr_arg
))) {
643 args
->args
[i
].skip
= true;
645 args
->args
[i
].offset
= vgpr_reg
;
646 vgpr_reg
+= args
->args
[i
].size
;
653 aco_ptr
<Pseudo_instruction
> startpgm
{create_instruction
<Pseudo_instruction
>(aco_opcode::p_startpgm
, Format::PSEUDO
, 0, arg_count
+ 1)};
654 for (unsigned i
= 0, arg
= 0; i
< ctx
->args
->ac
.arg_count
; i
++) {
655 if (ctx
->args
->ac
.args
[i
].skip
)
658 enum ac_arg_regfile file
= ctx
->args
->ac
.args
[i
].file
;
659 unsigned size
= ctx
->args
->ac
.args
[i
].size
;
660 unsigned reg
= ctx
->args
->ac
.args
[i
].offset
;
661 RegClass type
= RegClass(file
== AC_ARG_SGPR
? RegType::sgpr
: RegType::vgpr
, size
);
662 Temp dst
= Temp
{ctx
->program
->allocateId(), type
};
663 ctx
->arg_temps
[i
] = dst
;
664 startpgm
->definitions
[arg
] = Definition(dst
);
665 startpgm
->definitions
[arg
].setFixed(PhysReg
{file
== AC_ARG_SGPR
? reg
: reg
+ 256});
668 startpgm
->definitions
[arg_count
] = Definition
{ctx
->program
->allocateId(), exec
, ctx
->program
->lane_mask
};
669 Pseudo_instruction
*instr
= startpgm
.get();
670 ctx
->block
->instructions
.push_back(std::move(startpgm
));
672 /* Stash these in the program so that they can be accessed later when
675 ctx
->program
->private_segment_buffer
= get_arg(ctx
, ctx
->args
->ring_offsets
);
676 ctx
->program
->scratch_offset
= get_arg(ctx
, ctx
->args
->scratch_offset
);
682 type_size(const struct glsl_type
*type
, bool bindless
)
684 // TODO: don't we need type->std430_base_alignment() here?
685 return glsl_count_attribute_slots(type
, false);
689 shared_var_info(const struct glsl_type
*type
, unsigned *size
, unsigned *align
)
691 assert(glsl_type_is_vector_or_scalar(type
));
693 uint32_t comp_size
= glsl_type_is_boolean(type
)
694 ? 4 : glsl_get_bit_size(type
) / 8;
695 unsigned length
= glsl_get_vector_elements(type
);
696 *size
= comp_size
* length
,
701 mem_vectorize_callback(unsigned align
, unsigned bit_size
,
702 unsigned num_components
, unsigned high_offset
,
703 nir_intrinsic_instr
*low
, nir_intrinsic_instr
*high
)
705 if ((bit_size
!= 32 && bit_size
!= 64) || num_components
> 4)
708 /* >128 bit loads are split except with SMEM */
709 if (bit_size
* num_components
> 128)
712 switch (low
->intrinsic
) {
713 case nir_intrinsic_store_ssbo
:
714 if (low
->src
[0].ssa
->bit_size
< 32 || high
->src
[0].ssa
->bit_size
< 32)
716 return align
% 4 == 0;
717 case nir_intrinsic_load_ssbo
:
718 if (low
->dest
.ssa
.bit_size
< 32 || high
->dest
.ssa
.bit_size
< 32)
720 case nir_intrinsic_load_ubo
:
721 case nir_intrinsic_load_push_constant
:
722 return align
% 4 == 0;
723 case nir_intrinsic_load_deref
:
724 case nir_intrinsic_store_deref
:
725 assert(nir_src_as_deref(low
->src
[0])->mode
== nir_var_mem_shared
);
727 case nir_intrinsic_load_shared
:
728 case nir_intrinsic_store_shared
:
729 if (bit_size
* num_components
> 64) /* 96 and 128 bit loads require 128 bit alignment and are split otherwise */
730 return align
% 16 == 0;
732 return align
% 4 == 0;
740 setup_vs_output_info(isel_context
*ctx
, nir_shader
*nir
,
741 bool export_prim_id
, bool export_clip_dists
,
742 radv_vs_output_info
*outinfo
)
744 memset(outinfo
->vs_output_param_offset
, AC_EXP_PARAM_UNDEFINED
,
745 sizeof(outinfo
->vs_output_param_offset
));
747 outinfo
->param_exports
= 0;
748 int pos_written
= 0x1;
749 if (outinfo
->writes_pointsize
|| outinfo
->writes_viewport_index
|| outinfo
->writes_layer
)
750 pos_written
|= 1 << 1;
752 uint64_t mask
= ctx
->output_masks
[nir
->info
.stage
];
754 int idx
= u_bit_scan64(&mask
);
755 if (idx
>= VARYING_SLOT_VAR0
|| idx
== VARYING_SLOT_LAYER
|| idx
== VARYING_SLOT_PRIMITIVE_ID
||
756 ((idx
== VARYING_SLOT_CLIP_DIST0
|| idx
== VARYING_SLOT_CLIP_DIST1
) && export_clip_dists
)) {
757 if (outinfo
->vs_output_param_offset
[idx
] == AC_EXP_PARAM_UNDEFINED
)
758 outinfo
->vs_output_param_offset
[idx
] = outinfo
->param_exports
++;
761 if (outinfo
->writes_layer
&&
762 outinfo
->vs_output_param_offset
[VARYING_SLOT_LAYER
] == AC_EXP_PARAM_UNDEFINED
) {
763 /* when ctx->options->key.has_multiview_view_index = true, the layer
764 * variable isn't declared in NIR and it's isel's job to get the layer */
765 outinfo
->vs_output_param_offset
[VARYING_SLOT_LAYER
] = outinfo
->param_exports
++;
768 if (export_prim_id
) {
769 assert(outinfo
->vs_output_param_offset
[VARYING_SLOT_PRIMITIVE_ID
] == AC_EXP_PARAM_UNDEFINED
);
770 outinfo
->vs_output_param_offset
[VARYING_SLOT_PRIMITIVE_ID
] = outinfo
->param_exports
++;
773 ctx
->export_clip_dists
= export_clip_dists
;
774 ctx
->num_clip_distances
= util_bitcount(outinfo
->clip_dist_mask
);
775 ctx
->num_cull_distances
= util_bitcount(outinfo
->cull_dist_mask
);
777 assert(ctx
->num_clip_distances
+ ctx
->num_cull_distances
<= 8);
779 if (ctx
->num_clip_distances
+ ctx
->num_cull_distances
> 0)
780 pos_written
|= 1 << 2;
781 if (ctx
->num_clip_distances
+ ctx
->num_cull_distances
> 4)
782 pos_written
|= 1 << 3;
784 outinfo
->pos_exports
= util_bitcount(pos_written
);
788 setup_vs_variables(isel_context
*ctx
, nir_shader
*nir
)
790 nir_foreach_variable(variable
, &nir
->inputs
)
792 variable
->data
.driver_location
= variable
->data
.location
* 4;
794 nir_foreach_variable(variable
, &nir
->outputs
)
796 if (ctx
->stage
== vertex_geometry_gs
)
797 variable
->data
.driver_location
= util_bitcount64(ctx
->output_masks
[nir
->info
.stage
] & ((1ull << variable
->data
.location
) - 1ull)) * 4;
798 else if (ctx
->stage
== vertex_es
||
799 ctx
->stage
== vertex_ls
||
800 ctx
->stage
== vertex_tess_control_hs
)
801 // TODO: make this more compact
802 variable
->data
.driver_location
= shader_io_get_unique_index((gl_varying_slot
) variable
->data
.location
) * 4;
803 else if (ctx
->stage
== vertex_vs
)
804 variable
->data
.driver_location
= variable
->data
.location
* 4;
806 unreachable("Unsupported VS stage");
809 if (ctx
->stage
== vertex_vs
) {
810 radv_vs_output_info
*outinfo
= &ctx
->program
->info
->vs
.outinfo
;
811 setup_vs_output_info(ctx
, nir
, outinfo
->export_prim_id
,
812 ctx
->options
->key
.vs_common_out
.export_clip_dists
, outinfo
);
813 } else if (ctx
->stage
== vertex_geometry_gs
|| ctx
->stage
== vertex_es
) {
814 /* TODO: radv_nir_shader_info_pass() already sets this but it's larger
815 * than it needs to be in order to set it better, we have to improve
816 * radv_nir_shader_info_pass() because gfx9_get_gs_info() uses
817 * esgs_itemsize and has to be done before compilation
819 /* radv_es_output_info *outinfo = &ctx->program->info->vs.es_info;
820 outinfo->esgs_itemsize = util_bitcount64(ctx->output_masks[nir->info.stage]) * 16u; */
824 void setup_gs_variables(isel_context
*ctx
, nir_shader
*nir
)
826 if (ctx
->stage
== vertex_geometry_gs
|| ctx
->stage
== tess_eval_geometry_gs
) {
827 nir_foreach_variable(variable
, &nir
->inputs
) {
828 variable
->data
.driver_location
= util_bitcount64(ctx
->input_masks
[nir
->info
.stage
] & ((1ull << variable
->data
.location
) - 1ull)) * 4;
830 } else if (ctx
->stage
== geometry_gs
) {
831 //TODO: make this more compact
832 nir_foreach_variable(variable
, &nir
->inputs
) {
833 variable
->data
.driver_location
= shader_io_get_unique_index((gl_varying_slot
)variable
->data
.location
) * 4;
836 unreachable("Unsupported GS stage.");
839 nir_foreach_variable(variable
, &nir
->outputs
) {
840 variable
->data
.driver_location
= variable
->data
.location
* 4;
843 if (ctx
->stage
== vertex_geometry_gs
)
844 ctx
->program
->info
->gs
.es_type
= MESA_SHADER_VERTEX
;
845 else if (ctx
->stage
== tess_eval_geometry_gs
)
846 ctx
->program
->info
->gs
.es_type
= MESA_SHADER_TESS_EVAL
;
850 setup_tcs_info(isel_context
*ctx
, nir_shader
*nir
)
852 /* When the number of TCS input and output vertices are the same (typically 3):
853 * - There is an equal amount of LS and HS invocations
854 * - In case of merged LSHS shaders, the LS and HS halves of the shader
855 * always process the exact same vertex. We can use this knowledge to optimize them.
858 ctx
->stage
== vertex_tess_control_hs
&&
859 ctx
->args
->options
->key
.tcs
.input_vertices
== nir
->info
.tess
.tcs_vertices_out
;
861 if (ctx
->stage
== tess_control_hs
) {
862 ctx
->tcs_num_inputs
= ctx
->args
->options
->key
.tcs
.num_inputs
;
863 } else if (ctx
->stage
== vertex_tess_control_hs
) {
864 ctx
->tcs_num_inputs
= util_last_bit64(ctx
->args
->shader_info
->vs
.ls_outputs_written
);
866 if (ctx
->tcs_in_out_eq
) {
867 ctx
->tcs_temp_only_inputs
= ~nir
->info
.tess
.tcs_cross_invocation_inputs_read
&
868 ~nir
->info
.inputs_read_indirectly
&
869 nir
->info
.inputs_read
;
872 unreachable("Unsupported TCS shader stage");
875 ctx
->tcs_num_patches
= get_tcs_num_patches(
876 ctx
->args
->options
->key
.tcs
.input_vertices
,
877 nir
->info
.tess
.tcs_vertices_out
,
879 ctx
->args
->shader_info
->tcs
.outputs_written
,
880 ctx
->args
->shader_info
->tcs
.patch_outputs_written
,
881 ctx
->args
->options
->tess_offchip_block_dw_size
,
882 ctx
->args
->options
->chip_class
,
883 ctx
->args
->options
->family
);
884 unsigned lds_size
= calculate_tess_lds_size(
885 ctx
->args
->options
->key
.tcs
.input_vertices
,
886 nir
->info
.tess
.tcs_vertices_out
,
888 ctx
->tcs_num_patches
,
889 ctx
->args
->shader_info
->tcs
.outputs_written
,
890 ctx
->args
->shader_info
->tcs
.patch_outputs_written
);
892 ctx
->args
->shader_info
->tcs
.num_patches
= ctx
->tcs_num_patches
;
893 ctx
->args
->shader_info
->tcs
.lds_size
= lds_size
;
894 ctx
->program
->config
->lds_size
= (lds_size
+ ctx
->program
->lds_alloc_granule
- 1) /
895 ctx
->program
->lds_alloc_granule
;
899 setup_tcs_variables(isel_context
*ctx
, nir_shader
*nir
)
901 nir_foreach_variable(variable
, &nir
->inputs
) {
902 variable
->data
.driver_location
= shader_io_get_unique_index((gl_varying_slot
) variable
->data
.location
) * 4;
905 nir_foreach_variable(variable
, &nir
->outputs
) {
906 variable
->data
.driver_location
= shader_io_get_unique_index((gl_varying_slot
) variable
->data
.location
) * 4;
909 ctx
->tcs_tess_lvl_out_loc
= shader_io_get_unique_index(VARYING_SLOT_TESS_LEVEL_OUTER
) * 16u;
910 ctx
->tcs_tess_lvl_in_loc
= shader_io_get_unique_index(VARYING_SLOT_TESS_LEVEL_INNER
) * 16u;
914 setup_tes_variables(isel_context
*ctx
, nir_shader
*nir
)
916 ctx
->tcs_num_patches
= ctx
->args
->options
->key
.tes
.num_patches
;
918 nir_foreach_variable(variable
, &nir
->inputs
) {
919 variable
->data
.driver_location
= shader_io_get_unique_index((gl_varying_slot
) variable
->data
.location
) * 4;
922 nir_foreach_variable(variable
, &nir
->outputs
) {
923 if (ctx
->stage
== tess_eval_vs
)
924 variable
->data
.driver_location
= variable
->data
.location
* 4;
925 else if (ctx
->stage
== tess_eval_es
)
926 variable
->data
.driver_location
= shader_io_get_unique_index((gl_varying_slot
) variable
->data
.location
) * 4;
927 else if (ctx
->stage
== tess_eval_geometry_gs
)
928 variable
->data
.driver_location
= util_bitcount64(ctx
->output_masks
[nir
->info
.stage
] & ((1ull << variable
->data
.location
) - 1ull)) * 4;
930 unreachable("Unsupported TES shader stage");
933 if (ctx
->stage
== tess_eval_vs
) {
934 radv_vs_output_info
*outinfo
= &ctx
->program
->info
->tes
.outinfo
;
935 setup_vs_output_info(ctx
, nir
, outinfo
->export_prim_id
,
936 ctx
->options
->key
.vs_common_out
.export_clip_dists
, outinfo
);
941 setup_variables(isel_context
*ctx
, nir_shader
*nir
)
943 switch (nir
->info
.stage
) {
944 case MESA_SHADER_FRAGMENT
: {
945 nir_foreach_variable(variable
, &nir
->outputs
)
947 int idx
= variable
->data
.location
+ variable
->data
.index
;
948 variable
->data
.driver_location
= idx
* 4;
952 case MESA_SHADER_COMPUTE
: {
953 ctx
->program
->config
->lds_size
= (nir
->info
.cs
.shared_size
+ ctx
->program
->lds_alloc_granule
- 1) /
954 ctx
->program
->lds_alloc_granule
;
957 case MESA_SHADER_VERTEX
: {
958 setup_vs_variables(ctx
, nir
);
961 case MESA_SHADER_GEOMETRY
: {
962 setup_gs_variables(ctx
, nir
);
965 case MESA_SHADER_TESS_CTRL
: {
966 setup_tcs_variables(ctx
, nir
);
969 case MESA_SHADER_TESS_EVAL
: {
970 setup_tes_variables(ctx
, nir
);
974 unreachable("Unhandled shader stage.");
979 get_io_masks(isel_context
*ctx
, unsigned shader_count
, struct nir_shader
*const *shaders
)
981 for (unsigned i
= 0; i
< shader_count
; i
++) {
982 nir_shader
*nir
= shaders
[i
];
983 if (nir
->info
.stage
== MESA_SHADER_COMPUTE
)
986 uint64_t output_mask
= 0;
987 nir_foreach_variable(variable
, &nir
->outputs
) {
988 const glsl_type
*type
= variable
->type
;
989 if (nir_is_per_vertex_io(variable
, nir
->info
.stage
))
990 type
= type
->fields
.array
;
991 unsigned slots
= type
->count_attribute_slots(false);
992 if (variable
->data
.compact
) {
993 unsigned component_count
= variable
->data
.location_frac
+ type
->length
;
994 slots
= (component_count
+ 3) / 4;
996 output_mask
|= ((1ull << slots
) - 1) << variable
->data
.location
;
999 uint64_t input_mask
= 0;
1000 nir_foreach_variable(variable
, &nir
->inputs
) {
1001 const glsl_type
*type
= variable
->type
;
1002 if (nir_is_per_vertex_io(variable
, nir
->info
.stage
))
1003 type
= type
->fields
.array
;
1004 unsigned slots
= type
->count_attribute_slots(false);
1005 if (variable
->data
.compact
) {
1006 unsigned component_count
= variable
->data
.location_frac
+ type
->length
;
1007 slots
= (component_count
+ 3) / 4;
1009 input_mask
|= ((1ull << slots
) - 1) << variable
->data
.location
;
1012 ctx
->output_masks
[nir
->info
.stage
] |= output_mask
;
1013 if (i
+ 1 < shader_count
)
1014 ctx
->input_masks
[shaders
[i
+ 1]->info
.stage
] |= output_mask
;
1016 ctx
->input_masks
[nir
->info
.stage
] |= input_mask
;
1018 ctx
->output_masks
[shaders
[i
- 1]->info
.stage
] |= input_mask
;
1023 setup_nir(isel_context
*ctx
, nir_shader
*nir
)
1025 Program
*program
= ctx
->program
;
1027 /* align and copy constant data */
1028 while (program
->constant_data
.size() % 4u)
1029 program
->constant_data
.push_back(0);
1030 ctx
->constant_data_offset
= program
->constant_data
.size();
1031 program
->constant_data
.insert(program
->constant_data
.end(),
1032 (uint8_t*)nir
->constant_data
,
1033 (uint8_t*)nir
->constant_data
+ nir
->constant_data_size
);
1035 /* the variable setup has to be done before lower_io / CSE */
1036 setup_variables(ctx
, nir
);
1038 /* optimize and lower memory operations */
1039 bool lower_to_scalar
= false;
1040 bool lower_pack
= false;
1041 if (nir_opt_load_store_vectorize(nir
,
1042 (nir_variable_mode
)(nir_var_mem_ssbo
| nir_var_mem_ubo
|
1043 nir_var_mem_push_const
| nir_var_mem_shared
),
1044 mem_vectorize_callback
)) {
1045 lower_to_scalar
= true;
1048 if (nir
->info
.stage
!= MESA_SHADER_COMPUTE
)
1049 nir_lower_io(nir
, (nir_variable_mode
)(nir_var_shader_in
| nir_var_shader_out
), type_size
, (nir_lower_io_options
)0);
1050 nir_lower_explicit_io(nir
, nir_var_mem_global
, nir_address_format_64bit_global
);
1052 if (lower_to_scalar
)
1053 nir_lower_alu_to_scalar(nir
, NULL
, NULL
);
1055 nir_lower_pack(nir
);
1057 /* lower ALU operations */
1058 // TODO: implement logic64 in aco, it's more effective for sgprs
1059 nir_lower_int64(nir
, nir
->options
->lower_int64_options
);
1061 nir_opt_idiv_const(nir
, 32);
1062 nir_lower_idiv(nir
, nir_lower_idiv_precise
);
1064 /* optimize the lowered ALU operations */
1065 bool more_algebraic
= true;
1066 while (more_algebraic
) {
1067 more_algebraic
= false;
1068 NIR_PASS_V(nir
, nir_copy_prop
);
1069 NIR_PASS_V(nir
, nir_opt_dce
);
1070 NIR_PASS_V(nir
, nir_opt_constant_folding
);
1071 NIR_PASS(more_algebraic
, nir
, nir_opt_algebraic
);
1074 /* Do late algebraic optimization to turn add(a, neg(b)) back into
1075 * subs, then the mandatory cleanup after algebraic. Note that it may
1076 * produce fnegs, and if so then we need to keep running to squash
1079 bool more_late_algebraic
= true;
1080 while (more_late_algebraic
) {
1081 more_late_algebraic
= false;
1082 NIR_PASS(more_late_algebraic
, nir
, nir_opt_algebraic_late
);
1083 NIR_PASS_V(nir
, nir_opt_constant_folding
);
1084 NIR_PASS_V(nir
, nir_copy_prop
);
1085 NIR_PASS_V(nir
, nir_opt_dce
);
1086 NIR_PASS_V(nir
, nir_opt_cse
);
1089 /* cleanup passes */
1090 nir_lower_load_const_to_scalar(nir
);
1091 nir_opt_shrink_load(nir
);
1092 nir_move_options move_opts
= (nir_move_options
)(
1093 nir_move_const_undef
| nir_move_load_ubo
| nir_move_load_input
|
1094 nir_move_comparisons
| nir_move_copies
);
1095 nir_opt_sink(nir
, move_opts
);
1096 nir_opt_move(nir
, move_opts
);
1097 nir_convert_to_lcssa(nir
, true, false);
1098 nir_lower_phis_to_scalar(nir
);
1100 nir_function_impl
*func
= nir_shader_get_entrypoint(nir
);
1101 nir_index_ssa_defs(func
);
1105 setup_xnack(Program
*program
)
1107 switch (program
->family
) {
1115 program
->xnack_enabled
= true;
1123 setup_isel_context(Program
* program
,
1124 unsigned shader_count
,
1125 struct nir_shader
*const *shaders
,
1126 ac_shader_config
* config
,
1127 struct radv_shader_args
*args
,
1128 bool is_gs_copy_shader
)
1131 for (unsigned i
= 0; i
< shader_count
; i
++) {
1132 switch (shaders
[i
]->info
.stage
) {
1133 case MESA_SHADER_VERTEX
:
1134 program
->stage
|= sw_vs
;
1136 case MESA_SHADER_TESS_CTRL
:
1137 program
->stage
|= sw_tcs
;
1139 case MESA_SHADER_TESS_EVAL
:
1140 program
->stage
|= sw_tes
;
1142 case MESA_SHADER_GEOMETRY
:
1143 program
->stage
|= is_gs_copy_shader
? sw_gs_copy
: sw_gs
;
1145 case MESA_SHADER_FRAGMENT
:
1146 program
->stage
|= sw_fs
;
1148 case MESA_SHADER_COMPUTE
:
1149 program
->stage
|= sw_cs
;
1152 unreachable("Shader stage not implemented");
1155 bool gfx9_plus
= args
->options
->chip_class
>= GFX9
;
1156 bool ngg
= args
->shader_info
->is_ngg
&& args
->options
->chip_class
>= GFX10
;
1157 if (program
->stage
== sw_vs
&& args
->shader_info
->vs
.as_es
)
1158 program
->stage
|= hw_es
;
1159 else if (program
->stage
== sw_vs
&& !args
->shader_info
->vs
.as_ls
)
1160 program
->stage
|= hw_vs
;
1161 else if (program
->stage
== sw_gs
)
1162 program
->stage
|= hw_gs
;
1163 else if (program
->stage
== sw_fs
)
1164 program
->stage
|= hw_fs
;
1165 else if (program
->stage
== sw_cs
)
1166 program
->stage
|= hw_cs
;
1167 else if (program
->stage
== sw_gs_copy
)
1168 program
->stage
|= hw_vs
;
1169 else if (program
->stage
== (sw_vs
| sw_gs
) && gfx9_plus
&& !ngg
)
1170 program
->stage
|= hw_gs
;
1171 else if (program
->stage
== sw_vs
&& args
->shader_info
->vs
.as_ls
)
1172 program
->stage
|= hw_ls
; /* GFX6-8: VS is a Local Shader, when tessellation is used */
1173 else if (program
->stage
== sw_tcs
)
1174 program
->stage
|= hw_hs
; /* GFX6-8: TCS is a Hull Shader */
1175 else if (program
->stage
== (sw_vs
| sw_tcs
))
1176 program
->stage
|= hw_hs
; /* GFX9-10: VS+TCS merged into a Hull Shader */
1177 else if (program
->stage
== sw_tes
&& !args
->shader_info
->tes
.as_es
&& !ngg
)
1178 program
->stage
|= hw_vs
; /* GFX6-9: TES without GS uses the HW VS stage (and GFX10/legacy) */
1179 else if (program
->stage
== sw_tes
&& args
->shader_info
->tes
.as_es
&& !ngg
)
1180 program
->stage
|= hw_es
; /* GFX6-8: TES is an Export Shader */
1181 else if (program
->stage
== (sw_tes
| sw_gs
) && gfx9_plus
&& !ngg
)
1182 program
->stage
|= hw_gs
; /* GFX9: TES+GS merged into a GS (and GFX10/legacy) */
1184 unreachable("Shader stage not implemented");
1186 program
->config
= config
;
1187 program
->info
= args
->shader_info
;
1188 program
->chip_class
= args
->options
->chip_class
;
1189 program
->family
= args
->options
->family
;
1190 program
->wave_size
= args
->shader_info
->wave_size
;
1191 program
->lane_mask
= program
->wave_size
== 32 ? s1
: s2
;
1193 program
->lds_alloc_granule
= args
->options
->chip_class
>= GFX7
? 512 : 256;
1194 program
->lds_limit
= args
->options
->chip_class
>= GFX7
? 65536 : 32768;
1195 /* apparently gfx702 also has 16-bank LDS but I can't find a family for that */
1196 program
->has_16bank_lds
= args
->options
->family
== CHIP_KABINI
|| args
->options
->family
== CHIP_STONEY
;
1198 program
->vgpr_limit
= 256;
1199 program
->vgpr_alloc_granule
= 3;
1201 if (args
->options
->chip_class
>= GFX10
) {
1202 program
->physical_sgprs
= 2560; /* doesn't matter as long as it's at least 128 * 20 */
1203 program
->sgpr_alloc_granule
= 127;
1204 program
->sgpr_limit
= 106;
1205 program
->vgpr_alloc_granule
= program
->wave_size
== 32 ? 7 : 3;
1206 } else if (program
->chip_class
>= GFX8
) {
1207 program
->physical_sgprs
= 800;
1208 program
->sgpr_alloc_granule
= 15;
1209 if (args
->options
->family
== CHIP_TONGA
|| args
->options
->family
== CHIP_ICELAND
)
1210 program
->sgpr_limit
= 94; /* workaround hardware bug */
1212 program
->sgpr_limit
= 102;
1214 program
->physical_sgprs
= 512;
1215 program
->sgpr_alloc_granule
= 7;
1216 program
->sgpr_limit
= 104;
1219 isel_context ctx
= {};
1220 ctx
.program
= program
;
1222 ctx
.options
= args
->options
;
1223 ctx
.stage
= program
->stage
;
1225 /* TODO: Check if we need to adjust min_waves for unknown workgroup sizes. */
1226 if (program
->stage
& (hw_vs
| hw_fs
)) {
1227 /* PS and legacy VS have separate waves, no workgroups */
1228 program
->workgroup_size
= program
->wave_size
;
1229 } else if (program
->stage
== compute_cs
) {
1230 /* CS sets the workgroup size explicitly */
1231 unsigned* bsize
= program
->info
->cs
.block_size
;
1232 program
->workgroup_size
= bsize
[0] * bsize
[1] * bsize
[2];
1233 } else if ((program
->stage
& hw_es
) || program
->stage
== geometry_gs
) {
1234 /* Unmerged ESGS operate in workgroups if on-chip GS (LDS rings) are enabled on GFX7-8 (not implemented in Mesa) */
1235 program
->workgroup_size
= program
->wave_size
;
1236 } else if (program
->stage
& hw_gs
) {
1237 /* If on-chip GS (LDS rings) are enabled on GFX9 or later, merged GS operates in workgroups */
1238 program
->workgroup_size
= UINT_MAX
; /* TODO: set by VGT_GS_ONCHIP_CNTL, which is not plumbed to ACO */
1239 } else if (program
->stage
== vertex_ls
) {
1240 /* Unmerged LS operates in workgroups */
1241 program
->workgroup_size
= UINT_MAX
; /* TODO: probably tcs_num_patches * tcs_vertices_in, but those are not plumbed to ACO for LS */
1242 } else if (program
->stage
== tess_control_hs
) {
1243 /* Unmerged HS operates in workgroups, size is determined by the output vertices */
1244 setup_tcs_info(&ctx
, shaders
[0]);
1245 program
->workgroup_size
= ctx
.tcs_num_patches
* shaders
[0]->info
.tess
.tcs_vertices_out
;
1246 } else if (program
->stage
== vertex_tess_control_hs
) {
1247 /* Merged LSHS operates in workgroups, but can still have a different number of LS and HS invocations */
1248 setup_tcs_info(&ctx
, shaders
[1]);
1249 program
->workgroup_size
= ctx
.tcs_num_patches
* MAX2(shaders
[1]->info
.tess
.tcs_vertices_out
, ctx
.args
->options
->key
.tcs
.input_vertices
);
1251 unreachable("Unsupported shader stage.");
1254 calc_min_waves(program
);
1255 program
->vgpr_limit
= get_addr_vgpr_from_waves(program
, program
->min_waves
);
1256 program
->sgpr_limit
= get_addr_sgpr_from_waves(program
, program
->min_waves
);
1258 get_io_masks(&ctx
, shader_count
, shaders
);
1260 unsigned scratch_size
= 0;
1261 if (program
->stage
== gs_copy_vs
) {
1262 assert(shader_count
== 1);
1263 setup_vs_output_info(&ctx
, shaders
[0], false, true, &args
->shader_info
->vs
.outinfo
);
1265 for (unsigned i
= 0; i
< shader_count
; i
++) {
1266 nir_shader
*nir
= shaders
[i
];
1267 setup_nir(&ctx
, nir
);
1270 for (unsigned i
= 0; i
< shader_count
; i
++)
1271 scratch_size
= std::max(scratch_size
, shaders
[i
]->scratch_size
);
1274 ctx
.program
->config
->scratch_bytes_per_wave
= align(scratch_size
* ctx
.program
->wave_size
, 1024);
1276 ctx
.block
= ctx
.program
->create_and_insert_block();
1277 ctx
.block
->loop_nest_depth
= 0;
1278 ctx
.block
->kind
= block_kind_top_level
;
1280 setup_xnack(program
);