2 * Copyright © 2018 Valve Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
26 #include <unordered_map>
29 #include "vulkan/radv_shader.h"
30 #include "vulkan/radv_descriptor_set.h"
31 #include "vulkan/radv_shader_args.h"
33 #include "ac_exp_param.h"
34 #include "ac_shader_util.h"
36 #include "util/u_math.h"
38 #define MAX_INLINE_PUSH_CONSTS 8
43 uint8_t mask
[VARYING_SLOT_VAR31
+ 1];
44 Temp outputs
[VARYING_SLOT_VAR31
+ 1][4];
48 const struct radv_nir_compiler_options
*options
;
49 struct radv_shader_args
*args
;
52 uint32_t constant_data_offset
;
55 std::unique_ptr
<Temp
[]> allocated
;
56 std::unordered_map
<unsigned, std::array
<Temp
,NIR_MAX_VEC_COMPONENTS
>> allocated_vec
;
57 Stage stage
; /* Stage */
58 bool has_gfx10_wave64_bpermute
= false;
61 uint16_t loop_nest_depth
= 0;
65 bool has_divergent_continue
= false;
66 bool has_divergent_branch
= false;
69 bool is_divergent
= false;
71 bool exec_potentially_empty
= false;
72 std::unique_ptr
<unsigned[]> nir_to_aco
; /* NIR block index to ACO block index */
75 Temp arg_temps
[AC_MAX_ARGS
];
78 Temp persp_centroid
, linear_centroid
;
83 /* gathered information */
84 uint64_t input_masks
[MESA_SHADER_COMPUTE
];
85 uint64_t output_masks
[MESA_SHADER_COMPUTE
];
87 /* VS output information */
88 bool export_clip_dists
;
89 unsigned num_clip_distances
;
90 unsigned num_cull_distances
;
92 /* VS, FS or GS output information */
96 Temp
get_arg(isel_context
*ctx
, struct ac_arg arg
)
99 return ctx
->arg_temps
[arg
.arg_index
];
102 unsigned get_interp_input(nir_intrinsic_op intrin
, enum glsl_interp_mode interp
)
105 case INTERP_MODE_SMOOTH
:
106 case INTERP_MODE_NONE
:
107 if (intrin
== nir_intrinsic_load_barycentric_pixel
||
108 intrin
== nir_intrinsic_load_barycentric_at_sample
||
109 intrin
== nir_intrinsic_load_barycentric_at_offset
)
110 return S_0286CC_PERSP_CENTER_ENA(1);
111 else if (intrin
== nir_intrinsic_load_barycentric_centroid
)
112 return S_0286CC_PERSP_CENTROID_ENA(1);
113 else if (intrin
== nir_intrinsic_load_barycentric_sample
)
114 return S_0286CC_PERSP_SAMPLE_ENA(1);
116 case INTERP_MODE_NOPERSPECTIVE
:
117 if (intrin
== nir_intrinsic_load_barycentric_pixel
)
118 return S_0286CC_LINEAR_CENTER_ENA(1);
119 else if (intrin
== nir_intrinsic_load_barycentric_centroid
)
120 return S_0286CC_LINEAR_CENTROID_ENA(1);
121 else if (intrin
== nir_intrinsic_load_barycentric_sample
)
122 return S_0286CC_LINEAR_SAMPLE_ENA(1);
130 void init_context(isel_context
*ctx
, nir_shader
*shader
)
132 nir_function_impl
*impl
= nir_shader_get_entrypoint(shader
);
133 unsigned lane_mask_size
= ctx
->program
->lane_mask
.size();
135 ctx
->shader
= shader
;
136 ctx
->divergent_vals
= nir_divergence_analysis(shader
, nir_divergence_view_index_uniform
);
138 std::unique_ptr
<Temp
[]> allocated
{new Temp
[impl
->ssa_alloc
]()};
140 unsigned spi_ps_inputs
= 0;
142 std::unique_ptr
<unsigned[]> nir_to_aco
{new unsigned[impl
->num_blocks
]()};
147 nir_foreach_block(block
, impl
) {
148 nir_foreach_instr(instr
, block
) {
149 switch(instr
->type
) {
150 case nir_instr_type_alu
: {
151 nir_alu_instr
*alu_instr
= nir_instr_as_alu(instr
);
152 unsigned size
= alu_instr
->dest
.dest
.ssa
.num_components
;
153 if (alu_instr
->dest
.dest
.ssa
.bit_size
== 64)
155 RegType type
= RegType::sgpr
;
156 switch(alu_instr
->op
) {
178 case nir_op_fround_even
:
187 case nir_op_pack_half_2x16
:
188 case nir_op_unpack_half_2x16_split_x
:
189 case nir_op_unpack_half_2x16_split_y
:
192 case nir_op_fddx_fine
:
193 case nir_op_fddy_fine
:
194 case nir_op_fddx_coarse
:
195 case nir_op_fddy_coarse
:
196 case nir_op_fquantize2f16
:
198 case nir_op_frexp_sig
:
199 case nir_op_frexp_exp
:
200 case nir_op_cube_face_index
:
201 case nir_op_cube_face_coord
:
202 type
= RegType::vgpr
;
215 size
= lane_mask_size
;
223 type
= ctx
->divergent_vals
[alu_instr
->dest
.dest
.ssa
.index
] ? RegType::vgpr
: RegType::sgpr
;
226 if (alu_instr
->dest
.dest
.ssa
.bit_size
== 1) {
227 size
= lane_mask_size
;
229 if (ctx
->divergent_vals
[alu_instr
->dest
.dest
.ssa
.index
]) {
230 type
= RegType::vgpr
;
232 if (allocated
[alu_instr
->src
[1].src
.ssa
->index
].type() == RegType::vgpr
||
233 allocated
[alu_instr
->src
[2].src
.ssa
->index
].type() == RegType::vgpr
) {
234 type
= RegType::vgpr
;
237 if (alu_instr
->src
[1].src
.ssa
->num_components
== 1 && alu_instr
->src
[2].src
.ssa
->num_components
== 1) {
238 assert(allocated
[alu_instr
->src
[1].src
.ssa
->index
].size() == allocated
[alu_instr
->src
[2].src
.ssa
->index
].size());
239 size
= allocated
[alu_instr
->src
[1].src
.ssa
->index
].size();
244 if (alu_instr
->dest
.dest
.ssa
.bit_size
== 1) {
245 size
= lane_mask_size
;
247 type
= ctx
->divergent_vals
[alu_instr
->dest
.dest
.ssa
.index
] ? RegType::vgpr
: RegType::sgpr
;
251 if (alu_instr
->dest
.dest
.ssa
.bit_size
== 1) {
252 size
= lane_mask_size
;
254 for (unsigned i
= 0; i
< nir_op_infos
[alu_instr
->op
].num_inputs
; i
++) {
255 if (allocated
[alu_instr
->src
[i
].src
.ssa
->index
].type() == RegType::vgpr
)
256 type
= RegType::vgpr
;
261 allocated
[alu_instr
->dest
.dest
.ssa
.index
] = Temp(0, RegClass(type
, size
));
264 case nir_instr_type_load_const
: {
265 unsigned size
= nir_instr_as_load_const(instr
)->def
.num_components
;
266 if (nir_instr_as_load_const(instr
)->def
.bit_size
== 64)
268 else if (nir_instr_as_load_const(instr
)->def
.bit_size
== 1)
269 size
*= lane_mask_size
;
270 allocated
[nir_instr_as_load_const(instr
)->def
.index
] = Temp(0, RegClass(RegType::sgpr
, size
));
273 case nir_instr_type_intrinsic
: {
274 nir_intrinsic_instr
*intrinsic
= nir_instr_as_intrinsic(instr
);
275 if (!nir_intrinsic_infos
[intrinsic
->intrinsic
].has_dest
)
277 unsigned size
= intrinsic
->dest
.ssa
.num_components
;
278 if (intrinsic
->dest
.ssa
.bit_size
== 64)
280 RegType type
= RegType::sgpr
;
281 switch(intrinsic
->intrinsic
) {
282 case nir_intrinsic_load_push_constant
:
283 case nir_intrinsic_load_work_group_id
:
284 case nir_intrinsic_load_num_work_groups
:
285 case nir_intrinsic_load_subgroup_id
:
286 case nir_intrinsic_load_num_subgroups
:
287 case nir_intrinsic_load_first_vertex
:
288 case nir_intrinsic_load_base_instance
:
289 case nir_intrinsic_get_buffer_size
:
290 case nir_intrinsic_vote_all
:
291 case nir_intrinsic_vote_any
:
292 case nir_intrinsic_read_first_invocation
:
293 case nir_intrinsic_read_invocation
:
294 case nir_intrinsic_first_invocation
:
295 type
= RegType::sgpr
;
296 if (intrinsic
->dest
.ssa
.bit_size
== 1)
297 size
= lane_mask_size
;
299 case nir_intrinsic_ballot
:
300 type
= RegType::sgpr
;
302 case nir_intrinsic_load_sample_id
:
303 case nir_intrinsic_load_sample_mask_in
:
304 case nir_intrinsic_load_input
:
305 case nir_intrinsic_load_input_vertex
:
306 case nir_intrinsic_load_per_vertex_input
:
307 case nir_intrinsic_load_vertex_id
:
308 case nir_intrinsic_load_vertex_id_zero_base
:
309 case nir_intrinsic_load_barycentric_sample
:
310 case nir_intrinsic_load_barycentric_pixel
:
311 case nir_intrinsic_load_barycentric_model
:
312 case nir_intrinsic_load_barycentric_centroid
:
313 case nir_intrinsic_load_barycentric_at_sample
:
314 case nir_intrinsic_load_barycentric_at_offset
:
315 case nir_intrinsic_load_interpolated_input
:
316 case nir_intrinsic_load_frag_coord
:
317 case nir_intrinsic_load_sample_pos
:
318 case nir_intrinsic_load_layer_id
:
319 case nir_intrinsic_load_local_invocation_id
:
320 case nir_intrinsic_load_local_invocation_index
:
321 case nir_intrinsic_load_subgroup_invocation
:
322 case nir_intrinsic_write_invocation_amd
:
323 case nir_intrinsic_mbcnt_amd
:
324 case nir_intrinsic_load_instance_id
:
325 case nir_intrinsic_ssbo_atomic_add
:
326 case nir_intrinsic_ssbo_atomic_imin
:
327 case nir_intrinsic_ssbo_atomic_umin
:
328 case nir_intrinsic_ssbo_atomic_imax
:
329 case nir_intrinsic_ssbo_atomic_umax
:
330 case nir_intrinsic_ssbo_atomic_and
:
331 case nir_intrinsic_ssbo_atomic_or
:
332 case nir_intrinsic_ssbo_atomic_xor
:
333 case nir_intrinsic_ssbo_atomic_exchange
:
334 case nir_intrinsic_ssbo_atomic_comp_swap
:
335 case nir_intrinsic_global_atomic_add
:
336 case nir_intrinsic_global_atomic_imin
:
337 case nir_intrinsic_global_atomic_umin
:
338 case nir_intrinsic_global_atomic_imax
:
339 case nir_intrinsic_global_atomic_umax
:
340 case nir_intrinsic_global_atomic_and
:
341 case nir_intrinsic_global_atomic_or
:
342 case nir_intrinsic_global_atomic_xor
:
343 case nir_intrinsic_global_atomic_exchange
:
344 case nir_intrinsic_global_atomic_comp_swap
:
345 case nir_intrinsic_image_deref_atomic_add
:
346 case nir_intrinsic_image_deref_atomic_umin
:
347 case nir_intrinsic_image_deref_atomic_imin
:
348 case nir_intrinsic_image_deref_atomic_umax
:
349 case nir_intrinsic_image_deref_atomic_imax
:
350 case nir_intrinsic_image_deref_atomic_and
:
351 case nir_intrinsic_image_deref_atomic_or
:
352 case nir_intrinsic_image_deref_atomic_xor
:
353 case nir_intrinsic_image_deref_atomic_exchange
:
354 case nir_intrinsic_image_deref_atomic_comp_swap
:
355 case nir_intrinsic_image_deref_size
:
356 case nir_intrinsic_shared_atomic_add
:
357 case nir_intrinsic_shared_atomic_imin
:
358 case nir_intrinsic_shared_atomic_umin
:
359 case nir_intrinsic_shared_atomic_imax
:
360 case nir_intrinsic_shared_atomic_umax
:
361 case nir_intrinsic_shared_atomic_and
:
362 case nir_intrinsic_shared_atomic_or
:
363 case nir_intrinsic_shared_atomic_xor
:
364 case nir_intrinsic_shared_atomic_exchange
:
365 case nir_intrinsic_shared_atomic_comp_swap
:
366 case nir_intrinsic_load_scratch
:
367 case nir_intrinsic_load_invocation_id
:
368 case nir_intrinsic_load_primitive_id
:
369 type
= RegType::vgpr
;
371 case nir_intrinsic_shuffle
:
372 case nir_intrinsic_quad_broadcast
:
373 case nir_intrinsic_quad_swap_horizontal
:
374 case nir_intrinsic_quad_swap_vertical
:
375 case nir_intrinsic_quad_swap_diagonal
:
376 case nir_intrinsic_quad_swizzle_amd
:
377 case nir_intrinsic_masked_swizzle_amd
:
378 case nir_intrinsic_inclusive_scan
:
379 case nir_intrinsic_exclusive_scan
:
380 if (intrinsic
->dest
.ssa
.bit_size
== 1) {
381 size
= lane_mask_size
;
382 type
= RegType::sgpr
;
383 } else if (!ctx
->divergent_vals
[intrinsic
->dest
.ssa
.index
]) {
384 type
= RegType::sgpr
;
386 type
= RegType::vgpr
;
389 case nir_intrinsic_load_view_index
:
390 type
= ctx
->stage
== fragment_fs
? RegType::vgpr
: RegType::sgpr
;
392 case nir_intrinsic_load_front_face
:
393 case nir_intrinsic_load_helper_invocation
:
394 case nir_intrinsic_is_helper_invocation
:
395 type
= RegType::sgpr
;
396 size
= lane_mask_size
;
398 case nir_intrinsic_reduce
:
399 if (intrinsic
->dest
.ssa
.bit_size
== 1) {
400 size
= lane_mask_size
;
401 type
= RegType::sgpr
;
402 } else if (!ctx
->divergent_vals
[intrinsic
->dest
.ssa
.index
]) {
403 type
= RegType::sgpr
;
405 type
= RegType::vgpr
;
408 case nir_intrinsic_load_ubo
:
409 case nir_intrinsic_load_ssbo
:
410 case nir_intrinsic_load_global
:
411 case nir_intrinsic_vulkan_resource_index
:
412 type
= ctx
->divergent_vals
[intrinsic
->dest
.ssa
.index
] ? RegType::vgpr
: RegType::sgpr
;
414 /* due to copy propagation, the swizzled imov is removed if num dest components == 1 */
415 case nir_intrinsic_load_shared
:
416 if (ctx
->divergent_vals
[intrinsic
->dest
.ssa
.index
])
417 type
= RegType::vgpr
;
419 type
= RegType::sgpr
;
422 for (unsigned i
= 0; i
< nir_intrinsic_infos
[intrinsic
->intrinsic
].num_srcs
; i
++) {
423 if (allocated
[intrinsic
->src
[i
].ssa
->index
].type() == RegType::vgpr
)
424 type
= RegType::vgpr
;
428 allocated
[intrinsic
->dest
.ssa
.index
] = Temp(0, RegClass(type
, size
));
430 switch(intrinsic
->intrinsic
) {
431 case nir_intrinsic_load_barycentric_sample
:
432 case nir_intrinsic_load_barycentric_pixel
:
433 case nir_intrinsic_load_barycentric_centroid
:
434 case nir_intrinsic_load_barycentric_at_sample
:
435 case nir_intrinsic_load_barycentric_at_offset
: {
436 glsl_interp_mode mode
= (glsl_interp_mode
)nir_intrinsic_interp_mode(intrinsic
);
437 spi_ps_inputs
|= get_interp_input(intrinsic
->intrinsic
, mode
);
440 case nir_intrinsic_load_barycentric_model
:
441 spi_ps_inputs
|= S_0286CC_PERSP_PULL_MODEL_ENA(1);
443 case nir_intrinsic_load_front_face
:
444 spi_ps_inputs
|= S_0286CC_FRONT_FACE_ENA(1);
446 case nir_intrinsic_load_frag_coord
:
447 case nir_intrinsic_load_sample_pos
: {
448 uint8_t mask
= nir_ssa_def_components_read(&intrinsic
->dest
.ssa
);
449 for (unsigned i
= 0; i
< 4; i
++) {
451 spi_ps_inputs
|= S_0286CC_POS_X_FLOAT_ENA(1) << i
;
456 case nir_intrinsic_load_sample_id
:
457 spi_ps_inputs
|= S_0286CC_ANCILLARY_ENA(1);
459 case nir_intrinsic_load_sample_mask_in
:
460 spi_ps_inputs
|= S_0286CC_ANCILLARY_ENA(1);
461 spi_ps_inputs
|= S_0286CC_SAMPLE_COVERAGE_ENA(1);
468 case nir_instr_type_tex
: {
469 nir_tex_instr
* tex
= nir_instr_as_tex(instr
);
470 unsigned size
= tex
->dest
.ssa
.num_components
;
472 if (tex
->dest
.ssa
.bit_size
== 64)
474 if (tex
->op
== nir_texop_texture_samples
)
475 assert(!ctx
->divergent_vals
[tex
->dest
.ssa
.index
]);
476 if (ctx
->divergent_vals
[tex
->dest
.ssa
.index
])
477 allocated
[tex
->dest
.ssa
.index
] = Temp(0, RegClass(RegType::vgpr
, size
));
479 allocated
[tex
->dest
.ssa
.index
] = Temp(0, RegClass(RegType::sgpr
, size
));
482 case nir_instr_type_parallel_copy
: {
483 nir_foreach_parallel_copy_entry(entry
, nir_instr_as_parallel_copy(instr
)) {
484 allocated
[entry
->dest
.ssa
.index
] = allocated
[entry
->src
.ssa
->index
];
488 case nir_instr_type_ssa_undef
: {
489 unsigned size
= nir_instr_as_ssa_undef(instr
)->def
.num_components
;
490 if (nir_instr_as_ssa_undef(instr
)->def
.bit_size
== 64)
492 allocated
[nir_instr_as_ssa_undef(instr
)->def
.index
] = Temp(0, RegClass(RegType::sgpr
, size
));
495 case nir_instr_type_phi
: {
496 nir_phi_instr
* phi
= nir_instr_as_phi(instr
);
498 unsigned size
= phi
->dest
.ssa
.num_components
;
500 if (phi
->dest
.ssa
.bit_size
== 1) {
501 assert(size
== 1 && "multiple components not yet supported on boolean phis.");
502 type
= RegType::sgpr
;
503 size
*= lane_mask_size
;
504 allocated
[phi
->dest
.ssa
.index
] = Temp(0, RegClass(type
, size
));
508 if (ctx
->divergent_vals
[phi
->dest
.ssa
.index
]) {
509 type
= RegType::vgpr
;
511 type
= RegType::sgpr
;
512 nir_foreach_phi_src (src
, phi
) {
513 if (allocated
[src
->src
.ssa
->index
].type() == RegType::vgpr
)
514 type
= RegType::vgpr
;
515 if (allocated
[src
->src
.ssa
->index
].type() == RegType::none
)
520 size
*= phi
->dest
.ssa
.bit_size
== 64 ? 2 : 1;
521 RegClass rc
= RegClass(type
, size
);
522 if (rc
!= allocated
[phi
->dest
.ssa
.index
].regClass()) {
525 nir_foreach_phi_src(src
, phi
)
526 assert(allocated
[src
->src
.ssa
->index
].size() == rc
.size());
528 allocated
[phi
->dest
.ssa
.index
] = Temp(0, rc
);
538 if (G_0286CC_POS_W_FLOAT_ENA(spi_ps_inputs
)) {
539 /* If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be enabled too */
540 spi_ps_inputs
|= S_0286CC_PERSP_CENTER_ENA(1);
543 if (!(spi_ps_inputs
& 0x7F)) {
544 /* At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled */
545 spi_ps_inputs
|= S_0286CC_PERSP_CENTER_ENA(1);
548 ctx
->program
->config
->spi_ps_input_ena
= spi_ps_inputs
;
549 ctx
->program
->config
->spi_ps_input_addr
= spi_ps_inputs
;
551 for (unsigned i
= 0; i
< impl
->ssa_alloc
; i
++)
552 allocated
[i
] = Temp(ctx
->program
->allocateId(), allocated
[i
].regClass());
554 ctx
->allocated
.reset(allocated
.release());
555 ctx
->cf_info
.nir_to_aco
.reset(nir_to_aco
.release());
558 Pseudo_instruction
*add_startpgm(struct isel_context
*ctx
)
560 unsigned arg_count
= ctx
->args
->ac
.arg_count
;
561 if (ctx
->stage
== fragment_fs
) {
562 /* LLVM optimizes away unused FS inputs and computes spi_ps_input_addr
563 * itself and then communicates the results back via the ELF binary.
564 * Mirror what LLVM does by re-mapping the VGPR arguments here.
566 * TODO: If we made the FS input scanning code into a separate pass that
567 * could run before argument setup, then this wouldn't be necessary
570 struct ac_shader_args
*args
= &ctx
->args
->ac
;
572 for (unsigned i
= 0, vgpr_arg
= 0, vgpr_reg
= 0; i
< args
->arg_count
; i
++) {
573 if (args
->args
[i
].file
!= AC_ARG_VGPR
) {
578 if (!(ctx
->program
->config
->spi_ps_input_addr
& (1 << vgpr_arg
))) {
579 args
->args
[i
].skip
= true;
581 args
->args
[i
].offset
= vgpr_reg
;
582 vgpr_reg
+= args
->args
[i
].size
;
589 aco_ptr
<Pseudo_instruction
> startpgm
{create_instruction
<Pseudo_instruction
>(aco_opcode::p_startpgm
, Format::PSEUDO
, 0, arg_count
+ 1)};
590 for (unsigned i
= 0, arg
= 0; i
< ctx
->args
->ac
.arg_count
; i
++) {
591 if (ctx
->args
->ac
.args
[i
].skip
)
594 enum ac_arg_regfile file
= ctx
->args
->ac
.args
[i
].file
;
595 unsigned size
= ctx
->args
->ac
.args
[i
].size
;
596 unsigned reg
= ctx
->args
->ac
.args
[i
].offset
;
597 RegClass type
= RegClass(file
== AC_ARG_SGPR
? RegType::sgpr
: RegType::vgpr
, size
);
598 Temp dst
= Temp
{ctx
->program
->allocateId(), type
};
599 ctx
->arg_temps
[i
] = dst
;
600 startpgm
->definitions
[arg
] = Definition(dst
);
601 startpgm
->definitions
[arg
].setFixed(PhysReg
{file
== AC_ARG_SGPR
? reg
: reg
+ 256});
604 startpgm
->definitions
[arg_count
] = Definition
{ctx
->program
->allocateId(), exec
, ctx
->program
->lane_mask
};
605 Pseudo_instruction
*instr
= startpgm
.get();
606 ctx
->block
->instructions
.push_back(std::move(startpgm
));
608 /* Stash these in the program so that they can be accessed later when
611 ctx
->program
->private_segment_buffer
= get_arg(ctx
, ctx
->args
->ring_offsets
);
612 ctx
->program
->scratch_offset
= get_arg(ctx
, ctx
->args
->scratch_offset
);
618 type_size(const struct glsl_type
*type
, bool bindless
)
620 // TODO: don't we need type->std430_base_alignment() here?
621 return glsl_count_attribute_slots(type
, false);
625 shared_var_info(const struct glsl_type
*type
, unsigned *size
, unsigned *align
)
627 assert(glsl_type_is_vector_or_scalar(type
));
629 uint32_t comp_size
= glsl_type_is_boolean(type
)
630 ? 4 : glsl_get_bit_size(type
) / 8;
631 unsigned length
= glsl_get_vector_elements(type
);
632 *size
= comp_size
* length
,
637 mem_vectorize_callback(unsigned align
, unsigned bit_size
,
638 unsigned num_components
, unsigned high_offset
,
639 nir_intrinsic_instr
*low
, nir_intrinsic_instr
*high
)
641 if ((bit_size
!= 32 && bit_size
!= 64) || num_components
> 4)
644 /* >128 bit loads are split except with SMEM */
645 if (bit_size
* num_components
> 128)
648 switch (low
->intrinsic
) {
649 case nir_intrinsic_load_ubo
:
650 case nir_intrinsic_load_ssbo
:
651 case nir_intrinsic_store_ssbo
:
652 case nir_intrinsic_load_push_constant
:
653 return align
% 4 == 0;
654 case nir_intrinsic_load_deref
:
655 case nir_intrinsic_store_deref
:
656 assert(nir_src_as_deref(low
->src
[0])->mode
== nir_var_mem_shared
);
658 case nir_intrinsic_load_shared
:
659 case nir_intrinsic_store_shared
:
660 if (bit_size
* num_components
> 64) /* 96 and 128 bit loads require 128 bit alignment and are split otherwise */
661 return align
% 16 == 0;
663 return align
% 4 == 0;
671 setup_vs_output_info(isel_context
*ctx
, nir_shader
*nir
,
672 bool export_prim_id
, bool export_clip_dists
,
673 radv_vs_output_info
*outinfo
)
675 memset(outinfo
->vs_output_param_offset
, AC_EXP_PARAM_UNDEFINED
,
676 sizeof(outinfo
->vs_output_param_offset
));
678 outinfo
->param_exports
= 0;
679 int pos_written
= 0x1;
680 if (outinfo
->writes_pointsize
|| outinfo
->writes_viewport_index
|| outinfo
->writes_layer
)
681 pos_written
|= 1 << 1;
683 uint64_t mask
= ctx
->output_masks
[nir
->info
.stage
];
685 int idx
= u_bit_scan64(&mask
);
686 if (idx
>= VARYING_SLOT_VAR0
|| idx
== VARYING_SLOT_LAYER
|| idx
== VARYING_SLOT_PRIMITIVE_ID
||
687 ((idx
== VARYING_SLOT_CLIP_DIST0
|| idx
== VARYING_SLOT_CLIP_DIST1
) && export_clip_dists
)) {
688 if (outinfo
->vs_output_param_offset
[idx
] == AC_EXP_PARAM_UNDEFINED
)
689 outinfo
->vs_output_param_offset
[idx
] = outinfo
->param_exports
++;
692 if (outinfo
->writes_layer
&&
693 outinfo
->vs_output_param_offset
[VARYING_SLOT_LAYER
] == AC_EXP_PARAM_UNDEFINED
) {
694 /* when ctx->options->key.has_multiview_view_index = true, the layer
695 * variable isn't declared in NIR and it's isel's job to get the layer */
696 outinfo
->vs_output_param_offset
[VARYING_SLOT_LAYER
] = outinfo
->param_exports
++;
699 if (export_prim_id
) {
700 assert(outinfo
->vs_output_param_offset
[VARYING_SLOT_PRIMITIVE_ID
] == AC_EXP_PARAM_UNDEFINED
);
701 outinfo
->vs_output_param_offset
[VARYING_SLOT_PRIMITIVE_ID
] = outinfo
->param_exports
++;
704 ctx
->export_clip_dists
= export_clip_dists
;
705 ctx
->num_clip_distances
= util_bitcount(outinfo
->clip_dist_mask
);
706 ctx
->num_cull_distances
= util_bitcount(outinfo
->cull_dist_mask
);
708 assert(ctx
->num_clip_distances
+ ctx
->num_cull_distances
<= 8);
710 if (ctx
->num_clip_distances
+ ctx
->num_cull_distances
> 0)
711 pos_written
|= 1 << 2;
712 if (ctx
->num_clip_distances
+ ctx
->num_cull_distances
> 4)
713 pos_written
|= 1 << 3;
715 outinfo
->pos_exports
= util_bitcount(pos_written
);
719 setup_vs_variables(isel_context
*ctx
, nir_shader
*nir
)
721 nir_foreach_variable(variable
, &nir
->inputs
)
723 variable
->data
.driver_location
= variable
->data
.location
* 4;
725 nir_foreach_variable(variable
, &nir
->outputs
)
727 if (ctx
->stage
== vertex_geometry_gs
)
728 variable
->data
.driver_location
= util_bitcount64(ctx
->output_masks
[nir
->info
.stage
] & ((1ull << variable
->data
.location
) - 1ull)) * 4;
729 else if (ctx
->stage
== vertex_es
)
730 //TODO: make this more compact
731 variable
->data
.driver_location
= shader_io_get_unique_index((gl_varying_slot
)variable
->data
.location
) * 4;
733 variable
->data
.driver_location
= variable
->data
.location
* 4;
736 if (ctx
->stage
== vertex_vs
) {
737 radv_vs_output_info
*outinfo
= &ctx
->program
->info
->vs
.outinfo
;
738 setup_vs_output_info(ctx
, nir
, outinfo
->export_prim_id
,
739 ctx
->options
->key
.vs_common_out
.export_clip_dists
, outinfo
);
740 } else if (ctx
->stage
== vertex_geometry_gs
|| ctx
->stage
== vertex_es
) {
741 /* TODO: radv_nir_shader_info_pass() already sets this but it's larger
742 * than it needs to be in order to set it better, we have to improve
743 * radv_nir_shader_info_pass() because gfx9_get_gs_info() uses
744 * esgs_itemsize and has to be done before compilation
746 /* radv_es_output_info *outinfo = &ctx->program->info->vs.es_info;
747 outinfo->esgs_itemsize = util_bitcount64(ctx->output_masks[nir->info.stage]) * 16u; */
752 setup_variables(isel_context
*ctx
, nir_shader
*nir
)
754 switch (nir
->info
.stage
) {
755 case MESA_SHADER_FRAGMENT
: {
756 nir_foreach_variable(variable
, &nir
->outputs
)
758 int idx
= variable
->data
.location
+ variable
->data
.index
;
759 variable
->data
.driver_location
= idx
* 4;
763 case MESA_SHADER_COMPUTE
: {
764 ctx
->program
->config
->lds_size
= (nir
->info
.cs
.shared_size
+ ctx
->program
->lds_alloc_granule
- 1) /
765 ctx
->program
->lds_alloc_granule
;
768 case MESA_SHADER_VERTEX
: {
769 setup_vs_variables(ctx
, nir
);
772 case MESA_SHADER_GEOMETRY
: {
773 assert(ctx
->stage
== vertex_geometry_gs
|| ctx
->stage
== geometry_gs
);
774 if (ctx
->stage
== vertex_geometry_gs
) {
775 nir_foreach_variable(variable
, &nir
->inputs
) {
776 variable
->data
.driver_location
= util_bitcount64(ctx
->input_masks
[nir
->info
.stage
] & ((1ull << variable
->data
.location
) - 1ull)) * 4;
779 //TODO: make this more compact
780 nir_foreach_variable(variable
, &nir
->inputs
) {
781 variable
->data
.driver_location
= shader_io_get_unique_index((gl_varying_slot
)variable
->data
.location
) * 4;
784 nir_foreach_variable(variable
, &nir
->outputs
) {
785 variable
->data
.driver_location
= variable
->data
.location
* 4;
787 if (ctx
->stage
== vertex_geometry_gs
)
788 ctx
->program
->info
->gs
.es_type
= MESA_SHADER_VERTEX
; /* tesselation shaders are not yet supported */
792 unreachable("Unhandled shader stage.");
797 get_io_masks(isel_context
*ctx
, unsigned shader_count
, struct nir_shader
*const *shaders
)
799 for (unsigned i
= 0; i
< shader_count
; i
++) {
800 nir_shader
*nir
= shaders
[i
];
801 if (nir
->info
.stage
== MESA_SHADER_COMPUTE
)
804 uint64_t output_mask
= 0;
805 nir_foreach_variable(variable
, &nir
->outputs
) {
806 const glsl_type
*type
= variable
->type
;
807 if (nir_is_per_vertex_io(variable
, nir
->info
.stage
))
808 type
= type
->fields
.array
;
809 unsigned slots
= type
->count_attribute_slots(false);
810 if (variable
->data
.compact
) {
811 unsigned component_count
= variable
->data
.location_frac
+ type
->length
;
812 slots
= (component_count
+ 3) / 4;
814 output_mask
|= ((1ull << slots
) - 1) << variable
->data
.location
;
817 uint64_t input_mask
= 0;
818 nir_foreach_variable(variable
, &nir
->inputs
) {
819 const glsl_type
*type
= variable
->type
;
820 if (nir_is_per_vertex_io(variable
, nir
->info
.stage
))
821 type
= type
->fields
.array
;
822 unsigned slots
= type
->count_attribute_slots(false);
823 if (variable
->data
.compact
) {
824 unsigned component_count
= variable
->data
.location_frac
+ type
->length
;
825 slots
= (component_count
+ 3) / 4;
827 input_mask
|= ((1ull << slots
) - 1) << variable
->data
.location
;
830 ctx
->output_masks
[nir
->info
.stage
] |= output_mask
;
831 if (i
+ 1 < shader_count
)
832 ctx
->input_masks
[shaders
[i
+ 1]->info
.stage
] |= output_mask
;
834 ctx
->input_masks
[nir
->info
.stage
] |= input_mask
;
836 ctx
->output_masks
[shaders
[i
- 1]->info
.stage
] |= input_mask
;
841 setup_nir(isel_context
*ctx
, nir_shader
*nir
)
843 Program
*program
= ctx
->program
;
845 /* align and copy constant data */
846 while (program
->constant_data
.size() % 4u)
847 program
->constant_data
.push_back(0);
848 ctx
->constant_data_offset
= program
->constant_data
.size();
849 program
->constant_data
.insert(program
->constant_data
.end(),
850 (uint8_t*)nir
->constant_data
,
851 (uint8_t*)nir
->constant_data
+ nir
->constant_data_size
);
853 /* the variable setup has to be done before lower_io / CSE */
854 setup_variables(ctx
, nir
);
856 /* optimize and lower memory operations */
857 bool lower_to_scalar
= false;
858 bool lower_pack
= false;
859 if (nir_opt_load_store_vectorize(nir
,
860 (nir_variable_mode
)(nir_var_mem_ssbo
| nir_var_mem_ubo
|
861 nir_var_mem_push_const
| nir_var_mem_shared
),
862 mem_vectorize_callback
)) {
863 lower_to_scalar
= true;
866 if (nir
->info
.stage
!= MESA_SHADER_COMPUTE
)
867 nir_lower_io(nir
, (nir_variable_mode
)(nir_var_shader_in
| nir_var_shader_out
), type_size
, (nir_lower_io_options
)0);
868 nir_lower_explicit_io(nir
, nir_var_mem_global
, nir_address_format_64bit_global
);
871 nir_lower_alu_to_scalar(nir
, NULL
, NULL
);
875 /* lower ALU operations */
876 // TODO: implement logic64 in aco, it's more effective for sgprs
877 nir_lower_int64(nir
, nir
->options
->lower_int64_options
);
879 nir_opt_idiv_const(nir
, 32);
880 nir_lower_idiv(nir
, nir_lower_idiv_precise
);
882 /* optimize the lowered ALU operations */
883 bool more_algebraic
= true;
884 while (more_algebraic
) {
885 more_algebraic
= false;
886 NIR_PASS_V(nir
, nir_copy_prop
);
887 NIR_PASS_V(nir
, nir_opt_dce
);
888 NIR_PASS_V(nir
, nir_opt_constant_folding
);
889 NIR_PASS(more_algebraic
, nir
, nir_opt_algebraic
);
892 /* Do late algebraic optimization to turn add(a, neg(b)) back into
893 * subs, then the mandatory cleanup after algebraic. Note that it may
894 * produce fnegs, and if so then we need to keep running to squash
897 bool more_late_algebraic
= true;
898 while (more_late_algebraic
) {
899 more_late_algebraic
= false;
900 NIR_PASS(more_late_algebraic
, nir
, nir_opt_algebraic_late
);
901 NIR_PASS_V(nir
, nir_opt_constant_folding
);
902 NIR_PASS_V(nir
, nir_copy_prop
);
903 NIR_PASS_V(nir
, nir_opt_dce
);
904 NIR_PASS_V(nir
, nir_opt_cse
);
908 nir_lower_load_const_to_scalar(nir
);
909 nir_opt_shrink_load(nir
);
910 nir_move_options move_opts
= (nir_move_options
)(
911 nir_move_const_undef
| nir_move_load_ubo
| nir_move_load_input
|
912 nir_move_comparisons
| nir_move_copies
);
913 nir_opt_sink(nir
, move_opts
);
914 nir_opt_move(nir
, move_opts
);
915 nir_convert_to_lcssa(nir
, true, false);
916 nir_lower_phis_to_scalar(nir
);
918 nir_function_impl
*func
= nir_shader_get_entrypoint(nir
);
919 nir_index_ssa_defs(func
);
920 nir_metadata_require(func
, nir_metadata_block_index
);
924 setup_isel_context(Program
* program
,
925 unsigned shader_count
,
926 struct nir_shader
*const *shaders
,
927 ac_shader_config
* config
,
928 struct radv_shader_args
*args
,
929 bool is_gs_copy_shader
)
932 for (unsigned i
= 0; i
< shader_count
; i
++) {
933 switch (shaders
[i
]->info
.stage
) {
934 case MESA_SHADER_VERTEX
:
935 program
->stage
|= sw_vs
;
937 case MESA_SHADER_TESS_CTRL
:
938 program
->stage
|= sw_tcs
;
940 case MESA_SHADER_TESS_EVAL
:
941 program
->stage
|= sw_tes
;
943 case MESA_SHADER_GEOMETRY
:
944 program
->stage
|= is_gs_copy_shader
? sw_gs_copy
: sw_gs
;
946 case MESA_SHADER_FRAGMENT
:
947 program
->stage
|= sw_fs
;
949 case MESA_SHADER_COMPUTE
:
950 program
->stage
|= sw_cs
;
953 unreachable("Shader stage not implemented");
956 bool gfx9_plus
= args
->options
->chip_class
>= GFX9
;
957 bool ngg
= args
->shader_info
->is_ngg
&& args
->options
->chip_class
>= GFX10
;
958 if (program
->stage
== sw_vs
&& args
->shader_info
->vs
.as_es
)
959 program
->stage
|= hw_es
;
960 else if (program
->stage
== sw_vs
&& !args
->shader_info
->vs
.as_ls
)
961 program
->stage
|= hw_vs
;
962 else if (program
->stage
== sw_gs
)
963 program
->stage
|= hw_gs
;
964 else if (program
->stage
== sw_fs
)
965 program
->stage
|= hw_fs
;
966 else if (program
->stage
== sw_cs
)
967 program
->stage
|= hw_cs
;
968 else if (program
->stage
== sw_gs_copy
)
969 program
->stage
|= hw_vs
;
970 else if (program
->stage
== (sw_vs
| sw_gs
) && gfx9_plus
&& !ngg
)
971 program
->stage
|= hw_gs
;
973 unreachable("Shader stage not implemented");
975 program
->config
= config
;
976 program
->info
= args
->shader_info
;
977 program
->chip_class
= args
->options
->chip_class
;
978 program
->family
= args
->options
->family
;
979 program
->wave_size
= args
->shader_info
->wave_size
;
980 program
->lane_mask
= program
->wave_size
== 32 ? s1
: s2
;
982 program
->lds_alloc_granule
= args
->options
->chip_class
>= GFX7
? 512 : 256;
983 program
->lds_limit
= args
->options
->chip_class
>= GFX7
? 65536 : 32768;
984 program
->vgpr_limit
= 256;
985 program
->vgpr_alloc_granule
= 3;
987 if (args
->options
->chip_class
>= GFX10
) {
988 program
->physical_sgprs
= 2560; /* doesn't matter as long as it's at least 128 * 20 */
989 program
->sgpr_alloc_granule
= 127;
990 program
->sgpr_limit
= 106;
991 program
->vgpr_alloc_granule
= program
->wave_size
== 32 ? 7 : 3;
992 } else if (program
->chip_class
>= GFX8
) {
993 program
->physical_sgprs
= 800;
994 program
->sgpr_alloc_granule
= 15;
995 if (args
->options
->family
== CHIP_TONGA
|| args
->options
->family
== CHIP_ICELAND
)
996 program
->sgpr_limit
= 94; /* workaround hardware bug */
998 program
->sgpr_limit
= 102;
1000 program
->physical_sgprs
= 512;
1001 program
->sgpr_alloc_granule
= 7;
1002 program
->sgpr_limit
= 104;
1005 /* TODO: we don't have to allocate VCC if we don't need it */
1006 program
->needs_vcc
= true;
1008 calc_min_waves(program
);
1009 program
->vgpr_limit
= get_addr_vgpr_from_waves(program
, program
->min_waves
);
1010 program
->sgpr_limit
= get_addr_sgpr_from_waves(program
, program
->min_waves
);
1012 isel_context ctx
= {};
1013 ctx
.program
= program
;
1015 ctx
.options
= args
->options
;
1016 ctx
.stage
= program
->stage
;
1018 get_io_masks(&ctx
, shader_count
, shaders
);
1020 unsigned scratch_size
= 0;
1021 if (program
->stage
== gs_copy_vs
) {
1022 assert(shader_count
== 1);
1023 setup_vs_output_info(&ctx
, shaders
[0], false, true, &args
->shader_info
->vs
.outinfo
);
1025 for (unsigned i
= 0; i
< shader_count
; i
++) {
1026 nir_shader
*nir
= shaders
[i
];
1027 setup_nir(&ctx
, nir
);
1029 if (args
->options
->dump_preoptir
) {
1030 fprintf(stderr
, "NIR shader before instruction selection:\n");
1031 nir_print_shader(nir
, stderr
);
1035 for (unsigned i
= 0; i
< shader_count
; i
++)
1036 scratch_size
= std::max(scratch_size
, shaders
[i
]->scratch_size
);
1039 ctx
.program
->config
->scratch_bytes_per_wave
= align(scratch_size
* ctx
.program
->wave_size
, 1024);
1041 ctx
.block
= ctx
.program
->create_and_insert_block();
1042 ctx
.block
->loop_nest_depth
= 0;
1043 ctx
.block
->kind
= block_kind_top_level
;