aco: fix boolean undef regclass
[mesa.git] / src / amd / compiler / aco_instruction_selection_setup.cpp
1 /*
2 * Copyright © 2018 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25 #include <array>
26 #include <unordered_map>
27 #include "aco_ir.h"
28 #include "nir.h"
29 #include "nir_control_flow.h"
30 #include "vulkan/radv_shader.h"
31 #include "vulkan/radv_descriptor_set.h"
32 #include "vulkan/radv_shader_args.h"
33 #include "sid.h"
34 #include "ac_exp_param.h"
35 #include "ac_shader_util.h"
36
37 #include "util/u_math.h"
38
39 #define MAX_INLINE_PUSH_CONSTS 8
40
41 namespace aco {
42
43 struct output_state {
44 uint8_t mask[VARYING_SLOT_VAR31 + 1];
45 Temp outputs[VARYING_SLOT_VAR31 + 1][4];
46 };
47
48 struct isel_context {
49 const struct radv_nir_compiler_options *options;
50 struct radv_shader_args *args;
51 Program *program;
52 nir_shader *shader;
53 uint32_t constant_data_offset;
54 Block *block;
55 bool *divergent_vals;
56 std::unique_ptr<Temp[]> allocated;
57 std::unordered_map<unsigned, std::array<Temp,NIR_MAX_VEC_COMPONENTS>> allocated_vec;
58 Stage stage; /* Stage */
59 bool has_gfx10_wave64_bpermute = false;
60 struct {
61 bool has_branch;
62 uint16_t loop_nest_depth = 0;
63 struct {
64 unsigned header_idx;
65 Block* exit;
66 bool has_divergent_continue = false;
67 bool has_divergent_branch = false;
68 } parent_loop;
69 struct {
70 bool is_divergent = false;
71 } parent_if;
72 bool exec_potentially_empty_discard = false; /* set to false when loop_nest_depth==0 && parent_if.is_divergent==false */
73 uint16_t exec_potentially_empty_break_depth = UINT16_MAX;
74 /* Set to false when loop_nest_depth==exec_potentially_empty_break_depth
75 * and parent_if.is_divergent==false. Called _break but it's also used for
76 * loop continues. */
77 bool exec_potentially_empty_break = false;
78 std::unique_ptr<unsigned[]> nir_to_aco; /* NIR block index to ACO block index */
79 } cf_info;
80
81 Temp arg_temps[AC_MAX_ARGS];
82
83 /* FS inputs */
84 Temp persp_centroid, linear_centroid;
85
86 /* GS inputs */
87 Temp gs_wave_id;
88
89 /* gathered information */
90 uint64_t input_masks[MESA_SHADER_COMPUTE];
91 uint64_t output_masks[MESA_SHADER_COMPUTE];
92
93 /* VS output information */
94 bool export_clip_dists;
95 unsigned num_clip_distances;
96 unsigned num_cull_distances;
97
98 /* tessellation information */
99 uint32_t tcs_num_inputs;
100 uint32_t tcs_num_patches;
101
102 /* VS, FS or GS output information */
103 output_state outputs;
104 };
105
106 Temp get_arg(isel_context *ctx, struct ac_arg arg)
107 {
108 assert(arg.used);
109 return ctx->arg_temps[arg.arg_index];
110 }
111
112 unsigned get_interp_input(nir_intrinsic_op intrin, enum glsl_interp_mode interp)
113 {
114 switch (interp) {
115 case INTERP_MODE_SMOOTH:
116 case INTERP_MODE_NONE:
117 if (intrin == nir_intrinsic_load_barycentric_pixel ||
118 intrin == nir_intrinsic_load_barycentric_at_sample ||
119 intrin == nir_intrinsic_load_barycentric_at_offset)
120 return S_0286CC_PERSP_CENTER_ENA(1);
121 else if (intrin == nir_intrinsic_load_barycentric_centroid)
122 return S_0286CC_PERSP_CENTROID_ENA(1);
123 else if (intrin == nir_intrinsic_load_barycentric_sample)
124 return S_0286CC_PERSP_SAMPLE_ENA(1);
125 break;
126 case INTERP_MODE_NOPERSPECTIVE:
127 if (intrin == nir_intrinsic_load_barycentric_pixel)
128 return S_0286CC_LINEAR_CENTER_ENA(1);
129 else if (intrin == nir_intrinsic_load_barycentric_centroid)
130 return S_0286CC_LINEAR_CENTROID_ENA(1);
131 else if (intrin == nir_intrinsic_load_barycentric_sample)
132 return S_0286CC_LINEAR_SAMPLE_ENA(1);
133 break;
134 default:
135 break;
136 }
137 return 0;
138 }
139
140 /* If one side of a divergent IF ends in a branch and the other doesn't, we
141 * might have to emit the contents of the side without the branch at the merge
142 * block instead. This is so that we can use any SGPR live-out of the side
143 * without the branch without creating a linear phi in the invert or merge block. */
144 bool
145 sanitize_if(nir_function_impl *impl, bool *divergent, nir_if *nif)
146 {
147 if (!divergent[nif->condition.ssa->index])
148 return false;
149
150 nir_block *then_block = nir_if_last_then_block(nif);
151 nir_block *else_block = nir_if_last_else_block(nif);
152 bool then_jump = nir_block_ends_in_jump(then_block) || nir_block_is_unreachable(then_block);
153 bool else_jump = nir_block_ends_in_jump(else_block) || nir_block_is_unreachable(else_block);
154 if (then_jump == else_jump)
155 return false;
156
157 /* If the continue from block is empty then return as there is nothing to
158 * move.
159 */
160 if (nir_cf_list_is_empty_block(else_jump ? &nif->then_list : &nif->else_list))
161 return false;
162
163 /* Even though this if statement has a jump on one side, we may still have
164 * phis afterwards. Single-source phis can be produced by loop unrolling
165 * or dead control-flow passes and are perfectly legal. Run a quick phi
166 * removal on the block after the if to clean up any such phis.
167 */
168 nir_opt_remove_phis_block(nir_cf_node_as_block(nir_cf_node_next(&nif->cf_node)));
169
170 /* Finally, move the continue from branch after the if-statement. */
171 nir_block *last_continue_from_blk = else_jump ? then_block : else_block;
172 nir_block *first_continue_from_blk = else_jump ?
173 nir_if_first_then_block(nif) : nir_if_first_else_block(nif);
174
175 nir_cf_list tmp;
176 nir_cf_extract(&tmp, nir_before_block(first_continue_from_blk),
177 nir_after_block(last_continue_from_blk));
178 nir_cf_reinsert(&tmp, nir_after_cf_node(&nif->cf_node));
179
180 /* nir_cf_extract() invalidates dominance metadata, but it should still be
181 * correct because of the specific type of transformation we did. Block
182 * indices are not valid except for block_0's, which is all we care about for
183 * nir_block_is_unreachable(). */
184 impl->valid_metadata =
185 (nir_metadata)(impl->valid_metadata | nir_metadata_dominance | nir_metadata_block_index);
186
187 return true;
188 }
189
190 bool
191 sanitize_cf_list(nir_function_impl *impl, bool *divergent, struct exec_list *cf_list)
192 {
193 bool progress = false;
194 foreach_list_typed(nir_cf_node, cf_node, node, cf_list) {
195 switch (cf_node->type) {
196 case nir_cf_node_block:
197 break;
198 case nir_cf_node_if: {
199 nir_if *nif = nir_cf_node_as_if(cf_node);
200 progress |= sanitize_cf_list(impl, divergent, &nif->then_list);
201 progress |= sanitize_cf_list(impl, divergent, &nif->else_list);
202 progress |= sanitize_if(impl, divergent, nif);
203 break;
204 }
205 case nir_cf_node_loop: {
206 nir_loop *loop = nir_cf_node_as_loop(cf_node);
207 progress |= sanitize_cf_list(impl, divergent, &loop->body);
208 break;
209 }
210 case nir_cf_node_function:
211 unreachable("Invalid cf type");
212 }
213 }
214
215 return progress;
216 }
217
218 void init_context(isel_context *ctx, nir_shader *shader)
219 {
220 nir_function_impl *impl = nir_shader_get_entrypoint(shader);
221 unsigned lane_mask_size = ctx->program->lane_mask.size();
222
223 ctx->shader = shader;
224 ctx->divergent_vals = nir_divergence_analysis(shader, nir_divergence_view_index_uniform);
225
226 /* sanitize control flow */
227 nir_metadata_require(impl, nir_metadata_dominance);
228 sanitize_cf_list(impl, ctx->divergent_vals, &impl->body);
229 nir_metadata_preserve(impl, (nir_metadata)~nir_metadata_block_index);
230
231 /* we'll need this for isel */
232 nir_metadata_require(impl, nir_metadata_block_index);
233
234 if (!(ctx->stage & sw_gs_copy) && ctx->options->dump_preoptir) {
235 fprintf(stderr, "NIR shader before instruction selection:\n");
236 nir_print_shader(shader, stderr);
237 }
238
239 std::unique_ptr<Temp[]> allocated{new Temp[impl->ssa_alloc]()};
240
241 unsigned spi_ps_inputs = 0;
242
243 std::unique_ptr<unsigned[]> nir_to_aco{new unsigned[impl->num_blocks]()};
244
245 bool done = false;
246 while (!done) {
247 done = true;
248 nir_foreach_block(block, impl) {
249 nir_foreach_instr(instr, block) {
250 switch(instr->type) {
251 case nir_instr_type_alu: {
252 nir_alu_instr *alu_instr = nir_instr_as_alu(instr);
253 unsigned size = alu_instr->dest.dest.ssa.num_components;
254 if (alu_instr->dest.dest.ssa.bit_size == 64)
255 size *= 2;
256 RegType type = RegType::sgpr;
257 switch(alu_instr->op) {
258 case nir_op_fmul:
259 case nir_op_fadd:
260 case nir_op_fsub:
261 case nir_op_fmax:
262 case nir_op_fmin:
263 case nir_op_fmax3:
264 case nir_op_fmin3:
265 case nir_op_fmed3:
266 case nir_op_fneg:
267 case nir_op_fabs:
268 case nir_op_fsat:
269 case nir_op_fsign:
270 case nir_op_frcp:
271 case nir_op_frsq:
272 case nir_op_fsqrt:
273 case nir_op_fexp2:
274 case nir_op_flog2:
275 case nir_op_ffract:
276 case nir_op_ffloor:
277 case nir_op_fceil:
278 case nir_op_ftrunc:
279 case nir_op_fround_even:
280 case nir_op_fsin:
281 case nir_op_fcos:
282 case nir_op_f2f32:
283 case nir_op_f2f64:
284 case nir_op_u2f32:
285 case nir_op_u2f64:
286 case nir_op_i2f32:
287 case nir_op_i2f64:
288 case nir_op_pack_half_2x16:
289 case nir_op_unpack_half_2x16_split_x:
290 case nir_op_unpack_half_2x16_split_y:
291 case nir_op_fddx:
292 case nir_op_fddy:
293 case nir_op_fddx_fine:
294 case nir_op_fddy_fine:
295 case nir_op_fddx_coarse:
296 case nir_op_fddy_coarse:
297 case nir_op_fquantize2f16:
298 case nir_op_ldexp:
299 case nir_op_frexp_sig:
300 case nir_op_frexp_exp:
301 case nir_op_cube_face_index:
302 case nir_op_cube_face_coord:
303 type = RegType::vgpr;
304 break;
305 case nir_op_flt:
306 case nir_op_fge:
307 case nir_op_feq:
308 case nir_op_fne:
309 case nir_op_ilt:
310 case nir_op_ige:
311 case nir_op_ult:
312 case nir_op_uge:
313 case nir_op_ieq:
314 case nir_op_ine:
315 case nir_op_i2b1:
316 size = lane_mask_size;
317 break;
318 case nir_op_f2i64:
319 case nir_op_f2u64:
320 case nir_op_b2i32:
321 case nir_op_b2f32:
322 case nir_op_f2i32:
323 case nir_op_f2u32:
324 type = ctx->divergent_vals[alu_instr->dest.dest.ssa.index] ? RegType::vgpr : RegType::sgpr;
325 break;
326 case nir_op_bcsel:
327 if (alu_instr->dest.dest.ssa.bit_size == 1) {
328 size = lane_mask_size;
329 } else {
330 if (ctx->divergent_vals[alu_instr->dest.dest.ssa.index]) {
331 type = RegType::vgpr;
332 } else {
333 if (allocated[alu_instr->src[1].src.ssa->index].type() == RegType::vgpr ||
334 allocated[alu_instr->src[2].src.ssa->index].type() == RegType::vgpr) {
335 type = RegType::vgpr;
336 }
337 }
338 if (alu_instr->src[1].src.ssa->num_components == 1 && alu_instr->src[2].src.ssa->num_components == 1) {
339 assert(allocated[alu_instr->src[1].src.ssa->index].size() == allocated[alu_instr->src[2].src.ssa->index].size());
340 size = allocated[alu_instr->src[1].src.ssa->index].size();
341 }
342 }
343 break;
344 case nir_op_mov:
345 if (alu_instr->dest.dest.ssa.bit_size == 1) {
346 size = lane_mask_size;
347 } else {
348 type = ctx->divergent_vals[alu_instr->dest.dest.ssa.index] ? RegType::vgpr : RegType::sgpr;
349 }
350 break;
351 default:
352 if (alu_instr->dest.dest.ssa.bit_size == 1) {
353 size = lane_mask_size;
354 } else {
355 for (unsigned i = 0; i < nir_op_infos[alu_instr->op].num_inputs; i++) {
356 if (allocated[alu_instr->src[i].src.ssa->index].type() == RegType::vgpr)
357 type = RegType::vgpr;
358 }
359 }
360 break;
361 }
362 allocated[alu_instr->dest.dest.ssa.index] = Temp(0, RegClass(type, size));
363 break;
364 }
365 case nir_instr_type_load_const: {
366 unsigned size = nir_instr_as_load_const(instr)->def.num_components;
367 if (nir_instr_as_load_const(instr)->def.bit_size == 64)
368 size *= 2;
369 else if (nir_instr_as_load_const(instr)->def.bit_size == 1)
370 size *= lane_mask_size;
371 allocated[nir_instr_as_load_const(instr)->def.index] = Temp(0, RegClass(RegType::sgpr, size));
372 break;
373 }
374 case nir_instr_type_intrinsic: {
375 nir_intrinsic_instr *intrinsic = nir_instr_as_intrinsic(instr);
376 if (!nir_intrinsic_infos[intrinsic->intrinsic].has_dest)
377 break;
378 unsigned size = intrinsic->dest.ssa.num_components;
379 if (intrinsic->dest.ssa.bit_size == 64)
380 size *= 2;
381 RegType type = RegType::sgpr;
382 switch(intrinsic->intrinsic) {
383 case nir_intrinsic_load_push_constant:
384 case nir_intrinsic_load_work_group_id:
385 case nir_intrinsic_load_num_work_groups:
386 case nir_intrinsic_load_subgroup_id:
387 case nir_intrinsic_load_num_subgroups:
388 case nir_intrinsic_load_first_vertex:
389 case nir_intrinsic_load_base_instance:
390 case nir_intrinsic_get_buffer_size:
391 case nir_intrinsic_vote_all:
392 case nir_intrinsic_vote_any:
393 case nir_intrinsic_read_first_invocation:
394 case nir_intrinsic_read_invocation:
395 case nir_intrinsic_first_invocation:
396 type = RegType::sgpr;
397 if (intrinsic->dest.ssa.bit_size == 1)
398 size = lane_mask_size;
399 break;
400 case nir_intrinsic_ballot:
401 type = RegType::sgpr;
402 break;
403 case nir_intrinsic_load_sample_id:
404 case nir_intrinsic_load_sample_mask_in:
405 case nir_intrinsic_load_input:
406 case nir_intrinsic_load_output:
407 case nir_intrinsic_load_input_vertex:
408 case nir_intrinsic_load_per_vertex_input:
409 case nir_intrinsic_load_per_vertex_output:
410 case nir_intrinsic_load_vertex_id:
411 case nir_intrinsic_load_vertex_id_zero_base:
412 case nir_intrinsic_load_barycentric_sample:
413 case nir_intrinsic_load_barycentric_pixel:
414 case nir_intrinsic_load_barycentric_model:
415 case nir_intrinsic_load_barycentric_centroid:
416 case nir_intrinsic_load_barycentric_at_sample:
417 case nir_intrinsic_load_barycentric_at_offset:
418 case nir_intrinsic_load_interpolated_input:
419 case nir_intrinsic_load_frag_coord:
420 case nir_intrinsic_load_sample_pos:
421 case nir_intrinsic_load_layer_id:
422 case nir_intrinsic_load_local_invocation_id:
423 case nir_intrinsic_load_local_invocation_index:
424 case nir_intrinsic_load_subgroup_invocation:
425 case nir_intrinsic_load_tess_coord:
426 case nir_intrinsic_write_invocation_amd:
427 case nir_intrinsic_mbcnt_amd:
428 case nir_intrinsic_load_instance_id:
429 case nir_intrinsic_ssbo_atomic_add:
430 case nir_intrinsic_ssbo_atomic_imin:
431 case nir_intrinsic_ssbo_atomic_umin:
432 case nir_intrinsic_ssbo_atomic_imax:
433 case nir_intrinsic_ssbo_atomic_umax:
434 case nir_intrinsic_ssbo_atomic_and:
435 case nir_intrinsic_ssbo_atomic_or:
436 case nir_intrinsic_ssbo_atomic_xor:
437 case nir_intrinsic_ssbo_atomic_exchange:
438 case nir_intrinsic_ssbo_atomic_comp_swap:
439 case nir_intrinsic_global_atomic_add:
440 case nir_intrinsic_global_atomic_imin:
441 case nir_intrinsic_global_atomic_umin:
442 case nir_intrinsic_global_atomic_imax:
443 case nir_intrinsic_global_atomic_umax:
444 case nir_intrinsic_global_atomic_and:
445 case nir_intrinsic_global_atomic_or:
446 case nir_intrinsic_global_atomic_xor:
447 case nir_intrinsic_global_atomic_exchange:
448 case nir_intrinsic_global_atomic_comp_swap:
449 case nir_intrinsic_image_deref_atomic_add:
450 case nir_intrinsic_image_deref_atomic_umin:
451 case nir_intrinsic_image_deref_atomic_imin:
452 case nir_intrinsic_image_deref_atomic_umax:
453 case nir_intrinsic_image_deref_atomic_imax:
454 case nir_intrinsic_image_deref_atomic_and:
455 case nir_intrinsic_image_deref_atomic_or:
456 case nir_intrinsic_image_deref_atomic_xor:
457 case nir_intrinsic_image_deref_atomic_exchange:
458 case nir_intrinsic_image_deref_atomic_comp_swap:
459 case nir_intrinsic_image_deref_size:
460 case nir_intrinsic_shared_atomic_add:
461 case nir_intrinsic_shared_atomic_imin:
462 case nir_intrinsic_shared_atomic_umin:
463 case nir_intrinsic_shared_atomic_imax:
464 case nir_intrinsic_shared_atomic_umax:
465 case nir_intrinsic_shared_atomic_and:
466 case nir_intrinsic_shared_atomic_or:
467 case nir_intrinsic_shared_atomic_xor:
468 case nir_intrinsic_shared_atomic_exchange:
469 case nir_intrinsic_shared_atomic_comp_swap:
470 case nir_intrinsic_load_scratch:
471 case nir_intrinsic_load_invocation_id:
472 case nir_intrinsic_load_primitive_id:
473 type = RegType::vgpr;
474 break;
475 case nir_intrinsic_shuffle:
476 case nir_intrinsic_quad_broadcast:
477 case nir_intrinsic_quad_swap_horizontal:
478 case nir_intrinsic_quad_swap_vertical:
479 case nir_intrinsic_quad_swap_diagonal:
480 case nir_intrinsic_quad_swizzle_amd:
481 case nir_intrinsic_masked_swizzle_amd:
482 case nir_intrinsic_inclusive_scan:
483 case nir_intrinsic_exclusive_scan:
484 if (intrinsic->dest.ssa.bit_size == 1) {
485 size = lane_mask_size;
486 type = RegType::sgpr;
487 } else if (!ctx->divergent_vals[intrinsic->dest.ssa.index]) {
488 type = RegType::sgpr;
489 } else {
490 type = RegType::vgpr;
491 }
492 break;
493 case nir_intrinsic_load_view_index:
494 type = ctx->stage == fragment_fs ? RegType::vgpr : RegType::sgpr;
495 break;
496 case nir_intrinsic_load_front_face:
497 case nir_intrinsic_load_helper_invocation:
498 case nir_intrinsic_is_helper_invocation:
499 type = RegType::sgpr;
500 size = lane_mask_size;
501 break;
502 case nir_intrinsic_reduce:
503 if (intrinsic->dest.ssa.bit_size == 1) {
504 size = lane_mask_size;
505 type = RegType::sgpr;
506 } else if (!ctx->divergent_vals[intrinsic->dest.ssa.index]) {
507 type = RegType::sgpr;
508 } else {
509 type = RegType::vgpr;
510 }
511 break;
512 case nir_intrinsic_load_ubo:
513 case nir_intrinsic_load_ssbo:
514 case nir_intrinsic_load_global:
515 case nir_intrinsic_vulkan_resource_index:
516 type = ctx->divergent_vals[intrinsic->dest.ssa.index] ? RegType::vgpr : RegType::sgpr;
517 break;
518 /* due to copy propagation, the swizzled imov is removed if num dest components == 1 */
519 case nir_intrinsic_load_shared:
520 if (ctx->divergent_vals[intrinsic->dest.ssa.index])
521 type = RegType::vgpr;
522 else
523 type = RegType::sgpr;
524 break;
525 default:
526 for (unsigned i = 0; i < nir_intrinsic_infos[intrinsic->intrinsic].num_srcs; i++) {
527 if (allocated[intrinsic->src[i].ssa->index].type() == RegType::vgpr)
528 type = RegType::vgpr;
529 }
530 break;
531 }
532 allocated[intrinsic->dest.ssa.index] = Temp(0, RegClass(type, size));
533
534 switch(intrinsic->intrinsic) {
535 case nir_intrinsic_load_barycentric_sample:
536 case nir_intrinsic_load_barycentric_pixel:
537 case nir_intrinsic_load_barycentric_centroid:
538 case nir_intrinsic_load_barycentric_at_sample:
539 case nir_intrinsic_load_barycentric_at_offset: {
540 glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(intrinsic);
541 spi_ps_inputs |= get_interp_input(intrinsic->intrinsic, mode);
542 break;
543 }
544 case nir_intrinsic_load_barycentric_model:
545 spi_ps_inputs |= S_0286CC_PERSP_PULL_MODEL_ENA(1);
546 break;
547 case nir_intrinsic_load_front_face:
548 spi_ps_inputs |= S_0286CC_FRONT_FACE_ENA(1);
549 break;
550 case nir_intrinsic_load_frag_coord:
551 case nir_intrinsic_load_sample_pos: {
552 uint8_t mask = nir_ssa_def_components_read(&intrinsic->dest.ssa);
553 for (unsigned i = 0; i < 4; i++) {
554 if (mask & (1 << i))
555 spi_ps_inputs |= S_0286CC_POS_X_FLOAT_ENA(1) << i;
556
557 }
558 break;
559 }
560 case nir_intrinsic_load_sample_id:
561 spi_ps_inputs |= S_0286CC_ANCILLARY_ENA(1);
562 break;
563 case nir_intrinsic_load_sample_mask_in:
564 spi_ps_inputs |= S_0286CC_ANCILLARY_ENA(1);
565 spi_ps_inputs |= S_0286CC_SAMPLE_COVERAGE_ENA(1);
566 break;
567 default:
568 break;
569 }
570 break;
571 }
572 case nir_instr_type_tex: {
573 nir_tex_instr* tex = nir_instr_as_tex(instr);
574 unsigned size = tex->dest.ssa.num_components;
575
576 if (tex->dest.ssa.bit_size == 64)
577 size *= 2;
578 if (tex->op == nir_texop_texture_samples)
579 assert(!ctx->divergent_vals[tex->dest.ssa.index]);
580 if (ctx->divergent_vals[tex->dest.ssa.index])
581 allocated[tex->dest.ssa.index] = Temp(0, RegClass(RegType::vgpr, size));
582 else
583 allocated[tex->dest.ssa.index] = Temp(0, RegClass(RegType::sgpr, size));
584 break;
585 }
586 case nir_instr_type_parallel_copy: {
587 nir_foreach_parallel_copy_entry(entry, nir_instr_as_parallel_copy(instr)) {
588 allocated[entry->dest.ssa.index] = allocated[entry->src.ssa->index];
589 }
590 break;
591 }
592 case nir_instr_type_ssa_undef: {
593 unsigned size = nir_instr_as_ssa_undef(instr)->def.num_components;
594 if (nir_instr_as_ssa_undef(instr)->def.bit_size == 64)
595 size *= 2;
596 else if (nir_instr_as_ssa_undef(instr)->def.bit_size == 1)
597 size *= lane_mask_size;
598 allocated[nir_instr_as_ssa_undef(instr)->def.index] = Temp(0, RegClass(RegType::sgpr, size));
599 break;
600 }
601 case nir_instr_type_phi: {
602 nir_phi_instr* phi = nir_instr_as_phi(instr);
603 RegType type;
604 unsigned size = phi->dest.ssa.num_components;
605
606 if (phi->dest.ssa.bit_size == 1) {
607 assert(size == 1 && "multiple components not yet supported on boolean phis.");
608 type = RegType::sgpr;
609 size *= lane_mask_size;
610 allocated[phi->dest.ssa.index] = Temp(0, RegClass(type, size));
611 break;
612 }
613
614 if (ctx->divergent_vals[phi->dest.ssa.index]) {
615 type = RegType::vgpr;
616 } else {
617 type = RegType::sgpr;
618 nir_foreach_phi_src (src, phi) {
619 if (allocated[src->src.ssa->index].type() == RegType::vgpr)
620 type = RegType::vgpr;
621 if (allocated[src->src.ssa->index].type() == RegType::none)
622 done = false;
623 }
624 }
625
626 size *= phi->dest.ssa.bit_size == 64 ? 2 : 1;
627 RegClass rc = RegClass(type, size);
628 if (rc != allocated[phi->dest.ssa.index].regClass()) {
629 done = false;
630 } else {
631 nir_foreach_phi_src(src, phi)
632 assert(allocated[src->src.ssa->index].size() == rc.size());
633 }
634 allocated[phi->dest.ssa.index] = Temp(0, rc);
635 break;
636 }
637 default:
638 break;
639 }
640 }
641 }
642 }
643
644 if (G_0286CC_POS_W_FLOAT_ENA(spi_ps_inputs)) {
645 /* If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be enabled too */
646 spi_ps_inputs |= S_0286CC_PERSP_CENTER_ENA(1);
647 }
648
649 if (!(spi_ps_inputs & 0x7F)) {
650 /* At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled */
651 spi_ps_inputs |= S_0286CC_PERSP_CENTER_ENA(1);
652 }
653
654 ctx->program->config->spi_ps_input_ena = spi_ps_inputs;
655 ctx->program->config->spi_ps_input_addr = spi_ps_inputs;
656
657 for (unsigned i = 0; i < impl->ssa_alloc; i++)
658 allocated[i] = Temp(ctx->program->allocateId(), allocated[i].regClass());
659
660 ctx->allocated.reset(allocated.release());
661 ctx->cf_info.nir_to_aco.reset(nir_to_aco.release());
662 }
663
664 Pseudo_instruction *add_startpgm(struct isel_context *ctx)
665 {
666 unsigned arg_count = ctx->args->ac.arg_count;
667 if (ctx->stage == fragment_fs) {
668 /* LLVM optimizes away unused FS inputs and computes spi_ps_input_addr
669 * itself and then communicates the results back via the ELF binary.
670 * Mirror what LLVM does by re-mapping the VGPR arguments here.
671 *
672 * TODO: If we made the FS input scanning code into a separate pass that
673 * could run before argument setup, then this wouldn't be necessary
674 * anymore.
675 */
676 struct ac_shader_args *args = &ctx->args->ac;
677 arg_count = 0;
678 for (unsigned i = 0, vgpr_arg = 0, vgpr_reg = 0; i < args->arg_count; i++) {
679 if (args->args[i].file != AC_ARG_VGPR) {
680 arg_count++;
681 continue;
682 }
683
684 if (!(ctx->program->config->spi_ps_input_addr & (1 << vgpr_arg))) {
685 args->args[i].skip = true;
686 } else {
687 args->args[i].offset = vgpr_reg;
688 vgpr_reg += args->args[i].size;
689 arg_count++;
690 }
691 vgpr_arg++;
692 }
693 }
694
695 aco_ptr<Pseudo_instruction> startpgm{create_instruction<Pseudo_instruction>(aco_opcode::p_startpgm, Format::PSEUDO, 0, arg_count + 1)};
696 for (unsigned i = 0, arg = 0; i < ctx->args->ac.arg_count; i++) {
697 if (ctx->args->ac.args[i].skip)
698 continue;
699
700 enum ac_arg_regfile file = ctx->args->ac.args[i].file;
701 unsigned size = ctx->args->ac.args[i].size;
702 unsigned reg = ctx->args->ac.args[i].offset;
703 RegClass type = RegClass(file == AC_ARG_SGPR ? RegType::sgpr : RegType::vgpr, size);
704 Temp dst = Temp{ctx->program->allocateId(), type};
705 ctx->arg_temps[i] = dst;
706 startpgm->definitions[arg] = Definition(dst);
707 startpgm->definitions[arg].setFixed(PhysReg{file == AC_ARG_SGPR ? reg : reg + 256});
708 arg++;
709 }
710 startpgm->definitions[arg_count] = Definition{ctx->program->allocateId(), exec, ctx->program->lane_mask};
711 Pseudo_instruction *instr = startpgm.get();
712 ctx->block->instructions.push_back(std::move(startpgm));
713
714 /* Stash these in the program so that they can be accessed later when
715 * handling spilling.
716 */
717 ctx->program->private_segment_buffer = get_arg(ctx, ctx->args->ring_offsets);
718 ctx->program->scratch_offset = get_arg(ctx, ctx->args->scratch_offset);
719
720 return instr;
721 }
722
723 int
724 type_size(const struct glsl_type *type, bool bindless)
725 {
726 // TODO: don't we need type->std430_base_alignment() here?
727 return glsl_count_attribute_slots(type, false);
728 }
729
730 void
731 shared_var_info(const struct glsl_type *type, unsigned *size, unsigned *align)
732 {
733 assert(glsl_type_is_vector_or_scalar(type));
734
735 uint32_t comp_size = glsl_type_is_boolean(type)
736 ? 4 : glsl_get_bit_size(type) / 8;
737 unsigned length = glsl_get_vector_elements(type);
738 *size = comp_size * length,
739 *align = comp_size;
740 }
741
742 static bool
743 mem_vectorize_callback(unsigned align, unsigned bit_size,
744 unsigned num_components, unsigned high_offset,
745 nir_intrinsic_instr *low, nir_intrinsic_instr *high)
746 {
747 if ((bit_size != 32 && bit_size != 64) || num_components > 4)
748 return false;
749
750 /* >128 bit loads are split except with SMEM */
751 if (bit_size * num_components > 128)
752 return false;
753
754 switch (low->intrinsic) {
755 case nir_intrinsic_load_ubo:
756 case nir_intrinsic_load_ssbo:
757 case nir_intrinsic_store_ssbo:
758 case nir_intrinsic_load_push_constant:
759 return align % 4 == 0;
760 case nir_intrinsic_load_deref:
761 case nir_intrinsic_store_deref:
762 assert(nir_src_as_deref(low->src[0])->mode == nir_var_mem_shared);
763 /* fallthrough */
764 case nir_intrinsic_load_shared:
765 case nir_intrinsic_store_shared:
766 if (bit_size * num_components > 64) /* 96 and 128 bit loads require 128 bit alignment and are split otherwise */
767 return align % 16 == 0;
768 else
769 return align % 4 == 0;
770 default:
771 return false;
772 }
773 return false;
774 }
775
776 void
777 setup_vs_output_info(isel_context *ctx, nir_shader *nir,
778 bool export_prim_id, bool export_clip_dists,
779 radv_vs_output_info *outinfo)
780 {
781 memset(outinfo->vs_output_param_offset, AC_EXP_PARAM_UNDEFINED,
782 sizeof(outinfo->vs_output_param_offset));
783
784 outinfo->param_exports = 0;
785 int pos_written = 0x1;
786 if (outinfo->writes_pointsize || outinfo->writes_viewport_index || outinfo->writes_layer)
787 pos_written |= 1 << 1;
788
789 uint64_t mask = ctx->output_masks[nir->info.stage];
790 while (mask) {
791 int idx = u_bit_scan64(&mask);
792 if (idx >= VARYING_SLOT_VAR0 || idx == VARYING_SLOT_LAYER || idx == VARYING_SLOT_PRIMITIVE_ID ||
793 ((idx == VARYING_SLOT_CLIP_DIST0 || idx == VARYING_SLOT_CLIP_DIST1) && export_clip_dists)) {
794 if (outinfo->vs_output_param_offset[idx] == AC_EXP_PARAM_UNDEFINED)
795 outinfo->vs_output_param_offset[idx] = outinfo->param_exports++;
796 }
797 }
798 if (outinfo->writes_layer &&
799 outinfo->vs_output_param_offset[VARYING_SLOT_LAYER] == AC_EXP_PARAM_UNDEFINED) {
800 /* when ctx->options->key.has_multiview_view_index = true, the layer
801 * variable isn't declared in NIR and it's isel's job to get the layer */
802 outinfo->vs_output_param_offset[VARYING_SLOT_LAYER] = outinfo->param_exports++;
803 }
804
805 if (export_prim_id) {
806 assert(outinfo->vs_output_param_offset[VARYING_SLOT_PRIMITIVE_ID] == AC_EXP_PARAM_UNDEFINED);
807 outinfo->vs_output_param_offset[VARYING_SLOT_PRIMITIVE_ID] = outinfo->param_exports++;
808 }
809
810 ctx->export_clip_dists = export_clip_dists;
811 ctx->num_clip_distances = util_bitcount(outinfo->clip_dist_mask);
812 ctx->num_cull_distances = util_bitcount(outinfo->cull_dist_mask);
813
814 assert(ctx->num_clip_distances + ctx->num_cull_distances <= 8);
815
816 if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
817 pos_written |= 1 << 2;
818 if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
819 pos_written |= 1 << 3;
820
821 outinfo->pos_exports = util_bitcount(pos_written);
822 }
823
824 void
825 setup_vs_variables(isel_context *ctx, nir_shader *nir)
826 {
827 nir_foreach_variable(variable, &nir->inputs)
828 {
829 variable->data.driver_location = variable->data.location * 4;
830 }
831 nir_foreach_variable(variable, &nir->outputs)
832 {
833 if (ctx->stage == vertex_geometry_gs)
834 variable->data.driver_location = util_bitcount64(ctx->output_masks[nir->info.stage] & ((1ull << variable->data.location) - 1ull)) * 4;
835 else if (ctx->stage == vertex_es ||
836 ctx->stage == vertex_ls ||
837 ctx->stage == vertex_tess_control_hs)
838 // TODO: make this more compact
839 variable->data.driver_location = shader_io_get_unique_index((gl_varying_slot) variable->data.location) * 4;
840 else if (ctx->stage == vertex_vs)
841 variable->data.driver_location = variable->data.location * 4;
842 else
843 unreachable("Unsupported VS stage");
844 }
845
846 if (ctx->stage == vertex_vs) {
847 radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
848 setup_vs_output_info(ctx, nir, outinfo->export_prim_id,
849 ctx->options->key.vs_common_out.export_clip_dists, outinfo);
850 } else if (ctx->stage == vertex_geometry_gs || ctx->stage == vertex_es) {
851 /* TODO: radv_nir_shader_info_pass() already sets this but it's larger
852 * than it needs to be in order to set it better, we have to improve
853 * radv_nir_shader_info_pass() because gfx9_get_gs_info() uses
854 * esgs_itemsize and has to be done before compilation
855 */
856 /* radv_es_output_info *outinfo = &ctx->program->info->vs.es_info;
857 outinfo->esgs_itemsize = util_bitcount64(ctx->output_masks[nir->info.stage]) * 16u; */
858 }
859 }
860
861 void setup_gs_variables(isel_context *ctx, nir_shader *nir)
862 {
863 if (ctx->stage == vertex_geometry_gs || ctx->stage == tess_eval_geometry_gs) {
864 nir_foreach_variable(variable, &nir->inputs) {
865 variable->data.driver_location = util_bitcount64(ctx->input_masks[nir->info.stage] & ((1ull << variable->data.location) - 1ull)) * 4;
866 }
867 } else if (ctx->stage == geometry_gs) {
868 //TODO: make this more compact
869 nir_foreach_variable(variable, &nir->inputs) {
870 variable->data.driver_location = shader_io_get_unique_index((gl_varying_slot)variable->data.location) * 4;
871 }
872 } else {
873 unreachable("Unsupported GS stage.");
874 }
875
876 nir_foreach_variable(variable, &nir->outputs) {
877 variable->data.driver_location = variable->data.location * 4;
878 }
879
880 if (ctx->stage == vertex_geometry_gs)
881 ctx->program->info->gs.es_type = MESA_SHADER_VERTEX;
882 else if (ctx->stage == tess_eval_geometry_gs)
883 ctx->program->info->gs.es_type = MESA_SHADER_TESS_EVAL;
884 }
885
886 void
887 setup_tcs_variables(isel_context *ctx, nir_shader *nir)
888 {
889 switch (ctx->stage) {
890 case tess_control_hs:
891 ctx->tcs_num_inputs = ctx->args->options->key.tcs.num_inputs;
892 break;
893 case vertex_tess_control_hs:
894 ctx->tcs_num_inputs = util_last_bit64(ctx->args->shader_info->vs.ls_outputs_written);
895 break;
896 default:
897 unreachable("Unsupported TCS shader stage");
898 }
899
900 ctx->tcs_num_patches = get_tcs_num_patches(
901 ctx->args->options->key.tcs.input_vertices,
902 nir->info.tess.tcs_vertices_out,
903 ctx->tcs_num_inputs,
904 ctx->args->shader_info->tcs.outputs_written,
905 ctx->args->shader_info->tcs.patch_outputs_written,
906 ctx->args->options->tess_offchip_block_dw_size,
907 ctx->args->options->chip_class,
908 ctx->args->options->family);
909 unsigned lds_size = calculate_tess_lds_size(
910 ctx->args->options->key.tcs.input_vertices,
911 nir->info.tess.tcs_vertices_out,
912 ctx->tcs_num_inputs,
913 ctx->tcs_num_patches,
914 ctx->args->shader_info->tcs.outputs_written,
915 ctx->args->shader_info->tcs.patch_outputs_written);
916
917 ctx->args->shader_info->tcs.num_patches = ctx->tcs_num_patches;
918 ctx->args->shader_info->tcs.lds_size = lds_size;
919 ctx->program->config->lds_size = (lds_size + ctx->program->lds_alloc_granule - 1) /
920 ctx->program->lds_alloc_granule;
921
922 nir_foreach_variable(variable, &nir->inputs) {
923 variable->data.driver_location = shader_io_get_unique_index((gl_varying_slot) variable->data.location) * 4;
924 }
925
926 nir_foreach_variable(variable, &nir->outputs) {
927 variable->data.driver_location = shader_io_get_unique_index((gl_varying_slot) variable->data.location) * 4;
928 }
929 }
930
931 void
932 setup_tes_variables(isel_context *ctx, nir_shader *nir)
933 {
934 ctx->tcs_num_patches = ctx->args->options->key.tes.num_patches;
935
936 nir_foreach_variable(variable, &nir->inputs) {
937 variable->data.driver_location = shader_io_get_unique_index((gl_varying_slot) variable->data.location) * 4;
938 }
939
940 nir_foreach_variable(variable, &nir->outputs) {
941 if (ctx->stage == tess_eval_vs)
942 variable->data.driver_location = variable->data.location * 4;
943 else if (ctx->stage == tess_eval_es)
944 variable->data.driver_location = shader_io_get_unique_index((gl_varying_slot) variable->data.location) * 4;
945 else if (ctx->stage == tess_eval_geometry_gs)
946 variable->data.driver_location = util_bitcount64(ctx->output_masks[nir->info.stage] & ((1ull << variable->data.location) - 1ull)) * 4;
947 else
948 unreachable("Unsupported TES shader stage");
949 }
950
951 if (ctx->stage == tess_eval_vs) {
952 radv_vs_output_info *outinfo = &ctx->program->info->tes.outinfo;
953 setup_vs_output_info(ctx, nir, outinfo->export_prim_id,
954 ctx->options->key.vs_common_out.export_clip_dists, outinfo);
955 }
956 }
957
958 void
959 setup_variables(isel_context *ctx, nir_shader *nir)
960 {
961 switch (nir->info.stage) {
962 case MESA_SHADER_FRAGMENT: {
963 nir_foreach_variable(variable, &nir->outputs)
964 {
965 int idx = variable->data.location + variable->data.index;
966 variable->data.driver_location = idx * 4;
967 }
968 break;
969 }
970 case MESA_SHADER_COMPUTE: {
971 ctx->program->config->lds_size = (nir->info.cs.shared_size + ctx->program->lds_alloc_granule - 1) /
972 ctx->program->lds_alloc_granule;
973 break;
974 }
975 case MESA_SHADER_VERTEX: {
976 setup_vs_variables(ctx, nir);
977 break;
978 }
979 case MESA_SHADER_GEOMETRY: {
980 setup_gs_variables(ctx, nir);
981 break;
982 }
983 case MESA_SHADER_TESS_CTRL: {
984 setup_tcs_variables(ctx, nir);
985 break;
986 }
987 case MESA_SHADER_TESS_EVAL: {
988 setup_tes_variables(ctx, nir);
989 break;
990 }
991 default:
992 unreachable("Unhandled shader stage.");
993 }
994 }
995
996 void
997 get_io_masks(isel_context *ctx, unsigned shader_count, struct nir_shader *const *shaders)
998 {
999 for (unsigned i = 0; i < shader_count; i++) {
1000 nir_shader *nir = shaders[i];
1001 if (nir->info.stage == MESA_SHADER_COMPUTE)
1002 continue;
1003
1004 uint64_t output_mask = 0;
1005 nir_foreach_variable(variable, &nir->outputs) {
1006 const glsl_type *type = variable->type;
1007 if (nir_is_per_vertex_io(variable, nir->info.stage))
1008 type = type->fields.array;
1009 unsigned slots = type->count_attribute_slots(false);
1010 if (variable->data.compact) {
1011 unsigned component_count = variable->data.location_frac + type->length;
1012 slots = (component_count + 3) / 4;
1013 }
1014 output_mask |= ((1ull << slots) - 1) << variable->data.location;
1015 }
1016
1017 uint64_t input_mask = 0;
1018 nir_foreach_variable(variable, &nir->inputs) {
1019 const glsl_type *type = variable->type;
1020 if (nir_is_per_vertex_io(variable, nir->info.stage))
1021 type = type->fields.array;
1022 unsigned slots = type->count_attribute_slots(false);
1023 if (variable->data.compact) {
1024 unsigned component_count = variable->data.location_frac + type->length;
1025 slots = (component_count + 3) / 4;
1026 }
1027 input_mask |= ((1ull << slots) - 1) << variable->data.location;
1028 }
1029
1030 ctx->output_masks[nir->info.stage] |= output_mask;
1031 if (i + 1 < shader_count)
1032 ctx->input_masks[shaders[i + 1]->info.stage] |= output_mask;
1033
1034 ctx->input_masks[nir->info.stage] |= input_mask;
1035 if (i)
1036 ctx->output_masks[shaders[i - 1]->info.stage] |= input_mask;
1037 }
1038 }
1039
1040 void
1041 setup_nir(isel_context *ctx, nir_shader *nir)
1042 {
1043 Program *program = ctx->program;
1044
1045 /* align and copy constant data */
1046 while (program->constant_data.size() % 4u)
1047 program->constant_data.push_back(0);
1048 ctx->constant_data_offset = program->constant_data.size();
1049 program->constant_data.insert(program->constant_data.end(),
1050 (uint8_t*)nir->constant_data,
1051 (uint8_t*)nir->constant_data + nir->constant_data_size);
1052
1053 /* the variable setup has to be done before lower_io / CSE */
1054 setup_variables(ctx, nir);
1055
1056 /* optimize and lower memory operations */
1057 bool lower_to_scalar = false;
1058 bool lower_pack = false;
1059 if (nir_opt_load_store_vectorize(nir,
1060 (nir_variable_mode)(nir_var_mem_ssbo | nir_var_mem_ubo |
1061 nir_var_mem_push_const | nir_var_mem_shared),
1062 mem_vectorize_callback)) {
1063 lower_to_scalar = true;
1064 lower_pack = true;
1065 }
1066 if (nir->info.stage != MESA_SHADER_COMPUTE)
1067 nir_lower_io(nir, (nir_variable_mode)(nir_var_shader_in | nir_var_shader_out), type_size, (nir_lower_io_options)0);
1068 nir_lower_explicit_io(nir, nir_var_mem_global, nir_address_format_64bit_global);
1069
1070 if (lower_to_scalar)
1071 nir_lower_alu_to_scalar(nir, NULL, NULL);
1072 if (lower_pack)
1073 nir_lower_pack(nir);
1074
1075 /* lower ALU operations */
1076 // TODO: implement logic64 in aco, it's more effective for sgprs
1077 nir_lower_int64(nir, nir->options->lower_int64_options);
1078
1079 nir_opt_idiv_const(nir, 32);
1080 nir_lower_idiv(nir, nir_lower_idiv_precise);
1081
1082 /* optimize the lowered ALU operations */
1083 bool more_algebraic = true;
1084 while (more_algebraic) {
1085 more_algebraic = false;
1086 NIR_PASS_V(nir, nir_copy_prop);
1087 NIR_PASS_V(nir, nir_opt_dce);
1088 NIR_PASS_V(nir, nir_opt_constant_folding);
1089 NIR_PASS(more_algebraic, nir, nir_opt_algebraic);
1090 }
1091
1092 /* Do late algebraic optimization to turn add(a, neg(b)) back into
1093 * subs, then the mandatory cleanup after algebraic. Note that it may
1094 * produce fnegs, and if so then we need to keep running to squash
1095 * fneg(fneg(a)).
1096 */
1097 bool more_late_algebraic = true;
1098 while (more_late_algebraic) {
1099 more_late_algebraic = false;
1100 NIR_PASS(more_late_algebraic, nir, nir_opt_algebraic_late);
1101 NIR_PASS_V(nir, nir_opt_constant_folding);
1102 NIR_PASS_V(nir, nir_copy_prop);
1103 NIR_PASS_V(nir, nir_opt_dce);
1104 NIR_PASS_V(nir, nir_opt_cse);
1105 }
1106
1107 /* cleanup passes */
1108 nir_lower_load_const_to_scalar(nir);
1109 nir_opt_shrink_load(nir);
1110 nir_move_options move_opts = (nir_move_options)(
1111 nir_move_const_undef | nir_move_load_ubo | nir_move_load_input |
1112 nir_move_comparisons | nir_move_copies);
1113 nir_opt_sink(nir, move_opts);
1114 nir_opt_move(nir, move_opts);
1115 nir_convert_to_lcssa(nir, true, false);
1116 nir_lower_phis_to_scalar(nir);
1117
1118 nir_function_impl *func = nir_shader_get_entrypoint(nir);
1119 nir_index_ssa_defs(func);
1120 }
1121
1122 isel_context
1123 setup_isel_context(Program* program,
1124 unsigned shader_count,
1125 struct nir_shader *const *shaders,
1126 ac_shader_config* config,
1127 struct radv_shader_args *args,
1128 bool is_gs_copy_shader)
1129 {
1130 program->stage = 0;
1131 for (unsigned i = 0; i < shader_count; i++) {
1132 switch (shaders[i]->info.stage) {
1133 case MESA_SHADER_VERTEX:
1134 program->stage |= sw_vs;
1135 break;
1136 case MESA_SHADER_TESS_CTRL:
1137 program->stage |= sw_tcs;
1138 break;
1139 case MESA_SHADER_TESS_EVAL:
1140 program->stage |= sw_tes;
1141 break;
1142 case MESA_SHADER_GEOMETRY:
1143 program->stage |= is_gs_copy_shader ? sw_gs_copy : sw_gs;
1144 break;
1145 case MESA_SHADER_FRAGMENT:
1146 program->stage |= sw_fs;
1147 break;
1148 case MESA_SHADER_COMPUTE:
1149 program->stage |= sw_cs;
1150 break;
1151 default:
1152 unreachable("Shader stage not implemented");
1153 }
1154 }
1155 bool gfx9_plus = args->options->chip_class >= GFX9;
1156 bool ngg = args->shader_info->is_ngg && args->options->chip_class >= GFX10;
1157 if (program->stage == sw_vs && args->shader_info->vs.as_es)
1158 program->stage |= hw_es;
1159 else if (program->stage == sw_vs && !args->shader_info->vs.as_ls)
1160 program->stage |= hw_vs;
1161 else if (program->stage == sw_gs)
1162 program->stage |= hw_gs;
1163 else if (program->stage == sw_fs)
1164 program->stage |= hw_fs;
1165 else if (program->stage == sw_cs)
1166 program->stage |= hw_cs;
1167 else if (program->stage == sw_gs_copy)
1168 program->stage |= hw_vs;
1169 else if (program->stage == (sw_vs | sw_gs) && gfx9_plus && !ngg)
1170 program->stage |= hw_gs;
1171 else if (program->stage == sw_vs && args->shader_info->vs.as_ls)
1172 program->stage |= hw_ls; /* GFX6-8: VS is a Local Shader, when tessellation is used */
1173 else if (program->stage == sw_tcs)
1174 program->stage |= hw_hs; /* GFX6-8: TCS is a Hull Shader */
1175 else if (program->stage == (sw_vs | sw_tcs))
1176 program->stage |= hw_hs; /* GFX9-10: VS+TCS merged into a Hull Shader */
1177 else if (program->stage == sw_tes && !args->shader_info->tes.as_es && !ngg)
1178 program->stage |= hw_vs; /* GFX6-9: TES without GS uses the HW VS stage (and GFX10/legacy) */
1179 else if (program->stage == sw_tes && args->shader_info->tes.as_es && !ngg)
1180 program->stage |= hw_es; /* GFX6-8: TES is an Export Shader */
1181 else if (program->stage == (sw_tes | sw_gs) && gfx9_plus && !ngg)
1182 program->stage |= hw_gs; /* GFX9: TES+GS merged into a GS (and GFX10/legacy) */
1183 else
1184 unreachable("Shader stage not implemented");
1185
1186 program->config = config;
1187 program->info = args->shader_info;
1188 program->chip_class = args->options->chip_class;
1189 program->family = args->options->family;
1190 program->wave_size = args->shader_info->wave_size;
1191 program->lane_mask = program->wave_size == 32 ? s1 : s2;
1192
1193 program->lds_alloc_granule = args->options->chip_class >= GFX7 ? 512 : 256;
1194 program->lds_limit = args->options->chip_class >= GFX7 ? 65536 : 32768;
1195 /* apparently gfx702 also has 16-bank LDS but I can't find a family for that */
1196 program->has_16bank_lds = args->options->family == CHIP_KABINI || args->options->family == CHIP_STONEY;
1197
1198 program->vgpr_limit = 256;
1199 program->vgpr_alloc_granule = 3;
1200
1201 if (args->options->chip_class >= GFX10) {
1202 program->physical_sgprs = 2560; /* doesn't matter as long as it's at least 128 * 20 */
1203 program->sgpr_alloc_granule = 127;
1204 program->sgpr_limit = 106;
1205 program->vgpr_alloc_granule = program->wave_size == 32 ? 7 : 3;
1206 } else if (program->chip_class >= GFX8) {
1207 program->physical_sgprs = 800;
1208 program->sgpr_alloc_granule = 15;
1209 if (args->options->family == CHIP_TONGA || args->options->family == CHIP_ICELAND)
1210 program->sgpr_limit = 94; /* workaround hardware bug */
1211 else
1212 program->sgpr_limit = 102;
1213 } else {
1214 program->physical_sgprs = 512;
1215 program->sgpr_alloc_granule = 7;
1216 program->sgpr_limit = 104;
1217 }
1218
1219 calc_min_waves(program);
1220 program->vgpr_limit = get_addr_vgpr_from_waves(program, program->min_waves);
1221 program->sgpr_limit = get_addr_sgpr_from_waves(program, program->min_waves);
1222
1223 isel_context ctx = {};
1224 ctx.program = program;
1225 ctx.args = args;
1226 ctx.options = args->options;
1227 ctx.stage = program->stage;
1228
1229 get_io_masks(&ctx, shader_count, shaders);
1230
1231 unsigned scratch_size = 0;
1232 if (program->stage == gs_copy_vs) {
1233 assert(shader_count == 1);
1234 setup_vs_output_info(&ctx, shaders[0], false, true, &args->shader_info->vs.outinfo);
1235 } else {
1236 for (unsigned i = 0; i < shader_count; i++) {
1237 nir_shader *nir = shaders[i];
1238 setup_nir(&ctx, nir);
1239 }
1240
1241 for (unsigned i = 0; i < shader_count; i++)
1242 scratch_size = std::max(scratch_size, shaders[i]->scratch_size);
1243 }
1244
1245 ctx.program->config->scratch_bytes_per_wave = align(scratch_size * ctx.program->wave_size, 1024);
1246
1247 ctx.block = ctx.program->create_and_insert_block();
1248 ctx.block->loop_nest_depth = 0;
1249 ctx.block->kind = block_kind_top_level;
1250
1251 return ctx;
1252 }
1253
1254 }