projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
aco: limit register usage for large work groups
[mesa.git] / src / amd / compiler / aco_instruction_selection_setup.cpp
1 /*
2 * Copyright © 2018 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25 #include <array>
26 #include <unordered_map>
27 #include "aco_ir.h"
28 #include "nir.h"
29 #include "vulkan/radv_shader.h"
30 #include "vulkan/radv_descriptor_set.h"
31 #include "vulkan/radv_shader_args.h"
32 #include "sid.h"
33 #include "ac_exp_param.h"
34 #include "ac_shader_util.h"
35
36 #include "util/u_math.h"
37
38 #define MAX_INLINE_PUSH_CONSTS 8
39
40 namespace aco {
41
42 struct vs_output_state {
43 uint8_t mask[VARYING_SLOT_VAR31 + 1];
44 Temp outputs[VARYING_SLOT_VAR31 + 1][4];
45 };
46
47 struct isel_context {
48 const struct radv_nir_compiler_options *options;
49 struct radv_shader_args *args;
50 Program *program;
51 nir_shader *shader;
52 uint32_t constant_data_offset;
53 Block *block;
54 bool *divergent_vals;
55 std::unique_ptr<Temp[]> allocated;
56 std::unordered_map<unsigned, std::array<Temp,NIR_MAX_VEC_COMPONENTS>> allocated_vec;
57 Stage stage; /* Stage */
58 bool has_gfx10_wave64_bpermute = false;
59 struct {
60 bool has_branch;
61 uint16_t loop_nest_depth = 0;
62 struct {
63 unsigned header_idx;
64 Block* exit;
65 bool has_divergent_continue = false;
66 bool has_divergent_branch = false;
67 } parent_loop;
68 struct {
69 bool is_divergent = false;
70 } parent_if;
71 bool exec_potentially_empty = false;
72 std::unique_ptr<unsigned[]> nir_to_aco; /* NIR block index to ACO block index */
73 } cf_info;
74
75 Temp arg_temps[AC_MAX_ARGS];
76
77 /* inputs common for merged stages */
78 Temp merged_wave_info = Temp(0, s1);
79
80 /* FS inputs */
81 Temp persp_centroid, linear_centroid;
82
83 /* VS inputs */
84 bool needs_instance_id;
85
86 /* VS output information */
87 unsigned num_clip_distances;
88 unsigned num_cull_distances;
89 vs_output_state vs_output;
90 };
91
92 Temp get_arg(isel_context *ctx, struct ac_arg arg)
93 {
94 assert(arg.used);
95 return ctx->arg_temps[arg.arg_index];
96 }
97
98 unsigned get_interp_input(nir_intrinsic_op intrin, enum glsl_interp_mode interp)
99 {
100 switch (interp) {
101 case INTERP_MODE_SMOOTH:
102 case INTERP_MODE_NONE:
103 if (intrin == nir_intrinsic_load_barycentric_pixel ||
104 intrin == nir_intrinsic_load_barycentric_at_sample ||
105 intrin == nir_intrinsic_load_barycentric_at_offset)
106 return S_0286CC_PERSP_CENTER_ENA(1);
107 else if (intrin == nir_intrinsic_load_barycentric_centroid)
108 return S_0286CC_PERSP_CENTROID_ENA(1);
109 else if (intrin == nir_intrinsic_load_barycentric_sample)
110 return S_0286CC_PERSP_SAMPLE_ENA(1);
111 break;
112 case INTERP_MODE_NOPERSPECTIVE:
113 if (intrin == nir_intrinsic_load_barycentric_pixel)
114 return S_0286CC_LINEAR_CENTER_ENA(1);
115 else if (intrin == nir_intrinsic_load_barycentric_centroid)
116 return S_0286CC_LINEAR_CENTROID_ENA(1);
117 else if (intrin == nir_intrinsic_load_barycentric_sample)
118 return S_0286CC_LINEAR_SAMPLE_ENA(1);
119 break;
120 default:
121 break;
122 }
123 return 0;
124 }
125
126 void init_context(isel_context *ctx, nir_shader *shader)
127 {
128 nir_function_impl *impl = nir_shader_get_entrypoint(shader);
129 unsigned lane_mask_size = ctx->program->lane_mask.size();
130
131 ctx->shader = shader;
132 ctx->divergent_vals = nir_divergence_analysis(shader, nir_divergence_view_index_uniform);
133
134 std::unique_ptr<Temp[]> allocated{new Temp[impl->ssa_alloc]()};
135
136 unsigned spi_ps_inputs = 0;
137
138 std::unique_ptr<unsigned[]> nir_to_aco{new unsigned[impl->num_blocks]()};
139
140 bool done = false;
141 while (!done) {
142 done = true;
143 nir_foreach_block(block, impl) {
144 nir_foreach_instr(instr, block) {
145 switch(instr->type) {
146 case nir_instr_type_alu: {
147 nir_alu_instr *alu_instr = nir_instr_as_alu(instr);
148 unsigned size = alu_instr->dest.dest.ssa.num_components;
149 if (alu_instr->dest.dest.ssa.bit_size == 64)
150 size *= 2;
151 RegType type = RegType::sgpr;
152 switch(alu_instr->op) {
153 case nir_op_fmul:
154 case nir_op_fadd:
155 case nir_op_fsub:
156 case nir_op_fmax:
157 case nir_op_fmin:
158 case nir_op_fmax3:
159 case nir_op_fmin3:
160 case nir_op_fmed3:
161 case nir_op_fneg:
162 case nir_op_fabs:
163 case nir_op_fsat:
164 case nir_op_fsign:
165 case nir_op_frcp:
166 case nir_op_frsq:
167 case nir_op_fsqrt:
168 case nir_op_fexp2:
169 case nir_op_flog2:
170 case nir_op_ffract:
171 case nir_op_ffloor:
172 case nir_op_fceil:
173 case nir_op_ftrunc:
174 case nir_op_fround_even:
175 case nir_op_fsin:
176 case nir_op_fcos:
177 case nir_op_f2f32:
178 case nir_op_f2f64:
179 case nir_op_u2f32:
180 case nir_op_u2f64:
181 case nir_op_i2f32:
182 case nir_op_i2f64:
183 case nir_op_pack_half_2x16:
184 case nir_op_unpack_half_2x16_split_x:
185 case nir_op_unpack_half_2x16_split_y:
186 case nir_op_fddx:
187 case nir_op_fddy:
188 case nir_op_fddx_fine:
189 case nir_op_fddy_fine:
190 case nir_op_fddx_coarse:
191 case nir_op_fddy_coarse:
192 case nir_op_fquantize2f16:
193 case nir_op_ldexp:
194 case nir_op_frexp_sig:
195 case nir_op_frexp_exp:
196 case nir_op_cube_face_index:
197 case nir_op_cube_face_coord:
198 type = RegType::vgpr;
199 break;
200 case nir_op_flt:
201 case nir_op_fge:
202 case nir_op_feq:
203 case nir_op_fne:
204 case nir_op_ilt:
205 case nir_op_ige:
206 case nir_op_ult:
207 case nir_op_uge:
208 case nir_op_ieq:
209 case nir_op_ine:
210 case nir_op_i2b1:
211 size = lane_mask_size;
212 break;
213 case nir_op_f2i64:
214 case nir_op_f2u64:
215 case nir_op_b2i32:
216 case nir_op_b2f32:
217 case nir_op_f2i32:
218 case nir_op_f2u32:
219 type = ctx->divergent_vals[alu_instr->dest.dest.ssa.index] ? RegType::vgpr : RegType::sgpr;
220 break;
221 case nir_op_bcsel:
222 if (alu_instr->dest.dest.ssa.bit_size == 1) {
223 size = lane_mask_size;
224 } else {
225 if (ctx->divergent_vals[alu_instr->dest.dest.ssa.index]) {
226 type = RegType::vgpr;
227 } else {
228 if (allocated[alu_instr->src[1].src.ssa->index].type() == RegType::vgpr ||
229 allocated[alu_instr->src[2].src.ssa->index].type() == RegType::vgpr) {
230 type = RegType::vgpr;
231 }
232 }
233 if (alu_instr->src[1].src.ssa->num_components == 1 && alu_instr->src[2].src.ssa->num_components == 1) {
234 assert(allocated[alu_instr->src[1].src.ssa->index].size() == allocated[alu_instr->src[2].src.ssa->index].size());
235 size = allocated[alu_instr->src[1].src.ssa->index].size();
236 }
237 }
238 break;
239 case nir_op_mov:
240 if (alu_instr->dest.dest.ssa.bit_size == 1) {
241 size = lane_mask_size;
242 } else {
243 type = ctx->divergent_vals[alu_instr->dest.dest.ssa.index] ? RegType::vgpr : RegType::sgpr;
244 }
245 break;
246 default:
247 if (alu_instr->dest.dest.ssa.bit_size == 1) {
248 size = lane_mask_size;
249 } else {
250 for (unsigned i = 0; i < nir_op_infos[alu_instr->op].num_inputs; i++) {
251 if (allocated[alu_instr->src[i].src.ssa->index].type() == RegType::vgpr)
252 type = RegType::vgpr;
253 }
254 }
255 break;
256 }
257 allocated[alu_instr->dest.dest.ssa.index] = Temp(0, RegClass(type, size));
258 break;
259 }
260 case nir_instr_type_load_const: {
261 unsigned size = nir_instr_as_load_const(instr)->def.num_components;
262 if (nir_instr_as_load_const(instr)->def.bit_size == 64)
263 size *= 2;
264 else if (nir_instr_as_load_const(instr)->def.bit_size == 1)
265 size *= lane_mask_size;
266 allocated[nir_instr_as_load_const(instr)->def.index] = Temp(0, RegClass(RegType::sgpr, size));
267 break;
268 }
269 case nir_instr_type_intrinsic: {
270 nir_intrinsic_instr *intrinsic = nir_instr_as_intrinsic(instr);
271 if (!nir_intrinsic_infos[intrinsic->intrinsic].has_dest)
272 break;
273 unsigned size = intrinsic->dest.ssa.num_components;
274 if (intrinsic->dest.ssa.bit_size == 64)
275 size *= 2;
276 RegType type = RegType::sgpr;
277 switch(intrinsic->intrinsic) {
278 case nir_intrinsic_load_push_constant:
279 case nir_intrinsic_load_work_group_id:
280 case nir_intrinsic_load_num_work_groups:
281 case nir_intrinsic_load_subgroup_id:
282 case nir_intrinsic_load_num_subgroups:
283 case nir_intrinsic_load_first_vertex:
284 case nir_intrinsic_load_base_instance:
285 case nir_intrinsic_get_buffer_size:
286 case nir_intrinsic_vote_all:
287 case nir_intrinsic_vote_any:
288 case nir_intrinsic_read_first_invocation:
289 case nir_intrinsic_read_invocation:
290 case nir_intrinsic_first_invocation:
291 type = RegType::sgpr;
292 if (intrinsic->dest.ssa.bit_size == 1)
293 size = lane_mask_size;
294 break;
295 case nir_intrinsic_ballot:
296 type = RegType::sgpr;
297 break;
298 case nir_intrinsic_load_sample_id:
299 case nir_intrinsic_load_sample_mask_in:
300 case nir_intrinsic_load_input:
301 case nir_intrinsic_load_vertex_id:
302 case nir_intrinsic_load_vertex_id_zero_base:
303 case nir_intrinsic_load_barycentric_sample:
304 case nir_intrinsic_load_barycentric_pixel:
305 case nir_intrinsic_load_barycentric_centroid:
306 case nir_intrinsic_load_barycentric_at_sample:
307 case nir_intrinsic_load_barycentric_at_offset:
308 case nir_intrinsic_load_interpolated_input:
309 case nir_intrinsic_load_frag_coord:
310 case nir_intrinsic_load_sample_pos:
311 case nir_intrinsic_load_layer_id:
312 case nir_intrinsic_load_local_invocation_id:
313 case nir_intrinsic_load_local_invocation_index:
314 case nir_intrinsic_load_subgroup_invocation:
315 case nir_intrinsic_write_invocation_amd:
316 case nir_intrinsic_mbcnt_amd:
317 case nir_intrinsic_load_instance_id:
318 case nir_intrinsic_ssbo_atomic_add:
319 case nir_intrinsic_ssbo_atomic_imin:
320 case nir_intrinsic_ssbo_atomic_umin:
321 case nir_intrinsic_ssbo_atomic_imax:
322 case nir_intrinsic_ssbo_atomic_umax:
323 case nir_intrinsic_ssbo_atomic_and:
324 case nir_intrinsic_ssbo_atomic_or:
325 case nir_intrinsic_ssbo_atomic_xor:
326 case nir_intrinsic_ssbo_atomic_exchange:
327 case nir_intrinsic_ssbo_atomic_comp_swap:
328 case nir_intrinsic_global_atomic_add:
329 case nir_intrinsic_global_atomic_imin:
330 case nir_intrinsic_global_atomic_umin:
331 case nir_intrinsic_global_atomic_imax:
332 case nir_intrinsic_global_atomic_umax:
333 case nir_intrinsic_global_atomic_and:
334 case nir_intrinsic_global_atomic_or:
335 case nir_intrinsic_global_atomic_xor:
336 case nir_intrinsic_global_atomic_exchange:
337 case nir_intrinsic_global_atomic_comp_swap:
338 case nir_intrinsic_image_deref_atomic_add:
339 case nir_intrinsic_image_deref_atomic_umin:
340 case nir_intrinsic_image_deref_atomic_imin:
341 case nir_intrinsic_image_deref_atomic_umax:
342 case nir_intrinsic_image_deref_atomic_imax:
343 case nir_intrinsic_image_deref_atomic_and:
344 case nir_intrinsic_image_deref_atomic_or:
345 case nir_intrinsic_image_deref_atomic_xor:
346 case nir_intrinsic_image_deref_atomic_exchange:
347 case nir_intrinsic_image_deref_atomic_comp_swap:
348 case nir_intrinsic_image_deref_size:
349 case nir_intrinsic_shared_atomic_add:
350 case nir_intrinsic_shared_atomic_imin:
351 case nir_intrinsic_shared_atomic_umin:
352 case nir_intrinsic_shared_atomic_imax:
353 case nir_intrinsic_shared_atomic_umax:
354 case nir_intrinsic_shared_atomic_and:
355 case nir_intrinsic_shared_atomic_or:
356 case nir_intrinsic_shared_atomic_xor:
357 case nir_intrinsic_shared_atomic_exchange:
358 case nir_intrinsic_shared_atomic_comp_swap:
359 case nir_intrinsic_load_scratch:
360 type = RegType::vgpr;
361 break;
362 case nir_intrinsic_shuffle:
363 case nir_intrinsic_quad_broadcast:
364 case nir_intrinsic_quad_swap_horizontal:
365 case nir_intrinsic_quad_swap_vertical:
366 case nir_intrinsic_quad_swap_diagonal:
367 case nir_intrinsic_quad_swizzle_amd:
368 case nir_intrinsic_masked_swizzle_amd:
369 case nir_intrinsic_inclusive_scan:
370 case nir_intrinsic_exclusive_scan:
371 if (intrinsic->dest.ssa.bit_size == 1) {
372 size = lane_mask_size;
373 type = RegType::sgpr;
374 } else if (!ctx->divergent_vals[intrinsic->dest.ssa.index]) {
375 type = RegType::sgpr;
376 } else {
377 type = RegType::vgpr;
378 }
379 break;
380 case nir_intrinsic_load_view_index:
381 type = ctx->stage == fragment_fs ? RegType::vgpr : RegType::sgpr;
382 break;
383 case nir_intrinsic_load_front_face:
384 case nir_intrinsic_load_helper_invocation:
385 case nir_intrinsic_is_helper_invocation:
386 type = RegType::sgpr;
387 size = lane_mask_size;
388 break;
389 case nir_intrinsic_reduce:
390 if (intrinsic->dest.ssa.bit_size == 1) {
391 size = lane_mask_size;
392 type = RegType::sgpr;
393 } else if (!ctx->divergent_vals[intrinsic->dest.ssa.index]) {
394 type = RegType::sgpr;
395 } else {
396 type = RegType::vgpr;
397 }
398 break;
399 case nir_intrinsic_load_ubo:
400 case nir_intrinsic_load_ssbo:
401 case nir_intrinsic_load_global:
402 case nir_intrinsic_vulkan_resource_index:
403 type = ctx->divergent_vals[intrinsic->dest.ssa.index] ? RegType::vgpr : RegType::sgpr;
404 break;
405 /* due to copy propagation, the swizzled imov is removed if num dest components == 1 */
406 case nir_intrinsic_load_shared:
407 if (ctx->divergent_vals[intrinsic->dest.ssa.index])
408 type = RegType::vgpr;
409 else
410 type = RegType::sgpr;
411 break;
412 default:
413 for (unsigned i = 0; i < nir_intrinsic_infos[intrinsic->intrinsic].num_srcs; i++) {
414 if (allocated[intrinsic->src[i].ssa->index].type() == RegType::vgpr)
415 type = RegType::vgpr;
416 }
417 break;
418 }
419 allocated[intrinsic->dest.ssa.index] = Temp(0, RegClass(type, size));
420
421 switch(intrinsic->intrinsic) {
422 case nir_intrinsic_load_barycentric_sample:
423 case nir_intrinsic_load_barycentric_pixel:
424 case nir_intrinsic_load_barycentric_centroid:
425 case nir_intrinsic_load_barycentric_at_sample:
426 case nir_intrinsic_load_barycentric_at_offset: {
427 glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(intrinsic);
428 spi_ps_inputs |= get_interp_input(intrinsic->intrinsic, mode);
429 break;
430 }
431 case nir_intrinsic_load_front_face:
432 spi_ps_inputs |= S_0286CC_FRONT_FACE_ENA(1);
433 break;
434 case nir_intrinsic_load_frag_coord:
435 case nir_intrinsic_load_sample_pos: {
436 uint8_t mask = nir_ssa_def_components_read(&intrinsic->dest.ssa);
437 for (unsigned i = 0; i < 4; i++) {
438 if (mask & (1 << i))
439 spi_ps_inputs |= S_0286CC_POS_X_FLOAT_ENA(1) << i;
440
441 }
442 break;
443 }
444 case nir_intrinsic_load_sample_id:
445 spi_ps_inputs |= S_0286CC_ANCILLARY_ENA(1);
446 break;
447 case nir_intrinsic_load_sample_mask_in:
448 spi_ps_inputs |= S_0286CC_ANCILLARY_ENA(1);
449 spi_ps_inputs |= S_0286CC_SAMPLE_COVERAGE_ENA(1);
450 break;
451 default:
452 break;
453 }
454 break;
455 }
456 case nir_instr_type_tex: {
457 nir_tex_instr* tex = nir_instr_as_tex(instr);
458 unsigned size = tex->dest.ssa.num_components;
459
460 if (tex->dest.ssa.bit_size == 64)
461 size *= 2;
462 if (tex->op == nir_texop_texture_samples)
463 assert(!ctx->divergent_vals[tex->dest.ssa.index]);
464 if (ctx->divergent_vals[tex->dest.ssa.index])
465 allocated[tex->dest.ssa.index] = Temp(0, RegClass(RegType::vgpr, size));
466 else
467 allocated[tex->dest.ssa.index] = Temp(0, RegClass(RegType::sgpr, size));
468 break;
469 }
470 case nir_instr_type_parallel_copy: {
471 nir_foreach_parallel_copy_entry(entry, nir_instr_as_parallel_copy(instr)) {
472 allocated[entry->dest.ssa.index] = allocated[entry->src.ssa->index];
473 }
474 break;
475 }
476 case nir_instr_type_ssa_undef: {
477 unsigned size = nir_instr_as_ssa_undef(instr)->def.num_components;
478 if (nir_instr_as_ssa_undef(instr)->def.bit_size == 64)
479 size *= 2;
480 allocated[nir_instr_as_ssa_undef(instr)->def.index] = Temp(0, RegClass(RegType::sgpr, size));
481 break;
482 }
483 case nir_instr_type_phi: {
484 nir_phi_instr* phi = nir_instr_as_phi(instr);
485 RegType type;
486 unsigned size = phi->dest.ssa.num_components;
487
488 if (phi->dest.ssa.bit_size == 1) {
489 assert(size == 1 && "multiple components not yet supported on boolean phis.");
490 type = RegType::sgpr;
491 size *= lane_mask_size;
492 allocated[phi->dest.ssa.index] = Temp(0, RegClass(type, size));
493 break;
494 }
495
496 if (ctx->divergent_vals[phi->dest.ssa.index]) {
497 type = RegType::vgpr;
498 } else {
499 type = RegType::sgpr;
500 nir_foreach_phi_src (src, phi) {
501 if (allocated[src->src.ssa->index].type() == RegType::vgpr)
502 type = RegType::vgpr;
503 if (allocated[src->src.ssa->index].type() == RegType::none)
504 done = false;
505 }
506 }
507
508 size *= phi->dest.ssa.bit_size == 64 ? 2 : 1;
509 RegClass rc = RegClass(type, size);
510 if (rc != allocated[phi->dest.ssa.index].regClass()) {
511 done = false;
512 } else {
513 nir_foreach_phi_src(src, phi)
514 assert(allocated[src->src.ssa->index].size() == rc.size());
515 }
516 allocated[phi->dest.ssa.index] = Temp(0, rc);
517 break;
518 }
519 default:
520 break;
521 }
522 }
523 }
524 }
525
526 if (G_0286CC_POS_W_FLOAT_ENA(spi_ps_inputs)) {
527 /* If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be enabled too */
528 spi_ps_inputs |= S_0286CC_PERSP_CENTER_ENA(1);
529 }
530
531 if (!(spi_ps_inputs & 0x7F)) {
532 /* At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled */
533 spi_ps_inputs |= S_0286CC_PERSP_CENTER_ENA(1);
534 }
535
536 ctx->program->config->spi_ps_input_ena = spi_ps_inputs;
537 ctx->program->config->spi_ps_input_addr = spi_ps_inputs;
538
539 for (unsigned i = 0; i < impl->ssa_alloc; i++)
540 allocated[i] = Temp(ctx->program->allocateId(), allocated[i].regClass());
541
542 ctx->allocated.reset(allocated.release());
543 ctx->cf_info.nir_to_aco.reset(nir_to_aco.release());
544 }
545
546 Pseudo_instruction *add_startpgm(struct isel_context *ctx)
547 {
548 unsigned arg_count = ctx->args->ac.arg_count;
549 if (ctx->stage == fragment_fs) {
550 /* LLVM optimizes away unused FS inputs and computes spi_ps_input_addr
551 * itself and then communicates the results back via the ELF binary.
552 * Mirror what LLVM does by re-mapping the VGPR arguments here.
553 *
554 * TODO: If we made the FS input scanning code into a separate pass that
555 * could run before argument setup, then this wouldn't be necessary
556 * anymore.
557 */
558 struct ac_shader_args *args = &ctx->args->ac;
559 arg_count = 0;
560 for (unsigned i = 0, vgpr_arg = 0, vgpr_reg = 0; i < args->arg_count; i++) {
561 if (args->args[i].file != AC_ARG_VGPR) {
562 arg_count++;
563 continue;
564 }
565
566 if (!(ctx->program->config->spi_ps_input_addr & (1 << vgpr_arg))) {
567 args->args[i].skip = true;
568 } else {
569 args->args[i].offset = vgpr_reg;
570 vgpr_reg += args->args[i].size;
571 arg_count++;
572 }
573 vgpr_arg++;
574 }
575 }
576
577 aco_ptr<Pseudo_instruction> startpgm{create_instruction<Pseudo_instruction>(aco_opcode::p_startpgm, Format::PSEUDO, 0, arg_count + 1)};
578 for (unsigned i = 0, arg = 0; i < ctx->args->ac.arg_count; i++) {
579 if (ctx->args->ac.args[i].skip)
580 continue;
581
582 enum ac_arg_regfile file = ctx->args->ac.args[i].file;
583 unsigned size = ctx->args->ac.args[i].size;
584 unsigned reg = ctx->args->ac.args[i].offset;
585 RegClass type = RegClass(file == AC_ARG_SGPR ? RegType::sgpr : RegType::vgpr, size);
586 Temp dst = Temp{ctx->program->allocateId(), type};
587 ctx->arg_temps[i] = dst;
588 startpgm->definitions[arg] = Definition(dst);
589 startpgm->definitions[arg].setFixed(PhysReg{file == AC_ARG_SGPR ? reg : reg + 256});
590 arg++;
591 }
592 startpgm->definitions[arg_count] = Definition{ctx->program->allocateId(), exec, ctx->program->lane_mask};
593 Pseudo_instruction *instr = startpgm.get();
594 ctx->block->instructions.push_back(std::move(startpgm));
595
596 /* Stash these in the program so that they can be accessed later when
597 * handling spilling.
598 */
599 ctx->program->private_segment_buffer = get_arg(ctx, ctx->args->ring_offsets);
600 ctx->program->scratch_offset = get_arg(ctx, ctx->args->scratch_offset);
601
602 return instr;
603 }
604
605 int
606 type_size(const struct glsl_type *type, bool bindless)
607 {
608 // TODO: don't we need type->std430_base_alignment() here?
609 return glsl_count_attribute_slots(type, false);
610 }
611
612 void
613 shared_var_info(const struct glsl_type *type, unsigned *size, unsigned *align)
614 {
615 assert(glsl_type_is_vector_or_scalar(type));
616
617 uint32_t comp_size = glsl_type_is_boolean(type)
618 ? 4 : glsl_get_bit_size(type) / 8;
619 unsigned length = glsl_get_vector_elements(type);
620 *size = comp_size * length,
621 *align = comp_size;
622 }
623
624 static bool
625 mem_vectorize_callback(unsigned align, unsigned bit_size,
626 unsigned num_components, unsigned high_offset,
627 nir_intrinsic_instr *low, nir_intrinsic_instr *high)
628 {
629 if ((bit_size != 32 && bit_size != 64) || num_components > 4)
630 return false;
631
632 /* >128 bit loads are split except with SMEM */
633 if (bit_size * num_components > 128)
634 return false;
635
636 switch (low->intrinsic) {
637 case nir_intrinsic_load_ubo:
638 case nir_intrinsic_load_ssbo:
639 case nir_intrinsic_store_ssbo:
640 case nir_intrinsic_load_push_constant:
641 return align % 4 == 0;
642 case nir_intrinsic_load_deref:
643 case nir_intrinsic_store_deref:
644 assert(nir_src_as_deref(low->src[0])->mode == nir_var_mem_shared);
645 /* fallthrough */
646 case nir_intrinsic_load_shared:
647 case nir_intrinsic_store_shared:
648 if (bit_size * num_components > 64) /* 96 and 128 bit loads require 128 bit alignment and are split otherwise */
649 return align % 16 == 0;
650 else
651 return align % 4 == 0;
652 default:
653 return false;
654 }
655 return false;
656 }
657
658 void
659 setup_vs_variables(isel_context *ctx, nir_shader *nir)
660 {
661 nir_foreach_variable(variable, &nir->inputs)
662 {
663 variable->data.driver_location = variable->data.location * 4;
664 }
665 nir_foreach_variable(variable, &nir->outputs)
666 {
667 variable->data.driver_location = variable->data.location * 4;
668 }
669
670 radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
671
672 memset(outinfo->vs_output_param_offset, AC_EXP_PARAM_UNDEFINED,
673 sizeof(outinfo->vs_output_param_offset));
674
675 ctx->needs_instance_id = ctx->program->info->vs.needs_instance_id;
676
677 bool export_clip_dists = ctx->options->key.vs_common_out.export_clip_dists;
678
679 outinfo->param_exports = 0;
680 int pos_written = 0x1;
681 if (outinfo->writes_pointsize || outinfo->writes_viewport_index || outinfo->writes_layer)
682 pos_written |= 1 << 1;
683
684 nir_foreach_variable(variable, &nir->outputs)
685 {
686 int idx = variable->data.location;
687 unsigned slots = variable->type->count_attribute_slots(false);
688 if (variable->data.compact) {
689 unsigned component_count = variable->data.location_frac + variable->type->length;
690 slots = (component_count + 3) / 4;
691 }
692
693 if (idx >= VARYING_SLOT_VAR0 || idx == VARYING_SLOT_LAYER || idx == VARYING_SLOT_PRIMITIVE_ID ||
694 ((idx == VARYING_SLOT_CLIP_DIST0 || idx == VARYING_SLOT_CLIP_DIST1) && export_clip_dists)) {
695 for (unsigned i = 0; i < slots; i++) {
696 if (outinfo->vs_output_param_offset[idx + i] == AC_EXP_PARAM_UNDEFINED)
697 outinfo->vs_output_param_offset[idx + i] = outinfo->param_exports++;
698 }
699 }
700 }
701 if (outinfo->writes_layer &&
702 outinfo->vs_output_param_offset[VARYING_SLOT_LAYER] == AC_EXP_PARAM_UNDEFINED) {
703 /* when ctx->options->key.has_multiview_view_index = true, the layer
704 * variable isn't declared in NIR and it's isel's job to get the layer */
705 outinfo->vs_output_param_offset[VARYING_SLOT_LAYER] = outinfo->param_exports++;
706 }
707
708 if (outinfo->export_prim_id) {
709 assert(outinfo->vs_output_param_offset[VARYING_SLOT_PRIMITIVE_ID] == AC_EXP_PARAM_UNDEFINED);
710 outinfo->vs_output_param_offset[VARYING_SLOT_PRIMITIVE_ID] = outinfo->param_exports++;
711 }
712
713 ctx->num_clip_distances = util_bitcount(outinfo->clip_dist_mask);
714 ctx->num_cull_distances = util_bitcount(outinfo->cull_dist_mask);
715
716 assert(ctx->num_clip_distances + ctx->num_cull_distances <= 8);
717
718 if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
719 pos_written |= 1 << 2;
720 if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
721 pos_written |= 1 << 3;
722
723 outinfo->pos_exports = util_bitcount(pos_written);
724 }
725
726 void
727 setup_variables(isel_context *ctx, nir_shader *nir)
728 {
729 switch (nir->info.stage) {
730 case MESA_SHADER_FRAGMENT: {
731 nir_foreach_variable(variable, &nir->outputs)
732 {
733 int idx = variable->data.location + variable->data.index;
734 variable->data.driver_location = idx * 4;
735 }
736 break;
737 }
738 case MESA_SHADER_COMPUTE: {
739 ctx->program->config->lds_size = (nir->info.cs.shared_size + ctx->program->lds_alloc_granule - 1) /
740 ctx->program->lds_alloc_granule;
741 break;
742 }
743 case MESA_SHADER_VERTEX: {
744 setup_vs_variables(ctx, nir);
745 break;
746 }
747 default:
748 unreachable("Unhandled shader stage.");
749 }
750 }
751
752 isel_context
753 setup_isel_context(Program* program,
754 unsigned shader_count,
755 struct nir_shader *const *shaders,
756 ac_shader_config* config,
757 struct radv_shader_args *args)
758 {
759 program->stage = 0;
760 for (unsigned i = 0; i < shader_count; i++) {
761 switch (shaders[i]->info.stage) {
762 case MESA_SHADER_VERTEX:
763 program->stage |= sw_vs;
764 break;
765 case MESA_SHADER_TESS_CTRL:
766 program->stage |= sw_tcs;
767 break;
768 case MESA_SHADER_TESS_EVAL:
769 program->stage |= sw_tes;
770 break;
771 case MESA_SHADER_GEOMETRY:
772 program->stage |= sw_gs;
773 break;
774 case MESA_SHADER_FRAGMENT:
775 program->stage |= sw_fs;
776 break;
777 case MESA_SHADER_COMPUTE:
778 program->stage |= sw_cs;
779 break;
780 default:
781 unreachable("Shader stage not implemented");
782 }
783 }
784 if (program->stage == sw_vs)
785 program->stage |= hw_vs;
786 else if (program->stage == sw_fs)
787 program->stage |= hw_fs;
788 else if (program->stage == sw_cs)
789 program->stage |= hw_cs;
790 else
791 unreachable("Shader stage not implemented");
792
793 program->config = config;
794 program->info = args->shader_info;
795 program->chip_class = args->options->chip_class;
796 program->family = args->options->family;
797 program->wave_size = args->shader_info->wave_size;
798 program->lane_mask = program->wave_size == 32 ? s1 : s2;
799
800 program->lds_alloc_granule = args->options->chip_class >= GFX7 ? 512 : 256;
801 program->lds_limit = args->options->chip_class >= GFX7 ? 65536 : 32768;
802 program->vgpr_limit = 256;
803 program->vgpr_alloc_granule = 3;
804
805 if (args->options->chip_class >= GFX10) {
806 program->physical_sgprs = 2560; /* doesn't matter as long as it's at least 128 * 20 */
807 program->sgpr_alloc_granule = 127;
808 program->sgpr_limit = 106;
809 program->vgpr_alloc_granule = program->wave_size == 32 ? 7 : 3;
810 } else if (program->chip_class >= GFX8) {
811 program->physical_sgprs = 800;
812 program->sgpr_alloc_granule = 15;
813 if (args->options->family == CHIP_TONGA || args->options->family == CHIP_ICELAND)
814 program->sgpr_limit = 94; /* workaround hardware bug */
815 else
816 program->sgpr_limit = 102;
817 } else {
818 program->physical_sgprs = 512;
819 program->sgpr_alloc_granule = 7;
820 program->sgpr_limit = 104;
821 }
822
823 /* TODO: we don't have to allocate VCC if we don't need it */
824 program->needs_vcc = true;
825
826 calc_min_waves(program);
827 program->vgpr_limit = get_addr_vgpr_from_waves(program, program->min_waves);
828 program->sgpr_limit = get_addr_sgpr_from_waves(program, program->min_waves);
829
830 isel_context ctx = {};
831 ctx.program = program;
832 ctx.args = args;
833 ctx.options = args->options;
834 ctx.stage = program->stage;
835
836 for (unsigned i = 0; i < shader_count; i++) {
837 nir_shader *nir = shaders[i];
838
839 /* align and copy constant data */
840 while (program->constant_data.size() % 4u)
841 program->constant_data.push_back(0);
842 ctx.constant_data_offset = program->constant_data.size();
843 program->constant_data.insert(program->constant_data.end(),
844 (uint8_t*)nir->constant_data,
845 (uint8_t*)nir->constant_data + nir->constant_data_size);
846
847 /* the variable setup has to be done before lower_io / CSE */
848 setup_variables(&ctx, nir);
849
850 /* optimize and lower memory operations */
851 bool lower_to_scalar = false;
852 bool lower_pack = false;
853 if (nir_opt_load_store_vectorize(nir,
854 (nir_variable_mode)(nir_var_mem_ssbo | nir_var_mem_ubo |
855 nir_var_mem_push_const | nir_var_mem_shared),
856 mem_vectorize_callback)) {
857 lower_to_scalar = true;
858 lower_pack = true;
859 }
860 if (nir->info.stage != MESA_SHADER_COMPUTE)
861 nir_lower_io(nir, (nir_variable_mode)(nir_var_shader_in | nir_var_shader_out), type_size, (nir_lower_io_options)0);
862 nir_lower_explicit_io(nir, nir_var_mem_global, nir_address_format_64bit_global);
863
864 if (lower_to_scalar)
865 nir_lower_alu_to_scalar(nir, NULL, NULL);
866 if (lower_pack)
867 nir_lower_pack(nir);
868
869 /* lower ALU operations */
870 // TODO: implement logic64 in aco, it's more effective for sgprs
871 nir_lower_int64(nir, nir->options->lower_int64_options);
872
873 nir_opt_idiv_const(nir, 32);
874 nir_lower_idiv(nir, nir_lower_idiv_precise);
875
876 /* optimize the lowered ALU operations */
877 bool more_algebraic = true;
878 while (more_algebraic) {
879 more_algebraic = false;
880 NIR_PASS_V(nir, nir_copy_prop);
881 NIR_PASS_V(nir, nir_opt_dce);
882 NIR_PASS_V(nir, nir_opt_constant_folding);
883 NIR_PASS(more_algebraic, nir, nir_opt_algebraic);
884 }
885
886 /* Do late algebraic optimization to turn add(a, neg(b)) back into
887 * subs, then the mandatory cleanup after algebraic. Note that it may
888 * produce fnegs, and if so then we need to keep running to squash
889 * fneg(fneg(a)).
890 */
891 bool more_late_algebraic = true;
892 while (more_late_algebraic) {
893 more_late_algebraic = false;
894 NIR_PASS(more_late_algebraic, nir, nir_opt_algebraic_late);
895 NIR_PASS_V(nir, nir_opt_constant_folding);
896 NIR_PASS_V(nir, nir_copy_prop);
897 NIR_PASS_V(nir, nir_opt_dce);
898 NIR_PASS_V(nir, nir_opt_cse);
899 }
900
901 /* cleanup passes */
902 nir_lower_load_const_to_scalar(nir);
903 nir_opt_shrink_load(nir);
904 nir_move_options move_opts = (nir_move_options)(
905 nir_move_const_undef | nir_move_load_ubo | nir_move_load_input | nir_move_comparisons);
906 nir_opt_sink(nir, move_opts);
907 nir_opt_move(nir, move_opts);
908 nir_convert_to_lcssa(nir, true, false);
909 nir_lower_phis_to_scalar(nir);
910
911 nir_function_impl *func = nir_shader_get_entrypoint(nir);
912 nir_index_ssa_defs(func);
913 nir_metadata_require(func, nir_metadata_block_index);
914
915 if (args->options->dump_preoptir) {
916 fprintf(stderr, "NIR shader before instruction selection:\n");
917 nir_print_shader(nir, stderr);
918 }
919 }
920
921 unsigned scratch_size = 0;
922 for (unsigned i = 0; i < shader_count; i++)
923 scratch_size = std::max(scratch_size, shaders[i]->scratch_size);
924 ctx.program->config->scratch_bytes_per_wave = align(scratch_size * ctx.program->wave_size, 1024);
925
926 ctx.block = ctx.program->create_and_insert_block();
927 ctx.block->loop_nest_depth = 0;
928 ctx.block->kind = block_kind_top_level;
929
930 return ctx;
931 }
932
933 }