projects / mesa.git / blob
? search:


d663343d747b4dd717e59e0230c3ef29c97f39be
[mesa.git] / src / amd / compiler / aco_instruction_selection_setup.cpp
1 /*
2 * Copyright © 2018 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25 #include <array>
26 #include <unordered_map>
27 #include "aco_ir.h"
28 #include "nir.h"
29 #include "vulkan/radv_shader.h"
30 #include "vulkan/radv_descriptor_set.h"
31 #include "vulkan/radv_shader_args.h"
32 #include "sid.h"
33 #include "ac_exp_param.h"
34 #include "ac_shader_util.h"
35
36 #include "util/u_math.h"
37
38 #define MAX_INLINE_PUSH_CONSTS 8
39
40 namespace aco {
41
42 struct vs_output_state {
43 uint8_t mask[VARYING_SLOT_VAR31 + 1];
44 Temp outputs[VARYING_SLOT_VAR31 + 1][4];
45 };
46
47 struct isel_context {
48 const struct radv_nir_compiler_options *options;
49 struct radv_shader_args *args;
50 Program *program;
51 nir_shader *shader;
52 uint32_t constant_data_offset;
53 Block *block;
54 bool *divergent_vals;
55 std::unique_ptr<Temp[]> allocated;
56 std::unordered_map<unsigned, std::array<Temp,4>> allocated_vec;
57 Stage stage; /* Stage */
58 bool has_gfx10_wave64_bpermute = false;
59 struct {
60 bool has_branch;
61 uint16_t loop_nest_depth = 0;
62 struct {
63 unsigned header_idx;
64 Block* exit;
65 bool has_divergent_continue = false;
66 bool has_divergent_branch = false;
67 } parent_loop;
68 struct {
69 bool is_divergent = false;
70 } parent_if;
71 bool exec_potentially_empty = false;
72 } cf_info;
73
74 Temp arg_temps[AC_MAX_ARGS];
75
76 /* inputs common for merged stages */
77 Temp merged_wave_info = Temp(0, s1);
78
79 /* FS inputs */
80 Temp persp_centroid, linear_centroid;
81
82 /* VS inputs */
83 bool needs_instance_id;
84
85 /* VS output information */
86 unsigned num_clip_distances;
87 unsigned num_cull_distances;
88 vs_output_state vs_output;
89 };
90
91 Temp get_arg(isel_context *ctx, struct ac_arg arg)
92 {
93 assert(arg.used);
94 return ctx->arg_temps[arg.arg_index];
95 }
96
97 unsigned get_interp_input(nir_intrinsic_op intrin, enum glsl_interp_mode interp)
98 {
99 switch (interp) {
100 case INTERP_MODE_SMOOTH:
101 case INTERP_MODE_NONE:
102 if (intrin == nir_intrinsic_load_barycentric_pixel ||
103 intrin == nir_intrinsic_load_barycentric_at_sample ||
104 intrin == nir_intrinsic_load_barycentric_at_offset)
105 return S_0286CC_PERSP_CENTER_ENA(1);
106 else if (intrin == nir_intrinsic_load_barycentric_centroid)
107 return S_0286CC_PERSP_CENTROID_ENA(1);
108 else if (intrin == nir_intrinsic_load_barycentric_sample)
109 return S_0286CC_PERSP_SAMPLE_ENA(1);
110 break;
111 case INTERP_MODE_NOPERSPECTIVE:
112 if (intrin == nir_intrinsic_load_barycentric_pixel)
113 return S_0286CC_LINEAR_CENTER_ENA(1);
114 else if (intrin == nir_intrinsic_load_barycentric_centroid)
115 return S_0286CC_LINEAR_CENTROID_ENA(1);
116 else if (intrin == nir_intrinsic_load_barycentric_sample)
117 return S_0286CC_LINEAR_SAMPLE_ENA(1);
118 break;
119 default:
120 break;
121 }
122 return 0;
123 }
124
125 void init_context(isel_context *ctx, nir_shader *shader)
126 {
127 nir_function_impl *impl = nir_shader_get_entrypoint(shader);
128
129 ctx->shader = shader;
130 ctx->divergent_vals = nir_divergence_analysis(shader, nir_divergence_view_index_uniform);
131
132 std::unique_ptr<Temp[]> allocated{new Temp[impl->ssa_alloc]()};
133
134 unsigned spi_ps_inputs = 0;
135
136 bool done = false;
137 while (!done) {
138 done = true;
139 nir_foreach_block(block, impl) {
140 nir_foreach_instr(instr, block) {
141 switch(instr->type) {
142 case nir_instr_type_alu: {
143 nir_alu_instr *alu_instr = nir_instr_as_alu(instr);
144 unsigned size = alu_instr->dest.dest.ssa.num_components;
145 if (alu_instr->dest.dest.ssa.bit_size == 64)
146 size *= 2;
147 RegType type = RegType::sgpr;
148 switch(alu_instr->op) {
149 case nir_op_fmul:
150 case nir_op_fadd:
151 case nir_op_fsub:
152 case nir_op_fmax:
153 case nir_op_fmin:
154 case nir_op_fmax3:
155 case nir_op_fmin3:
156 case nir_op_fmed3:
157 case nir_op_fneg:
158 case nir_op_fabs:
159 case nir_op_fsat:
160 case nir_op_fsign:
161 case nir_op_frcp:
162 case nir_op_frsq:
163 case nir_op_fsqrt:
164 case nir_op_fexp2:
165 case nir_op_flog2:
166 case nir_op_ffract:
167 case nir_op_ffloor:
168 case nir_op_fceil:
169 case nir_op_ftrunc:
170 case nir_op_fround_even:
171 case nir_op_fsin:
172 case nir_op_fcos:
173 case nir_op_f2f32:
174 case nir_op_f2f64:
175 case nir_op_u2f32:
176 case nir_op_u2f64:
177 case nir_op_i2f32:
178 case nir_op_i2f64:
179 case nir_op_pack_half_2x16:
180 case nir_op_unpack_half_2x16_split_x:
181 case nir_op_unpack_half_2x16_split_y:
182 case nir_op_fddx:
183 case nir_op_fddy:
184 case nir_op_fddx_fine:
185 case nir_op_fddy_fine:
186 case nir_op_fddx_coarse:
187 case nir_op_fddy_coarse:
188 case nir_op_fquantize2f16:
189 case nir_op_ldexp:
190 case nir_op_frexp_sig:
191 case nir_op_frexp_exp:
192 case nir_op_cube_face_index:
193 case nir_op_cube_face_coord:
194 type = RegType::vgpr;
195 break;
196 case nir_op_flt:
197 case nir_op_fge:
198 case nir_op_feq:
199 case nir_op_fne:
200 case nir_op_ilt:
201 case nir_op_ige:
202 case nir_op_ult:
203 case nir_op_uge:
204 case nir_op_ieq:
205 case nir_op_ine:
206 case nir_op_i2b1:
207 size = 2;
208 break;
209 case nir_op_f2i64:
210 case nir_op_f2u64:
211 case nir_op_b2i32:
212 case nir_op_b2f32:
213 case nir_op_f2i32:
214 case nir_op_f2u32:
215 type = ctx->divergent_vals[alu_instr->dest.dest.ssa.index] ? RegType::vgpr : RegType::sgpr;
216 break;
217 case nir_op_bcsel:
218 if (alu_instr->dest.dest.ssa.bit_size == 1) {
219 size = 2;
220 } else {
221 if (ctx->divergent_vals[alu_instr->dest.dest.ssa.index]) {
222 type = RegType::vgpr;
223 } else {
224 if (allocated[alu_instr->src[1].src.ssa->index].type() == RegType::vgpr ||
225 allocated[alu_instr->src[2].src.ssa->index].type() == RegType::vgpr) {
226 type = RegType::vgpr;
227 }
228 }
229 if (alu_instr->src[1].src.ssa->num_components == 1 && alu_instr->src[2].src.ssa->num_components == 1) {
230 assert(allocated[alu_instr->src[1].src.ssa->index].size() == allocated[alu_instr->src[2].src.ssa->index].size());
231 size = allocated[alu_instr->src[1].src.ssa->index].size();
232 }
233 }
234 break;
235 case nir_op_mov:
236 if (alu_instr->dest.dest.ssa.bit_size == 1) {
237 size = 2;
238 } else {
239 type = ctx->divergent_vals[alu_instr->dest.dest.ssa.index] ? RegType::vgpr : RegType::sgpr;
240 }
241 break;
242 default:
243 if (alu_instr->dest.dest.ssa.bit_size == 1) {
244 size = 2;
245 } else {
246 for (unsigned i = 0; i < nir_op_infos[alu_instr->op].num_inputs; i++) {
247 if (allocated[alu_instr->src[i].src.ssa->index].type() == RegType::vgpr)
248 type = RegType::vgpr;
249 }
250 }
251 break;
252 }
253 allocated[alu_instr->dest.dest.ssa.index] = Temp(0, RegClass(type, size));
254 break;
255 }
256 case nir_instr_type_load_const: {
257 unsigned size = nir_instr_as_load_const(instr)->def.num_components;
258 if (nir_instr_as_load_const(instr)->def.bit_size == 64)
259 size *= 2;
260 else if (nir_instr_as_load_const(instr)->def.bit_size == 1)
261 size *= 2;
262 allocated[nir_instr_as_load_const(instr)->def.index] = Temp(0, RegClass(RegType::sgpr, size));
263 break;
264 }
265 case nir_instr_type_intrinsic: {
266 nir_intrinsic_instr *intrinsic = nir_instr_as_intrinsic(instr);
267 if (!nir_intrinsic_infos[intrinsic->intrinsic].has_dest)
268 break;
269 unsigned size = intrinsic->dest.ssa.num_components;
270 if (intrinsic->dest.ssa.bit_size == 64)
271 size *= 2;
272 RegType type = RegType::sgpr;
273 switch(intrinsic->intrinsic) {
274 case nir_intrinsic_load_push_constant:
275 case nir_intrinsic_load_work_group_id:
276 case nir_intrinsic_load_num_work_groups:
277 case nir_intrinsic_load_subgroup_id:
278 case nir_intrinsic_load_num_subgroups:
279 case nir_intrinsic_load_first_vertex:
280 case nir_intrinsic_load_base_instance:
281 case nir_intrinsic_get_buffer_size:
282 case nir_intrinsic_vote_all:
283 case nir_intrinsic_vote_any:
284 case nir_intrinsic_read_first_invocation:
285 case nir_intrinsic_read_invocation:
286 case nir_intrinsic_first_invocation:
287 type = RegType::sgpr;
288 if (intrinsic->dest.ssa.bit_size == 1)
289 size = 2;
290 break;
291 case nir_intrinsic_ballot:
292 type = RegType::sgpr;
293 size = 2;
294 break;
295 case nir_intrinsic_load_sample_id:
296 case nir_intrinsic_load_sample_mask_in:
297 case nir_intrinsic_load_input:
298 case nir_intrinsic_load_vertex_id:
299 case nir_intrinsic_load_vertex_id_zero_base:
300 case nir_intrinsic_load_barycentric_sample:
301 case nir_intrinsic_load_barycentric_pixel:
302 case nir_intrinsic_load_barycentric_centroid:
303 case nir_intrinsic_load_barycentric_at_sample:
304 case nir_intrinsic_load_barycentric_at_offset:
305 case nir_intrinsic_load_interpolated_input:
306 case nir_intrinsic_load_frag_coord:
307 case nir_intrinsic_load_sample_pos:
308 case nir_intrinsic_load_layer_id:
309 case nir_intrinsic_load_local_invocation_id:
310 case nir_intrinsic_load_local_invocation_index:
311 case nir_intrinsic_load_subgroup_invocation:
312 case nir_intrinsic_write_invocation_amd:
313 case nir_intrinsic_mbcnt_amd:
314 case nir_intrinsic_load_instance_id:
315 case nir_intrinsic_ssbo_atomic_add:
316 case nir_intrinsic_ssbo_atomic_imin:
317 case nir_intrinsic_ssbo_atomic_umin:
318 case nir_intrinsic_ssbo_atomic_imax:
319 case nir_intrinsic_ssbo_atomic_umax:
320 case nir_intrinsic_ssbo_atomic_and:
321 case nir_intrinsic_ssbo_atomic_or:
322 case nir_intrinsic_ssbo_atomic_xor:
323 case nir_intrinsic_ssbo_atomic_exchange:
324 case nir_intrinsic_ssbo_atomic_comp_swap:
325 case nir_intrinsic_image_deref_atomic_add:
326 case nir_intrinsic_image_deref_atomic_umin:
327 case nir_intrinsic_image_deref_atomic_imin:
328 case nir_intrinsic_image_deref_atomic_umax:
329 case nir_intrinsic_image_deref_atomic_imax:
330 case nir_intrinsic_image_deref_atomic_and:
331 case nir_intrinsic_image_deref_atomic_or:
332 case nir_intrinsic_image_deref_atomic_xor:
333 case nir_intrinsic_image_deref_atomic_exchange:
334 case nir_intrinsic_image_deref_atomic_comp_swap:
335 case nir_intrinsic_image_deref_size:
336 case nir_intrinsic_shared_atomic_add:
337 case nir_intrinsic_shared_atomic_imin:
338 case nir_intrinsic_shared_atomic_umin:
339 case nir_intrinsic_shared_atomic_imax:
340 case nir_intrinsic_shared_atomic_umax:
341 case nir_intrinsic_shared_atomic_and:
342 case nir_intrinsic_shared_atomic_or:
343 case nir_intrinsic_shared_atomic_xor:
344 case nir_intrinsic_shared_atomic_exchange:
345 case nir_intrinsic_shared_atomic_comp_swap:
346 case nir_intrinsic_load_scratch:
347 type = RegType::vgpr;
348 break;
349 case nir_intrinsic_shuffle:
350 case nir_intrinsic_quad_broadcast:
351 case nir_intrinsic_quad_swap_horizontal:
352 case nir_intrinsic_quad_swap_vertical:
353 case nir_intrinsic_quad_swap_diagonal:
354 case nir_intrinsic_quad_swizzle_amd:
355 case nir_intrinsic_masked_swizzle_amd:
356 case nir_intrinsic_inclusive_scan:
357 case nir_intrinsic_exclusive_scan:
358 if (intrinsic->dest.ssa.bit_size == 1) {
359 size = 2;
360 type = RegType::sgpr;
361 } else if (!ctx->divergent_vals[intrinsic->dest.ssa.index]) {
362 type = RegType::sgpr;
363 } else {
364 type = RegType::vgpr;
365 }
366 break;
367 case nir_intrinsic_load_view_index:
368 type = ctx->stage == fragment_fs ? RegType::vgpr : RegType::sgpr;
369 break;
370 case nir_intrinsic_load_front_face:
371 case nir_intrinsic_load_helper_invocation:
372 case nir_intrinsic_is_helper_invocation:
373 type = RegType::sgpr;
374 size = 2;
375 break;
376 case nir_intrinsic_reduce:
377 if (intrinsic->dest.ssa.bit_size == 1) {
378 size = 2;
379 type = RegType::sgpr;
380 } else if (nir_intrinsic_cluster_size(intrinsic) == 0 ||
381 !ctx->divergent_vals[intrinsic->dest.ssa.index]) {
382 type = RegType::sgpr;
383 } else {
384 type = RegType::vgpr;
385 }
386 break;
387 case nir_intrinsic_load_ubo:
388 case nir_intrinsic_load_ssbo:
389 case nir_intrinsic_load_global:
390 case nir_intrinsic_vulkan_resource_index:
391 type = ctx->divergent_vals[intrinsic->dest.ssa.index] ? RegType::vgpr : RegType::sgpr;
392 break;
393 /* due to copy propagation, the swizzled imov is removed if num dest components == 1 */
394 case nir_intrinsic_load_shared:
395 if (ctx->divergent_vals[intrinsic->dest.ssa.index])
396 type = RegType::vgpr;
397 else
398 type = RegType::sgpr;
399 break;
400 default:
401 for (unsigned i = 0; i < nir_intrinsic_infos[intrinsic->intrinsic].num_srcs; i++) {
402 if (allocated[intrinsic->src[i].ssa->index].type() == RegType::vgpr)
403 type = RegType::vgpr;
404 }
405 break;
406 }
407 allocated[intrinsic->dest.ssa.index] = Temp(0, RegClass(type, size));
408
409 switch(intrinsic->intrinsic) {
410 case nir_intrinsic_load_barycentric_sample:
411 case nir_intrinsic_load_barycentric_pixel:
412 case nir_intrinsic_load_barycentric_centroid:
413 case nir_intrinsic_load_barycentric_at_sample:
414 case nir_intrinsic_load_barycentric_at_offset: {
415 glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(intrinsic);
416 spi_ps_inputs |= get_interp_input(intrinsic->intrinsic, mode);
417 break;
418 }
419 case nir_intrinsic_load_front_face:
420 spi_ps_inputs |= S_0286CC_FRONT_FACE_ENA(1);
421 break;
422 case nir_intrinsic_load_frag_coord:
423 case nir_intrinsic_load_sample_pos: {
424 uint8_t mask = nir_ssa_def_components_read(&intrinsic->dest.ssa);
425 for (unsigned i = 0; i < 4; i++) {
426 if (mask & (1 << i))
427 spi_ps_inputs |= S_0286CC_POS_X_FLOAT_ENA(1) << i;
428
429 }
430 break;
431 }
432 case nir_intrinsic_load_sample_id:
433 spi_ps_inputs |= S_0286CC_ANCILLARY_ENA(1);
434 break;
435 case nir_intrinsic_load_sample_mask_in:
436 spi_ps_inputs |= S_0286CC_ANCILLARY_ENA(1);
437 spi_ps_inputs |= S_0286CC_SAMPLE_COVERAGE_ENA(1);
438 break;
439 default:
440 break;
441 }
442 break;
443 }
444 case nir_instr_type_tex: {
445 nir_tex_instr* tex = nir_instr_as_tex(instr);
446 unsigned size = tex->dest.ssa.num_components;
447
448 if (tex->dest.ssa.bit_size == 64)
449 size *= 2;
450 if (tex->op == nir_texop_texture_samples)
451 assert(!ctx->divergent_vals[tex->dest.ssa.index]);
452 if (ctx->divergent_vals[tex->dest.ssa.index])
453 allocated[tex->dest.ssa.index] = Temp(0, RegClass(RegType::vgpr, size));
454 else
455 allocated[tex->dest.ssa.index] = Temp(0, RegClass(RegType::sgpr, size));
456 break;
457 }
458 case nir_instr_type_parallel_copy: {
459 nir_foreach_parallel_copy_entry(entry, nir_instr_as_parallel_copy(instr)) {
460 allocated[entry->dest.ssa.index] = allocated[entry->src.ssa->index];
461 }
462 break;
463 }
464 case nir_instr_type_ssa_undef: {
465 unsigned size = nir_instr_as_ssa_undef(instr)->def.num_components;
466 if (nir_instr_as_ssa_undef(instr)->def.bit_size == 64)
467 size *= 2;
468 allocated[nir_instr_as_ssa_undef(instr)->def.index] = Temp(0, RegClass(RegType::sgpr, size));
469 break;
470 }
471 case nir_instr_type_phi: {
472 nir_phi_instr* phi = nir_instr_as_phi(instr);
473 RegType type;
474 unsigned size = phi->dest.ssa.num_components;
475
476 if (phi->dest.ssa.bit_size == 1) {
477 assert(size == 1 && "multiple components not yet supported on boolean phis.");
478 type = RegType::sgpr;
479 size *= 2;
480 allocated[phi->dest.ssa.index] = Temp(0, RegClass(type, size));
481 break;
482 }
483
484 if (ctx->divergent_vals[phi->dest.ssa.index]) {
485 type = RegType::vgpr;
486 } else {
487 type = RegType::sgpr;
488 nir_foreach_phi_src (src, phi) {
489 if (allocated[src->src.ssa->index].type() == RegType::vgpr)
490 type = RegType::vgpr;
491 if (allocated[src->src.ssa->index].type() == RegType::none)
492 done = false;
493 }
494 }
495
496 size *= phi->dest.ssa.bit_size == 64 ? 2 : 1;
497 RegClass rc = RegClass(type, size);
498 if (rc != allocated[phi->dest.ssa.index].regClass()) {
499 done = false;
500 } else {
501 nir_foreach_phi_src(src, phi)
502 assert(allocated[src->src.ssa->index].size() == rc.size());
503 }
504 allocated[phi->dest.ssa.index] = Temp(0, rc);
505 break;
506 }
507 default:
508 break;
509 }
510 }
511 }
512 }
513
514 if (G_0286CC_POS_W_FLOAT_ENA(spi_ps_inputs)) {
515 /* If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be enabled too */
516 spi_ps_inputs |= S_0286CC_PERSP_CENTER_ENA(1);
517 }
518
519 if (!(spi_ps_inputs & 0x7F)) {
520 /* At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled */
521 spi_ps_inputs |= S_0286CC_PERSP_CENTER_ENA(1);
522 }
523
524 ctx->program->config->spi_ps_input_ena = spi_ps_inputs;
525 ctx->program->config->spi_ps_input_addr = spi_ps_inputs;
526
527 for (unsigned i = 0; i < impl->ssa_alloc; i++)
528 allocated[i] = Temp(ctx->program->allocateId(), allocated[i].regClass());
529
530 ctx->allocated.reset(allocated.release());
531 }
532
533 Pseudo_instruction *add_startpgm(struct isel_context *ctx)
534 {
535 unsigned arg_count = ctx->args->ac.arg_count;
536 if (ctx->stage == fragment_fs) {
537 /* LLVM optimizes away unused FS inputs and computes spi_ps_input_addr
538 * itself and then communicates the results back via the ELF binary.
539 * Mirror what LLVM does by re-mapping the VGPR arguments here.
540 *
541 * TODO: If we made the FS input scanning code into a separate pass that
542 * could run before argument setup, then this wouldn't be necessary
543 * anymore.
544 */
545 struct ac_shader_args *args = &ctx->args->ac;
546 arg_count = 0;
547 for (unsigned i = 0, vgpr_arg = 0, vgpr_reg = 0; i < args->arg_count; i++) {
548 if (args->args[i].file != AC_ARG_VGPR) {
549 arg_count++;
550 continue;
551 }
552
553 if (!(ctx->program->config->spi_ps_input_addr & (1 << vgpr_arg))) {
554 args->args[i].skip = true;
555 } else {
556 args->args[i].offset = vgpr_reg;
557 vgpr_reg += args->args[i].size;
558 arg_count++;
559 }
560 vgpr_arg++;
561 }
562 }
563
564 aco_ptr<Pseudo_instruction> startpgm{create_instruction<Pseudo_instruction>(aco_opcode::p_startpgm, Format::PSEUDO, 0, arg_count + 1)};
565 for (unsigned i = 0, arg = 0; i < ctx->args->ac.arg_count; i++) {
566 if (ctx->args->ac.args[i].skip)
567 continue;
568
569 enum ac_arg_regfile file = ctx->args->ac.args[i].file;
570 unsigned size = ctx->args->ac.args[i].size;
571 unsigned reg = ctx->args->ac.args[i].offset;
572 RegClass type = RegClass(file == AC_ARG_SGPR ? RegType::sgpr : RegType::vgpr, size);
573 Temp dst = Temp{ctx->program->allocateId(), type};
574 ctx->arg_temps[i] = dst;
575 startpgm->definitions[arg] = Definition(dst);
576 startpgm->definitions[arg].setFixed(PhysReg{file == AC_ARG_SGPR ? reg : reg + 256});
577 arg++;
578 }
579 startpgm->definitions[arg_count] = Definition{ctx->program->allocateId(), exec, s2};
580 Pseudo_instruction *instr = startpgm.get();
581 ctx->block->instructions.push_back(std::move(startpgm));
582
583 /* Stash these in the program so that they can be accessed later when
584 * handling spilling.
585 */
586 ctx->program->private_segment_buffer = get_arg(ctx, ctx->args->ring_offsets);
587 ctx->program->scratch_offset = get_arg(ctx, ctx->args->scratch_offset);
588
589 return instr;
590 }
591
592 int
593 type_size(const struct glsl_type *type, bool bindless)
594 {
595 // TODO: don't we need type->std430_base_alignment() here?
596 return glsl_count_attribute_slots(type, false);
597 }
598
599 void
600 shared_var_info(const struct glsl_type *type, unsigned *size, unsigned *align)
601 {
602 assert(glsl_type_is_vector_or_scalar(type));
603
604 uint32_t comp_size = glsl_type_is_boolean(type)
605 ? 4 : glsl_get_bit_size(type) / 8;
606 unsigned length = glsl_get_vector_elements(type);
607 *size = comp_size * length,
608 *align = comp_size;
609 }
610
611 static bool
612 mem_vectorize_callback(unsigned align, unsigned bit_size,
613 unsigned num_components, unsigned high_offset,
614 nir_intrinsic_instr *low, nir_intrinsic_instr *high)
615 {
616 if ((bit_size != 32 && bit_size != 64) || num_components > 4)
617 return false;
618
619 /* >128 bit loads are split except with SMEM */
620 if (bit_size * num_components > 128)
621 return false;
622
623 switch (low->intrinsic) {
624 case nir_intrinsic_load_ubo:
625 case nir_intrinsic_load_ssbo:
626 case nir_intrinsic_store_ssbo:
627 case nir_intrinsic_load_push_constant:
628 return align % 4 == 0;
629 case nir_intrinsic_load_deref:
630 case nir_intrinsic_store_deref:
631 assert(nir_src_as_deref(low->src[0])->mode == nir_var_mem_shared);
632 /* fallthrough */
633 case nir_intrinsic_load_shared:
634 case nir_intrinsic_store_shared:
635 if (bit_size * num_components > 64) /* 96 and 128 bit loads require 128 bit alignment and are split otherwise */
636 return align % 16 == 0;
637 else
638 return align % 4 == 0;
639 default:
640 return false;
641 }
642 return false;
643 }
644
645 void
646 setup_vs_variables(isel_context *ctx, nir_shader *nir)
647 {
648 nir_foreach_variable(variable, &nir->inputs)
649 {
650 variable->data.driver_location = variable->data.location * 4;
651 }
652 nir_foreach_variable(variable, &nir->outputs)
653 {
654 variable->data.driver_location = variable->data.location * 4;
655 }
656
657 radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
658
659 memset(outinfo->vs_output_param_offset, AC_EXP_PARAM_UNDEFINED,
660 sizeof(outinfo->vs_output_param_offset));
661
662 ctx->needs_instance_id = ctx->program->info->vs.needs_instance_id;
663
664 bool export_clip_dists = ctx->options->key.vs_common_out.export_clip_dists;
665
666 outinfo->param_exports = 0;
667 int pos_written = 0x1;
668 if (outinfo->writes_pointsize || outinfo->writes_viewport_index || outinfo->writes_layer)
669 pos_written |= 1 << 1;
670
671 nir_foreach_variable(variable, &nir->outputs)
672 {
673 int idx = variable->data.location;
674 unsigned slots = variable->type->count_attribute_slots(false);
675 if (variable->data.compact) {
676 unsigned component_count = variable->data.location_frac + variable->type->length;
677 slots = (component_count + 3) / 4;
678 }
679
680 if (idx >= VARYING_SLOT_VAR0 || idx == VARYING_SLOT_LAYER || idx == VARYING_SLOT_PRIMITIVE_ID ||
681 ((idx == VARYING_SLOT_CLIP_DIST0 || idx == VARYING_SLOT_CLIP_DIST1) && export_clip_dists)) {
682 for (unsigned i = 0; i < slots; i++) {
683 if (outinfo->vs_output_param_offset[idx + i] == AC_EXP_PARAM_UNDEFINED)
684 outinfo->vs_output_param_offset[idx + i] = outinfo->param_exports++;
685 }
686 }
687 }
688 if (outinfo->writes_layer &&
689 outinfo->vs_output_param_offset[VARYING_SLOT_LAYER] == AC_EXP_PARAM_UNDEFINED) {
690 /* when ctx->options->key.has_multiview_view_index = true, the layer
691 * variable isn't declared in NIR and it's isel's job to get the layer */
692 outinfo->vs_output_param_offset[VARYING_SLOT_LAYER] = outinfo->param_exports++;
693 }
694
695 if (outinfo->export_prim_id) {
696 assert(outinfo->vs_output_param_offset[VARYING_SLOT_PRIMITIVE_ID] == AC_EXP_PARAM_UNDEFINED);
697 outinfo->vs_output_param_offset[VARYING_SLOT_PRIMITIVE_ID] = outinfo->param_exports++;
698 }
699
700 ctx->num_clip_distances = util_bitcount(outinfo->clip_dist_mask);
701 ctx->num_cull_distances = util_bitcount(outinfo->cull_dist_mask);
702
703 assert(ctx->num_clip_distances + ctx->num_cull_distances <= 8);
704
705 if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
706 pos_written |= 1 << 2;
707 if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
708 pos_written |= 1 << 3;
709
710 outinfo->pos_exports = util_bitcount(pos_written);
711 }
712
713 void
714 setup_variables(isel_context *ctx, nir_shader *nir)
715 {
716 switch (nir->info.stage) {
717 case MESA_SHADER_FRAGMENT: {
718 nir_foreach_variable(variable, &nir->outputs)
719 {
720 int idx = variable->data.location + variable->data.index;
721 variable->data.driver_location = idx * 4;
722 }
723 break;
724 }
725 case MESA_SHADER_COMPUTE: {
726 ctx->program->config->lds_size = (nir->info.cs.shared_size + ctx->program->lds_alloc_granule - 1) /
727 ctx->program->lds_alloc_granule;
728 break;
729 }
730 case MESA_SHADER_VERTEX: {
731 setup_vs_variables(ctx, nir);
732 break;
733 }
734 default:
735 unreachable("Unhandled shader stage.");
736 }
737 }
738
739 isel_context
740 setup_isel_context(Program* program,
741 unsigned shader_count,
742 struct nir_shader *const *shaders,
743 ac_shader_config* config,
744 struct radv_shader_args *args)
745 {
746 program->stage = 0;
747 for (unsigned i = 0; i < shader_count; i++) {
748 switch (shaders[i]->info.stage) {
749 case MESA_SHADER_VERTEX:
750 program->stage |= sw_vs;
751 break;
752 case MESA_SHADER_TESS_CTRL:
753 program->stage |= sw_tcs;
754 break;
755 case MESA_SHADER_TESS_EVAL:
756 program->stage |= sw_tes;
757 break;
758 case MESA_SHADER_GEOMETRY:
759 program->stage |= sw_gs;
760 break;
761 case MESA_SHADER_FRAGMENT:
762 program->stage |= sw_fs;
763 break;
764 case MESA_SHADER_COMPUTE:
765 program->stage |= sw_cs;
766 break;
767 default:
768 unreachable("Shader stage not implemented");
769 }
770 }
771 if (program->stage == sw_vs)
772 program->stage |= hw_vs;
773 else if (program->stage == sw_fs)
774 program->stage |= hw_fs;
775 else if (program->stage == sw_cs)
776 program->stage |= hw_cs;
777 else
778 unreachable("Shader stage not implemented");
779
780 program->config = config;
781 program->info = args->shader_info;
782 program->chip_class = args->options->chip_class;
783 program->family = args->options->family;
784 program->wave_size = args->shader_info->wave_size;
785
786 program->lds_alloc_granule = args->options->chip_class >= GFX7 ? 512 : 256;
787 program->lds_limit = args->options->chip_class >= GFX7 ? 65536 : 32768;
788 program->vgpr_limit = 256;
789
790 if (args->options->chip_class >= GFX10) {
791 program->physical_sgprs = 2560; /* doesn't matter as long as it's at least 128 * 20 */
792 program->sgpr_alloc_granule = 127;
793 program->sgpr_limit = 106;
794 } else if (program->chip_class >= GFX8) {
795 program->physical_sgprs = 800;
796 program->sgpr_alloc_granule = 15;
797 if (args->options->family == CHIP_TONGA || args->options->family == CHIP_ICELAND)
798 program->sgpr_limit = 94; /* workaround hardware bug */
799 else
800 program->sgpr_limit = 102;
801 } else {
802 program->physical_sgprs = 512;
803 program->sgpr_alloc_granule = 7;
804 program->sgpr_limit = 104;
805 }
806 /* TODO: we don't have to allocate VCC if we don't need it */
807 program->needs_vcc = true;
808
809 isel_context ctx = {};
810 ctx.program = program;
811 ctx.args = args;
812 ctx.options = args->options;
813 ctx.stage = program->stage;
814
815 for (unsigned i = 0; i < shader_count; i++) {
816 nir_shader *nir = shaders[i];
817
818 /* align and copy constant data */
819 while (program->constant_data.size() % 4u)
820 program->constant_data.push_back(0);
821 ctx.constant_data_offset = program->constant_data.size();
822 program->constant_data.insert(program->constant_data.end(),
823 (uint8_t*)nir->constant_data,
824 (uint8_t*)nir->constant_data + nir->constant_data_size);
825
826 /* the variable setup has to be done before lower_io / CSE */
827 if (nir->info.stage == MESA_SHADER_COMPUTE)
828 nir_lower_vars_to_explicit_types(nir, nir_var_mem_shared, shared_var_info);
829 setup_variables(&ctx, nir);
830
831 /* optimize and lower memory operations */
832 bool lower_to_scalar = false;
833 bool lower_pack = false;
834 if (nir_opt_load_store_vectorize(nir,
835 (nir_variable_mode)(nir_var_mem_ssbo | nir_var_mem_ubo |
836 nir_var_mem_push_const | nir_var_mem_shared),
837 mem_vectorize_callback)) {
838 lower_to_scalar = true;
839 lower_pack = true;
840 }
841 if (nir->info.stage == MESA_SHADER_COMPUTE)
842 lower_to_scalar |= nir_lower_explicit_io(nir, nir_var_mem_shared, nir_address_format_32bit_offset);
843 else
844 nir_lower_io(nir, (nir_variable_mode)(nir_var_shader_in | nir_var_shader_out), type_size, (nir_lower_io_options)0);
845 nir_lower_explicit_io(nir, nir_var_mem_global, nir_address_format_64bit_global);
846
847 if (lower_to_scalar)
848 nir_lower_alu_to_scalar(nir, NULL, NULL);
849 if (lower_pack)
850 nir_lower_pack(nir);
851
852 /* lower ALU operations */
853 // TODO: implement logic64 in aco, it's more effective for sgprs
854 nir_lower_int64(nir, nir->options->lower_int64_options);
855
856 nir_opt_idiv_const(nir, 32);
857 nir_lower_idiv(nir, nir_lower_idiv_precise);
858
859 /* optimize the lowered ALU operations */
860 bool more_algebraic = true;
861 while (more_algebraic) {
862 more_algebraic = false;
863 NIR_PASS_V(nir, nir_copy_prop);
864 NIR_PASS_V(nir, nir_opt_dce);
865 NIR_PASS_V(nir, nir_opt_constant_folding);
866 NIR_PASS(more_algebraic, nir, nir_opt_algebraic);
867 }
868
869 /* Do late algebraic optimization to turn add(a, neg(b)) back into
870 * subs, then the mandatory cleanup after algebraic. Note that it may
871 * produce fnegs, and if so then we need to keep running to squash
872 * fneg(fneg(a)).
873 */
874 bool more_late_algebraic = true;
875 while (more_late_algebraic) {
876 more_late_algebraic = false;
877 NIR_PASS(more_late_algebraic, nir, nir_opt_algebraic_late);
878 NIR_PASS_V(nir, nir_opt_constant_folding);
879 NIR_PASS_V(nir, nir_copy_prop);
880 NIR_PASS_V(nir, nir_opt_dce);
881 NIR_PASS_V(nir, nir_opt_cse);
882 }
883
884 /* cleanup passes */
885 nir_lower_load_const_to_scalar(nir);
886 nir_opt_shrink_load(nir);
887 nir_move_options move_opts = (nir_move_options)(
888 nir_move_const_undef | nir_move_load_ubo | nir_move_load_input | nir_move_comparisons);
889 nir_opt_sink(nir, move_opts);
890 nir_opt_move(nir, move_opts);
891 nir_convert_to_lcssa(nir, true, false);
892 nir_lower_phis_to_scalar(nir);
893
894 nir_function_impl *func = nir_shader_get_entrypoint(nir);
895 nir_index_ssa_defs(func);
896
897 if (args->options->dump_preoptir) {
898 fprintf(stderr, "NIR shader before instruction selection:\n");
899 nir_print_shader(nir, stderr);
900 }
901 }
902
903 unsigned scratch_size = 0;
904 for (unsigned i = 0; i < shader_count; i++)
905 scratch_size = std::max(scratch_size, shaders[i]->scratch_size);
906 ctx.program->config->scratch_bytes_per_wave = align(scratch_size * ctx.program->wave_size, 1024);
907
908 ctx.block = ctx.program->create_and_insert_block();
909 ctx.block->loop_nest_depth = 0;
910 ctx.block->kind = block_kind_top_level;
911
912 return ctx;
913 }
914
915 }