projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
ac/radeonsi: add num_work_groups to the abi
[mesa.git] / src / gallium / drivers / radeonsi / si_shader.c
1 /*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "gallivm/lp_bld_const.h"
25 #include "gallivm/lp_bld_gather.h"
26 #include "gallivm/lp_bld_intr.h"
27 #include "gallivm/lp_bld_logic.h"
28 #include "gallivm/lp_bld_arit.h"
29 #include "gallivm/lp_bld_flow.h"
30 #include "gallivm/lp_bld_misc.h"
31 #include "util/u_memory.h"
32 #include "util/u_string.h"
33 #include "tgsi/tgsi_build.h"
34 #include "tgsi/tgsi_util.h"
35 #include "tgsi/tgsi_dump.h"
36
37 #include "ac_binary.h"
38 #include "ac_llvm_util.h"
39 #include "ac_exp_param.h"
40 #include "ac_shader_util.h"
41 #include "si_shader_internal.h"
42 #include "si_pipe.h"
43 #include "sid.h"
44
45 #include "compiler/nir/nir.h"
46
47 static const char *scratch_rsrc_dword0_symbol =
48 "SCRATCH_RSRC_DWORD0";
49
50 static const char *scratch_rsrc_dword1_symbol =
51 "SCRATCH_RSRC_DWORD1";
52
53 struct si_shader_output_values
54 {
55 LLVMValueRef values[4];
56 unsigned semantic_name;
57 unsigned semantic_index;
58 ubyte vertex_stream[4];
59 };
60
61 /**
62 * Used to collect types and other info about arguments of the LLVM function
63 * before the function is created.
64 */
65 struct si_function_info {
66 LLVMTypeRef types[100];
67 LLVMValueRef *assign[100];
68 unsigned num_sgpr_params;
69 unsigned num_params;
70 };
71
72 enum si_arg_regfile {
73 ARG_SGPR,
74 ARG_VGPR
75 };
76
77 static void si_init_shader_ctx(struct si_shader_context *ctx,
78 struct si_screen *sscreen,
79 LLVMTargetMachineRef tm);
80
81 static void si_llvm_emit_barrier(const struct lp_build_tgsi_action *action,
82 struct lp_build_tgsi_context *bld_base,
83 struct lp_build_emit_data *emit_data);
84
85 static void si_dump_shader_key(unsigned processor, const struct si_shader *shader,
86 FILE *f);
87
88 static void si_build_vs_prolog_function(struct si_shader_context *ctx,
89 union si_shader_part_key *key);
90 static void si_build_tcs_epilog_function(struct si_shader_context *ctx,
91 union si_shader_part_key *key);
92 static void si_build_ps_prolog_function(struct si_shader_context *ctx,
93 union si_shader_part_key *key);
94 static void si_build_ps_epilog_function(struct si_shader_context *ctx,
95 union si_shader_part_key *key);
96
97 /* Ideally pass the sample mask input to the PS epilog as v14, which
98 * is its usual location, so that the shader doesn't have to add v_mov.
99 */
100 #define PS_EPILOG_SAMPLEMASK_MIN_LOC 14
101
102 static bool llvm_type_is_64bit(struct si_shader_context *ctx,
103 LLVMTypeRef type)
104 {
105 if (type == ctx->ac.i64 || type == ctx->ac.f64)
106 return true;
107
108 return false;
109 }
110
111 static bool is_merged_shader(struct si_shader *shader)
112 {
113 if (shader->selector->screen->info.chip_class <= VI)
114 return false;
115
116 return shader->key.as_ls ||
117 shader->key.as_es ||
118 shader->selector->type == PIPE_SHADER_TESS_CTRL ||
119 shader->selector->type == PIPE_SHADER_GEOMETRY;
120 }
121
122 static void si_init_function_info(struct si_function_info *fninfo)
123 {
124 fninfo->num_params = 0;
125 fninfo->num_sgpr_params = 0;
126 }
127
128 static unsigned add_arg_assign(struct si_function_info *fninfo,
129 enum si_arg_regfile regfile, LLVMTypeRef type,
130 LLVMValueRef *assign)
131 {
132 assert(regfile != ARG_SGPR || fninfo->num_sgpr_params == fninfo->num_params);
133
134 unsigned idx = fninfo->num_params++;
135 assert(idx < ARRAY_SIZE(fninfo->types));
136
137 if (regfile == ARG_SGPR)
138 fninfo->num_sgpr_params = fninfo->num_params;
139
140 fninfo->types[idx] = type;
141 fninfo->assign[idx] = assign;
142 return idx;
143 }
144
145 static unsigned add_arg(struct si_function_info *fninfo,
146 enum si_arg_regfile regfile, LLVMTypeRef type)
147 {
148 return add_arg_assign(fninfo, regfile, type, NULL);
149 }
150
151 static void add_arg_assign_checked(struct si_function_info *fninfo,
152 enum si_arg_regfile regfile, LLVMTypeRef type,
153 LLVMValueRef *assign, unsigned idx)
154 {
155 MAYBE_UNUSED unsigned actual = add_arg_assign(fninfo, regfile, type, assign);
156 assert(actual == idx);
157 }
158
159 static void add_arg_checked(struct si_function_info *fninfo,
160 enum si_arg_regfile regfile, LLVMTypeRef type,
161 unsigned idx)
162 {
163 add_arg_assign_checked(fninfo, regfile, type, NULL, idx);
164 }
165
166 /**
167 * Returns a unique index for a per-patch semantic name and index. The index
168 * must be less than 32, so that a 32-bit bitmask of used inputs or outputs
169 * can be calculated.
170 */
171 unsigned si_shader_io_get_unique_index_patch(unsigned semantic_name, unsigned index)
172 {
173 switch (semantic_name) {
174 case TGSI_SEMANTIC_TESSOUTER:
175 return 0;
176 case TGSI_SEMANTIC_TESSINNER:
177 return 1;
178 case TGSI_SEMANTIC_PATCH:
179 assert(index < 30);
180 return 2 + index;
181
182 default:
183 assert(!"invalid semantic name");
184 return 0;
185 }
186 }
187
188 /**
189 * Returns a unique index for a semantic name and index. The index must be
190 * less than 64, so that a 64-bit bitmask of used inputs or outputs can be
191 * calculated.
192 */
193 unsigned si_shader_io_get_unique_index(unsigned semantic_name, unsigned index)
194 {
195 switch (semantic_name) {
196 case TGSI_SEMANTIC_POSITION:
197 return 0;
198 case TGSI_SEMANTIC_GENERIC:
199 /* Since some shader stages use the the highest used IO index
200 * to determine the size to allocate for inputs/outputs
201 * (in LDS, tess and GS rings). GENERIC should be placed right
202 * after POSITION to make that size as small as possible.
203 */
204 if (index < SI_MAX_IO_GENERIC)
205 return 1 + index;
206
207 assert(!"invalid generic index");
208 return 0;
209 case TGSI_SEMANTIC_PSIZE:
210 return SI_MAX_IO_GENERIC + 1;
211 case TGSI_SEMANTIC_CLIPDIST:
212 assert(index <= 1);
213 return SI_MAX_IO_GENERIC + 2 + index;
214 case TGSI_SEMANTIC_FOG:
215 return SI_MAX_IO_GENERIC + 4;
216 case TGSI_SEMANTIC_LAYER:
217 return SI_MAX_IO_GENERIC + 5;
218 case TGSI_SEMANTIC_VIEWPORT_INDEX:
219 return SI_MAX_IO_GENERIC + 6;
220 case TGSI_SEMANTIC_PRIMID:
221 return SI_MAX_IO_GENERIC + 7;
222 case TGSI_SEMANTIC_COLOR: /* these alias */
223 case TGSI_SEMANTIC_BCOLOR:
224 assert(index < 2);
225 return SI_MAX_IO_GENERIC + 8 + index;
226 case TGSI_SEMANTIC_TEXCOORD:
227 assert(index < 8);
228 assert(SI_MAX_IO_GENERIC + 10 + index < 64);
229 return SI_MAX_IO_GENERIC + 10 + index;
230 default:
231 assert(!"invalid semantic name");
232 return 0;
233 }
234 }
235
236 /**
237 * Get the value of a shader input parameter and extract a bitfield.
238 */
239 static LLVMValueRef unpack_llvm_param(struct si_shader_context *ctx,
240 LLVMValueRef value, unsigned rshift,
241 unsigned bitwidth)
242 {
243 if (LLVMGetTypeKind(LLVMTypeOf(value)) == LLVMFloatTypeKind)
244 value = ac_to_integer(&ctx->ac, value);
245
246 if (rshift)
247 value = LLVMBuildLShr(ctx->ac.builder, value,
248 LLVMConstInt(ctx->i32, rshift, 0), "");
249
250 if (rshift + bitwidth < 32) {
251 unsigned mask = (1 << bitwidth) - 1;
252 value = LLVMBuildAnd(ctx->ac.builder, value,
253 LLVMConstInt(ctx->i32, mask, 0), "");
254 }
255
256 return value;
257 }
258
259 static LLVMValueRef unpack_param(struct si_shader_context *ctx,
260 unsigned param, unsigned rshift,
261 unsigned bitwidth)
262 {
263 LLVMValueRef value = LLVMGetParam(ctx->main_fn, param);
264
265 return unpack_llvm_param(ctx, value, rshift, bitwidth);
266 }
267
268 static LLVMValueRef get_rel_patch_id(struct si_shader_context *ctx)
269 {
270 switch (ctx->type) {
271 case PIPE_SHADER_TESS_CTRL:
272 return unpack_llvm_param(ctx, ctx->abi.tcs_rel_ids, 0, 8);
273
274 case PIPE_SHADER_TESS_EVAL:
275 return LLVMGetParam(ctx->main_fn,
276 ctx->param_tes_rel_patch_id);
277
278 default:
279 assert(0);
280 return NULL;
281 }
282 }
283
284 /* Tessellation shaders pass outputs to the next shader using LDS.
285 *
286 * LS outputs = TCS inputs
287 * TCS outputs = TES inputs
288 *
289 * The LDS layout is:
290 * - TCS inputs for patch 0
291 * - TCS inputs for patch 1
292 * - TCS inputs for patch 2 = get_tcs_in_current_patch_offset (if RelPatchID==2)
293 * - ...
294 * - TCS outputs for patch 0 = get_tcs_out_patch0_offset
295 * - Per-patch TCS outputs for patch 0 = get_tcs_out_patch0_patch_data_offset
296 * - TCS outputs for patch 1
297 * - Per-patch TCS outputs for patch 1
298 * - TCS outputs for patch 2 = get_tcs_out_current_patch_offset (if RelPatchID==2)
299 * - Per-patch TCS outputs for patch 2 = get_tcs_out_current_patch_data_offset (if RelPatchID==2)
300 * - ...
301 *
302 * All three shaders VS(LS), TCS, TES share the same LDS space.
303 */
304
305 static LLVMValueRef
306 get_tcs_in_patch_stride(struct si_shader_context *ctx)
307 {
308 return unpack_param(ctx, ctx->param_vs_state_bits, 8, 13);
309 }
310
311 static unsigned get_tcs_out_vertex_dw_stride_constant(struct si_shader_context *ctx)
312 {
313 assert(ctx->type == PIPE_SHADER_TESS_CTRL);
314
315 if (ctx->shader->key.mono.u.ff_tcs_inputs_to_copy)
316 return util_last_bit64(ctx->shader->key.mono.u.ff_tcs_inputs_to_copy) * 4;
317
318 return util_last_bit64(ctx->shader->selector->outputs_written) * 4;
319 }
320
321 static LLVMValueRef get_tcs_out_vertex_dw_stride(struct si_shader_context *ctx)
322 {
323 unsigned stride = get_tcs_out_vertex_dw_stride_constant(ctx);
324
325 return LLVMConstInt(ctx->i32, stride, 0);
326 }
327
328 static LLVMValueRef get_tcs_out_patch_stride(struct si_shader_context *ctx)
329 {
330 if (ctx->shader->key.mono.u.ff_tcs_inputs_to_copy)
331 return unpack_param(ctx, ctx->param_tcs_out_lds_layout, 0, 13);
332
333 const struct tgsi_shader_info *info = &ctx->shader->selector->info;
334 unsigned tcs_out_vertices = info->properties[TGSI_PROPERTY_TCS_VERTICES_OUT];
335 unsigned vertex_dw_stride = get_tcs_out_vertex_dw_stride_constant(ctx);
336 unsigned num_patch_outputs = util_last_bit64(ctx->shader->selector->patch_outputs_written);
337 unsigned patch_dw_stride = tcs_out_vertices * vertex_dw_stride +
338 num_patch_outputs * 4;
339 return LLVMConstInt(ctx->i32, patch_dw_stride, 0);
340 }
341
342 static LLVMValueRef
343 get_tcs_out_patch0_offset(struct si_shader_context *ctx)
344 {
345 return lp_build_mul_imm(&ctx->bld_base.uint_bld,
346 unpack_param(ctx,
347 ctx->param_tcs_out_lds_offsets,
348 0, 16),
349 4);
350 }
351
352 static LLVMValueRef
353 get_tcs_out_patch0_patch_data_offset(struct si_shader_context *ctx)
354 {
355 return lp_build_mul_imm(&ctx->bld_base.uint_bld,
356 unpack_param(ctx,
357 ctx->param_tcs_out_lds_offsets,
358 16, 16),
359 4);
360 }
361
362 static LLVMValueRef
363 get_tcs_in_current_patch_offset(struct si_shader_context *ctx)
364 {
365 LLVMValueRef patch_stride = get_tcs_in_patch_stride(ctx);
366 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
367
368 return LLVMBuildMul(ctx->ac.builder, patch_stride, rel_patch_id, "");
369 }
370
371 static LLVMValueRef
372 get_tcs_out_current_patch_offset(struct si_shader_context *ctx)
373 {
374 LLVMValueRef patch0_offset = get_tcs_out_patch0_offset(ctx);
375 LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
376 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
377
378 return LLVMBuildAdd(ctx->ac.builder, patch0_offset,
379 LLVMBuildMul(ctx->ac.builder, patch_stride,
380 rel_patch_id, ""),
381 "");
382 }
383
384 static LLVMValueRef
385 get_tcs_out_current_patch_data_offset(struct si_shader_context *ctx)
386 {
387 LLVMValueRef patch0_patch_data_offset =
388 get_tcs_out_patch0_patch_data_offset(ctx);
389 LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
390 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
391
392 return LLVMBuildAdd(ctx->ac.builder, patch0_patch_data_offset,
393 LLVMBuildMul(ctx->ac.builder, patch_stride,
394 rel_patch_id, ""),
395 "");
396 }
397
398 static LLVMValueRef get_num_tcs_out_vertices(struct si_shader_context *ctx)
399 {
400 unsigned tcs_out_vertices =
401 ctx->shader->selector ?
402 ctx->shader->selector->info.properties[TGSI_PROPERTY_TCS_VERTICES_OUT] : 0;
403
404 /* If !tcs_out_vertices, it's either the fixed-func TCS or the TCS epilog. */
405 if (ctx->type == PIPE_SHADER_TESS_CTRL && tcs_out_vertices)
406 return LLVMConstInt(ctx->i32, tcs_out_vertices, 0);
407
408 return unpack_param(ctx, ctx->param_tcs_offchip_layout, 6, 6);
409 }
410
411 static LLVMValueRef get_tcs_in_vertex_dw_stride(struct si_shader_context *ctx)
412 {
413 unsigned stride;
414
415 switch (ctx->type) {
416 case PIPE_SHADER_VERTEX:
417 stride = util_last_bit64(ctx->shader->selector->outputs_written);
418 return LLVMConstInt(ctx->i32, stride * 4, 0);
419
420 case PIPE_SHADER_TESS_CTRL:
421 if (ctx->screen->info.chip_class >= GFX9 &&
422 ctx->shader->is_monolithic) {
423 stride = util_last_bit64(ctx->shader->key.part.tcs.ls->outputs_written);
424 return LLVMConstInt(ctx->i32, stride * 4, 0);
425 }
426 return unpack_param(ctx, ctx->param_vs_state_bits, 24, 8);
427
428 default:
429 assert(0);
430 return NULL;
431 }
432 }
433
434 static LLVMValueRef get_instance_index_for_fetch(
435 struct si_shader_context *ctx,
436 unsigned param_start_instance, LLVMValueRef divisor)
437 {
438 LLVMValueRef result = ctx->abi.instance_id;
439
440 /* The division must be done before START_INSTANCE is added. */
441 if (divisor != ctx->i32_1)
442 result = LLVMBuildUDiv(ctx->ac.builder, result, divisor, "");
443
444 return LLVMBuildAdd(ctx->ac.builder, result,
445 LLVMGetParam(ctx->main_fn, param_start_instance), "");
446 }
447
448 /* Bitcast <4 x float> to <2 x double>, extract the component, and convert
449 * to float. */
450 static LLVMValueRef extract_double_to_float(struct si_shader_context *ctx,
451 LLVMValueRef vec4,
452 unsigned double_index)
453 {
454 LLVMBuilderRef builder = ctx->ac.builder;
455 LLVMTypeRef f64 = LLVMDoubleTypeInContext(ctx->ac.context);
456 LLVMValueRef dvec2 = LLVMBuildBitCast(builder, vec4,
457 LLVMVectorType(f64, 2), "");
458 LLVMValueRef index = LLVMConstInt(ctx->i32, double_index, 0);
459 LLVMValueRef value = LLVMBuildExtractElement(builder, dvec2, index, "");
460 return LLVMBuildFPTrunc(builder, value, ctx->f32, "");
461 }
462
463 static LLVMValueRef unpack_sint16(struct si_shader_context *ctx,
464 LLVMValueRef i32, unsigned index)
465 {
466 assert(index <= 1);
467
468 if (index == 1)
469 return LLVMBuildAShr(ctx->ac.builder, i32,
470 LLVMConstInt(ctx->i32, 16, 0), "");
471
472 return LLVMBuildSExt(ctx->ac.builder,
473 LLVMBuildTrunc(ctx->ac.builder, i32,
474 ctx->ac.i16, ""),
475 ctx->i32, "");
476 }
477
478 void si_llvm_load_input_vs(
479 struct si_shader_context *ctx,
480 unsigned input_index,
481 LLVMValueRef out[4])
482 {
483 const struct tgsi_shader_info *info = &ctx->shader->selector->info;
484 unsigned vs_blit_property = info->properties[TGSI_PROPERTY_VS_BLIT_SGPRS];
485
486 if (vs_blit_property) {
487 LLVMValueRef vertex_id = ctx->abi.vertex_id;
488 LLVMValueRef sel_x1 = LLVMBuildICmp(ctx->ac.builder,
489 LLVMIntULE, vertex_id,
490 ctx->i32_1, "");
491 /* Use LLVMIntNE, because we have 3 vertices and only
492 * the middle one should use y2.
493 */
494 LLVMValueRef sel_y1 = LLVMBuildICmp(ctx->ac.builder,
495 LLVMIntNE, vertex_id,
496 ctx->i32_1, "");
497
498 if (input_index == 0) {
499 /* Position: */
500 LLVMValueRef x1y1 = LLVMGetParam(ctx->main_fn,
501 ctx->param_vs_blit_inputs);
502 LLVMValueRef x2y2 = LLVMGetParam(ctx->main_fn,
503 ctx->param_vs_blit_inputs + 1);
504
505 LLVMValueRef x1 = unpack_sint16(ctx, x1y1, 0);
506 LLVMValueRef y1 = unpack_sint16(ctx, x1y1, 1);
507 LLVMValueRef x2 = unpack_sint16(ctx, x2y2, 0);
508 LLVMValueRef y2 = unpack_sint16(ctx, x2y2, 1);
509
510 LLVMValueRef x = LLVMBuildSelect(ctx->ac.builder, sel_x1,
511 x1, x2, "");
512 LLVMValueRef y = LLVMBuildSelect(ctx->ac.builder, sel_y1,
513 y1, y2, "");
514
515 out[0] = LLVMBuildSIToFP(ctx->ac.builder, x, ctx->f32, "");
516 out[1] = LLVMBuildSIToFP(ctx->ac.builder, y, ctx->f32, "");
517 out[2] = LLVMGetParam(ctx->main_fn,
518 ctx->param_vs_blit_inputs + 2);
519 out[3] = ctx->ac.f32_1;
520 return;
521 }
522
523 /* Color or texture coordinates: */
524 assert(input_index == 1);
525
526 if (vs_blit_property == SI_VS_BLIT_SGPRS_POS_COLOR) {
527 for (int i = 0; i < 4; i++) {
528 out[i] = LLVMGetParam(ctx->main_fn,
529 ctx->param_vs_blit_inputs + 3 + i);
530 }
531 } else {
532 assert(vs_blit_property == SI_VS_BLIT_SGPRS_POS_TEXCOORD);
533 LLVMValueRef x1 = LLVMGetParam(ctx->main_fn,
534 ctx->param_vs_blit_inputs + 3);
535 LLVMValueRef y1 = LLVMGetParam(ctx->main_fn,
536 ctx->param_vs_blit_inputs + 4);
537 LLVMValueRef x2 = LLVMGetParam(ctx->main_fn,
538 ctx->param_vs_blit_inputs + 5);
539 LLVMValueRef y2 = LLVMGetParam(ctx->main_fn,
540 ctx->param_vs_blit_inputs + 6);
541
542 out[0] = LLVMBuildSelect(ctx->ac.builder, sel_x1,
543 x1, x2, "");
544 out[1] = LLVMBuildSelect(ctx->ac.builder, sel_y1,
545 y1, y2, "");
546 out[2] = LLVMGetParam(ctx->main_fn,
547 ctx->param_vs_blit_inputs + 7);
548 out[3] = LLVMGetParam(ctx->main_fn,
549 ctx->param_vs_blit_inputs + 8);
550 }
551 return;
552 }
553
554 unsigned chan;
555 unsigned fix_fetch;
556 unsigned num_fetches;
557 unsigned fetch_stride;
558 unsigned num_channels;
559
560 LLVMValueRef t_list_ptr;
561 LLVMValueRef t_offset;
562 LLVMValueRef t_list;
563 LLVMValueRef vertex_index;
564 LLVMValueRef input[3];
565
566 /* Load the T list */
567 t_list_ptr = LLVMGetParam(ctx->main_fn, ctx->param_vertex_buffers);
568
569 t_offset = LLVMConstInt(ctx->i32, input_index, 0);
570
571 t_list = ac_build_load_to_sgpr(&ctx->ac, t_list_ptr, t_offset);
572
573 vertex_index = LLVMGetParam(ctx->main_fn,
574 ctx->param_vertex_index0 +
575 input_index);
576
577 fix_fetch = ctx->shader->key.mono.vs_fix_fetch[input_index];
578
579 /* Do multiple loads for special formats. */
580 switch (fix_fetch) {
581 case SI_FIX_FETCH_RGB_64_FLOAT:
582 num_fetches = 3; /* 3 2-dword loads */
583 fetch_stride = 8;
584 num_channels = 2;
585 break;
586 case SI_FIX_FETCH_RGBA_64_FLOAT:
587 num_fetches = 2; /* 2 4-dword loads */
588 fetch_stride = 16;
589 num_channels = 4;
590 break;
591 case SI_FIX_FETCH_RGB_8:
592 case SI_FIX_FETCH_RGB_8_INT:
593 num_fetches = 3;
594 fetch_stride = 1;
595 num_channels = 1;
596 break;
597 case SI_FIX_FETCH_RGB_16:
598 case SI_FIX_FETCH_RGB_16_INT:
599 num_fetches = 3;
600 fetch_stride = 2;
601 num_channels = 1;
602 break;
603 default:
604 num_fetches = 1;
605 fetch_stride = 0;
606 num_channels = util_last_bit(info->input_usage_mask[input_index]);
607 }
608
609 for (unsigned i = 0; i < num_fetches; i++) {
610 LLVMValueRef voffset = LLVMConstInt(ctx->i32, fetch_stride * i, 0);
611
612 input[i] = ac_build_buffer_load_format(&ctx->ac, t_list,
613 vertex_index, voffset,
614 num_channels, false, true);
615 input[i] = ac_build_expand_to_vec4(&ctx->ac, input[i], num_channels);
616 }
617
618 /* Break up the vec4 into individual components */
619 for (chan = 0; chan < 4; chan++) {
620 LLVMValueRef llvm_chan = LLVMConstInt(ctx->i32, chan, 0);
621 out[chan] = LLVMBuildExtractElement(ctx->ac.builder,
622 input[0], llvm_chan, "");
623 }
624
625 switch (fix_fetch) {
626 case SI_FIX_FETCH_A2_SNORM:
627 case SI_FIX_FETCH_A2_SSCALED:
628 case SI_FIX_FETCH_A2_SINT: {
629 /* The hardware returns an unsigned value; convert it to a
630 * signed one.
631 */
632 LLVMValueRef tmp = out[3];
633 LLVMValueRef c30 = LLVMConstInt(ctx->i32, 30, 0);
634
635 /* First, recover the sign-extended signed integer value. */
636 if (fix_fetch == SI_FIX_FETCH_A2_SSCALED)
637 tmp = LLVMBuildFPToUI(ctx->ac.builder, tmp, ctx->i32, "");
638 else
639 tmp = ac_to_integer(&ctx->ac, tmp);
640
641 /* For the integer-like cases, do a natural sign extension.
642 *
643 * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
644 * and happen to contain 0, 1, 2, 3 as the two LSBs of the
645 * exponent.
646 */
647 tmp = LLVMBuildShl(ctx->ac.builder, tmp,
648 fix_fetch == SI_FIX_FETCH_A2_SNORM ?
649 LLVMConstInt(ctx->i32, 7, 0) : c30, "");
650 tmp = LLVMBuildAShr(ctx->ac.builder, tmp, c30, "");
651
652 /* Convert back to the right type. */
653 if (fix_fetch == SI_FIX_FETCH_A2_SNORM) {
654 LLVMValueRef clamp;
655 LLVMValueRef neg_one = LLVMConstReal(ctx->f32, -1.0);
656 tmp = LLVMBuildSIToFP(ctx->ac.builder, tmp, ctx->f32, "");
657 clamp = LLVMBuildFCmp(ctx->ac.builder, LLVMRealULT, tmp, neg_one, "");
658 tmp = LLVMBuildSelect(ctx->ac.builder, clamp, neg_one, tmp, "");
659 } else if (fix_fetch == SI_FIX_FETCH_A2_SSCALED) {
660 tmp = LLVMBuildSIToFP(ctx->ac.builder, tmp, ctx->f32, "");
661 }
662
663 out[3] = tmp;
664 break;
665 }
666 case SI_FIX_FETCH_RGBA_32_UNORM:
667 case SI_FIX_FETCH_RGBX_32_UNORM:
668 for (chan = 0; chan < 4; chan++) {
669 out[chan] = ac_to_integer(&ctx->ac, out[chan]);
670 out[chan] = LLVMBuildUIToFP(ctx->ac.builder,
671 out[chan], ctx->f32, "");
672 out[chan] = LLVMBuildFMul(ctx->ac.builder, out[chan],
673 LLVMConstReal(ctx->f32, 1.0 / UINT_MAX), "");
674 }
675 /* RGBX UINT returns 1 in alpha, which would be rounded to 0 by normalizing. */
676 if (fix_fetch == SI_FIX_FETCH_RGBX_32_UNORM)
677 out[3] = LLVMConstReal(ctx->f32, 1);
678 break;
679 case SI_FIX_FETCH_RGBA_32_SNORM:
680 case SI_FIX_FETCH_RGBX_32_SNORM:
681 case SI_FIX_FETCH_RGBA_32_FIXED:
682 case SI_FIX_FETCH_RGBX_32_FIXED: {
683 double scale;
684 if (fix_fetch >= SI_FIX_FETCH_RGBA_32_FIXED)
685 scale = 1.0 / 0x10000;
686 else
687 scale = 1.0 / INT_MAX;
688
689 for (chan = 0; chan < 4; chan++) {
690 out[chan] = ac_to_integer(&ctx->ac, out[chan]);
691 out[chan] = LLVMBuildSIToFP(ctx->ac.builder,
692 out[chan], ctx->f32, "");
693 out[chan] = LLVMBuildFMul(ctx->ac.builder, out[chan],
694 LLVMConstReal(ctx->f32, scale), "");
695 }
696 /* RGBX SINT returns 1 in alpha, which would be rounded to 0 by normalizing. */
697 if (fix_fetch == SI_FIX_FETCH_RGBX_32_SNORM ||
698 fix_fetch == SI_FIX_FETCH_RGBX_32_FIXED)
699 out[3] = LLVMConstReal(ctx->f32, 1);
700 break;
701 }
702 case SI_FIX_FETCH_RGBA_32_USCALED:
703 for (chan = 0; chan < 4; chan++) {
704 out[chan] = ac_to_integer(&ctx->ac, out[chan]);
705 out[chan] = LLVMBuildUIToFP(ctx->ac.builder,
706 out[chan], ctx->f32, "");
707 }
708 break;
709 case SI_FIX_FETCH_RGBA_32_SSCALED:
710 for (chan = 0; chan < 4; chan++) {
711 out[chan] = ac_to_integer(&ctx->ac, out[chan]);
712 out[chan] = LLVMBuildSIToFP(ctx->ac.builder,
713 out[chan], ctx->f32, "");
714 }
715 break;
716 case SI_FIX_FETCH_RG_64_FLOAT:
717 for (chan = 0; chan < 2; chan++)
718 out[chan] = extract_double_to_float(ctx, input[0], chan);
719
720 out[2] = LLVMConstReal(ctx->f32, 0);
721 out[3] = LLVMConstReal(ctx->f32, 1);
722 break;
723 case SI_FIX_FETCH_RGB_64_FLOAT:
724 for (chan = 0; chan < 3; chan++)
725 out[chan] = extract_double_to_float(ctx, input[chan], 0);
726
727 out[3] = LLVMConstReal(ctx->f32, 1);
728 break;
729 case SI_FIX_FETCH_RGBA_64_FLOAT:
730 for (chan = 0; chan < 4; chan++) {
731 out[chan] = extract_double_to_float(ctx, input[chan / 2],
732 chan % 2);
733 }
734 break;
735 case SI_FIX_FETCH_RGB_8:
736 case SI_FIX_FETCH_RGB_8_INT:
737 case SI_FIX_FETCH_RGB_16:
738 case SI_FIX_FETCH_RGB_16_INT:
739 for (chan = 0; chan < 3; chan++) {
740 out[chan] = LLVMBuildExtractElement(ctx->ac.builder,
741 input[chan],
742 ctx->i32_0, "");
743 }
744 if (fix_fetch == SI_FIX_FETCH_RGB_8 ||
745 fix_fetch == SI_FIX_FETCH_RGB_16) {
746 out[3] = LLVMConstReal(ctx->f32, 1);
747 } else {
748 out[3] = ac_to_float(&ctx->ac, ctx->i32_1);
749 }
750 break;
751 }
752 }
753
754 static void declare_input_vs(
755 struct si_shader_context *ctx,
756 unsigned input_index,
757 const struct tgsi_full_declaration *decl,
758 LLVMValueRef out[4])
759 {
760 si_llvm_load_input_vs(ctx, input_index, out);
761 }
762
763 static LLVMValueRef get_primitive_id(struct si_shader_context *ctx,
764 unsigned swizzle)
765 {
766 if (swizzle > 0)
767 return ctx->i32_0;
768
769 switch (ctx->type) {
770 case PIPE_SHADER_VERTEX:
771 return LLVMGetParam(ctx->main_fn,
772 ctx->param_vs_prim_id);
773 case PIPE_SHADER_TESS_CTRL:
774 return ctx->abi.tcs_patch_id;
775 case PIPE_SHADER_TESS_EVAL:
776 return ctx->abi.tes_patch_id;
777 case PIPE_SHADER_GEOMETRY:
778 return ctx->abi.gs_prim_id;
779 default:
780 assert(0);
781 return ctx->i32_0;
782 }
783 }
784
785 /**
786 * Return the value of tgsi_ind_register for indexing.
787 * This is the indirect index with the constant offset added to it.
788 */
789 LLVMValueRef si_get_indirect_index(struct si_shader_context *ctx,
790 const struct tgsi_ind_register *ind,
791 unsigned addr_mul,
792 int rel_index)
793 {
794 LLVMValueRef result;
795
796 if (ind->File == TGSI_FILE_ADDRESS) {
797 result = ctx->addrs[ind->Index][ind->Swizzle];
798 result = LLVMBuildLoad(ctx->ac.builder, result, "");
799 } else {
800 struct tgsi_full_src_register src = {};
801
802 src.Register.File = ind->File;
803 src.Register.Index = ind->Index;
804
805 /* Set the second index to 0 for constants. */
806 if (ind->File == TGSI_FILE_CONSTANT)
807 src.Register.Dimension = 1;
808
809 result = ctx->bld_base.emit_fetch_funcs[ind->File](&ctx->bld_base, &src,
810 TGSI_TYPE_SIGNED,
811 ind->Swizzle);
812 result = ac_to_integer(&ctx->ac, result);
813 }
814
815 if (addr_mul != 1)
816 result = LLVMBuildMul(ctx->ac.builder, result,
817 LLVMConstInt(ctx->i32, addr_mul, 0), "");
818 result = LLVMBuildAdd(ctx->ac.builder, result,
819 LLVMConstInt(ctx->i32, rel_index, 0), "");
820 return result;
821 }
822
823 /**
824 * Like si_get_indirect_index, but restricts the return value to a (possibly
825 * undefined) value inside [0..num).
826 */
827 LLVMValueRef si_get_bounded_indirect_index(struct si_shader_context *ctx,
828 const struct tgsi_ind_register *ind,
829 int rel_index, unsigned num)
830 {
831 LLVMValueRef result = si_get_indirect_index(ctx, ind, 1, rel_index);
832
833 return si_llvm_bound_index(ctx, result, num);
834 }
835
836 static LLVMValueRef get_dw_address_from_generic_indices(struct si_shader_context *ctx,
837 LLVMValueRef vertex_dw_stride,
838 LLVMValueRef base_addr,
839 LLVMValueRef vertex_index,
840 LLVMValueRef param_index,
841 unsigned input_index,
842 ubyte *name,
843 ubyte *index,
844 bool is_patch)
845 {
846 if (vertex_dw_stride) {
847 base_addr = LLVMBuildAdd(ctx->ac.builder, base_addr,
848 LLVMBuildMul(ctx->ac.builder, vertex_index,
849 vertex_dw_stride, ""), "");
850 }
851
852 if (param_index) {
853 base_addr = LLVMBuildAdd(ctx->ac.builder, base_addr,
854 LLVMBuildMul(ctx->ac.builder, param_index,
855 LLVMConstInt(ctx->i32, 4, 0), ""), "");
856 }
857
858 int param = is_patch ?
859 si_shader_io_get_unique_index_patch(name[input_index],
860 index[input_index]) :
861 si_shader_io_get_unique_index(name[input_index],
862 index[input_index]);
863
864 /* Add the base address of the element. */
865 return LLVMBuildAdd(ctx->ac.builder, base_addr,
866 LLVMConstInt(ctx->i32, param * 4, 0), "");
867 }
868
869 /**
870 * Calculate a dword address given an input or output register and a stride.
871 */
872 static LLVMValueRef get_dw_address(struct si_shader_context *ctx,
873 const struct tgsi_full_dst_register *dst,
874 const struct tgsi_full_src_register *src,
875 LLVMValueRef vertex_dw_stride,
876 LLVMValueRef base_addr)
877 {
878 struct tgsi_shader_info *info = &ctx->shader->selector->info;
879 ubyte *name, *index, *array_first;
880 int input_index;
881 struct tgsi_full_dst_register reg;
882 LLVMValueRef vertex_index = NULL;
883 LLVMValueRef ind_index = NULL;
884
885 /* Set the register description. The address computation is the same
886 * for sources and destinations. */
887 if (src) {
888 reg.Register.File = src->Register.File;
889 reg.Register.Index = src->Register.Index;
890 reg.Register.Indirect = src->Register.Indirect;
891 reg.Register.Dimension = src->Register.Dimension;
892 reg.Indirect = src->Indirect;
893 reg.Dimension = src->Dimension;
894 reg.DimIndirect = src->DimIndirect;
895 } else
896 reg = *dst;
897
898 /* If the register is 2-dimensional (e.g. an array of vertices
899 * in a primitive), calculate the base address of the vertex. */
900 if (reg.Register.Dimension) {
901 if (reg.Dimension.Indirect)
902 vertex_index = si_get_indirect_index(ctx, &reg.DimIndirect,
903 1, reg.Dimension.Index);
904 else
905 vertex_index = LLVMConstInt(ctx->i32, reg.Dimension.Index, 0);
906 }
907
908 /* Get information about the register. */
909 if (reg.Register.File == TGSI_FILE_INPUT) {
910 name = info->input_semantic_name;
911 index = info->input_semantic_index;
912 array_first = info->input_array_first;
913 } else if (reg.Register.File == TGSI_FILE_OUTPUT) {
914 name = info->output_semantic_name;
915 index = info->output_semantic_index;
916 array_first = info->output_array_first;
917 } else {
918 assert(0);
919 return NULL;
920 }
921
922 if (reg.Register.Indirect) {
923 /* Add the relative address of the element. */
924 if (reg.Indirect.ArrayID)
925 input_index = array_first[reg.Indirect.ArrayID];
926 else
927 input_index = reg.Register.Index;
928
929 ind_index = si_get_indirect_index(ctx, &reg.Indirect,
930 1, reg.Register.Index - input_index);
931 } else {
932 input_index = reg.Register.Index;
933 }
934
935 return get_dw_address_from_generic_indices(ctx, vertex_dw_stride,
936 base_addr, vertex_index,
937 ind_index, input_index,
938 name, index,
939 !reg.Register.Dimension);
940 }
941
942 /* The offchip buffer layout for TCS->TES is
943 *
944 * - attribute 0 of patch 0 vertex 0
945 * - attribute 0 of patch 0 vertex 1
946 * - attribute 0 of patch 0 vertex 2
947 * ...
948 * - attribute 0 of patch 1 vertex 0
949 * - attribute 0 of patch 1 vertex 1
950 * ...
951 * - attribute 1 of patch 0 vertex 0
952 * - attribute 1 of patch 0 vertex 1
953 * ...
954 * - per patch attribute 0 of patch 0
955 * - per patch attribute 0 of patch 1
956 * ...
957 *
958 * Note that every attribute has 4 components.
959 */
960 static LLVMValueRef get_tcs_tes_buffer_address(struct si_shader_context *ctx,
961 LLVMValueRef rel_patch_id,
962 LLVMValueRef vertex_index,
963 LLVMValueRef param_index)
964 {
965 LLVMValueRef base_addr, vertices_per_patch, num_patches, total_vertices;
966 LLVMValueRef param_stride, constant16;
967
968 vertices_per_patch = get_num_tcs_out_vertices(ctx);
969 num_patches = unpack_param(ctx, ctx->param_tcs_offchip_layout, 0, 6);
970 total_vertices = LLVMBuildMul(ctx->ac.builder, vertices_per_patch,
971 num_patches, "");
972
973 constant16 = LLVMConstInt(ctx->i32, 16, 0);
974 if (vertex_index) {
975 base_addr = LLVMBuildMul(ctx->ac.builder, rel_patch_id,
976 vertices_per_patch, "");
977
978 base_addr = LLVMBuildAdd(ctx->ac.builder, base_addr,
979 vertex_index, "");
980
981 param_stride = total_vertices;
982 } else {
983 base_addr = rel_patch_id;
984 param_stride = num_patches;
985 }
986
987 base_addr = LLVMBuildAdd(ctx->ac.builder, base_addr,
988 LLVMBuildMul(ctx->ac.builder, param_index,
989 param_stride, ""), "");
990
991 base_addr = LLVMBuildMul(ctx->ac.builder, base_addr, constant16, "");
992
993 if (!vertex_index) {
994 LLVMValueRef patch_data_offset =
995 unpack_param(ctx, ctx->param_tcs_offchip_layout, 12, 20);
996
997 base_addr = LLVMBuildAdd(ctx->ac.builder, base_addr,
998 patch_data_offset, "");
999 }
1000 return base_addr;
1001 }
1002
1003 /* This is a generic helper that can be shared by the NIR and TGSI backends */
1004 static LLVMValueRef get_tcs_tes_buffer_address_from_generic_indices(
1005 struct si_shader_context *ctx,
1006 LLVMValueRef vertex_index,
1007 LLVMValueRef param_index,
1008 unsigned param_base,
1009 ubyte *name,
1010 ubyte *index,
1011 bool is_patch)
1012 {
1013 unsigned param_index_base;
1014
1015 param_index_base = is_patch ?
1016 si_shader_io_get_unique_index_patch(name[param_base], index[param_base]) :
1017 si_shader_io_get_unique_index(name[param_base], index[param_base]);
1018
1019 if (param_index) {
1020 param_index = LLVMBuildAdd(ctx->ac.builder, param_index,
1021 LLVMConstInt(ctx->i32, param_index_base, 0),
1022 "");
1023 } else {
1024 param_index = LLVMConstInt(ctx->i32, param_index_base, 0);
1025 }
1026
1027 return get_tcs_tes_buffer_address(ctx, get_rel_patch_id(ctx),
1028 vertex_index, param_index);
1029 }
1030
1031 static LLVMValueRef get_tcs_tes_buffer_address_from_reg(
1032 struct si_shader_context *ctx,
1033 const struct tgsi_full_dst_register *dst,
1034 const struct tgsi_full_src_register *src)
1035 {
1036 struct tgsi_shader_info *info = &ctx->shader->selector->info;
1037 ubyte *name, *index, *array_first;
1038 struct tgsi_full_src_register reg;
1039 LLVMValueRef vertex_index = NULL;
1040 LLVMValueRef param_index = NULL;
1041 unsigned param_base;
1042
1043 reg = src ? *src : tgsi_full_src_register_from_dst(dst);
1044
1045 if (reg.Register.Dimension) {
1046
1047 if (reg.Dimension.Indirect)
1048 vertex_index = si_get_indirect_index(ctx, &reg.DimIndirect,
1049 1, reg.Dimension.Index);
1050 else
1051 vertex_index = LLVMConstInt(ctx->i32, reg.Dimension.Index, 0);
1052 }
1053
1054 /* Get information about the register. */
1055 if (reg.Register.File == TGSI_FILE_INPUT) {
1056 name = info->input_semantic_name;
1057 index = info->input_semantic_index;
1058 array_first = info->input_array_first;
1059 } else if (reg.Register.File == TGSI_FILE_OUTPUT) {
1060 name = info->output_semantic_name;
1061 index = info->output_semantic_index;
1062 array_first = info->output_array_first;
1063 } else {
1064 assert(0);
1065 return NULL;
1066 }
1067
1068 if (reg.Register.Indirect) {
1069 if (reg.Indirect.ArrayID)
1070 param_base = array_first[reg.Indirect.ArrayID];
1071 else
1072 param_base = reg.Register.Index;
1073
1074 param_index = si_get_indirect_index(ctx, &reg.Indirect,
1075 1, reg.Register.Index - param_base);
1076
1077 } else {
1078 param_base = reg.Register.Index;
1079 }
1080
1081 return get_tcs_tes_buffer_address_from_generic_indices(ctx, vertex_index,
1082 param_index, param_base,
1083 name, index, !reg.Register.Dimension);
1084 }
1085
1086 static LLVMValueRef buffer_load(struct lp_build_tgsi_context *bld_base,
1087 LLVMTypeRef type, unsigned swizzle,
1088 LLVMValueRef buffer, LLVMValueRef offset,
1089 LLVMValueRef base, bool can_speculate)
1090 {
1091 struct si_shader_context *ctx = si_shader_context(bld_base);
1092 LLVMValueRef value, value2;
1093 LLVMTypeRef vec_type = LLVMVectorType(type, 4);
1094
1095 if (swizzle == ~0) {
1096 value = ac_build_buffer_load(&ctx->ac, buffer, 4, NULL, base, offset,
1097 0, 1, 0, can_speculate, false);
1098
1099 return LLVMBuildBitCast(ctx->ac.builder, value, vec_type, "");
1100 }
1101
1102 if (!llvm_type_is_64bit(ctx, type)) {
1103 value = ac_build_buffer_load(&ctx->ac, buffer, 4, NULL, base, offset,
1104 0, 1, 0, can_speculate, false);
1105
1106 value = LLVMBuildBitCast(ctx->ac.builder, value, vec_type, "");
1107 return LLVMBuildExtractElement(ctx->ac.builder, value,
1108 LLVMConstInt(ctx->i32, swizzle, 0), "");
1109 }
1110
1111 value = ac_build_buffer_load(&ctx->ac, buffer, 1, NULL, base, offset,
1112 swizzle * 4, 1, 0, can_speculate, false);
1113
1114 value2 = ac_build_buffer_load(&ctx->ac, buffer, 1, NULL, base, offset,
1115 swizzle * 4 + 4, 1, 0, can_speculate, false);
1116
1117 return si_llvm_emit_fetch_64bit(bld_base, type, value, value2);
1118 }
1119
1120 /**
1121 * Load from LDS.
1122 *
1123 * \param type output value type
1124 * \param swizzle offset (typically 0..3); it can be ~0, which loads a vec4
1125 * \param dw_addr address in dwords
1126 */
1127 static LLVMValueRef lds_load(struct lp_build_tgsi_context *bld_base,
1128 LLVMTypeRef type, unsigned swizzle,
1129 LLVMValueRef dw_addr)
1130 {
1131 struct si_shader_context *ctx = si_shader_context(bld_base);
1132 LLVMValueRef value;
1133
1134 if (swizzle == ~0) {
1135 LLVMValueRef values[TGSI_NUM_CHANNELS];
1136
1137 for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; chan++)
1138 values[chan] = lds_load(bld_base, type, chan, dw_addr);
1139
1140 return lp_build_gather_values(&ctx->gallivm, values,
1141 TGSI_NUM_CHANNELS);
1142 }
1143
1144 /* Split 64-bit loads. */
1145 if (llvm_type_is_64bit(ctx, type)) {
1146 LLVMValueRef lo, hi;
1147
1148 lo = lds_load(bld_base, ctx->i32, swizzle, dw_addr);
1149 hi = lds_load(bld_base, ctx->i32, swizzle + 1, dw_addr);
1150 return si_llvm_emit_fetch_64bit(bld_base, type, lo, hi);
1151 }
1152
1153 dw_addr = lp_build_add(&bld_base->uint_bld, dw_addr,
1154 LLVMConstInt(ctx->i32, swizzle, 0));
1155
1156 value = ac_lds_load(&ctx->ac, dw_addr);
1157
1158 return LLVMBuildBitCast(ctx->ac.builder, value, type, "");
1159 }
1160
1161 /**
1162 * Store to LDS.
1163 *
1164 * \param swizzle offset (typically 0..3)
1165 * \param dw_addr address in dwords
1166 * \param value value to store
1167 */
1168 static void lds_store(struct si_shader_context *ctx,
1169 unsigned dw_offset_imm, LLVMValueRef dw_addr,
1170 LLVMValueRef value)
1171 {
1172 dw_addr = lp_build_add(&ctx->bld_base.uint_bld, dw_addr,
1173 LLVMConstInt(ctx->i32, dw_offset_imm, 0));
1174
1175 ac_lds_store(&ctx->ac, dw_addr, value);
1176 }
1177
1178 static LLVMValueRef desc_from_addr_base64k(struct si_shader_context *ctx,
1179 unsigned param)
1180 {
1181 LLVMBuilderRef builder = ctx->ac.builder;
1182
1183 LLVMValueRef addr = LLVMGetParam(ctx->main_fn, param);
1184 addr = LLVMBuildZExt(builder, addr, ctx->i64, "");
1185 addr = LLVMBuildShl(builder, addr, LLVMConstInt(ctx->i64, 16, 0), "");
1186
1187 uint64_t desc2 = 0xffffffff;
1188 uint64_t desc3 = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1189 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1190 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1191 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1192 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1193 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
1194 LLVMValueRef hi = LLVMConstInt(ctx->i64, desc2 | (desc3 << 32), 0);
1195
1196 LLVMValueRef desc = LLVMGetUndef(LLVMVectorType(ctx->i64, 2));
1197 desc = LLVMBuildInsertElement(builder, desc, addr, ctx->i32_0, "");
1198 desc = LLVMBuildInsertElement(builder, desc, hi, ctx->i32_1, "");
1199 return LLVMBuildBitCast(builder, desc, ctx->v4i32, "");
1200 }
1201
1202 static LLVMValueRef fetch_input_tcs(
1203 struct lp_build_tgsi_context *bld_base,
1204 const struct tgsi_full_src_register *reg,
1205 enum tgsi_opcode_type type, unsigned swizzle)
1206 {
1207 struct si_shader_context *ctx = si_shader_context(bld_base);
1208 LLVMValueRef dw_addr, stride;
1209
1210 stride = get_tcs_in_vertex_dw_stride(ctx);
1211 dw_addr = get_tcs_in_current_patch_offset(ctx);
1212 dw_addr = get_dw_address(ctx, NULL, reg, stride, dw_addr);
1213
1214 return lds_load(bld_base, tgsi2llvmtype(bld_base, type), swizzle, dw_addr);
1215 }
1216
1217 static LLVMValueRef si_nir_load_tcs_varyings(struct ac_shader_abi *abi,
1218 LLVMValueRef vertex_index,
1219 LLVMValueRef param_index,
1220 unsigned const_index,
1221 unsigned location,
1222 unsigned driver_location,
1223 unsigned component,
1224 unsigned num_components,
1225 bool is_patch,
1226 bool is_compact,
1227 bool load_input)
1228 {
1229 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1230 struct tgsi_shader_info *info = &ctx->shader->selector->info;
1231 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
1232 LLVMValueRef dw_addr, stride;
1233
1234 driver_location = driver_location / 4;
1235
1236 if (load_input) {
1237 stride = get_tcs_in_vertex_dw_stride(ctx);
1238 dw_addr = get_tcs_in_current_patch_offset(ctx);
1239 } else {
1240 if (is_patch) {
1241 stride = NULL;
1242 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1243 } else {
1244 stride = get_tcs_out_vertex_dw_stride(ctx);
1245 dw_addr = get_tcs_out_current_patch_offset(ctx);
1246 }
1247 }
1248
1249 if (param_index) {
1250 /* Add the constant index to the indirect index */
1251 param_index = LLVMBuildAdd(ctx->ac.builder, param_index,
1252 LLVMConstInt(ctx->i32, const_index, 0), "");
1253 } else {
1254 param_index = LLVMConstInt(ctx->i32, const_index, 0);
1255 }
1256
1257 dw_addr = get_dw_address_from_generic_indices(ctx, stride, dw_addr,
1258 vertex_index, param_index,
1259 driver_location,
1260 info->input_semantic_name,
1261 info->input_semantic_index,
1262 is_patch);
1263
1264 LLVMValueRef value[4];
1265 for (unsigned i = 0; i < num_components + component; i++) {
1266 value[i] = lds_load(bld_base, ctx->i32, i, dw_addr);
1267 }
1268
1269 return ac_build_varying_gather_values(&ctx->ac, value, num_components, component);
1270 }
1271
1272 static LLVMValueRef fetch_output_tcs(
1273 struct lp_build_tgsi_context *bld_base,
1274 const struct tgsi_full_src_register *reg,
1275 enum tgsi_opcode_type type, unsigned swizzle)
1276 {
1277 struct si_shader_context *ctx = si_shader_context(bld_base);
1278 LLVMValueRef dw_addr, stride;
1279
1280 if (reg->Register.Dimension) {
1281 stride = get_tcs_out_vertex_dw_stride(ctx);
1282 dw_addr = get_tcs_out_current_patch_offset(ctx);
1283 dw_addr = get_dw_address(ctx, NULL, reg, stride, dw_addr);
1284 } else {
1285 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1286 dw_addr = get_dw_address(ctx, NULL, reg, NULL, dw_addr);
1287 }
1288
1289 return lds_load(bld_base, tgsi2llvmtype(bld_base, type), swizzle, dw_addr);
1290 }
1291
1292 static LLVMValueRef fetch_input_tes(
1293 struct lp_build_tgsi_context *bld_base,
1294 const struct tgsi_full_src_register *reg,
1295 enum tgsi_opcode_type type, unsigned swizzle)
1296 {
1297 struct si_shader_context *ctx = si_shader_context(bld_base);
1298 LLVMValueRef buffer, base, addr;
1299
1300 buffer = desc_from_addr_base64k(ctx, ctx->param_tcs_offchip_addr_base64k);
1301
1302 base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
1303 addr = get_tcs_tes_buffer_address_from_reg(ctx, NULL, reg);
1304
1305 return buffer_load(bld_base, tgsi2llvmtype(bld_base, type), swizzle,
1306 buffer, base, addr, true);
1307 }
1308
1309 LLVMValueRef si_nir_load_input_tes(struct ac_shader_abi *abi,
1310 LLVMValueRef vertex_index,
1311 LLVMValueRef param_index,
1312 unsigned const_index,
1313 unsigned location,
1314 unsigned driver_location,
1315 unsigned component,
1316 unsigned num_components,
1317 bool is_patch,
1318 bool is_compact,
1319 bool load_input)
1320 {
1321 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1322 struct tgsi_shader_info *info = &ctx->shader->selector->info;
1323 LLVMValueRef buffer, base, addr;
1324
1325 driver_location = driver_location / 4;
1326
1327 buffer = desc_from_addr_base64k(ctx, ctx->param_tcs_offchip_addr_base64k);
1328
1329 base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
1330
1331 if (param_index) {
1332 /* Add the constant index to the indirect index */
1333 param_index = LLVMBuildAdd(ctx->ac.builder, param_index,
1334 LLVMConstInt(ctx->i32, const_index, 0), "");
1335 } else {
1336 param_index = LLVMConstInt(ctx->i32, const_index, 0);
1337 }
1338
1339 addr = get_tcs_tes_buffer_address_from_generic_indices(ctx, vertex_index,
1340 param_index, driver_location,
1341 info->input_semantic_name,
1342 info->input_semantic_index,
1343 is_patch);
1344
1345 /* TODO: This will generate rather ordinary llvm code, although it
1346 * should be easy for the optimiser to fix up. In future we might want
1347 * to refactor buffer_load(), but for now this maximises code sharing
1348 * between the NIR and TGSI backends.
1349 */
1350 LLVMValueRef value[4];
1351 for (unsigned i = component; i < num_components + component; i++) {
1352 value[i] = buffer_load(&ctx->bld_base, ctx->i32, i, buffer, base, addr, true);
1353 }
1354
1355 return ac_build_varying_gather_values(&ctx->ac, value, num_components, component);
1356 }
1357
1358 static void store_output_tcs(struct lp_build_tgsi_context *bld_base,
1359 const struct tgsi_full_instruction *inst,
1360 const struct tgsi_opcode_info *info,
1361 unsigned index,
1362 LLVMValueRef dst[4])
1363 {
1364 struct si_shader_context *ctx = si_shader_context(bld_base);
1365 const struct tgsi_full_dst_register *reg = &inst->Dst[index];
1366 const struct tgsi_shader_info *sh_info = &ctx->shader->selector->info;
1367 unsigned chan_index;
1368 LLVMValueRef dw_addr, stride;
1369 LLVMValueRef buffer, base, buf_addr;
1370 LLVMValueRef values[4];
1371 bool skip_lds_store;
1372 bool is_tess_factor = false, is_tess_inner = false;
1373
1374 /* Only handle per-patch and per-vertex outputs here.
1375 * Vectors will be lowered to scalars and this function will be called again.
1376 */
1377 if (reg->Register.File != TGSI_FILE_OUTPUT ||
1378 (dst[0] && LLVMGetTypeKind(LLVMTypeOf(dst[0])) == LLVMVectorTypeKind)) {
1379 si_llvm_emit_store(bld_base, inst, info, index, dst);
1380 return;
1381 }
1382
1383 if (reg->Register.Dimension) {
1384 stride = get_tcs_out_vertex_dw_stride(ctx);
1385 dw_addr = get_tcs_out_current_patch_offset(ctx);
1386 dw_addr = get_dw_address(ctx, reg, NULL, stride, dw_addr);
1387 skip_lds_store = !sh_info->reads_pervertex_outputs;
1388 } else {
1389 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1390 dw_addr = get_dw_address(ctx, reg, NULL, NULL, dw_addr);
1391 skip_lds_store = !sh_info->reads_perpatch_outputs;
1392
1393 if (!reg->Register.Indirect) {
1394 int name = sh_info->output_semantic_name[reg->Register.Index];
1395
1396 /* Always write tess factors into LDS for the TCS epilog. */
1397 if (name == TGSI_SEMANTIC_TESSINNER ||
1398 name == TGSI_SEMANTIC_TESSOUTER) {
1399 /* The epilog doesn't read LDS if invocation 0 defines tess factors. */
1400 skip_lds_store = !sh_info->reads_tessfactor_outputs &&
1401 ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs;
1402 is_tess_factor = true;
1403 is_tess_inner = name == TGSI_SEMANTIC_TESSINNER;
1404 }
1405 }
1406 }
1407
1408 buffer = desc_from_addr_base64k(ctx, ctx->param_tcs_offchip_addr_base64k);
1409
1410 base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
1411 buf_addr = get_tcs_tes_buffer_address_from_reg(ctx, reg, NULL);
1412
1413 uint32_t writemask = reg->Register.WriteMask;
1414 while (writemask) {
1415 chan_index = u_bit_scan(&writemask);
1416 LLVMValueRef value = dst[chan_index];
1417
1418 if (inst->Instruction.Saturate)
1419 value = ac_build_clamp(&ctx->ac, value);
1420
1421 /* Skip LDS stores if there is no LDS read of this output. */
1422 if (!skip_lds_store)
1423 lds_store(ctx, chan_index, dw_addr, value);
1424
1425 value = ac_to_integer(&ctx->ac, value);
1426 values[chan_index] = value;
1427
1428 if (reg->Register.WriteMask != 0xF && !is_tess_factor) {
1429 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 1,
1430 buf_addr, base,
1431 4 * chan_index, 1, 0, true, false);
1432 }
1433
1434 /* Write tess factors into VGPRs for the epilog. */
1435 if (is_tess_factor &&
1436 ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs) {
1437 if (!is_tess_inner) {
1438 LLVMBuildStore(ctx->ac.builder, value, /* outer */
1439 ctx->invoc0_tess_factors[chan_index]);
1440 } else if (chan_index < 2) {
1441 LLVMBuildStore(ctx->ac.builder, value, /* inner */
1442 ctx->invoc0_tess_factors[4 + chan_index]);
1443 }
1444 }
1445 }
1446
1447 if (reg->Register.WriteMask == 0xF && !is_tess_factor) {
1448 LLVMValueRef value = lp_build_gather_values(&ctx->gallivm,
1449 values, 4);
1450 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, buf_addr,
1451 base, 0, 1, 0, true, false);
1452 }
1453 }
1454
1455 static void si_nir_store_output_tcs(struct ac_shader_abi *abi,
1456 LLVMValueRef vertex_index,
1457 LLVMValueRef param_index,
1458 unsigned const_index,
1459 unsigned location,
1460 unsigned driver_location,
1461 LLVMValueRef src,
1462 unsigned component,
1463 bool is_patch,
1464 bool is_compact,
1465 unsigned writemask)
1466 {
1467 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1468 struct tgsi_shader_info *info = &ctx->shader->selector->info;
1469 LLVMValueRef dw_addr, stride;
1470 LLVMValueRef buffer, base, addr;
1471 LLVMValueRef values[4];
1472 bool skip_lds_store;
1473 bool is_tess_factor = false, is_tess_inner = false;
1474
1475 driver_location = driver_location / 4;
1476
1477 if (param_index) {
1478 /* Add the constant index to the indirect index */
1479 param_index = LLVMBuildAdd(ctx->ac.builder, param_index,
1480 LLVMConstInt(ctx->i32, const_index, 0), "");
1481 } else {
1482 if (const_index != 0)
1483 param_index = LLVMConstInt(ctx->i32, const_index, 0);
1484 }
1485
1486 if (!is_patch) {
1487 stride = get_tcs_out_vertex_dw_stride(ctx);
1488 dw_addr = get_tcs_out_current_patch_offset(ctx);
1489 dw_addr = get_dw_address_from_generic_indices(ctx, stride, dw_addr,
1490 vertex_index, param_index,
1491 driver_location,
1492 info->output_semantic_name,
1493 info->output_semantic_index,
1494 is_patch);
1495
1496 skip_lds_store = !info->reads_pervertex_outputs;
1497 } else {
1498 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1499 dw_addr = get_dw_address_from_generic_indices(ctx, NULL, dw_addr,
1500 vertex_index, param_index,
1501 driver_location,
1502 info->output_semantic_name,
1503 info->output_semantic_index,
1504 is_patch);
1505
1506 skip_lds_store = !info->reads_perpatch_outputs;
1507
1508 if (!param_index) {
1509 int name = info->output_semantic_name[driver_location];
1510
1511 /* Always write tess factors into LDS for the TCS epilog. */
1512 if (name == TGSI_SEMANTIC_TESSINNER ||
1513 name == TGSI_SEMANTIC_TESSOUTER) {
1514 /* The epilog doesn't read LDS if invocation 0 defines tess factors. */
1515 skip_lds_store = !info->reads_tessfactor_outputs &&
1516 ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs;
1517 is_tess_factor = true;
1518 is_tess_inner = name == TGSI_SEMANTIC_TESSINNER;
1519 }
1520 }
1521 }
1522
1523 buffer = desc_from_addr_base64k(ctx, ctx->param_tcs_offchip_addr_base64k);
1524
1525 base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
1526
1527 addr = get_tcs_tes_buffer_address_from_generic_indices(ctx, vertex_index,
1528 param_index, driver_location,
1529 info->output_semantic_name,
1530 info->output_semantic_index,
1531 is_patch);
1532
1533 for (unsigned chan = 0; chan < 4; chan++) {
1534 if (!(writemask & (1 << chan)))
1535 continue;
1536 LLVMValueRef value = ac_llvm_extract_elem(&ctx->ac, src, chan - component);
1537
1538 /* Skip LDS stores if there is no LDS read of this output. */
1539 if (!skip_lds_store)
1540 ac_lds_store(&ctx->ac, dw_addr, value);
1541
1542 value = ac_to_integer(&ctx->ac, value);
1543 values[chan] = value;
1544
1545 if (writemask != 0xF && !is_tess_factor) {
1546 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 1,
1547 addr, base,
1548 4 * chan, 1, 0, true, false);
1549 }
1550
1551 /* Write tess factors into VGPRs for the epilog. */
1552 if (is_tess_factor &&
1553 ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs) {
1554 if (!is_tess_inner) {
1555 LLVMBuildStore(ctx->ac.builder, value, /* outer */
1556 ctx->invoc0_tess_factors[chan]);
1557 } else if (chan < 2) {
1558 LLVMBuildStore(ctx->ac.builder, value, /* inner */
1559 ctx->invoc0_tess_factors[4 + chan]);
1560 }
1561 }
1562 }
1563
1564 if (writemask == 0xF && !is_tess_factor) {
1565 LLVMValueRef value = lp_build_gather_values(&ctx->gallivm,
1566 values, 4);
1567 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, addr,
1568 base, 0, 1, 0, true, false);
1569 }
1570 }
1571
1572 LLVMValueRef si_llvm_load_input_gs(struct ac_shader_abi *abi,
1573 unsigned input_index,
1574 unsigned vtx_offset_param,
1575 LLVMTypeRef type,
1576 unsigned swizzle)
1577 {
1578 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1579 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
1580 struct si_shader *shader = ctx->shader;
1581 struct lp_build_context *uint = &ctx->bld_base.uint_bld;
1582 LLVMValueRef vtx_offset, soffset;
1583 struct tgsi_shader_info *info = &shader->selector->info;
1584 unsigned semantic_name = info->input_semantic_name[input_index];
1585 unsigned semantic_index = info->input_semantic_index[input_index];
1586 unsigned param;
1587 LLVMValueRef value;
1588
1589 param = si_shader_io_get_unique_index(semantic_name, semantic_index);
1590
1591 /* GFX9 has the ESGS ring in LDS. */
1592 if (ctx->screen->info.chip_class >= GFX9) {
1593 unsigned index = vtx_offset_param;
1594
1595 switch (index / 2) {
1596 case 0:
1597 vtx_offset = unpack_param(ctx, ctx->param_gs_vtx01_offset,
1598 index % 2 ? 16 : 0, 16);
1599 break;
1600 case 1:
1601 vtx_offset = unpack_param(ctx, ctx->param_gs_vtx23_offset,
1602 index % 2 ? 16 : 0, 16);
1603 break;
1604 case 2:
1605 vtx_offset = unpack_param(ctx, ctx->param_gs_vtx45_offset,
1606 index % 2 ? 16 : 0, 16);
1607 break;
1608 default:
1609 assert(0);
1610 return NULL;
1611 }
1612
1613 vtx_offset = LLVMBuildAdd(ctx->ac.builder, vtx_offset,
1614 LLVMConstInt(ctx->i32, param * 4, 0), "");
1615 return lds_load(bld_base, type, swizzle, vtx_offset);
1616 }
1617
1618 /* GFX6: input load from the ESGS ring in memory. */
1619 if (swizzle == ~0) {
1620 LLVMValueRef values[TGSI_NUM_CHANNELS];
1621 unsigned chan;
1622 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1623 values[chan] = si_llvm_load_input_gs(abi, input_index, vtx_offset_param,
1624 type, chan);
1625 }
1626 return lp_build_gather_values(&ctx->gallivm, values,
1627 TGSI_NUM_CHANNELS);
1628 }
1629
1630 /* Get the vertex offset parameter on GFX6. */
1631 LLVMValueRef gs_vtx_offset = ctx->gs_vtx_offset[vtx_offset_param];
1632
1633 vtx_offset = lp_build_mul_imm(uint, gs_vtx_offset, 4);
1634
1635 soffset = LLVMConstInt(ctx->i32, (param * 4 + swizzle) * 256, 0);
1636
1637 value = ac_build_buffer_load(&ctx->ac, ctx->esgs_ring, 1, ctx->i32_0,
1638 vtx_offset, soffset, 0, 1, 0, true, false);
1639 if (llvm_type_is_64bit(ctx, type)) {
1640 LLVMValueRef value2;
1641 soffset = LLVMConstInt(ctx->i32, (param * 4 + swizzle + 1) * 256, 0);
1642
1643 value2 = ac_build_buffer_load(&ctx->ac, ctx->esgs_ring, 1,
1644 ctx->i32_0, vtx_offset, soffset,
1645 0, 1, 0, true, false);
1646 return si_llvm_emit_fetch_64bit(bld_base, type, value, value2);
1647 }
1648 return LLVMBuildBitCast(ctx->ac.builder, value, type, "");
1649 }
1650
1651 static LLVMValueRef fetch_input_gs(
1652 struct lp_build_tgsi_context *bld_base,
1653 const struct tgsi_full_src_register *reg,
1654 enum tgsi_opcode_type type,
1655 unsigned swizzle)
1656 {
1657 struct si_shader_context *ctx = si_shader_context(bld_base);
1658 struct tgsi_shader_info *info = &ctx->shader->selector->info;
1659
1660 unsigned semantic_name = info->input_semantic_name[reg->Register.Index];
1661 if (swizzle != ~0 && semantic_name == TGSI_SEMANTIC_PRIMID)
1662 return get_primitive_id(ctx, swizzle);
1663
1664 if (!reg->Register.Dimension)
1665 return NULL;
1666
1667 return si_llvm_load_input_gs(&ctx->abi, reg->Register.Index,
1668 reg->Dimension.Index,
1669 tgsi2llvmtype(bld_base, type),
1670 swizzle);
1671 }
1672
1673 static int lookup_interp_param_index(unsigned interpolate, unsigned location)
1674 {
1675 switch (interpolate) {
1676 case TGSI_INTERPOLATE_CONSTANT:
1677 return 0;
1678
1679 case TGSI_INTERPOLATE_LINEAR:
1680 if (location == TGSI_INTERPOLATE_LOC_SAMPLE)
1681 return SI_PARAM_LINEAR_SAMPLE;
1682 else if (location == TGSI_INTERPOLATE_LOC_CENTROID)
1683 return SI_PARAM_LINEAR_CENTROID;
1684 else
1685 return SI_PARAM_LINEAR_CENTER;
1686 break;
1687 case TGSI_INTERPOLATE_COLOR:
1688 case TGSI_INTERPOLATE_PERSPECTIVE:
1689 if (location == TGSI_INTERPOLATE_LOC_SAMPLE)
1690 return SI_PARAM_PERSP_SAMPLE;
1691 else if (location == TGSI_INTERPOLATE_LOC_CENTROID)
1692 return SI_PARAM_PERSP_CENTROID;
1693 else
1694 return SI_PARAM_PERSP_CENTER;
1695 break;
1696 default:
1697 fprintf(stderr, "Warning: Unhandled interpolation mode.\n");
1698 return -1;
1699 }
1700 }
1701
1702 static LLVMValueRef si_build_fs_interp(struct si_shader_context *ctx,
1703 unsigned attr_index, unsigned chan,
1704 LLVMValueRef prim_mask,
1705 LLVMValueRef i, LLVMValueRef j)
1706 {
1707 if (i || j) {
1708 return ac_build_fs_interp(&ctx->ac,
1709 LLVMConstInt(ctx->i32, chan, 0),
1710 LLVMConstInt(ctx->i32, attr_index, 0),
1711 prim_mask, i, j);
1712 }
1713 return ac_build_fs_interp_mov(&ctx->ac,
1714 LLVMConstInt(ctx->i32, 2, 0), /* P0 */
1715 LLVMConstInt(ctx->i32, chan, 0),
1716 LLVMConstInt(ctx->i32, attr_index, 0),
1717 prim_mask);
1718 }
1719
1720 /**
1721 * Interpolate a fragment shader input.
1722 *
1723 * @param ctx context
1724 * @param input_index index of the input in hardware
1725 * @param semantic_name TGSI_SEMANTIC_*
1726 * @param semantic_index semantic index
1727 * @param num_interp_inputs number of all interpolated inputs (= BCOLOR offset)
1728 * @param colors_read_mask color components read (4 bits for each color, 8 bits in total)
1729 * @param interp_param interpolation weights (i,j)
1730 * @param prim_mask SI_PARAM_PRIM_MASK
1731 * @param face SI_PARAM_FRONT_FACE
1732 * @param result the return value (4 components)
1733 */
1734 static void interp_fs_input(struct si_shader_context *ctx,
1735 unsigned input_index,
1736 unsigned semantic_name,
1737 unsigned semantic_index,
1738 unsigned num_interp_inputs,
1739 unsigned colors_read_mask,
1740 LLVMValueRef interp_param,
1741 LLVMValueRef prim_mask,
1742 LLVMValueRef face,
1743 LLVMValueRef result[4])
1744 {
1745 LLVMValueRef i = NULL, j = NULL;
1746 unsigned chan;
1747
1748 /* fs.constant returns the param from the middle vertex, so it's not
1749 * really useful for flat shading. It's meant to be used for custom
1750 * interpolation (but the intrinsic can't fetch from the other two
1751 * vertices).
1752 *
1753 * Luckily, it doesn't matter, because we rely on the FLAT_SHADE state
1754 * to do the right thing. The only reason we use fs.constant is that
1755 * fs.interp cannot be used on integers, because they can be equal
1756 * to NaN.
1757 *
1758 * When interp is false we will use fs.constant or for newer llvm,
1759 * amdgcn.interp.mov.
1760 */
1761 bool interp = interp_param != NULL;
1762
1763 if (interp) {
1764 interp_param = LLVMBuildBitCast(ctx->ac.builder, interp_param,
1765 LLVMVectorType(ctx->f32, 2), "");
1766
1767 i = LLVMBuildExtractElement(ctx->ac.builder, interp_param,
1768 ctx->i32_0, "");
1769 j = LLVMBuildExtractElement(ctx->ac.builder, interp_param,
1770 ctx->i32_1, "");
1771 }
1772
1773 if (semantic_name == TGSI_SEMANTIC_COLOR &&
1774 ctx->shader->key.part.ps.prolog.color_two_side) {
1775 LLVMValueRef is_face_positive;
1776
1777 /* If BCOLOR0 is used, BCOLOR1 is at offset "num_inputs + 1",
1778 * otherwise it's at offset "num_inputs".
1779 */
1780 unsigned back_attr_offset = num_interp_inputs;
1781 if (semantic_index == 1 && colors_read_mask & 0xf)
1782 back_attr_offset += 1;
1783
1784 is_face_positive = LLVMBuildICmp(ctx->ac.builder, LLVMIntNE,
1785 face, ctx->i32_0, "");
1786
1787 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1788 LLVMValueRef front, back;
1789
1790 front = si_build_fs_interp(ctx,
1791 input_index, chan,
1792 prim_mask, i, j);
1793 back = si_build_fs_interp(ctx,
1794 back_attr_offset, chan,
1795 prim_mask, i, j);
1796
1797 result[chan] = LLVMBuildSelect(ctx->ac.builder,
1798 is_face_positive,
1799 front,
1800 back,
1801 "");
1802 }
1803 } else if (semantic_name == TGSI_SEMANTIC_FOG) {
1804 result[0] = si_build_fs_interp(ctx, input_index,
1805 0, prim_mask, i, j);
1806 result[1] =
1807 result[2] = LLVMConstReal(ctx->f32, 0.0f);
1808 result[3] = LLVMConstReal(ctx->f32, 1.0f);
1809 } else {
1810 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1811 result[chan] = si_build_fs_interp(ctx,
1812 input_index, chan,
1813 prim_mask, i, j);
1814 }
1815 }
1816 }
1817
1818 void si_llvm_load_input_fs(
1819 struct si_shader_context *ctx,
1820 unsigned input_index,
1821 LLVMValueRef out[4])
1822 {
1823 struct lp_build_context *base = &ctx->bld_base.base;
1824 struct si_shader *shader = ctx->shader;
1825 struct tgsi_shader_info *info = &shader->selector->info;
1826 LLVMValueRef main_fn = ctx->main_fn;
1827 LLVMValueRef interp_param = NULL;
1828 int interp_param_idx;
1829 enum tgsi_semantic semantic_name = info->input_semantic_name[input_index];
1830 unsigned semantic_index = info->input_semantic_index[input_index];
1831 enum tgsi_interpolate_mode interp_mode = info->input_interpolate[input_index];
1832 enum tgsi_interpolate_loc interp_loc = info->input_interpolate_loc[input_index];
1833
1834 /* Get colors from input VGPRs (set by the prolog). */
1835 if (semantic_name == TGSI_SEMANTIC_COLOR) {
1836 unsigned colors_read = shader->selector->info.colors_read;
1837 unsigned mask = colors_read >> (semantic_index * 4);
1838 unsigned offset = SI_PARAM_POS_FIXED_PT + 1 +
1839 (semantic_index ? util_bitcount(colors_read & 0xf) : 0);
1840
1841 out[0] = mask & 0x1 ? LLVMGetParam(main_fn, offset++) : base->undef;
1842 out[1] = mask & 0x2 ? LLVMGetParam(main_fn, offset++) : base->undef;
1843 out[2] = mask & 0x4 ? LLVMGetParam(main_fn, offset++) : base->undef;
1844 out[3] = mask & 0x8 ? LLVMGetParam(main_fn, offset++) : base->undef;
1845 return;
1846 }
1847
1848 interp_param_idx = lookup_interp_param_index(interp_mode, interp_loc);
1849 if (interp_param_idx == -1)
1850 return;
1851 else if (interp_param_idx) {
1852 interp_param = LLVMGetParam(ctx->main_fn, interp_param_idx);
1853 }
1854
1855 interp_fs_input(ctx, input_index, semantic_name,
1856 semantic_index, 0, /* this param is unused */
1857 shader->selector->info.colors_read, interp_param,
1858 ctx->abi.prim_mask,
1859 LLVMGetParam(main_fn, SI_PARAM_FRONT_FACE),
1860 &out[0]);
1861 }
1862
1863 static void declare_input_fs(
1864 struct si_shader_context *ctx,
1865 unsigned input_index,
1866 const struct tgsi_full_declaration *decl,
1867 LLVMValueRef out[4])
1868 {
1869 si_llvm_load_input_fs(ctx, input_index, out);
1870 }
1871
1872 static LLVMValueRef get_sample_id(struct si_shader_context *ctx)
1873 {
1874 return unpack_param(ctx, SI_PARAM_ANCILLARY, 8, 4);
1875 }
1876
1877 static LLVMValueRef get_block_size(struct ac_shader_abi *abi)
1878 {
1879 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1880
1881 LLVMValueRef values[3];
1882 LLVMValueRef result;
1883 unsigned i;
1884 unsigned *properties = ctx->shader->selector->info.properties;
1885
1886 if (properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH] != 0) {
1887 unsigned sizes[3] = {
1888 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH],
1889 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT],
1890 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH]
1891 };
1892
1893 for (i = 0; i < 3; ++i)
1894 values[i] = LLVMConstInt(ctx->i32, sizes[i], 0);
1895
1896 result = lp_build_gather_values(&ctx->gallivm, values, 3);
1897 } else {
1898 result = LLVMGetParam(ctx->main_fn, ctx->param_block_size);
1899 }
1900
1901 return result;
1902 }
1903
1904 /**
1905 * Load a dword from a constant buffer.
1906 */
1907 static LLVMValueRef buffer_load_const(struct si_shader_context *ctx,
1908 LLVMValueRef resource,
1909 LLVMValueRef offset)
1910 {
1911 return ac_build_buffer_load(&ctx->ac, resource, 1, NULL, offset, NULL,
1912 0, 0, 0, true, true);
1913 }
1914
1915 static LLVMValueRef load_sample_position(struct ac_shader_abi *abi, LLVMValueRef sample_id)
1916 {
1917 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1918 struct lp_build_context *uint_bld = &ctx->bld_base.uint_bld;
1919 LLVMValueRef desc = LLVMGetParam(ctx->main_fn, ctx->param_rw_buffers);
1920 LLVMValueRef buf_index = LLVMConstInt(ctx->i32, SI_PS_CONST_SAMPLE_POSITIONS, 0);
1921 LLVMValueRef resource = ac_build_load_to_sgpr(&ctx->ac, desc, buf_index);
1922
1923 /* offset = sample_id * 8 (8 = 2 floats containing samplepos.xy) */
1924 LLVMValueRef offset0 = lp_build_mul_imm(uint_bld, sample_id, 8);
1925 LLVMValueRef offset1 = LLVMBuildAdd(ctx->ac.builder, offset0, LLVMConstInt(ctx->i32, 4, 0), "");
1926
1927 LLVMValueRef pos[4] = {
1928 buffer_load_const(ctx, resource, offset0),
1929 buffer_load_const(ctx, resource, offset1),
1930 LLVMConstReal(ctx->f32, 0),
1931 LLVMConstReal(ctx->f32, 0)
1932 };
1933
1934 return lp_build_gather_values(&ctx->gallivm, pos, 4);
1935 }
1936
1937 static LLVMValueRef si_load_tess_coord(struct ac_shader_abi *abi,
1938 LLVMTypeRef type,
1939 unsigned num_components)
1940 {
1941 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1942 struct lp_build_context *bld = &ctx->bld_base.base;
1943
1944 LLVMValueRef coord[4] = {
1945 LLVMGetParam(ctx->main_fn, ctx->param_tes_u),
1946 LLVMGetParam(ctx->main_fn, ctx->param_tes_v),
1947 ctx->ac.f32_0,
1948 ctx->ac.f32_0
1949 };
1950
1951 /* For triangles, the vector should be (u, v, 1-u-v). */
1952 if (ctx->shader->selector->info.properties[TGSI_PROPERTY_TES_PRIM_MODE] ==
1953 PIPE_PRIM_TRIANGLES)
1954 coord[2] = lp_build_sub(bld, ctx->ac.f32_1,
1955 lp_build_add(bld, coord[0], coord[1]));
1956
1957 return lp_build_gather_values(&ctx->gallivm, coord, 4);
1958 }
1959
1960 static LLVMValueRef load_tess_level(struct si_shader_context *ctx,
1961 unsigned semantic_name)
1962 {
1963 LLVMValueRef buffer, base, addr;
1964
1965 int param = si_shader_io_get_unique_index_patch(semantic_name, 0);
1966
1967 buffer = desc_from_addr_base64k(ctx, ctx->param_tcs_offchip_addr_base64k);
1968
1969 base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
1970 addr = get_tcs_tes_buffer_address(ctx, get_rel_patch_id(ctx), NULL,
1971 LLVMConstInt(ctx->i32, param, 0));
1972
1973 return buffer_load(&ctx->bld_base, ctx->f32,
1974 ~0, buffer, base, addr, true);
1975
1976 }
1977
1978 static LLVMValueRef si_load_tess_level(struct ac_shader_abi *abi,
1979 unsigned varying_id)
1980 {
1981 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1982 unsigned semantic_name;
1983
1984 switch (varying_id) {
1985 case VARYING_SLOT_TESS_LEVEL_INNER:
1986 semantic_name = TGSI_SEMANTIC_TESSINNER;
1987 break;
1988 case VARYING_SLOT_TESS_LEVEL_OUTER:
1989 semantic_name = TGSI_SEMANTIC_TESSOUTER;
1990 break;
1991 default:
1992 unreachable("unknown tess level");
1993 }
1994
1995 return load_tess_level(ctx, semantic_name);
1996
1997 }
1998
1999 static LLVMValueRef si_load_patch_vertices_in(struct ac_shader_abi *abi)
2000 {
2001 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
2002 if (ctx->type == PIPE_SHADER_TESS_CTRL)
2003 return unpack_param(ctx, ctx->param_tcs_out_lds_layout, 26, 6);
2004 else if (ctx->type == PIPE_SHADER_TESS_EVAL)
2005 return get_num_tcs_out_vertices(ctx);
2006 else
2007 unreachable("invalid shader stage for TGSI_SEMANTIC_VERTICESIN");
2008 }
2009
2010 void si_load_system_value(struct si_shader_context *ctx,
2011 unsigned index,
2012 const struct tgsi_full_declaration *decl)
2013 {
2014 LLVMValueRef value = 0;
2015
2016 assert(index < RADEON_LLVM_MAX_SYSTEM_VALUES);
2017
2018 switch (decl->Semantic.Name) {
2019 case TGSI_SEMANTIC_INSTANCEID:
2020 value = ctx->abi.instance_id;
2021 break;
2022
2023 case TGSI_SEMANTIC_VERTEXID:
2024 value = LLVMBuildAdd(ctx->ac.builder,
2025 ctx->abi.vertex_id,
2026 ctx->abi.base_vertex, "");
2027 break;
2028
2029 case TGSI_SEMANTIC_VERTEXID_NOBASE:
2030 /* Unused. Clarify the meaning in indexed vs. non-indexed
2031 * draws if this is ever used again. */
2032 assert(false);
2033 break;
2034
2035 case TGSI_SEMANTIC_BASEVERTEX:
2036 {
2037 /* For non-indexed draws, the base vertex set by the driver
2038 * (for direct draws) or the CP (for indirect draws) is the
2039 * first vertex ID, but GLSL expects 0 to be returned.
2040 */
2041 LLVMValueRef vs_state = LLVMGetParam(ctx->main_fn, ctx->param_vs_state_bits);
2042 LLVMValueRef indexed;
2043
2044 indexed = LLVMBuildLShr(ctx->ac.builder, vs_state, ctx->i32_1, "");
2045 indexed = LLVMBuildTrunc(ctx->ac.builder, indexed, ctx->i1, "");
2046
2047 value = LLVMBuildSelect(ctx->ac.builder, indexed,
2048 ctx->abi.base_vertex, ctx->i32_0, "");
2049 break;
2050 }
2051
2052 case TGSI_SEMANTIC_BASEINSTANCE:
2053 value = ctx->abi.start_instance;
2054 break;
2055
2056 case TGSI_SEMANTIC_DRAWID:
2057 value = ctx->abi.draw_id;
2058 break;
2059
2060 case TGSI_SEMANTIC_INVOCATIONID:
2061 if (ctx->type == PIPE_SHADER_TESS_CTRL)
2062 value = unpack_llvm_param(ctx, ctx->abi.tcs_rel_ids, 8, 5);
2063 else if (ctx->type == PIPE_SHADER_GEOMETRY)
2064 value = ctx->abi.gs_invocation_id;
2065 else
2066 assert(!"INVOCATIONID not implemented");
2067 break;
2068
2069 case TGSI_SEMANTIC_POSITION:
2070 {
2071 LLVMValueRef pos[4] = {
2072 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_X_FLOAT),
2073 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Y_FLOAT),
2074 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Z_FLOAT),
2075 lp_build_emit_llvm_unary(&ctx->bld_base, TGSI_OPCODE_RCP,
2076 LLVMGetParam(ctx->main_fn,
2077 SI_PARAM_POS_W_FLOAT)),
2078 };
2079 value = lp_build_gather_values(&ctx->gallivm, pos, 4);
2080 break;
2081 }
2082
2083 case TGSI_SEMANTIC_FACE:
2084 value = ctx->abi.front_face;
2085 break;
2086
2087 case TGSI_SEMANTIC_SAMPLEID:
2088 value = get_sample_id(ctx);
2089 break;
2090
2091 case TGSI_SEMANTIC_SAMPLEPOS: {
2092 LLVMValueRef pos[4] = {
2093 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_X_FLOAT),
2094 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Y_FLOAT),
2095 LLVMConstReal(ctx->f32, 0),
2096 LLVMConstReal(ctx->f32, 0)
2097 };
2098 pos[0] = lp_build_emit_llvm_unary(&ctx->bld_base,
2099 TGSI_OPCODE_FRC, pos[0]);
2100 pos[1] = lp_build_emit_llvm_unary(&ctx->bld_base,
2101 TGSI_OPCODE_FRC, pos[1]);
2102 value = lp_build_gather_values(&ctx->gallivm, pos, 4);
2103 break;
2104 }
2105
2106 case TGSI_SEMANTIC_SAMPLEMASK:
2107 /* This can only occur with the OpenGL Core profile, which
2108 * doesn't support smoothing.
2109 */
2110 value = LLVMGetParam(ctx->main_fn, SI_PARAM_SAMPLE_COVERAGE);
2111 break;
2112
2113 case TGSI_SEMANTIC_TESSCOORD:
2114 value = si_load_tess_coord(&ctx->abi, NULL, 4);
2115 break;
2116
2117 case TGSI_SEMANTIC_VERTICESIN:
2118 value = si_load_patch_vertices_in(&ctx->abi);
2119 break;
2120
2121 case TGSI_SEMANTIC_TESSINNER:
2122 case TGSI_SEMANTIC_TESSOUTER:
2123 value = load_tess_level(ctx, decl->Semantic.Name);
2124 break;
2125
2126 case TGSI_SEMANTIC_DEFAULT_TESSOUTER_SI:
2127 case TGSI_SEMANTIC_DEFAULT_TESSINNER_SI:
2128 {
2129 LLVMValueRef buf, slot, val[4];
2130 int i, offset;
2131
2132 slot = LLVMConstInt(ctx->i32, SI_HS_CONST_DEFAULT_TESS_LEVELS, 0);
2133 buf = LLVMGetParam(ctx->main_fn, ctx->param_rw_buffers);
2134 buf = ac_build_load_to_sgpr(&ctx->ac, buf, slot);
2135 offset = decl->Semantic.Name == TGSI_SEMANTIC_DEFAULT_TESSINNER_SI ? 4 : 0;
2136
2137 for (i = 0; i < 4; i++)
2138 val[i] = buffer_load_const(ctx, buf,
2139 LLVMConstInt(ctx->i32, (offset + i) * 4, 0));
2140 value = lp_build_gather_values(&ctx->gallivm, val, 4);
2141 break;
2142 }
2143
2144 case TGSI_SEMANTIC_PRIMID:
2145 value = get_primitive_id(ctx, 0);
2146 break;
2147
2148 case TGSI_SEMANTIC_GRID_SIZE:
2149 value = ctx->abi.num_work_groups;
2150 break;
2151
2152 case TGSI_SEMANTIC_BLOCK_SIZE:
2153 value = get_block_size(&ctx->abi);
2154 break;
2155
2156 case TGSI_SEMANTIC_BLOCK_ID:
2157 {
2158 LLVMValueRef values[3];
2159
2160 for (int i = 0; i < 3; i++) {
2161 values[i] = ctx->i32_0;
2162 if (ctx->abi.workgroup_ids[i]) {
2163 values[i] = ctx->abi.workgroup_ids[i];
2164 }
2165 }
2166 value = lp_build_gather_values(&ctx->gallivm, values, 3);
2167 break;
2168 }
2169
2170 case TGSI_SEMANTIC_THREAD_ID:
2171 value = ctx->abi.local_invocation_ids;
2172 break;
2173
2174 case TGSI_SEMANTIC_HELPER_INVOCATION:
2175 value = lp_build_intrinsic(ctx->ac.builder,
2176 "llvm.amdgcn.ps.live",
2177 ctx->i1, NULL, 0,
2178 LP_FUNC_ATTR_READNONE);
2179 value = LLVMBuildNot(ctx->ac.builder, value, "");
2180 value = LLVMBuildSExt(ctx->ac.builder, value, ctx->i32, "");
2181 break;
2182
2183 case TGSI_SEMANTIC_SUBGROUP_SIZE:
2184 value = LLVMConstInt(ctx->i32, 64, 0);
2185 break;
2186
2187 case TGSI_SEMANTIC_SUBGROUP_INVOCATION:
2188 value = ac_get_thread_id(&ctx->ac);
2189 break;
2190
2191 case TGSI_SEMANTIC_SUBGROUP_EQ_MASK:
2192 {
2193 LLVMValueRef id = ac_get_thread_id(&ctx->ac);
2194 id = LLVMBuildZExt(ctx->ac.builder, id, ctx->i64, "");
2195 value = LLVMBuildShl(ctx->ac.builder, LLVMConstInt(ctx->i64, 1, 0), id, "");
2196 value = LLVMBuildBitCast(ctx->ac.builder, value, ctx->v2i32, "");
2197 break;
2198 }
2199
2200 case TGSI_SEMANTIC_SUBGROUP_GE_MASK:
2201 case TGSI_SEMANTIC_SUBGROUP_GT_MASK:
2202 case TGSI_SEMANTIC_SUBGROUP_LE_MASK:
2203 case TGSI_SEMANTIC_SUBGROUP_LT_MASK:
2204 {
2205 LLVMValueRef id = ac_get_thread_id(&ctx->ac);
2206 if (decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_GT_MASK ||
2207 decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_LE_MASK) {
2208 /* All bits set except LSB */
2209 value = LLVMConstInt(ctx->i64, -2, 0);
2210 } else {
2211 /* All bits set */
2212 value = LLVMConstInt(ctx->i64, -1, 0);
2213 }
2214 id = LLVMBuildZExt(ctx->ac.builder, id, ctx->i64, "");
2215 value = LLVMBuildShl(ctx->ac.builder, value, id, "");
2216 if (decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_LE_MASK ||
2217 decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_LT_MASK)
2218 value = LLVMBuildNot(ctx->ac.builder, value, "");
2219 value = LLVMBuildBitCast(ctx->ac.builder, value, ctx->v2i32, "");
2220 break;
2221 }
2222
2223 default:
2224 assert(!"unknown system value");
2225 return;
2226 }
2227
2228 ctx->system_values[index] = value;
2229 }
2230
2231 void si_declare_compute_memory(struct si_shader_context *ctx,
2232 const struct tgsi_full_declaration *decl)
2233 {
2234 struct si_shader_selector *sel = ctx->shader->selector;
2235
2236 LLVMTypeRef i8p = LLVMPointerType(ctx->i8, AC_LOCAL_ADDR_SPACE);
2237 LLVMValueRef var;
2238
2239 assert(decl->Declaration.MemType == TGSI_MEMORY_TYPE_SHARED);
2240 assert(decl->Range.First == decl->Range.Last);
2241 assert(!ctx->ac.lds);
2242
2243 var = LLVMAddGlobalInAddressSpace(ctx->ac.module,
2244 LLVMArrayType(ctx->i8, sel->local_size),
2245 "compute_lds",
2246 AC_LOCAL_ADDR_SPACE);
2247 LLVMSetAlignment(var, 4);
2248
2249 ctx->ac.lds = LLVMBuildBitCast(ctx->ac.builder, var, i8p, "");
2250 }
2251
2252 static LLVMValueRef load_const_buffer_desc(struct si_shader_context *ctx, int i)
2253 {
2254 LLVMValueRef list_ptr = LLVMGetParam(ctx->main_fn,
2255 ctx->param_const_and_shader_buffers);
2256
2257 return ac_build_load_to_sgpr(&ctx->ac, list_ptr,
2258 LLVMConstInt(ctx->i32, si_get_constbuf_slot(i), 0));
2259 }
2260
2261 static LLVMValueRef load_ubo(struct ac_shader_abi *abi, LLVMValueRef index)
2262 {
2263 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
2264 LLVMValueRef ptr = LLVMGetParam(ctx->main_fn, ctx->param_const_and_shader_buffers);
2265
2266 index = si_llvm_bound_index(ctx, index, ctx->num_const_buffers);
2267 index = LLVMBuildAdd(ctx->ac.builder, index,
2268 LLVMConstInt(ctx->i32, SI_NUM_SHADER_BUFFERS, 0), "");
2269
2270 return ac_build_load_to_sgpr(&ctx->ac, ptr, index);
2271 }
2272
2273 static LLVMValueRef
2274 load_ssbo(struct ac_shader_abi *abi, LLVMValueRef index, bool write)
2275 {
2276 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
2277 LLVMValueRef rsrc_ptr = LLVMGetParam(ctx->main_fn,
2278 ctx->param_const_and_shader_buffers);
2279
2280 index = si_llvm_bound_index(ctx, index, ctx->num_shader_buffers);
2281 index = LLVMBuildSub(ctx->ac.builder,
2282 LLVMConstInt(ctx->i32, SI_NUM_SHADER_BUFFERS - 1, 0),
2283 index, "");
2284
2285 return ac_build_load_to_sgpr(&ctx->ac, rsrc_ptr, index);
2286 }
2287
2288 static LLVMValueRef fetch_constant(
2289 struct lp_build_tgsi_context *bld_base,
2290 const struct tgsi_full_src_register *reg,
2291 enum tgsi_opcode_type type,
2292 unsigned swizzle)
2293 {
2294 struct si_shader_context *ctx = si_shader_context(bld_base);
2295 struct si_shader_selector *sel = ctx->shader->selector;
2296 const struct tgsi_ind_register *ireg = &reg->Indirect;
2297 unsigned buf, idx;
2298
2299 LLVMValueRef addr, bufp;
2300
2301 if (swizzle == LP_CHAN_ALL) {
2302 unsigned chan;
2303 LLVMValueRef values[4];
2304 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan)
2305 values[chan] = fetch_constant(bld_base, reg, type, chan);
2306
2307 return lp_build_gather_values(&ctx->gallivm, values, 4);
2308 }
2309
2310 /* Split 64-bit loads. */
2311 if (tgsi_type_is_64bit(type)) {
2312 LLVMValueRef lo, hi;
2313
2314 lo = fetch_constant(bld_base, reg, TGSI_TYPE_UNSIGNED, swizzle);
2315 hi = fetch_constant(bld_base, reg, TGSI_TYPE_UNSIGNED, swizzle + 1);
2316 return si_llvm_emit_fetch_64bit(bld_base, tgsi2llvmtype(bld_base, type),
2317 lo, hi);
2318 }
2319
2320 idx = reg->Register.Index * 4 + swizzle;
2321 if (reg->Register.Indirect) {
2322 addr = si_get_indirect_index(ctx, ireg, 16, idx * 4);
2323 } else {
2324 addr = LLVMConstInt(ctx->i32, idx * 4, 0);
2325 }
2326
2327 /* Fast path when user data SGPRs point to constant buffer 0 directly. */
2328 if (sel->info.const_buffers_declared == 1 &&
2329 sel->info.shader_buffers_declared == 0) {
2330 LLVMValueRef ptr =
2331 LLVMGetParam(ctx->main_fn, ctx->param_const_and_shader_buffers);
2332
2333 /* This enables use of s_load_dword and flat_load_dword for const buffer 0
2334 * loads, and up to x4 load opcode merging. However, it leads to horrible
2335 * code reducing SIMD wave occupancy from 8 to 2 in many cases.
2336 *
2337 * Using s_buffer_load_dword (x1) seems to be the best option right now.
2338 *
2339 * LLVM 5.0 on SI doesn't insert a required s_nop between SALU setting
2340 * a descriptor and s_buffer_load_dword using it, so we can't expand
2341 * the pointer into a full descriptor like below. We have to use
2342 * s_load_dword instead. The only case when LLVM 5.0 would select
2343 * s_buffer_load_dword (that we have to prevent) is when we use use
2344 * a literal offset where we don't need bounds checking.
2345 */
2346 if (ctx->screen->info.chip_class == SI &&
2347 HAVE_LLVM < 0x0600 &&
2348 !reg->Register.Indirect) {
2349 addr = LLVMBuildLShr(ctx->ac.builder, addr, LLVMConstInt(ctx->i32, 2, 0), "");
2350 LLVMValueRef result = ac_build_load_invariant(&ctx->ac, ptr, addr);
2351 return bitcast(bld_base, type, result);
2352 }
2353
2354 /* Do the bounds checking with a descriptor, because
2355 * doing computation and manual bounds checking of 64-bit
2356 * addresses generates horrible VALU code with very high
2357 * VGPR usage and very low SIMD occupancy.
2358 */
2359 ptr = LLVMBuildPtrToInt(ctx->ac.builder, ptr, ctx->i64, "");
2360 ptr = LLVMBuildBitCast(ctx->ac.builder, ptr, ctx->v2i32, "");
2361
2362 LLVMValueRef desc_elems[] = {
2363 LLVMBuildExtractElement(ctx->ac.builder, ptr, ctx->i32_0, ""),
2364 LLVMBuildExtractElement(ctx->ac.builder, ptr, ctx->i32_1, ""),
2365 LLVMConstInt(ctx->i32, (sel->info.const_file_max[0] + 1) * 16, 0),
2366 LLVMConstInt(ctx->i32,
2367 S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
2368 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
2369 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
2370 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
2371 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
2372 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32), 0)
2373 };
2374 LLVMValueRef desc = ac_build_gather_values(&ctx->ac, desc_elems, 4);
2375 LLVMValueRef result = buffer_load_const(ctx, desc, addr);
2376 return bitcast(bld_base, type, result);
2377 }
2378
2379 assert(reg->Register.Dimension);
2380 buf = reg->Dimension.Index;
2381
2382 if (reg->Dimension.Indirect) {
2383 LLVMValueRef ptr = LLVMGetParam(ctx->main_fn, ctx->param_const_and_shader_buffers);
2384 LLVMValueRef index;
2385 index = si_get_bounded_indirect_index(ctx, &reg->DimIndirect,
2386 reg->Dimension.Index,
2387 ctx->num_const_buffers);
2388 index = LLVMBuildAdd(ctx->ac.builder, index,
2389 LLVMConstInt(ctx->i32, SI_NUM_SHADER_BUFFERS, 0), "");
2390 bufp = ac_build_load_to_sgpr(&ctx->ac, ptr, index);
2391 } else
2392 bufp = load_const_buffer_desc(ctx, buf);
2393
2394 return bitcast(bld_base, type, buffer_load_const(ctx, bufp, addr));
2395 }
2396
2397 /* Initialize arguments for the shader export intrinsic */
2398 static void si_llvm_init_export_args(struct si_shader_context *ctx,
2399 LLVMValueRef *values,
2400 unsigned target,
2401 struct ac_export_args *args)
2402 {
2403 LLVMValueRef f32undef = LLVMGetUndef(ctx->ac.f32);
2404 unsigned spi_shader_col_format = V_028714_SPI_SHADER_32_ABGR;
2405 unsigned chan;
2406 bool is_int8, is_int10;
2407
2408 /* Default is 0xf. Adjusted below depending on the format. */
2409 args->enabled_channels = 0xf; /* writemask */
2410
2411 /* Specify whether the EXEC mask represents the valid mask */
2412 args->valid_mask = 0;
2413
2414 /* Specify whether this is the last export */
2415 args->done = 0;
2416
2417 /* Specify the target we are exporting */
2418 args->target = target;
2419
2420 if (ctx->type == PIPE_SHADER_FRAGMENT) {
2421 const struct si_shader_key *key = &ctx->shader->key;
2422 unsigned col_formats = key->part.ps.epilog.spi_shader_col_format;
2423 int cbuf = target - V_008DFC_SQ_EXP_MRT;
2424
2425 assert(cbuf >= 0 && cbuf < 8);
2426 spi_shader_col_format = (col_formats >> (cbuf * 4)) & 0xf;
2427 is_int8 = (key->part.ps.epilog.color_is_int8 >> cbuf) & 0x1;
2428 is_int10 = (key->part.ps.epilog.color_is_int10 >> cbuf) & 0x1;
2429 }
2430
2431 args->compr = false;
2432 args->out[0] = f32undef;
2433 args->out[1] = f32undef;
2434 args->out[2] = f32undef;
2435 args->out[3] = f32undef;
2436
2437 LLVMValueRef (*packf)(struct ac_llvm_context *ctx, LLVMValueRef args[2]) = NULL;
2438 LLVMValueRef (*packi)(struct ac_llvm_context *ctx, LLVMValueRef args[2],
2439 unsigned bits, bool hi) = NULL;
2440
2441 switch (spi_shader_col_format) {
2442 case V_028714_SPI_SHADER_ZERO:
2443 args->enabled_channels = 0; /* writemask */
2444 args->target = V_008DFC_SQ_EXP_NULL;
2445 break;
2446
2447 case V_028714_SPI_SHADER_32_R:
2448 args->enabled_channels = 1; /* writemask */
2449 args->out[0] = values[0];
2450 break;
2451
2452 case V_028714_SPI_SHADER_32_GR:
2453 args->enabled_channels = 0x3; /* writemask */
2454 args->out[0] = values[0];
2455 args->out[1] = values[1];
2456 break;
2457
2458 case V_028714_SPI_SHADER_32_AR:
2459 args->enabled_channels = 0x9; /* writemask */
2460 args->out[0] = values[0];
2461 args->out[3] = values[3];
2462 break;
2463
2464 case V_028714_SPI_SHADER_FP16_ABGR:
2465 packf = ac_build_cvt_pkrtz_f16;
2466 break;
2467
2468 case V_028714_SPI_SHADER_UNORM16_ABGR:
2469 packf = ac_build_cvt_pknorm_u16;
2470 break;
2471
2472 case V_028714_SPI_SHADER_SNORM16_ABGR:
2473 packf = ac_build_cvt_pknorm_i16;
2474 break;
2475
2476 case V_028714_SPI_SHADER_UINT16_ABGR:
2477 packi = ac_build_cvt_pk_u16;
2478 break;
2479
2480 case V_028714_SPI_SHADER_SINT16_ABGR:
2481 packi = ac_build_cvt_pk_i16;
2482 break;
2483
2484 case V_028714_SPI_SHADER_32_ABGR:
2485 memcpy(&args->out[0], values, sizeof(values[0]) * 4);
2486 break;
2487 }
2488
2489 /* Pack f16 or norm_i16/u16. */
2490 if (packf) {
2491 for (chan = 0; chan < 2; chan++) {
2492 LLVMValueRef pack_args[2] = {
2493 values[2 * chan],
2494 values[2 * chan + 1]
2495 };
2496 LLVMValueRef packed;
2497
2498 packed = packf(&ctx->ac, pack_args);
2499 args->out[chan] = ac_to_float(&ctx->ac, packed);
2500 }
2501 args->compr = 1; /* COMPR flag */
2502 }
2503 /* Pack i16/u16. */
2504 if (packi) {
2505 for (chan = 0; chan < 2; chan++) {
2506 LLVMValueRef pack_args[2] = {
2507 ac_to_integer(&ctx->ac, values[2 * chan]),
2508 ac_to_integer(&ctx->ac, values[2 * chan + 1])
2509 };
2510 LLVMValueRef packed;
2511
2512 packed = packi(&ctx->ac, pack_args,
2513 is_int8 ? 8 : is_int10 ? 10 : 16,
2514 chan == 1);
2515 args->out[chan] = ac_to_float(&ctx->ac, packed);
2516 }
2517 args->compr = 1; /* COMPR flag */
2518 }
2519 }
2520
2521 static void si_alpha_test(struct lp_build_tgsi_context *bld_base,
2522 LLVMValueRef alpha)
2523 {
2524 struct si_shader_context *ctx = si_shader_context(bld_base);
2525
2526 if (ctx->shader->key.part.ps.epilog.alpha_func != PIPE_FUNC_NEVER) {
2527 static LLVMRealPredicate cond_map[PIPE_FUNC_ALWAYS + 1] = {
2528 [PIPE_FUNC_LESS] = LLVMRealOLT,
2529 [PIPE_FUNC_EQUAL] = LLVMRealOEQ,
2530 [PIPE_FUNC_LEQUAL] = LLVMRealOLE,
2531 [PIPE_FUNC_GREATER] = LLVMRealOGT,
2532 [PIPE_FUNC_NOTEQUAL] = LLVMRealONE,
2533 [PIPE_FUNC_GEQUAL] = LLVMRealOGE,
2534 };
2535 LLVMRealPredicate cond = cond_map[ctx->shader->key.part.ps.epilog.alpha_func];
2536 assert(cond);
2537
2538 LLVMValueRef alpha_ref = LLVMGetParam(ctx->main_fn,
2539 SI_PARAM_ALPHA_REF);
2540 LLVMValueRef alpha_pass =
2541 LLVMBuildFCmp(ctx->ac.builder, cond, alpha, alpha_ref, "");
2542 ac_build_kill_if_false(&ctx->ac, alpha_pass);
2543 } else {
2544 ac_build_kill_if_false(&ctx->ac, LLVMConstInt(ctx->i1, 0, 0));
2545 }
2546 }
2547
2548 static LLVMValueRef si_scale_alpha_by_sample_mask(struct lp_build_tgsi_context *bld_base,
2549 LLVMValueRef alpha,
2550 unsigned samplemask_param)
2551 {
2552 struct si_shader_context *ctx = si_shader_context(bld_base);
2553 LLVMValueRef coverage;
2554
2555 /* alpha = alpha * popcount(coverage) / SI_NUM_SMOOTH_AA_SAMPLES */
2556 coverage = LLVMGetParam(ctx->main_fn,
2557 samplemask_param);
2558 coverage = ac_to_integer(&ctx->ac, coverage);
2559
2560 coverage = lp_build_intrinsic(ctx->ac.builder, "llvm.ctpop.i32",
2561 ctx->i32,
2562 &coverage, 1, LP_FUNC_ATTR_READNONE);
2563
2564 coverage = LLVMBuildUIToFP(ctx->ac.builder, coverage,
2565 ctx->f32, "");
2566
2567 coverage = LLVMBuildFMul(ctx->ac.builder, coverage,
2568 LLVMConstReal(ctx->f32,
2569 1.0 / SI_NUM_SMOOTH_AA_SAMPLES), "");
2570
2571 return LLVMBuildFMul(ctx->ac.builder, alpha, coverage, "");
2572 }
2573
2574 static void si_llvm_emit_clipvertex(struct si_shader_context *ctx,
2575 struct ac_export_args *pos, LLVMValueRef *out_elts)
2576 {
2577 unsigned reg_index;
2578 unsigned chan;
2579 unsigned const_chan;
2580 LLVMValueRef base_elt;
2581 LLVMValueRef ptr = LLVMGetParam(ctx->main_fn, ctx->param_rw_buffers);
2582 LLVMValueRef constbuf_index = LLVMConstInt(ctx->i32,
2583 SI_VS_CONST_CLIP_PLANES, 0);
2584 LLVMValueRef const_resource = ac_build_load_to_sgpr(&ctx->ac, ptr, constbuf_index);
2585
2586 for (reg_index = 0; reg_index < 2; reg_index ++) {
2587 struct ac_export_args *args = &pos[2 + reg_index];
2588
2589 args->out[0] =
2590 args->out[1] =
2591 args->out[2] =
2592 args->out[3] = LLVMConstReal(ctx->f32, 0.0f);
2593
2594 /* Compute dot products of position and user clip plane vectors */
2595 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
2596 for (const_chan = 0; const_chan < TGSI_NUM_CHANNELS; const_chan++) {
2597 LLVMValueRef addr =
2598 LLVMConstInt(ctx->i32, ((reg_index * 4 + chan) * 4 +
2599 const_chan) * 4, 0);
2600 base_elt = buffer_load_const(ctx, const_resource,
2601 addr);
2602 args->out[chan] =
2603 lp_build_add(&ctx->bld_base.base, args->out[chan],
2604 lp_build_mul(&ctx->bld_base.base, base_elt,
2605 out_elts[const_chan]));
2606 }
2607 }
2608
2609 args->enabled_channels = 0xf;
2610 args->valid_mask = 0;
2611 args->done = 0;
2612 args->target = V_008DFC_SQ_EXP_POS + 2 + reg_index;
2613 args->compr = 0;
2614 }
2615 }
2616
2617 static void si_dump_streamout(struct pipe_stream_output_info *so)
2618 {
2619 unsigned i;
2620
2621 if (so->num_outputs)
2622 fprintf(stderr, "STREAMOUT\n");
2623
2624 for (i = 0; i < so->num_outputs; i++) {
2625 unsigned mask = ((1 << so->output[i].num_components) - 1) <<
2626 so->output[i].start_component;
2627 fprintf(stderr, " %i: BUF%i[%i..%i] <- OUT[%i].%s%s%s%s\n",
2628 i, so->output[i].output_buffer,
2629 so->output[i].dst_offset, so->output[i].dst_offset + so->output[i].num_components - 1,
2630 so->output[i].register_index,
2631 mask & 1 ? "x" : "",
2632 mask & 2 ? "y" : "",
2633 mask & 4 ? "z" : "",
2634 mask & 8 ? "w" : "");
2635 }
2636 }
2637
2638 static void emit_streamout_output(struct si_shader_context *ctx,
2639 LLVMValueRef const *so_buffers,
2640 LLVMValueRef const *so_write_offsets,
2641 struct pipe_stream_output *stream_out,
2642 struct si_shader_output_values *shader_out)
2643 {
2644 unsigned buf_idx = stream_out->output_buffer;
2645 unsigned start = stream_out->start_component;
2646 unsigned num_comps = stream_out->num_components;
2647 LLVMValueRef out[4];
2648
2649 assert(num_comps && num_comps <= 4);
2650 if (!num_comps || num_comps > 4)
2651 return;
2652
2653 /* Load the output as int. */
2654 for (int j = 0; j < num_comps; j++) {
2655 assert(stream_out->stream == shader_out->vertex_stream[start + j]);
2656
2657 out[j] = ac_to_integer(&ctx->ac, shader_out->values[start + j]);
2658 }
2659
2660 /* Pack the output. */
2661 LLVMValueRef vdata = NULL;
2662
2663 switch (num_comps) {
2664 case 1: /* as i32 */
2665 vdata = out[0];
2666 break;
2667 case 2: /* as v2i32 */
2668 case 3: /* as v4i32 (aligned to 4) */
2669 case 4: /* as v4i32 */
2670 vdata = LLVMGetUndef(LLVMVectorType(ctx->i32, util_next_power_of_two(num_comps)));
2671 for (int j = 0; j < num_comps; j++) {
2672 vdata = LLVMBuildInsertElement(ctx->ac.builder, vdata, out[j],
2673 LLVMConstInt(ctx->i32, j, 0), "");
2674 }
2675 break;
2676 }
2677
2678 ac_build_buffer_store_dword(&ctx->ac, so_buffers[buf_idx],
2679 vdata, num_comps,
2680 so_write_offsets[buf_idx],
2681 ctx->i32_0,
2682 stream_out->dst_offset * 4, 1, 1, true, false);
2683 }
2684
2685 /**
2686 * Write streamout data to buffers for vertex stream @p stream (different
2687 * vertex streams can occur for GS copy shaders).
2688 */
2689 static void si_llvm_emit_streamout(struct si_shader_context *ctx,
2690 struct si_shader_output_values *outputs,
2691 unsigned noutput, unsigned stream)
2692 {
2693 struct si_shader_selector *sel = ctx->shader->selector;
2694 struct pipe_stream_output_info *so = &sel->so;
2695 LLVMBuilderRef builder = ctx->ac.builder;
2696 int i;
2697 struct lp_build_if_state if_ctx;
2698
2699 /* Get bits [22:16], i.e. (so_param >> 16) & 127; */
2700 LLVMValueRef so_vtx_count =
2701 unpack_param(ctx, ctx->param_streamout_config, 16, 7);
2702
2703 LLVMValueRef tid = ac_get_thread_id(&ctx->ac);
2704
2705 /* can_emit = tid < so_vtx_count; */
2706 LLVMValueRef can_emit =
2707 LLVMBuildICmp(builder, LLVMIntULT, tid, so_vtx_count, "");
2708
2709 /* Emit the streamout code conditionally. This actually avoids
2710 * out-of-bounds buffer access. The hw tells us via the SGPR
2711 * (so_vtx_count) which threads are allowed to emit streamout data. */
2712 lp_build_if(&if_ctx, &ctx->gallivm, can_emit);
2713 {
2714 /* The buffer offset is computed as follows:
2715 * ByteOffset = streamout_offset[buffer_id]*4 +
2716 * (streamout_write_index + thread_id)*stride[buffer_id] +
2717 * attrib_offset
2718 */
2719
2720 LLVMValueRef so_write_index =
2721 LLVMGetParam(ctx->main_fn,
2722 ctx->param_streamout_write_index);
2723
2724 /* Compute (streamout_write_index + thread_id). */
2725 so_write_index = LLVMBuildAdd(builder, so_write_index, tid, "");
2726
2727 /* Load the descriptor and compute the write offset for each
2728 * enabled buffer. */
2729 LLVMValueRef so_write_offset[4] = {};
2730 LLVMValueRef so_buffers[4];
2731 LLVMValueRef buf_ptr = LLVMGetParam(ctx->main_fn,
2732 ctx->param_rw_buffers);
2733
2734 for (i = 0; i < 4; i++) {
2735 if (!so->stride[i])
2736 continue;
2737
2738 LLVMValueRef offset = LLVMConstInt(ctx->i32,
2739 SI_VS_STREAMOUT_BUF0 + i, 0);
2740
2741 so_buffers[i] = ac_build_load_to_sgpr(&ctx->ac, buf_ptr, offset);
2742
2743 LLVMValueRef so_offset = LLVMGetParam(ctx->main_fn,
2744 ctx->param_streamout_offset[i]);
2745 so_offset = LLVMBuildMul(builder, so_offset, LLVMConstInt(ctx->i32, 4, 0), "");
2746
2747 so_write_offset[i] = LLVMBuildMul(builder, so_write_index,
2748 LLVMConstInt(ctx->i32, so->stride[i]*4, 0), "");
2749 so_write_offset[i] = LLVMBuildAdd(builder, so_write_offset[i], so_offset, "");
2750 }
2751
2752 /* Write streamout data. */
2753 for (i = 0; i < so->num_outputs; i++) {
2754 unsigned reg = so->output[i].register_index;
2755
2756 if (reg >= noutput)
2757 continue;
2758
2759 if (stream != so->output[i].stream)
2760 continue;
2761
2762 emit_streamout_output(ctx, so_buffers, so_write_offset,
2763 &so->output[i], &outputs[reg]);
2764 }
2765 }
2766 lp_build_endif(&if_ctx);
2767 }
2768
2769 static void si_export_param(struct si_shader_context *ctx, unsigned index,
2770 LLVMValueRef *values)
2771 {
2772 struct ac_export_args args;
2773
2774 si_llvm_init_export_args(ctx, values,
2775 V_008DFC_SQ_EXP_PARAM + index, &args);
2776 ac_build_export(&ctx->ac, &args);
2777 }
2778
2779 static void si_build_param_exports(struct si_shader_context *ctx,
2780 struct si_shader_output_values *outputs,
2781 unsigned noutput)
2782 {
2783 struct si_shader *shader = ctx->shader;
2784 unsigned param_count = 0;
2785
2786 for (unsigned i = 0; i < noutput; i++) {
2787 unsigned semantic_name = outputs[i].semantic_name;
2788 unsigned semantic_index = outputs[i].semantic_index;
2789
2790 if (outputs[i].vertex_stream[0] != 0 &&
2791 outputs[i].vertex_stream[1] != 0 &&
2792 outputs[i].vertex_stream[2] != 0 &&
2793 outputs[i].vertex_stream[3] != 0)
2794 continue;
2795
2796 switch (semantic_name) {
2797 case TGSI_SEMANTIC_LAYER:
2798 case TGSI_SEMANTIC_VIEWPORT_INDEX:
2799 case TGSI_SEMANTIC_CLIPDIST:
2800 case TGSI_SEMANTIC_COLOR:
2801 case TGSI_SEMANTIC_BCOLOR:
2802 case TGSI_SEMANTIC_PRIMID:
2803 case TGSI_SEMANTIC_FOG:
2804 case TGSI_SEMANTIC_TEXCOORD:
2805 case TGSI_SEMANTIC_GENERIC:
2806 break;
2807 default:
2808 continue;
2809 }
2810
2811 if ((semantic_name != TGSI_SEMANTIC_GENERIC ||
2812 semantic_index < SI_MAX_IO_GENERIC) &&
2813 shader->key.opt.kill_outputs &
2814 (1ull << si_shader_io_get_unique_index(semantic_name, semantic_index)))
2815 continue;
2816
2817 si_export_param(ctx, param_count, outputs[i].values);
2818
2819 assert(i < ARRAY_SIZE(shader->info.vs_output_param_offset));
2820 shader->info.vs_output_param_offset[i] = param_count++;
2821 }
2822
2823 shader->info.nr_param_exports = param_count;
2824 }
2825
2826 /* Generate export instructions for hardware VS shader stage */
2827 static void si_llvm_export_vs(struct si_shader_context *ctx,
2828 struct si_shader_output_values *outputs,
2829 unsigned noutput)
2830 {
2831 struct si_shader *shader = ctx->shader;
2832 struct ac_export_args pos_args[4] = {};
2833 LLVMValueRef psize_value = NULL, edgeflag_value = NULL, layer_value = NULL, viewport_index_value = NULL;
2834 unsigned pos_idx;
2835 int i;
2836
2837 /* Build position exports. */
2838 for (i = 0; i < noutput; i++) {
2839 switch (outputs[i].semantic_name) {
2840 case TGSI_SEMANTIC_POSITION:
2841 si_llvm_init_export_args(ctx, outputs[i].values,
2842 V_008DFC_SQ_EXP_POS, &pos_args[0]);
2843 break;
2844 case TGSI_SEMANTIC_PSIZE:
2845 psize_value = outputs[i].values[0];
2846 break;
2847 case TGSI_SEMANTIC_LAYER:
2848 layer_value = outputs[i].values[0];
2849 break;
2850 case TGSI_SEMANTIC_VIEWPORT_INDEX:
2851 viewport_index_value = outputs[i].values[0];
2852 break;
2853 case TGSI_SEMANTIC_EDGEFLAG:
2854 edgeflag_value = outputs[i].values[0];
2855 break;
2856 case TGSI_SEMANTIC_CLIPDIST:
2857 if (!shader->key.opt.clip_disable) {
2858 unsigned index = 2 + outputs[i].semantic_index;
2859 si_llvm_init_export_args(ctx, outputs[i].values,
2860 V_008DFC_SQ_EXP_POS + index,
2861 &pos_args[index]);
2862 }
2863 break;
2864 case TGSI_SEMANTIC_CLIPVERTEX:
2865 if (!shader->key.opt.clip_disable) {
2866 si_llvm_emit_clipvertex(ctx, pos_args,
2867 outputs[i].values);
2868 }
2869 break;
2870 }
2871 }
2872
2873 /* We need to add the position output manually if it's missing. */
2874 if (!pos_args[0].out[0]) {
2875 pos_args[0].enabled_channels = 0xf; /* writemask */
2876 pos_args[0].valid_mask = 0; /* EXEC mask */
2877 pos_args[0].done = 0; /* last export? */
2878 pos_args[0].target = V_008DFC_SQ_EXP_POS;
2879 pos_args[0].compr = 0; /* COMPR flag */
2880 pos_args[0].out[0] = ctx->ac.f32_0; /* X */
2881 pos_args[0].out[1] = ctx->ac.f32_0; /* Y */
2882 pos_args[0].out[2] = ctx->ac.f32_0; /* Z */
2883 pos_args[0].out[3] = ctx->ac.f32_1; /* W */
2884 }
2885
2886 /* Write the misc vector (point size, edgeflag, layer, viewport). */
2887 if (shader->selector->info.writes_psize ||
2888 shader->selector->info.writes_edgeflag ||
2889 shader->selector->info.writes_viewport_index ||
2890 shader->selector->info.writes_layer) {
2891 pos_args[1].enabled_channels = shader->selector->info.writes_psize |
2892 (shader->selector->info.writes_edgeflag << 1) |
2893 (shader->selector->info.writes_layer << 2);
2894
2895 pos_args[1].valid_mask = 0; /* EXEC mask */
2896 pos_args[1].done = 0; /* last export? */
2897 pos_args[1].target = V_008DFC_SQ_EXP_POS + 1;
2898 pos_args[1].compr = 0; /* COMPR flag */
2899 pos_args[1].out[0] = ctx->ac.f32_0; /* X */
2900 pos_args[1].out[1] = ctx->ac.f32_0; /* Y */
2901 pos_args[1].out[2] = ctx->ac.f32_0; /* Z */
2902 pos_args[1].out[3] = ctx->ac.f32_0; /* W */
2903
2904 if (shader->selector->info.writes_psize)
2905 pos_args[1].out[0] = psize_value;
2906
2907 if (shader->selector->info.writes_edgeflag) {
2908 /* The output is a float, but the hw expects an integer
2909 * with the first bit containing the edge flag. */
2910 edgeflag_value = LLVMBuildFPToUI(ctx->ac.builder,
2911 edgeflag_value,
2912 ctx->i32, "");
2913 edgeflag_value = ac_build_umin(&ctx->ac,
2914 edgeflag_value,
2915 ctx->i32_1);
2916
2917 /* The LLVM intrinsic expects a float. */
2918 pos_args[1].out[1] = ac_to_float(&ctx->ac, edgeflag_value);
2919 }
2920
2921 if (ctx->screen->info.chip_class >= GFX9) {
2922 /* GFX9 has the layer in out.z[10:0] and the viewport
2923 * index in out.z[19:16].
2924 */
2925 if (shader->selector->info.writes_layer)
2926 pos_args[1].out[2] = layer_value;
2927
2928 if (shader->selector->info.writes_viewport_index) {
2929 LLVMValueRef v = viewport_index_value;
2930
2931 v = ac_to_integer(&ctx->ac, v);
2932 v = LLVMBuildShl(ctx->ac.builder, v,
2933 LLVMConstInt(ctx->i32, 16, 0), "");
2934 v = LLVMBuildOr(ctx->ac.builder, v,
2935 ac_to_integer(&ctx->ac, pos_args[1].out[2]), "");
2936 pos_args[1].out[2] = ac_to_float(&ctx->ac, v);
2937 pos_args[1].enabled_channels |= 1 << 2;
2938 }
2939 } else {
2940 if (shader->selector->info.writes_layer)
2941 pos_args[1].out[2] = layer_value;
2942
2943 if (shader->selector->info.writes_viewport_index) {
2944 pos_args[1].out[3] = viewport_index_value;
2945 pos_args[1].enabled_channels |= 1 << 3;
2946 }
2947 }
2948 }
2949
2950 for (i = 0; i < 4; i++)
2951 if (pos_args[i].out[0])
2952 shader->info.nr_pos_exports++;
2953
2954 pos_idx = 0;
2955 for (i = 0; i < 4; i++) {
2956 if (!pos_args[i].out[0])
2957 continue;
2958
2959 /* Specify the target we are exporting */
2960 pos_args[i].target = V_008DFC_SQ_EXP_POS + pos_idx++;
2961
2962 if (pos_idx == shader->info.nr_pos_exports)
2963 /* Specify that this is the last export */
2964 pos_args[i].done = 1;
2965
2966 ac_build_export(&ctx->ac, &pos_args[i]);
2967 }
2968
2969 /* Build parameter exports. */
2970 si_build_param_exports(ctx, outputs, noutput);
2971 }
2972
2973 /**
2974 * Forward all outputs from the vertex shader to the TES. This is only used
2975 * for the fixed function TCS.
2976 */
2977 static void si_copy_tcs_inputs(struct lp_build_tgsi_context *bld_base)
2978 {
2979 struct si_shader_context *ctx = si_shader_context(bld_base);
2980 LLVMValueRef invocation_id, buffer, buffer_offset;
2981 LLVMValueRef lds_vertex_stride, lds_vertex_offset, lds_base;
2982 uint64_t inputs;
2983
2984 invocation_id = unpack_llvm_param(ctx, ctx->abi.tcs_rel_ids, 8, 5);
2985 buffer = desc_from_addr_base64k(ctx, ctx->param_tcs_offchip_addr_base64k);
2986 buffer_offset = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
2987
2988 lds_vertex_stride = get_tcs_in_vertex_dw_stride(ctx);
2989 lds_vertex_offset = LLVMBuildMul(ctx->ac.builder, invocation_id,
2990 lds_vertex_stride, "");
2991 lds_base = get_tcs_in_current_patch_offset(ctx);
2992 lds_base = LLVMBuildAdd(ctx->ac.builder, lds_base, lds_vertex_offset, "");
2993
2994 inputs = ctx->shader->key.mono.u.ff_tcs_inputs_to_copy;
2995 while (inputs) {
2996 unsigned i = u_bit_scan64(&inputs);
2997
2998 LLVMValueRef lds_ptr = LLVMBuildAdd(ctx->ac.builder, lds_base,
2999 LLVMConstInt(ctx->i32, 4 * i, 0),
3000 "");
3001
3002 LLVMValueRef buffer_addr = get_tcs_tes_buffer_address(ctx,
3003 get_rel_patch_id(ctx),
3004 invocation_id,
3005 LLVMConstInt(ctx->i32, i, 0));
3006
3007 LLVMValueRef value = lds_load(bld_base, ctx->ac.i32, ~0,
3008 lds_ptr);
3009
3010 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, buffer_addr,
3011 buffer_offset, 0, 1, 0, true, false);
3012 }
3013 }
3014
3015 static void si_write_tess_factors(struct lp_build_tgsi_context *bld_base,
3016 LLVMValueRef rel_patch_id,
3017 LLVMValueRef invocation_id,
3018 LLVMValueRef tcs_out_current_patch_data_offset,
3019 LLVMValueRef invoc0_tf_outer[4],
3020 LLVMValueRef invoc0_tf_inner[2])
3021 {
3022 struct si_shader_context *ctx = si_shader_context(bld_base);
3023 struct si_shader *shader = ctx->shader;
3024 unsigned tess_inner_index, tess_outer_index;
3025 LLVMValueRef lds_base, lds_inner, lds_outer, byteoffset, buffer;
3026 LLVMValueRef out[6], vec0, vec1, tf_base, inner[4], outer[4];
3027 unsigned stride, outer_comps, inner_comps, i, offset;
3028 struct lp_build_if_state if_ctx, inner_if_ctx;
3029
3030 /* Add a barrier before loading tess factors from LDS. */
3031 if (!shader->key.part.tcs.epilog.invoc0_tess_factors_are_def)
3032 si_llvm_emit_barrier(NULL, bld_base, NULL);
3033
3034 /* Do this only for invocation 0, because the tess levels are per-patch,
3035 * not per-vertex.
3036 *
3037 * This can't jump, because invocation 0 executes this. It should
3038 * at least mask out the loads and stores for other invocations.
3039 */
3040 lp_build_if(&if_ctx, &ctx->gallivm,
3041 LLVMBuildICmp(ctx->ac.builder, LLVMIntEQ,
3042 invocation_id, ctx->i32_0, ""));
3043
3044 /* Determine the layout of one tess factor element in the buffer. */
3045 switch (shader->key.part.tcs.epilog.prim_mode) {
3046 case PIPE_PRIM_LINES:
3047 stride = 2; /* 2 dwords, 1 vec2 store */
3048 outer_comps = 2;
3049 inner_comps = 0;
3050 break;
3051 case PIPE_PRIM_TRIANGLES:
3052 stride = 4; /* 4 dwords, 1 vec4 store */
3053 outer_comps = 3;
3054 inner_comps = 1;
3055 break;
3056 case PIPE_PRIM_QUADS:
3057 stride = 6; /* 6 dwords, 2 stores (vec4 + vec2) */
3058 outer_comps = 4;
3059 inner_comps = 2;
3060 break;
3061 default:
3062 assert(0);
3063 return;
3064 }
3065
3066 for (i = 0; i < 4; i++) {
3067 inner[i] = LLVMGetUndef(ctx->i32);
3068 outer[i] = LLVMGetUndef(ctx->i32);
3069 }
3070
3071 if (shader->key.part.tcs.epilog.invoc0_tess_factors_are_def) {
3072 /* Tess factors are in VGPRs. */
3073 for (i = 0; i < outer_comps; i++)
3074 outer[i] = out[i] = invoc0_tf_outer[i];
3075 for (i = 0; i < inner_comps; i++)
3076 inner[i] = out[outer_comps+i] = invoc0_tf_inner[i];
3077 } else {
3078 /* Load tess_inner and tess_outer from LDS.
3079 * Any invocation can write them, so we can't get them from a temporary.
3080 */
3081 tess_inner_index = si_shader_io_get_unique_index_patch(TGSI_SEMANTIC_TESSINNER, 0);
3082 tess_outer_index = si_shader_io_get_unique_index_patch(TGSI_SEMANTIC_TESSOUTER, 0);
3083
3084 lds_base = tcs_out_current_patch_data_offset;
3085 lds_inner = LLVMBuildAdd(ctx->ac.builder, lds_base,
3086 LLVMConstInt(ctx->i32,
3087 tess_inner_index * 4, 0), "");
3088 lds_outer = LLVMBuildAdd(ctx->ac.builder, lds_base,
3089 LLVMConstInt(ctx->i32,
3090 tess_outer_index * 4, 0), "");
3091
3092 for (i = 0; i < outer_comps; i++) {
3093 outer[i] = out[i] =
3094 lds_load(bld_base, ctx->ac.i32, i, lds_outer);
3095 }
3096 for (i = 0; i < inner_comps; i++) {
3097 inner[i] = out[outer_comps+i] =
3098 lds_load(bld_base, ctx->ac.i32, i, lds_inner);
3099 }
3100 }
3101
3102 if (shader->key.part.tcs.epilog.prim_mode == PIPE_PRIM_LINES) {
3103 /* For isolines, the hardware expects tess factors in the
3104 * reverse order from what GLSL / TGSI specify.
3105 */
3106 LLVMValueRef tmp = out[0];
3107 out[0] = out[1];
3108 out[1] = tmp;
3109 }
3110
3111 /* Convert the outputs to vectors for stores. */
3112 vec0 = lp_build_gather_values(&ctx->gallivm, out, MIN2(stride, 4));
3113 vec1 = NULL;
3114
3115 if (stride > 4)
3116 vec1 = lp_build_gather_values(&ctx->gallivm, out+4, stride - 4);
3117
3118 /* Get the buffer. */
3119 buffer = desc_from_addr_base64k(ctx, ctx->param_tcs_factor_addr_base64k);
3120
3121 /* Get the offset. */
3122 tf_base = LLVMGetParam(ctx->main_fn,
3123 ctx->param_tcs_factor_offset);
3124 byteoffset = LLVMBuildMul(ctx->ac.builder, rel_patch_id,
3125 LLVMConstInt(ctx->i32, 4 * stride, 0), "");
3126
3127 lp_build_if(&inner_if_ctx, &ctx->gallivm,
3128 LLVMBuildICmp(ctx->ac.builder, LLVMIntEQ,
3129 rel_patch_id, ctx->i32_0, ""));
3130
3131 /* Store the dynamic HS control word. */
3132 offset = 0;
3133 if (ctx->screen->info.chip_class <= VI) {
3134 ac_build_buffer_store_dword(&ctx->ac, buffer,
3135 LLVMConstInt(ctx->i32, 0x80000000, 0),
3136 1, ctx->i32_0, tf_base,
3137 offset, 1, 0, true, false);
3138 offset += 4;
3139 }
3140
3141 lp_build_endif(&inner_if_ctx);
3142
3143 /* Store the tessellation factors. */
3144 ac_build_buffer_store_dword(&ctx->ac, buffer, vec0,
3145 MIN2(stride, 4), byteoffset, tf_base,
3146 offset, 1, 0, true, false);
3147 offset += 16;
3148 if (vec1)
3149 ac_build_buffer_store_dword(&ctx->ac, buffer, vec1,
3150 stride - 4, byteoffset, tf_base,
3151 offset, 1, 0, true, false);
3152
3153 /* Store the tess factors into the offchip buffer if TES reads them. */
3154 if (shader->key.part.tcs.epilog.tes_reads_tess_factors) {
3155 LLVMValueRef buf, base, inner_vec, outer_vec, tf_outer_offset;
3156 LLVMValueRef tf_inner_offset;
3157 unsigned param_outer, param_inner;
3158
3159 buf = desc_from_addr_base64k(ctx, ctx->param_tcs_offchip_addr_base64k);
3160 base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
3161
3162 param_outer = si_shader_io_get_unique_index_patch(
3163 TGSI_SEMANTIC_TESSOUTER, 0);
3164 tf_outer_offset = get_tcs_tes_buffer_address(ctx, rel_patch_id, NULL,
3165 LLVMConstInt(ctx->i32, param_outer, 0));
3166
3167 outer_vec = lp_build_gather_values(&ctx->gallivm, outer,
3168 util_next_power_of_two(outer_comps));
3169
3170 ac_build_buffer_store_dword(&ctx->ac, buf, outer_vec,
3171 outer_comps, tf_outer_offset,
3172 base, 0, 1, 0, true, false);
3173 if (inner_comps) {
3174 param_inner = si_shader_io_get_unique_index_patch(
3175 TGSI_SEMANTIC_TESSINNER, 0);
3176 tf_inner_offset = get_tcs_tes_buffer_address(ctx, rel_patch_id, NULL,
3177 LLVMConstInt(ctx->i32, param_inner, 0));
3178
3179 inner_vec = inner_comps == 1 ? inner[0] :
3180 lp_build_gather_values(&ctx->gallivm, inner, inner_comps);
3181 ac_build_buffer_store_dword(&ctx->ac, buf, inner_vec,
3182 inner_comps, tf_inner_offset,
3183 base, 0, 1, 0, true, false);
3184 }
3185 }
3186
3187 lp_build_endif(&if_ctx);
3188 }
3189
3190 static LLVMValueRef
3191 si_insert_input_ret(struct si_shader_context *ctx, LLVMValueRef ret,
3192 unsigned param, unsigned return_index)
3193 {
3194 return LLVMBuildInsertValue(ctx->ac.builder, ret,
3195 LLVMGetParam(ctx->main_fn, param),
3196 return_index, "");
3197 }
3198
3199 static LLVMValueRef
3200 si_insert_input_ret_float(struct si_shader_context *ctx, LLVMValueRef ret,
3201 unsigned param, unsigned return_index)
3202 {
3203 LLVMBuilderRef builder = ctx->ac.builder;
3204 LLVMValueRef p = LLVMGetParam(ctx->main_fn, param);
3205
3206 return LLVMBuildInsertValue(builder, ret,
3207 ac_to_float(&ctx->ac, p),
3208 return_index, "");
3209 }
3210
3211 static LLVMValueRef
3212 si_insert_input_ptr_as_2xi32(struct si_shader_context *ctx, LLVMValueRef ret,
3213 unsigned param, unsigned return_index)
3214 {
3215 LLVMBuilderRef builder = ctx->ac.builder;
3216 LLVMValueRef ptr, lo, hi;
3217
3218 ptr = LLVMGetParam(ctx->main_fn, param);
3219 ptr = LLVMBuildPtrToInt(builder, ptr, ctx->i64, "");
3220 ptr = LLVMBuildBitCast(builder, ptr, ctx->v2i32, "");
3221 lo = LLVMBuildExtractElement(builder, ptr, ctx->i32_0, "");
3222 hi = LLVMBuildExtractElement(builder, ptr, ctx->i32_1, "");
3223 ret = LLVMBuildInsertValue(builder, ret, lo, return_index, "");
3224 return LLVMBuildInsertValue(builder, ret, hi, return_index + 1, "");
3225 }
3226
3227 /* This only writes the tessellation factor levels. */
3228 static void si_llvm_emit_tcs_epilogue(struct ac_shader_abi *abi,
3229 unsigned max_outputs,
3230 LLVMValueRef *addrs)
3231 {
3232 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
3233 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
3234 LLVMBuilderRef builder = ctx->ac.builder;
3235 LLVMValueRef rel_patch_id, invocation_id, tf_lds_offset;
3236
3237 si_copy_tcs_inputs(bld_base);
3238
3239 rel_patch_id = get_rel_patch_id(ctx);
3240 invocation_id = unpack_llvm_param(ctx, ctx->abi.tcs_rel_ids, 8, 5);
3241 tf_lds_offset = get_tcs_out_current_patch_data_offset(ctx);
3242
3243 if (ctx->screen->info.chip_class >= GFX9) {
3244 LLVMBasicBlockRef blocks[2] = {
3245 LLVMGetInsertBlock(builder),
3246 ctx->merged_wrap_if_state.entry_block
3247 };
3248 LLVMValueRef values[2];
3249
3250 lp_build_endif(&ctx->merged_wrap_if_state);
3251
3252 values[0] = rel_patch_id;
3253 values[1] = LLVMGetUndef(ctx->i32);
3254 rel_patch_id = ac_build_phi(&ctx->ac, ctx->i32, 2, values, blocks);
3255
3256 values[0] = tf_lds_offset;
3257 values[1] = LLVMGetUndef(ctx->i32);
3258 tf_lds_offset = ac_build_phi(&ctx->ac, ctx->i32, 2, values, blocks);
3259
3260 values[0] = invocation_id;
3261 values[1] = ctx->i32_1; /* cause the epilog to skip threads */
3262 invocation_id = ac_build_phi(&ctx->ac, ctx->i32, 2, values, blocks);
3263 }
3264
3265 /* Return epilog parameters from this function. */
3266 LLVMValueRef ret = ctx->return_value;
3267 unsigned vgpr;
3268
3269 if (ctx->screen->info.chip_class >= GFX9) {
3270 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_layout,
3271 8 + GFX9_SGPR_TCS_OFFCHIP_LAYOUT);
3272 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_addr_base64k,
3273 8 + GFX9_SGPR_TCS_OFFCHIP_ADDR_BASE64K);
3274 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_addr_base64k,
3275 8 + GFX9_SGPR_TCS_FACTOR_ADDR_BASE64K);
3276 /* Tess offchip and tess factor offsets are at the beginning. */
3277 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_offset, 2);
3278 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_offset, 4);
3279 vgpr = 8 + GFX9_SGPR_TCS_FACTOR_ADDR_BASE64K + 1;
3280 } else {
3281 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_layout,
3282 GFX6_SGPR_TCS_OFFCHIP_LAYOUT);
3283 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_addr_base64k,
3284 GFX6_SGPR_TCS_OFFCHIP_ADDR_BASE64K);
3285 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_addr_base64k,
3286 GFX6_SGPR_TCS_FACTOR_ADDR_BASE64K);
3287 /* Tess offchip and tess factor offsets are after user SGPRs. */
3288 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_offset,
3289 GFX6_TCS_NUM_USER_SGPR);
3290 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_offset,
3291 GFX6_TCS_NUM_USER_SGPR + 1);
3292 vgpr = GFX6_TCS_NUM_USER_SGPR + 2;
3293 }
3294
3295 /* VGPRs */
3296 rel_patch_id = ac_to_float(&ctx->ac, rel_patch_id);
3297 invocation_id = ac_to_float(&ctx->ac, invocation_id);
3298 tf_lds_offset = ac_to_float(&ctx->ac, tf_lds_offset);
3299
3300 /* Leave a hole corresponding to the two input VGPRs. This ensures that
3301 * the invocation_id output does not alias the tcs_rel_ids input,
3302 * which saves a V_MOV on gfx9.
3303 */
3304 vgpr += 2;
3305
3306 ret = LLVMBuildInsertValue(builder, ret, rel_patch_id, vgpr++, "");
3307 ret = LLVMBuildInsertValue(builder, ret, invocation_id, vgpr++, "");
3308
3309 if (ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs) {
3310 vgpr++; /* skip the tess factor LDS offset */
3311 for (unsigned i = 0; i < 6; i++) {
3312 LLVMValueRef value =
3313 LLVMBuildLoad(builder, ctx->invoc0_tess_factors[i], "");
3314 value = ac_to_float(&ctx->ac, value);
3315 ret = LLVMBuildInsertValue(builder, ret, value, vgpr++, "");
3316 }
3317 } else {
3318 ret = LLVMBuildInsertValue(builder, ret, tf_lds_offset, vgpr++, "");
3319 }
3320 ctx->return_value = ret;
3321 }
3322
3323 /* Pass TCS inputs from LS to TCS on GFX9. */
3324 static void si_set_ls_return_value_for_tcs(struct si_shader_context *ctx)
3325 {
3326 LLVMValueRef ret = ctx->return_value;
3327
3328 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_offset, 2);
3329 ret = si_insert_input_ret(ctx, ret, ctx->param_merged_wave_info, 3);
3330 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_offset, 4);
3331 ret = si_insert_input_ret(ctx, ret, ctx->param_merged_scratch_offset, 5);
3332
3333 ret = si_insert_input_ptr_as_2xi32(ctx, ret, ctx->param_rw_buffers,
3334 8 + SI_SGPR_RW_BUFFERS);
3335 ret = si_insert_input_ptr_as_2xi32(ctx, ret,
3336 ctx->param_bindless_samplers_and_images,
3337 8 + SI_SGPR_BINDLESS_SAMPLERS_AND_IMAGES);
3338
3339 ret = si_insert_input_ret(ctx, ret, ctx->param_vs_state_bits,
3340 8 + SI_SGPR_VS_STATE_BITS);
3341 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_layout,
3342 8 + GFX9_SGPR_TCS_OFFCHIP_LAYOUT);
3343 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_out_lds_offsets,
3344 8 + GFX9_SGPR_TCS_OUT_OFFSETS);
3345 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_out_lds_layout,
3346 8 + GFX9_SGPR_TCS_OUT_LAYOUT);
3347 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_addr_base64k,
3348 8 + GFX9_SGPR_TCS_OFFCHIP_ADDR_BASE64K);
3349 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_addr_base64k,
3350 8 + GFX9_SGPR_TCS_FACTOR_ADDR_BASE64K);
3351
3352 unsigned desc_param = ctx->param_tcs_factor_addr_base64k + 2;
3353 ret = si_insert_input_ptr_as_2xi32(ctx, ret, desc_param,
3354 8 + GFX9_SGPR_TCS_CONST_AND_SHADER_BUFFERS);
3355 ret = si_insert_input_ptr_as_2xi32(ctx, ret, desc_param + 1,
3356 8 + GFX9_SGPR_TCS_SAMPLERS_AND_IMAGES);
3357
3358 unsigned vgpr = 8 + GFX9_TCS_NUM_USER_SGPR;
3359 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
3360 ac_to_float(&ctx->ac, ctx->abi.tcs_patch_id),
3361 vgpr++, "");
3362 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
3363 ac_to_float(&ctx->ac, ctx->abi.tcs_rel_ids),
3364 vgpr++, "");
3365 ctx->return_value = ret;
3366 }
3367
3368 /* Pass GS inputs from ES to GS on GFX9. */
3369 static void si_set_es_return_value_for_gs(struct si_shader_context *ctx)
3370 {
3371 LLVMValueRef ret = ctx->return_value;
3372
3373 ret = si_insert_input_ret(ctx, ret, ctx->param_gs2vs_offset, 2);
3374 ret = si_insert_input_ret(ctx, ret, ctx->param_merged_wave_info, 3);
3375 ret = si_insert_input_ret(ctx, ret, ctx->param_merged_scratch_offset, 5);
3376
3377 ret = si_insert_input_ptr_as_2xi32(ctx, ret, ctx->param_rw_buffers,
3378 8 + SI_SGPR_RW_BUFFERS);
3379 ret = si_insert_input_ptr_as_2xi32(ctx, ret,
3380 ctx->param_bindless_samplers_and_images,
3381 8 + SI_SGPR_BINDLESS_SAMPLERS_AND_IMAGES);
3382
3383 unsigned desc_param = ctx->param_vs_state_bits + 1;
3384 ret = si_insert_input_ptr_as_2xi32(ctx, ret, desc_param,
3385 8 + GFX9_SGPR_GS_CONST_AND_SHADER_BUFFERS);
3386 ret = si_insert_input_ptr_as_2xi32(ctx, ret, desc_param + 1,
3387 8 + GFX9_SGPR_GS_SAMPLERS_AND_IMAGES);
3388
3389 unsigned vgpr = 8 + GFX9_GS_NUM_USER_SGPR;
3390 for (unsigned i = 0; i < 5; i++) {
3391 unsigned param = ctx->param_gs_vtx01_offset + i;
3392 ret = si_insert_input_ret_float(ctx, ret, param, vgpr++);
3393 }
3394 ctx->return_value = ret;
3395 }
3396
3397 static void si_llvm_emit_ls_epilogue(struct ac_shader_abi *abi,
3398 unsigned max_outputs,
3399 LLVMValueRef *addrs)
3400 {
3401 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
3402 struct si_shader *shader = ctx->shader;
3403 struct tgsi_shader_info *info = &shader->selector->info;
3404 unsigned i, chan;
3405 LLVMValueRef vertex_id = LLVMGetParam(ctx->main_fn,
3406 ctx->param_rel_auto_id);
3407 LLVMValueRef vertex_dw_stride = get_tcs_in_vertex_dw_stride(ctx);
3408 LLVMValueRef base_dw_addr = LLVMBuildMul(ctx->ac.builder, vertex_id,
3409 vertex_dw_stride, "");
3410
3411 /* Write outputs to LDS. The next shader (TCS aka HS) will read
3412 * its inputs from it. */
3413 for (i = 0; i < info->num_outputs; i++) {
3414 unsigned name = info->output_semantic_name[i];
3415 unsigned index = info->output_semantic_index[i];
3416
3417 /* The ARB_shader_viewport_layer_array spec contains the
3418 * following issue:
3419 *
3420 * 2) What happens if gl_ViewportIndex or gl_Layer is
3421 * written in the vertex shader and a geometry shader is
3422 * present?
3423 *
3424 * RESOLVED: The value written by the last vertex processing
3425 * stage is used. If the last vertex processing stage
3426 * (vertex, tessellation evaluation or geometry) does not
3427 * statically assign to gl_ViewportIndex or gl_Layer, index
3428 * or layer zero is assumed.
3429 *
3430 * So writes to those outputs in VS-as-LS are simply ignored.
3431 */
3432 if (name == TGSI_SEMANTIC_LAYER ||
3433 name == TGSI_SEMANTIC_VIEWPORT_INDEX)
3434 continue;
3435
3436 int param = si_shader_io_get_unique_index(name, index);
3437 LLVMValueRef dw_addr = LLVMBuildAdd(ctx->ac.builder, base_dw_addr,
3438 LLVMConstInt(ctx->i32, param * 4, 0), "");
3439
3440 for (chan = 0; chan < 4; chan++) {
3441 if (!(info->output_usagemask[i] & (1 << chan)))
3442 continue;
3443
3444 lds_store(ctx, chan, dw_addr,
3445 LLVMBuildLoad(ctx->ac.builder, addrs[4 * i + chan], ""));
3446 }
3447 }
3448
3449 if (ctx->screen->info.chip_class >= GFX9)
3450 si_set_ls_return_value_for_tcs(ctx);
3451 }
3452
3453 static void si_llvm_emit_es_epilogue(struct ac_shader_abi *abi,
3454 unsigned max_outputs,
3455 LLVMValueRef *addrs)
3456 {
3457 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
3458 struct si_shader *es = ctx->shader;
3459 struct tgsi_shader_info *info = &es->selector->info;
3460 LLVMValueRef soffset = LLVMGetParam(ctx->main_fn,
3461 ctx->param_es2gs_offset);
3462 LLVMValueRef lds_base = NULL;
3463 unsigned chan;
3464 int i;
3465
3466 if (ctx->screen->info.chip_class >= GFX9 && info->num_outputs) {
3467 unsigned itemsize_dw = es->selector->esgs_itemsize / 4;
3468 LLVMValueRef vertex_idx = ac_get_thread_id(&ctx->ac);
3469 LLVMValueRef wave_idx = unpack_param(ctx, ctx->param_merged_wave_info, 24, 4);
3470 vertex_idx = LLVMBuildOr(ctx->ac.builder, vertex_idx,
3471 LLVMBuildMul(ctx->ac.builder, wave_idx,
3472 LLVMConstInt(ctx->i32, 64, false), ""), "");
3473 lds_base = LLVMBuildMul(ctx->ac.builder, vertex_idx,
3474 LLVMConstInt(ctx->i32, itemsize_dw, 0), "");
3475 }
3476
3477 for (i = 0; i < info->num_outputs; i++) {
3478 int param;
3479
3480 if (info->output_semantic_name[i] == TGSI_SEMANTIC_VIEWPORT_INDEX ||
3481 info->output_semantic_name[i] == TGSI_SEMANTIC_LAYER)
3482 continue;
3483
3484 param = si_shader_io_get_unique_index(info->output_semantic_name[i],
3485 info->output_semantic_index[i]);
3486
3487 for (chan = 0; chan < 4; chan++) {
3488 LLVMValueRef out_val = LLVMBuildLoad(ctx->ac.builder, addrs[4 * i + chan], "");
3489 out_val = ac_to_integer(&ctx->ac, out_val);
3490
3491 /* GFX9 has the ESGS ring in LDS. */
3492 if (ctx->screen->info.chip_class >= GFX9) {
3493 lds_store(ctx, param * 4 + chan, lds_base, out_val);
3494 continue;
3495 }
3496
3497 ac_build_buffer_store_dword(&ctx->ac,
3498 ctx->esgs_ring,
3499 out_val, 1, NULL, soffset,
3500 (4 * param + chan) * 4,
3501 1, 1, true, true);
3502 }
3503 }
3504
3505 if (ctx->screen->info.chip_class >= GFX9)
3506 si_set_es_return_value_for_gs(ctx);
3507 }
3508
3509 static LLVMValueRef si_get_gs_wave_id(struct si_shader_context *ctx)
3510 {
3511 if (ctx->screen->info.chip_class >= GFX9)
3512 return unpack_param(ctx, ctx->param_merged_wave_info, 16, 8);
3513 else
3514 return LLVMGetParam(ctx->main_fn, ctx->param_gs_wave_id);
3515 }
3516
3517 static void emit_gs_epilogue(struct si_shader_context *ctx)
3518 {
3519 ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_NOP | AC_SENDMSG_GS_DONE,
3520 si_get_gs_wave_id(ctx));
3521
3522 if (ctx->screen->info.chip_class >= GFX9)
3523 lp_build_endif(&ctx->merged_wrap_if_state);
3524 }
3525
3526 static void si_llvm_emit_gs_epilogue(struct ac_shader_abi *abi,
3527 unsigned max_outputs,
3528 LLVMValueRef *addrs)
3529 {
3530 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
3531 struct tgsi_shader_info UNUSED *info = &ctx->shader->selector->info;
3532
3533 assert(info->num_outputs <= max_outputs);
3534
3535 emit_gs_epilogue(ctx);
3536 }
3537
3538 static void si_tgsi_emit_gs_epilogue(struct lp_build_tgsi_context *bld_base)
3539 {
3540 struct si_shader_context *ctx = si_shader_context(bld_base);
3541 emit_gs_epilogue(ctx);
3542 }
3543
3544 static void si_llvm_emit_vs_epilogue(struct ac_shader_abi *abi,
3545 unsigned max_outputs,
3546 LLVMValueRef *addrs)
3547 {
3548 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
3549 struct tgsi_shader_info *info = &ctx->shader->selector->info;
3550 struct si_shader_output_values *outputs = NULL;
3551 int i,j;
3552
3553 assert(!ctx->shader->is_gs_copy_shader);
3554 assert(info->num_outputs <= max_outputs);
3555
3556 outputs = MALLOC((info->num_outputs + 1) * sizeof(outputs[0]));
3557
3558 /* Vertex color clamping.
3559 *
3560 * This uses a state constant loaded in a user data SGPR and
3561 * an IF statement is added that clamps all colors if the constant
3562 * is true.
3563 */
3564 if (ctx->type == PIPE_SHADER_VERTEX) {
3565 struct lp_build_if_state if_ctx;
3566 LLVMValueRef cond = NULL;
3567 LLVMValueRef addr, val;
3568
3569 for (i = 0; i < info->num_outputs; i++) {
3570 if (info->output_semantic_name[i] != TGSI_SEMANTIC_COLOR &&
3571 info->output_semantic_name[i] != TGSI_SEMANTIC_BCOLOR)
3572 continue;
3573
3574 /* We've found a color. */
3575 if (!cond) {
3576 /* The state is in the first bit of the user SGPR. */
3577 cond = LLVMGetParam(ctx->main_fn,
3578 ctx->param_vs_state_bits);
3579 cond = LLVMBuildTrunc(ctx->ac.builder, cond,
3580 ctx->i1, "");
3581 lp_build_if(&if_ctx, &ctx->gallivm, cond);
3582 }
3583
3584 for (j = 0; j < 4; j++) {
3585 addr = addrs[4 * i + j];
3586 val = LLVMBuildLoad(ctx->ac.builder, addr, "");
3587 val = ac_build_clamp(&ctx->ac, val);
3588 LLVMBuildStore(ctx->ac.builder, val, addr);
3589 }
3590 }
3591
3592 if (cond)
3593 lp_build_endif(&if_ctx);
3594 }
3595
3596 for (i = 0; i < info->num_outputs; i++) {
3597 outputs[i].semantic_name = info->output_semantic_name[i];
3598 outputs[i].semantic_index = info->output_semantic_index[i];
3599
3600 for (j = 0; j < 4; j++) {
3601 outputs[i].values[j] =
3602 LLVMBuildLoad(ctx->ac.builder,
3603 addrs[4 * i + j],
3604 "");
3605 outputs[i].vertex_stream[j] =
3606 (info->output_streams[i] >> (2 * j)) & 3;
3607 }
3608 }
3609
3610 if (ctx->shader->selector->so.num_outputs)
3611 si_llvm_emit_streamout(ctx, outputs, i, 0);
3612
3613 /* Export PrimitiveID. */
3614 if (ctx->shader->key.mono.u.vs_export_prim_id) {
3615 outputs[i].semantic_name = TGSI_SEMANTIC_PRIMID;
3616 outputs[i].semantic_index = 0;
3617 outputs[i].values[0] = ac_to_float(&ctx->ac, get_primitive_id(ctx, 0));
3618 for (j = 1; j < 4; j++)
3619 outputs[i].values[j] = LLVMConstReal(ctx->f32, 0);
3620
3621 memset(outputs[i].vertex_stream, 0,
3622 sizeof(outputs[i].vertex_stream));
3623 i++;
3624 }
3625
3626 si_llvm_export_vs(ctx, outputs, i);
3627 FREE(outputs);
3628 }
3629
3630 static void si_tgsi_emit_epilogue(struct lp_build_tgsi_context *bld_base)
3631 {
3632 struct si_shader_context *ctx = si_shader_context(bld_base);
3633
3634 ctx->abi.emit_outputs(&ctx->abi, RADEON_LLVM_MAX_OUTPUTS,
3635 &ctx->outputs[0][0]);
3636 }
3637
3638 struct si_ps_exports {
3639 unsigned num;
3640 struct ac_export_args args[10];
3641 };
3642
3643 static void si_export_mrt_z(struct lp_build_tgsi_context *bld_base,
3644 LLVMValueRef depth, LLVMValueRef stencil,
3645 LLVMValueRef samplemask, struct si_ps_exports *exp)
3646 {
3647 struct si_shader_context *ctx = si_shader_context(bld_base);
3648 struct ac_export_args args;
3649
3650 ac_export_mrt_z(&ctx->ac, depth, stencil, samplemask, &args);
3651
3652 memcpy(&exp->args[exp->num++], &args, sizeof(args));
3653 }
3654
3655 static void si_export_mrt_color(struct lp_build_tgsi_context *bld_base,
3656 LLVMValueRef *color, unsigned index,
3657 unsigned samplemask_param,
3658 bool is_last, struct si_ps_exports *exp)
3659 {
3660 struct si_shader_context *ctx = si_shader_context(bld_base);
3661 int i;
3662
3663 /* Clamp color */
3664 if (ctx->shader->key.part.ps.epilog.clamp_color)
3665 for (i = 0; i < 4; i++)
3666 color[i] = ac_build_clamp(&ctx->ac, color[i]);
3667
3668 /* Alpha to one */
3669 if (ctx->shader->key.part.ps.epilog.alpha_to_one)
3670 color[3] = ctx->ac.f32_1;
3671
3672 /* Alpha test */
3673 if (index == 0 &&
3674 ctx->shader->key.part.ps.epilog.alpha_func != PIPE_FUNC_ALWAYS)
3675 si_alpha_test(bld_base, color[3]);
3676
3677 /* Line & polygon smoothing */
3678 if (ctx->shader->key.part.ps.epilog.poly_line_smoothing)
3679 color[3] = si_scale_alpha_by_sample_mask(bld_base, color[3],
3680 samplemask_param);
3681
3682 /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
3683 if (ctx->shader->key.part.ps.epilog.last_cbuf > 0) {
3684 struct ac_export_args args[8];
3685 int c, last = -1;
3686
3687 /* Get the export arguments, also find out what the last one is. */
3688 for (c = 0; c <= ctx->shader->key.part.ps.epilog.last_cbuf; c++) {
3689 si_llvm_init_export_args(ctx, color,
3690 V_008DFC_SQ_EXP_MRT + c, &args[c]);
3691 if (args[c].enabled_channels)
3692 last = c;
3693 }
3694
3695 /* Emit all exports. */
3696 for (c = 0; c <= ctx->shader->key.part.ps.epilog.last_cbuf; c++) {
3697 if (is_last && last == c) {
3698 args[c].valid_mask = 1; /* whether the EXEC mask is valid */
3699 args[c].done = 1; /* DONE bit */
3700 } else if (!args[c].enabled_channels)
3701 continue; /* unnecessary NULL export */
3702
3703 memcpy(&exp->args[exp->num++], &args[c], sizeof(args[c]));
3704 }
3705 } else {
3706 struct ac_export_args args;
3707
3708 /* Export */
3709 si_llvm_init_export_args(ctx, color, V_008DFC_SQ_EXP_MRT + index,
3710 &args);
3711 if (is_last) {
3712 args.valid_mask = 1; /* whether the EXEC mask is valid */
3713 args.done = 1; /* DONE bit */
3714 } else if (!args.enabled_channels)
3715 return; /* unnecessary NULL export */
3716
3717 memcpy(&exp->args[exp->num++], &args, sizeof(args));
3718 }
3719 }
3720
3721 static void si_emit_ps_exports(struct si_shader_context *ctx,
3722 struct si_ps_exports *exp)
3723 {
3724 for (unsigned i = 0; i < exp->num; i++)
3725 ac_build_export(&ctx->ac, &exp->args[i]);
3726 }
3727
3728 static void si_export_null(struct lp_build_tgsi_context *bld_base)
3729 {
3730 struct si_shader_context *ctx = si_shader_context(bld_base);
3731 struct lp_build_context *base = &bld_base->base;
3732 struct ac_export_args args;
3733
3734 args.enabled_channels = 0x0; /* enabled channels */
3735 args.valid_mask = 1; /* whether the EXEC mask is valid */
3736 args.done = 1; /* DONE bit */
3737 args.target = V_008DFC_SQ_EXP_NULL;
3738 args.compr = 0; /* COMPR flag (0 = 32-bit export) */
3739 args.out[0] = base->undef; /* R */
3740 args.out[1] = base->undef; /* G */
3741 args.out[2] = base->undef; /* B */
3742 args.out[3] = base->undef; /* A */
3743
3744 ac_build_export(&ctx->ac, &args);
3745 }
3746
3747 /**
3748 * Return PS outputs in this order:
3749 *
3750 * v[0:3] = color0.xyzw
3751 * v[4:7] = color1.xyzw
3752 * ...
3753 * vN+0 = Depth
3754 * vN+1 = Stencil
3755 * vN+2 = SampleMask
3756 * vN+3 = SampleMaskIn (used for OpenGL smoothing)
3757 *
3758 * The alpha-ref SGPR is returned via its original location.
3759 */
3760 static void si_llvm_return_fs_outputs(struct ac_shader_abi *abi,
3761 unsigned max_outputs,
3762 LLVMValueRef *addrs)
3763 {
3764 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
3765 struct si_shader *shader = ctx->shader;
3766 struct tgsi_shader_info *info = &shader->selector->info;
3767 LLVMBuilderRef builder = ctx->ac.builder;
3768 unsigned i, j, first_vgpr, vgpr;
3769
3770 LLVMValueRef color[8][4] = {};
3771 LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
3772 LLVMValueRef ret;
3773
3774 if (ctx->postponed_kill)
3775 ac_build_kill_if_false(&ctx->ac, LLVMBuildLoad(builder, ctx->postponed_kill, ""));
3776
3777 /* Read the output values. */
3778 for (i = 0; i < info->num_outputs; i++) {
3779 unsigned semantic_name = info->output_semantic_name[i];
3780 unsigned semantic_index = info->output_semantic_index[i];
3781
3782 switch (semantic_name) {
3783 case TGSI_SEMANTIC_COLOR:
3784 assert(semantic_index < 8);
3785 for (j = 0; j < 4; j++) {
3786 LLVMValueRef ptr = addrs[4 * i + j];
3787 LLVMValueRef result = LLVMBuildLoad(builder, ptr, "");
3788 color[semantic_index][j] = result;
3789 }
3790 break;
3791 case TGSI_SEMANTIC_POSITION:
3792 depth = LLVMBuildLoad(builder,
3793 addrs[4 * i + 2], "");
3794 break;
3795 case TGSI_SEMANTIC_STENCIL:
3796 stencil = LLVMBuildLoad(builder,
3797 addrs[4 * i + 1], "");
3798 break;
3799 case TGSI_SEMANTIC_SAMPLEMASK:
3800 samplemask = LLVMBuildLoad(builder,
3801 addrs[4 * i + 0], "");
3802 break;
3803 default:
3804 fprintf(stderr, "Warning: SI unhandled fs output type:%d\n",
3805 semantic_name);
3806 }
3807 }
3808
3809 /* Fill the return structure. */
3810 ret = ctx->return_value;
3811
3812 /* Set SGPRs. */
3813 ret = LLVMBuildInsertValue(builder, ret,
3814 ac_to_integer(&ctx->ac,
3815 LLVMGetParam(ctx->main_fn,
3816 SI_PARAM_ALPHA_REF)),
3817 SI_SGPR_ALPHA_REF, "");
3818
3819 /* Set VGPRs */
3820 first_vgpr = vgpr = SI_SGPR_ALPHA_REF + 1;
3821 for (i = 0; i < ARRAY_SIZE(color); i++) {
3822 if (!color[i][0])
3823 continue;
3824
3825 for (j = 0; j < 4; j++)
3826 ret = LLVMBuildInsertValue(builder, ret, color[i][j], vgpr++, "");
3827 }
3828 if (depth)
3829 ret = LLVMBuildInsertValue(builder, ret, depth, vgpr++, "");
3830 if (stencil)
3831 ret = LLVMBuildInsertValue(builder, ret, stencil, vgpr++, "");
3832 if (samplemask)
3833 ret = LLVMBuildInsertValue(builder, ret, samplemask, vgpr++, "");
3834
3835 /* Add the input sample mask for smoothing at the end. */
3836 if (vgpr < first_vgpr + PS_EPILOG_SAMPLEMASK_MIN_LOC)
3837 vgpr = first_vgpr + PS_EPILOG_SAMPLEMASK_MIN_LOC;
3838 ret = LLVMBuildInsertValue(builder, ret,
3839 LLVMGetParam(ctx->main_fn,
3840 SI_PARAM_SAMPLE_COVERAGE), vgpr++, "");
3841
3842 ctx->return_value = ret;
3843 }
3844
3845 static void membar_emit(
3846 const struct lp_build_tgsi_action *action,
3847 struct lp_build_tgsi_context *bld_base,
3848 struct lp_build_emit_data *emit_data)
3849 {
3850 struct si_shader_context *ctx = si_shader_context(bld_base);
3851 LLVMValueRef src0 = lp_build_emit_fetch(bld_base, emit_data->inst, 0, 0);
3852 unsigned flags = LLVMConstIntGetZExtValue(src0);
3853 unsigned waitcnt = NOOP_WAITCNT;
3854
3855 if (flags & TGSI_MEMBAR_THREAD_GROUP)
3856 waitcnt &= VM_CNT & LGKM_CNT;
3857
3858 if (flags & (TGSI_MEMBAR_ATOMIC_BUFFER |
3859 TGSI_MEMBAR_SHADER_BUFFER |
3860 TGSI_MEMBAR_SHADER_IMAGE))
3861 waitcnt &= VM_CNT;
3862
3863 if (flags & TGSI_MEMBAR_SHARED)
3864 waitcnt &= LGKM_CNT;
3865
3866 if (waitcnt != NOOP_WAITCNT)
3867 ac_build_waitcnt(&ctx->ac, waitcnt);
3868 }
3869
3870 static void clock_emit(
3871 const struct lp_build_tgsi_action *action,
3872 struct lp_build_tgsi_context *bld_base,
3873 struct lp_build_emit_data *emit_data)
3874 {
3875 struct si_shader_context *ctx = si_shader_context(bld_base);
3876 LLVMValueRef tmp = ac_build_shader_clock(&ctx->ac);
3877
3878 emit_data->output[0] =
3879 LLVMBuildExtractElement(ctx->ac.builder, tmp, ctx->i32_0, "");
3880 emit_data->output[1] =
3881 LLVMBuildExtractElement(ctx->ac.builder, tmp, ctx->i32_1, "");
3882 }
3883
3884 static void si_llvm_emit_ddxy(
3885 const struct lp_build_tgsi_action *action,
3886 struct lp_build_tgsi_context *bld_base,
3887 struct lp_build_emit_data *emit_data)
3888 {
3889 struct si_shader_context *ctx = si_shader_context(bld_base);
3890 unsigned opcode = emit_data->info->opcode;
3891 LLVMValueRef val;
3892 int idx;
3893 unsigned mask;
3894
3895 if (opcode == TGSI_OPCODE_DDX_FINE)
3896 mask = AC_TID_MASK_LEFT;
3897 else if (opcode == TGSI_OPCODE_DDY_FINE)
3898 mask = AC_TID_MASK_TOP;
3899 else
3900 mask = AC_TID_MASK_TOP_LEFT;
3901
3902 /* for DDX we want to next X pixel, DDY next Y pixel. */
3903 idx = (opcode == TGSI_OPCODE_DDX || opcode == TGSI_OPCODE_DDX_FINE) ? 1 : 2;
3904
3905 val = ac_to_integer(&ctx->ac, emit_data->args[0]);
3906 val = ac_build_ddxy(&ctx->ac, mask, idx, val);
3907 emit_data->output[emit_data->chan] = val;
3908 }
3909
3910 /*
3911 * this takes an I,J coordinate pair,
3912 * and works out the X and Y derivatives.
3913 * it returns DDX(I), DDX(J), DDY(I), DDY(J).
3914 */
3915 static LLVMValueRef si_llvm_emit_ddxy_interp(
3916 struct lp_build_tgsi_context *bld_base,
3917 LLVMValueRef interp_ij)
3918 {
3919 struct si_shader_context *ctx = si_shader_context(bld_base);
3920 LLVMValueRef result[4], a;
3921 unsigned i;
3922
3923 for (i = 0; i < 2; i++) {
3924 a = LLVMBuildExtractElement(ctx->ac.builder, interp_ij,
3925 LLVMConstInt(ctx->i32, i, 0), "");
3926 result[i] = lp_build_emit_llvm_unary(bld_base, TGSI_OPCODE_DDX, a);
3927 result[2+i] = lp_build_emit_llvm_unary(bld_base, TGSI_OPCODE_DDY, a);
3928 }
3929
3930 return lp_build_gather_values(&ctx->gallivm, result, 4);
3931 }
3932
3933 static void interp_fetch_args(
3934 struct lp_build_tgsi_context *bld_base,
3935 struct lp_build_emit_data *emit_data)
3936 {
3937 struct si_shader_context *ctx = si_shader_context(bld_base);
3938 const struct tgsi_full_instruction *inst = emit_data->inst;
3939
3940 if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET) {
3941 /* offset is in second src, first two channels */
3942 emit_data->args[0] = lp_build_emit_fetch(bld_base,
3943 emit_data->inst, 1,
3944 TGSI_CHAN_X);
3945 emit_data->args[1] = lp_build_emit_fetch(bld_base,
3946 emit_data->inst, 1,
3947 TGSI_CHAN_Y);
3948 emit_data->arg_count = 2;
3949 } else if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE) {
3950 LLVMValueRef sample_position;
3951 LLVMValueRef sample_id;
3952 LLVMValueRef halfval = LLVMConstReal(ctx->f32, 0.5f);
3953
3954 /* fetch sample ID, then fetch its sample position,
3955 * and place into first two channels.
3956 */
3957 sample_id = lp_build_emit_fetch(bld_base,
3958 emit_data->inst, 1, TGSI_CHAN_X);
3959 sample_id = ac_to_integer(&ctx->ac, sample_id);
3960
3961 /* Section 8.13.2 (Interpolation Functions) of the OpenGL Shading
3962 * Language 4.50 spec says about interpolateAtSample:
3963 *
3964 * "Returns the value of the input interpolant variable at
3965 * the location of sample number sample. If multisample
3966 * buffers are not available, the input variable will be
3967 * evaluated at the center of the pixel. If sample sample
3968 * does not exist, the position used to interpolate the
3969 * input variable is undefined."
3970 *
3971 * This means that sample_id values outside of the valid are
3972 * in fact valid input, and the usual mechanism for loading the
3973 * sample position doesn't work.
3974 */
3975 if (ctx->shader->key.mono.u.ps.interpolate_at_sample_force_center) {
3976 LLVMValueRef center[4] = {
3977 LLVMConstReal(ctx->f32, 0.5),
3978 LLVMConstReal(ctx->f32, 0.5),
3979 ctx->ac.f32_0,
3980 ctx->ac.f32_0,
3981 };
3982
3983 sample_position = lp_build_gather_values(&ctx->gallivm, center, 4);
3984 } else {
3985 sample_position = load_sample_position(&ctx->abi, sample_id);
3986 }
3987
3988 emit_data->args[0] = LLVMBuildExtractElement(ctx->ac.builder,
3989 sample_position,
3990 ctx->i32_0, "");
3991
3992 emit_data->args[0] = LLVMBuildFSub(ctx->ac.builder, emit_data->args[0], halfval, "");
3993 emit_data->args[1] = LLVMBuildExtractElement(ctx->ac.builder,
3994 sample_position,
3995 ctx->i32_1, "");
3996 emit_data->args[1] = LLVMBuildFSub(ctx->ac.builder, emit_data->args[1], halfval, "");
3997 emit_data->arg_count = 2;
3998 }
3999 }
4000
4001 static void build_interp_intrinsic(const struct lp_build_tgsi_action *action,
4002 struct lp_build_tgsi_context *bld_base,
4003 struct lp_build_emit_data *emit_data)
4004 {
4005 struct si_shader_context *ctx = si_shader_context(bld_base);
4006 struct si_shader *shader = ctx->shader;
4007 const struct tgsi_shader_info *info = &shader->selector->info;
4008 LLVMValueRef interp_param;
4009 const struct tgsi_full_instruction *inst = emit_data->inst;
4010 const struct tgsi_full_src_register *input = &inst->Src[0];
4011 int input_base, input_array_size;
4012 int chan;
4013 int i;
4014 LLVMValueRef prim_mask = ctx->abi.prim_mask;
4015 LLVMValueRef array_idx;
4016 int interp_param_idx;
4017 unsigned interp;
4018 unsigned location;
4019
4020 assert(input->Register.File == TGSI_FILE_INPUT);
4021
4022 if (input->Register.Indirect) {
4023 unsigned array_id = input->Indirect.ArrayID;
4024
4025 if (array_id) {
4026 input_base = info->input_array_first[array_id];
4027 input_array_size = info->input_array_last[array_id] - input_base + 1;
4028 } else {
4029 input_base = inst->Src[0].Register.Index;
4030 input_array_size = info->num_inputs - input_base;
4031 }
4032
4033 array_idx = si_get_indirect_index(ctx, &input->Indirect,
4034 1, input->Register.Index - input_base);
4035 } else {
4036 input_base = inst->Src[0].Register.Index;
4037 input_array_size = 1;
4038 array_idx = ctx->i32_0;
4039 }
4040
4041 interp = shader->selector->info.input_interpolate[input_base];
4042
4043 if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET ||
4044 inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE)
4045 location = TGSI_INTERPOLATE_LOC_CENTER;
4046 else
4047 location = TGSI_INTERPOLATE_LOC_CENTROID;
4048
4049 interp_param_idx = lookup_interp_param_index(interp, location);
4050 if (interp_param_idx == -1)
4051 return;
4052 else if (interp_param_idx)
4053 interp_param = LLVMGetParam(ctx->main_fn, interp_param_idx);
4054 else
4055 interp_param = NULL;
4056
4057 if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET ||
4058 inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE) {
4059 LLVMValueRef ij_out[2];
4060 LLVMValueRef ddxy_out = si_llvm_emit_ddxy_interp(bld_base, interp_param);
4061
4062 /*
4063 * take the I then J parameters, and the DDX/Y for it, and
4064 * calculate the IJ inputs for the interpolator.
4065 * temp1 = ddx * offset/sample.x + I;
4066 * interp_param.I = ddy * offset/sample.y + temp1;
4067 * temp1 = ddx * offset/sample.x + J;
4068 * interp_param.J = ddy * offset/sample.y + temp1;
4069 */
4070 for (i = 0; i < 2; i++) {
4071 LLVMValueRef ix_ll = LLVMConstInt(ctx->i32, i, 0);
4072 LLVMValueRef iy_ll = LLVMConstInt(ctx->i32, i + 2, 0);
4073 LLVMValueRef ddx_el = LLVMBuildExtractElement(ctx->ac.builder,
4074 ddxy_out, ix_ll, "");
4075 LLVMValueRef ddy_el = LLVMBuildExtractElement(ctx->ac.builder,
4076 ddxy_out, iy_ll, "");
4077 LLVMValueRef interp_el = LLVMBuildExtractElement(ctx->ac.builder,
4078 interp_param, ix_ll, "");
4079 LLVMValueRef temp1, temp2;
4080
4081 interp_el = ac_to_float(&ctx->ac, interp_el);
4082
4083 temp1 = LLVMBuildFMul(ctx->ac.builder, ddx_el, emit_data->args[0], "");
4084
4085 temp1 = LLVMBuildFAdd(ctx->ac.builder, temp1, interp_el, "");
4086
4087 temp2 = LLVMBuildFMul(ctx->ac.builder, ddy_el, emit_data->args[1], "");
4088
4089 ij_out[i] = LLVMBuildFAdd(ctx->ac.builder, temp2, temp1, "");
4090 }
4091 interp_param = lp_build_gather_values(&ctx->gallivm, ij_out, 2);
4092 }
4093
4094 if (interp_param)
4095 interp_param = ac_to_float(&ctx->ac, interp_param);
4096
4097 for (chan = 0; chan < 4; chan++) {
4098 LLVMValueRef gather = LLVMGetUndef(LLVMVectorType(ctx->f32, input_array_size));
4099 unsigned schan = tgsi_util_get_full_src_register_swizzle(&inst->Src[0], chan);
4100
4101 for (unsigned idx = 0; idx < input_array_size; ++idx) {
4102 LLVMValueRef v, i = NULL, j = NULL;
4103
4104 if (interp_param) {
4105 i = LLVMBuildExtractElement(
4106 ctx->ac.builder, interp_param, ctx->i32_0, "");
4107 j = LLVMBuildExtractElement(
4108 ctx->ac.builder, interp_param, ctx->i32_1, "");
4109 }
4110 v = si_build_fs_interp(ctx, input_base + idx, schan,
4111 prim_mask, i, j);
4112
4113 gather = LLVMBuildInsertElement(ctx->ac.builder,
4114 gather, v, LLVMConstInt(ctx->i32, idx, false), "");
4115 }
4116
4117 emit_data->output[chan] = LLVMBuildExtractElement(
4118 ctx->ac.builder, gather, array_idx, "");
4119 }
4120 }
4121
4122 static void vote_all_emit(
4123 const struct lp_build_tgsi_action *action,
4124 struct lp_build_tgsi_context *bld_base,
4125 struct lp_build_emit_data *emit_data)
4126 {
4127 struct si_shader_context *ctx = si_shader_context(bld_base);
4128
4129 LLVMValueRef tmp = ac_build_vote_all(&ctx->ac, emit_data->args[0]);
4130 emit_data->output[emit_data->chan] =
4131 LLVMBuildSExt(ctx->ac.builder, tmp, ctx->i32, "");
4132 }
4133
4134 static void vote_any_emit(
4135 const struct lp_build_tgsi_action *action,
4136 struct lp_build_tgsi_context *bld_base,
4137 struct lp_build_emit_data *emit_data)
4138 {
4139 struct si_shader_context *ctx = si_shader_context(bld_base);
4140
4141 LLVMValueRef tmp = ac_build_vote_any(&ctx->ac, emit_data->args[0]);
4142 emit_data->output[emit_data->chan] =
4143 LLVMBuildSExt(ctx->ac.builder, tmp, ctx->i32, "");
4144 }
4145
4146 static void vote_eq_emit(
4147 const struct lp_build_tgsi_action *action,
4148 struct lp_build_tgsi_context *bld_base,
4149 struct lp_build_emit_data *emit_data)
4150 {
4151 struct si_shader_context *ctx = si_shader_context(bld_base);
4152
4153 LLVMValueRef tmp = ac_build_vote_eq(&ctx->ac, emit_data->args[0]);
4154 emit_data->output[emit_data->chan] =
4155 LLVMBuildSExt(ctx->ac.builder, tmp, ctx->i32, "");
4156 }
4157
4158 static void ballot_emit(
4159 const struct lp_build_tgsi_action *action,
4160 struct lp_build_tgsi_context *bld_base,
4161 struct lp_build_emit_data *emit_data)
4162 {
4163 struct si_shader_context *ctx = si_shader_context(bld_base);
4164 LLVMBuilderRef builder = ctx->ac.builder;
4165 LLVMValueRef tmp;
4166
4167 tmp = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_X);
4168 tmp = ac_build_ballot(&ctx->ac, tmp);
4169 tmp = LLVMBuildBitCast(builder, tmp, ctx->v2i32, "");
4170
4171 emit_data->output[0] = LLVMBuildExtractElement(builder, tmp, ctx->i32_0, "");
4172 emit_data->output[1] = LLVMBuildExtractElement(builder, tmp, ctx->i32_1, "");
4173 }
4174
4175 static void read_invoc_fetch_args(
4176 struct lp_build_tgsi_context *bld_base,
4177 struct lp_build_emit_data *emit_data)
4178 {
4179 emit_data->args[0] = lp_build_emit_fetch(bld_base, emit_data->inst,
4180 0, emit_data->src_chan);
4181
4182 /* Always read the source invocation (= lane) from the X channel. */
4183 emit_data->args[1] = lp_build_emit_fetch(bld_base, emit_data->inst,
4184 1, TGSI_CHAN_X);
4185 emit_data->arg_count = 2;
4186 }
4187
4188 static void read_lane_emit(
4189 const struct lp_build_tgsi_action *action,
4190 struct lp_build_tgsi_context *bld_base,
4191 struct lp_build_emit_data *emit_data)
4192 {
4193 struct si_shader_context *ctx = si_shader_context(bld_base);
4194
4195 /* We currently have no other way to prevent LLVM from lifting the icmp
4196 * calls to a dominating basic block.
4197 */
4198 ac_build_optimization_barrier(&ctx->ac, &emit_data->args[0]);
4199
4200 for (unsigned i = 0; i < emit_data->arg_count; ++i)
4201 emit_data->args[i] = ac_to_integer(&ctx->ac, emit_data->args[i]);
4202
4203 emit_data->output[emit_data->chan] =
4204 ac_build_intrinsic(&ctx->ac, action->intr_name,
4205 ctx->i32, emit_data->args, emit_data->arg_count,
4206 AC_FUNC_ATTR_READNONE |
4207 AC_FUNC_ATTR_CONVERGENT);
4208 }
4209
4210 static unsigned si_llvm_get_stream(struct lp_build_tgsi_context *bld_base,
4211 struct lp_build_emit_data *emit_data)
4212 {
4213 struct si_shader_context *ctx = si_shader_context(bld_base);
4214 struct tgsi_src_register src0 = emit_data->inst->Src[0].Register;
4215 LLVMValueRef imm;
4216 unsigned stream;
4217
4218 assert(src0.File == TGSI_FILE_IMMEDIATE);
4219
4220 imm = ctx->imms[src0.Index * TGSI_NUM_CHANNELS + src0.SwizzleX];
4221 stream = LLVMConstIntGetZExtValue(imm) & 0x3;
4222 return stream;
4223 }
4224
4225 /* Emit one vertex from the geometry shader */
4226 static void si_llvm_emit_vertex(struct ac_shader_abi *abi,
4227 unsigned stream,
4228 LLVMValueRef *addrs)
4229 {
4230 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
4231 struct tgsi_shader_info *info = &ctx->shader->selector->info;
4232 struct lp_build_context *uint = &ctx->bld_base.uint_bld;
4233 struct si_shader *shader = ctx->shader;
4234 struct lp_build_if_state if_state;
4235 LLVMValueRef soffset = LLVMGetParam(ctx->main_fn,
4236 ctx->param_gs2vs_offset);
4237 LLVMValueRef gs_next_vertex;
4238 LLVMValueRef can_emit;
4239 unsigned chan, offset;
4240 int i;
4241
4242 /* Write vertex attribute values to GSVS ring */
4243 gs_next_vertex = LLVMBuildLoad(ctx->ac.builder,
4244 ctx->gs_next_vertex[stream],
4245 "");
4246
4247 /* If this thread has already emitted the declared maximum number of
4248 * vertices, skip the write: excessive vertex emissions are not
4249 * supposed to have any effect.
4250 *
4251 * If the shader has no writes to memory, kill it instead. This skips
4252 * further memory loads and may allow LLVM to skip to the end
4253 * altogether.
4254 */
4255 can_emit = LLVMBuildICmp(ctx->ac.builder, LLVMIntULT, gs_next_vertex,
4256 LLVMConstInt(ctx->i32,
4257 shader->selector->gs_max_out_vertices, 0), "");
4258
4259 bool use_kill = !info->writes_memory;
4260 if (use_kill) {
4261 ac_build_kill_if_false(&ctx->ac, can_emit);
4262 } else {
4263 lp_build_if(&if_state, &ctx->gallivm, can_emit);
4264 }
4265
4266 offset = 0;
4267 for (i = 0; i < info->num_outputs; i++) {
4268 for (chan = 0; chan < 4; chan++) {
4269 if (!(info->output_usagemask[i] & (1 << chan)) ||
4270 ((info->output_streams[i] >> (2 * chan)) & 3) != stream)
4271 continue;
4272
4273 LLVMValueRef out_val = LLVMBuildLoad(ctx->ac.builder, addrs[4 * i + chan], "");
4274 LLVMValueRef voffset =
4275 LLVMConstInt(ctx->i32, offset *
4276 shader->selector->gs_max_out_vertices, 0);
4277 offset++;
4278
4279 voffset = lp_build_add(uint, voffset, gs_next_vertex);
4280 voffset = lp_build_mul_imm(uint, voffset, 4);
4281
4282 out_val = ac_to_integer(&ctx->ac, out_val);
4283
4284 ac_build_buffer_store_dword(&ctx->ac,
4285 ctx->gsvs_ring[stream],
4286 out_val, 1,
4287 voffset, soffset, 0,
4288 1, 1, true, true);
4289 }
4290 }
4291
4292 gs_next_vertex = lp_build_add(uint, gs_next_vertex,
4293 ctx->i32_1);
4294
4295 LLVMBuildStore(ctx->ac.builder, gs_next_vertex, ctx->gs_next_vertex[stream]);
4296
4297 /* Signal vertex emission */
4298 ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_EMIT | AC_SENDMSG_GS | (stream << 8),
4299 si_get_gs_wave_id(ctx));
4300 if (!use_kill)
4301 lp_build_endif(&if_state);
4302 }
4303
4304 /* Emit one vertex from the geometry shader */
4305 static void si_tgsi_emit_vertex(
4306 const struct lp_build_tgsi_action *action,
4307 struct lp_build_tgsi_context *bld_base,
4308 struct lp_build_emit_data *emit_data)
4309 {
4310 struct si_shader_context *ctx = si_shader_context(bld_base);
4311 unsigned stream = si_llvm_get_stream(bld_base, emit_data);
4312
4313 si_llvm_emit_vertex(&ctx->abi, stream, ctx->outputs[0]);
4314 }
4315
4316 /* Cut one primitive from the geometry shader */
4317 static void si_llvm_emit_primitive(struct ac_shader_abi *abi,
4318 unsigned stream)
4319 {
4320 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
4321
4322 /* Signal primitive cut */
4323 ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_CUT | AC_SENDMSG_GS | (stream << 8),
4324 si_get_gs_wave_id(ctx));
4325 }
4326
4327 /* Cut one primitive from the geometry shader */
4328 static void si_tgsi_emit_primitive(
4329 const struct lp_build_tgsi_action *action,
4330 struct lp_build_tgsi_context *bld_base,
4331 struct lp_build_emit_data *emit_data)
4332 {
4333 struct si_shader_context *ctx = si_shader_context(bld_base);
4334
4335 si_llvm_emit_primitive(&ctx->abi, si_llvm_get_stream(bld_base, emit_data));
4336 }
4337
4338 static void si_llvm_emit_barrier(const struct lp_build_tgsi_action *action,
4339 struct lp_build_tgsi_context *bld_base,
4340 struct lp_build_emit_data *emit_data)
4341 {
4342 struct si_shader_context *ctx = si_shader_context(bld_base);
4343
4344 /* SI only (thanks to a hw bug workaround):
4345 * The real barrier instruction isn’t needed, because an entire patch
4346 * always fits into a single wave.
4347 */
4348 if (ctx->screen->info.chip_class == SI &&
4349 ctx->type == PIPE_SHADER_TESS_CTRL) {
4350 ac_build_waitcnt(&ctx->ac, LGKM_CNT & VM_CNT);
4351 return;
4352 }
4353
4354 lp_build_intrinsic(ctx->ac.builder,
4355 "llvm.amdgcn.s.barrier",
4356 ctx->voidt, NULL, 0, LP_FUNC_ATTR_CONVERGENT);
4357 }
4358
4359 static const struct lp_build_tgsi_action interp_action = {
4360 .fetch_args = interp_fetch_args,
4361 .emit = build_interp_intrinsic,
4362 };
4363
4364 static void si_create_function(struct si_shader_context *ctx,
4365 const char *name,
4366 LLVMTypeRef *returns, unsigned num_returns,
4367 struct si_function_info *fninfo,
4368 unsigned max_workgroup_size)
4369 {
4370 int i;
4371
4372 si_llvm_create_func(ctx, name, returns, num_returns,
4373 fninfo->types, fninfo->num_params);
4374 ctx->return_value = LLVMGetUndef(ctx->return_type);
4375
4376 for (i = 0; i < fninfo->num_sgpr_params; ++i) {
4377 LLVMValueRef P = LLVMGetParam(ctx->main_fn, i);
4378
4379 /* The combination of:
4380 * - noalias
4381 * - dereferenceable
4382 * - invariant.load
4383 * allows the optimization passes to move loads and reduces
4384 * SGPR spilling significantly.
4385 */
4386 lp_add_function_attr(ctx->main_fn, i + 1, LP_FUNC_ATTR_INREG);
4387
4388 if (LLVMGetTypeKind(LLVMTypeOf(P)) == LLVMPointerTypeKind) {
4389 lp_add_function_attr(ctx->main_fn, i + 1, LP_FUNC_ATTR_NOALIAS);
4390 ac_add_attr_dereferenceable(P, UINT64_MAX);
4391 }
4392 }
4393
4394 for (i = 0; i < fninfo->num_params; ++i) {
4395 if (fninfo->assign[i])
4396 *fninfo->assign[i] = LLVMGetParam(ctx->main_fn, i);
4397 }
4398
4399 if (max_workgroup_size) {
4400 si_llvm_add_attribute(ctx->main_fn, "amdgpu-max-work-group-size",
4401 max_workgroup_size);
4402 }
4403 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
4404 "no-signed-zeros-fp-math",
4405 "true");
4406
4407 if (ctx->screen->debug_flags & DBG(UNSAFE_MATH)) {
4408 /* These were copied from some LLVM test. */
4409 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
4410 "less-precise-fpmad",
4411 "true");
4412 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
4413 "no-infs-fp-math",
4414 "true");
4415 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
4416 "no-nans-fp-math",
4417 "true");
4418 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
4419 "unsafe-fp-math",
4420 "true");
4421 }
4422 }
4423
4424 static void declare_streamout_params(struct si_shader_context *ctx,
4425 struct pipe_stream_output_info *so,
4426 struct si_function_info *fninfo)
4427 {
4428 int i;
4429
4430 /* Streamout SGPRs. */
4431 if (so->num_outputs) {
4432 if (ctx->type != PIPE_SHADER_TESS_EVAL)
4433 ctx->param_streamout_config = add_arg(fninfo, ARG_SGPR, ctx->ac.i32);
4434 else
4435 ctx->param_streamout_config = fninfo->num_params - 1;
4436
4437 ctx->param_streamout_write_index = add_arg(fninfo, ARG_SGPR, ctx->ac.i32);
4438 }
4439 /* A streamout buffer offset is loaded if the stride is non-zero. */
4440 for (i = 0; i < 4; i++) {
4441 if (!so->stride[i])
4442 continue;
4443
4444 ctx->param_streamout_offset[i] = add_arg(fninfo, ARG_SGPR, ctx->ac.i32);
4445 }
4446 }
4447
4448 static unsigned si_get_max_workgroup_size(const struct si_shader *shader)
4449 {
4450 switch (shader->selector->type) {
4451 case PIPE_SHADER_TESS_CTRL:
4452 /* Return this so that LLVM doesn't remove s_barrier
4453 * instructions on chips where we use s_barrier. */
4454 return shader->selector->screen->info.chip_class >= CIK ? 128 : 64;
4455
4456 case PIPE_SHADER_GEOMETRY:
4457 return shader->selector->screen->info.chip_class >= GFX9 ? 128 : 64;
4458
4459 case PIPE_SHADER_COMPUTE:
4460 break; /* see below */
4461
4462 default:
4463 return 0;
4464 }
4465
4466 const unsigned *properties = shader->selector->info.properties;
4467 unsigned max_work_group_size =
4468 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH] *
4469 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT] *
4470 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH];
4471
4472 if (!max_work_group_size) {
4473 /* This is a variable group size compute shader,
4474 * compile it for the maximum possible group size.
4475 */
4476 max_work_group_size = SI_MAX_VARIABLE_THREADS_PER_BLOCK;
4477 }
4478 return max_work_group_size;
4479 }
4480
4481 static void declare_per_stage_desc_pointers(struct si_shader_context *ctx,
4482 struct si_function_info *fninfo,
4483 bool assign_params)
4484 {
4485 LLVMTypeRef const_shader_buf_type;
4486
4487 if (ctx->shader->selector->info.const_buffers_declared == 1 &&
4488 ctx->shader->selector->info.shader_buffers_declared == 0)
4489 const_shader_buf_type = ctx->f32;
4490 else
4491 const_shader_buf_type = ctx->v4i32;
4492
4493 unsigned const_and_shader_buffers =
4494 add_arg(fninfo, ARG_SGPR,
4495 ac_array_in_const_addr_space(const_shader_buf_type));
4496
4497 unsigned samplers_and_images =
4498 add_arg(fninfo, ARG_SGPR,
4499 ac_array_in_const_addr_space(ctx->v8i32));
4500
4501 if (assign_params) {
4502 ctx->param_const_and_shader_buffers = const_and_shader_buffers;
4503 ctx->param_samplers_and_images = samplers_and_images;
4504 }
4505 }
4506
4507 static void declare_global_desc_pointers(struct si_shader_context *ctx,
4508 struct si_function_info *fninfo)
4509 {
4510 ctx->param_rw_buffers = add_arg(fninfo, ARG_SGPR,
4511 ac_array_in_const_addr_space(ctx->v4i32));
4512 ctx->param_bindless_samplers_and_images = add_arg(fninfo, ARG_SGPR,
4513 ac_array_in_const_addr_space(ctx->v8i32));
4514 }
4515
4516 static void declare_vs_specific_input_sgprs(struct si_shader_context *ctx,
4517 struct si_function_info *fninfo)
4518 {
4519 ctx->param_vertex_buffers = add_arg(fninfo, ARG_SGPR,
4520 ac_array_in_const_addr_space(ctx->v4i32));
4521 add_arg_assign(fninfo, ARG_SGPR, ctx->i32, &ctx->abi.base_vertex);
4522 add_arg_assign(fninfo, ARG_SGPR, ctx->i32, &ctx->abi.start_instance);
4523 add_arg_assign(fninfo, ARG_SGPR, ctx->i32, &ctx->abi.draw_id);
4524 ctx->param_vs_state_bits = add_arg(fninfo, ARG_SGPR, ctx->i32);
4525 }
4526
4527 static void declare_vs_input_vgprs(struct si_shader_context *ctx,
4528 struct si_function_info *fninfo,
4529 unsigned *num_prolog_vgprs)
4530 {
4531 struct si_shader *shader = ctx->shader;
4532
4533 add_arg_assign(fninfo, ARG_VGPR, ctx->i32, &ctx->abi.vertex_id);
4534 if (shader->key.as_ls) {
4535 ctx->param_rel_auto_id = add_arg(fninfo, ARG_VGPR, ctx->i32);
4536 add_arg_assign(fninfo, ARG_VGPR, ctx->i32, &ctx->abi.instance_id);
4537 } else {
4538 add_arg_assign(fninfo, ARG_VGPR, ctx->i32, &ctx->abi.instance_id);
4539 ctx->param_vs_prim_id = add_arg(fninfo, ARG_VGPR, ctx->i32);
4540 }
4541 add_arg(fninfo, ARG_VGPR, ctx->i32); /* unused */
4542
4543 if (!shader->is_gs_copy_shader) {
4544 /* Vertex load indices. */
4545 ctx->param_vertex_index0 = fninfo->num_params;
4546 for (unsigned i = 0; i < shader->selector->info.num_inputs; i++)
4547 add_arg(fninfo, ARG_VGPR, ctx->i32);
4548 *num_prolog_vgprs += shader->selector->info.num_inputs;
4549 }
4550 }
4551
4552 static void declare_tes_input_vgprs(struct si_shader_context *ctx,
4553 struct si_function_info *fninfo)
4554 {
4555 ctx->param_tes_u = add_arg(fninfo, ARG_VGPR, ctx->f32);
4556 ctx->param_tes_v = add_arg(fninfo, ARG_VGPR, ctx->f32);
4557 ctx->param_tes_rel_patch_id = add_arg(fninfo, ARG_VGPR, ctx->i32);
4558 add_arg_assign(fninfo, ARG_VGPR, ctx->i32, &ctx->abi.tes_patch_id);
4559 }
4560
4561 enum {
4562 /* Convenient merged shader definitions. */
4563 SI_SHADER_MERGED_VERTEX_TESSCTRL = PIPE_SHADER_TYPES,
4564 SI_SHADER_MERGED_VERTEX_OR_TESSEVAL_GEOMETRY,
4565 };
4566
4567 static void create_function(struct si_shader_context *ctx)
4568 {
4569 struct si_shader *shader = ctx->shader;
4570 struct si_function_info fninfo;
4571 LLVMTypeRef returns[16+32*4];
4572 unsigned i, num_return_sgprs;
4573 unsigned num_returns = 0;
4574 unsigned num_prolog_vgprs = 0;
4575 unsigned type = ctx->type;
4576 unsigned vs_blit_property =
4577 shader->selector->info.properties[TGSI_PROPERTY_VS_BLIT_SGPRS];
4578
4579 si_init_function_info(&fninfo);
4580
4581 /* Set MERGED shaders. */
4582 if (ctx->screen->info.chip_class >= GFX9) {
4583 if (shader->key.as_ls || type == PIPE_SHADER_TESS_CTRL)
4584 type = SI_SHADER_MERGED_VERTEX_TESSCTRL; /* LS or HS */
4585 else if (shader->key.as_es || type == PIPE_SHADER_GEOMETRY)
4586 type = SI_SHADER_MERGED_VERTEX_OR_TESSEVAL_GEOMETRY;
4587 }
4588
4589 LLVMTypeRef v3i32 = LLVMVectorType(ctx->i32, 3);
4590
4591 switch (type) {
4592 case PIPE_SHADER_VERTEX:
4593 declare_global_desc_pointers(ctx, &fninfo);
4594
4595 if (vs_blit_property) {
4596 ctx->param_vs_blit_inputs = fninfo.num_params;
4597 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* i16 x1, y1 */
4598 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* i16 x2, y2 */
4599 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* depth */
4600
4601 if (vs_blit_property == SI_VS_BLIT_SGPRS_POS_COLOR) {
4602 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* color0 */
4603 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* color1 */
4604 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* color2 */
4605 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* color3 */
4606 } else if (vs_blit_property == SI_VS_BLIT_SGPRS_POS_TEXCOORD) {
4607 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* texcoord.x1 */
4608 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* texcoord.y1 */
4609 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* texcoord.x2 */
4610 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* texcoord.y2 */
4611 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* texcoord.z */
4612 add_arg(&fninfo, ARG_SGPR, ctx->f32); /* texcoord.w */
4613 }
4614
4615 /* VGPRs */
4616 declare_vs_input_vgprs(ctx, &fninfo, &num_prolog_vgprs);
4617 break;
4618 }
4619
4620 declare_per_stage_desc_pointers(ctx, &fninfo, true);
4621 declare_vs_specific_input_sgprs(ctx, &fninfo);
4622
4623 if (shader->key.as_es) {
4624 ctx->param_es2gs_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4625 } else if (shader->key.as_ls) {
4626 /* no extra parameters */
4627 } else {
4628 if (shader->is_gs_copy_shader) {
4629 fninfo.num_params = ctx->param_rw_buffers + 1;
4630 fninfo.num_sgpr_params = fninfo.num_params;
4631 }
4632
4633 /* The locations of the other parameters are assigned dynamically. */
4634 declare_streamout_params(ctx, &shader->selector->so,
4635 &fninfo);
4636 }
4637
4638 /* VGPRs */
4639 declare_vs_input_vgprs(ctx, &fninfo, &num_prolog_vgprs);
4640 break;
4641
4642 case PIPE_SHADER_TESS_CTRL: /* SI-CI-VI */
4643 declare_global_desc_pointers(ctx, &fninfo);
4644 declare_per_stage_desc_pointers(ctx, &fninfo, true);
4645 ctx->param_tcs_offchip_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4646 ctx->param_tcs_out_lds_offsets = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4647 ctx->param_tcs_out_lds_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4648 ctx->param_vs_state_bits = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4649 ctx->param_tcs_offchip_addr_base64k = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4650 ctx->param_tcs_factor_addr_base64k = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4651 ctx->param_tcs_offchip_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4652 ctx->param_tcs_factor_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4653
4654 /* VGPRs */
4655 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->abi.tcs_patch_id);
4656 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->abi.tcs_rel_ids);
4657
4658 /* param_tcs_offchip_offset and param_tcs_factor_offset are
4659 * placed after the user SGPRs.
4660 */
4661 for (i = 0; i < GFX6_TCS_NUM_USER_SGPR + 2; i++)
4662 returns[num_returns++] = ctx->i32; /* SGPRs */
4663 for (i = 0; i < 11; i++)
4664 returns[num_returns++] = ctx->f32; /* VGPRs */
4665 break;
4666
4667 case SI_SHADER_MERGED_VERTEX_TESSCTRL:
4668 /* Merged stages have 8 system SGPRs at the beginning. */
4669 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* SPI_SHADER_USER_DATA_ADDR_LO_HS */
4670 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* SPI_SHADER_USER_DATA_ADDR_HI_HS */
4671 ctx->param_tcs_offchip_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4672 ctx->param_merged_wave_info = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4673 ctx->param_tcs_factor_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4674 ctx->param_merged_scratch_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4675 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4676 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4677
4678 declare_global_desc_pointers(ctx, &fninfo);
4679 declare_per_stage_desc_pointers(ctx, &fninfo,
4680 ctx->type == PIPE_SHADER_VERTEX);
4681 declare_vs_specific_input_sgprs(ctx, &fninfo);
4682
4683 ctx->param_tcs_offchip_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4684 ctx->param_tcs_out_lds_offsets = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4685 ctx->param_tcs_out_lds_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4686 ctx->param_tcs_offchip_addr_base64k = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4687 ctx->param_tcs_factor_addr_base64k = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4688 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4689
4690 declare_per_stage_desc_pointers(ctx, &fninfo,
4691 ctx->type == PIPE_SHADER_TESS_CTRL);
4692
4693 /* VGPRs (first TCS, then VS) */
4694 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->abi.tcs_patch_id);
4695 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->abi.tcs_rel_ids);
4696
4697 if (ctx->type == PIPE_SHADER_VERTEX) {
4698 declare_vs_input_vgprs(ctx, &fninfo,
4699 &num_prolog_vgprs);
4700
4701 /* LS return values are inputs to the TCS main shader part. */
4702 for (i = 0; i < 8 + GFX9_TCS_NUM_USER_SGPR; i++)
4703 returns[num_returns++] = ctx->i32; /* SGPRs */
4704 for (i = 0; i < 2; i++)
4705 returns[num_returns++] = ctx->f32; /* VGPRs */
4706 } else {
4707 /* TCS return values are inputs to the TCS epilog.
4708 *
4709 * param_tcs_offchip_offset, param_tcs_factor_offset,
4710 * param_tcs_offchip_layout, and param_rw_buffers
4711 * should be passed to the epilog.
4712 */
4713 for (i = 0; i <= 8 + GFX9_SGPR_TCS_FACTOR_ADDR_BASE64K; i++)
4714 returns[num_returns++] = ctx->i32; /* SGPRs */
4715 for (i = 0; i < 11; i++)
4716 returns[num_returns++] = ctx->f32; /* VGPRs */
4717 }
4718 break;
4719
4720 case SI_SHADER_MERGED_VERTEX_OR_TESSEVAL_GEOMETRY:
4721 /* Merged stages have 8 system SGPRs at the beginning. */
4722 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused (SPI_SHADER_USER_DATA_ADDR_LO_GS) */
4723 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused (SPI_SHADER_USER_DATA_ADDR_HI_GS) */
4724 ctx->param_gs2vs_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4725 ctx->param_merged_wave_info = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4726 ctx->param_tcs_offchip_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4727 ctx->param_merged_scratch_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4728 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused (SPI_SHADER_PGM_LO/HI_GS << 8) */
4729 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused (SPI_SHADER_PGM_LO/HI_GS >> 24) */
4730
4731 declare_global_desc_pointers(ctx, &fninfo);
4732 declare_per_stage_desc_pointers(ctx, &fninfo,
4733 (ctx->type == PIPE_SHADER_VERTEX ||
4734 ctx->type == PIPE_SHADER_TESS_EVAL));
4735 if (ctx->type == PIPE_SHADER_VERTEX) {
4736 declare_vs_specific_input_sgprs(ctx, &fninfo);
4737 } else {
4738 /* TESS_EVAL (and also GEOMETRY):
4739 * Declare as many input SGPRs as the VS has. */
4740 ctx->param_tcs_offchip_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4741 ctx->param_tcs_offchip_addr_base64k = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4742 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4743 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4744 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4745 ctx->param_vs_state_bits = add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4746 }
4747
4748 declare_per_stage_desc_pointers(ctx, &fninfo,
4749 ctx->type == PIPE_SHADER_GEOMETRY);
4750
4751 /* VGPRs (first GS, then VS/TES) */
4752 ctx->param_gs_vtx01_offset = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4753 ctx->param_gs_vtx23_offset = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4754 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->abi.gs_prim_id);
4755 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->abi.gs_invocation_id);
4756 ctx->param_gs_vtx45_offset = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4757
4758 if (ctx->type == PIPE_SHADER_VERTEX) {
4759 declare_vs_input_vgprs(ctx, &fninfo,
4760 &num_prolog_vgprs);
4761 } else if (ctx->type == PIPE_SHADER_TESS_EVAL) {
4762 declare_tes_input_vgprs(ctx, &fninfo);
4763 }
4764
4765 if (ctx->type == PIPE_SHADER_VERTEX ||
4766 ctx->type == PIPE_SHADER_TESS_EVAL) {
4767 /* ES return values are inputs to GS. */
4768 for (i = 0; i < 8 + GFX9_GS_NUM_USER_SGPR; i++)
4769 returns[num_returns++] = ctx->i32; /* SGPRs */
4770 for (i = 0; i < 5; i++)
4771 returns[num_returns++] = ctx->f32; /* VGPRs */
4772 }
4773 break;
4774
4775 case PIPE_SHADER_TESS_EVAL:
4776 declare_global_desc_pointers(ctx, &fninfo);
4777 declare_per_stage_desc_pointers(ctx, &fninfo, true);
4778 ctx->param_tcs_offchip_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4779 ctx->param_tcs_offchip_addr_base64k = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4780
4781 if (shader->key.as_es) {
4782 ctx->param_tcs_offchip_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4783 add_arg(&fninfo, ARG_SGPR, ctx->i32);
4784 ctx->param_es2gs_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4785 } else {
4786 add_arg(&fninfo, ARG_SGPR, ctx->i32);
4787 declare_streamout_params(ctx, &shader->selector->so,
4788 &fninfo);
4789 ctx->param_tcs_offchip_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4790 }
4791
4792 /* VGPRs */
4793 declare_tes_input_vgprs(ctx, &fninfo);
4794 break;
4795
4796 case PIPE_SHADER_GEOMETRY:
4797 declare_global_desc_pointers(ctx, &fninfo);
4798 declare_per_stage_desc_pointers(ctx, &fninfo, true);
4799 ctx->param_gs2vs_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4800 ctx->param_gs_wave_id = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4801
4802 /* VGPRs */
4803 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->gs_vtx_offset[0]);
4804 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->gs_vtx_offset[1]);
4805 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->abi.gs_prim_id);
4806 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->gs_vtx_offset[2]);
4807 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->gs_vtx_offset[3]);
4808 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->gs_vtx_offset[4]);
4809 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->gs_vtx_offset[5]);
4810 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &ctx->abi.gs_invocation_id);
4811 break;
4812
4813 case PIPE_SHADER_FRAGMENT:
4814 declare_global_desc_pointers(ctx, &fninfo);
4815 declare_per_stage_desc_pointers(ctx, &fninfo, true);
4816 add_arg_checked(&fninfo, ARG_SGPR, ctx->f32, SI_PARAM_ALPHA_REF);
4817 add_arg_assign_checked(&fninfo, ARG_SGPR, ctx->i32,
4818 &ctx->abi.prim_mask, SI_PARAM_PRIM_MASK);
4819
4820 add_arg_checked(&fninfo, ARG_VGPR, ctx->v2i32, SI_PARAM_PERSP_SAMPLE);
4821 add_arg_checked(&fninfo, ARG_VGPR, ctx->v2i32, SI_PARAM_PERSP_CENTER);
4822 add_arg_checked(&fninfo, ARG_VGPR, ctx->v2i32, SI_PARAM_PERSP_CENTROID);
4823 add_arg_checked(&fninfo, ARG_VGPR, v3i32, SI_PARAM_PERSP_PULL_MODEL);
4824 add_arg_checked(&fninfo, ARG_VGPR, ctx->v2i32, SI_PARAM_LINEAR_SAMPLE);
4825 add_arg_checked(&fninfo, ARG_VGPR, ctx->v2i32, SI_PARAM_LINEAR_CENTER);
4826 add_arg_checked(&fninfo, ARG_VGPR, ctx->v2i32, SI_PARAM_LINEAR_CENTROID);
4827 add_arg_checked(&fninfo, ARG_VGPR, ctx->f32, SI_PARAM_LINE_STIPPLE_TEX);
4828 add_arg_assign_checked(&fninfo, ARG_VGPR, ctx->f32,
4829 &ctx->abi.frag_pos[0], SI_PARAM_POS_X_FLOAT);
4830 add_arg_assign_checked(&fninfo, ARG_VGPR, ctx->f32,
4831 &ctx->abi.frag_pos[1], SI_PARAM_POS_Y_FLOAT);
4832 add_arg_assign_checked(&fninfo, ARG_VGPR, ctx->f32,
4833 &ctx->abi.frag_pos[2], SI_PARAM_POS_Z_FLOAT);
4834 add_arg_assign_checked(&fninfo, ARG_VGPR, ctx->f32,
4835 &ctx->abi.frag_pos[3], SI_PARAM_POS_W_FLOAT);
4836 add_arg_assign_checked(&fninfo, ARG_VGPR, ctx->i32,
4837 &ctx->abi.front_face, SI_PARAM_FRONT_FACE);
4838 shader->info.face_vgpr_index = 20;
4839 add_arg_assign_checked(&fninfo, ARG_VGPR, ctx->i32,
4840 &ctx->abi.ancillary, SI_PARAM_ANCILLARY);
4841 shader->info.ancillary_vgpr_index = 21;
4842 add_arg_assign_checked(&fninfo, ARG_VGPR, ctx->f32,
4843 &ctx->abi.sample_coverage, SI_PARAM_SAMPLE_COVERAGE);
4844 add_arg_checked(&fninfo, ARG_VGPR, ctx->i32, SI_PARAM_POS_FIXED_PT);
4845
4846 /* Color inputs from the prolog. */
4847 if (shader->selector->info.colors_read) {
4848 unsigned num_color_elements =
4849 util_bitcount(shader->selector->info.colors_read);
4850
4851 assert(fninfo.num_params + num_color_elements <= ARRAY_SIZE(fninfo.types));
4852 for (i = 0; i < num_color_elements; i++)
4853 add_arg(&fninfo, ARG_VGPR, ctx->f32);
4854
4855 num_prolog_vgprs += num_color_elements;
4856 }
4857
4858 /* Outputs for the epilog. */
4859 num_return_sgprs = SI_SGPR_ALPHA_REF + 1;
4860 num_returns =
4861 num_return_sgprs +
4862 util_bitcount(shader->selector->info.colors_written) * 4 +
4863 shader->selector->info.writes_z +
4864 shader->selector->info.writes_stencil +
4865 shader->selector->info.writes_samplemask +
4866 1 /* SampleMaskIn */;
4867
4868 num_returns = MAX2(num_returns,
4869 num_return_sgprs +
4870 PS_EPILOG_SAMPLEMASK_MIN_LOC + 1);
4871
4872 for (i = 0; i < num_return_sgprs; i++)
4873 returns[i] = ctx->i32;
4874 for (; i < num_returns; i++)
4875 returns[i] = ctx->f32;
4876 break;
4877
4878 case PIPE_SHADER_COMPUTE:
4879 declare_global_desc_pointers(ctx, &fninfo);
4880 declare_per_stage_desc_pointers(ctx, &fninfo, true);
4881 if (shader->selector->info.uses_grid_size)
4882 add_arg_assign(&fninfo, ARG_SGPR, v3i32, &ctx->abi.num_work_groups);
4883 if (shader->selector->info.uses_block_size)
4884 ctx->param_block_size = add_arg(&fninfo, ARG_SGPR, v3i32);
4885
4886 for (i = 0; i < 3; i++) {
4887 ctx->abi.workgroup_ids[i] = NULL;
4888 if (shader->selector->info.uses_block_id[i])
4889 add_arg_assign(&fninfo, ARG_SGPR, ctx->i32, &ctx->abi.workgroup_ids[i]);
4890 }
4891
4892 add_arg_assign(&fninfo, ARG_VGPR, v3i32, &ctx->abi.local_invocation_ids);
4893 break;
4894 default:
4895 assert(0 && "unimplemented shader");
4896 return;
4897 }
4898
4899 si_create_function(ctx, "main", returns, num_returns, &fninfo,
4900 si_get_max_workgroup_size(shader));
4901
4902 /* Reserve register locations for VGPR inputs the PS prolog may need. */
4903 if (ctx->type == PIPE_SHADER_FRAGMENT &&
4904 ctx->separate_prolog) {
4905 si_llvm_add_attribute(ctx->main_fn,
4906 "InitialPSInputAddr",
4907 S_0286D0_PERSP_SAMPLE_ENA(1) |
4908 S_0286D0_PERSP_CENTER_ENA(1) |
4909 S_0286D0_PERSP_CENTROID_ENA(1) |
4910 S_0286D0_LINEAR_SAMPLE_ENA(1) |
4911 S_0286D0_LINEAR_CENTER_ENA(1) |
4912 S_0286D0_LINEAR_CENTROID_ENA(1) |
4913 S_0286D0_FRONT_FACE_ENA(1) |
4914 S_0286D0_ANCILLARY_ENA(1) |
4915 S_0286D0_POS_FIXED_PT_ENA(1));
4916 }
4917
4918 shader->info.num_input_sgprs = 0;
4919 shader->info.num_input_vgprs = 0;
4920
4921 for (i = 0; i < fninfo.num_sgpr_params; ++i)
4922 shader->info.num_input_sgprs += ac_get_type_size(fninfo.types[i]) / 4;
4923
4924 for (; i < fninfo.num_params; ++i)
4925 shader->info.num_input_vgprs += ac_get_type_size(fninfo.types[i]) / 4;
4926
4927 assert(shader->info.num_input_vgprs >= num_prolog_vgprs);
4928 shader->info.num_input_vgprs -= num_prolog_vgprs;
4929
4930 if (shader->key.as_ls ||
4931 ctx->type == PIPE_SHADER_TESS_CTRL ||
4932 /* GFX9 has the ESGS ring buffer in LDS. */
4933 type == SI_SHADER_MERGED_VERTEX_OR_TESSEVAL_GEOMETRY)
4934 ac_declare_lds_as_pointer(&ctx->ac);
4935 }
4936
4937 /**
4938 * Load ESGS and GSVS ring buffer resource descriptors and save the variables
4939 * for later use.
4940 */
4941 static void preload_ring_buffers(struct si_shader_context *ctx)
4942 {
4943 LLVMBuilderRef builder = ctx->ac.builder;
4944
4945 LLVMValueRef buf_ptr = LLVMGetParam(ctx->main_fn,
4946 ctx->param_rw_buffers);
4947
4948 if (ctx->screen->info.chip_class <= VI &&
4949 (ctx->shader->key.as_es || ctx->type == PIPE_SHADER_GEOMETRY)) {
4950 unsigned ring =
4951 ctx->type == PIPE_SHADER_GEOMETRY ? SI_GS_RING_ESGS
4952 : SI_ES_RING_ESGS;
4953 LLVMValueRef offset = LLVMConstInt(ctx->i32, ring, 0);
4954
4955 ctx->esgs_ring =
4956 ac_build_load_to_sgpr(&ctx->ac, buf_ptr, offset);
4957 }
4958
4959 if (ctx->shader->is_gs_copy_shader) {
4960 LLVMValueRef offset = LLVMConstInt(ctx->i32, SI_RING_GSVS, 0);
4961
4962 ctx->gsvs_ring[0] =
4963 ac_build_load_to_sgpr(&ctx->ac, buf_ptr, offset);
4964 } else if (ctx->type == PIPE_SHADER_GEOMETRY) {
4965 const struct si_shader_selector *sel = ctx->shader->selector;
4966 LLVMValueRef offset = LLVMConstInt(ctx->i32, SI_RING_GSVS, 0);
4967 LLVMValueRef base_ring;
4968
4969 base_ring = ac_build_load_to_sgpr(&ctx->ac, buf_ptr, offset);
4970
4971 /* The conceptual layout of the GSVS ring is
4972 * v0c0 .. vLv0 v0c1 .. vLc1 ..
4973 * but the real memory layout is swizzled across
4974 * threads:
4975 * t0v0c0 .. t15v0c0 t0v1c0 .. t15v1c0 ... t15vLcL
4976 * t16v0c0 ..
4977 * Override the buffer descriptor accordingly.
4978 */
4979 LLVMTypeRef v2i64 = LLVMVectorType(ctx->i64, 2);
4980 uint64_t stream_offset = 0;
4981
4982 for (unsigned stream = 0; stream < 4; ++stream) {
4983 unsigned num_components;
4984 unsigned stride;
4985 unsigned num_records;
4986 LLVMValueRef ring, tmp;
4987
4988 num_components = sel->info.num_stream_output_components[stream];
4989 if (!num_components)
4990 continue;
4991
4992 stride = 4 * num_components * sel->gs_max_out_vertices;
4993
4994 /* Limit on the stride field for <= CIK. */
4995 assert(stride < (1 << 14));
4996
4997 num_records = 64;
4998
4999 ring = LLVMBuildBitCast(builder, base_ring, v2i64, "");
5000 tmp = LLVMBuildExtractElement(builder, ring, ctx->i32_0, "");
5001 tmp = LLVMBuildAdd(builder, tmp,
5002 LLVMConstInt(ctx->i64,
5003 stream_offset, 0), "");
5004 stream_offset += stride * 64;
5005
5006 ring = LLVMBuildInsertElement(builder, ring, tmp, ctx->i32_0, "");
5007 ring = LLVMBuildBitCast(builder, ring, ctx->v4i32, "");
5008 tmp = LLVMBuildExtractElement(builder, ring, ctx->i32_1, "");
5009 tmp = LLVMBuildOr(builder, tmp,
5010 LLVMConstInt(ctx->i32,
5011 S_008F04_STRIDE(stride) |
5012 S_008F04_SWIZZLE_ENABLE(1), 0), "");
5013 ring = LLVMBuildInsertElement(builder, ring, tmp, ctx->i32_1, "");
5014 ring = LLVMBuildInsertElement(builder, ring,
5015 LLVMConstInt(ctx->i32, num_records, 0),
5016 LLVMConstInt(ctx->i32, 2, 0), "");
5017 ring = LLVMBuildInsertElement(builder, ring,
5018 LLVMConstInt(ctx->i32,
5019 S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
5020 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
5021 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
5022 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
5023 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5024 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) |
5025 S_008F0C_ELEMENT_SIZE(1) | /* element_size = 4 (bytes) */
5026 S_008F0C_INDEX_STRIDE(1) | /* index_stride = 16 (elements) */
5027 S_008F0C_ADD_TID_ENABLE(1),
5028 0),
5029 LLVMConstInt(ctx->i32, 3, 0), "");
5030
5031 ctx->gsvs_ring[stream] = ring;
5032 }
5033 }
5034 }
5035
5036 static void si_llvm_emit_polygon_stipple(struct si_shader_context *ctx,
5037 LLVMValueRef param_rw_buffers,
5038 unsigned param_pos_fixed_pt)
5039 {
5040 LLVMBuilderRef builder = ctx->ac.builder;
5041 LLVMValueRef slot, desc, offset, row, bit, address[2];
5042
5043 /* Use the fixed-point gl_FragCoord input.
5044 * Since the stipple pattern is 32x32 and it repeats, just get 5 bits
5045 * per coordinate to get the repeating effect.
5046 */
5047 address[0] = unpack_param(ctx, param_pos_fixed_pt, 0, 5);
5048 address[1] = unpack_param(ctx, param_pos_fixed_pt, 16, 5);
5049
5050 /* Load the buffer descriptor. */
5051 slot = LLVMConstInt(ctx->i32, SI_PS_CONST_POLY_STIPPLE, 0);
5052 desc = ac_build_load_to_sgpr(&ctx->ac, param_rw_buffers, slot);
5053
5054 /* The stipple pattern is 32x32, each row has 32 bits. */
5055 offset = LLVMBuildMul(builder, address[1],
5056 LLVMConstInt(ctx->i32, 4, 0), "");
5057 row = buffer_load_const(ctx, desc, offset);
5058 row = ac_to_integer(&ctx->ac, row);
5059 bit = LLVMBuildLShr(builder, row, address[0], "");
5060 bit = LLVMBuildTrunc(builder, bit, ctx->i1, "");
5061 ac_build_kill_if_false(&ctx->ac, bit);
5062 }
5063
5064 void si_shader_binary_read_config(struct ac_shader_binary *binary,
5065 struct si_shader_config *conf,
5066 unsigned symbol_offset)
5067 {
5068 unsigned i;
5069 const unsigned char *config =
5070 ac_shader_binary_config_start(binary, symbol_offset);
5071 bool really_needs_scratch = false;
5072
5073 /* LLVM adds SGPR spills to the scratch size.
5074 * Find out if we really need the scratch buffer.
5075 */
5076 for (i = 0; i < binary->reloc_count; i++) {
5077 const struct ac_shader_reloc *reloc = &binary->relocs[i];
5078
5079 if (!strcmp(scratch_rsrc_dword0_symbol, reloc->name) ||
5080 !strcmp(scratch_rsrc_dword1_symbol, reloc->name)) {
5081 really_needs_scratch = true;
5082 break;
5083 }
5084 }
5085
5086 /* XXX: We may be able to emit some of these values directly rather than
5087 * extracting fields to be emitted later.
5088 */
5089
5090 for (i = 0; i < binary->config_size_per_symbol; i+= 8) {
5091 unsigned reg = util_le32_to_cpu(*(uint32_t*)(config + i));
5092 unsigned value = util_le32_to_cpu(*(uint32_t*)(config + i + 4));
5093 switch (reg) {
5094 case R_00B028_SPI_SHADER_PGM_RSRC1_PS:
5095 case R_00B128_SPI_SHADER_PGM_RSRC1_VS:
5096 case R_00B228_SPI_SHADER_PGM_RSRC1_GS:
5097 case R_00B428_SPI_SHADER_PGM_RSRC1_HS:
5098 case R_00B848_COMPUTE_PGM_RSRC1:
5099 conf->num_sgprs = MAX2(conf->num_sgprs, (G_00B028_SGPRS(value) + 1) * 8);
5100 conf->num_vgprs = MAX2(conf->num_vgprs, (G_00B028_VGPRS(value) + 1) * 4);
5101 conf->float_mode = G_00B028_FLOAT_MODE(value);
5102 conf->rsrc1 = value;
5103 break;
5104 case R_00B02C_SPI_SHADER_PGM_RSRC2_PS:
5105 conf->lds_size = MAX2(conf->lds_size, G_00B02C_EXTRA_LDS_SIZE(value));
5106 break;
5107 case R_00B84C_COMPUTE_PGM_RSRC2:
5108 conf->lds_size = MAX2(conf->lds_size, G_00B84C_LDS_SIZE(value));
5109 conf->rsrc2 = value;
5110 break;
5111 case R_0286CC_SPI_PS_INPUT_ENA:
5112 conf->spi_ps_input_ena = value;
5113 break;
5114 case R_0286D0_SPI_PS_INPUT_ADDR:
5115 conf->spi_ps_input_addr = value;
5116 break;
5117 case R_0286E8_SPI_TMPRING_SIZE:
5118 case R_00B860_COMPUTE_TMPRING_SIZE:
5119 /* WAVESIZE is in units of 256 dwords. */
5120 if (really_needs_scratch)
5121 conf->scratch_bytes_per_wave =
5122 G_00B860_WAVESIZE(value) * 256 * 4;
5123 break;
5124 case 0x4: /* SPILLED_SGPRS */
5125 conf->spilled_sgprs = value;
5126 break;
5127 case 0x8: /* SPILLED_VGPRS */
5128 conf->spilled_vgprs = value;
5129 break;
5130 default:
5131 {
5132 static bool printed;
5133
5134 if (!printed) {
5135 fprintf(stderr, "Warning: LLVM emitted unknown "
5136 "config register: 0x%x\n", reg);
5137 printed = true;
5138 }
5139 }
5140 break;
5141 }
5142 }
5143
5144 if (!conf->spi_ps_input_addr)
5145 conf->spi_ps_input_addr = conf->spi_ps_input_ena;
5146 }
5147
5148 void si_shader_apply_scratch_relocs(struct si_shader *shader,
5149 uint64_t scratch_va)
5150 {
5151 unsigned i;
5152 uint32_t scratch_rsrc_dword0 = scratch_va;
5153 uint32_t scratch_rsrc_dword1 =
5154 S_008F04_BASE_ADDRESS_HI(scratch_va >> 32);
5155
5156 /* Enable scratch coalescing. */
5157 scratch_rsrc_dword1 |= S_008F04_SWIZZLE_ENABLE(1);
5158
5159 for (i = 0 ; i < shader->binary.reloc_count; i++) {
5160 const struct ac_shader_reloc *reloc =
5161 &shader->binary.relocs[i];
5162 if (!strcmp(scratch_rsrc_dword0_symbol, reloc->name)) {
5163 util_memcpy_cpu_to_le32(shader->binary.code + reloc->offset,
5164 &scratch_rsrc_dword0, 4);
5165 } else if (!strcmp(scratch_rsrc_dword1_symbol, reloc->name)) {
5166 util_memcpy_cpu_to_le32(shader->binary.code + reloc->offset,
5167 &scratch_rsrc_dword1, 4);
5168 }
5169 }
5170 }
5171
5172 static unsigned si_get_shader_binary_size(const struct si_shader *shader)
5173 {
5174 unsigned size = shader->binary.code_size;
5175
5176 if (shader->prolog)
5177 size += shader->prolog->binary.code_size;
5178 if (shader->previous_stage)
5179 size += shader->previous_stage->binary.code_size;
5180 if (shader->prolog2)
5181 size += shader->prolog2->binary.code_size;
5182 if (shader->epilog)
5183 size += shader->epilog->binary.code_size;
5184 return size;
5185 }
5186
5187 int si_shader_binary_upload(struct si_screen *sscreen, struct si_shader *shader)
5188 {
5189 const struct ac_shader_binary *prolog =
5190 shader->prolog ? &shader->prolog->binary : NULL;
5191 const struct ac_shader_binary *previous_stage =
5192 shader->previous_stage ? &shader->previous_stage->binary : NULL;
5193 const struct ac_shader_binary *prolog2 =
5194 shader->prolog2 ? &shader->prolog2->binary : NULL;
5195 const struct ac_shader_binary *epilog =
5196 shader->epilog ? &shader->epilog->binary : NULL;
5197 const struct ac_shader_binary *mainb = &shader->binary;
5198 unsigned bo_size = si_get_shader_binary_size(shader) +
5199 (!epilog ? mainb->rodata_size : 0);
5200 unsigned char *ptr;
5201
5202 assert(!prolog || !prolog->rodata_size);
5203 assert(!previous_stage || !previous_stage->rodata_size);
5204 assert(!prolog2 || !prolog2->rodata_size);
5205 assert((!prolog && !previous_stage && !prolog2 && !epilog) ||
5206 !mainb->rodata_size);
5207 assert(!epilog || !epilog->rodata_size);
5208
5209 r600_resource_reference(&shader->bo, NULL);
5210 shader->bo = (struct r600_resource*)
5211 si_aligned_buffer_create(&sscreen->b,
5212 sscreen->cpdma_prefetch_writes_memory ?
5213 0 : R600_RESOURCE_FLAG_READ_ONLY,
5214 PIPE_USAGE_IMMUTABLE,
5215 align(bo_size, SI_CPDMA_ALIGNMENT),
5216 256);
5217 if (!shader->bo)
5218 return -ENOMEM;
5219
5220 /* Upload. */
5221 ptr = sscreen->ws->buffer_map(shader->bo->buf, NULL,
5222 PIPE_TRANSFER_READ_WRITE |
5223 PIPE_TRANSFER_UNSYNCHRONIZED);
5224
5225 /* Don't use util_memcpy_cpu_to_le32. LLVM binaries are
5226 * endian-independent. */
5227 if (prolog) {
5228 memcpy(ptr, prolog->code, prolog->code_size);
5229 ptr += prolog->code_size;
5230 }
5231 if (previous_stage) {
5232 memcpy(ptr, previous_stage->code, previous_stage->code_size);
5233 ptr += previous_stage->code_size;
5234 }
5235 if (prolog2) {
5236 memcpy(ptr, prolog2->code, prolog2->code_size);
5237 ptr += prolog2->code_size;
5238 }
5239
5240 memcpy(ptr, mainb->code, mainb->code_size);
5241 ptr += mainb->code_size;
5242
5243 if (epilog)
5244 memcpy(ptr, epilog->code, epilog->code_size);
5245 else if (mainb->rodata_size > 0)
5246 memcpy(ptr, mainb->rodata, mainb->rodata_size);
5247
5248 sscreen->ws->buffer_unmap(shader->bo->buf);
5249 return 0;
5250 }
5251
5252 static void si_shader_dump_disassembly(const struct ac_shader_binary *binary,
5253 struct pipe_debug_callback *debug,
5254 const char *name, FILE *file)
5255 {
5256 char *line, *p;
5257 unsigned i, count;
5258
5259 if (binary->disasm_string) {
5260 fprintf(file, "Shader %s disassembly:\n", name);
5261 fprintf(file, "%s", binary->disasm_string);
5262
5263 if (debug && debug->debug_message) {
5264 /* Very long debug messages are cut off, so send the
5265 * disassembly one line at a time. This causes more
5266 * overhead, but on the plus side it simplifies
5267 * parsing of resulting logs.
5268 */
5269 pipe_debug_message(debug, SHADER_INFO,
5270 "Shader Disassembly Begin");
5271
5272 line = binary->disasm_string;
5273 while (*line) {
5274 p = util_strchrnul(line, '\n');
5275 count = p - line;
5276
5277 if (count) {
5278 pipe_debug_message(debug, SHADER_INFO,
5279 "%.*s", count, line);
5280 }
5281
5282 if (!*p)
5283 break;
5284 line = p + 1;
5285 }
5286
5287 pipe_debug_message(debug, SHADER_INFO,
5288 "Shader Disassembly End");
5289 }
5290 } else {
5291 fprintf(file, "Shader %s binary:\n", name);
5292 for (i = 0; i < binary->code_size; i += 4) {
5293 fprintf(file, "@0x%x: %02x%02x%02x%02x\n", i,
5294 binary->code[i + 3], binary->code[i + 2],
5295 binary->code[i + 1], binary->code[i]);
5296 }
5297 }
5298 }
5299
5300 static void si_calculate_max_simd_waves(struct si_shader *shader)
5301 {
5302 struct si_screen *sscreen = shader->selector->screen;
5303 struct si_shader_config *conf = &shader->config;
5304 unsigned num_inputs = shader->selector->info.num_inputs;
5305 unsigned lds_increment = sscreen->info.chip_class >= CIK ? 512 : 256;
5306 unsigned lds_per_wave = 0;
5307 unsigned max_simd_waves;
5308
5309 switch (sscreen->info.family) {
5310 /* These always have 8 waves: */
5311 case CHIP_POLARIS10:
5312 case CHIP_POLARIS11:
5313 case CHIP_POLARIS12:
5314 max_simd_waves = 8;
5315 break;
5316 default:
5317 max_simd_waves = 10;
5318 }
5319
5320 /* Compute LDS usage for PS. */
5321 switch (shader->selector->type) {
5322 case PIPE_SHADER_FRAGMENT:
5323 /* The minimum usage per wave is (num_inputs * 48). The maximum
5324 * usage is (num_inputs * 48 * 16).
5325 * We can get anything in between and it varies between waves.
5326 *
5327 * The 48 bytes per input for a single primitive is equal to
5328 * 4 bytes/component * 4 components/input * 3 points.
5329 *
5330 * Other stages don't know the size at compile time or don't
5331 * allocate LDS per wave, but instead they do it per thread group.
5332 */
5333 lds_per_wave = conf->lds_size * lds_increment +
5334 align(num_inputs * 48, lds_increment);
5335 break;
5336 case PIPE_SHADER_COMPUTE:
5337 if (shader->selector) {
5338 unsigned max_workgroup_size =
5339 si_get_max_workgroup_size(shader);
5340 lds_per_wave = (conf->lds_size * lds_increment) /
5341 DIV_ROUND_UP(max_workgroup_size, 64);
5342 }
5343 break;
5344 }
5345
5346 /* Compute the per-SIMD wave counts. */
5347 if (conf->num_sgprs) {
5348 if (sscreen->info.chip_class >= VI)
5349 max_simd_waves = MIN2(max_simd_waves, 800 / conf->num_sgprs);
5350 else
5351 max_simd_waves = MIN2(max_simd_waves, 512 / conf->num_sgprs);
5352 }
5353
5354 if (conf->num_vgprs)
5355 max_simd_waves = MIN2(max_simd_waves, 256 / conf->num_vgprs);
5356
5357 /* LDS is 64KB per CU (4 SIMDs), which is 16KB per SIMD (usage above
5358 * 16KB makes some SIMDs unoccupied). */
5359 if (lds_per_wave)
5360 max_simd_waves = MIN2(max_simd_waves, 16384 / lds_per_wave);
5361
5362 conf->max_simd_waves = max_simd_waves;
5363 }
5364
5365 void si_shader_dump_stats_for_shader_db(const struct si_shader *shader,
5366 struct pipe_debug_callback *debug)
5367 {
5368 const struct si_shader_config *conf = &shader->config;
5369
5370 pipe_debug_message(debug, SHADER_INFO,
5371 "Shader Stats: SGPRS: %d VGPRS: %d Code Size: %d "
5372 "LDS: %d Scratch: %d Max Waves: %d Spilled SGPRs: %d "
5373 "Spilled VGPRs: %d PrivMem VGPRs: %d",
5374 conf->num_sgprs, conf->num_vgprs,
5375 si_get_shader_binary_size(shader),
5376 conf->lds_size, conf->scratch_bytes_per_wave,
5377 conf->max_simd_waves, conf->spilled_sgprs,
5378 conf->spilled_vgprs, conf->private_mem_vgprs);
5379 }
5380
5381 static void si_shader_dump_stats(struct si_screen *sscreen,
5382 const struct si_shader *shader,
5383 unsigned processor,
5384 FILE *file,
5385 bool check_debug_option)
5386 {
5387 const struct si_shader_config *conf = &shader->config;
5388
5389 if (!check_debug_option ||
5390 si_can_dump_shader(sscreen, processor)) {
5391 if (processor == PIPE_SHADER_FRAGMENT) {
5392 fprintf(file, "*** SHADER CONFIG ***\n"
5393 "SPI_PS_INPUT_ADDR = 0x%04x\n"
5394 "SPI_PS_INPUT_ENA = 0x%04x\n",
5395 conf->spi_ps_input_addr, conf->spi_ps_input_ena);
5396 }
5397
5398 fprintf(file, "*** SHADER STATS ***\n"
5399 "SGPRS: %d\n"
5400 "VGPRS: %d\n"
5401 "Spilled SGPRs: %d\n"
5402 "Spilled VGPRs: %d\n"
5403 "Private memory VGPRs: %d\n"
5404 "Code Size: %d bytes\n"
5405 "LDS: %d blocks\n"
5406 "Scratch: %d bytes per wave\n"
5407 "Max Waves: %d\n"
5408 "********************\n\n\n",
5409 conf->num_sgprs, conf->num_vgprs,
5410 conf->spilled_sgprs, conf->spilled_vgprs,
5411 conf->private_mem_vgprs,
5412 si_get_shader_binary_size(shader),
5413 conf->lds_size, conf->scratch_bytes_per_wave,
5414 conf->max_simd_waves);
5415 }
5416 }
5417
5418 const char *si_get_shader_name(const struct si_shader *shader, unsigned processor)
5419 {
5420 switch (processor) {
5421 case PIPE_SHADER_VERTEX:
5422 if (shader->key.as_es)
5423 return "Vertex Shader as ES";
5424 else if (shader->key.as_ls)
5425 return "Vertex Shader as LS";
5426 else
5427 return "Vertex Shader as VS";
5428 case PIPE_SHADER_TESS_CTRL:
5429 return "Tessellation Control Shader";
5430 case PIPE_SHADER_TESS_EVAL:
5431 if (shader->key.as_es)
5432 return "Tessellation Evaluation Shader as ES";
5433 else
5434 return "Tessellation Evaluation Shader as VS";
5435 case PIPE_SHADER_GEOMETRY:
5436 if (shader->is_gs_copy_shader)
5437 return "GS Copy Shader as VS";
5438 else
5439 return "Geometry Shader";
5440 case PIPE_SHADER_FRAGMENT:
5441 return "Pixel Shader";
5442 case PIPE_SHADER_COMPUTE:
5443 return "Compute Shader";
5444 default:
5445 return "Unknown Shader";
5446 }
5447 }
5448
5449 void si_shader_dump(struct si_screen *sscreen, const struct si_shader *shader,
5450 struct pipe_debug_callback *debug, unsigned processor,
5451 FILE *file, bool check_debug_option)
5452 {
5453 if (!check_debug_option ||
5454 si_can_dump_shader(sscreen, processor))
5455 si_dump_shader_key(processor, shader, file);
5456
5457 if (!check_debug_option && shader->binary.llvm_ir_string) {
5458 if (shader->previous_stage &&
5459 shader->previous_stage->binary.llvm_ir_string) {
5460 fprintf(file, "\n%s - previous stage - LLVM IR:\n\n",
5461 si_get_shader_name(shader, processor));
5462 fprintf(file, "%s\n", shader->previous_stage->binary.llvm_ir_string);
5463 }
5464
5465 fprintf(file, "\n%s - main shader part - LLVM IR:\n\n",
5466 si_get_shader_name(shader, processor));
5467 fprintf(file, "%s\n", shader->binary.llvm_ir_string);
5468 }
5469
5470 if (!check_debug_option ||
5471 (si_can_dump_shader(sscreen, processor) &&
5472 !(sscreen->debug_flags & DBG(NO_ASM)))) {
5473 fprintf(file, "\n%s:\n", si_get_shader_name(shader, processor));
5474
5475 if (shader->prolog)
5476 si_shader_dump_disassembly(&shader->prolog->binary,
5477 debug, "prolog", file);
5478 if (shader->previous_stage)
5479 si_shader_dump_disassembly(&shader->previous_stage->binary,
5480 debug, "previous stage", file);
5481 if (shader->prolog2)
5482 si_shader_dump_disassembly(&shader->prolog2->binary,
5483 debug, "prolog2", file);
5484
5485 si_shader_dump_disassembly(&shader->binary, debug, "main", file);
5486
5487 if (shader->epilog)
5488 si_shader_dump_disassembly(&shader->epilog->binary,
5489 debug, "epilog", file);
5490 fprintf(file, "\n");
5491 }
5492
5493 si_shader_dump_stats(sscreen, shader, processor, file,
5494 check_debug_option);
5495 }
5496
5497 static int si_compile_llvm(struct si_screen *sscreen,
5498 struct ac_shader_binary *binary,
5499 struct si_shader_config *conf,
5500 LLVMTargetMachineRef tm,
5501 LLVMModuleRef mod,
5502 struct pipe_debug_callback *debug,
5503 unsigned processor,
5504 const char *name)
5505 {
5506 int r = 0;
5507 unsigned count = p_atomic_inc_return(&sscreen->num_compilations);
5508
5509 if (si_can_dump_shader(sscreen, processor)) {
5510 fprintf(stderr, "radeonsi: Compiling shader %d\n", count);
5511
5512 if (!(sscreen->debug_flags & (DBG(NO_IR) | DBG(PREOPT_IR)))) {
5513 fprintf(stderr, "%s LLVM IR:\n\n", name);
5514 ac_dump_module(mod);
5515 fprintf(stderr, "\n");
5516 }
5517 }
5518
5519 if (sscreen->record_llvm_ir) {
5520 char *ir = LLVMPrintModuleToString(mod);
5521 binary->llvm_ir_string = strdup(ir);
5522 LLVMDisposeMessage(ir);
5523 }
5524
5525 if (!si_replace_shader(count, binary)) {
5526 r = si_llvm_compile(mod, binary, tm, debug);
5527 if (r)
5528 return r;
5529 }
5530
5531 si_shader_binary_read_config(binary, conf, 0);
5532
5533 /* Enable 64-bit and 16-bit denormals, because there is no performance
5534 * cost.
5535 *
5536 * If denormals are enabled, all floating-point output modifiers are
5537 * ignored.
5538 *
5539 * Don't enable denormals for 32-bit floats, because:
5540 * - Floating-point output modifiers would be ignored by the hw.
5541 * - Some opcodes don't support denormals, such as v_mad_f32. We would
5542 * have to stop using those.
5543 * - SI & CI would be very slow.
5544 */
5545 conf->float_mode |= V_00B028_FP_64_DENORMS;
5546
5547 FREE(binary->config);
5548 FREE(binary->global_symbol_offsets);
5549 binary->config = NULL;
5550 binary->global_symbol_offsets = NULL;
5551
5552 /* Some shaders can't have rodata because their binaries can be
5553 * concatenated.
5554 */
5555 if (binary->rodata_size &&
5556 (processor == PIPE_SHADER_VERTEX ||
5557 processor == PIPE_SHADER_TESS_CTRL ||
5558 processor == PIPE_SHADER_TESS_EVAL ||
5559 processor == PIPE_SHADER_FRAGMENT)) {
5560 fprintf(stderr, "radeonsi: The shader can't have rodata.");
5561 return -EINVAL;
5562 }
5563
5564 return r;
5565 }
5566
5567 static void si_llvm_build_ret(struct si_shader_context *ctx, LLVMValueRef ret)
5568 {
5569 if (LLVMGetTypeKind(LLVMTypeOf(ret)) == LLVMVoidTypeKind)
5570 LLVMBuildRetVoid(ctx->ac.builder);
5571 else
5572 LLVMBuildRet(ctx->ac.builder, ret);
5573 }
5574
5575 /* Generate code for the hardware VS shader stage to go with a geometry shader */
5576 struct si_shader *
5577 si_generate_gs_copy_shader(struct si_screen *sscreen,
5578 LLVMTargetMachineRef tm,
5579 struct si_shader_selector *gs_selector,
5580 struct pipe_debug_callback *debug)
5581 {
5582 struct si_shader_context ctx;
5583 struct si_shader *shader;
5584 LLVMBuilderRef builder;
5585 struct lp_build_tgsi_context *bld_base = &ctx.bld_base;
5586 struct lp_build_context *uint = &bld_base->uint_bld;
5587 struct si_shader_output_values *outputs;
5588 struct tgsi_shader_info *gsinfo = &gs_selector->info;
5589 int i, r;
5590
5591 outputs = MALLOC(gsinfo->num_outputs * sizeof(outputs[0]));
5592
5593 if (!outputs)
5594 return NULL;
5595
5596 shader = CALLOC_STRUCT(si_shader);
5597 if (!shader) {
5598 FREE(outputs);
5599 return NULL;
5600 }
5601
5602 /* We can leave the fence as permanently signaled because the GS copy
5603 * shader only becomes visible globally after it has been compiled. */
5604 util_queue_fence_init(&shader->ready);
5605
5606 shader->selector = gs_selector;
5607 shader->is_gs_copy_shader = true;
5608
5609 si_init_shader_ctx(&ctx, sscreen, tm);
5610 ctx.shader = shader;
5611 ctx.type = PIPE_SHADER_VERTEX;
5612
5613 builder = ctx.ac.builder;
5614
5615 create_function(&ctx);
5616 preload_ring_buffers(&ctx);
5617
5618 LLVMValueRef voffset =
5619 lp_build_mul_imm(uint, ctx.abi.vertex_id, 4);
5620
5621 /* Fetch the vertex stream ID.*/
5622 LLVMValueRef stream_id;
5623
5624 if (gs_selector->so.num_outputs)
5625 stream_id = unpack_param(&ctx, ctx.param_streamout_config, 24, 2);
5626 else
5627 stream_id = ctx.i32_0;
5628
5629 /* Fill in output information. */
5630 for (i = 0; i < gsinfo->num_outputs; ++i) {
5631 outputs[i].semantic_name = gsinfo->output_semantic_name[i];
5632 outputs[i].semantic_index = gsinfo->output_semantic_index[i];
5633
5634 for (int chan = 0; chan < 4; chan++) {
5635 outputs[i].vertex_stream[chan] =
5636 (gsinfo->output_streams[i] >> (2 * chan)) & 3;
5637 }
5638 }
5639
5640 LLVMBasicBlockRef end_bb;
5641 LLVMValueRef switch_inst;
5642
5643 end_bb = LLVMAppendBasicBlockInContext(ctx.ac.context, ctx.main_fn, "end");
5644 switch_inst = LLVMBuildSwitch(builder, stream_id, end_bb, 4);
5645
5646 for (int stream = 0; stream < 4; stream++) {
5647 LLVMBasicBlockRef bb;
5648 unsigned offset;
5649
5650 if (!gsinfo->num_stream_output_components[stream])
5651 continue;
5652
5653 if (stream > 0 && !gs_selector->so.num_outputs)
5654 continue;
5655
5656 bb = LLVMInsertBasicBlockInContext(ctx.ac.context, end_bb, "out");
5657 LLVMAddCase(switch_inst, LLVMConstInt(ctx.i32, stream, 0), bb);
5658 LLVMPositionBuilderAtEnd(builder, bb);
5659
5660 /* Fetch vertex data from GSVS ring */
5661 offset = 0;
5662 for (i = 0; i < gsinfo->num_outputs; ++i) {
5663 for (unsigned chan = 0; chan < 4; chan++) {
5664 if (!(gsinfo->output_usagemask[i] & (1 << chan)) ||
5665 outputs[i].vertex_stream[chan] != stream) {
5666 outputs[i].values[chan] = ctx.bld_base.base.undef;
5667 continue;
5668 }
5669
5670 LLVMValueRef soffset = LLVMConstInt(ctx.i32,
5671 offset * gs_selector->gs_max_out_vertices * 16 * 4, 0);
5672 offset++;
5673
5674 outputs[i].values[chan] =
5675 ac_build_buffer_load(&ctx.ac,
5676 ctx.gsvs_ring[0], 1,
5677 ctx.i32_0, voffset,
5678 soffset, 0, 1, 1,
5679 true, false);
5680 }
5681 }
5682
5683 /* Streamout and exports. */
5684 if (gs_selector->so.num_outputs) {
5685 si_llvm_emit_streamout(&ctx, outputs,
5686 gsinfo->num_outputs,
5687 stream);
5688 }
5689
5690 if (stream == 0)
5691 si_llvm_export_vs(&ctx, outputs, gsinfo->num_outputs);
5692
5693 LLVMBuildBr(builder, end_bb);
5694 }
5695
5696 LLVMPositionBuilderAtEnd(builder, end_bb);
5697
5698 LLVMBuildRetVoid(ctx.ac.builder);
5699
5700 ctx.type = PIPE_SHADER_GEOMETRY; /* override for shader dumping */
5701 si_llvm_optimize_module(&ctx);
5702
5703 r = si_compile_llvm(sscreen, &ctx.shader->binary,
5704 &ctx.shader->config, ctx.tm,
5705 ctx.gallivm.module,
5706 debug, PIPE_SHADER_GEOMETRY,
5707 "GS Copy Shader");
5708 if (!r) {
5709 if (si_can_dump_shader(sscreen, PIPE_SHADER_GEOMETRY))
5710 fprintf(stderr, "GS Copy Shader:\n");
5711 si_shader_dump(sscreen, ctx.shader, debug,
5712 PIPE_SHADER_GEOMETRY, stderr, true);
5713 r = si_shader_binary_upload(sscreen, ctx.shader);
5714 }
5715
5716 si_llvm_dispose(&ctx);
5717
5718 FREE(outputs);
5719
5720 if (r != 0) {
5721 FREE(shader);
5722 shader = NULL;
5723 }
5724 return shader;
5725 }
5726
5727 static void si_dump_shader_key_vs(const struct si_shader_key *key,
5728 const struct si_vs_prolog_bits *prolog,
5729 const char *prefix, FILE *f)
5730 {
5731 fprintf(f, " %s.instance_divisor_is_one = %u\n",
5732 prefix, prolog->instance_divisor_is_one);
5733 fprintf(f, " %s.instance_divisor_is_fetched = %u\n",
5734 prefix, prolog->instance_divisor_is_fetched);
5735 fprintf(f, " %s.ls_vgpr_fix = %u\n",
5736 prefix, prolog->ls_vgpr_fix);
5737
5738 fprintf(f, " mono.vs.fix_fetch = {");
5739 for (int i = 0; i < SI_MAX_ATTRIBS; i++)
5740 fprintf(f, !i ? "%u" : ", %u", key->mono.vs_fix_fetch[i]);
5741 fprintf(f, "}\n");
5742 }
5743
5744 static void si_dump_shader_key(unsigned processor, const struct si_shader *shader,
5745 FILE *f)
5746 {
5747 const struct si_shader_key *key = &shader->key;
5748
5749 fprintf(f, "SHADER KEY\n");
5750
5751 switch (processor) {
5752 case PIPE_SHADER_VERTEX:
5753 si_dump_shader_key_vs(key, &key->part.vs.prolog,
5754 "part.vs.prolog", f);
5755 fprintf(f, " as_es = %u\n", key->as_es);
5756 fprintf(f, " as_ls = %u\n", key->as_ls);
5757 fprintf(f, " mono.u.vs_export_prim_id = %u\n",
5758 key->mono.u.vs_export_prim_id);
5759 break;
5760
5761 case PIPE_SHADER_TESS_CTRL:
5762 if (shader->selector->screen->info.chip_class >= GFX9) {
5763 si_dump_shader_key_vs(key, &key->part.tcs.ls_prolog,
5764 "part.tcs.ls_prolog", f);
5765 }
5766 fprintf(f, " part.tcs.epilog.prim_mode = %u\n", key->part.tcs.epilog.prim_mode);
5767 fprintf(f, " mono.u.ff_tcs_inputs_to_copy = 0x%"PRIx64"\n", key->mono.u.ff_tcs_inputs_to_copy);
5768 break;
5769
5770 case PIPE_SHADER_TESS_EVAL:
5771 fprintf(f, " as_es = %u\n", key->as_es);
5772 fprintf(f, " mono.u.vs_export_prim_id = %u\n",
5773 key->mono.u.vs_export_prim_id);
5774 break;
5775
5776 case PIPE_SHADER_GEOMETRY:
5777 if (shader->is_gs_copy_shader)
5778 break;
5779
5780 if (shader->selector->screen->info.chip_class >= GFX9 &&
5781 key->part.gs.es->type == PIPE_SHADER_VERTEX) {
5782 si_dump_shader_key_vs(key, &key->part.gs.vs_prolog,
5783 "part.gs.vs_prolog", f);
5784 }
5785 fprintf(f, " part.gs.prolog.tri_strip_adj_fix = %u\n", key->part.gs.prolog.tri_strip_adj_fix);
5786 break;
5787
5788 case PIPE_SHADER_COMPUTE:
5789 break;
5790
5791 case PIPE_SHADER_FRAGMENT:
5792 fprintf(f, " part.ps.prolog.color_two_side = %u\n", key->part.ps.prolog.color_two_side);
5793 fprintf(f, " part.ps.prolog.flatshade_colors = %u\n", key->part.ps.prolog.flatshade_colors);
5794 fprintf(f, " part.ps.prolog.poly_stipple = %u\n", key->part.ps.prolog.poly_stipple);
5795 fprintf(f, " part.ps.prolog.force_persp_sample_interp = %u\n", key->part.ps.prolog.force_persp_sample_interp);
5796 fprintf(f, " part.ps.prolog.force_linear_sample_interp = %u\n", key->part.ps.prolog.force_linear_sample_interp);
5797 fprintf(f, " part.ps.prolog.force_persp_center_interp = %u\n", key->part.ps.prolog.force_persp_center_interp);
5798 fprintf(f, " part.ps.prolog.force_linear_center_interp = %u\n", key->part.ps.prolog.force_linear_center_interp);
5799 fprintf(f, " part.ps.prolog.bc_optimize_for_persp = %u\n", key->part.ps.prolog.bc_optimize_for_persp);
5800 fprintf(f, " part.ps.prolog.bc_optimize_for_linear = %u\n", key->part.ps.prolog.bc_optimize_for_linear);
5801 fprintf(f, " part.ps.epilog.spi_shader_col_format = 0x%x\n", key->part.ps.epilog.spi_shader_col_format);
5802 fprintf(f, " part.ps.epilog.color_is_int8 = 0x%X\n", key->part.ps.epilog.color_is_int8);
5803 fprintf(f, " part.ps.epilog.color_is_int10 = 0x%X\n", key->part.ps.epilog.color_is_int10);
5804 fprintf(f, " part.ps.epilog.last_cbuf = %u\n", key->part.ps.epilog.last_cbuf);
5805 fprintf(f, " part.ps.epilog.alpha_func = %u\n", key->part.ps.epilog.alpha_func);
5806 fprintf(f, " part.ps.epilog.alpha_to_one = %u\n", key->part.ps.epilog.alpha_to_one);
5807 fprintf(f, " part.ps.epilog.poly_line_smoothing = %u\n", key->part.ps.epilog.poly_line_smoothing);
5808 fprintf(f, " part.ps.epilog.clamp_color = %u\n", key->part.ps.epilog.clamp_color);
5809 break;
5810
5811 default:
5812 assert(0);
5813 }
5814
5815 if ((processor == PIPE_SHADER_GEOMETRY ||
5816 processor == PIPE_SHADER_TESS_EVAL ||
5817 processor == PIPE_SHADER_VERTEX) &&
5818 !key->as_es && !key->as_ls) {
5819 fprintf(f, " opt.kill_outputs = 0x%"PRIx64"\n", key->opt.kill_outputs);
5820 fprintf(f, " opt.clip_disable = %u\n", key->opt.clip_disable);
5821 }
5822 }
5823
5824 static void si_init_shader_ctx(struct si_shader_context *ctx,
5825 struct si_screen *sscreen,
5826 LLVMTargetMachineRef tm)
5827 {
5828 struct lp_build_tgsi_context *bld_base;
5829
5830 si_llvm_context_init(ctx, sscreen, tm);
5831
5832 bld_base = &ctx->bld_base;
5833 bld_base->emit_fetch_funcs[TGSI_FILE_CONSTANT] = fetch_constant;
5834
5835 bld_base->op_actions[TGSI_OPCODE_INTERP_CENTROID] = interp_action;
5836 bld_base->op_actions[TGSI_OPCODE_INTERP_SAMPLE] = interp_action;
5837 bld_base->op_actions[TGSI_OPCODE_INTERP_OFFSET] = interp_action;
5838
5839 bld_base->op_actions[TGSI_OPCODE_MEMBAR].emit = membar_emit;
5840
5841 bld_base->op_actions[TGSI_OPCODE_CLOCK].emit = clock_emit;
5842
5843 bld_base->op_actions[TGSI_OPCODE_DDX].emit = si_llvm_emit_ddxy;
5844 bld_base->op_actions[TGSI_OPCODE_DDY].emit = si_llvm_emit_ddxy;
5845 bld_base->op_actions[TGSI_OPCODE_DDX_FINE].emit = si_llvm_emit_ddxy;
5846 bld_base->op_actions[TGSI_OPCODE_DDY_FINE].emit = si_llvm_emit_ddxy;
5847
5848 bld_base->op_actions[TGSI_OPCODE_VOTE_ALL].emit = vote_all_emit;
5849 bld_base->op_actions[TGSI_OPCODE_VOTE_ANY].emit = vote_any_emit;
5850 bld_base->op_actions[TGSI_OPCODE_VOTE_EQ].emit = vote_eq_emit;
5851 bld_base->op_actions[TGSI_OPCODE_BALLOT].emit = ballot_emit;
5852 bld_base->op_actions[TGSI_OPCODE_READ_FIRST].intr_name = "llvm.amdgcn.readfirstlane";
5853 bld_base->op_actions[TGSI_OPCODE_READ_FIRST].emit = read_lane_emit;
5854 bld_base->op_actions[TGSI_OPCODE_READ_INVOC].intr_name = "llvm.amdgcn.readlane";
5855 bld_base->op_actions[TGSI_OPCODE_READ_INVOC].fetch_args = read_invoc_fetch_args;
5856 bld_base->op_actions[TGSI_OPCODE_READ_INVOC].emit = read_lane_emit;
5857
5858 bld_base->op_actions[TGSI_OPCODE_EMIT].emit = si_tgsi_emit_vertex;
5859 bld_base->op_actions[TGSI_OPCODE_ENDPRIM].emit = si_tgsi_emit_primitive;
5860 bld_base->op_actions[TGSI_OPCODE_BARRIER].emit = si_llvm_emit_barrier;
5861 }
5862
5863 static void si_optimize_vs_outputs(struct si_shader_context *ctx)
5864 {
5865 struct si_shader *shader = ctx->shader;
5866 struct tgsi_shader_info *info = &shader->selector->info;
5867
5868 if ((ctx->type != PIPE_SHADER_VERTEX &&
5869 ctx->type != PIPE_SHADER_TESS_EVAL) ||
5870 shader->key.as_ls ||
5871 shader->key.as_es)
5872 return;
5873
5874 ac_optimize_vs_outputs(&ctx->ac,
5875 ctx->main_fn,
5876 shader->info.vs_output_param_offset,
5877 info->num_outputs,
5878 &shader->info.nr_param_exports);
5879 }
5880
5881 static void si_count_scratch_private_memory(struct si_shader_context *ctx)
5882 {
5883 ctx->shader->config.private_mem_vgprs = 0;
5884
5885 /* Process all LLVM instructions. */
5886 LLVMBasicBlockRef bb = LLVMGetFirstBasicBlock(ctx->main_fn);
5887 while (bb) {
5888 LLVMValueRef next = LLVMGetFirstInstruction(bb);
5889
5890 while (next) {
5891 LLVMValueRef inst = next;
5892 next = LLVMGetNextInstruction(next);
5893
5894 if (LLVMGetInstructionOpcode(inst) != LLVMAlloca)
5895 continue;
5896
5897 LLVMTypeRef type = LLVMGetElementType(LLVMTypeOf(inst));
5898 /* No idea why LLVM aligns allocas to 4 elements. */
5899 unsigned alignment = LLVMGetAlignment(inst);
5900 unsigned dw_size = align(ac_get_type_size(type) / 4, alignment);
5901 ctx->shader->config.private_mem_vgprs += dw_size;
5902 }
5903 bb = LLVMGetNextBasicBlock(bb);
5904 }
5905 }
5906
5907 static void si_init_exec_from_input(struct si_shader_context *ctx,
5908 unsigned param, unsigned bitoffset)
5909 {
5910 LLVMValueRef args[] = {
5911 LLVMGetParam(ctx->main_fn, param),
5912 LLVMConstInt(ctx->i32, bitoffset, 0),
5913 };
5914 lp_build_intrinsic(ctx->ac.builder,
5915 "llvm.amdgcn.init.exec.from.input",
5916 ctx->voidt, args, 2, LP_FUNC_ATTR_CONVERGENT);
5917 }
5918
5919 static bool si_vs_needs_prolog(const struct si_shader_selector *sel,
5920 const struct si_vs_prolog_bits *key)
5921 {
5922 /* VGPR initialization fixup for Vega10 and Raven is always done in the
5923 * VS prolog. */
5924 return sel->vs_needs_prolog || key->ls_vgpr_fix;
5925 }
5926
5927 static bool si_compile_tgsi_main(struct si_shader_context *ctx,
5928 bool is_monolithic)
5929 {
5930 struct si_shader *shader = ctx->shader;
5931 struct si_shader_selector *sel = shader->selector;
5932 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
5933
5934 // TODO clean all this up!
5935 switch (ctx->type) {
5936 case PIPE_SHADER_VERTEX:
5937 ctx->load_input = declare_input_vs;
5938 if (shader->key.as_ls)
5939 ctx->abi.emit_outputs = si_llvm_emit_ls_epilogue;
5940 else if (shader->key.as_es)
5941 ctx->abi.emit_outputs = si_llvm_emit_es_epilogue;
5942 else
5943 ctx->abi.emit_outputs = si_llvm_emit_vs_epilogue;
5944 bld_base->emit_epilogue = si_tgsi_emit_epilogue;
5945 break;
5946 case PIPE_SHADER_TESS_CTRL:
5947 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_tcs;
5948 ctx->abi.load_tess_varyings = si_nir_load_tcs_varyings;
5949 bld_base->emit_fetch_funcs[TGSI_FILE_OUTPUT] = fetch_output_tcs;
5950 bld_base->emit_store = store_output_tcs;
5951 ctx->abi.store_tcs_outputs = si_nir_store_output_tcs;
5952 ctx->abi.emit_outputs = si_llvm_emit_tcs_epilogue;
5953 ctx->abi.load_patch_vertices_in = si_load_patch_vertices_in;
5954 bld_base->emit_epilogue = si_tgsi_emit_epilogue;
5955 break;
5956 case PIPE_SHADER_TESS_EVAL:
5957 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_tes;
5958 ctx->abi.load_tess_varyings = si_nir_load_input_tes;
5959 ctx->abi.load_tess_coord = si_load_tess_coord;
5960 ctx->abi.load_tess_level = si_load_tess_level;
5961 ctx->abi.load_patch_vertices_in = si_load_patch_vertices_in;
5962 if (shader->key.as_es)
5963 ctx->abi.emit_outputs = si_llvm_emit_es_epilogue;
5964 else
5965 ctx->abi.emit_outputs = si_llvm_emit_vs_epilogue;
5966 bld_base->emit_epilogue = si_tgsi_emit_epilogue;
5967 break;
5968 case PIPE_SHADER_GEOMETRY:
5969 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_gs;
5970 ctx->abi.load_inputs = si_nir_load_input_gs;
5971 ctx->abi.emit_vertex = si_llvm_emit_vertex;
5972 ctx->abi.emit_primitive = si_llvm_emit_primitive;
5973 ctx->abi.emit_outputs = si_llvm_emit_gs_epilogue;
5974 bld_base->emit_epilogue = si_tgsi_emit_gs_epilogue;
5975 break;
5976 case PIPE_SHADER_FRAGMENT:
5977 ctx->load_input = declare_input_fs;
5978 ctx->abi.emit_outputs = si_llvm_return_fs_outputs;
5979 bld_base->emit_epilogue = si_tgsi_emit_epilogue;
5980 ctx->abi.lookup_interp_param = si_nir_lookup_interp_param;
5981 ctx->abi.load_sample_position = load_sample_position;
5982 break;
5983 case PIPE_SHADER_COMPUTE:
5984 ctx->abi.load_local_group_size = get_block_size;
5985 break;
5986 default:
5987 assert(!"Unsupported shader type");
5988 return false;
5989 }
5990
5991 ctx->abi.load_ubo = load_ubo;
5992 ctx->abi.load_ssbo = load_ssbo;
5993
5994 create_function(ctx);
5995 preload_ring_buffers(ctx);
5996
5997 /* For GFX9 merged shaders:
5998 * - Set EXEC for the first shader. If the prolog is present, set
5999 * EXEC there instead.
6000 * - Add a barrier before the second shader.
6001 * - In the second shader, reset EXEC to ~0 and wrap the main part in
6002 * an if-statement. This is required for correctness in geometry
6003 * shaders, to ensure that empty GS waves do not send GS_EMIT and
6004 * GS_CUT messages.
6005 *
6006 * For monolithic merged shaders, the first shader is wrapped in an
6007 * if-block together with its prolog in si_build_wrapper_function.
6008 */
6009 if (ctx->screen->info.chip_class >= GFX9) {
6010 if (!is_monolithic &&
6011 sel->info.num_instructions > 1 && /* not empty shader */
6012 (shader->key.as_es || shader->key.as_ls) &&
6013 (ctx->type == PIPE_SHADER_TESS_EVAL ||
6014 (ctx->type == PIPE_SHADER_VERTEX &&
6015 !si_vs_needs_prolog(sel, &shader->key.part.vs.prolog)))) {
6016 si_init_exec_from_input(ctx,
6017 ctx->param_merged_wave_info, 0);
6018 } else if (ctx->type == PIPE_SHADER_TESS_CTRL ||
6019 ctx->type == PIPE_SHADER_GEOMETRY) {
6020 if (!is_monolithic)
6021 ac_init_exec_full_mask(&ctx->ac);
6022
6023 /* The barrier must execute for all shaders in a
6024 * threadgroup.
6025 */
6026 si_llvm_emit_barrier(NULL, bld_base, NULL);
6027
6028 LLVMValueRef num_threads = unpack_param(ctx, ctx->param_merged_wave_info, 8, 8);
6029 LLVMValueRef ena =
6030 LLVMBuildICmp(ctx->ac.builder, LLVMIntULT,
6031 ac_get_thread_id(&ctx->ac), num_threads, "");
6032 lp_build_if(&ctx->merged_wrap_if_state, &ctx->gallivm, ena);
6033 }
6034 }
6035
6036 if (ctx->type == PIPE_SHADER_TESS_CTRL &&
6037 sel->tcs_info.tessfactors_are_def_in_all_invocs) {
6038 for (unsigned i = 0; i < 6; i++) {
6039 ctx->invoc0_tess_factors[i] =
6040 lp_build_alloca_undef(&ctx->gallivm, ctx->i32, "");
6041 }
6042 }
6043
6044 if (ctx->type == PIPE_SHADER_GEOMETRY) {
6045 int i;
6046 for (i = 0; i < 4; i++) {
6047 ctx->gs_next_vertex[i] =
6048 lp_build_alloca(&ctx->gallivm,
6049 ctx->i32, "");
6050 }
6051 }
6052
6053 if (sel->force_correct_derivs_after_kill) {
6054 ctx->postponed_kill = lp_build_alloca_undef(&ctx->gallivm, ctx->i1, "");
6055 /* true = don't kill. */
6056 LLVMBuildStore(ctx->ac.builder, LLVMConstInt(ctx->i1, 1, 0),
6057 ctx->postponed_kill);
6058 }
6059
6060 if (sel->tokens) {
6061 if (!lp_build_tgsi_llvm(bld_base, sel->tokens)) {
6062 fprintf(stderr, "Failed to translate shader from TGSI to LLVM\n");
6063 return false;
6064 }
6065 } else {
6066 if (!si_nir_build_llvm(ctx, sel->nir)) {
6067 fprintf(stderr, "Failed to translate shader from NIR to LLVM\n");
6068 return false;
6069 }
6070 }
6071
6072 si_llvm_build_ret(ctx, ctx->return_value);
6073 return true;
6074 }
6075
6076 /**
6077 * Compute the VS prolog key, which contains all the information needed to
6078 * build the VS prolog function, and set shader->info bits where needed.
6079 *
6080 * \param info Shader info of the vertex shader.
6081 * \param num_input_sgprs Number of input SGPRs for the vertex shader.
6082 * \param prolog_key Key of the VS prolog
6083 * \param shader_out The vertex shader, or the next shader if merging LS+HS or ES+GS.
6084 * \param key Output shader part key.
6085 */
6086 static void si_get_vs_prolog_key(const struct tgsi_shader_info *info,
6087 unsigned num_input_sgprs,
6088 const struct si_vs_prolog_bits *prolog_key,
6089 struct si_shader *shader_out,
6090 union si_shader_part_key *key)
6091 {
6092 memset(key, 0, sizeof(*key));
6093 key->vs_prolog.states = *prolog_key;
6094 key->vs_prolog.num_input_sgprs = num_input_sgprs;
6095 key->vs_prolog.last_input = MAX2(1, info->num_inputs) - 1;
6096 key->vs_prolog.as_ls = shader_out->key.as_ls;
6097 key->vs_prolog.as_es = shader_out->key.as_es;
6098
6099 if (shader_out->selector->type == PIPE_SHADER_TESS_CTRL) {
6100 key->vs_prolog.as_ls = 1;
6101 key->vs_prolog.num_merged_next_stage_vgprs = 2;
6102 } else if (shader_out->selector->type == PIPE_SHADER_GEOMETRY) {
6103 key->vs_prolog.as_es = 1;
6104 key->vs_prolog.num_merged_next_stage_vgprs = 5;
6105 }
6106
6107 /* Enable loading the InstanceID VGPR. */
6108 uint16_t input_mask = u_bit_consecutive(0, info->num_inputs);
6109
6110 if ((key->vs_prolog.states.instance_divisor_is_one |
6111 key->vs_prolog.states.instance_divisor_is_fetched) & input_mask)
6112 shader_out->info.uses_instanceid = true;
6113 }
6114
6115 /**
6116 * Compute the PS prolog key, which contains all the information needed to
6117 * build the PS prolog function, and set related bits in shader->config.
6118 */
6119 static void si_get_ps_prolog_key(struct si_shader *shader,
6120 union si_shader_part_key *key,
6121 bool separate_prolog)
6122 {
6123 struct tgsi_shader_info *info = &shader->selector->info;
6124
6125 memset(key, 0, sizeof(*key));
6126 key->ps_prolog.states = shader->key.part.ps.prolog;
6127 key->ps_prolog.colors_read = info->colors_read;
6128 key->ps_prolog.num_input_sgprs = shader->info.num_input_sgprs;
6129 key->ps_prolog.num_input_vgprs = shader->info.num_input_vgprs;
6130 key->ps_prolog.wqm = info->uses_derivatives &&
6131 (key->ps_prolog.colors_read ||
6132 key->ps_prolog.states.force_persp_sample_interp ||
6133 key->ps_prolog.states.force_linear_sample_interp ||
6134 key->ps_prolog.states.force_persp_center_interp ||
6135 key->ps_prolog.states.force_linear_center_interp ||
6136 key->ps_prolog.states.bc_optimize_for_persp ||
6137 key->ps_prolog.states.bc_optimize_for_linear);
6138 key->ps_prolog.ancillary_vgpr_index = shader->info.ancillary_vgpr_index;
6139
6140 if (info->colors_read) {
6141 unsigned *color = shader->selector->color_attr_index;
6142
6143 if (shader->key.part.ps.prolog.color_two_side) {
6144 /* BCOLORs are stored after the last input. */
6145 key->ps_prolog.num_interp_inputs = info->num_inputs;
6146 key->ps_prolog.face_vgpr_index = shader->info.face_vgpr_index;
6147 shader->config.spi_ps_input_ena |= S_0286CC_FRONT_FACE_ENA(1);
6148 }
6149
6150 for (unsigned i = 0; i < 2; i++) {
6151 unsigned interp = info->input_interpolate[color[i]];
6152 unsigned location = info->input_interpolate_loc[color[i]];
6153
6154 if (!(info->colors_read & (0xf << i*4)))
6155 continue;
6156
6157 key->ps_prolog.color_attr_index[i] = color[i];
6158
6159 if (shader->key.part.ps.prolog.flatshade_colors &&
6160 interp == TGSI_INTERPOLATE_COLOR)
6161 interp = TGSI_INTERPOLATE_CONSTANT;
6162
6163 switch (interp) {
6164 case TGSI_INTERPOLATE_CONSTANT:
6165 key->ps_prolog.color_interp_vgpr_index[i] = -1;
6166 break;
6167 case TGSI_INTERPOLATE_PERSPECTIVE:
6168 case TGSI_INTERPOLATE_COLOR:
6169 /* Force the interpolation location for colors here. */
6170 if (shader->key.part.ps.prolog.force_persp_sample_interp)
6171 location = TGSI_INTERPOLATE_LOC_SAMPLE;
6172 if (shader->key.part.ps.prolog.force_persp_center_interp)
6173 location = TGSI_INTERPOLATE_LOC_CENTER;
6174
6175 switch (location) {
6176 case TGSI_INTERPOLATE_LOC_SAMPLE:
6177 key->ps_prolog.color_interp_vgpr_index[i] = 0;
6178 shader->config.spi_ps_input_ena |=
6179 S_0286CC_PERSP_SAMPLE_ENA(1);
6180 break;
6181 case TGSI_INTERPOLATE_LOC_CENTER:
6182 key->ps_prolog.color_interp_vgpr_index[i] = 2;
6183 shader->config.spi_ps_input_ena |=
6184 S_0286CC_PERSP_CENTER_ENA(1);
6185 break;
6186 case TGSI_INTERPOLATE_LOC_CENTROID:
6187 key->ps_prolog.color_interp_vgpr_index[i] = 4;
6188 shader->config.spi_ps_input_ena |=
6189 S_0286CC_PERSP_CENTROID_ENA(1);
6190 break;
6191 default:
6192 assert(0);
6193 }
6194 break;
6195 case TGSI_INTERPOLATE_LINEAR:
6196 /* Force the interpolation location for colors here. */
6197 if (shader->key.part.ps.prolog.force_linear_sample_interp)
6198 location = TGSI_INTERPOLATE_LOC_SAMPLE;
6199 if (shader->key.part.ps.prolog.force_linear_center_interp)
6200 location = TGSI_INTERPOLATE_LOC_CENTER;
6201
6202 /* The VGPR assignment for non-monolithic shaders
6203 * works because InitialPSInputAddr is set on the
6204 * main shader and PERSP_PULL_MODEL is never used.
6205 */
6206 switch (location) {
6207 case TGSI_INTERPOLATE_LOC_SAMPLE:
6208 key->ps_prolog.color_interp_vgpr_index[i] =
6209 separate_prolog ? 6 : 9;
6210 shader->config.spi_ps_input_ena |=
6211 S_0286CC_LINEAR_SAMPLE_ENA(1);
6212 break;
6213 case TGSI_INTERPOLATE_LOC_CENTER:
6214 key->ps_prolog.color_interp_vgpr_index[i] =
6215 separate_prolog ? 8 : 11;
6216 shader->config.spi_ps_input_ena |=
6217 S_0286CC_LINEAR_CENTER_ENA(1);
6218 break;
6219 case TGSI_INTERPOLATE_LOC_CENTROID:
6220 key->ps_prolog.color_interp_vgpr_index[i] =
6221 separate_prolog ? 10 : 13;
6222 shader->config.spi_ps_input_ena |=
6223 S_0286CC_LINEAR_CENTROID_ENA(1);
6224 break;
6225 default:
6226 assert(0);
6227 }
6228 break;
6229 default:
6230 assert(0);
6231 }
6232 }
6233 }
6234 }
6235
6236 /**
6237 * Check whether a PS prolog is required based on the key.
6238 */
6239 static bool si_need_ps_prolog(const union si_shader_part_key *key)
6240 {
6241 return key->ps_prolog.colors_read ||
6242 key->ps_prolog.states.force_persp_sample_interp ||
6243 key->ps_prolog.states.force_linear_sample_interp ||
6244 key->ps_prolog.states.force_persp_center_interp ||
6245 key->ps_prolog.states.force_linear_center_interp ||
6246 key->ps_prolog.states.bc_optimize_for_persp ||
6247 key->ps_prolog.states.bc_optimize_for_linear ||
6248 key->ps_prolog.states.poly_stipple ||
6249 key->ps_prolog.states.samplemask_log_ps_iter;
6250 }
6251
6252 /**
6253 * Compute the PS epilog key, which contains all the information needed to
6254 * build the PS epilog function.
6255 */
6256 static void si_get_ps_epilog_key(struct si_shader *shader,
6257 union si_shader_part_key *key)
6258 {
6259 struct tgsi_shader_info *info = &shader->selector->info;
6260 memset(key, 0, sizeof(*key));
6261 key->ps_epilog.colors_written = info->colors_written;
6262 key->ps_epilog.writes_z = info->writes_z;
6263 key->ps_epilog.writes_stencil = info->writes_stencil;
6264 key->ps_epilog.writes_samplemask = info->writes_samplemask;
6265 key->ps_epilog.states = shader->key.part.ps.epilog;
6266 }
6267
6268 /**
6269 * Build the GS prolog function. Rotate the input vertices for triangle strips
6270 * with adjacency.
6271 */
6272 static void si_build_gs_prolog_function(struct si_shader_context *ctx,
6273 union si_shader_part_key *key)
6274 {
6275 unsigned num_sgprs, num_vgprs;
6276 struct si_function_info fninfo;
6277 LLVMBuilderRef builder = ctx->ac.builder;
6278 LLVMTypeRef returns[48];
6279 LLVMValueRef func, ret;
6280
6281 si_init_function_info(&fninfo);
6282
6283 if (ctx->screen->info.chip_class >= GFX9) {
6284 num_sgprs = 8 + GFX9_GS_NUM_USER_SGPR;
6285 num_vgprs = 5; /* ES inputs are not needed by GS */
6286 } else {
6287 num_sgprs = GFX6_GS_NUM_USER_SGPR + 2;
6288 num_vgprs = 8;
6289 }
6290
6291 for (unsigned i = 0; i < num_sgprs; ++i) {
6292 add_arg(&fninfo, ARG_SGPR, ctx->i32);
6293 returns[i] = ctx->i32;
6294 }
6295
6296 for (unsigned i = 0; i < num_vgprs; ++i) {
6297 add_arg(&fninfo, ARG_VGPR, ctx->i32);
6298 returns[num_sgprs + i] = ctx->f32;
6299 }
6300
6301 /* Create the function. */
6302 si_create_function(ctx, "gs_prolog", returns, num_sgprs + num_vgprs,
6303 &fninfo, 0);
6304 func = ctx->main_fn;
6305
6306 /* Set the full EXEC mask for the prolog, because we are only fiddling
6307 * with registers here. The main shader part will set the correct EXEC
6308 * mask.
6309 */
6310 if (ctx->screen->info.chip_class >= GFX9 && !key->gs_prolog.is_monolithic)
6311 ac_init_exec_full_mask(&ctx->ac);
6312
6313 /* Copy inputs to outputs. This should be no-op, as the registers match,
6314 * but it will prevent the compiler from overwriting them unintentionally.
6315 */
6316 ret = ctx->return_value;
6317 for (unsigned i = 0; i < num_sgprs; i++) {
6318 LLVMValueRef p = LLVMGetParam(func, i);
6319 ret = LLVMBuildInsertValue(builder, ret, p, i, "");
6320 }
6321 for (unsigned i = 0; i < num_vgprs; i++) {
6322 LLVMValueRef p = LLVMGetParam(func, num_sgprs + i);
6323 p = ac_to_float(&ctx->ac, p);
6324 ret = LLVMBuildInsertValue(builder, ret, p, num_sgprs + i, "");
6325 }
6326
6327 if (key->gs_prolog.states.tri_strip_adj_fix) {
6328 /* Remap the input vertices for every other primitive. */
6329 const unsigned gfx6_vtx_params[6] = {
6330 num_sgprs,
6331 num_sgprs + 1,
6332 num_sgprs + 3,
6333 num_sgprs + 4,
6334 num_sgprs + 5,
6335 num_sgprs + 6
6336 };
6337 const unsigned gfx9_vtx_params[3] = {
6338 num_sgprs,
6339 num_sgprs + 1,
6340 num_sgprs + 4,
6341 };
6342 LLVMValueRef vtx_in[6], vtx_out[6];
6343 LLVMValueRef prim_id, rotate;
6344
6345 if (ctx->screen->info.chip_class >= GFX9) {
6346 for (unsigned i = 0; i < 3; i++) {
6347 vtx_in[i*2] = unpack_param(ctx, gfx9_vtx_params[i], 0, 16);
6348 vtx_in[i*2+1] = unpack_param(ctx, gfx9_vtx_params[i], 16, 16);
6349 }
6350 } else {
6351 for (unsigned i = 0; i < 6; i++)
6352 vtx_in[i] = LLVMGetParam(func, gfx6_vtx_params[i]);
6353 }
6354
6355 prim_id = LLVMGetParam(func, num_sgprs + 2);
6356 rotate = LLVMBuildTrunc(builder, prim_id, ctx->i1, "");
6357
6358 for (unsigned i = 0; i < 6; ++i) {
6359 LLVMValueRef base, rotated;
6360 base = vtx_in[i];
6361 rotated = vtx_in[(i + 4) % 6];
6362 vtx_out[i] = LLVMBuildSelect(builder, rotate, rotated, base, "");
6363 }
6364
6365 if (ctx->screen->info.chip_class >= GFX9) {
6366 for (unsigned i = 0; i < 3; i++) {
6367 LLVMValueRef hi, out;
6368
6369 hi = LLVMBuildShl(builder, vtx_out[i*2+1],
6370 LLVMConstInt(ctx->i32, 16, 0), "");
6371 out = LLVMBuildOr(builder, vtx_out[i*2], hi, "");
6372 out = ac_to_float(&ctx->ac, out);
6373 ret = LLVMBuildInsertValue(builder, ret, out,
6374 gfx9_vtx_params[i], "");
6375 }
6376 } else {
6377 for (unsigned i = 0; i < 6; i++) {
6378 LLVMValueRef out;
6379
6380 out = ac_to_float(&ctx->ac, vtx_out[i]);
6381 ret = LLVMBuildInsertValue(builder, ret, out,
6382 gfx6_vtx_params[i], "");
6383 }
6384 }
6385 }
6386
6387 LLVMBuildRet(builder, ret);
6388 }
6389
6390 /**
6391 * Given a list of shader part functions, build a wrapper function that
6392 * runs them in sequence to form a monolithic shader.
6393 */
6394 static void si_build_wrapper_function(struct si_shader_context *ctx,
6395 LLVMValueRef *parts,
6396 unsigned num_parts,
6397 unsigned main_part,
6398 unsigned next_shader_first_part)
6399 {
6400 LLVMBuilderRef builder = ctx->ac.builder;
6401 /* PS epilog has one arg per color component; gfx9 merged shader
6402 * prologs need to forward 32 user SGPRs.
6403 */
6404 struct si_function_info fninfo;
6405 LLVMValueRef initial[64], out[64];
6406 LLVMTypeRef function_type;
6407 unsigned num_first_params;
6408 unsigned num_out, initial_num_out;
6409 MAYBE_UNUSED unsigned num_out_sgpr; /* used in debug checks */
6410 MAYBE_UNUSED unsigned initial_num_out_sgpr; /* used in debug checks */
6411 unsigned num_sgprs, num_vgprs;
6412 unsigned gprs;
6413 struct lp_build_if_state if_state;
6414
6415 si_init_function_info(&fninfo);
6416
6417 for (unsigned i = 0; i < num_parts; ++i) {
6418 lp_add_function_attr(parts[i], -1, LP_FUNC_ATTR_ALWAYSINLINE);
6419 LLVMSetLinkage(parts[i], LLVMPrivateLinkage);
6420 }
6421
6422 /* The parameters of the wrapper function correspond to those of the
6423 * first part in terms of SGPRs and VGPRs, but we use the types of the
6424 * main part to get the right types. This is relevant for the
6425 * dereferenceable attribute on descriptor table pointers.
6426 */
6427 num_sgprs = 0;
6428 num_vgprs = 0;
6429
6430 function_type = LLVMGetElementType(LLVMTypeOf(parts[0]));
6431 num_first_params = LLVMCountParamTypes(function_type);
6432
6433 for (unsigned i = 0; i < num_first_params; ++i) {
6434 LLVMValueRef param = LLVMGetParam(parts[0], i);
6435
6436 if (ac_is_sgpr_param(param)) {
6437 assert(num_vgprs == 0);
6438 num_sgprs += ac_get_type_size(LLVMTypeOf(param)) / 4;
6439 } else {
6440 num_vgprs += ac_get_type_size(LLVMTypeOf(param)) / 4;
6441 }
6442 }
6443
6444 gprs = 0;
6445 while (gprs < num_sgprs + num_vgprs) {
6446 LLVMValueRef param = LLVMGetParam(parts[main_part], fninfo.num_params);
6447 LLVMTypeRef type = LLVMTypeOf(param);
6448 unsigned size = ac_get_type_size(type) / 4;
6449
6450 add_arg(&fninfo, gprs < num_sgprs ? ARG_SGPR : ARG_VGPR, type);
6451
6452 assert(ac_is_sgpr_param(param) == (gprs < num_sgprs));
6453 assert(gprs + size <= num_sgprs + num_vgprs &&
6454 (gprs >= num_sgprs || gprs + size <= num_sgprs));
6455
6456 gprs += size;
6457 }
6458
6459 si_create_function(ctx, "wrapper", NULL, 0, &fninfo,
6460 si_get_max_workgroup_size(ctx->shader));
6461
6462 if (is_merged_shader(ctx->shader))
6463 ac_init_exec_full_mask(&ctx->ac);
6464
6465 /* Record the arguments of the function as if they were an output of
6466 * a previous part.
6467 */
6468 num_out = 0;
6469 num_out_sgpr = 0;
6470
6471 for (unsigned i = 0; i < fninfo.num_params; ++i) {
6472 LLVMValueRef param = LLVMGetParam(ctx->main_fn, i);
6473 LLVMTypeRef param_type = LLVMTypeOf(param);
6474 LLVMTypeRef out_type = i < fninfo.num_sgpr_params ? ctx->i32 : ctx->f32;
6475 unsigned size = ac_get_type_size(param_type) / 4;
6476
6477 if (size == 1) {
6478 if (param_type != out_type)
6479 param = LLVMBuildBitCast(builder, param, out_type, "");
6480 out[num_out++] = param;
6481 } else {
6482 LLVMTypeRef vector_type = LLVMVectorType(out_type, size);
6483
6484 if (LLVMGetTypeKind(param_type) == LLVMPointerTypeKind) {
6485 param = LLVMBuildPtrToInt(builder, param, ctx->i64, "");
6486 param_type = ctx->i64;
6487 }
6488
6489 if (param_type != vector_type)
6490 param = LLVMBuildBitCast(builder, param, vector_type, "");
6491
6492 for (unsigned j = 0; j < size; ++j)
6493 out[num_out++] = LLVMBuildExtractElement(
6494 builder, param, LLVMConstInt(ctx->i32, j, 0), "");
6495 }
6496
6497 if (i < fninfo.num_sgpr_params)
6498 num_out_sgpr = num_out;
6499 }
6500
6501 memcpy(initial, out, sizeof(out));
6502 initial_num_out = num_out;
6503 initial_num_out_sgpr = num_out_sgpr;
6504
6505 /* Now chain the parts. */
6506 for (unsigned part = 0; part < num_parts; ++part) {
6507 LLVMValueRef in[48];
6508 LLVMValueRef ret;
6509 LLVMTypeRef ret_type;
6510 unsigned out_idx = 0;
6511 unsigned num_params = LLVMCountParams(parts[part]);
6512
6513 /* Merged shaders are executed conditionally depending
6514 * on the number of enabled threads passed in the input SGPRs. */
6515 if (is_merged_shader(ctx->shader) && part == 0) {
6516 LLVMValueRef ena, count = initial[3];
6517
6518 count = LLVMBuildAnd(builder, count,
6519 LLVMConstInt(ctx->i32, 0x7f, 0), "");
6520 ena = LLVMBuildICmp(builder, LLVMIntULT,
6521 ac_get_thread_id(&ctx->ac), count, "");
6522 lp_build_if(&if_state, &ctx->gallivm, ena);
6523 }
6524
6525 /* Derive arguments for the next part from outputs of the
6526 * previous one.
6527 */
6528 for (unsigned param_idx = 0; param_idx < num_params; ++param_idx) {
6529 LLVMValueRef param;
6530 LLVMTypeRef param_type;
6531 bool is_sgpr;
6532 unsigned param_size;
6533 LLVMValueRef arg = NULL;
6534
6535 param = LLVMGetParam(parts[part], param_idx);
6536 param_type = LLVMTypeOf(param);
6537 param_size = ac_get_type_size(param_type) / 4;
6538 is_sgpr = ac_is_sgpr_param(param);
6539
6540 if (is_sgpr)
6541 lp_add_function_attr(parts[part], param_idx + 1, LP_FUNC_ATTR_INREG);
6542
6543 assert(out_idx + param_size <= (is_sgpr ? num_out_sgpr : num_out));
6544 assert(is_sgpr || out_idx >= num_out_sgpr);
6545
6546 if (param_size == 1)
6547 arg = out[out_idx];
6548 else
6549 arg = lp_build_gather_values(&ctx->gallivm, &out[out_idx], param_size);
6550
6551 if (LLVMTypeOf(arg) != param_type) {
6552 if (LLVMGetTypeKind(param_type) == LLVMPointerTypeKind) {
6553 arg = LLVMBuildBitCast(builder, arg, ctx->i64, "");
6554 arg = LLVMBuildIntToPtr(builder, arg, param_type, "");
6555 } else {
6556 arg = LLVMBuildBitCast(builder, arg, param_type, "");
6557 }
6558 }
6559
6560 in[param_idx] = arg;
6561 out_idx += param_size;
6562 }
6563
6564 ret = LLVMBuildCall(builder, parts[part], in, num_params, "");
6565
6566 if (is_merged_shader(ctx->shader) &&
6567 part + 1 == next_shader_first_part) {
6568 lp_build_endif(&if_state);
6569
6570 /* The second half of the merged shader should use
6571 * the inputs from the toplevel (wrapper) function,
6572 * not the return value from the last call.
6573 *
6574 * That's because the last call was executed condi-
6575 * tionally, so we can't consume it in the main
6576 * block.
6577 */
6578 memcpy(out, initial, sizeof(initial));
6579 num_out = initial_num_out;
6580 num_out_sgpr = initial_num_out_sgpr;
6581 continue;
6582 }
6583
6584 /* Extract the returned GPRs. */
6585 ret_type = LLVMTypeOf(ret);
6586 num_out = 0;
6587 num_out_sgpr = 0;
6588
6589 if (LLVMGetTypeKind(ret_type) != LLVMVoidTypeKind) {
6590 assert(LLVMGetTypeKind(ret_type) == LLVMStructTypeKind);
6591
6592 unsigned ret_size = LLVMCountStructElementTypes(ret_type);
6593
6594 for (unsigned i = 0; i < ret_size; ++i) {
6595 LLVMValueRef val =
6596 LLVMBuildExtractValue(builder, ret, i, "");
6597
6598 assert(num_out < ARRAY_SIZE(out));
6599 out[num_out++] = val;
6600
6601 if (LLVMTypeOf(val) == ctx->i32) {
6602 assert(num_out_sgpr + 1 == num_out);
6603 num_out_sgpr = num_out;
6604 }
6605 }
6606 }
6607 }
6608
6609 LLVMBuildRetVoid(builder);
6610 }
6611
6612 int si_compile_tgsi_shader(struct si_screen *sscreen,
6613 LLVMTargetMachineRef tm,
6614 struct si_shader *shader,
6615 bool is_monolithic,
6616 struct pipe_debug_callback *debug)
6617 {
6618 struct si_shader_selector *sel = shader->selector;
6619 struct si_shader_context ctx;
6620 int r = -1;
6621
6622 /* Dump TGSI code before doing TGSI->LLVM conversion in case the
6623 * conversion fails. */
6624 if (si_can_dump_shader(sscreen, sel->info.processor) &&
6625 !(sscreen->debug_flags & DBG(NO_TGSI))) {
6626 if (sel->tokens)
6627 tgsi_dump(sel->tokens, 0);
6628 else
6629 nir_print_shader(sel->nir, stderr);
6630 si_dump_streamout(&sel->so);
6631 }
6632
6633 si_init_shader_ctx(&ctx, sscreen, tm);
6634 si_llvm_context_set_tgsi(&ctx, shader);
6635 ctx.separate_prolog = !is_monolithic;
6636
6637 memset(shader->info.vs_output_param_offset, AC_EXP_PARAM_UNDEFINED,
6638 sizeof(shader->info.vs_output_param_offset));
6639
6640 shader->info.uses_instanceid = sel->info.uses_instanceid;
6641
6642 if (!si_compile_tgsi_main(&ctx, is_monolithic)) {
6643 si_llvm_dispose(&ctx);
6644 return -1;
6645 }
6646
6647 if (is_monolithic && ctx.type == PIPE_SHADER_VERTEX) {
6648 LLVMValueRef parts[2];
6649 bool need_prolog = sel->vs_needs_prolog;
6650
6651 parts[1] = ctx.main_fn;
6652
6653 if (need_prolog) {
6654 union si_shader_part_key prolog_key;
6655 si_get_vs_prolog_key(&sel->info,
6656 shader->info.num_input_sgprs,
6657 &shader->key.part.vs.prolog,
6658 shader, &prolog_key);
6659 si_build_vs_prolog_function(&ctx, &prolog_key);
6660 parts[0] = ctx.main_fn;
6661 }
6662
6663 si_build_wrapper_function(&ctx, parts + !need_prolog,
6664 1 + need_prolog, need_prolog, 0);
6665 } else if (is_monolithic && ctx.type == PIPE_SHADER_TESS_CTRL) {
6666 if (sscreen->info.chip_class >= GFX9) {
6667 struct si_shader_selector *ls = shader->key.part.tcs.ls;
6668 LLVMValueRef parts[4];
6669 bool vs_needs_prolog =
6670 si_vs_needs_prolog(ls, &shader->key.part.tcs.ls_prolog);
6671
6672 /* TCS main part */
6673 parts[2] = ctx.main_fn;
6674
6675 /* TCS epilog */
6676 union si_shader_part_key tcs_epilog_key;
6677 memset(&tcs_epilog_key, 0, sizeof(tcs_epilog_key));
6678 tcs_epilog_key.tcs_epilog.states = shader->key.part.tcs.epilog;
6679 si_build_tcs_epilog_function(&ctx, &tcs_epilog_key);
6680 parts[3] = ctx.main_fn;
6681
6682 /* VS prolog */
6683 if (vs_needs_prolog) {
6684 union si_shader_part_key vs_prolog_key;
6685 si_get_vs_prolog_key(&ls->info,
6686 shader->info.num_input_sgprs,
6687 &shader->key.part.tcs.ls_prolog,
6688 shader, &vs_prolog_key);
6689 vs_prolog_key.vs_prolog.is_monolithic = true;
6690 si_build_vs_prolog_function(&ctx, &vs_prolog_key);
6691 parts[0] = ctx.main_fn;
6692 }
6693
6694 /* VS as LS main part */
6695 struct si_shader shader_ls = {};
6696 shader_ls.selector = ls;
6697 shader_ls.key.as_ls = 1;
6698 shader_ls.key.mono = shader->key.mono;
6699 shader_ls.key.opt = shader->key.opt;
6700 si_llvm_context_set_tgsi(&ctx, &shader_ls);
6701
6702 if (!si_compile_tgsi_main(&ctx, true)) {
6703 si_llvm_dispose(&ctx);
6704 return -1;
6705 }
6706 shader->info.uses_instanceid |= ls->info.uses_instanceid;
6707 parts[1] = ctx.main_fn;
6708
6709 /* Reset the shader context. */
6710 ctx.shader = shader;
6711 ctx.type = PIPE_SHADER_TESS_CTRL;
6712
6713 si_build_wrapper_function(&ctx,
6714 parts + !vs_needs_prolog,
6715 4 - !vs_needs_prolog, 0,
6716 vs_needs_prolog ? 2 : 1);
6717 } else {
6718 LLVMValueRef parts[2];
6719 union si_shader_part_key epilog_key;
6720
6721 parts[0] = ctx.main_fn;
6722
6723 memset(&epilog_key, 0, sizeof(epilog_key));
6724 epilog_key.tcs_epilog.states = shader->key.part.tcs.epilog;
6725 si_build_tcs_epilog_function(&ctx, &epilog_key);
6726 parts[1] = ctx.main_fn;
6727
6728 si_build_wrapper_function(&ctx, parts, 2, 0, 0);
6729 }
6730 } else if (is_monolithic && ctx.type == PIPE_SHADER_GEOMETRY) {
6731 if (ctx.screen->info.chip_class >= GFX9) {
6732 struct si_shader_selector *es = shader->key.part.gs.es;
6733 LLVMValueRef es_prolog = NULL;
6734 LLVMValueRef es_main = NULL;
6735 LLVMValueRef gs_prolog = NULL;
6736 LLVMValueRef gs_main = ctx.main_fn;
6737
6738 /* GS prolog */
6739 union si_shader_part_key gs_prolog_key;
6740 memset(&gs_prolog_key, 0, sizeof(gs_prolog_key));
6741 gs_prolog_key.gs_prolog.states = shader->key.part.gs.prolog;
6742 gs_prolog_key.gs_prolog.is_monolithic = true;
6743 si_build_gs_prolog_function(&ctx, &gs_prolog_key);
6744 gs_prolog = ctx.main_fn;
6745
6746 /* ES prolog */
6747 if (es->vs_needs_prolog) {
6748 union si_shader_part_key vs_prolog_key;
6749 si_get_vs_prolog_key(&es->info,
6750 shader->info.num_input_sgprs,
6751 &shader->key.part.gs.vs_prolog,
6752 shader, &vs_prolog_key);
6753 vs_prolog_key.vs_prolog.is_monolithic = true;
6754 si_build_vs_prolog_function(&ctx, &vs_prolog_key);
6755 es_prolog = ctx.main_fn;
6756 }
6757
6758 /* ES main part */
6759 struct si_shader shader_es = {};
6760 shader_es.selector = es;
6761 shader_es.key.as_es = 1;
6762 shader_es.key.mono = shader->key.mono;
6763 shader_es.key.opt = shader->key.opt;
6764 si_llvm_context_set_tgsi(&ctx, &shader_es);
6765
6766 if (!si_compile_tgsi_main(&ctx, true)) {
6767 si_llvm_dispose(&ctx);
6768 return -1;
6769 }
6770 shader->info.uses_instanceid |= es->info.uses_instanceid;
6771 es_main = ctx.main_fn;
6772
6773 /* Reset the shader context. */
6774 ctx.shader = shader;
6775 ctx.type = PIPE_SHADER_GEOMETRY;
6776
6777 /* Prepare the array of shader parts. */
6778 LLVMValueRef parts[4];
6779 unsigned num_parts = 0, main_part, next_first_part;
6780
6781 if (es_prolog)
6782 parts[num_parts++] = es_prolog;
6783
6784 parts[main_part = num_parts++] = es_main;
6785 parts[next_first_part = num_parts++] = gs_prolog;
6786 parts[num_parts++] = gs_main;
6787
6788 si_build_wrapper_function(&ctx, parts, num_parts,
6789 main_part, next_first_part);
6790 } else {
6791 LLVMValueRef parts[2];
6792 union si_shader_part_key prolog_key;
6793
6794 parts[1] = ctx.main_fn;
6795
6796 memset(&prolog_key, 0, sizeof(prolog_key));
6797 prolog_key.gs_prolog.states = shader->key.part.gs.prolog;
6798 si_build_gs_prolog_function(&ctx, &prolog_key);
6799 parts[0] = ctx.main_fn;
6800
6801 si_build_wrapper_function(&ctx, parts, 2, 1, 0);
6802 }
6803 } else if (is_monolithic && ctx.type == PIPE_SHADER_FRAGMENT) {
6804 LLVMValueRef parts[3];
6805 union si_shader_part_key prolog_key;
6806 union si_shader_part_key epilog_key;
6807 bool need_prolog;
6808
6809 si_get_ps_prolog_key(shader, &prolog_key, false);
6810 need_prolog = si_need_ps_prolog(&prolog_key);
6811
6812 parts[need_prolog ? 1 : 0] = ctx.main_fn;
6813
6814 if (need_prolog) {
6815 si_build_ps_prolog_function(&ctx, &prolog_key);
6816 parts[0] = ctx.main_fn;
6817 }
6818
6819 si_get_ps_epilog_key(shader, &epilog_key);
6820 si_build_ps_epilog_function(&ctx, &epilog_key);
6821 parts[need_prolog ? 2 : 1] = ctx.main_fn;
6822
6823 si_build_wrapper_function(&ctx, parts, need_prolog ? 3 : 2,
6824 need_prolog ? 1 : 0, 0);
6825 }
6826
6827 si_llvm_optimize_module(&ctx);
6828
6829 /* Post-optimization transformations and analysis. */
6830 si_optimize_vs_outputs(&ctx);
6831
6832 if ((debug && debug->debug_message) ||
6833 si_can_dump_shader(sscreen, ctx.type))
6834 si_count_scratch_private_memory(&ctx);
6835
6836 /* Compile to bytecode. */
6837 r = si_compile_llvm(sscreen, &shader->binary, &shader->config, tm,
6838 ctx.gallivm.module, debug, ctx.type, "TGSI shader");
6839 si_llvm_dispose(&ctx);
6840 if (r) {
6841 fprintf(stderr, "LLVM failed to compile shader\n");
6842 return r;
6843 }
6844
6845 /* Validate SGPR and VGPR usage for compute to detect compiler bugs.
6846 * LLVM 3.9svn has this bug.
6847 */
6848 if (sel->type == PIPE_SHADER_COMPUTE) {
6849 unsigned wave_size = 64;
6850 unsigned max_vgprs = 256;
6851 unsigned max_sgprs = sscreen->info.chip_class >= VI ? 800 : 512;
6852 unsigned max_sgprs_per_wave = 128;
6853 unsigned max_block_threads = si_get_max_workgroup_size(shader);
6854 unsigned min_waves_per_cu = DIV_ROUND_UP(max_block_threads, wave_size);
6855 unsigned min_waves_per_simd = DIV_ROUND_UP(min_waves_per_cu, 4);
6856
6857 max_vgprs = max_vgprs / min_waves_per_simd;
6858 max_sgprs = MIN2(max_sgprs / min_waves_per_simd, max_sgprs_per_wave);
6859
6860 if (shader->config.num_sgprs > max_sgprs ||
6861 shader->config.num_vgprs > max_vgprs) {
6862 fprintf(stderr, "LLVM failed to compile a shader correctly: "
6863 "SGPR:VGPR usage is %u:%u, but the hw limit is %u:%u\n",
6864 shader->config.num_sgprs, shader->config.num_vgprs,
6865 max_sgprs, max_vgprs);
6866
6867 /* Just terminate the process, because dependent
6868 * shaders can hang due to bad input data, but use
6869 * the env var to allow shader-db to work.
6870 */
6871 if (!debug_get_bool_option("SI_PASS_BAD_SHADERS", false))
6872 abort();
6873 }
6874 }
6875
6876 /* Add the scratch offset to input SGPRs. */
6877 if (shader->config.scratch_bytes_per_wave && !is_merged_shader(shader))
6878 shader->info.num_input_sgprs += 1; /* scratch byte offset */
6879
6880 /* Calculate the number of fragment input VGPRs. */
6881 if (ctx.type == PIPE_SHADER_FRAGMENT) {
6882 shader->info.num_input_vgprs = 0;
6883 shader->info.face_vgpr_index = -1;
6884 shader->info.ancillary_vgpr_index = -1;
6885
6886 if (G_0286CC_PERSP_SAMPLE_ENA(shader->config.spi_ps_input_addr))
6887 shader->info.num_input_vgprs += 2;
6888 if (G_0286CC_PERSP_CENTER_ENA(shader->config.spi_ps_input_addr))
6889 shader->info.num_input_vgprs += 2;
6890 if (G_0286CC_PERSP_CENTROID_ENA(shader->config.spi_ps_input_addr))
6891 shader->info.num_input_vgprs += 2;
6892 if (G_0286CC_PERSP_PULL_MODEL_ENA(shader->config.spi_ps_input_addr))
6893 shader->info.num_input_vgprs += 3;
6894 if (G_0286CC_LINEAR_SAMPLE_ENA(shader->config.spi_ps_input_addr))
6895 shader->info.num_input_vgprs += 2;
6896 if (G_0286CC_LINEAR_CENTER_ENA(shader->config.spi_ps_input_addr))
6897 shader->info.num_input_vgprs += 2;
6898 if (G_0286CC_LINEAR_CENTROID_ENA(shader->config.spi_ps_input_addr))
6899 shader->info.num_input_vgprs += 2;
6900 if (G_0286CC_LINE_STIPPLE_TEX_ENA(shader->config.spi_ps_input_addr))
6901 shader->info.num_input_vgprs += 1;
6902 if (G_0286CC_POS_X_FLOAT_ENA(shader->config.spi_ps_input_addr))
6903 shader->info.num_input_vgprs += 1;
6904 if (G_0286CC_POS_Y_FLOAT_ENA(shader->config.spi_ps_input_addr))
6905 shader->info.num_input_vgprs += 1;
6906 if (G_0286CC_POS_Z_FLOAT_ENA(shader->config.spi_ps_input_addr))
6907 shader->info.num_input_vgprs += 1;
6908 if (G_0286CC_POS_W_FLOAT_ENA(shader->config.spi_ps_input_addr))
6909 shader->info.num_input_vgprs += 1;
6910 if (G_0286CC_FRONT_FACE_ENA(shader->config.spi_ps_input_addr)) {
6911 shader->info.face_vgpr_index = shader->info.num_input_vgprs;
6912 shader->info.num_input_vgprs += 1;
6913 }
6914 if (G_0286CC_ANCILLARY_ENA(shader->config.spi_ps_input_addr)) {
6915 shader->info.ancillary_vgpr_index = shader->info.num_input_vgprs;
6916 shader->info.num_input_vgprs += 1;
6917 }
6918 if (G_0286CC_SAMPLE_COVERAGE_ENA(shader->config.spi_ps_input_addr))
6919 shader->info.num_input_vgprs += 1;
6920 if (G_0286CC_POS_FIXED_PT_ENA(shader->config.spi_ps_input_addr))
6921 shader->info.num_input_vgprs += 1;
6922 }
6923
6924 si_calculate_max_simd_waves(shader);
6925 si_shader_dump_stats_for_shader_db(shader, debug);
6926 return 0;
6927 }
6928
6929 /**
6930 * Create, compile and return a shader part (prolog or epilog).
6931 *
6932 * \param sscreen screen
6933 * \param list list of shader parts of the same category
6934 * \param type shader type
6935 * \param key shader part key
6936 * \param prolog whether the part being requested is a prolog
6937 * \param tm LLVM target machine
6938 * \param debug debug callback
6939 * \param build the callback responsible for building the main function
6940 * \return non-NULL on success
6941 */
6942 static struct si_shader_part *
6943 si_get_shader_part(struct si_screen *sscreen,
6944 struct si_shader_part **list,
6945 enum pipe_shader_type type,
6946 bool prolog,
6947 union si_shader_part_key *key,
6948 LLVMTargetMachineRef tm,
6949 struct pipe_debug_callback *debug,
6950 void (*build)(struct si_shader_context *,
6951 union si_shader_part_key *),
6952 const char *name)
6953 {
6954 struct si_shader_part *result;
6955
6956 mtx_lock(&sscreen->shader_parts_mutex);
6957
6958 /* Find existing. */
6959 for (result = *list; result; result = result->next) {
6960 if (memcmp(&result->key, key, sizeof(*key)) == 0) {
6961 mtx_unlock(&sscreen->shader_parts_mutex);
6962 return result;
6963 }
6964 }
6965
6966 /* Compile a new one. */
6967 result = CALLOC_STRUCT(si_shader_part);
6968 result->key = *key;
6969
6970 struct si_shader shader = {};
6971 struct si_shader_context ctx;
6972
6973 si_init_shader_ctx(&ctx, sscreen, tm);
6974 ctx.shader = &shader;
6975 ctx.type = type;
6976
6977 switch (type) {
6978 case PIPE_SHADER_VERTEX:
6979 shader.key.as_ls = key->vs_prolog.as_ls;
6980 shader.key.as_es = key->vs_prolog.as_es;
6981 break;
6982 case PIPE_SHADER_TESS_CTRL:
6983 assert(!prolog);
6984 shader.key.part.tcs.epilog = key->tcs_epilog.states;
6985 break;
6986 case PIPE_SHADER_GEOMETRY:
6987 assert(prolog);
6988 break;
6989 case PIPE_SHADER_FRAGMENT:
6990 if (prolog)
6991 shader.key.part.ps.prolog = key->ps_prolog.states;
6992 else
6993 shader.key.part.ps.epilog = key->ps_epilog.states;
6994 break;
6995 default:
6996 unreachable("bad shader part");
6997 }
6998
6999 build(&ctx, key);
7000
7001 /* Compile. */
7002 si_llvm_optimize_module(&ctx);
7003
7004 if (si_compile_llvm(sscreen, &result->binary, &result->config, tm,
7005 ctx.ac.module, debug, ctx.type, name)) {
7006 FREE(result);
7007 result = NULL;
7008 goto out;
7009 }
7010
7011 result->next = *list;
7012 *list = result;
7013
7014 out:
7015 si_llvm_dispose(&ctx);
7016 mtx_unlock(&sscreen->shader_parts_mutex);
7017 return result;
7018 }
7019
7020 static LLVMValueRef si_prolog_get_rw_buffers(struct si_shader_context *ctx)
7021 {
7022 LLVMValueRef ptr[2], list;
7023 bool is_merged_shader =
7024 ctx->screen->info.chip_class >= GFX9 &&
7025 (ctx->type == PIPE_SHADER_TESS_CTRL ||
7026 ctx->type == PIPE_SHADER_GEOMETRY ||
7027 ctx->shader->key.as_ls || ctx->shader->key.as_es);
7028
7029 /* Get the pointer to rw buffers. */
7030 ptr[0] = LLVMGetParam(ctx->main_fn, (is_merged_shader ? 8 : 0) + SI_SGPR_RW_BUFFERS);
7031 ptr[1] = LLVMGetParam(ctx->main_fn, (is_merged_shader ? 8 : 0) + SI_SGPR_RW_BUFFERS_HI);
7032 list = lp_build_gather_values(&ctx->gallivm, ptr, 2);
7033 list = LLVMBuildBitCast(ctx->ac.builder, list, ctx->i64, "");
7034 list = LLVMBuildIntToPtr(ctx->ac.builder, list,
7035 ac_array_in_const_addr_space(ctx->v4i32), "");
7036 return list;
7037 }
7038
7039 /**
7040 * Build the vertex shader prolog function.
7041 *
7042 * The inputs are the same as VS (a lot of SGPRs and 4 VGPR system values).
7043 * All inputs are returned unmodified. The vertex load indices are
7044 * stored after them, which will be used by the API VS for fetching inputs.
7045 *
7046 * For example, the expected outputs for instance_divisors[] = {0, 1, 2} are:
7047 * input_v0,
7048 * input_v1,
7049 * input_v2,
7050 * input_v3,
7051 * (VertexID + BaseVertex),
7052 * (InstanceID + StartInstance),
7053 * (InstanceID / 2 + StartInstance)
7054 */
7055 static void si_build_vs_prolog_function(struct si_shader_context *ctx,
7056 union si_shader_part_key *key)
7057 {
7058 struct si_function_info fninfo;
7059 LLVMTypeRef *returns;
7060 LLVMValueRef ret, func;
7061 int num_returns, i;
7062 unsigned first_vs_vgpr = key->vs_prolog.num_merged_next_stage_vgprs;
7063 unsigned num_input_vgprs = key->vs_prolog.num_merged_next_stage_vgprs + 4;
7064 LLVMValueRef input_vgprs[9];
7065 unsigned num_all_input_regs = key->vs_prolog.num_input_sgprs +
7066 num_input_vgprs;
7067 unsigned user_sgpr_base = key->vs_prolog.num_merged_next_stage_vgprs ? 8 : 0;
7068
7069 si_init_function_info(&fninfo);
7070
7071 /* 4 preloaded VGPRs + vertex load indices as prolog outputs */
7072 returns = alloca((num_all_input_regs + key->vs_prolog.last_input + 1) *
7073 sizeof(LLVMTypeRef));
7074 num_returns = 0;
7075
7076 /* Declare input and output SGPRs. */
7077 for (i = 0; i < key->vs_prolog.num_input_sgprs; i++) {
7078 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7079 returns[num_returns++] = ctx->i32;
7080 }
7081
7082 /* Preloaded VGPRs (outputs must be floats) */
7083 for (i = 0; i < num_input_vgprs; i++) {
7084 add_arg_assign(&fninfo, ARG_VGPR, ctx->i32, &input_vgprs[i]);
7085 returns[num_returns++] = ctx->f32;
7086 }
7087
7088 /* Vertex load indices. */
7089 for (i = 0; i <= key->vs_prolog.last_input; i++)
7090 returns[num_returns++] = ctx->f32;
7091
7092 /* Create the function. */
7093 si_create_function(ctx, "vs_prolog", returns, num_returns, &fninfo, 0);
7094 func = ctx->main_fn;
7095
7096 if (key->vs_prolog.num_merged_next_stage_vgprs) {
7097 if (!key->vs_prolog.is_monolithic)
7098 si_init_exec_from_input(ctx, 3, 0);
7099
7100 if (key->vs_prolog.as_ls &&
7101 ctx->screen->has_ls_vgpr_init_bug) {
7102 /* If there are no HS threads, SPI loads the LS VGPRs
7103 * starting at VGPR 0. Shift them back to where they
7104 * belong.
7105 */
7106 LLVMValueRef has_hs_threads =
7107 LLVMBuildICmp(ctx->ac.builder, LLVMIntNE,
7108 unpack_param(ctx, 3, 8, 8),
7109 ctx->i32_0, "");
7110
7111 for (i = 4; i > 0; --i) {
7112 input_vgprs[i + 1] =
7113 LLVMBuildSelect(ctx->ac.builder, has_hs_threads,
7114 input_vgprs[i + 1],
7115 input_vgprs[i - 1], "");
7116 }
7117 }
7118 }
7119
7120 ctx->abi.vertex_id = input_vgprs[first_vs_vgpr];
7121 ctx->abi.instance_id = input_vgprs[first_vs_vgpr + (key->vs_prolog.as_ls ? 2 : 1)];
7122
7123 /* Copy inputs to outputs. This should be no-op, as the registers match,
7124 * but it will prevent the compiler from overwriting them unintentionally.
7125 */
7126 ret = ctx->return_value;
7127 for (i = 0; i < key->vs_prolog.num_input_sgprs; i++) {
7128 LLVMValueRef p = LLVMGetParam(func, i);
7129 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, p, i, "");
7130 }
7131 for (i = 0; i < num_input_vgprs; i++) {
7132 LLVMValueRef p = input_vgprs[i];
7133 p = ac_to_float(&ctx->ac, p);
7134 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, p,
7135 key->vs_prolog.num_input_sgprs + i, "");
7136 }
7137
7138 /* Compute vertex load indices from instance divisors. */
7139 LLVMValueRef instance_divisor_constbuf = NULL;
7140
7141 if (key->vs_prolog.states.instance_divisor_is_fetched) {
7142 LLVMValueRef list = si_prolog_get_rw_buffers(ctx);
7143 LLVMValueRef buf_index =
7144 LLVMConstInt(ctx->i32, SI_VS_CONST_INSTANCE_DIVISORS, 0);
7145 instance_divisor_constbuf =
7146 ac_build_load_to_sgpr(&ctx->ac, list, buf_index);
7147 }
7148
7149 for (i = 0; i <= key->vs_prolog.last_input; i++) {
7150 bool divisor_is_one =
7151 key->vs_prolog.states.instance_divisor_is_one & (1u << i);
7152 bool divisor_is_fetched =
7153 key->vs_prolog.states.instance_divisor_is_fetched & (1u << i);
7154 LLVMValueRef index;
7155
7156 if (divisor_is_one || divisor_is_fetched) {
7157 LLVMValueRef divisor = ctx->i32_1;
7158
7159 if (divisor_is_fetched) {
7160 divisor = buffer_load_const(ctx, instance_divisor_constbuf,
7161 LLVMConstInt(ctx->i32, i * 4, 0));
7162 divisor = ac_to_integer(&ctx->ac, divisor);
7163 }
7164
7165 /* InstanceID / Divisor + StartInstance */
7166 index = get_instance_index_for_fetch(ctx,
7167 user_sgpr_base +
7168 SI_SGPR_START_INSTANCE,
7169 divisor);
7170 } else {
7171 /* VertexID + BaseVertex */
7172 index = LLVMBuildAdd(ctx->ac.builder,
7173 ctx->abi.vertex_id,
7174 LLVMGetParam(func, user_sgpr_base +
7175 SI_SGPR_BASE_VERTEX), "");
7176 }
7177
7178 index = ac_to_float(&ctx->ac, index);
7179 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, index,
7180 fninfo.num_params + i, "");
7181 }
7182
7183 si_llvm_build_ret(ctx, ret);
7184 }
7185
7186 static bool si_get_vs_prolog(struct si_screen *sscreen,
7187 LLVMTargetMachineRef tm,
7188 struct si_shader *shader,
7189 struct pipe_debug_callback *debug,
7190 struct si_shader *main_part,
7191 const struct si_vs_prolog_bits *key)
7192 {
7193 struct si_shader_selector *vs = main_part->selector;
7194
7195 if (!si_vs_needs_prolog(vs, key))
7196 return true;
7197
7198 /* Get the prolog. */
7199 union si_shader_part_key prolog_key;
7200 si_get_vs_prolog_key(&vs->info, main_part->info.num_input_sgprs,
7201 key, shader, &prolog_key);
7202
7203 shader->prolog =
7204 si_get_shader_part(sscreen, &sscreen->vs_prologs,
7205 PIPE_SHADER_VERTEX, true, &prolog_key, tm,
7206 debug, si_build_vs_prolog_function,
7207 "Vertex Shader Prolog");
7208 return shader->prolog != NULL;
7209 }
7210
7211 /**
7212 * Select and compile (or reuse) vertex shader parts (prolog & epilog).
7213 */
7214 static bool si_shader_select_vs_parts(struct si_screen *sscreen,
7215 LLVMTargetMachineRef tm,
7216 struct si_shader *shader,
7217 struct pipe_debug_callback *debug)
7218 {
7219 return si_get_vs_prolog(sscreen, tm, shader, debug, shader,
7220 &shader->key.part.vs.prolog);
7221 }
7222
7223 /**
7224 * Compile the TCS epilog function. This writes tesselation factors to memory
7225 * based on the output primitive type of the tesselator (determined by TES).
7226 */
7227 static void si_build_tcs_epilog_function(struct si_shader_context *ctx,
7228 union si_shader_part_key *key)
7229 {
7230 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
7231 struct si_function_info fninfo;
7232 LLVMValueRef func;
7233
7234 si_init_function_info(&fninfo);
7235
7236 if (ctx->screen->info.chip_class >= GFX9) {
7237 add_arg(&fninfo, ARG_SGPR, ctx->i64);
7238 ctx->param_tcs_offchip_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
7239 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* wave info */
7240 ctx->param_tcs_factor_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
7241 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7242 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7243 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7244 add_arg(&fninfo, ARG_SGPR, ctx->i64);
7245 add_arg(&fninfo, ARG_SGPR, ctx->i64);
7246 add_arg(&fninfo, ARG_SGPR, ctx->i64);
7247 add_arg(&fninfo, ARG_SGPR, ctx->i64);
7248 add_arg(&fninfo, ARG_SGPR, ctx->i64);
7249 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7250 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7251 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7252 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7253 ctx->param_tcs_offchip_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
7254 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7255 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7256 ctx->param_tcs_offchip_addr_base64k = add_arg(&fninfo, ARG_SGPR, ctx->i32);
7257 ctx->param_tcs_factor_addr_base64k = add_arg(&fninfo, ARG_SGPR, ctx->i32);
7258 } else {
7259 add_arg(&fninfo, ARG_SGPR, ctx->i64);
7260 add_arg(&fninfo, ARG_SGPR, ctx->i64);
7261 add_arg(&fninfo, ARG_SGPR, ctx->i64);
7262 add_arg(&fninfo, ARG_SGPR, ctx->i64);
7263 ctx->param_tcs_offchip_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
7264 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7265 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7266 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7267 ctx->param_tcs_offchip_addr_base64k = add_arg(&fninfo, ARG_SGPR, ctx->i32);
7268 ctx->param_tcs_factor_addr_base64k = add_arg(&fninfo, ARG_SGPR, ctx->i32);
7269 ctx->param_tcs_offchip_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
7270 ctx->param_tcs_factor_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
7271 }
7272
7273 add_arg(&fninfo, ARG_VGPR, ctx->i32); /* VGPR gap */
7274 add_arg(&fninfo, ARG_VGPR, ctx->i32); /* VGPR gap */
7275 unsigned tess_factors_idx =
7276 add_arg(&fninfo, ARG_VGPR, ctx->i32); /* patch index within the wave (REL_PATCH_ID) */
7277 add_arg(&fninfo, ARG_VGPR, ctx->i32); /* invocation ID within the patch */
7278 add_arg(&fninfo, ARG_VGPR, ctx->i32); /* LDS offset where tess factors should be loaded from */
7279
7280 for (unsigned i = 0; i < 6; i++)
7281 add_arg(&fninfo, ARG_VGPR, ctx->i32); /* tess factors */
7282
7283 /* Create the function. */
7284 si_create_function(ctx, "tcs_epilog", NULL, 0, &fninfo,
7285 ctx->screen->info.chip_class >= CIK ? 128 : 64);
7286 ac_declare_lds_as_pointer(&ctx->ac);
7287 func = ctx->main_fn;
7288
7289 LLVMValueRef invoc0_tess_factors[6];
7290 for (unsigned i = 0; i < 6; i++)
7291 invoc0_tess_factors[i] = LLVMGetParam(func, tess_factors_idx + 3 + i);
7292
7293 si_write_tess_factors(bld_base,
7294 LLVMGetParam(func, tess_factors_idx),
7295 LLVMGetParam(func, tess_factors_idx + 1),
7296 LLVMGetParam(func, tess_factors_idx + 2),
7297 invoc0_tess_factors, invoc0_tess_factors + 4);
7298
7299 LLVMBuildRetVoid(ctx->ac.builder);
7300 }
7301
7302 /**
7303 * Select and compile (or reuse) TCS parts (epilog).
7304 */
7305 static bool si_shader_select_tcs_parts(struct si_screen *sscreen,
7306 LLVMTargetMachineRef tm,
7307 struct si_shader *shader,
7308 struct pipe_debug_callback *debug)
7309 {
7310 if (sscreen->info.chip_class >= GFX9) {
7311 struct si_shader *ls_main_part =
7312 shader->key.part.tcs.ls->main_shader_part_ls;
7313
7314 if (!si_get_vs_prolog(sscreen, tm, shader, debug, ls_main_part,
7315 &shader->key.part.tcs.ls_prolog))
7316 return false;
7317
7318 shader->previous_stage = ls_main_part;
7319 }
7320
7321 /* Get the epilog. */
7322 union si_shader_part_key epilog_key;
7323 memset(&epilog_key, 0, sizeof(epilog_key));
7324 epilog_key.tcs_epilog.states = shader->key.part.tcs.epilog;
7325
7326 shader->epilog = si_get_shader_part(sscreen, &sscreen->tcs_epilogs,
7327 PIPE_SHADER_TESS_CTRL, false,
7328 &epilog_key, tm, debug,
7329 si_build_tcs_epilog_function,
7330 "Tessellation Control Shader Epilog");
7331 return shader->epilog != NULL;
7332 }
7333
7334 /**
7335 * Select and compile (or reuse) GS parts (prolog).
7336 */
7337 static bool si_shader_select_gs_parts(struct si_screen *sscreen,
7338 LLVMTargetMachineRef tm,
7339 struct si_shader *shader,
7340 struct pipe_debug_callback *debug)
7341 {
7342 if (sscreen->info.chip_class >= GFX9) {
7343 struct si_shader *es_main_part =
7344 shader->key.part.gs.es->main_shader_part_es;
7345
7346 if (shader->key.part.gs.es->type == PIPE_SHADER_VERTEX &&
7347 !si_get_vs_prolog(sscreen, tm, shader, debug, es_main_part,
7348 &shader->key.part.gs.vs_prolog))
7349 return false;
7350
7351 shader->previous_stage = es_main_part;
7352 }
7353
7354 if (!shader->key.part.gs.prolog.tri_strip_adj_fix)
7355 return true;
7356
7357 union si_shader_part_key prolog_key;
7358 memset(&prolog_key, 0, sizeof(prolog_key));
7359 prolog_key.gs_prolog.states = shader->key.part.gs.prolog;
7360
7361 shader->prolog2 = si_get_shader_part(sscreen, &sscreen->gs_prologs,
7362 PIPE_SHADER_GEOMETRY, true,
7363 &prolog_key, tm, debug,
7364 si_build_gs_prolog_function,
7365 "Geometry Shader Prolog");
7366 return shader->prolog2 != NULL;
7367 }
7368
7369 /**
7370 * Build the pixel shader prolog function. This handles:
7371 * - two-side color selection and interpolation
7372 * - overriding interpolation parameters for the API PS
7373 * - polygon stippling
7374 *
7375 * All preloaded SGPRs and VGPRs are passed through unmodified unless they are
7376 * overriden by other states. (e.g. per-sample interpolation)
7377 * Interpolated colors are stored after the preloaded VGPRs.
7378 */
7379 static void si_build_ps_prolog_function(struct si_shader_context *ctx,
7380 union si_shader_part_key *key)
7381 {
7382 struct si_function_info fninfo;
7383 LLVMValueRef ret, func;
7384 int num_returns, i, num_color_channels;
7385
7386 assert(si_need_ps_prolog(key));
7387
7388 si_init_function_info(&fninfo);
7389
7390 /* Declare inputs. */
7391 for (i = 0; i < key->ps_prolog.num_input_sgprs; i++)
7392 add_arg(&fninfo, ARG_SGPR, ctx->i32);
7393
7394 for (i = 0; i < key->ps_prolog.num_input_vgprs; i++)
7395 add_arg(&fninfo, ARG_VGPR, ctx->f32);
7396
7397 /* Declare outputs (same as inputs + add colors if needed) */
7398 num_returns = fninfo.num_params;
7399 num_color_channels = util_bitcount(key->ps_prolog.colors_read);
7400 for (i = 0; i < num_color_channels; i++)
7401 fninfo.types[num_returns++] = ctx->f32;
7402
7403 /* Create the function. */
7404 si_create_function(ctx, "ps_prolog", fninfo.types, num_returns,
7405 &fninfo, 0);
7406 func = ctx->main_fn;
7407
7408 /* Copy inputs to outputs. This should be no-op, as the registers match,
7409 * but it will prevent the compiler from overwriting them unintentionally.
7410 */
7411 ret = ctx->return_value;
7412 for (i = 0; i < fninfo.num_params; i++) {
7413 LLVMValueRef p = LLVMGetParam(func, i);
7414 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, p, i, "");
7415 }
7416
7417 /* Polygon stippling. */
7418 if (key->ps_prolog.states.poly_stipple) {
7419 /* POS_FIXED_PT is always last. */
7420 unsigned pos = key->ps_prolog.num_input_sgprs +
7421 key->ps_prolog.num_input_vgprs - 1;
7422 LLVMValueRef list = si_prolog_get_rw_buffers(ctx);
7423
7424 si_llvm_emit_polygon_stipple(ctx, list, pos);
7425 }
7426
7427 if (key->ps_prolog.states.bc_optimize_for_persp ||
7428 key->ps_prolog.states.bc_optimize_for_linear) {
7429 unsigned i, base = key->ps_prolog.num_input_sgprs;
7430 LLVMValueRef center[2], centroid[2], tmp, bc_optimize;
7431
7432 /* The shader should do: if (PRIM_MASK[31]) CENTROID = CENTER;
7433 * The hw doesn't compute CENTROID if the whole wave only
7434 * contains fully-covered quads.
7435 *
7436 * PRIM_MASK is after user SGPRs.
7437 */
7438 bc_optimize = LLVMGetParam(func, SI_PS_NUM_USER_SGPR);
7439 bc_optimize = LLVMBuildLShr(ctx->ac.builder, bc_optimize,
7440 LLVMConstInt(ctx->i32, 31, 0), "");
7441 bc_optimize = LLVMBuildTrunc(ctx->ac.builder, bc_optimize,
7442 ctx->i1, "");
7443
7444 if (key->ps_prolog.states.bc_optimize_for_persp) {
7445 /* Read PERSP_CENTER. */
7446 for (i = 0; i < 2; i++)
7447 center[i] = LLVMGetParam(func, base + 2 + i);
7448 /* Read PERSP_CENTROID. */
7449 for (i = 0; i < 2; i++)
7450 centroid[i] = LLVMGetParam(func, base + 4 + i);
7451 /* Select PERSP_CENTROID. */
7452 for (i = 0; i < 2; i++) {
7453 tmp = LLVMBuildSelect(ctx->ac.builder, bc_optimize,
7454 center[i], centroid[i], "");
7455 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7456 tmp, base + 4 + i, "");
7457 }
7458 }
7459 if (key->ps_prolog.states.bc_optimize_for_linear) {
7460 /* Read LINEAR_CENTER. */
7461 for (i = 0; i < 2; i++)
7462 center[i] = LLVMGetParam(func, base + 8 + i);
7463 /* Read LINEAR_CENTROID. */
7464 for (i = 0; i < 2; i++)
7465 centroid[i] = LLVMGetParam(func, base + 10 + i);
7466 /* Select LINEAR_CENTROID. */
7467 for (i = 0; i < 2; i++) {
7468 tmp = LLVMBuildSelect(ctx->ac.builder, bc_optimize,
7469 center[i], centroid[i], "");
7470 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7471 tmp, base + 10 + i, "");
7472 }
7473 }
7474 }
7475
7476 /* Force per-sample interpolation. */
7477 if (key->ps_prolog.states.force_persp_sample_interp) {
7478 unsigned i, base = key->ps_prolog.num_input_sgprs;
7479 LLVMValueRef persp_sample[2];
7480
7481 /* Read PERSP_SAMPLE. */
7482 for (i = 0; i < 2; i++)
7483 persp_sample[i] = LLVMGetParam(func, base + i);
7484 /* Overwrite PERSP_CENTER. */
7485 for (i = 0; i < 2; i++)
7486 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7487 persp_sample[i], base + 2 + i, "");
7488 /* Overwrite PERSP_CENTROID. */
7489 for (i = 0; i < 2; i++)
7490 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7491 persp_sample[i], base + 4 + i, "");
7492 }
7493 if (key->ps_prolog.states.force_linear_sample_interp) {
7494 unsigned i, base = key->ps_prolog.num_input_sgprs;
7495 LLVMValueRef linear_sample[2];
7496
7497 /* Read LINEAR_SAMPLE. */
7498 for (i = 0; i < 2; i++)
7499 linear_sample[i] = LLVMGetParam(func, base + 6 + i);
7500 /* Overwrite LINEAR_CENTER. */
7501 for (i = 0; i < 2; i++)
7502 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7503 linear_sample[i], base + 8 + i, "");
7504 /* Overwrite LINEAR_CENTROID. */
7505 for (i = 0; i < 2; i++)
7506 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7507 linear_sample[i], base + 10 + i, "");
7508 }
7509
7510 /* Force center interpolation. */
7511 if (key->ps_prolog.states.force_persp_center_interp) {
7512 unsigned i, base = key->ps_prolog.num_input_sgprs;
7513 LLVMValueRef persp_center[2];
7514
7515 /* Read PERSP_CENTER. */
7516 for (i = 0; i < 2; i++)
7517 persp_center[i] = LLVMGetParam(func, base + 2 + i);
7518 /* Overwrite PERSP_SAMPLE. */
7519 for (i = 0; i < 2; i++)
7520 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7521 persp_center[i], base + i, "");
7522 /* Overwrite PERSP_CENTROID. */
7523 for (i = 0; i < 2; i++)
7524 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7525 persp_center[i], base + 4 + i, "");
7526 }
7527 if (key->ps_prolog.states.force_linear_center_interp) {
7528 unsigned i, base = key->ps_prolog.num_input_sgprs;
7529 LLVMValueRef linear_center[2];
7530
7531 /* Read LINEAR_CENTER. */
7532 for (i = 0; i < 2; i++)
7533 linear_center[i] = LLVMGetParam(func, base + 8 + i);
7534 /* Overwrite LINEAR_SAMPLE. */
7535 for (i = 0; i < 2; i++)
7536 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7537 linear_center[i], base + 6 + i, "");
7538 /* Overwrite LINEAR_CENTROID. */
7539 for (i = 0; i < 2; i++)
7540 ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
7541 linear_center[i], base + 10 + i, "");
7542 }
7543
7544 /* Interpolate colors. */
7545 unsigned color_out_idx = 0;
7546 for (i = 0; i < 2; i++) {
7547 unsigned writemask = (key->ps_prolog.colors_read >> (i * 4)) & 0xf;
7548 unsigned face_vgpr = key->ps_prolog.num_input_sgprs +
7549 key->ps_prolog.face_vgpr_index;
7550 LLVMValueRef interp[2], color[4];
7551 LLVMValueRef interp_ij = NULL, prim_mask = NULL, face = NULL;
7552
7553 if (!writemask)
7554 continue;
7555
7556 /* If the interpolation qualifier is not CONSTANT (-1). */
7557 if (key->ps_prolog.color_interp_vgpr_index[i] != -1) {
7558 unsigned interp_vgpr = key->ps_prolog.num_input_sgprs +
7559 key->ps_prolog.color_interp_vgpr_index[i];
7560
7561 /* Get the (i,j) updated by bc_optimize handling. */
7562 interp[0] = LLVMBuildExtractValue(ctx->ac.builder, ret,
7563 interp_vgpr, "");
7564 interp[1] = LLVMBuildExtractValue(ctx->ac.builder, ret,
7565 interp_vgpr + 1, "");
7566 interp_ij = lp_build_gather_values(&ctx->gallivm, interp, 2);
7567 }
7568
7569 /* Use the absolute location of the input. */
7570 prim_mask = LLVMGetParam(func, SI_PS_NUM_USER_SGPR);
7571
7572 if (key->ps_prolog.states.color_two_side) {
7573 face = LLVMGetParam(func, face_vgpr);
7574 face = ac_to_integer(&ctx->ac, face);
7575 }
7576
7577 interp_fs_input(ctx,
7578 key->ps_prolog.color_attr_index[i],
7579 TGSI_SEMANTIC_COLOR, i,
7580 key->ps_prolog.num_interp_inputs,
7581 key->ps_prolog.colors_read, interp_ij,
7582 prim_mask, face, color);
7583
7584 while (writemask) {
7585 unsigned chan = u_bit_scan(&writemask);
7586 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, color[chan],
7587 fninfo.num_params + color_out_idx++, "");
7588 }
7589 }
7590
7591 /* Section 15.2.2 (Shader Inputs) of the OpenGL 4.5 (Core Profile) spec
7592 * says:
7593 *
7594 * "When per-sample shading is active due to the use of a fragment
7595 * input qualified by sample or due to the use of the gl_SampleID
7596 * or gl_SamplePosition variables, only the bit for the current
7597 * sample is set in gl_SampleMaskIn. When state specifies multiple
7598 * fragment shader invocations for a given fragment, the sample
7599 * mask for any single fragment shader invocation may specify a
7600 * subset of the covered samples for the fragment. In this case,
7601 * the bit corresponding to each covered sample will be set in
7602 * exactly one fragment shader invocation."
7603 *
7604 * The samplemask loaded by hardware is always the coverage of the
7605 * entire pixel/fragment, so mask bits out based on the sample ID.
7606 */
7607 if (key->ps_prolog.states.samplemask_log_ps_iter) {
7608 /* The bit pattern matches that used by fixed function fragment
7609 * processing. */
7610 static const uint16_t ps_iter_masks[] = {
7611 0xffff, /* not used */
7612 0x5555,
7613 0x1111,
7614 0x0101,
7615 0x0001,
7616 };
7617 assert(key->ps_prolog.states.samplemask_log_ps_iter < ARRAY_SIZE(ps_iter_masks));
7618
7619 uint32_t ps_iter_mask = ps_iter_masks[key->ps_prolog.states.samplemask_log_ps_iter];
7620 unsigned ancillary_vgpr = key->ps_prolog.num_input_sgprs +
7621 key->ps_prolog.ancillary_vgpr_index;
7622 LLVMValueRef sampleid = unpack_param(ctx, ancillary_vgpr, 8, 4);
7623 LLVMValueRef samplemask = LLVMGetParam(func, ancillary_vgpr + 1);
7624
7625 samplemask = ac_to_integer(&ctx->ac, samplemask);
7626 samplemask = LLVMBuildAnd(
7627 ctx->ac.builder,
7628 samplemask,
7629 LLVMBuildShl(ctx->ac.builder,
7630 LLVMConstInt(ctx->i32, ps_iter_mask, false),
7631 sampleid, ""),
7632 "");
7633 samplemask = ac_to_float(&ctx->ac, samplemask);
7634
7635 ret = LLVMBuildInsertValue(ctx->ac.builder, ret, samplemask,
7636 ancillary_vgpr + 1, "");
7637 }
7638
7639 /* Tell LLVM to insert WQM instruction sequence when needed. */
7640 if (key->ps_prolog.wqm) {
7641 LLVMAddTargetDependentFunctionAttr(func,
7642 "amdgpu-ps-wqm-outputs", "");
7643 }
7644
7645 si_llvm_build_ret(ctx, ret);
7646 }
7647
7648 /**
7649 * Build the pixel shader epilog function. This handles everything that must be
7650 * emulated for pixel shader exports. (alpha-test, format conversions, etc)
7651 */
7652 static void si_build_ps_epilog_function(struct si_shader_context *ctx,
7653 union si_shader_part_key *key)
7654 {
7655 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
7656 struct si_function_info fninfo;
7657 LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
7658 int i;
7659 struct si_ps_exports exp = {};
7660
7661 si_init_function_info(&fninfo);
7662
7663 /* Declare input SGPRs. */
7664 ctx->param_rw_buffers = add_arg(&fninfo, ARG_SGPR, ctx->i64);
7665 ctx->param_bindless_samplers_and_images = add_arg(&fninfo, ARG_SGPR, ctx->i64);
7666 ctx->param_const_and_shader_buffers = add_arg(&fninfo, ARG_SGPR, ctx->i64);
7667 ctx->param_samplers_and_images = add_arg(&fninfo, ARG_SGPR, ctx->i64);
7668 add_arg_checked(&fninfo, ARG_SGPR, ctx->f32, SI_PARAM_ALPHA_REF);
7669
7670 /* Declare input VGPRs. */
7671 unsigned required_num_params =
7672 fninfo.num_sgpr_params +
7673 util_bitcount(key->ps_epilog.colors_written) * 4 +
7674 key->ps_epilog.writes_z +
7675 key->ps_epilog.writes_stencil +
7676 key->ps_epilog.writes_samplemask;
7677
7678 required_num_params = MAX2(required_num_params,
7679 fninfo.num_sgpr_params + PS_EPILOG_SAMPLEMASK_MIN_LOC + 1);
7680
7681 while (fninfo.num_params < required_num_params)
7682 add_arg(&fninfo, ARG_VGPR, ctx->f32);
7683
7684 /* Create the function. */
7685 si_create_function(ctx, "ps_epilog", NULL, 0, &fninfo, 0);
7686 /* Disable elimination of unused inputs. */
7687 si_llvm_add_attribute(ctx->main_fn,
7688 "InitialPSInputAddr", 0xffffff);
7689
7690 /* Process colors. */
7691 unsigned vgpr = fninfo.num_sgpr_params;
7692 unsigned colors_written = key->ps_epilog.colors_written;
7693 int last_color_export = -1;
7694
7695 /* Find the last color export. */
7696 if (!key->ps_epilog.writes_z &&
7697 !key->ps_epilog.writes_stencil &&
7698 !key->ps_epilog.writes_samplemask) {
7699 unsigned spi_format = key->ps_epilog.states.spi_shader_col_format;
7700
7701 /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
7702 if (colors_written == 0x1 && key->ps_epilog.states.last_cbuf > 0) {
7703 /* Just set this if any of the colorbuffers are enabled. */
7704 if (spi_format &
7705 ((1ull << (4 * (key->ps_epilog.states.last_cbuf + 1))) - 1))
7706 last_color_export = 0;
7707 } else {
7708 for (i = 0; i < 8; i++)
7709 if (colors_written & (1 << i) &&
7710 (spi_format >> (i * 4)) & 0xf)
7711 last_color_export = i;
7712 }
7713 }
7714
7715 while (colors_written) {
7716 LLVMValueRef color[4];
7717 int mrt = u_bit_scan(&colors_written);
7718
7719 for (i = 0; i < 4; i++)
7720 color[i] = LLVMGetParam(ctx->main_fn, vgpr++);
7721
7722 si_export_mrt_color(bld_base, color, mrt,
7723 fninfo.num_params - 1,
7724 mrt == last_color_export, &exp);
7725 }
7726
7727 /* Process depth, stencil, samplemask. */
7728 if (key->ps_epilog.writes_z)
7729 depth = LLVMGetParam(ctx->main_fn, vgpr++);
7730 if (key->ps_epilog.writes_stencil)
7731 stencil = LLVMGetParam(ctx->main_fn, vgpr++);
7732 if (key->ps_epilog.writes_samplemask)
7733 samplemask = LLVMGetParam(ctx->main_fn, vgpr++);
7734
7735 if (depth || stencil || samplemask)
7736 si_export_mrt_z(bld_base, depth, stencil, samplemask, &exp);
7737 else if (last_color_export == -1)
7738 si_export_null(bld_base);
7739
7740 if (exp.num)
7741 si_emit_ps_exports(ctx, &exp);
7742
7743 /* Compile. */
7744 LLVMBuildRetVoid(ctx->ac.builder);
7745 }
7746
7747 /**
7748 * Select and compile (or reuse) pixel shader parts (prolog & epilog).
7749 */
7750 static bool si_shader_select_ps_parts(struct si_screen *sscreen,
7751 LLVMTargetMachineRef tm,
7752 struct si_shader *shader,
7753 struct pipe_debug_callback *debug)
7754 {
7755 union si_shader_part_key prolog_key;
7756 union si_shader_part_key epilog_key;
7757
7758 /* Get the prolog. */
7759 si_get_ps_prolog_key(shader, &prolog_key, true);
7760
7761 /* The prolog is a no-op if these aren't set. */
7762 if (si_need_ps_prolog(&prolog_key)) {
7763 shader->prolog =
7764 si_get_shader_part(sscreen, &sscreen->ps_prologs,
7765 PIPE_SHADER_FRAGMENT, true,
7766 &prolog_key, tm, debug,
7767 si_build_ps_prolog_function,
7768 "Fragment Shader Prolog");
7769 if (!shader->prolog)
7770 return false;
7771 }
7772
7773 /* Get the epilog. */
7774 si_get_ps_epilog_key(shader, &epilog_key);
7775
7776 shader->epilog =
7777 si_get_shader_part(sscreen, &sscreen->ps_epilogs,
7778 PIPE_SHADER_FRAGMENT, false,
7779 &epilog_key, tm, debug,
7780 si_build_ps_epilog_function,
7781 "Fragment Shader Epilog");
7782 if (!shader->epilog)
7783 return false;
7784
7785 /* Enable POS_FIXED_PT if polygon stippling is enabled. */
7786 if (shader->key.part.ps.prolog.poly_stipple) {
7787 shader->config.spi_ps_input_ena |= S_0286CC_POS_FIXED_PT_ENA(1);
7788 assert(G_0286CC_POS_FIXED_PT_ENA(shader->config.spi_ps_input_addr));
7789 }
7790
7791 /* Set up the enable bits for per-sample shading if needed. */
7792 if (shader->key.part.ps.prolog.force_persp_sample_interp &&
7793 (G_0286CC_PERSP_CENTER_ENA(shader->config.spi_ps_input_ena) ||
7794 G_0286CC_PERSP_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
7795 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_CENTER_ENA;
7796 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_CENTROID_ENA;
7797 shader->config.spi_ps_input_ena |= S_0286CC_PERSP_SAMPLE_ENA(1);
7798 }
7799 if (shader->key.part.ps.prolog.force_linear_sample_interp &&
7800 (G_0286CC_LINEAR_CENTER_ENA(shader->config.spi_ps_input_ena) ||
7801 G_0286CC_LINEAR_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
7802 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_CENTER_ENA;
7803 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_CENTROID_ENA;
7804 shader->config.spi_ps_input_ena |= S_0286CC_LINEAR_SAMPLE_ENA(1);
7805 }
7806 if (shader->key.part.ps.prolog.force_persp_center_interp &&
7807 (G_0286CC_PERSP_SAMPLE_ENA(shader->config.spi_ps_input_ena) ||
7808 G_0286CC_PERSP_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
7809 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_SAMPLE_ENA;
7810 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_CENTROID_ENA;
7811 shader->config.spi_ps_input_ena |= S_0286CC_PERSP_CENTER_ENA(1);
7812 }
7813 if (shader->key.part.ps.prolog.force_linear_center_interp &&
7814 (G_0286CC_LINEAR_SAMPLE_ENA(shader->config.spi_ps_input_ena) ||
7815 G_0286CC_LINEAR_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
7816 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_SAMPLE_ENA;
7817 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_CENTROID_ENA;
7818 shader->config.spi_ps_input_ena |= S_0286CC_LINEAR_CENTER_ENA(1);
7819 }
7820
7821 /* POW_W_FLOAT requires that one of the perspective weights is enabled. */
7822 if (G_0286CC_POS_W_FLOAT_ENA(shader->config.spi_ps_input_ena) &&
7823 !(shader->config.spi_ps_input_ena & 0xf)) {
7824 shader->config.spi_ps_input_ena |= S_0286CC_PERSP_CENTER_ENA(1);
7825 assert(G_0286CC_PERSP_CENTER_ENA(shader->config.spi_ps_input_addr));
7826 }
7827
7828 /* At least one pair of interpolation weights must be enabled. */
7829 if (!(shader->config.spi_ps_input_ena & 0x7f)) {
7830 shader->config.spi_ps_input_ena |= S_0286CC_LINEAR_CENTER_ENA(1);
7831 assert(G_0286CC_LINEAR_CENTER_ENA(shader->config.spi_ps_input_addr));
7832 }
7833
7834 /* Samplemask fixup requires the sample ID. */
7835 if (shader->key.part.ps.prolog.samplemask_log_ps_iter) {
7836 shader->config.spi_ps_input_ena |= S_0286CC_ANCILLARY_ENA(1);
7837 assert(G_0286CC_ANCILLARY_ENA(shader->config.spi_ps_input_addr));
7838 }
7839
7840 /* The sample mask input is always enabled, because the API shader always
7841 * passes it through to the epilog. Disable it here if it's unused.
7842 */
7843 if (!shader->key.part.ps.epilog.poly_line_smoothing &&
7844 !shader->selector->info.reads_samplemask)
7845 shader->config.spi_ps_input_ena &= C_0286CC_SAMPLE_COVERAGE_ENA;
7846
7847 return true;
7848 }
7849
7850 void si_multiwave_lds_size_workaround(struct si_screen *sscreen,
7851 unsigned *lds_size)
7852 {
7853 /* SPI barrier management bug:
7854 * Make sure we have at least 4k of LDS in use to avoid the bug.
7855 * It applies to workgroup sizes of more than one wavefront.
7856 */
7857 if (sscreen->info.family == CHIP_BONAIRE ||
7858 sscreen->info.family == CHIP_KABINI ||
7859 sscreen->info.family == CHIP_MULLINS)
7860 *lds_size = MAX2(*lds_size, 8);
7861 }
7862
7863 static void si_fix_resource_usage(struct si_screen *sscreen,
7864 struct si_shader *shader)
7865 {
7866 unsigned min_sgprs = shader->info.num_input_sgprs + 2; /* VCC */
7867
7868 shader->config.num_sgprs = MAX2(shader->config.num_sgprs, min_sgprs);
7869
7870 if (shader->selector->type == PIPE_SHADER_COMPUTE &&
7871 si_get_max_workgroup_size(shader) > 64) {
7872 si_multiwave_lds_size_workaround(sscreen,
7873 &shader->config.lds_size);
7874 }
7875 }
7876
7877 int si_shader_create(struct si_screen *sscreen, LLVMTargetMachineRef tm,
7878 struct si_shader *shader,
7879 struct pipe_debug_callback *debug)
7880 {
7881 struct si_shader_selector *sel = shader->selector;
7882 struct si_shader *mainp = *si_get_main_shader_part(sel, &shader->key);
7883 int r;
7884
7885 /* LS, ES, VS are compiled on demand if the main part hasn't been
7886 * compiled for that stage.
7887 *
7888 * Vertex shaders are compiled on demand when a vertex fetch
7889 * workaround must be applied.
7890 */
7891 if (shader->is_monolithic) {
7892 /* Monolithic shader (compiled as a whole, has many variants,
7893 * may take a long time to compile).
7894 */
7895 r = si_compile_tgsi_shader(sscreen, tm, shader, true, debug);
7896 if (r)
7897 return r;
7898 } else {
7899 /* The shader consists of several parts:
7900 *
7901 * - the middle part is the user shader, it has 1 variant only
7902 * and it was compiled during the creation of the shader
7903 * selector
7904 * - the prolog part is inserted at the beginning
7905 * - the epilog part is inserted at the end
7906 *
7907 * The prolog and epilog have many (but simple) variants.
7908 *
7909 * Starting with gfx9, geometry and tessellation control
7910 * shaders also contain the prolog and user shader parts of
7911 * the previous shader stage.
7912 */
7913
7914 if (!mainp)
7915 return -1;
7916
7917 /* Copy the compiled TGSI shader data over. */
7918 shader->is_binary_shared = true;
7919 shader->binary = mainp->binary;
7920 shader->config = mainp->config;
7921 shader->info.num_input_sgprs = mainp->info.num_input_sgprs;
7922 shader->info.num_input_vgprs = mainp->info.num_input_vgprs;
7923 shader->info.face_vgpr_index = mainp->info.face_vgpr_index;
7924 shader->info.ancillary_vgpr_index = mainp->info.ancillary_vgpr_index;
7925 memcpy(shader->info.vs_output_param_offset,
7926 mainp->info.vs_output_param_offset,
7927 sizeof(mainp->info.vs_output_param_offset));
7928 shader->info.uses_instanceid = mainp->info.uses_instanceid;
7929 shader->info.nr_pos_exports = mainp->info.nr_pos_exports;
7930 shader->info.nr_param_exports = mainp->info.nr_param_exports;
7931
7932 /* Select prologs and/or epilogs. */
7933 switch (sel->type) {
7934 case PIPE_SHADER_VERTEX:
7935 if (!si_shader_select_vs_parts(sscreen, tm, shader, debug))
7936 return -1;
7937 break;
7938 case PIPE_SHADER_TESS_CTRL:
7939 if (!si_shader_select_tcs_parts(sscreen, tm, shader, debug))
7940 return -1;
7941 break;
7942 case PIPE_SHADER_TESS_EVAL:
7943 break;
7944 case PIPE_SHADER_GEOMETRY:
7945 if (!si_shader_select_gs_parts(sscreen, tm, shader, debug))
7946 return -1;
7947 break;
7948 case PIPE_SHADER_FRAGMENT:
7949 if (!si_shader_select_ps_parts(sscreen, tm, shader, debug))
7950 return -1;
7951
7952 /* Make sure we have at least as many VGPRs as there
7953 * are allocated inputs.
7954 */
7955 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
7956 shader->info.num_input_vgprs);
7957 break;
7958 }
7959
7960 /* Update SGPR and VGPR counts. */
7961 if (shader->prolog) {
7962 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
7963 shader->prolog->config.num_sgprs);
7964 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
7965 shader->prolog->config.num_vgprs);
7966 }
7967 if (shader->previous_stage) {
7968 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
7969 shader->previous_stage->config.num_sgprs);
7970 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
7971 shader->previous_stage->config.num_vgprs);
7972 shader->config.spilled_sgprs =
7973 MAX2(shader->config.spilled_sgprs,
7974 shader->previous_stage->config.spilled_sgprs);
7975 shader->config.spilled_vgprs =
7976 MAX2(shader->config.spilled_vgprs,
7977 shader->previous_stage->config.spilled_vgprs);
7978 shader->config.private_mem_vgprs =
7979 MAX2(shader->config.private_mem_vgprs,
7980 shader->previous_stage->config.private_mem_vgprs);
7981 shader->config.scratch_bytes_per_wave =
7982 MAX2(shader->config.scratch_bytes_per_wave,
7983 shader->previous_stage->config.scratch_bytes_per_wave);
7984 shader->info.uses_instanceid |=
7985 shader->previous_stage->info.uses_instanceid;
7986 }
7987 if (shader->prolog2) {
7988 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
7989 shader->prolog2->config.num_sgprs);
7990 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
7991 shader->prolog2->config.num_vgprs);
7992 }
7993 if (shader->epilog) {
7994 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
7995 shader->epilog->config.num_sgprs);
7996 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
7997 shader->epilog->config.num_vgprs);
7998 }
7999 si_calculate_max_simd_waves(shader);
8000 }
8001
8002 si_fix_resource_usage(sscreen, shader);
8003 si_shader_dump(sscreen, shader, debug, sel->info.processor,
8004 stderr, true);
8005
8006 /* Upload. */
8007 r = si_shader_binary_upload(sscreen, shader);
8008 if (r) {
8009 fprintf(stderr, "LLVM failed to upload shader\n");
8010 return r;
8011 }
8012
8013 return 0;
8014 }
8015
8016 void si_shader_destroy(struct si_shader *shader)
8017 {
8018 if (shader->scratch_bo)
8019 r600_resource_reference(&shader->scratch_bo, NULL);
8020
8021 r600_resource_reference(&shader->bo, NULL);
8022
8023 if (!shader->is_binary_shared)
8024 ac_shader_binary_clean(&shader->binary);
8025
8026 free(shader->shader_log);
8027 }