projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
ac,radeonsi: move some VS input descriptions to ac_shader_abi
[mesa.git] / src / gallium / drivers / radeonsi / si_shader.c
1 /*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 *
23 * Authors:
24 * Tom Stellard <thomas.stellard@amd.com>
25 * Michel Dänzer <michel.daenzer@amd.com>
26 * Christian König <christian.koenig@amd.com>
27 */
28
29 #include "gallivm/lp_bld_const.h"
30 #include "gallivm/lp_bld_gather.h"
31 #include "gallivm/lp_bld_intr.h"
32 #include "gallivm/lp_bld_logic.h"
33 #include "gallivm/lp_bld_arit.h"
34 #include "gallivm/lp_bld_flow.h"
35 #include "gallivm/lp_bld_misc.h"
36 #include "util/u_memory.h"
37 #include "util/u_string.h"
38 #include "tgsi/tgsi_build.h"
39 #include "tgsi/tgsi_util.h"
40 #include "tgsi/tgsi_dump.h"
41
42 #include "ac_binary.h"
43 #include "ac_llvm_util.h"
44 #include "ac_exp_param.h"
45 #include "si_shader_internal.h"
46 #include "si_pipe.h"
47 #include "sid.h"
48
49
50 static const char *scratch_rsrc_dword0_symbol =
51 "SCRATCH_RSRC_DWORD0";
52
53 static const char *scratch_rsrc_dword1_symbol =
54 "SCRATCH_RSRC_DWORD1";
55
56 struct si_shader_output_values
57 {
58 LLVMValueRef values[4];
59 unsigned semantic_name;
60 unsigned semantic_index;
61 ubyte vertex_stream[4];
62 };
63
64 /**
65 * Used to collect types and other info about arguments of the LLVM function
66 * before the function is created.
67 */
68 struct si_function_info {
69 LLVMTypeRef types[100];
70 LLVMValueRef *assign[100];
71 unsigned num_sgpr_params;
72 unsigned num_params;
73 };
74
75 enum si_arg_regfile {
76 ARG_SGPR,
77 ARG_VGPR
78 };
79
80 static void si_init_shader_ctx(struct si_shader_context *ctx,
81 struct si_screen *sscreen,
82 LLVMTargetMachineRef tm);
83
84 static void si_llvm_emit_barrier(const struct lp_build_tgsi_action *action,
85 struct lp_build_tgsi_context *bld_base,
86 struct lp_build_emit_data *emit_data);
87
88 static void si_dump_shader_key(unsigned processor, const struct si_shader *shader,
89 FILE *f);
90
91 static unsigned llvm_get_type_size(LLVMTypeRef type);
92
93 static void si_build_vs_prolog_function(struct si_shader_context *ctx,
94 union si_shader_part_key *key);
95 static void si_build_tcs_epilog_function(struct si_shader_context *ctx,
96 union si_shader_part_key *key);
97 static void si_build_ps_prolog_function(struct si_shader_context *ctx,
98 union si_shader_part_key *key);
99 static void si_build_ps_epilog_function(struct si_shader_context *ctx,
100 union si_shader_part_key *key);
101
102 /* Ideally pass the sample mask input to the PS epilog as v13, which
103 * is its usual location, so that the shader doesn't have to add v_mov.
104 */
105 #define PS_EPILOG_SAMPLEMASK_MIN_LOC 13
106
107 enum {
108 CONST_ADDR_SPACE = 2,
109 LOCAL_ADDR_SPACE = 3,
110 };
111
112 static bool is_merged_shader(struct si_shader *shader)
113 {
114 if (shader->selector->screen->b.chip_class <= VI)
115 return false;
116
117 return shader->key.as_ls ||
118 shader->key.as_es ||
119 shader->selector->type == PIPE_SHADER_TESS_CTRL ||
120 shader->selector->type == PIPE_SHADER_GEOMETRY;
121 }
122
123 static void si_init_function_info(struct si_function_info *fninfo)
124 {
125 fninfo->num_params = 0;
126 fninfo->num_sgpr_params = 0;
127 }
128
129 static unsigned add_arg_assign(struct si_function_info *fninfo,
130 enum si_arg_regfile regfile, LLVMTypeRef type,
131 LLVMValueRef *assign)
132 {
133 assert(regfile != ARG_SGPR || fninfo->num_sgpr_params == fninfo->num_params);
134
135 unsigned idx = fninfo->num_params++;
136 assert(idx < ARRAY_SIZE(fninfo->types));
137
138 if (regfile == ARG_SGPR)
139 fninfo->num_sgpr_params = fninfo->num_params;
140
141 fninfo->types[idx] = type;
142 fninfo->assign[idx] = assign;
143 return idx;
144 }
145
146 static unsigned add_arg(struct si_function_info *fninfo,
147 enum si_arg_regfile regfile, LLVMTypeRef type)
148 {
149 return add_arg_assign(fninfo, regfile, type, NULL);
150 }
151
152 static void add_arg_checked(struct si_function_info *fninfo,
153 enum si_arg_regfile regfile, LLVMTypeRef type,
154 unsigned idx)
155 {
156 MAYBE_UNUSED unsigned actual = add_arg(fninfo, regfile, type);
157 assert(actual == idx);
158 }
159
160 /**
161 * Returns a unique index for a per-patch semantic name and index. The index
162 * must be less than 32, so that a 32-bit bitmask of used inputs or outputs
163 * can be calculated.
164 */
165 unsigned si_shader_io_get_unique_index_patch(unsigned semantic_name, unsigned index)
166 {
167 switch (semantic_name) {
168 case TGSI_SEMANTIC_TESSOUTER:
169 return 0;
170 case TGSI_SEMANTIC_TESSINNER:
171 return 1;
172 case TGSI_SEMANTIC_PATCH:
173 assert(index < 30);
174 return 2 + index;
175
176 default:
177 assert(!"invalid semantic name");
178 return 0;
179 }
180 }
181
182 /**
183 * Returns a unique index for a semantic name and index. The index must be
184 * less than 64, so that a 64-bit bitmask of used inputs or outputs can be
185 * calculated.
186 */
187 unsigned si_shader_io_get_unique_index(unsigned semantic_name, unsigned index)
188 {
189 switch (semantic_name) {
190 case TGSI_SEMANTIC_POSITION:
191 return 0;
192 case TGSI_SEMANTIC_GENERIC:
193 /* Since some shader stages use the the highest used IO index
194 * to determine the size to allocate for inputs/outputs
195 * (in LDS, tess and GS rings). GENERIC should be placed right
196 * after POSITION to make that size as small as possible.
197 */
198 if (index < SI_MAX_IO_GENERIC)
199 return 1 + index;
200
201 assert(!"invalid generic index");
202 return 0;
203 case TGSI_SEMANTIC_PSIZE:
204 return SI_MAX_IO_GENERIC + 1;
205 case TGSI_SEMANTIC_CLIPDIST:
206 assert(index <= 1);
207 return SI_MAX_IO_GENERIC + 2 + index;
208 case TGSI_SEMANTIC_FOG:
209 return SI_MAX_IO_GENERIC + 4;
210 case TGSI_SEMANTIC_LAYER:
211 return SI_MAX_IO_GENERIC + 5;
212 case TGSI_SEMANTIC_VIEWPORT_INDEX:
213 return SI_MAX_IO_GENERIC + 6;
214 case TGSI_SEMANTIC_PRIMID:
215 return SI_MAX_IO_GENERIC + 7;
216 case TGSI_SEMANTIC_COLOR: /* these alias */
217 case TGSI_SEMANTIC_BCOLOR:
218 assert(index < 2);
219 return SI_MAX_IO_GENERIC + 8 + index;
220 case TGSI_SEMANTIC_TEXCOORD:
221 assert(index < 8);
222 assert(SI_MAX_IO_GENERIC + 10 + index < 64);
223 return SI_MAX_IO_GENERIC + 10 + index;
224 default:
225 assert(!"invalid semantic name");
226 return 0;
227 }
228 }
229
230 /**
231 * Helper function that builds an LLVM IR PHI node and immediately adds
232 * incoming edges.
233 */
234 static LLVMValueRef
235 build_phi(struct ac_llvm_context *ctx, LLVMTypeRef type,
236 unsigned count_incoming, LLVMValueRef *values,
237 LLVMBasicBlockRef *blocks)
238 {
239 LLVMValueRef phi = LLVMBuildPhi(ctx->builder, type, "");
240 LLVMAddIncoming(phi, values, blocks, count_incoming);
241 return phi;
242 }
243
244 /**
245 * Get the value of a shader input parameter and extract a bitfield.
246 */
247 static LLVMValueRef unpack_param(struct si_shader_context *ctx,
248 unsigned param, unsigned rshift,
249 unsigned bitwidth)
250 {
251 struct gallivm_state *gallivm = &ctx->gallivm;
252 LLVMValueRef value = LLVMGetParam(ctx->main_fn,
253 param);
254
255 if (LLVMGetTypeKind(LLVMTypeOf(value)) == LLVMFloatTypeKind)
256 value = bitcast(&ctx->bld_base,
257 TGSI_TYPE_UNSIGNED, value);
258
259 if (rshift)
260 value = LLVMBuildLShr(gallivm->builder, value,
261 LLVMConstInt(ctx->i32, rshift, 0), "");
262
263 if (rshift + bitwidth < 32) {
264 unsigned mask = (1 << bitwidth) - 1;
265 value = LLVMBuildAnd(gallivm->builder, value,
266 LLVMConstInt(ctx->i32, mask, 0), "");
267 }
268
269 return value;
270 }
271
272 static LLVMValueRef get_rel_patch_id(struct si_shader_context *ctx)
273 {
274 switch (ctx->type) {
275 case PIPE_SHADER_TESS_CTRL:
276 return unpack_param(ctx, ctx->param_tcs_rel_ids, 0, 8);
277
278 case PIPE_SHADER_TESS_EVAL:
279 return LLVMGetParam(ctx->main_fn,
280 ctx->param_tes_rel_patch_id);
281
282 default:
283 assert(0);
284 return NULL;
285 }
286 }
287
288 /* Tessellation shaders pass outputs to the next shader using LDS.
289 *
290 * LS outputs = TCS inputs
291 * TCS outputs = TES inputs
292 *
293 * The LDS layout is:
294 * - TCS inputs for patch 0
295 * - TCS inputs for patch 1
296 * - TCS inputs for patch 2 = get_tcs_in_current_patch_offset (if RelPatchID==2)
297 * - ...
298 * - TCS outputs for patch 0 = get_tcs_out_patch0_offset
299 * - Per-patch TCS outputs for patch 0 = get_tcs_out_patch0_patch_data_offset
300 * - TCS outputs for patch 1
301 * - Per-patch TCS outputs for patch 1
302 * - TCS outputs for patch 2 = get_tcs_out_current_patch_offset (if RelPatchID==2)
303 * - Per-patch TCS outputs for patch 2 = get_tcs_out_current_patch_data_offset (if RelPatchID==2)
304 * - ...
305 *
306 * All three shaders VS(LS), TCS, TES share the same LDS space.
307 */
308
309 static LLVMValueRef
310 get_tcs_in_patch_stride(struct si_shader_context *ctx)
311 {
312 return unpack_param(ctx, ctx->param_vs_state_bits, 8, 13);
313 }
314
315 static LLVMValueRef
316 get_tcs_out_patch_stride(struct si_shader_context *ctx)
317 {
318 return unpack_param(ctx, ctx->param_tcs_out_lds_layout, 0, 13);
319 }
320
321 static LLVMValueRef
322 get_tcs_out_patch0_offset(struct si_shader_context *ctx)
323 {
324 return lp_build_mul_imm(&ctx->bld_base.uint_bld,
325 unpack_param(ctx,
326 ctx->param_tcs_out_lds_offsets,
327 0, 16),
328 4);
329 }
330
331 static LLVMValueRef
332 get_tcs_out_patch0_patch_data_offset(struct si_shader_context *ctx)
333 {
334 return lp_build_mul_imm(&ctx->bld_base.uint_bld,
335 unpack_param(ctx,
336 ctx->param_tcs_out_lds_offsets,
337 16, 16),
338 4);
339 }
340
341 static LLVMValueRef
342 get_tcs_in_current_patch_offset(struct si_shader_context *ctx)
343 {
344 struct gallivm_state *gallivm = &ctx->gallivm;
345 LLVMValueRef patch_stride = get_tcs_in_patch_stride(ctx);
346 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
347
348 return LLVMBuildMul(gallivm->builder, patch_stride, rel_patch_id, "");
349 }
350
351 static LLVMValueRef
352 get_tcs_out_current_patch_offset(struct si_shader_context *ctx)
353 {
354 struct gallivm_state *gallivm = &ctx->gallivm;
355 LLVMValueRef patch0_offset = get_tcs_out_patch0_offset(ctx);
356 LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
357 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
358
359 return LLVMBuildAdd(gallivm->builder, patch0_offset,
360 LLVMBuildMul(gallivm->builder, patch_stride,
361 rel_patch_id, ""),
362 "");
363 }
364
365 static LLVMValueRef
366 get_tcs_out_current_patch_data_offset(struct si_shader_context *ctx)
367 {
368 struct gallivm_state *gallivm = &ctx->gallivm;
369 LLVMValueRef patch0_patch_data_offset =
370 get_tcs_out_patch0_patch_data_offset(ctx);
371 LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
372 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
373
374 return LLVMBuildAdd(gallivm->builder, patch0_patch_data_offset,
375 LLVMBuildMul(gallivm->builder, patch_stride,
376 rel_patch_id, ""),
377 "");
378 }
379
380 static LLVMValueRef get_instance_index_for_fetch(
381 struct si_shader_context *ctx,
382 unsigned param_start_instance, LLVMValueRef divisor)
383 {
384 struct gallivm_state *gallivm = &ctx->gallivm;
385
386 LLVMValueRef result = ctx->abi.instance_id;
387
388 /* The division must be done before START_INSTANCE is added. */
389 if (divisor != ctx->i32_1)
390 result = LLVMBuildUDiv(gallivm->builder, result, divisor, "");
391
392 return LLVMBuildAdd(gallivm->builder, result,
393 LLVMGetParam(ctx->main_fn, param_start_instance), "");
394 }
395
396 /* Bitcast <4 x float> to <2 x double>, extract the component, and convert
397 * to float. */
398 static LLVMValueRef extract_double_to_float(struct si_shader_context *ctx,
399 LLVMValueRef vec4,
400 unsigned double_index)
401 {
402 LLVMBuilderRef builder = ctx->gallivm.builder;
403 LLVMTypeRef f64 = LLVMDoubleTypeInContext(ctx->gallivm.context);
404 LLVMValueRef dvec2 = LLVMBuildBitCast(builder, vec4,
405 LLVMVectorType(f64, 2), "");
406 LLVMValueRef index = LLVMConstInt(ctx->i32, double_index, 0);
407 LLVMValueRef value = LLVMBuildExtractElement(builder, dvec2, index, "");
408 return LLVMBuildFPTrunc(builder, value, ctx->f32, "");
409 }
410
411 static void declare_input_vs(
412 struct si_shader_context *ctx,
413 unsigned input_index,
414 const struct tgsi_full_declaration *decl,
415 LLVMValueRef out[4])
416 {
417 struct gallivm_state *gallivm = &ctx->gallivm;
418
419 unsigned chan;
420 unsigned fix_fetch;
421 unsigned num_fetches;
422 unsigned fetch_stride;
423
424 LLVMValueRef t_list_ptr;
425 LLVMValueRef t_offset;
426 LLVMValueRef t_list;
427 LLVMValueRef vertex_index;
428 LLVMValueRef input[3];
429
430 /* Load the T list */
431 t_list_ptr = LLVMGetParam(ctx->main_fn, ctx->param_vertex_buffers);
432
433 t_offset = LLVMConstInt(ctx->i32, input_index, 0);
434
435 t_list = ac_build_indexed_load_const(&ctx->ac, t_list_ptr, t_offset);
436
437 vertex_index = LLVMGetParam(ctx->main_fn,
438 ctx->param_vertex_index0 +
439 input_index);
440
441 fix_fetch = ctx->shader->key.mono.vs_fix_fetch[input_index];
442
443 /* Do multiple loads for special formats. */
444 switch (fix_fetch) {
445 case SI_FIX_FETCH_RGB_64_FLOAT:
446 num_fetches = 3; /* 3 2-dword loads */
447 fetch_stride = 8;
448 break;
449 case SI_FIX_FETCH_RGBA_64_FLOAT:
450 num_fetches = 2; /* 2 4-dword loads */
451 fetch_stride = 16;
452 break;
453 case SI_FIX_FETCH_RGB_8:
454 case SI_FIX_FETCH_RGB_8_INT:
455 num_fetches = 3;
456 fetch_stride = 1;
457 break;
458 case SI_FIX_FETCH_RGB_16:
459 case SI_FIX_FETCH_RGB_16_INT:
460 num_fetches = 3;
461 fetch_stride = 2;
462 break;
463 default:
464 num_fetches = 1;
465 fetch_stride = 0;
466 }
467
468 for (unsigned i = 0; i < num_fetches; i++) {
469 LLVMValueRef voffset = LLVMConstInt(ctx->i32, fetch_stride * i, 0);
470
471 input[i] = ac_build_buffer_load_format(&ctx->ac, t_list,
472 vertex_index, voffset,
473 true);
474 }
475
476 /* Break up the vec4 into individual components */
477 for (chan = 0; chan < 4; chan++) {
478 LLVMValueRef llvm_chan = LLVMConstInt(ctx->i32, chan, 0);
479 out[chan] = LLVMBuildExtractElement(gallivm->builder,
480 input[0], llvm_chan, "");
481 }
482
483 switch (fix_fetch) {
484 case SI_FIX_FETCH_A2_SNORM:
485 case SI_FIX_FETCH_A2_SSCALED:
486 case SI_FIX_FETCH_A2_SINT: {
487 /* The hardware returns an unsigned value; convert it to a
488 * signed one.
489 */
490 LLVMValueRef tmp = out[3];
491 LLVMValueRef c30 = LLVMConstInt(ctx->i32, 30, 0);
492
493 /* First, recover the sign-extended signed integer value. */
494 if (fix_fetch == SI_FIX_FETCH_A2_SSCALED)
495 tmp = LLVMBuildFPToUI(gallivm->builder, tmp, ctx->i32, "");
496 else
497 tmp = LLVMBuildBitCast(gallivm->builder, tmp, ctx->i32, "");
498
499 /* For the integer-like cases, do a natural sign extension.
500 *
501 * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
502 * and happen to contain 0, 1, 2, 3 as the two LSBs of the
503 * exponent.
504 */
505 tmp = LLVMBuildShl(gallivm->builder, tmp,
506 fix_fetch == SI_FIX_FETCH_A2_SNORM ?
507 LLVMConstInt(ctx->i32, 7, 0) : c30, "");
508 tmp = LLVMBuildAShr(gallivm->builder, tmp, c30, "");
509
510 /* Convert back to the right type. */
511 if (fix_fetch == SI_FIX_FETCH_A2_SNORM) {
512 LLVMValueRef clamp;
513 LLVMValueRef neg_one = LLVMConstReal(ctx->f32, -1.0);
514 tmp = LLVMBuildSIToFP(gallivm->builder, tmp, ctx->f32, "");
515 clamp = LLVMBuildFCmp(gallivm->builder, LLVMRealULT, tmp, neg_one, "");
516 tmp = LLVMBuildSelect(gallivm->builder, clamp, neg_one, tmp, "");
517 } else if (fix_fetch == SI_FIX_FETCH_A2_SSCALED) {
518 tmp = LLVMBuildSIToFP(gallivm->builder, tmp, ctx->f32, "");
519 }
520
521 out[3] = tmp;
522 break;
523 }
524 case SI_FIX_FETCH_RGBA_32_UNORM:
525 case SI_FIX_FETCH_RGBX_32_UNORM:
526 for (chan = 0; chan < 4; chan++) {
527 out[chan] = LLVMBuildBitCast(gallivm->builder, out[chan],
528 ctx->i32, "");
529 out[chan] = LLVMBuildUIToFP(gallivm->builder,
530 out[chan], ctx->f32, "");
531 out[chan] = LLVMBuildFMul(gallivm->builder, out[chan],
532 LLVMConstReal(ctx->f32, 1.0 / UINT_MAX), "");
533 }
534 /* RGBX UINT returns 1 in alpha, which would be rounded to 0 by normalizing. */
535 if (fix_fetch == SI_FIX_FETCH_RGBX_32_UNORM)
536 out[3] = LLVMConstReal(ctx->f32, 1);
537 break;
538 case SI_FIX_FETCH_RGBA_32_SNORM:
539 case SI_FIX_FETCH_RGBX_32_SNORM:
540 case SI_FIX_FETCH_RGBA_32_FIXED:
541 case SI_FIX_FETCH_RGBX_32_FIXED: {
542 double scale;
543 if (fix_fetch >= SI_FIX_FETCH_RGBA_32_FIXED)
544 scale = 1.0 / 0x10000;
545 else
546 scale = 1.0 / INT_MAX;
547
548 for (chan = 0; chan < 4; chan++) {
549 out[chan] = LLVMBuildBitCast(gallivm->builder, out[chan],
550 ctx->i32, "");
551 out[chan] = LLVMBuildSIToFP(gallivm->builder,
552 out[chan], ctx->f32, "");
553 out[chan] = LLVMBuildFMul(gallivm->builder, out[chan],
554 LLVMConstReal(ctx->f32, scale), "");
555 }
556 /* RGBX SINT returns 1 in alpha, which would be rounded to 0 by normalizing. */
557 if (fix_fetch == SI_FIX_FETCH_RGBX_32_SNORM ||
558 fix_fetch == SI_FIX_FETCH_RGBX_32_FIXED)
559 out[3] = LLVMConstReal(ctx->f32, 1);
560 break;
561 }
562 case SI_FIX_FETCH_RGBA_32_USCALED:
563 for (chan = 0; chan < 4; chan++) {
564 out[chan] = LLVMBuildBitCast(gallivm->builder, out[chan],
565 ctx->i32, "");
566 out[chan] = LLVMBuildUIToFP(gallivm->builder,
567 out[chan], ctx->f32, "");
568 }
569 break;
570 case SI_FIX_FETCH_RGBA_32_SSCALED:
571 for (chan = 0; chan < 4; chan++) {
572 out[chan] = LLVMBuildBitCast(gallivm->builder, out[chan],
573 ctx->i32, "");
574 out[chan] = LLVMBuildSIToFP(gallivm->builder,
575 out[chan], ctx->f32, "");
576 }
577 break;
578 case SI_FIX_FETCH_RG_64_FLOAT:
579 for (chan = 0; chan < 2; chan++)
580 out[chan] = extract_double_to_float(ctx, input[0], chan);
581
582 out[2] = LLVMConstReal(ctx->f32, 0);
583 out[3] = LLVMConstReal(ctx->f32, 1);
584 break;
585 case SI_FIX_FETCH_RGB_64_FLOAT:
586 for (chan = 0; chan < 3; chan++)
587 out[chan] = extract_double_to_float(ctx, input[chan], 0);
588
589 out[3] = LLVMConstReal(ctx->f32, 1);
590 break;
591 case SI_FIX_FETCH_RGBA_64_FLOAT:
592 for (chan = 0; chan < 4; chan++) {
593 out[chan] = extract_double_to_float(ctx, input[chan / 2],
594 chan % 2);
595 }
596 break;
597 case SI_FIX_FETCH_RGB_8:
598 case SI_FIX_FETCH_RGB_8_INT:
599 case SI_FIX_FETCH_RGB_16:
600 case SI_FIX_FETCH_RGB_16_INT:
601 for (chan = 0; chan < 3; chan++) {
602 out[chan] = LLVMBuildExtractElement(gallivm->builder,
603 input[chan],
604 ctx->i32_0, "");
605 }
606 if (fix_fetch == SI_FIX_FETCH_RGB_8 ||
607 fix_fetch == SI_FIX_FETCH_RGB_16) {
608 out[3] = LLVMConstReal(ctx->f32, 1);
609 } else {
610 out[3] = LLVMBuildBitCast(gallivm->builder, ctx->i32_1,
611 ctx->f32, "");
612 }
613 break;
614 }
615 }
616
617 static LLVMValueRef get_primitive_id(struct lp_build_tgsi_context *bld_base,
618 unsigned swizzle)
619 {
620 struct si_shader_context *ctx = si_shader_context(bld_base);
621
622 if (swizzle > 0)
623 return ctx->i32_0;
624
625 switch (ctx->type) {
626 case PIPE_SHADER_VERTEX:
627 return LLVMGetParam(ctx->main_fn,
628 ctx->param_vs_prim_id);
629 case PIPE_SHADER_TESS_CTRL:
630 return LLVMGetParam(ctx->main_fn,
631 ctx->param_tcs_patch_id);
632 case PIPE_SHADER_TESS_EVAL:
633 return LLVMGetParam(ctx->main_fn,
634 ctx->param_tes_patch_id);
635 case PIPE_SHADER_GEOMETRY:
636 return LLVMGetParam(ctx->main_fn,
637 ctx->param_gs_prim_id);
638 default:
639 assert(0);
640 return ctx->i32_0;
641 }
642 }
643
644 /**
645 * Return the value of tgsi_ind_register for indexing.
646 * This is the indirect index with the constant offset added to it.
647 */
648 static LLVMValueRef get_indirect_index(struct si_shader_context *ctx,
649 const struct tgsi_ind_register *ind,
650 int rel_index)
651 {
652 struct gallivm_state *gallivm = &ctx->gallivm;
653 LLVMValueRef result;
654
655 result = ctx->addrs[ind->Index][ind->Swizzle];
656 result = LLVMBuildLoad(gallivm->builder, result, "");
657 result = LLVMBuildAdd(gallivm->builder, result,
658 LLVMConstInt(ctx->i32, rel_index, 0), "");
659 return result;
660 }
661
662 /**
663 * Like get_indirect_index, but restricts the return value to a (possibly
664 * undefined) value inside [0..num).
665 */
666 LLVMValueRef si_get_bounded_indirect_index(struct si_shader_context *ctx,
667 const struct tgsi_ind_register *ind,
668 int rel_index, unsigned num)
669 {
670 LLVMValueRef result = get_indirect_index(ctx, ind, rel_index);
671
672 return si_llvm_bound_index(ctx, result, num);
673 }
674
675
676 /**
677 * Calculate a dword address given an input or output register and a stride.
678 */
679 static LLVMValueRef get_dw_address(struct si_shader_context *ctx,
680 const struct tgsi_full_dst_register *dst,
681 const struct tgsi_full_src_register *src,
682 LLVMValueRef vertex_dw_stride,
683 LLVMValueRef base_addr)
684 {
685 struct gallivm_state *gallivm = &ctx->gallivm;
686 struct tgsi_shader_info *info = &ctx->shader->selector->info;
687 ubyte *name, *index, *array_first;
688 int first, param;
689 struct tgsi_full_dst_register reg;
690
691 /* Set the register description. The address computation is the same
692 * for sources and destinations. */
693 if (src) {
694 reg.Register.File = src->Register.File;
695 reg.Register.Index = src->Register.Index;
696 reg.Register.Indirect = src->Register.Indirect;
697 reg.Register.Dimension = src->Register.Dimension;
698 reg.Indirect = src->Indirect;
699 reg.Dimension = src->Dimension;
700 reg.DimIndirect = src->DimIndirect;
701 } else
702 reg = *dst;
703
704 /* If the register is 2-dimensional (e.g. an array of vertices
705 * in a primitive), calculate the base address of the vertex. */
706 if (reg.Register.Dimension) {
707 LLVMValueRef index;
708
709 if (reg.Dimension.Indirect)
710 index = get_indirect_index(ctx, &reg.DimIndirect,
711 reg.Dimension.Index);
712 else
713 index = LLVMConstInt(ctx->i32, reg.Dimension.Index, 0);
714
715 base_addr = LLVMBuildAdd(gallivm->builder, base_addr,
716 LLVMBuildMul(gallivm->builder, index,
717 vertex_dw_stride, ""), "");
718 }
719
720 /* Get information about the register. */
721 if (reg.Register.File == TGSI_FILE_INPUT) {
722 name = info->input_semantic_name;
723 index = info->input_semantic_index;
724 array_first = info->input_array_first;
725 } else if (reg.Register.File == TGSI_FILE_OUTPUT) {
726 name = info->output_semantic_name;
727 index = info->output_semantic_index;
728 array_first = info->output_array_first;
729 } else {
730 assert(0);
731 return NULL;
732 }
733
734 if (reg.Register.Indirect) {
735 /* Add the relative address of the element. */
736 LLVMValueRef ind_index;
737
738 if (reg.Indirect.ArrayID)
739 first = array_first[reg.Indirect.ArrayID];
740 else
741 first = reg.Register.Index;
742
743 ind_index = get_indirect_index(ctx, &reg.Indirect,
744 reg.Register.Index - first);
745
746 base_addr = LLVMBuildAdd(gallivm->builder, base_addr,
747 LLVMBuildMul(gallivm->builder, ind_index,
748 LLVMConstInt(ctx->i32, 4, 0), ""), "");
749
750 param = reg.Register.Dimension ?
751 si_shader_io_get_unique_index(name[first], index[first]) :
752 si_shader_io_get_unique_index_patch(name[first], index[first]);
753 } else {
754 param = reg.Register.Dimension ?
755 si_shader_io_get_unique_index(name[reg.Register.Index],
756 index[reg.Register.Index]) :
757 si_shader_io_get_unique_index_patch(name[reg.Register.Index],
758 index[reg.Register.Index]);
759 }
760
761 /* Add the base address of the element. */
762 return LLVMBuildAdd(gallivm->builder, base_addr,
763 LLVMConstInt(ctx->i32, param * 4, 0), "");
764 }
765
766 /* The offchip buffer layout for TCS->TES is
767 *
768 * - attribute 0 of patch 0 vertex 0
769 * - attribute 0 of patch 0 vertex 1
770 * - attribute 0 of patch 0 vertex 2
771 * ...
772 * - attribute 0 of patch 1 vertex 0
773 * - attribute 0 of patch 1 vertex 1
774 * ...
775 * - attribute 1 of patch 0 vertex 0
776 * - attribute 1 of patch 0 vertex 1
777 * ...
778 * - per patch attribute 0 of patch 0
779 * - per patch attribute 0 of patch 1
780 * ...
781 *
782 * Note that every attribute has 4 components.
783 */
784 static LLVMValueRef get_tcs_tes_buffer_address(struct si_shader_context *ctx,
785 LLVMValueRef rel_patch_id,
786 LLVMValueRef vertex_index,
787 LLVMValueRef param_index)
788 {
789 struct gallivm_state *gallivm = &ctx->gallivm;
790 LLVMValueRef base_addr, vertices_per_patch, num_patches, total_vertices;
791 LLVMValueRef param_stride, constant16;
792
793 vertices_per_patch = unpack_param(ctx, ctx->param_tcs_offchip_layout, 6, 6);
794 num_patches = unpack_param(ctx, ctx->param_tcs_offchip_layout, 0, 6);
795 total_vertices = LLVMBuildMul(gallivm->builder, vertices_per_patch,
796 num_patches, "");
797
798 constant16 = LLVMConstInt(ctx->i32, 16, 0);
799 if (vertex_index) {
800 base_addr = LLVMBuildMul(gallivm->builder, rel_patch_id,
801 vertices_per_patch, "");
802
803 base_addr = LLVMBuildAdd(gallivm->builder, base_addr,
804 vertex_index, "");
805
806 param_stride = total_vertices;
807 } else {
808 base_addr = rel_patch_id;
809 param_stride = num_patches;
810 }
811
812 base_addr = LLVMBuildAdd(gallivm->builder, base_addr,
813 LLVMBuildMul(gallivm->builder, param_index,
814 param_stride, ""), "");
815
816 base_addr = LLVMBuildMul(gallivm->builder, base_addr, constant16, "");
817
818 if (!vertex_index) {
819 LLVMValueRef patch_data_offset =
820 unpack_param(ctx, ctx->param_tcs_offchip_layout, 12, 20);
821
822 base_addr = LLVMBuildAdd(gallivm->builder, base_addr,
823 patch_data_offset, "");
824 }
825 return base_addr;
826 }
827
828 static LLVMValueRef get_tcs_tes_buffer_address_from_reg(
829 struct si_shader_context *ctx,
830 const struct tgsi_full_dst_register *dst,
831 const struct tgsi_full_src_register *src)
832 {
833 struct gallivm_state *gallivm = &ctx->gallivm;
834 struct tgsi_shader_info *info = &ctx->shader->selector->info;
835 ubyte *name, *index, *array_first;
836 struct tgsi_full_src_register reg;
837 LLVMValueRef vertex_index = NULL;
838 LLVMValueRef param_index = NULL;
839 unsigned param_index_base, param_base;
840
841 reg = src ? *src : tgsi_full_src_register_from_dst(dst);
842
843 if (reg.Register.Dimension) {
844
845 if (reg.Dimension.Indirect)
846 vertex_index = get_indirect_index(ctx, &reg.DimIndirect,
847 reg.Dimension.Index);
848 else
849 vertex_index = LLVMConstInt(ctx->i32, reg.Dimension.Index, 0);
850 }
851
852 /* Get information about the register. */
853 if (reg.Register.File == TGSI_FILE_INPUT) {
854 name = info->input_semantic_name;
855 index = info->input_semantic_index;
856 array_first = info->input_array_first;
857 } else if (reg.Register.File == TGSI_FILE_OUTPUT) {
858 name = info->output_semantic_name;
859 index = info->output_semantic_index;
860 array_first = info->output_array_first;
861 } else {
862 assert(0);
863 return NULL;
864 }
865
866 if (reg.Register.Indirect) {
867 if (reg.Indirect.ArrayID)
868 param_base = array_first[reg.Indirect.ArrayID];
869 else
870 param_base = reg.Register.Index;
871
872 param_index = get_indirect_index(ctx, &reg.Indirect,
873 reg.Register.Index - param_base);
874
875 } else {
876 param_base = reg.Register.Index;
877 param_index = ctx->i32_0;
878 }
879
880 param_index_base = reg.Register.Dimension ?
881 si_shader_io_get_unique_index(name[param_base], index[param_base]) :
882 si_shader_io_get_unique_index_patch(name[param_base], index[param_base]);
883
884 param_index = LLVMBuildAdd(gallivm->builder, param_index,
885 LLVMConstInt(ctx->i32, param_index_base, 0),
886 "");
887
888 return get_tcs_tes_buffer_address(ctx, get_rel_patch_id(ctx),
889 vertex_index, param_index);
890 }
891
892 static LLVMValueRef buffer_load(struct lp_build_tgsi_context *bld_base,
893 enum tgsi_opcode_type type, unsigned swizzle,
894 LLVMValueRef buffer, LLVMValueRef offset,
895 LLVMValueRef base, bool can_speculate)
896 {
897 struct si_shader_context *ctx = si_shader_context(bld_base);
898 struct gallivm_state *gallivm = &ctx->gallivm;
899 LLVMValueRef value, value2;
900 LLVMTypeRef llvm_type = tgsi2llvmtype(bld_base, type);
901 LLVMTypeRef vec_type = LLVMVectorType(llvm_type, 4);
902
903 if (swizzle == ~0) {
904 value = ac_build_buffer_load(&ctx->ac, buffer, 4, NULL, base, offset,
905 0, 1, 0, can_speculate, false);
906
907 return LLVMBuildBitCast(gallivm->builder, value, vec_type, "");
908 }
909
910 if (!tgsi_type_is_64bit(type)) {
911 value = ac_build_buffer_load(&ctx->ac, buffer, 4, NULL, base, offset,
912 0, 1, 0, can_speculate, false);
913
914 value = LLVMBuildBitCast(gallivm->builder, value, vec_type, "");
915 return LLVMBuildExtractElement(gallivm->builder, value,
916 LLVMConstInt(ctx->i32, swizzle, 0), "");
917 }
918
919 value = ac_build_buffer_load(&ctx->ac, buffer, 1, NULL, base, offset,
920 swizzle * 4, 1, 0, can_speculate, false);
921
922 value2 = ac_build_buffer_load(&ctx->ac, buffer, 1, NULL, base, offset,
923 swizzle * 4 + 4, 1, 0, can_speculate, false);
924
925 return si_llvm_emit_fetch_64bit(bld_base, type, value, value2);
926 }
927
928 /**
929 * Load from LDS.
930 *
931 * \param type output value type
932 * \param swizzle offset (typically 0..3); it can be ~0, which loads a vec4
933 * \param dw_addr address in dwords
934 */
935 static LLVMValueRef lds_load(struct lp_build_tgsi_context *bld_base,
936 enum tgsi_opcode_type type, unsigned swizzle,
937 LLVMValueRef dw_addr)
938 {
939 struct si_shader_context *ctx = si_shader_context(bld_base);
940 struct gallivm_state *gallivm = &ctx->gallivm;
941 LLVMValueRef value;
942
943 if (swizzle == ~0) {
944 LLVMValueRef values[TGSI_NUM_CHANNELS];
945
946 for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; chan++)
947 values[chan] = lds_load(bld_base, type, chan, dw_addr);
948
949 return lp_build_gather_values(gallivm, values,
950 TGSI_NUM_CHANNELS);
951 }
952
953 dw_addr = lp_build_add(&bld_base->uint_bld, dw_addr,
954 LLVMConstInt(ctx->i32, swizzle, 0));
955
956 value = ac_build_indexed_load(&ctx->ac, ctx->lds, dw_addr, false);
957 if (tgsi_type_is_64bit(type)) {
958 LLVMValueRef value2;
959 dw_addr = lp_build_add(&bld_base->uint_bld, dw_addr,
960 ctx->i32_1);
961 value2 = ac_build_indexed_load(&ctx->ac, ctx->lds, dw_addr, false);
962 return si_llvm_emit_fetch_64bit(bld_base, type, value, value2);
963 }
964
965 return LLVMBuildBitCast(gallivm->builder, value,
966 tgsi2llvmtype(bld_base, type), "");
967 }
968
969 /**
970 * Store to LDS.
971 *
972 * \param swizzle offset (typically 0..3)
973 * \param dw_addr address in dwords
974 * \param value value to store
975 */
976 static void lds_store(struct lp_build_tgsi_context *bld_base,
977 unsigned dw_offset_imm, LLVMValueRef dw_addr,
978 LLVMValueRef value)
979 {
980 struct si_shader_context *ctx = si_shader_context(bld_base);
981 struct gallivm_state *gallivm = &ctx->gallivm;
982
983 dw_addr = lp_build_add(&bld_base->uint_bld, dw_addr,
984 LLVMConstInt(ctx->i32, dw_offset_imm, 0));
985
986 value = LLVMBuildBitCast(gallivm->builder, value, ctx->i32, "");
987 ac_build_indexed_store(&ctx->ac, ctx->lds,
988 dw_addr, value);
989 }
990
991 static LLVMValueRef desc_from_addr_base64k(struct si_shader_context *ctx,
992 unsigned param)
993 {
994 LLVMBuilderRef builder = ctx->gallivm.builder;
995
996 LLVMValueRef addr = LLVMGetParam(ctx->main_fn, param);
997 addr = LLVMBuildZExt(builder, addr, ctx->i64, "");
998 addr = LLVMBuildShl(builder, addr, LLVMConstInt(ctx->i64, 16, 0), "");
999
1000 uint64_t desc2 = 0xffffffff;
1001 uint64_t desc3 = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1002 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1003 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1004 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1005 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1006 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
1007 LLVMValueRef hi = LLVMConstInt(ctx->i64, desc2 | (desc3 << 32), 0);
1008
1009 LLVMValueRef desc = LLVMGetUndef(LLVMVectorType(ctx->i64, 2));
1010 desc = LLVMBuildInsertElement(builder, desc, addr, ctx->i32_0, "");
1011 desc = LLVMBuildInsertElement(builder, desc, hi, ctx->i32_1, "");
1012 return LLVMBuildBitCast(builder, desc, ctx->v4i32, "");
1013 }
1014
1015 static LLVMValueRef fetch_input_tcs(
1016 struct lp_build_tgsi_context *bld_base,
1017 const struct tgsi_full_src_register *reg,
1018 enum tgsi_opcode_type type, unsigned swizzle)
1019 {
1020 struct si_shader_context *ctx = si_shader_context(bld_base);
1021 LLVMValueRef dw_addr, stride;
1022
1023 stride = unpack_param(ctx, ctx->param_vs_state_bits, 24, 8);
1024 dw_addr = get_tcs_in_current_patch_offset(ctx);
1025 dw_addr = get_dw_address(ctx, NULL, reg, stride, dw_addr);
1026
1027 return lds_load(bld_base, type, swizzle, dw_addr);
1028 }
1029
1030 static LLVMValueRef fetch_output_tcs(
1031 struct lp_build_tgsi_context *bld_base,
1032 const struct tgsi_full_src_register *reg,
1033 enum tgsi_opcode_type type, unsigned swizzle)
1034 {
1035 struct si_shader_context *ctx = si_shader_context(bld_base);
1036 LLVMValueRef dw_addr, stride;
1037
1038 if (reg->Register.Dimension) {
1039 stride = unpack_param(ctx, ctx->param_tcs_out_lds_layout, 13, 8);
1040 dw_addr = get_tcs_out_current_patch_offset(ctx);
1041 dw_addr = get_dw_address(ctx, NULL, reg, stride, dw_addr);
1042 } else {
1043 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1044 dw_addr = get_dw_address(ctx, NULL, reg, NULL, dw_addr);
1045 }
1046
1047 return lds_load(bld_base, type, swizzle, dw_addr);
1048 }
1049
1050 static LLVMValueRef fetch_input_tes(
1051 struct lp_build_tgsi_context *bld_base,
1052 const struct tgsi_full_src_register *reg,
1053 enum tgsi_opcode_type type, unsigned swizzle)
1054 {
1055 struct si_shader_context *ctx = si_shader_context(bld_base);
1056 LLVMValueRef buffer, base, addr;
1057
1058 buffer = desc_from_addr_base64k(ctx, ctx->param_tcs_offchip_addr_base64k);
1059
1060 base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
1061 addr = get_tcs_tes_buffer_address_from_reg(ctx, NULL, reg);
1062
1063 return buffer_load(bld_base, type, swizzle, buffer, base, addr, true);
1064 }
1065
1066 static void store_output_tcs(struct lp_build_tgsi_context *bld_base,
1067 const struct tgsi_full_instruction *inst,
1068 const struct tgsi_opcode_info *info,
1069 LLVMValueRef dst[4])
1070 {
1071 struct si_shader_context *ctx = si_shader_context(bld_base);
1072 struct gallivm_state *gallivm = &ctx->gallivm;
1073 const struct tgsi_full_dst_register *reg = &inst->Dst[0];
1074 const struct tgsi_shader_info *sh_info = &ctx->shader->selector->info;
1075 unsigned chan_index;
1076 LLVMValueRef dw_addr, stride;
1077 LLVMValueRef buffer, base, buf_addr;
1078 LLVMValueRef values[4];
1079 bool skip_lds_store;
1080 bool is_tess_factor = false;
1081
1082 /* Only handle per-patch and per-vertex outputs here.
1083 * Vectors will be lowered to scalars and this function will be called again.
1084 */
1085 if (reg->Register.File != TGSI_FILE_OUTPUT ||
1086 (dst[0] && LLVMGetTypeKind(LLVMTypeOf(dst[0])) == LLVMVectorTypeKind)) {
1087 si_llvm_emit_store(bld_base, inst, info, dst);
1088 return;
1089 }
1090
1091 if (reg->Register.Dimension) {
1092 stride = unpack_param(ctx, ctx->param_tcs_out_lds_layout, 13, 8);
1093 dw_addr = get_tcs_out_current_patch_offset(ctx);
1094 dw_addr = get_dw_address(ctx, reg, NULL, stride, dw_addr);
1095 skip_lds_store = !sh_info->reads_pervertex_outputs;
1096 } else {
1097 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1098 dw_addr = get_dw_address(ctx, reg, NULL, NULL, dw_addr);
1099 skip_lds_store = !sh_info->reads_perpatch_outputs;
1100
1101 if (!reg->Register.Indirect) {
1102 int name = sh_info->output_semantic_name[reg->Register.Index];
1103
1104 /* Always write tess factors into LDS for the TCS epilog. */
1105 if (name == TGSI_SEMANTIC_TESSINNER ||
1106 name == TGSI_SEMANTIC_TESSOUTER) {
1107 skip_lds_store = false;
1108 is_tess_factor = true;
1109 }
1110 }
1111 }
1112
1113 buffer = desc_from_addr_base64k(ctx, ctx->param_tcs_offchip_addr_base64k);
1114
1115 base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
1116 buf_addr = get_tcs_tes_buffer_address_from_reg(ctx, reg, NULL);
1117
1118
1119 TGSI_FOR_EACH_DST0_ENABLED_CHANNEL(inst, chan_index) {
1120 LLVMValueRef value = dst[chan_index];
1121
1122 if (inst->Instruction.Saturate)
1123 value = ac_build_clamp(&ctx->ac, value);
1124
1125 /* Skip LDS stores if there is no LDS read of this output. */
1126 if (!skip_lds_store)
1127 lds_store(bld_base, chan_index, dw_addr, value);
1128
1129 value = LLVMBuildBitCast(gallivm->builder, value, ctx->i32, "");
1130 values[chan_index] = value;
1131
1132 if (inst->Dst[0].Register.WriteMask != 0xF && !is_tess_factor) {
1133 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 1,
1134 buf_addr, base,
1135 4 * chan_index, 1, 0, true, false);
1136 }
1137 }
1138
1139 if (inst->Dst[0].Register.WriteMask == 0xF && !is_tess_factor) {
1140 LLVMValueRef value = lp_build_gather_values(gallivm,
1141 values, 4);
1142 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, buf_addr,
1143 base, 0, 1, 0, true, false);
1144 }
1145 }
1146
1147 static LLVMValueRef fetch_input_gs(
1148 struct lp_build_tgsi_context *bld_base,
1149 const struct tgsi_full_src_register *reg,
1150 enum tgsi_opcode_type type,
1151 unsigned swizzle)
1152 {
1153 struct si_shader_context *ctx = si_shader_context(bld_base);
1154 struct si_shader *shader = ctx->shader;
1155 struct lp_build_context *uint = &ctx->bld_base.uint_bld;
1156 struct gallivm_state *gallivm = &ctx->gallivm;
1157 LLVMValueRef vtx_offset, soffset;
1158 struct tgsi_shader_info *info = &shader->selector->info;
1159 unsigned semantic_name = info->input_semantic_name[reg->Register.Index];
1160 unsigned semantic_index = info->input_semantic_index[reg->Register.Index];
1161 unsigned param;
1162 LLVMValueRef value;
1163
1164 if (swizzle != ~0 && semantic_name == TGSI_SEMANTIC_PRIMID)
1165 return get_primitive_id(bld_base, swizzle);
1166
1167 if (!reg->Register.Dimension)
1168 return NULL;
1169
1170 param = si_shader_io_get_unique_index(semantic_name, semantic_index);
1171
1172 /* GFX9 has the ESGS ring in LDS. */
1173 if (ctx->screen->b.chip_class >= GFX9) {
1174 unsigned index = reg->Dimension.Index;
1175
1176 switch (index / 2) {
1177 case 0:
1178 vtx_offset = unpack_param(ctx, ctx->param_gs_vtx01_offset,
1179 index % 2 ? 16 : 0, 16);
1180 break;
1181 case 1:
1182 vtx_offset = unpack_param(ctx, ctx->param_gs_vtx23_offset,
1183 index % 2 ? 16 : 0, 16);
1184 break;
1185 case 2:
1186 vtx_offset = unpack_param(ctx, ctx->param_gs_vtx45_offset,
1187 index % 2 ? 16 : 0, 16);
1188 break;
1189 default:
1190 assert(0);
1191 return NULL;
1192 }
1193
1194 vtx_offset = LLVMBuildAdd(gallivm->builder, vtx_offset,
1195 LLVMConstInt(ctx->i32, param * 4, 0), "");
1196 return lds_load(bld_base, type, swizzle, vtx_offset);
1197 }
1198
1199 /* GFX6: input load from the ESGS ring in memory. */
1200 if (swizzle == ~0) {
1201 LLVMValueRef values[TGSI_NUM_CHANNELS];
1202 unsigned chan;
1203 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1204 values[chan] = fetch_input_gs(bld_base, reg, type, chan);
1205 }
1206 return lp_build_gather_values(gallivm, values,
1207 TGSI_NUM_CHANNELS);
1208 }
1209
1210 /* Get the vertex offset parameter on GFX6. */
1211 unsigned vtx_offset_param = reg->Dimension.Index;
1212 if (vtx_offset_param < 2) {
1213 vtx_offset_param += ctx->param_gs_vtx0_offset;
1214 } else {
1215 assert(vtx_offset_param < 6);
1216 vtx_offset_param += ctx->param_gs_vtx2_offset - 2;
1217 }
1218 vtx_offset = lp_build_mul_imm(uint,
1219 LLVMGetParam(ctx->main_fn,
1220 vtx_offset_param),
1221 4);
1222
1223 soffset = LLVMConstInt(ctx->i32, (param * 4 + swizzle) * 256, 0);
1224
1225 value = ac_build_buffer_load(&ctx->ac, ctx->esgs_ring, 1, ctx->i32_0,
1226 vtx_offset, soffset, 0, 1, 0, true, false);
1227 if (tgsi_type_is_64bit(type)) {
1228 LLVMValueRef value2;
1229 soffset = LLVMConstInt(ctx->i32, (param * 4 + swizzle + 1) * 256, 0);
1230
1231 value2 = ac_build_buffer_load(&ctx->ac, ctx->esgs_ring, 1,
1232 ctx->i32_0, vtx_offset, soffset,
1233 0, 1, 0, true, false);
1234 return si_llvm_emit_fetch_64bit(bld_base, type,
1235 value, value2);
1236 }
1237 return LLVMBuildBitCast(gallivm->builder,
1238 value,
1239 tgsi2llvmtype(bld_base, type), "");
1240 }
1241
1242 static int lookup_interp_param_index(unsigned interpolate, unsigned location)
1243 {
1244 switch (interpolate) {
1245 case TGSI_INTERPOLATE_CONSTANT:
1246 return 0;
1247
1248 case TGSI_INTERPOLATE_LINEAR:
1249 if (location == TGSI_INTERPOLATE_LOC_SAMPLE)
1250 return SI_PARAM_LINEAR_SAMPLE;
1251 else if (location == TGSI_INTERPOLATE_LOC_CENTROID)
1252 return SI_PARAM_LINEAR_CENTROID;
1253 else
1254 return SI_PARAM_LINEAR_CENTER;
1255 break;
1256 case TGSI_INTERPOLATE_COLOR:
1257 case TGSI_INTERPOLATE_PERSPECTIVE:
1258 if (location == TGSI_INTERPOLATE_LOC_SAMPLE)
1259 return SI_PARAM_PERSP_SAMPLE;
1260 else if (location == TGSI_INTERPOLATE_LOC_CENTROID)
1261 return SI_PARAM_PERSP_CENTROID;
1262 else
1263 return SI_PARAM_PERSP_CENTER;
1264 break;
1265 default:
1266 fprintf(stderr, "Warning: Unhandled interpolation mode.\n");
1267 return -1;
1268 }
1269 }
1270
1271 static LLVMValueRef si_build_fs_interp(struct si_shader_context *ctx,
1272 unsigned attr_index, unsigned chan,
1273 LLVMValueRef prim_mask,
1274 LLVMValueRef i, LLVMValueRef j)
1275 {
1276 if (i || j) {
1277 return ac_build_fs_interp(&ctx->ac,
1278 LLVMConstInt(ctx->i32, chan, 0),
1279 LLVMConstInt(ctx->i32, attr_index, 0),
1280 prim_mask, i, j);
1281 }
1282 return ac_build_fs_interp_mov(&ctx->ac,
1283 LLVMConstInt(ctx->i32, 2, 0), /* P0 */
1284 LLVMConstInt(ctx->i32, chan, 0),
1285 LLVMConstInt(ctx->i32, attr_index, 0),
1286 prim_mask);
1287 }
1288
1289 /**
1290 * Interpolate a fragment shader input.
1291 *
1292 * @param ctx context
1293 * @param input_index index of the input in hardware
1294 * @param semantic_name TGSI_SEMANTIC_*
1295 * @param semantic_index semantic index
1296 * @param num_interp_inputs number of all interpolated inputs (= BCOLOR offset)
1297 * @param colors_read_mask color components read (4 bits for each color, 8 bits in total)
1298 * @param interp_param interpolation weights (i,j)
1299 * @param prim_mask SI_PARAM_PRIM_MASK
1300 * @param face SI_PARAM_FRONT_FACE
1301 * @param result the return value (4 components)
1302 */
1303 static void interp_fs_input(struct si_shader_context *ctx,
1304 unsigned input_index,
1305 unsigned semantic_name,
1306 unsigned semantic_index,
1307 unsigned num_interp_inputs,
1308 unsigned colors_read_mask,
1309 LLVMValueRef interp_param,
1310 LLVMValueRef prim_mask,
1311 LLVMValueRef face,
1312 LLVMValueRef result[4])
1313 {
1314 struct gallivm_state *gallivm = &ctx->gallivm;
1315 LLVMValueRef i = NULL, j = NULL;
1316 unsigned chan;
1317
1318 /* fs.constant returns the param from the middle vertex, so it's not
1319 * really useful for flat shading. It's meant to be used for custom
1320 * interpolation (but the intrinsic can't fetch from the other two
1321 * vertices).
1322 *
1323 * Luckily, it doesn't matter, because we rely on the FLAT_SHADE state
1324 * to do the right thing. The only reason we use fs.constant is that
1325 * fs.interp cannot be used on integers, because they can be equal
1326 * to NaN.
1327 *
1328 * When interp is false we will use fs.constant or for newer llvm,
1329 * amdgcn.interp.mov.
1330 */
1331 bool interp = interp_param != NULL;
1332
1333 if (interp) {
1334 interp_param = LLVMBuildBitCast(gallivm->builder, interp_param,
1335 LLVMVectorType(ctx->f32, 2), "");
1336
1337 i = LLVMBuildExtractElement(gallivm->builder, interp_param,
1338 ctx->i32_0, "");
1339 j = LLVMBuildExtractElement(gallivm->builder, interp_param,
1340 ctx->i32_1, "");
1341 }
1342
1343 if (semantic_name == TGSI_SEMANTIC_COLOR &&
1344 ctx->shader->key.part.ps.prolog.color_two_side) {
1345 LLVMValueRef is_face_positive;
1346
1347 /* If BCOLOR0 is used, BCOLOR1 is at offset "num_inputs + 1",
1348 * otherwise it's at offset "num_inputs".
1349 */
1350 unsigned back_attr_offset = num_interp_inputs;
1351 if (semantic_index == 1 && colors_read_mask & 0xf)
1352 back_attr_offset += 1;
1353
1354 is_face_positive = LLVMBuildICmp(gallivm->builder, LLVMIntNE,
1355 face, ctx->i32_0, "");
1356
1357 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1358 LLVMValueRef front, back;
1359
1360 front = si_build_fs_interp(ctx,
1361 input_index, chan,
1362 prim_mask, i, j);
1363 back = si_build_fs_interp(ctx,
1364 back_attr_offset, chan,
1365 prim_mask, i, j);
1366
1367 result[chan] = LLVMBuildSelect(gallivm->builder,
1368 is_face_positive,
1369 front,
1370 back,
1371 "");
1372 }
1373 } else if (semantic_name == TGSI_SEMANTIC_FOG) {
1374 result[0] = si_build_fs_interp(ctx, input_index,
1375 0, prim_mask, i, j);
1376 result[1] =
1377 result[2] = LLVMConstReal(ctx->f32, 0.0f);
1378 result[3] = LLVMConstReal(ctx->f32, 1.0f);
1379 } else {
1380 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1381 result[chan] = si_build_fs_interp(ctx,
1382 input_index, chan,
1383 prim_mask, i, j);
1384 }
1385 }
1386 }
1387
1388 static void declare_input_fs(
1389 struct si_shader_context *ctx,
1390 unsigned input_index,
1391 const struct tgsi_full_declaration *decl,
1392 LLVMValueRef out[4])
1393 {
1394 struct lp_build_context *base = &ctx->bld_base.base;
1395 struct si_shader *shader = ctx->shader;
1396 LLVMValueRef main_fn = ctx->main_fn;
1397 LLVMValueRef interp_param = NULL;
1398 int interp_param_idx;
1399
1400 /* Get colors from input VGPRs (set by the prolog). */
1401 if (decl->Semantic.Name == TGSI_SEMANTIC_COLOR) {
1402 unsigned i = decl->Semantic.Index;
1403 unsigned colors_read = shader->selector->info.colors_read;
1404 unsigned mask = colors_read >> (i * 4);
1405 unsigned offset = SI_PARAM_POS_FIXED_PT + 1 +
1406 (i ? util_bitcount(colors_read & 0xf) : 0);
1407
1408 out[0] = mask & 0x1 ? LLVMGetParam(main_fn, offset++) : base->undef;
1409 out[1] = mask & 0x2 ? LLVMGetParam(main_fn, offset++) : base->undef;
1410 out[2] = mask & 0x4 ? LLVMGetParam(main_fn, offset++) : base->undef;
1411 out[3] = mask & 0x8 ? LLVMGetParam(main_fn, offset++) : base->undef;
1412 return;
1413 }
1414
1415 interp_param_idx = lookup_interp_param_index(decl->Interp.Interpolate,
1416 decl->Interp.Location);
1417 if (interp_param_idx == -1)
1418 return;
1419 else if (interp_param_idx) {
1420 interp_param = LLVMGetParam(ctx->main_fn, interp_param_idx);
1421 }
1422
1423 interp_fs_input(ctx, input_index, decl->Semantic.Name,
1424 decl->Semantic.Index, 0, /* this param is unused */
1425 shader->selector->info.colors_read, interp_param,
1426 LLVMGetParam(main_fn, SI_PARAM_PRIM_MASK),
1427 LLVMGetParam(main_fn, SI_PARAM_FRONT_FACE),
1428 &out[0]);
1429 }
1430
1431 static LLVMValueRef get_sample_id(struct si_shader_context *ctx)
1432 {
1433 return unpack_param(ctx, SI_PARAM_ANCILLARY, 8, 4);
1434 }
1435
1436
1437 /**
1438 * Load a dword from a constant buffer.
1439 */
1440 static LLVMValueRef buffer_load_const(struct si_shader_context *ctx,
1441 LLVMValueRef resource,
1442 LLVMValueRef offset)
1443 {
1444 return ac_build_buffer_load(&ctx->ac, resource, 1, NULL, offset, NULL,
1445 0, 0, 0, true, true);
1446 }
1447
1448 static LLVMValueRef load_sample_position(struct si_shader_context *ctx, LLVMValueRef sample_id)
1449 {
1450 struct lp_build_context *uint_bld = &ctx->bld_base.uint_bld;
1451 struct gallivm_state *gallivm = &ctx->gallivm;
1452 LLVMBuilderRef builder = gallivm->builder;
1453 LLVMValueRef desc = LLVMGetParam(ctx->main_fn, ctx->param_rw_buffers);
1454 LLVMValueRef buf_index = LLVMConstInt(ctx->i32, SI_PS_CONST_SAMPLE_POSITIONS, 0);
1455 LLVMValueRef resource = ac_build_indexed_load_const(&ctx->ac, desc, buf_index);
1456
1457 /* offset = sample_id * 8 (8 = 2 floats containing samplepos.xy) */
1458 LLVMValueRef offset0 = lp_build_mul_imm(uint_bld, sample_id, 8);
1459 LLVMValueRef offset1 = LLVMBuildAdd(builder, offset0, LLVMConstInt(ctx->i32, 4, 0), "");
1460
1461 LLVMValueRef pos[4] = {
1462 buffer_load_const(ctx, resource, offset0),
1463 buffer_load_const(ctx, resource, offset1),
1464 LLVMConstReal(ctx->f32, 0),
1465 LLVMConstReal(ctx->f32, 0)
1466 };
1467
1468 return lp_build_gather_values(gallivm, pos, 4);
1469 }
1470
1471 static void declare_system_value(struct si_shader_context *ctx,
1472 unsigned index,
1473 const struct tgsi_full_declaration *decl)
1474 {
1475 struct lp_build_context *bld = &ctx->bld_base.base;
1476 struct gallivm_state *gallivm = &ctx->gallivm;
1477 LLVMValueRef value = 0;
1478
1479 assert(index < RADEON_LLVM_MAX_SYSTEM_VALUES);
1480
1481 switch (decl->Semantic.Name) {
1482 case TGSI_SEMANTIC_INSTANCEID:
1483 value = ctx->abi.instance_id;
1484 break;
1485
1486 case TGSI_SEMANTIC_VERTEXID:
1487 value = LLVMBuildAdd(gallivm->builder,
1488 ctx->abi.vertex_id,
1489 ctx->abi.base_vertex, "");
1490 break;
1491
1492 case TGSI_SEMANTIC_VERTEXID_NOBASE:
1493 /* Unused. Clarify the meaning in indexed vs. non-indexed
1494 * draws if this is ever used again. */
1495 assert(false);
1496 break;
1497
1498 case TGSI_SEMANTIC_BASEVERTEX:
1499 {
1500 /* For non-indexed draws, the base vertex set by the driver
1501 * (for direct draws) or the CP (for indirect draws) is the
1502 * first vertex ID, but GLSL expects 0 to be returned.
1503 */
1504 LLVMValueRef vs_state = LLVMGetParam(ctx->main_fn, ctx->param_vs_state_bits);
1505 LLVMValueRef indexed;
1506
1507 indexed = LLVMBuildLShr(gallivm->builder, vs_state, ctx->i32_1, "");
1508 indexed = LLVMBuildTrunc(gallivm->builder, indexed, ctx->i1, "");
1509
1510 value = LLVMBuildSelect(gallivm->builder, indexed,
1511 ctx->abi.base_vertex, ctx->i32_0, "");
1512 break;
1513 }
1514
1515 case TGSI_SEMANTIC_BASEINSTANCE:
1516 value = ctx->abi.start_instance;
1517 break;
1518
1519 case TGSI_SEMANTIC_DRAWID:
1520 value = ctx->abi.draw_id;
1521 break;
1522
1523 case TGSI_SEMANTIC_INVOCATIONID:
1524 if (ctx->type == PIPE_SHADER_TESS_CTRL)
1525 value = unpack_param(ctx, ctx->param_tcs_rel_ids, 8, 5);
1526 else if (ctx->type == PIPE_SHADER_GEOMETRY)
1527 value = LLVMGetParam(ctx->main_fn,
1528 ctx->param_gs_instance_id);
1529 else
1530 assert(!"INVOCATIONID not implemented");
1531 break;
1532
1533 case TGSI_SEMANTIC_POSITION:
1534 {
1535 LLVMValueRef pos[4] = {
1536 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_X_FLOAT),
1537 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Y_FLOAT),
1538 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Z_FLOAT),
1539 lp_build_emit_llvm_unary(&ctx->bld_base, TGSI_OPCODE_RCP,
1540 LLVMGetParam(ctx->main_fn,
1541 SI_PARAM_POS_W_FLOAT)),
1542 };
1543 value = lp_build_gather_values(gallivm, pos, 4);
1544 break;
1545 }
1546
1547 case TGSI_SEMANTIC_FACE:
1548 value = LLVMGetParam(ctx->main_fn, SI_PARAM_FRONT_FACE);
1549 break;
1550
1551 case TGSI_SEMANTIC_SAMPLEID:
1552 value = get_sample_id(ctx);
1553 break;
1554
1555 case TGSI_SEMANTIC_SAMPLEPOS: {
1556 LLVMValueRef pos[4] = {
1557 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_X_FLOAT),
1558 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Y_FLOAT),
1559 LLVMConstReal(ctx->f32, 0),
1560 LLVMConstReal(ctx->f32, 0)
1561 };
1562 pos[0] = lp_build_emit_llvm_unary(&ctx->bld_base,
1563 TGSI_OPCODE_FRC, pos[0]);
1564 pos[1] = lp_build_emit_llvm_unary(&ctx->bld_base,
1565 TGSI_OPCODE_FRC, pos[1]);
1566 value = lp_build_gather_values(gallivm, pos, 4);
1567 break;
1568 }
1569
1570 case TGSI_SEMANTIC_SAMPLEMASK:
1571 /* This can only occur with the OpenGL Core profile, which
1572 * doesn't support smoothing.
1573 */
1574 value = LLVMGetParam(ctx->main_fn, SI_PARAM_SAMPLE_COVERAGE);
1575 break;
1576
1577 case TGSI_SEMANTIC_TESSCOORD:
1578 {
1579 LLVMValueRef coord[4] = {
1580 LLVMGetParam(ctx->main_fn, ctx->param_tes_u),
1581 LLVMGetParam(ctx->main_fn, ctx->param_tes_v),
1582 bld->zero,
1583 bld->zero
1584 };
1585
1586 /* For triangles, the vector should be (u, v, 1-u-v). */
1587 if (ctx->shader->selector->info.properties[TGSI_PROPERTY_TES_PRIM_MODE] ==
1588 PIPE_PRIM_TRIANGLES)
1589 coord[2] = lp_build_sub(bld, bld->one,
1590 lp_build_add(bld, coord[0], coord[1]));
1591
1592 value = lp_build_gather_values(gallivm, coord, 4);
1593 break;
1594 }
1595
1596 case TGSI_SEMANTIC_VERTICESIN:
1597 if (ctx->type == PIPE_SHADER_TESS_CTRL)
1598 value = unpack_param(ctx, ctx->param_tcs_out_lds_layout, 26, 6);
1599 else if (ctx->type == PIPE_SHADER_TESS_EVAL)
1600 value = unpack_param(ctx, ctx->param_tcs_offchip_layout, 6, 6);
1601 else
1602 assert(!"invalid shader stage for TGSI_SEMANTIC_VERTICESIN");
1603 break;
1604
1605 case TGSI_SEMANTIC_TESSINNER:
1606 case TGSI_SEMANTIC_TESSOUTER:
1607 {
1608 LLVMValueRef buffer, base, addr;
1609 int param = si_shader_io_get_unique_index_patch(decl->Semantic.Name, 0);
1610
1611 buffer = desc_from_addr_base64k(ctx, ctx->param_tcs_offchip_addr_base64k);
1612
1613 base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
1614 addr = get_tcs_tes_buffer_address(ctx, get_rel_patch_id(ctx), NULL,
1615 LLVMConstInt(ctx->i32, param, 0));
1616
1617 value = buffer_load(&ctx->bld_base, TGSI_TYPE_FLOAT,
1618 ~0, buffer, base, addr, true);
1619
1620 break;
1621 }
1622
1623 case TGSI_SEMANTIC_DEFAULT_TESSOUTER_SI:
1624 case TGSI_SEMANTIC_DEFAULT_TESSINNER_SI:
1625 {
1626 LLVMValueRef buf, slot, val[4];
1627 int i, offset;
1628
1629 slot = LLVMConstInt(ctx->i32, SI_HS_CONST_DEFAULT_TESS_LEVELS, 0);
1630 buf = LLVMGetParam(ctx->main_fn, ctx->param_rw_buffers);
1631 buf = ac_build_indexed_load_const(&ctx->ac, buf, slot);
1632 offset = decl->Semantic.Name == TGSI_SEMANTIC_DEFAULT_TESSINNER_SI ? 4 : 0;
1633
1634 for (i = 0; i < 4; i++)
1635 val[i] = buffer_load_const(ctx, buf,
1636 LLVMConstInt(ctx->i32, (offset + i) * 4, 0));
1637 value = lp_build_gather_values(gallivm, val, 4);
1638 break;
1639 }
1640
1641 case TGSI_SEMANTIC_PRIMID:
1642 value = get_primitive_id(&ctx->bld_base, 0);
1643 break;
1644
1645 case TGSI_SEMANTIC_GRID_SIZE:
1646 value = LLVMGetParam(ctx->main_fn, ctx->param_grid_size);
1647 break;
1648
1649 case TGSI_SEMANTIC_BLOCK_SIZE:
1650 {
1651 LLVMValueRef values[3];
1652 unsigned i;
1653 unsigned *properties = ctx->shader->selector->info.properties;
1654
1655 if (properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH] != 0) {
1656 unsigned sizes[3] = {
1657 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH],
1658 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT],
1659 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH]
1660 };
1661
1662 for (i = 0; i < 3; ++i)
1663 values[i] = LLVMConstInt(ctx->i32, sizes[i], 0);
1664
1665 value = lp_build_gather_values(gallivm, values, 3);
1666 } else {
1667 value = LLVMGetParam(ctx->main_fn, ctx->param_block_size);
1668 }
1669 break;
1670 }
1671
1672 case TGSI_SEMANTIC_BLOCK_ID:
1673 {
1674 LLVMValueRef values[3];
1675
1676 for (int i = 0; i < 3; i++) {
1677 values[i] = ctx->i32_0;
1678 if (ctx->param_block_id[i] >= 0) {
1679 values[i] = LLVMGetParam(ctx->main_fn,
1680 ctx->param_block_id[i]);
1681 }
1682 }
1683 value = lp_build_gather_values(gallivm, values, 3);
1684 break;
1685 }
1686
1687 case TGSI_SEMANTIC_THREAD_ID:
1688 value = LLVMGetParam(ctx->main_fn, ctx->param_thread_id);
1689 break;
1690
1691 case TGSI_SEMANTIC_HELPER_INVOCATION:
1692 value = lp_build_intrinsic(gallivm->builder,
1693 "llvm.amdgcn.ps.live",
1694 ctx->i1, NULL, 0,
1695 LP_FUNC_ATTR_READNONE);
1696 value = LLVMBuildNot(gallivm->builder, value, "");
1697 value = LLVMBuildSExt(gallivm->builder, value, ctx->i32, "");
1698 break;
1699
1700 case TGSI_SEMANTIC_SUBGROUP_SIZE:
1701 value = LLVMConstInt(ctx->i32, 64, 0);
1702 break;
1703
1704 case TGSI_SEMANTIC_SUBGROUP_INVOCATION:
1705 value = ac_get_thread_id(&ctx->ac);
1706 break;
1707
1708 case TGSI_SEMANTIC_SUBGROUP_EQ_MASK:
1709 {
1710 LLVMValueRef id = ac_get_thread_id(&ctx->ac);
1711 id = LLVMBuildZExt(gallivm->builder, id, ctx->i64, "");
1712 value = LLVMBuildShl(gallivm->builder, LLVMConstInt(ctx->i64, 1, 0), id, "");
1713 value = LLVMBuildBitCast(gallivm->builder, value, ctx->v2i32, "");
1714 break;
1715 }
1716
1717 case TGSI_SEMANTIC_SUBGROUP_GE_MASK:
1718 case TGSI_SEMANTIC_SUBGROUP_GT_MASK:
1719 case TGSI_SEMANTIC_SUBGROUP_LE_MASK:
1720 case TGSI_SEMANTIC_SUBGROUP_LT_MASK:
1721 {
1722 LLVMValueRef id = ac_get_thread_id(&ctx->ac);
1723 if (decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_GT_MASK ||
1724 decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_LE_MASK) {
1725 /* All bits set except LSB */
1726 value = LLVMConstInt(ctx->i64, -2, 0);
1727 } else {
1728 /* All bits set */
1729 value = LLVMConstInt(ctx->i64, -1, 0);
1730 }
1731 id = LLVMBuildZExt(gallivm->builder, id, ctx->i64, "");
1732 value = LLVMBuildShl(gallivm->builder, value, id, "");
1733 if (decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_LE_MASK ||
1734 decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_LT_MASK)
1735 value = LLVMBuildNot(gallivm->builder, value, "");
1736 value = LLVMBuildBitCast(gallivm->builder, value, ctx->v2i32, "");
1737 break;
1738 }
1739
1740 default:
1741 assert(!"unknown system value");
1742 return;
1743 }
1744
1745 ctx->system_values[index] = value;
1746 }
1747
1748 static void declare_compute_memory(struct si_shader_context *ctx,
1749 const struct tgsi_full_declaration *decl)
1750 {
1751 struct si_shader_selector *sel = ctx->shader->selector;
1752 struct gallivm_state *gallivm = &ctx->gallivm;
1753
1754 LLVMTypeRef i8p = LLVMPointerType(ctx->i8, LOCAL_ADDR_SPACE);
1755 LLVMValueRef var;
1756
1757 assert(decl->Declaration.MemType == TGSI_MEMORY_TYPE_SHARED);
1758 assert(decl->Range.First == decl->Range.Last);
1759 assert(!ctx->shared_memory);
1760
1761 var = LLVMAddGlobalInAddressSpace(gallivm->module,
1762 LLVMArrayType(ctx->i8, sel->local_size),
1763 "compute_lds",
1764 LOCAL_ADDR_SPACE);
1765 LLVMSetAlignment(var, 4);
1766
1767 ctx->shared_memory = LLVMBuildBitCast(gallivm->builder, var, i8p, "");
1768 }
1769
1770 static LLVMValueRef load_const_buffer_desc(struct si_shader_context *ctx, int i)
1771 {
1772 LLVMValueRef list_ptr = LLVMGetParam(ctx->main_fn,
1773 ctx->param_const_and_shader_buffers);
1774
1775 return ac_build_indexed_load_const(&ctx->ac, list_ptr,
1776 LLVMConstInt(ctx->i32, si_get_constbuf_slot(i), 0));
1777 }
1778
1779 static LLVMValueRef fetch_constant(
1780 struct lp_build_tgsi_context *bld_base,
1781 const struct tgsi_full_src_register *reg,
1782 enum tgsi_opcode_type type,
1783 unsigned swizzle)
1784 {
1785 struct si_shader_context *ctx = si_shader_context(bld_base);
1786 struct lp_build_context *base = &bld_base->base;
1787 const struct tgsi_ind_register *ireg = &reg->Indirect;
1788 unsigned buf, idx;
1789
1790 LLVMValueRef addr, bufp;
1791 LLVMValueRef result;
1792
1793 if (swizzle == LP_CHAN_ALL) {
1794 unsigned chan;
1795 LLVMValueRef values[4];
1796 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan)
1797 values[chan] = fetch_constant(bld_base, reg, type, chan);
1798
1799 return lp_build_gather_values(&ctx->gallivm, values, 4);
1800 }
1801
1802 buf = reg->Register.Dimension ? reg->Dimension.Index : 0;
1803 idx = reg->Register.Index * 4 + swizzle;
1804
1805 if (reg->Register.Dimension && reg->Dimension.Indirect) {
1806 LLVMValueRef ptr = LLVMGetParam(ctx->main_fn, ctx->param_const_and_shader_buffers);
1807 LLVMValueRef index;
1808 index = si_get_bounded_indirect_index(ctx, &reg->DimIndirect,
1809 reg->Dimension.Index,
1810 ctx->num_const_buffers);
1811 index = LLVMBuildAdd(ctx->gallivm.builder, index,
1812 LLVMConstInt(ctx->i32, SI_NUM_SHADER_BUFFERS, 0), "");
1813 bufp = ac_build_indexed_load_const(&ctx->ac, ptr, index);
1814 } else
1815 bufp = load_const_buffer_desc(ctx, buf);
1816
1817 if (reg->Register.Indirect) {
1818 addr = ctx->addrs[ireg->Index][ireg->Swizzle];
1819 addr = LLVMBuildLoad(base->gallivm->builder, addr, "load addr reg");
1820 addr = lp_build_mul_imm(&bld_base->uint_bld, addr, 16);
1821 addr = lp_build_add(&bld_base->uint_bld, addr,
1822 LLVMConstInt(ctx->i32, idx * 4, 0));
1823 } else {
1824 addr = LLVMConstInt(ctx->i32, idx * 4, 0);
1825 }
1826
1827 result = buffer_load_const(ctx, bufp, addr);
1828
1829 if (!tgsi_type_is_64bit(type))
1830 result = bitcast(bld_base, type, result);
1831 else {
1832 LLVMValueRef addr2, result2;
1833
1834 addr2 = lp_build_add(&bld_base->uint_bld, addr,
1835 LLVMConstInt(ctx->i32, 4, 0));
1836 result2 = buffer_load_const(ctx, bufp, addr2);
1837
1838 result = si_llvm_emit_fetch_64bit(bld_base, type,
1839 result, result2);
1840 }
1841 return result;
1842 }
1843
1844 /* Upper 16 bits must be zero. */
1845 static LLVMValueRef si_llvm_pack_two_int16(struct si_shader_context *ctx,
1846 LLVMValueRef val[2])
1847 {
1848 return LLVMBuildOr(ctx->gallivm.builder, val[0],
1849 LLVMBuildShl(ctx->gallivm.builder, val[1],
1850 LLVMConstInt(ctx->i32, 16, 0),
1851 ""), "");
1852 }
1853
1854 /* Upper 16 bits are ignored and will be dropped. */
1855 static LLVMValueRef si_llvm_pack_two_int32_as_int16(struct si_shader_context *ctx,
1856 LLVMValueRef val[2])
1857 {
1858 LLVMValueRef v[2] = {
1859 LLVMBuildAnd(ctx->gallivm.builder, val[0],
1860 LLVMConstInt(ctx->i32, 0xffff, 0), ""),
1861 val[1],
1862 };
1863 return si_llvm_pack_two_int16(ctx, v);
1864 }
1865
1866 /* Initialize arguments for the shader export intrinsic */
1867 static void si_llvm_init_export_args(struct lp_build_tgsi_context *bld_base,
1868 LLVMValueRef *values,
1869 unsigned target,
1870 struct ac_export_args *args)
1871 {
1872 struct si_shader_context *ctx = si_shader_context(bld_base);
1873 struct lp_build_context *base = &bld_base->base;
1874 LLVMBuilderRef builder = ctx->gallivm.builder;
1875 LLVMValueRef val[4];
1876 unsigned spi_shader_col_format = V_028714_SPI_SHADER_32_ABGR;
1877 unsigned chan;
1878 bool is_int8, is_int10;
1879
1880 /* Default is 0xf. Adjusted below depending on the format. */
1881 args->enabled_channels = 0xf; /* writemask */
1882
1883 /* Specify whether the EXEC mask represents the valid mask */
1884 args->valid_mask = 0;
1885
1886 /* Specify whether this is the last export */
1887 args->done = 0;
1888
1889 /* Specify the target we are exporting */
1890 args->target = target;
1891
1892 if (ctx->type == PIPE_SHADER_FRAGMENT) {
1893 const struct si_shader_key *key = &ctx->shader->key;
1894 unsigned col_formats = key->part.ps.epilog.spi_shader_col_format;
1895 int cbuf = target - V_008DFC_SQ_EXP_MRT;
1896
1897 assert(cbuf >= 0 && cbuf < 8);
1898 spi_shader_col_format = (col_formats >> (cbuf * 4)) & 0xf;
1899 is_int8 = (key->part.ps.epilog.color_is_int8 >> cbuf) & 0x1;
1900 is_int10 = (key->part.ps.epilog.color_is_int10 >> cbuf) & 0x1;
1901 }
1902
1903 args->compr = false;
1904 args->out[0] = base->undef;
1905 args->out[1] = base->undef;
1906 args->out[2] = base->undef;
1907 args->out[3] = base->undef;
1908
1909 switch (spi_shader_col_format) {
1910 case V_028714_SPI_SHADER_ZERO:
1911 args->enabled_channels = 0; /* writemask */
1912 args->target = V_008DFC_SQ_EXP_NULL;
1913 break;
1914
1915 case V_028714_SPI_SHADER_32_R:
1916 args->enabled_channels = 1; /* writemask */
1917 args->out[0] = values[0];
1918 break;
1919
1920 case V_028714_SPI_SHADER_32_GR:
1921 args->enabled_channels = 0x3; /* writemask */
1922 args->out[0] = values[0];
1923 args->out[1] = values[1];
1924 break;
1925
1926 case V_028714_SPI_SHADER_32_AR:
1927 args->enabled_channels = 0x9; /* writemask */
1928 args->out[0] = values[0];
1929 args->out[3] = values[3];
1930 break;
1931
1932 case V_028714_SPI_SHADER_FP16_ABGR:
1933 args->compr = 1; /* COMPR flag */
1934
1935 for (chan = 0; chan < 2; chan++) {
1936 LLVMValueRef pack_args[2] = {
1937 values[2 * chan],
1938 values[2 * chan + 1]
1939 };
1940 LLVMValueRef packed;
1941
1942 packed = ac_build_cvt_pkrtz_f16(&ctx->ac, pack_args);
1943 args->out[chan] =
1944 LLVMBuildBitCast(ctx->gallivm.builder,
1945 packed, ctx->f32, "");
1946 }
1947 break;
1948
1949 case V_028714_SPI_SHADER_UNORM16_ABGR:
1950 for (chan = 0; chan < 4; chan++) {
1951 val[chan] = ac_build_clamp(&ctx->ac, values[chan]);
1952 val[chan] = LLVMBuildFMul(builder, val[chan],
1953 LLVMConstReal(ctx->f32, 65535), "");
1954 val[chan] = LLVMBuildFAdd(builder, val[chan],
1955 LLVMConstReal(ctx->f32, 0.5), "");
1956 val[chan] = LLVMBuildFPToUI(builder, val[chan],
1957 ctx->i32, "");
1958 }
1959
1960 args->compr = 1; /* COMPR flag */
1961 args->out[0] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1962 si_llvm_pack_two_int16(ctx, val));
1963 args->out[1] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1964 si_llvm_pack_two_int16(ctx, val+2));
1965 break;
1966
1967 case V_028714_SPI_SHADER_SNORM16_ABGR:
1968 for (chan = 0; chan < 4; chan++) {
1969 /* Clamp between [-1, 1]. */
1970 val[chan] = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MIN,
1971 values[chan],
1972 LLVMConstReal(ctx->f32, 1));
1973 val[chan] = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MAX,
1974 val[chan],
1975 LLVMConstReal(ctx->f32, -1));
1976 /* Convert to a signed integer in [-32767, 32767]. */
1977 val[chan] = LLVMBuildFMul(builder, val[chan],
1978 LLVMConstReal(ctx->f32, 32767), "");
1979 /* If positive, add 0.5, else add -0.5. */
1980 val[chan] = LLVMBuildFAdd(builder, val[chan],
1981 LLVMBuildSelect(builder,
1982 LLVMBuildFCmp(builder, LLVMRealOGE,
1983 val[chan], base->zero, ""),
1984 LLVMConstReal(ctx->f32, 0.5),
1985 LLVMConstReal(ctx->f32, -0.5), ""), "");
1986 val[chan] = LLVMBuildFPToSI(builder, val[chan], ctx->i32, "");
1987 }
1988
1989 args->compr = 1; /* COMPR flag */
1990 args->out[0] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1991 si_llvm_pack_two_int32_as_int16(ctx, val));
1992 args->out[1] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1993 si_llvm_pack_two_int32_as_int16(ctx, val+2));
1994 break;
1995
1996 case V_028714_SPI_SHADER_UINT16_ABGR: {
1997 LLVMValueRef max_rgb = LLVMConstInt(ctx->i32,
1998 is_int8 ? 255 : is_int10 ? 1023 : 65535, 0);
1999 LLVMValueRef max_alpha =
2000 !is_int10 ? max_rgb : LLVMConstInt(ctx->i32, 3, 0);
2001
2002 /* Clamp. */
2003 for (chan = 0; chan < 4; chan++) {
2004 val[chan] = bitcast(bld_base, TGSI_TYPE_UNSIGNED, values[chan]);
2005 val[chan] = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_UMIN,
2006 val[chan],
2007 chan == 3 ? max_alpha : max_rgb);
2008 }
2009
2010 args->compr = 1; /* COMPR flag */
2011 args->out[0] = bitcast(bld_base, TGSI_TYPE_FLOAT,
2012 si_llvm_pack_two_int16(ctx, val));
2013 args->out[1] = bitcast(bld_base, TGSI_TYPE_FLOAT,
2014 si_llvm_pack_two_int16(ctx, val+2));
2015 break;
2016 }
2017
2018 case V_028714_SPI_SHADER_SINT16_ABGR: {
2019 LLVMValueRef max_rgb = LLVMConstInt(ctx->i32,
2020 is_int8 ? 127 : is_int10 ? 511 : 32767, 0);
2021 LLVMValueRef min_rgb = LLVMConstInt(ctx->i32,
2022 is_int8 ? -128 : is_int10 ? -512 : -32768, 0);
2023 LLVMValueRef max_alpha =
2024 !is_int10 ? max_rgb : ctx->i32_1;
2025 LLVMValueRef min_alpha =
2026 !is_int10 ? min_rgb : LLVMConstInt(ctx->i32, -2, 0);
2027
2028 /* Clamp. */
2029 for (chan = 0; chan < 4; chan++) {
2030 val[chan] = bitcast(bld_base, TGSI_TYPE_UNSIGNED, values[chan]);
2031 val[chan] = lp_build_emit_llvm_binary(bld_base,
2032 TGSI_OPCODE_IMIN,
2033 val[chan], chan == 3 ? max_alpha : max_rgb);
2034 val[chan] = lp_build_emit_llvm_binary(bld_base,
2035 TGSI_OPCODE_IMAX,
2036 val[chan], chan == 3 ? min_alpha : min_rgb);
2037 }
2038
2039 args->compr = 1; /* COMPR flag */
2040 args->out[0] = bitcast(bld_base, TGSI_TYPE_FLOAT,
2041 si_llvm_pack_two_int32_as_int16(ctx, val));
2042 args->out[1] = bitcast(bld_base, TGSI_TYPE_FLOAT,
2043 si_llvm_pack_two_int32_as_int16(ctx, val+2));
2044 break;
2045 }
2046
2047 case V_028714_SPI_SHADER_32_ABGR:
2048 memcpy(&args->out[0], values, sizeof(values[0]) * 4);
2049 break;
2050 }
2051 }
2052
2053 static void si_alpha_test(struct lp_build_tgsi_context *bld_base,
2054 LLVMValueRef alpha)
2055 {
2056 struct si_shader_context *ctx = si_shader_context(bld_base);
2057
2058 if (ctx->shader->key.part.ps.epilog.alpha_func != PIPE_FUNC_NEVER) {
2059 LLVMValueRef alpha_ref = LLVMGetParam(ctx->main_fn,
2060 SI_PARAM_ALPHA_REF);
2061
2062 LLVMValueRef alpha_pass =
2063 lp_build_cmp(&bld_base->base,
2064 ctx->shader->key.part.ps.epilog.alpha_func,
2065 alpha, alpha_ref);
2066 LLVMValueRef arg =
2067 lp_build_select(&bld_base->base,
2068 alpha_pass,
2069 LLVMConstReal(ctx->f32, 1.0f),
2070 LLVMConstReal(ctx->f32, -1.0f));
2071
2072 ac_build_kill(&ctx->ac, arg);
2073 } else {
2074 ac_build_kill(&ctx->ac, NULL);
2075 }
2076 }
2077
2078 static LLVMValueRef si_scale_alpha_by_sample_mask(struct lp_build_tgsi_context *bld_base,
2079 LLVMValueRef alpha,
2080 unsigned samplemask_param)
2081 {
2082 struct si_shader_context *ctx = si_shader_context(bld_base);
2083 struct gallivm_state *gallivm = &ctx->gallivm;
2084 LLVMValueRef coverage;
2085
2086 /* alpha = alpha * popcount(coverage) / SI_NUM_SMOOTH_AA_SAMPLES */
2087 coverage = LLVMGetParam(ctx->main_fn,
2088 samplemask_param);
2089 coverage = bitcast(bld_base, TGSI_TYPE_SIGNED, coverage);
2090
2091 coverage = lp_build_intrinsic(gallivm->builder, "llvm.ctpop.i32",
2092 ctx->i32,
2093 &coverage, 1, LP_FUNC_ATTR_READNONE);
2094
2095 coverage = LLVMBuildUIToFP(gallivm->builder, coverage,
2096 ctx->f32, "");
2097
2098 coverage = LLVMBuildFMul(gallivm->builder, coverage,
2099 LLVMConstReal(ctx->f32,
2100 1.0 / SI_NUM_SMOOTH_AA_SAMPLES), "");
2101
2102 return LLVMBuildFMul(gallivm->builder, alpha, coverage, "");
2103 }
2104
2105 static void si_llvm_emit_clipvertex(struct lp_build_tgsi_context *bld_base,
2106 struct ac_export_args *pos, LLVMValueRef *out_elts)
2107 {
2108 struct si_shader_context *ctx = si_shader_context(bld_base);
2109 struct lp_build_context *base = &bld_base->base;
2110 unsigned reg_index;
2111 unsigned chan;
2112 unsigned const_chan;
2113 LLVMValueRef base_elt;
2114 LLVMValueRef ptr = LLVMGetParam(ctx->main_fn, ctx->param_rw_buffers);
2115 LLVMValueRef constbuf_index = LLVMConstInt(ctx->i32,
2116 SI_VS_CONST_CLIP_PLANES, 0);
2117 LLVMValueRef const_resource = ac_build_indexed_load_const(&ctx->ac, ptr, constbuf_index);
2118
2119 for (reg_index = 0; reg_index < 2; reg_index ++) {
2120 struct ac_export_args *args = &pos[2 + reg_index];
2121
2122 args->out[0] =
2123 args->out[1] =
2124 args->out[2] =
2125 args->out[3] = LLVMConstReal(ctx->f32, 0.0f);
2126
2127 /* Compute dot products of position and user clip plane vectors */
2128 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
2129 for (const_chan = 0; const_chan < TGSI_NUM_CHANNELS; const_chan++) {
2130 LLVMValueRef addr =
2131 LLVMConstInt(ctx->i32, ((reg_index * 4 + chan) * 4 +
2132 const_chan) * 4, 0);
2133 base_elt = buffer_load_const(ctx, const_resource,
2134 addr);
2135 args->out[chan] =
2136 lp_build_add(base, args->out[chan],
2137 lp_build_mul(base, base_elt,
2138 out_elts[const_chan]));
2139 }
2140 }
2141
2142 args->enabled_channels = 0xf;
2143 args->valid_mask = 0;
2144 args->done = 0;
2145 args->target = V_008DFC_SQ_EXP_POS + 2 + reg_index;
2146 args->compr = 0;
2147 }
2148 }
2149
2150 static void si_dump_streamout(struct pipe_stream_output_info *so)
2151 {
2152 unsigned i;
2153
2154 if (so->num_outputs)
2155 fprintf(stderr, "STREAMOUT\n");
2156
2157 for (i = 0; i < so->num_outputs; i++) {
2158 unsigned mask = ((1 << so->output[i].num_components) - 1) <<
2159 so->output[i].start_component;
2160 fprintf(stderr, " %i: BUF%i[%i..%i] <- OUT[%i].%s%s%s%s\n",
2161 i, so->output[i].output_buffer,
2162 so->output[i].dst_offset, so->output[i].dst_offset + so->output[i].num_components - 1,
2163 so->output[i].register_index,
2164 mask & 1 ? "x" : "",
2165 mask & 2 ? "y" : "",
2166 mask & 4 ? "z" : "",
2167 mask & 8 ? "w" : "");
2168 }
2169 }
2170
2171 static void emit_streamout_output(struct si_shader_context *ctx,
2172 LLVMValueRef const *so_buffers,
2173 LLVMValueRef const *so_write_offsets,
2174 struct pipe_stream_output *stream_out,
2175 struct si_shader_output_values *shader_out)
2176 {
2177 struct gallivm_state *gallivm = &ctx->gallivm;
2178 LLVMBuilderRef builder = gallivm->builder;
2179 unsigned buf_idx = stream_out->output_buffer;
2180 unsigned start = stream_out->start_component;
2181 unsigned num_comps = stream_out->num_components;
2182 LLVMValueRef out[4];
2183
2184 assert(num_comps && num_comps <= 4);
2185 if (!num_comps || num_comps > 4)
2186 return;
2187
2188 /* Load the output as int. */
2189 for (int j = 0; j < num_comps; j++) {
2190 assert(stream_out->stream == shader_out->vertex_stream[start + j]);
2191
2192 out[j] = LLVMBuildBitCast(builder,
2193 shader_out->values[start + j],
2194 ctx->i32, "");
2195 }
2196
2197 /* Pack the output. */
2198 LLVMValueRef vdata = NULL;
2199
2200 switch (num_comps) {
2201 case 1: /* as i32 */
2202 vdata = out[0];
2203 break;
2204 case 2: /* as v2i32 */
2205 case 3: /* as v4i32 (aligned to 4) */
2206 case 4: /* as v4i32 */
2207 vdata = LLVMGetUndef(LLVMVectorType(ctx->i32, util_next_power_of_two(num_comps)));
2208 for (int j = 0; j < num_comps; j++) {
2209 vdata = LLVMBuildInsertElement(builder, vdata, out[j],
2210 LLVMConstInt(ctx->i32, j, 0), "");
2211 }
2212 break;
2213 }
2214
2215 ac_build_buffer_store_dword(&ctx->ac, so_buffers[buf_idx],
2216 vdata, num_comps,
2217 so_write_offsets[buf_idx],
2218 ctx->i32_0,
2219 stream_out->dst_offset * 4, 1, 1, true, false);
2220 }
2221
2222 /**
2223 * Write streamout data to buffers for vertex stream @p stream (different
2224 * vertex streams can occur for GS copy shaders).
2225 */
2226 static void si_llvm_emit_streamout(struct si_shader_context *ctx,
2227 struct si_shader_output_values *outputs,
2228 unsigned noutput, unsigned stream)
2229 {
2230 struct si_shader_selector *sel = ctx->shader->selector;
2231 struct pipe_stream_output_info *so = &sel->so;
2232 struct gallivm_state *gallivm = &ctx->gallivm;
2233 LLVMBuilderRef builder = gallivm->builder;
2234 int i;
2235 struct lp_build_if_state if_ctx;
2236
2237 /* Get bits [22:16], i.e. (so_param >> 16) & 127; */
2238 LLVMValueRef so_vtx_count =
2239 unpack_param(ctx, ctx->param_streamout_config, 16, 7);
2240
2241 LLVMValueRef tid = ac_get_thread_id(&ctx->ac);
2242
2243 /* can_emit = tid < so_vtx_count; */
2244 LLVMValueRef can_emit =
2245 LLVMBuildICmp(builder, LLVMIntULT, tid, so_vtx_count, "");
2246
2247 /* Emit the streamout code conditionally. This actually avoids
2248 * out-of-bounds buffer access. The hw tells us via the SGPR
2249 * (so_vtx_count) which threads are allowed to emit streamout data. */
2250 lp_build_if(&if_ctx, gallivm, can_emit);
2251 {
2252 /* The buffer offset is computed as follows:
2253 * ByteOffset = streamout_offset[buffer_id]*4 +
2254 * (streamout_write_index + thread_id)*stride[buffer_id] +
2255 * attrib_offset
2256 */
2257
2258 LLVMValueRef so_write_index =
2259 LLVMGetParam(ctx->main_fn,
2260 ctx->param_streamout_write_index);
2261
2262 /* Compute (streamout_write_index + thread_id). */
2263 so_write_index = LLVMBuildAdd(builder, so_write_index, tid, "");
2264
2265 /* Load the descriptor and compute the write offset for each
2266 * enabled buffer. */
2267 LLVMValueRef so_write_offset[4] = {};
2268 LLVMValueRef so_buffers[4];
2269 LLVMValueRef buf_ptr = LLVMGetParam(ctx->main_fn,
2270 ctx->param_rw_buffers);
2271
2272 for (i = 0; i < 4; i++) {
2273 if (!so->stride[i])
2274 continue;
2275
2276 LLVMValueRef offset = LLVMConstInt(ctx->i32,
2277 SI_VS_STREAMOUT_BUF0 + i, 0);
2278
2279 so_buffers[i] = ac_build_indexed_load_const(&ctx->ac, buf_ptr, offset);
2280
2281 LLVMValueRef so_offset = LLVMGetParam(ctx->main_fn,
2282 ctx->param_streamout_offset[i]);
2283 so_offset = LLVMBuildMul(builder, so_offset, LLVMConstInt(ctx->i32, 4, 0), "");
2284
2285 so_write_offset[i] = LLVMBuildMul(builder, so_write_index,
2286 LLVMConstInt(ctx->i32, so->stride[i]*4, 0), "");
2287 so_write_offset[i] = LLVMBuildAdd(builder, so_write_offset[i], so_offset, "");
2288 }
2289
2290 /* Write streamout data. */
2291 for (i = 0; i < so->num_outputs; i++) {
2292 unsigned reg = so->output[i].register_index;
2293
2294 if (reg >= noutput)
2295 continue;
2296
2297 if (stream != so->output[i].stream)
2298 continue;
2299
2300 emit_streamout_output(ctx, so_buffers, so_write_offset,
2301 &so->output[i], &outputs[reg]);
2302 }
2303 }
2304 lp_build_endif(&if_ctx);
2305 }
2306
2307 static void si_export_param(struct si_shader_context *ctx, unsigned index,
2308 LLVMValueRef *values)
2309 {
2310 struct ac_export_args args;
2311
2312 si_llvm_init_export_args(&ctx->bld_base, values,
2313 V_008DFC_SQ_EXP_PARAM + index, &args);
2314 ac_build_export(&ctx->ac, &args);
2315 }
2316
2317 static void si_build_param_exports(struct si_shader_context *ctx,
2318 struct si_shader_output_values *outputs,
2319 unsigned noutput)
2320 {
2321 struct si_shader *shader = ctx->shader;
2322 unsigned param_count = 0;
2323
2324 for (unsigned i = 0; i < noutput; i++) {
2325 unsigned semantic_name = outputs[i].semantic_name;
2326 unsigned semantic_index = outputs[i].semantic_index;
2327
2328 if (outputs[i].vertex_stream[0] != 0 &&
2329 outputs[i].vertex_stream[1] != 0 &&
2330 outputs[i].vertex_stream[2] != 0 &&
2331 outputs[i].vertex_stream[3] != 0)
2332 continue;
2333
2334 switch (semantic_name) {
2335 case TGSI_SEMANTIC_LAYER:
2336 case TGSI_SEMANTIC_VIEWPORT_INDEX:
2337 case TGSI_SEMANTIC_CLIPDIST:
2338 case TGSI_SEMANTIC_COLOR:
2339 case TGSI_SEMANTIC_BCOLOR:
2340 case TGSI_SEMANTIC_PRIMID:
2341 case TGSI_SEMANTIC_FOG:
2342 case TGSI_SEMANTIC_TEXCOORD:
2343 case TGSI_SEMANTIC_GENERIC:
2344 break;
2345 default:
2346 continue;
2347 }
2348
2349 if ((semantic_name != TGSI_SEMANTIC_GENERIC ||
2350 semantic_index < SI_MAX_IO_GENERIC) &&
2351 shader->key.opt.kill_outputs &
2352 (1ull << si_shader_io_get_unique_index(semantic_name, semantic_index)))
2353 continue;
2354
2355 si_export_param(ctx, param_count, outputs[i].values);
2356
2357 assert(i < ARRAY_SIZE(shader->info.vs_output_param_offset));
2358 shader->info.vs_output_param_offset[i] = param_count++;
2359 }
2360
2361 shader->info.nr_param_exports = param_count;
2362 }
2363
2364 /* Generate export instructions for hardware VS shader stage */
2365 static void si_llvm_export_vs(struct lp_build_tgsi_context *bld_base,
2366 struct si_shader_output_values *outputs,
2367 unsigned noutput)
2368 {
2369 struct si_shader_context *ctx = si_shader_context(bld_base);
2370 struct si_shader *shader = ctx->shader;
2371 struct lp_build_context *base = &bld_base->base;
2372 struct ac_export_args pos_args[4] = {};
2373 LLVMValueRef psize_value = NULL, edgeflag_value = NULL, layer_value = NULL, viewport_index_value = NULL;
2374 unsigned pos_idx;
2375 int i;
2376
2377 /* Build position exports. */
2378 for (i = 0; i < noutput; i++) {
2379 switch (outputs[i].semantic_name) {
2380 case TGSI_SEMANTIC_POSITION:
2381 si_llvm_init_export_args(bld_base, outputs[i].values,
2382 V_008DFC_SQ_EXP_POS, &pos_args[0]);
2383 break;
2384 case TGSI_SEMANTIC_PSIZE:
2385 psize_value = outputs[i].values[0];
2386 break;
2387 case TGSI_SEMANTIC_LAYER:
2388 layer_value = outputs[i].values[0];
2389 break;
2390 case TGSI_SEMANTIC_VIEWPORT_INDEX:
2391 viewport_index_value = outputs[i].values[0];
2392 break;
2393 case TGSI_SEMANTIC_EDGEFLAG:
2394 edgeflag_value = outputs[i].values[0];
2395 break;
2396 case TGSI_SEMANTIC_CLIPDIST:
2397 if (!shader->key.opt.clip_disable) {
2398 unsigned index = 2 + outputs[i].semantic_index;
2399 si_llvm_init_export_args(bld_base, outputs[i].values,
2400 V_008DFC_SQ_EXP_POS + index,
2401 &pos_args[index]);
2402 }
2403 break;
2404 case TGSI_SEMANTIC_CLIPVERTEX:
2405 if (!shader->key.opt.clip_disable) {
2406 si_llvm_emit_clipvertex(bld_base, pos_args,
2407 outputs[i].values);
2408 }
2409 break;
2410 }
2411 }
2412
2413 /* We need to add the position output manually if it's missing. */
2414 if (!pos_args[0].out[0]) {
2415 pos_args[0].enabled_channels = 0xf; /* writemask */
2416 pos_args[0].valid_mask = 0; /* EXEC mask */
2417 pos_args[0].done = 0; /* last export? */
2418 pos_args[0].target = V_008DFC_SQ_EXP_POS;
2419 pos_args[0].compr = 0; /* COMPR flag */
2420 pos_args[0].out[0] = base->zero; /* X */
2421 pos_args[0].out[1] = base->zero; /* Y */
2422 pos_args[0].out[2] = base->zero; /* Z */
2423 pos_args[0].out[3] = base->one; /* W */
2424 }
2425
2426 /* Write the misc vector (point size, edgeflag, layer, viewport). */
2427 if (shader->selector->info.writes_psize ||
2428 shader->selector->info.writes_edgeflag ||
2429 shader->selector->info.writes_viewport_index ||
2430 shader->selector->info.writes_layer) {
2431 pos_args[1].enabled_channels = shader->selector->info.writes_psize |
2432 (shader->selector->info.writes_edgeflag << 1) |
2433 (shader->selector->info.writes_layer << 2);
2434
2435 pos_args[1].valid_mask = 0; /* EXEC mask */
2436 pos_args[1].done = 0; /* last export? */
2437 pos_args[1].target = V_008DFC_SQ_EXP_POS + 1;
2438 pos_args[1].compr = 0; /* COMPR flag */
2439 pos_args[1].out[0] = base->zero; /* X */
2440 pos_args[1].out[1] = base->zero; /* Y */
2441 pos_args[1].out[2] = base->zero; /* Z */
2442 pos_args[1].out[3] = base->zero; /* W */
2443
2444 if (shader->selector->info.writes_psize)
2445 pos_args[1].out[0] = psize_value;
2446
2447 if (shader->selector->info.writes_edgeflag) {
2448 /* The output is a float, but the hw expects an integer
2449 * with the first bit containing the edge flag. */
2450 edgeflag_value = LLVMBuildFPToUI(ctx->gallivm.builder,
2451 edgeflag_value,
2452 ctx->i32, "");
2453 edgeflag_value = lp_build_min(&bld_base->int_bld,
2454 edgeflag_value,
2455 ctx->i32_1);
2456
2457 /* The LLVM intrinsic expects a float. */
2458 pos_args[1].out[1] = LLVMBuildBitCast(ctx->gallivm.builder,
2459 edgeflag_value,
2460 ctx->f32, "");
2461 }
2462
2463 if (ctx->screen->b.chip_class >= GFX9) {
2464 /* GFX9 has the layer in out.z[10:0] and the viewport
2465 * index in out.z[19:16].
2466 */
2467 if (shader->selector->info.writes_layer)
2468 pos_args[1].out[2] = layer_value;
2469
2470 if (shader->selector->info.writes_viewport_index) {
2471 LLVMValueRef v = viewport_index_value;
2472
2473 v = bitcast(bld_base, TGSI_TYPE_UNSIGNED, v);
2474 v = LLVMBuildShl(ctx->gallivm.builder, v,
2475 LLVMConstInt(ctx->i32, 16, 0), "");
2476 v = LLVMBuildOr(ctx->gallivm.builder, v,
2477 bitcast(bld_base, TGSI_TYPE_UNSIGNED,
2478 pos_args[1].out[2]), "");
2479 pos_args[1].out[2] = bitcast(bld_base, TGSI_TYPE_FLOAT, v);
2480 pos_args[1].enabled_channels |= 1 << 2;
2481 }
2482 } else {
2483 if (shader->selector->info.writes_layer)
2484 pos_args[1].out[2] = layer_value;
2485
2486 if (shader->selector->info.writes_viewport_index) {
2487 pos_args[1].out[3] = viewport_index_value;
2488 pos_args[1].enabled_channels |= 1 << 3;
2489 }
2490 }
2491 }
2492
2493 for (i = 0; i < 4; i++)
2494 if (pos_args[i].out[0])
2495 shader->info.nr_pos_exports++;
2496
2497 pos_idx = 0;
2498 for (i = 0; i < 4; i++) {
2499 if (!pos_args[i].out[0])
2500 continue;
2501
2502 /* Specify the target we are exporting */
2503 pos_args[i].target = V_008DFC_SQ_EXP_POS + pos_idx++;
2504
2505 if (pos_idx == shader->info.nr_pos_exports)
2506 /* Specify that this is the last export */
2507 pos_args[i].done = 1;
2508
2509 ac_build_export(&ctx->ac, &pos_args[i]);
2510 }
2511
2512 /* Build parameter exports. */
2513 si_build_param_exports(ctx, outputs, noutput);
2514 }
2515
2516 /**
2517 * Forward all outputs from the vertex shader to the TES. This is only used
2518 * for the fixed function TCS.
2519 */
2520 static void si_copy_tcs_inputs(struct lp_build_tgsi_context *bld_base)
2521 {
2522 struct si_shader_context *ctx = si_shader_context(bld_base);
2523 struct gallivm_state *gallivm = &ctx->gallivm;
2524 LLVMValueRef invocation_id, buffer, buffer_offset;
2525 LLVMValueRef lds_vertex_stride, lds_vertex_offset, lds_base;
2526 uint64_t inputs;
2527
2528 invocation_id = unpack_param(ctx, ctx->param_tcs_rel_ids, 8, 5);
2529 buffer = desc_from_addr_base64k(ctx, ctx->param_tcs_offchip_addr_base64k);
2530 buffer_offset = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
2531
2532 lds_vertex_stride = unpack_param(ctx, ctx->param_vs_state_bits, 24, 8);
2533 lds_vertex_offset = LLVMBuildMul(gallivm->builder, invocation_id,
2534 lds_vertex_stride, "");
2535 lds_base = get_tcs_in_current_patch_offset(ctx);
2536 lds_base = LLVMBuildAdd(gallivm->builder, lds_base, lds_vertex_offset, "");
2537
2538 inputs = ctx->shader->key.mono.u.ff_tcs_inputs_to_copy;
2539 while (inputs) {
2540 unsigned i = u_bit_scan64(&inputs);
2541
2542 LLVMValueRef lds_ptr = LLVMBuildAdd(gallivm->builder, lds_base,
2543 LLVMConstInt(ctx->i32, 4 * i, 0),
2544 "");
2545
2546 LLVMValueRef buffer_addr = get_tcs_tes_buffer_address(ctx,
2547 get_rel_patch_id(ctx),
2548 invocation_id,
2549 LLVMConstInt(ctx->i32, i, 0));
2550
2551 LLVMValueRef value = lds_load(bld_base, TGSI_TYPE_SIGNED, ~0,
2552 lds_ptr);
2553
2554 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, buffer_addr,
2555 buffer_offset, 0, 1, 0, true, false);
2556 }
2557 }
2558
2559 static void si_write_tess_factors(struct lp_build_tgsi_context *bld_base,
2560 LLVMValueRef rel_patch_id,
2561 LLVMValueRef invocation_id,
2562 LLVMValueRef tcs_out_current_patch_data_offset)
2563 {
2564 struct si_shader_context *ctx = si_shader_context(bld_base);
2565 struct gallivm_state *gallivm = &ctx->gallivm;
2566 struct si_shader *shader = ctx->shader;
2567 unsigned tess_inner_index, tess_outer_index;
2568 LLVMValueRef lds_base, lds_inner, lds_outer, byteoffset, buffer;
2569 LLVMValueRef out[6], vec0, vec1, tf_base, inner[4], outer[4];
2570 unsigned stride, outer_comps, inner_comps, i, offset;
2571 struct lp_build_if_state if_ctx, inner_if_ctx;
2572
2573 si_llvm_emit_barrier(NULL, bld_base, NULL);
2574
2575 /* Do this only for invocation 0, because the tess levels are per-patch,
2576 * not per-vertex.
2577 *
2578 * This can't jump, because invocation 0 executes this. It should
2579 * at least mask out the loads and stores for other invocations.
2580 */
2581 lp_build_if(&if_ctx, gallivm,
2582 LLVMBuildICmp(gallivm->builder, LLVMIntEQ,
2583 invocation_id, ctx->i32_0, ""));
2584
2585 /* Determine the layout of one tess factor element in the buffer. */
2586 switch (shader->key.part.tcs.epilog.prim_mode) {
2587 case PIPE_PRIM_LINES:
2588 stride = 2; /* 2 dwords, 1 vec2 store */
2589 outer_comps = 2;
2590 inner_comps = 0;
2591 break;
2592 case PIPE_PRIM_TRIANGLES:
2593 stride = 4; /* 4 dwords, 1 vec4 store */
2594 outer_comps = 3;
2595 inner_comps = 1;
2596 break;
2597 case PIPE_PRIM_QUADS:
2598 stride = 6; /* 6 dwords, 2 stores (vec4 + vec2) */
2599 outer_comps = 4;
2600 inner_comps = 2;
2601 break;
2602 default:
2603 assert(0);
2604 return;
2605 }
2606
2607 /* Load tess_inner and tess_outer from LDS.
2608 * Any invocation can write them, so we can't get them from a temporary.
2609 */
2610 tess_inner_index = si_shader_io_get_unique_index_patch(TGSI_SEMANTIC_TESSINNER, 0);
2611 tess_outer_index = si_shader_io_get_unique_index_patch(TGSI_SEMANTIC_TESSOUTER, 0);
2612
2613 lds_base = tcs_out_current_patch_data_offset;
2614 lds_inner = LLVMBuildAdd(gallivm->builder, lds_base,
2615 LLVMConstInt(ctx->i32,
2616 tess_inner_index * 4, 0), "");
2617 lds_outer = LLVMBuildAdd(gallivm->builder, lds_base,
2618 LLVMConstInt(ctx->i32,
2619 tess_outer_index * 4, 0), "");
2620
2621 for (i = 0; i < 4; i++) {
2622 inner[i] = LLVMGetUndef(ctx->i32);
2623 outer[i] = LLVMGetUndef(ctx->i32);
2624 }
2625
2626 if (shader->key.part.tcs.epilog.prim_mode == PIPE_PRIM_LINES) {
2627 /* For isolines, the hardware expects tess factors in the
2628 * reverse order from what GLSL / TGSI specify.
2629 */
2630 outer[0] = out[1] = lds_load(bld_base, TGSI_TYPE_SIGNED, 0, lds_outer);
2631 outer[1] = out[0] = lds_load(bld_base, TGSI_TYPE_SIGNED, 1, lds_outer);
2632 } else {
2633 for (i = 0; i < outer_comps; i++) {
2634 outer[i] = out[i] =
2635 lds_load(bld_base, TGSI_TYPE_SIGNED, i, lds_outer);
2636 }
2637 for (i = 0; i < inner_comps; i++) {
2638 inner[i] = out[outer_comps+i] =
2639 lds_load(bld_base, TGSI_TYPE_SIGNED, i, lds_inner);
2640 }
2641 }
2642
2643 /* Convert the outputs to vectors for stores. */
2644 vec0 = lp_build_gather_values(gallivm, out, MIN2(stride, 4));
2645 vec1 = NULL;
2646
2647 if (stride > 4)
2648 vec1 = lp_build_gather_values(gallivm, out+4, stride - 4);
2649
2650 /* Get the buffer. */
2651 buffer = desc_from_addr_base64k(ctx, ctx->param_tcs_factor_addr_base64k);
2652
2653 /* Get the offset. */
2654 tf_base = LLVMGetParam(ctx->main_fn,
2655 ctx->param_tcs_factor_offset);
2656 byteoffset = LLVMBuildMul(gallivm->builder, rel_patch_id,
2657 LLVMConstInt(ctx->i32, 4 * stride, 0), "");
2658
2659 lp_build_if(&inner_if_ctx, gallivm,
2660 LLVMBuildICmp(gallivm->builder, LLVMIntEQ,
2661 rel_patch_id, ctx->i32_0, ""));
2662
2663 /* Store the dynamic HS control word. */
2664 offset = 0;
2665 if (ctx->screen->b.chip_class <= VI) {
2666 ac_build_buffer_store_dword(&ctx->ac, buffer,
2667 LLVMConstInt(ctx->i32, 0x80000000, 0),
2668 1, ctx->i32_0, tf_base,
2669 offset, 1, 0, true, false);
2670 offset += 4;
2671 }
2672
2673 lp_build_endif(&inner_if_ctx);
2674
2675 /* Store the tessellation factors. */
2676 ac_build_buffer_store_dword(&ctx->ac, buffer, vec0,
2677 MIN2(stride, 4), byteoffset, tf_base,
2678 offset, 1, 0, true, false);
2679 offset += 16;
2680 if (vec1)
2681 ac_build_buffer_store_dword(&ctx->ac, buffer, vec1,
2682 stride - 4, byteoffset, tf_base,
2683 offset, 1, 0, true, false);
2684
2685 /* Store the tess factors into the offchip buffer if TES reads them. */
2686 if (shader->key.part.tcs.epilog.tes_reads_tess_factors) {
2687 LLVMValueRef buf, base, inner_vec, outer_vec, tf_outer_offset;
2688 LLVMValueRef tf_inner_offset;
2689 unsigned param_outer, param_inner;
2690
2691 buf = desc_from_addr_base64k(ctx, ctx->param_tcs_offchip_addr_base64k);
2692 base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
2693
2694 param_outer = si_shader_io_get_unique_index_patch(
2695 TGSI_SEMANTIC_TESSOUTER, 0);
2696 tf_outer_offset = get_tcs_tes_buffer_address(ctx, rel_patch_id, NULL,
2697 LLVMConstInt(ctx->i32, param_outer, 0));
2698
2699 outer_vec = lp_build_gather_values(gallivm, outer,
2700 util_next_power_of_two(outer_comps));
2701
2702 ac_build_buffer_store_dword(&ctx->ac, buf, outer_vec,
2703 outer_comps, tf_outer_offset,
2704 base, 0, 1, 0, true, false);
2705 if (inner_comps) {
2706 param_inner = si_shader_io_get_unique_index_patch(
2707 TGSI_SEMANTIC_TESSINNER, 0);
2708 tf_inner_offset = get_tcs_tes_buffer_address(ctx, rel_patch_id, NULL,
2709 LLVMConstInt(ctx->i32, param_inner, 0));
2710
2711 inner_vec = inner_comps == 1 ? inner[0] :
2712 lp_build_gather_values(gallivm, inner, inner_comps);
2713 ac_build_buffer_store_dword(&ctx->ac, buf, inner_vec,
2714 inner_comps, tf_inner_offset,
2715 base, 0, 1, 0, true, false);
2716 }
2717 }
2718
2719 lp_build_endif(&if_ctx);
2720 }
2721
2722 static LLVMValueRef
2723 si_insert_input_ret(struct si_shader_context *ctx, LLVMValueRef ret,
2724 unsigned param, unsigned return_index)
2725 {
2726 return LLVMBuildInsertValue(ctx->gallivm.builder, ret,
2727 LLVMGetParam(ctx->main_fn, param),
2728 return_index, "");
2729 }
2730
2731 static LLVMValueRef
2732 si_insert_input_ret_float(struct si_shader_context *ctx, LLVMValueRef ret,
2733 unsigned param, unsigned return_index)
2734 {
2735 LLVMBuilderRef builder = ctx->gallivm.builder;
2736 LLVMValueRef p = LLVMGetParam(ctx->main_fn, param);
2737
2738 return LLVMBuildInsertValue(builder, ret,
2739 LLVMBuildBitCast(builder, p, ctx->f32, ""),
2740 return_index, "");
2741 }
2742
2743 static LLVMValueRef
2744 si_insert_input_ptr_as_2xi32(struct si_shader_context *ctx, LLVMValueRef ret,
2745 unsigned param, unsigned return_index)
2746 {
2747 LLVMBuilderRef builder = ctx->gallivm.builder;
2748 LLVMValueRef ptr, lo, hi;
2749
2750 ptr = LLVMGetParam(ctx->main_fn, param);
2751 ptr = LLVMBuildPtrToInt(builder, ptr, ctx->i64, "");
2752 ptr = LLVMBuildBitCast(builder, ptr, ctx->v2i32, "");
2753 lo = LLVMBuildExtractElement(builder, ptr, ctx->i32_0, "");
2754 hi = LLVMBuildExtractElement(builder, ptr, ctx->i32_1, "");
2755 ret = LLVMBuildInsertValue(builder, ret, lo, return_index, "");
2756 return LLVMBuildInsertValue(builder, ret, hi, return_index + 1, "");
2757 }
2758
2759 /* This only writes the tessellation factor levels. */
2760 static void si_llvm_emit_tcs_epilogue(struct lp_build_tgsi_context *bld_base)
2761 {
2762 struct si_shader_context *ctx = si_shader_context(bld_base);
2763 LLVMBuilderRef builder = ctx->gallivm.builder;
2764 LLVMValueRef rel_patch_id, invocation_id, tf_lds_offset;
2765
2766 si_copy_tcs_inputs(bld_base);
2767
2768 rel_patch_id = get_rel_patch_id(ctx);
2769 invocation_id = unpack_param(ctx, ctx->param_tcs_rel_ids, 8, 5);
2770 tf_lds_offset = get_tcs_out_current_patch_data_offset(ctx);
2771
2772 if (ctx->screen->b.chip_class >= GFX9) {
2773 LLVMBasicBlockRef blocks[2] = {
2774 LLVMGetInsertBlock(builder),
2775 ctx->merged_wrap_if_state.entry_block
2776 };
2777 LLVMValueRef values[2];
2778
2779 lp_build_endif(&ctx->merged_wrap_if_state);
2780
2781 values[0] = rel_patch_id;
2782 values[1] = LLVMGetUndef(ctx->i32);
2783 rel_patch_id = build_phi(&ctx->ac, ctx->i32, 2, values, blocks);
2784
2785 values[0] = tf_lds_offset;
2786 values[1] = LLVMGetUndef(ctx->i32);
2787 tf_lds_offset = build_phi(&ctx->ac, ctx->i32, 2, values, blocks);
2788
2789 values[0] = invocation_id;
2790 values[1] = ctx->i32_1; /* cause the epilog to skip threads */
2791 invocation_id = build_phi(&ctx->ac, ctx->i32, 2, values, blocks);
2792 }
2793
2794 /* Return epilog parameters from this function. */
2795 LLVMValueRef ret = ctx->return_value;
2796 unsigned vgpr;
2797
2798 if (ctx->screen->b.chip_class >= GFX9) {
2799 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_layout,
2800 8 + GFX9_SGPR_TCS_OFFCHIP_LAYOUT);
2801 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_addr_base64k,
2802 8 + GFX9_SGPR_TCS_OFFCHIP_ADDR_BASE64K);
2803 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_addr_base64k,
2804 8 + GFX9_SGPR_TCS_FACTOR_ADDR_BASE64K);
2805 /* Tess offchip and tess factor offsets are at the beginning. */
2806 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_offset, 2);
2807 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_offset, 4);
2808 vgpr = 8 + GFX9_SGPR_TCS_FACTOR_ADDR_BASE64K + 1;
2809 } else {
2810 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_layout,
2811 GFX6_SGPR_TCS_OFFCHIP_LAYOUT);
2812 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_addr_base64k,
2813 GFX6_SGPR_TCS_OFFCHIP_ADDR_BASE64K);
2814 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_addr_base64k,
2815 GFX6_SGPR_TCS_FACTOR_ADDR_BASE64K);
2816 /* Tess offchip and tess factor offsets are after user SGPRs. */
2817 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_offset,
2818 GFX6_TCS_NUM_USER_SGPR);
2819 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_offset,
2820 GFX6_TCS_NUM_USER_SGPR + 1);
2821 vgpr = GFX6_TCS_NUM_USER_SGPR + 2;
2822 }
2823
2824 /* VGPRs */
2825 rel_patch_id = bitcast(bld_base, TGSI_TYPE_FLOAT, rel_patch_id);
2826 invocation_id = bitcast(bld_base, TGSI_TYPE_FLOAT, invocation_id);
2827 tf_lds_offset = bitcast(bld_base, TGSI_TYPE_FLOAT, tf_lds_offset);
2828
2829 /* Leave a hole corresponding to the two input VGPRs. This ensures that
2830 * the invocation_id output does not alias the param_tcs_rel_ids input,
2831 * which saves a V_MOV on gfx9.
2832 */
2833 vgpr += 2;
2834
2835 ret = LLVMBuildInsertValue(builder, ret, rel_patch_id, vgpr++, "");
2836 ret = LLVMBuildInsertValue(builder, ret, invocation_id, vgpr++, "");
2837 ret = LLVMBuildInsertValue(builder, ret, tf_lds_offset, vgpr++, "");
2838 ctx->return_value = ret;
2839 }
2840
2841 /* Pass TCS inputs from LS to TCS on GFX9. */
2842 static void si_set_ls_return_value_for_tcs(struct si_shader_context *ctx)
2843 {
2844 LLVMValueRef ret = ctx->return_value;
2845
2846 ret = si_insert_input_ptr_as_2xi32(ctx, ret, ctx->param_rw_buffers, 0);
2847 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_offset, 2);
2848 ret = si_insert_input_ret(ctx, ret, ctx->param_merged_wave_info, 3);
2849 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_offset, 4);
2850 ret = si_insert_input_ret(ctx, ret, ctx->param_merged_scratch_offset, 5);
2851
2852 ret = si_insert_input_ret(ctx, ret, ctx->param_vs_state_bits,
2853 8 + SI_SGPR_VS_STATE_BITS);
2854 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_layout,
2855 8 + GFX9_SGPR_TCS_OFFCHIP_LAYOUT);
2856 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_out_lds_offsets,
2857 8 + GFX9_SGPR_TCS_OUT_OFFSETS);
2858 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_out_lds_layout,
2859 8 + GFX9_SGPR_TCS_OUT_LAYOUT);
2860 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_addr_base64k,
2861 8 + GFX9_SGPR_TCS_OFFCHIP_ADDR_BASE64K);
2862 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_addr_base64k,
2863 8 + GFX9_SGPR_TCS_FACTOR_ADDR_BASE64K);
2864
2865 unsigned desc_param = ctx->param_tcs_factor_addr_base64k + 2;
2866 ret = si_insert_input_ptr_as_2xi32(ctx, ret, desc_param,
2867 8 + GFX9_SGPR_TCS_CONST_AND_SHADER_BUFFERS);
2868 ret = si_insert_input_ptr_as_2xi32(ctx, ret, desc_param + 1,
2869 8 + GFX9_SGPR_TCS_SAMPLERS_AND_IMAGES);
2870
2871 unsigned vgpr = 8 + GFX9_TCS_NUM_USER_SGPR;
2872 ret = si_insert_input_ret_float(ctx, ret,
2873 ctx->param_tcs_patch_id, vgpr++);
2874 ret = si_insert_input_ret_float(ctx, ret,
2875 ctx->param_tcs_rel_ids, vgpr++);
2876 ctx->return_value = ret;
2877 }
2878
2879 /* Pass GS inputs from ES to GS on GFX9. */
2880 static void si_set_es_return_value_for_gs(struct si_shader_context *ctx)
2881 {
2882 LLVMValueRef ret = ctx->return_value;
2883
2884 ret = si_insert_input_ptr_as_2xi32(ctx, ret, ctx->param_rw_buffers, 0);
2885 ret = si_insert_input_ret(ctx, ret, ctx->param_gs2vs_offset, 2);
2886 ret = si_insert_input_ret(ctx, ret, ctx->param_merged_wave_info, 3);
2887
2888 ret = si_insert_input_ret(ctx, ret, ctx->param_merged_scratch_offset, 5);
2889
2890 unsigned desc_param = ctx->param_vs_state_bits + 1;
2891 ret = si_insert_input_ptr_as_2xi32(ctx, ret, desc_param,
2892 8 + GFX9_SGPR_GS_CONST_AND_SHADER_BUFFERS);
2893 ret = si_insert_input_ptr_as_2xi32(ctx, ret, desc_param + 1,
2894 8 + GFX9_SGPR_GS_SAMPLERS_AND_IMAGES);
2895
2896 unsigned vgpr = 8 + GFX9_GS_NUM_USER_SGPR;
2897 for (unsigned i = 0; i < 5; i++) {
2898 unsigned param = ctx->param_gs_vtx01_offset + i;
2899 ret = si_insert_input_ret_float(ctx, ret, param, vgpr++);
2900 }
2901 ctx->return_value = ret;
2902 }
2903
2904 static void si_llvm_emit_ls_epilogue(struct lp_build_tgsi_context *bld_base)
2905 {
2906 struct si_shader_context *ctx = si_shader_context(bld_base);
2907 struct si_shader *shader = ctx->shader;
2908 struct tgsi_shader_info *info = &shader->selector->info;
2909 struct gallivm_state *gallivm = &ctx->gallivm;
2910 unsigned i, chan;
2911 LLVMValueRef vertex_id = LLVMGetParam(ctx->main_fn,
2912 ctx->param_rel_auto_id);
2913 LLVMValueRef vertex_dw_stride =
2914 unpack_param(ctx, ctx->param_vs_state_bits, 24, 8);
2915 LLVMValueRef base_dw_addr = LLVMBuildMul(gallivm->builder, vertex_id,
2916 vertex_dw_stride, "");
2917
2918 /* Write outputs to LDS. The next shader (TCS aka HS) will read
2919 * its inputs from it. */
2920 for (i = 0; i < info->num_outputs; i++) {
2921 LLVMValueRef *out_ptr = ctx->outputs[i];
2922 unsigned name = info->output_semantic_name[i];
2923 unsigned index = info->output_semantic_index[i];
2924
2925 /* The ARB_shader_viewport_layer_array spec contains the
2926 * following issue:
2927 *
2928 * 2) What happens if gl_ViewportIndex or gl_Layer is
2929 * written in the vertex shader and a geometry shader is
2930 * present?
2931 *
2932 * RESOLVED: The value written by the last vertex processing
2933 * stage is used. If the last vertex processing stage
2934 * (vertex, tessellation evaluation or geometry) does not
2935 * statically assign to gl_ViewportIndex or gl_Layer, index
2936 * or layer zero is assumed.
2937 *
2938 * So writes to those outputs in VS-as-LS are simply ignored.
2939 */
2940 if (name == TGSI_SEMANTIC_LAYER ||
2941 name == TGSI_SEMANTIC_VIEWPORT_INDEX)
2942 continue;
2943
2944 int param = si_shader_io_get_unique_index(name, index);
2945 LLVMValueRef dw_addr = LLVMBuildAdd(gallivm->builder, base_dw_addr,
2946 LLVMConstInt(ctx->i32, param * 4, 0), "");
2947
2948 for (chan = 0; chan < 4; chan++) {
2949 lds_store(bld_base, chan, dw_addr,
2950 LLVMBuildLoad(gallivm->builder, out_ptr[chan], ""));
2951 }
2952 }
2953
2954 if (ctx->screen->b.chip_class >= GFX9)
2955 si_set_ls_return_value_for_tcs(ctx);
2956 }
2957
2958 static void si_llvm_emit_es_epilogue(struct lp_build_tgsi_context *bld_base)
2959 {
2960 struct si_shader_context *ctx = si_shader_context(bld_base);
2961 struct gallivm_state *gallivm = &ctx->gallivm;
2962 struct si_shader *es = ctx->shader;
2963 struct tgsi_shader_info *info = &es->selector->info;
2964 LLVMValueRef soffset = LLVMGetParam(ctx->main_fn,
2965 ctx->param_es2gs_offset);
2966 LLVMValueRef lds_base = NULL;
2967 unsigned chan;
2968 int i;
2969
2970 if (ctx->screen->b.chip_class >= GFX9 && info->num_outputs) {
2971 unsigned itemsize_dw = es->selector->esgs_itemsize / 4;
2972 LLVMValueRef vertex_idx = ac_get_thread_id(&ctx->ac);
2973 LLVMValueRef wave_idx = unpack_param(ctx, ctx->param_merged_wave_info, 24, 4);
2974 vertex_idx = LLVMBuildOr(gallivm->builder, vertex_idx,
2975 LLVMBuildMul(gallivm->builder, wave_idx,
2976 LLVMConstInt(ctx->i32, 64, false), ""), "");
2977 lds_base = LLVMBuildMul(gallivm->builder, vertex_idx,
2978 LLVMConstInt(ctx->i32, itemsize_dw, 0), "");
2979 }
2980
2981 for (i = 0; i < info->num_outputs; i++) {
2982 LLVMValueRef *out_ptr = ctx->outputs[i];
2983 int param;
2984
2985 if (info->output_semantic_name[i] == TGSI_SEMANTIC_VIEWPORT_INDEX ||
2986 info->output_semantic_name[i] == TGSI_SEMANTIC_LAYER)
2987 continue;
2988
2989 param = si_shader_io_get_unique_index(info->output_semantic_name[i],
2990 info->output_semantic_index[i]);
2991
2992 for (chan = 0; chan < 4; chan++) {
2993 LLVMValueRef out_val = LLVMBuildLoad(gallivm->builder, out_ptr[chan], "");
2994 out_val = LLVMBuildBitCast(gallivm->builder, out_val, ctx->i32, "");
2995
2996 /* GFX9 has the ESGS ring in LDS. */
2997 if (ctx->screen->b.chip_class >= GFX9) {
2998 lds_store(bld_base, param * 4 + chan, lds_base, out_val);
2999 continue;
3000 }
3001
3002 ac_build_buffer_store_dword(&ctx->ac,
3003 ctx->esgs_ring,
3004 out_val, 1, NULL, soffset,
3005 (4 * param + chan) * 4,
3006 1, 1, true, true);
3007 }
3008 }
3009
3010 if (ctx->screen->b.chip_class >= GFX9)
3011 si_set_es_return_value_for_gs(ctx);
3012 }
3013
3014 static LLVMValueRef si_get_gs_wave_id(struct si_shader_context *ctx)
3015 {
3016 if (ctx->screen->b.chip_class >= GFX9)
3017 return unpack_param(ctx, ctx->param_merged_wave_info, 16, 8);
3018 else
3019 return LLVMGetParam(ctx->main_fn, ctx->param_gs_wave_id);
3020 }
3021
3022 static void si_llvm_emit_gs_epilogue(struct lp_build_tgsi_context *bld_base)
3023 {
3024 struct si_shader_context *ctx = si_shader_context(bld_base);
3025
3026 ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_NOP | AC_SENDMSG_GS_DONE,
3027 si_get_gs_wave_id(ctx));
3028
3029 if (ctx->screen->b.chip_class >= GFX9)
3030 lp_build_endif(&ctx->merged_wrap_if_state);
3031 }
3032
3033 static void si_llvm_emit_vs_epilogue(struct lp_build_tgsi_context *bld_base)
3034 {
3035 struct si_shader_context *ctx = si_shader_context(bld_base);
3036 struct gallivm_state *gallivm = &ctx->gallivm;
3037 struct tgsi_shader_info *info = &ctx->shader->selector->info;
3038 struct si_shader_output_values *outputs = NULL;
3039 int i,j;
3040
3041 assert(!ctx->shader->is_gs_copy_shader);
3042
3043 outputs = MALLOC((info->num_outputs + 1) * sizeof(outputs[0]));
3044
3045 /* Vertex color clamping.
3046 *
3047 * This uses a state constant loaded in a user data SGPR and
3048 * an IF statement is added that clamps all colors if the constant
3049 * is true.
3050 */
3051 if (ctx->type == PIPE_SHADER_VERTEX) {
3052 struct lp_build_if_state if_ctx;
3053 LLVMValueRef cond = NULL;
3054 LLVMValueRef addr, val;
3055
3056 for (i = 0; i < info->num_outputs; i++) {
3057 if (info->output_semantic_name[i] != TGSI_SEMANTIC_COLOR &&
3058 info->output_semantic_name[i] != TGSI_SEMANTIC_BCOLOR)
3059 continue;
3060
3061 /* We've found a color. */
3062 if (!cond) {
3063 /* The state is in the first bit of the user SGPR. */
3064 cond = LLVMGetParam(ctx->main_fn,
3065 ctx->param_vs_state_bits);
3066 cond = LLVMBuildTrunc(gallivm->builder, cond,
3067 ctx->i1, "");
3068 lp_build_if(&if_ctx, gallivm, cond);
3069 }
3070
3071 for (j = 0; j < 4; j++) {
3072 addr = ctx->outputs[i][j];
3073 val = LLVMBuildLoad(gallivm->builder, addr, "");
3074 val = ac_build_clamp(&ctx->ac, val);
3075 LLVMBuildStore(gallivm->builder, val, addr);
3076 }
3077 }
3078
3079 if (cond)
3080 lp_build_endif(&if_ctx);
3081 }
3082
3083 for (i = 0; i < info->num_outputs; i++) {
3084 outputs[i].semantic_name = info->output_semantic_name[i];
3085 outputs[i].semantic_index = info->output_semantic_index[i];
3086
3087 for (j = 0; j < 4; j++) {
3088 outputs[i].values[j] =
3089 LLVMBuildLoad(gallivm->builder,
3090 ctx->outputs[i][j],
3091 "");
3092 outputs[i].vertex_stream[j] =
3093 (info->output_streams[i] >> (2 * j)) & 3;
3094 }
3095 }
3096
3097 if (ctx->shader->selector->so.num_outputs)
3098 si_llvm_emit_streamout(ctx, outputs, i, 0);
3099
3100 /* Export PrimitiveID. */
3101 if (ctx->shader->key.mono.u.vs_export_prim_id) {
3102 outputs[i].semantic_name = TGSI_SEMANTIC_PRIMID;
3103 outputs[i].semantic_index = 0;
3104 outputs[i].values[0] = bitcast(bld_base, TGSI_TYPE_FLOAT,
3105 get_primitive_id(bld_base, 0));
3106 for (j = 1; j < 4; j++)
3107 outputs[i].values[j] = LLVMConstReal(ctx->f32, 0);
3108
3109 memset(outputs[i].vertex_stream, 0,
3110 sizeof(outputs[i].vertex_stream));
3111 i++;
3112 }
3113
3114 si_llvm_export_vs(bld_base, outputs, i);
3115 FREE(outputs);
3116 }
3117
3118 struct si_ps_exports {
3119 unsigned num;
3120 struct ac_export_args args[10];
3121 };
3122
3123 unsigned si_get_spi_shader_z_format(bool writes_z, bool writes_stencil,
3124 bool writes_samplemask)
3125 {
3126 if (writes_z) {
3127 /* Z needs 32 bits. */
3128 if (writes_samplemask)
3129 return V_028710_SPI_SHADER_32_ABGR;
3130 else if (writes_stencil)
3131 return V_028710_SPI_SHADER_32_GR;
3132 else
3133 return V_028710_SPI_SHADER_32_R;
3134 } else if (writes_stencil || writes_samplemask) {
3135 /* Both stencil and sample mask need only 16 bits. */
3136 return V_028710_SPI_SHADER_UINT16_ABGR;
3137 } else {
3138 return V_028710_SPI_SHADER_ZERO;
3139 }
3140 }
3141
3142 static void si_export_mrt_z(struct lp_build_tgsi_context *bld_base,
3143 LLVMValueRef depth, LLVMValueRef stencil,
3144 LLVMValueRef samplemask, struct si_ps_exports *exp)
3145 {
3146 struct si_shader_context *ctx = si_shader_context(bld_base);
3147 struct lp_build_context *base = &bld_base->base;
3148 struct ac_export_args args;
3149 unsigned mask = 0;
3150 unsigned format = si_get_spi_shader_z_format(depth != NULL,
3151 stencil != NULL,
3152 samplemask != NULL);
3153
3154 assert(depth || stencil || samplemask);
3155
3156 args.valid_mask = 1; /* whether the EXEC mask is valid */
3157 args.done = 1; /* DONE bit */
3158
3159 /* Specify the target we are exporting */
3160 args.target = V_008DFC_SQ_EXP_MRTZ;
3161
3162 args.compr = 0; /* COMP flag */
3163 args.out[0] = base->undef; /* R, depth */
3164 args.out[1] = base->undef; /* G, stencil test value[0:7], stencil op value[8:15] */
3165 args.out[2] = base->undef; /* B, sample mask */
3166 args.out[3] = base->undef; /* A, alpha to mask */
3167
3168 if (format == V_028710_SPI_SHADER_UINT16_ABGR) {
3169 assert(!depth);
3170 args.compr = 1; /* COMPR flag */
3171
3172 if (stencil) {
3173 /* Stencil should be in X[23:16]. */
3174 stencil = bitcast(bld_base, TGSI_TYPE_UNSIGNED, stencil);
3175 stencil = LLVMBuildShl(ctx->gallivm.builder, stencil,
3176 LLVMConstInt(ctx->i32, 16, 0), "");
3177 args.out[0] = bitcast(bld_base, TGSI_TYPE_FLOAT, stencil);
3178 mask |= 0x3;
3179 }
3180 if (samplemask) {
3181 /* SampleMask should be in Y[15:0]. */
3182 args.out[1] = samplemask;
3183 mask |= 0xc;
3184 }
3185 } else {
3186 if (depth) {
3187 args.out[0] = depth;
3188 mask |= 0x1;
3189 }
3190 if (stencil) {
3191 args.out[1] = stencil;
3192 mask |= 0x2;
3193 }
3194 if (samplemask) {
3195 args.out[2] = samplemask;
3196 mask |= 0x4;
3197 }
3198 }
3199
3200 /* SI (except OLAND and HAINAN) has a bug that it only looks
3201 * at the X writemask component. */
3202 if (ctx->screen->b.chip_class == SI &&
3203 ctx->screen->b.family != CHIP_OLAND &&
3204 ctx->screen->b.family != CHIP_HAINAN)
3205 mask |= 0x1;
3206
3207 /* Specify which components to enable */
3208 args.enabled_channels = mask;
3209
3210 memcpy(&exp->args[exp->num++], &args, sizeof(args));
3211 }
3212
3213 static void si_export_mrt_color(struct lp_build_tgsi_context *bld_base,
3214 LLVMValueRef *color, unsigned index,
3215 unsigned samplemask_param,
3216 bool is_last, struct si_ps_exports *exp)
3217 {
3218 struct si_shader_context *ctx = si_shader_context(bld_base);
3219 struct lp_build_context *base = &bld_base->base;
3220 int i;
3221
3222 /* Clamp color */
3223 if (ctx->shader->key.part.ps.epilog.clamp_color)
3224 for (i = 0; i < 4; i++)
3225 color[i] = ac_build_clamp(&ctx->ac, color[i]);
3226
3227 /* Alpha to one */
3228 if (ctx->shader->key.part.ps.epilog.alpha_to_one)
3229 color[3] = base->one;
3230
3231 /* Alpha test */
3232 if (index == 0 &&
3233 ctx->shader->key.part.ps.epilog.alpha_func != PIPE_FUNC_ALWAYS)
3234 si_alpha_test(bld_base, color[3]);
3235
3236 /* Line & polygon smoothing */
3237 if (ctx->shader->key.part.ps.epilog.poly_line_smoothing)
3238 color[3] = si_scale_alpha_by_sample_mask(bld_base, color[3],
3239 samplemask_param);
3240
3241 /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
3242 if (ctx->shader->key.part.ps.epilog.last_cbuf > 0) {
3243 struct ac_export_args args[8];
3244 int c, last = -1;
3245
3246 /* Get the export arguments, also find out what the last one is. */
3247 for (c = 0; c <= ctx->shader->key.part.ps.epilog.last_cbuf; c++) {
3248 si_llvm_init_export_args(bld_base, color,
3249 V_008DFC_SQ_EXP_MRT + c, &args[c]);
3250 if (args[c].enabled_channels)
3251 last = c;
3252 }
3253
3254 /* Emit all exports. */
3255 for (c = 0; c <= ctx->shader->key.part.ps.epilog.last_cbuf; c++) {
3256 if (is_last && last == c) {
3257 args[c].valid_mask = 1; /* whether the EXEC mask is valid */
3258 args[c].done = 1; /* DONE bit */
3259 } else if (!args[c].enabled_channels)
3260 continue; /* unnecessary NULL export */
3261
3262 memcpy(&exp->args[exp->num++], &args[c], sizeof(args[c]));
3263 }
3264 } else {
3265 struct ac_export_args args;
3266
3267 /* Export */
3268 si_llvm_init_export_args(bld_base, color, V_008DFC_SQ_EXP_MRT + index,
3269 &args);
3270 if (is_last) {
3271 args.valid_mask = 1; /* whether the EXEC mask is valid */
3272 args.done = 1; /* DONE bit */
3273 } else if (!args.enabled_channels)
3274 return; /* unnecessary NULL export */
3275
3276 memcpy(&exp->args[exp->num++], &args, sizeof(args));
3277 }
3278 }
3279
3280 static void si_emit_ps_exports(struct si_shader_context *ctx,
3281 struct si_ps_exports *exp)
3282 {
3283 for (unsigned i = 0; i < exp->num; i++)
3284 ac_build_export(&ctx->ac, &exp->args[i]);
3285 }
3286
3287 static void si_export_null(struct lp_build_tgsi_context *bld_base)
3288 {
3289 struct si_shader_context *ctx = si_shader_context(bld_base);
3290 struct lp_build_context *base = &bld_base->base;
3291 struct ac_export_args args;
3292
3293 args.enabled_channels = 0x0; /* enabled channels */
3294 args.valid_mask = 1; /* whether the EXEC mask is valid */
3295 args.done = 1; /* DONE bit */
3296 args.target = V_008DFC_SQ_EXP_NULL;
3297 args.compr = 0; /* COMPR flag (0 = 32-bit export) */
3298 args.out[0] = base->undef; /* R */
3299 args.out[1] = base->undef; /* G */
3300 args.out[2] = base->undef; /* B */
3301 args.out[3] = base->undef; /* A */
3302
3303 ac_build_export(&ctx->ac, &args);
3304 }
3305
3306 /**
3307 * Return PS outputs in this order:
3308 *
3309 * v[0:3] = color0.xyzw
3310 * v[4:7] = color1.xyzw
3311 * ...
3312 * vN+0 = Depth
3313 * vN+1 = Stencil
3314 * vN+2 = SampleMask
3315 * vN+3 = SampleMaskIn (used for OpenGL smoothing)
3316 *
3317 * The alpha-ref SGPR is returned via its original location.
3318 */
3319 static void si_llvm_return_fs_outputs(struct lp_build_tgsi_context *bld_base)
3320 {
3321 struct si_shader_context *ctx = si_shader_context(bld_base);
3322 struct si_shader *shader = ctx->shader;
3323 struct tgsi_shader_info *info = &shader->selector->info;
3324 LLVMBuilderRef builder = ctx->gallivm.builder;
3325 unsigned i, j, first_vgpr, vgpr;
3326
3327 LLVMValueRef color[8][4] = {};
3328 LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
3329 LLVMValueRef ret;
3330
3331 if (ctx->postponed_kill)
3332 ac_build_kill(&ctx->ac, LLVMBuildLoad(builder, ctx->postponed_kill, ""));
3333
3334 /* Read the output values. */
3335 for (i = 0; i < info->num_outputs; i++) {
3336 unsigned semantic_name = info->output_semantic_name[i];
3337 unsigned semantic_index = info->output_semantic_index[i];
3338
3339 switch (semantic_name) {
3340 case TGSI_SEMANTIC_COLOR:
3341 assert(semantic_index < 8);
3342 for (j = 0; j < 4; j++) {
3343 LLVMValueRef ptr = ctx->outputs[i][j];
3344 LLVMValueRef result = LLVMBuildLoad(builder, ptr, "");
3345 color[semantic_index][j] = result;
3346 }
3347 break;
3348 case TGSI_SEMANTIC_POSITION:
3349 depth = LLVMBuildLoad(builder,
3350 ctx->outputs[i][2], "");
3351 break;
3352 case TGSI_SEMANTIC_STENCIL:
3353 stencil = LLVMBuildLoad(builder,
3354 ctx->outputs[i][1], "");
3355 break;
3356 case TGSI_SEMANTIC_SAMPLEMASK:
3357 samplemask = LLVMBuildLoad(builder,
3358 ctx->outputs[i][0], "");
3359 break;
3360 default:
3361 fprintf(stderr, "Warning: SI unhandled fs output type:%d\n",
3362 semantic_name);
3363 }
3364 }
3365
3366 /* Fill the return structure. */
3367 ret = ctx->return_value;
3368
3369 /* Set SGPRs. */
3370 ret = LLVMBuildInsertValue(builder, ret,
3371 bitcast(bld_base, TGSI_TYPE_SIGNED,
3372 LLVMGetParam(ctx->main_fn,
3373 SI_PARAM_ALPHA_REF)),
3374 SI_SGPR_ALPHA_REF, "");
3375
3376 /* Set VGPRs */
3377 first_vgpr = vgpr = SI_SGPR_ALPHA_REF + 1;
3378 for (i = 0; i < ARRAY_SIZE(color); i++) {
3379 if (!color[i][0])
3380 continue;
3381
3382 for (j = 0; j < 4; j++)
3383 ret = LLVMBuildInsertValue(builder, ret, color[i][j], vgpr++, "");
3384 }
3385 if (depth)
3386 ret = LLVMBuildInsertValue(builder, ret, depth, vgpr++, "");
3387 if (stencil)
3388 ret = LLVMBuildInsertValue(builder, ret, stencil, vgpr++, "");
3389 if (samplemask)
3390 ret = LLVMBuildInsertValue(builder, ret, samplemask, vgpr++, "");
3391
3392 /* Add the input sample mask for smoothing at the end. */
3393 if (vgpr < first_vgpr + PS_EPILOG_SAMPLEMASK_MIN_LOC)
3394 vgpr = first_vgpr + PS_EPILOG_SAMPLEMASK_MIN_LOC;
3395 ret = LLVMBuildInsertValue(builder, ret,
3396 LLVMGetParam(ctx->main_fn,
3397 SI_PARAM_SAMPLE_COVERAGE), vgpr++, "");
3398
3399 ctx->return_value = ret;
3400 }
3401
3402 /* Prevent optimizations (at least of memory accesses) across the current
3403 * point in the program by emitting empty inline assembly that is marked as
3404 * having side effects.
3405 *
3406 * Optionally, a value can be passed through the inline assembly to prevent
3407 * LLVM from hoisting calls to ReadNone functions.
3408 */
3409 static void emit_optimization_barrier(struct si_shader_context *ctx,
3410 LLVMValueRef *pvgpr)
3411 {
3412 static int counter = 0;
3413
3414 LLVMBuilderRef builder = ctx->gallivm.builder;
3415 char code[16];
3416
3417 snprintf(code, sizeof(code), "; %d", p_atomic_inc_return(&counter));
3418
3419 if (!pvgpr) {
3420 LLVMTypeRef ftype = LLVMFunctionType(ctx->voidt, NULL, 0, false);
3421 LLVMValueRef inlineasm = LLVMConstInlineAsm(ftype, code, "", true, false);
3422 LLVMBuildCall(builder, inlineasm, NULL, 0, "");
3423 } else {
3424 LLVMTypeRef ftype = LLVMFunctionType(ctx->i32, &ctx->i32, 1, false);
3425 LLVMValueRef inlineasm = LLVMConstInlineAsm(ftype, code, "=v,0", true, false);
3426 LLVMValueRef vgpr = *pvgpr;
3427 LLVMTypeRef vgpr_type = LLVMTypeOf(vgpr);
3428 unsigned vgpr_size = llvm_get_type_size(vgpr_type);
3429 LLVMValueRef vgpr0;
3430
3431 assert(vgpr_size % 4 == 0);
3432
3433 vgpr = LLVMBuildBitCast(builder, vgpr, LLVMVectorType(ctx->i32, vgpr_size / 4), "");
3434 vgpr0 = LLVMBuildExtractElement(builder, vgpr, ctx->i32_0, "");
3435 vgpr0 = LLVMBuildCall(builder, inlineasm, &vgpr0, 1, "");
3436 vgpr = LLVMBuildInsertElement(builder, vgpr, vgpr0, ctx->i32_0, "");
3437 vgpr = LLVMBuildBitCast(builder, vgpr, vgpr_type, "");
3438
3439 *pvgpr = vgpr;
3440 }
3441 }
3442
3443 void si_emit_waitcnt(struct si_shader_context *ctx, unsigned simm16)
3444 {
3445 struct gallivm_state *gallivm = &ctx->gallivm;
3446 LLVMBuilderRef builder = gallivm->builder;
3447 LLVMValueRef args[1] = {
3448 LLVMConstInt(ctx->i32, simm16, 0)
3449 };
3450 lp_build_intrinsic(builder, "llvm.amdgcn.s.waitcnt",
3451 ctx->voidt, args, 1, 0);
3452 }
3453
3454 static void membar_emit(
3455 const struct lp_build_tgsi_action *action,
3456 struct lp_build_tgsi_context *bld_base,
3457 struct lp_build_emit_data *emit_data)
3458 {
3459 struct si_shader_context *ctx = si_shader_context(bld_base);
3460 LLVMValueRef src0 = lp_build_emit_fetch(bld_base, emit_data->inst, 0, 0);
3461 unsigned flags = LLVMConstIntGetZExtValue(src0);
3462 unsigned waitcnt = NOOP_WAITCNT;
3463
3464 if (flags & TGSI_MEMBAR_THREAD_GROUP)
3465 waitcnt &= VM_CNT & LGKM_CNT;
3466
3467 if (flags & (TGSI_MEMBAR_ATOMIC_BUFFER |
3468 TGSI_MEMBAR_SHADER_BUFFER |
3469 TGSI_MEMBAR_SHADER_IMAGE))
3470 waitcnt &= VM_CNT;
3471
3472 if (flags & TGSI_MEMBAR_SHARED)
3473 waitcnt &= LGKM_CNT;
3474
3475 if (waitcnt != NOOP_WAITCNT)
3476 si_emit_waitcnt(ctx, waitcnt);
3477 }
3478
3479 static void clock_emit(
3480 const struct lp_build_tgsi_action *action,
3481 struct lp_build_tgsi_context *bld_base,
3482 struct lp_build_emit_data *emit_data)
3483 {
3484 struct si_shader_context *ctx = si_shader_context(bld_base);
3485 struct gallivm_state *gallivm = &ctx->gallivm;
3486 LLVMValueRef tmp;
3487
3488 tmp = lp_build_intrinsic(gallivm->builder, "llvm.readcyclecounter",
3489 ctx->i64, NULL, 0, 0);
3490 tmp = LLVMBuildBitCast(gallivm->builder, tmp, ctx->v2i32, "");
3491
3492 emit_data->output[0] =
3493 LLVMBuildExtractElement(gallivm->builder, tmp, ctx->i32_0, "");
3494 emit_data->output[1] =
3495 LLVMBuildExtractElement(gallivm->builder, tmp, ctx->i32_1, "");
3496 }
3497
3498 LLVMTypeRef si_const_array(LLVMTypeRef elem_type, int num_elements)
3499 {
3500 return LLVMPointerType(LLVMArrayType(elem_type, num_elements),
3501 CONST_ADDR_SPACE);
3502 }
3503
3504 static void si_llvm_emit_ddxy(
3505 const struct lp_build_tgsi_action *action,
3506 struct lp_build_tgsi_context *bld_base,
3507 struct lp_build_emit_data *emit_data)
3508 {
3509 struct si_shader_context *ctx = si_shader_context(bld_base);
3510 struct gallivm_state *gallivm = &ctx->gallivm;
3511 unsigned opcode = emit_data->info->opcode;
3512 LLVMValueRef val;
3513 int idx;
3514 unsigned mask;
3515
3516 if (opcode == TGSI_OPCODE_DDX_FINE)
3517 mask = AC_TID_MASK_LEFT;
3518 else if (opcode == TGSI_OPCODE_DDY_FINE)
3519 mask = AC_TID_MASK_TOP;
3520 else
3521 mask = AC_TID_MASK_TOP_LEFT;
3522
3523 /* for DDX we want to next X pixel, DDY next Y pixel. */
3524 idx = (opcode == TGSI_OPCODE_DDX || opcode == TGSI_OPCODE_DDX_FINE) ? 1 : 2;
3525
3526 val = LLVMBuildBitCast(gallivm->builder, emit_data->args[0], ctx->i32, "");
3527 val = ac_build_ddxy(&ctx->ac, ctx->screen->has_ds_bpermute,
3528 mask, idx, ctx->lds, val);
3529 emit_data->output[emit_data->chan] = val;
3530 }
3531
3532 /*
3533 * this takes an I,J coordinate pair,
3534 * and works out the X and Y derivatives.
3535 * it returns DDX(I), DDX(J), DDY(I), DDY(J).
3536 */
3537 static LLVMValueRef si_llvm_emit_ddxy_interp(
3538 struct lp_build_tgsi_context *bld_base,
3539 LLVMValueRef interp_ij)
3540 {
3541 struct si_shader_context *ctx = si_shader_context(bld_base);
3542 struct gallivm_state *gallivm = &ctx->gallivm;
3543 LLVMValueRef result[4], a;
3544 unsigned i;
3545
3546 for (i = 0; i < 2; i++) {
3547 a = LLVMBuildExtractElement(gallivm->builder, interp_ij,
3548 LLVMConstInt(ctx->i32, i, 0), "");
3549 result[i] = lp_build_emit_llvm_unary(bld_base, TGSI_OPCODE_DDX, a);
3550 result[2+i] = lp_build_emit_llvm_unary(bld_base, TGSI_OPCODE_DDY, a);
3551 }
3552
3553 return lp_build_gather_values(gallivm, result, 4);
3554 }
3555
3556 static void interp_fetch_args(
3557 struct lp_build_tgsi_context *bld_base,
3558 struct lp_build_emit_data *emit_data)
3559 {
3560 struct si_shader_context *ctx = si_shader_context(bld_base);
3561 struct gallivm_state *gallivm = &ctx->gallivm;
3562 const struct tgsi_full_instruction *inst = emit_data->inst;
3563
3564 if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET) {
3565 /* offset is in second src, first two channels */
3566 emit_data->args[0] = lp_build_emit_fetch(bld_base,
3567 emit_data->inst, 1,
3568 TGSI_CHAN_X);
3569 emit_data->args[1] = lp_build_emit_fetch(bld_base,
3570 emit_data->inst, 1,
3571 TGSI_CHAN_Y);
3572 emit_data->arg_count = 2;
3573 } else if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE) {
3574 LLVMValueRef sample_position;
3575 LLVMValueRef sample_id;
3576 LLVMValueRef halfval = LLVMConstReal(ctx->f32, 0.5f);
3577
3578 /* fetch sample ID, then fetch its sample position,
3579 * and place into first two channels.
3580 */
3581 sample_id = lp_build_emit_fetch(bld_base,
3582 emit_data->inst, 1, TGSI_CHAN_X);
3583 sample_id = LLVMBuildBitCast(gallivm->builder, sample_id,
3584 ctx->i32, "");
3585 sample_position = load_sample_position(ctx, sample_id);
3586
3587 emit_data->args[0] = LLVMBuildExtractElement(gallivm->builder,
3588 sample_position,
3589 ctx->i32_0, "");
3590
3591 emit_data->args[0] = LLVMBuildFSub(gallivm->builder, emit_data->args[0], halfval, "");
3592 emit_data->args[1] = LLVMBuildExtractElement(gallivm->builder,
3593 sample_position,
3594 ctx->i32_1, "");
3595 emit_data->args[1] = LLVMBuildFSub(gallivm->builder, emit_data->args[1], halfval, "");
3596 emit_data->arg_count = 2;
3597 }
3598 }
3599
3600 static void build_interp_intrinsic(const struct lp_build_tgsi_action *action,
3601 struct lp_build_tgsi_context *bld_base,
3602 struct lp_build_emit_data *emit_data)
3603 {
3604 struct si_shader_context *ctx = si_shader_context(bld_base);
3605 struct si_shader *shader = ctx->shader;
3606 struct gallivm_state *gallivm = &ctx->gallivm;
3607 const struct tgsi_shader_info *info = &shader->selector->info;
3608 LLVMValueRef interp_param;
3609 const struct tgsi_full_instruction *inst = emit_data->inst;
3610 const struct tgsi_full_src_register *input = &inst->Src[0];
3611 int input_base, input_array_size;
3612 int chan;
3613 int i;
3614 LLVMValueRef prim_mask = LLVMGetParam(ctx->main_fn, SI_PARAM_PRIM_MASK);
3615 LLVMValueRef array_idx;
3616 int interp_param_idx;
3617 unsigned interp;
3618 unsigned location;
3619
3620 assert(input->Register.File == TGSI_FILE_INPUT);
3621
3622 if (input->Register.Indirect) {
3623 unsigned array_id = input->Indirect.ArrayID;
3624
3625 if (array_id) {
3626 input_base = info->input_array_first[array_id];
3627 input_array_size = info->input_array_last[array_id] - input_base + 1;
3628 } else {
3629 input_base = inst->Src[0].Register.Index;
3630 input_array_size = info->num_inputs - input_base;
3631 }
3632
3633 array_idx = get_indirect_index(ctx, &input->Indirect,
3634 input->Register.Index - input_base);
3635 } else {
3636 input_base = inst->Src[0].Register.Index;
3637 input_array_size = 1;
3638 array_idx = ctx->i32_0;
3639 }
3640
3641 interp = shader->selector->info.input_interpolate[input_base];
3642
3643 if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET ||
3644 inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE)
3645 location = TGSI_INTERPOLATE_LOC_CENTER;
3646 else
3647 location = TGSI_INTERPOLATE_LOC_CENTROID;
3648
3649 interp_param_idx = lookup_interp_param_index(interp, location);
3650 if (interp_param_idx == -1)
3651 return;
3652 else if (interp_param_idx)
3653 interp_param = LLVMGetParam(ctx->main_fn, interp_param_idx);
3654 else
3655 interp_param = NULL;
3656
3657 if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET ||
3658 inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE) {
3659 LLVMValueRef ij_out[2];
3660 LLVMValueRef ddxy_out = si_llvm_emit_ddxy_interp(bld_base, interp_param);
3661
3662 /*
3663 * take the I then J parameters, and the DDX/Y for it, and
3664 * calculate the IJ inputs for the interpolator.
3665 * temp1 = ddx * offset/sample.x + I;
3666 * interp_param.I = ddy * offset/sample.y + temp1;
3667 * temp1 = ddx * offset/sample.x + J;
3668 * interp_param.J = ddy * offset/sample.y + temp1;
3669 */
3670 for (i = 0; i < 2; i++) {
3671 LLVMValueRef ix_ll = LLVMConstInt(ctx->i32, i, 0);
3672 LLVMValueRef iy_ll = LLVMConstInt(ctx->i32, i + 2, 0);
3673 LLVMValueRef ddx_el = LLVMBuildExtractElement(gallivm->builder,
3674 ddxy_out, ix_ll, "");
3675 LLVMValueRef ddy_el = LLVMBuildExtractElement(gallivm->builder,
3676 ddxy_out, iy_ll, "");
3677 LLVMValueRef interp_el = LLVMBuildExtractElement(gallivm->builder,
3678 interp_param, ix_ll, "");
3679 LLVMValueRef temp1, temp2;
3680
3681 interp_el = LLVMBuildBitCast(gallivm->builder, interp_el,
3682 ctx->f32, "");
3683
3684 temp1 = LLVMBuildFMul(gallivm->builder, ddx_el, emit_data->args[0], "");
3685
3686 temp1 = LLVMBuildFAdd(gallivm->builder, temp1, interp_el, "");
3687
3688 temp2 = LLVMBuildFMul(gallivm->builder, ddy_el, emit_data->args[1], "");
3689
3690 ij_out[i] = LLVMBuildFAdd(gallivm->builder, temp2, temp1, "");
3691 }
3692 interp_param = lp_build_gather_values(gallivm, ij_out, 2);
3693 }
3694
3695 if (interp_param) {
3696 interp_param = LLVMBuildBitCast(gallivm->builder,
3697 interp_param, LLVMVectorType(ctx->f32, 2), "");
3698 }
3699
3700 for (chan = 0; chan < 4; chan++) {
3701 LLVMValueRef gather = LLVMGetUndef(LLVMVectorType(ctx->f32, input_array_size));
3702 unsigned schan = tgsi_util_get_full_src_register_swizzle(&inst->Src[0], chan);
3703
3704 for (unsigned idx = 0; idx < input_array_size; ++idx) {
3705 LLVMValueRef v, i = NULL, j = NULL;
3706
3707 if (interp_param) {
3708 interp_param = LLVMBuildBitCast(gallivm->builder,
3709 interp_param, LLVMVectorType(ctx->f32, 2), "");
3710 i = LLVMBuildExtractElement(
3711 gallivm->builder, interp_param, ctx->i32_0, "");
3712 j = LLVMBuildExtractElement(
3713 gallivm->builder, interp_param, ctx->i32_1, "");
3714 }
3715 v = si_build_fs_interp(ctx, input_base + idx, schan,
3716 prim_mask, i, j);
3717
3718 gather = LLVMBuildInsertElement(gallivm->builder,
3719 gather, v, LLVMConstInt(ctx->i32, idx, false), "");
3720 }
3721
3722 emit_data->output[chan] = LLVMBuildExtractElement(
3723 gallivm->builder, gather, array_idx, "");
3724 }
3725 }
3726
3727 static LLVMValueRef si_emit_ballot(struct si_shader_context *ctx,
3728 LLVMValueRef value)
3729 {
3730 struct gallivm_state *gallivm = &ctx->gallivm;
3731 LLVMValueRef args[3] = {
3732 value,
3733 ctx->i32_0,
3734 LLVMConstInt(ctx->i32, LLVMIntNE, 0)
3735 };
3736
3737 /* We currently have no other way to prevent LLVM from lifting the icmp
3738 * calls to a dominating basic block.
3739 */
3740 emit_optimization_barrier(ctx, &args[0]);
3741
3742 if (LLVMTypeOf(args[0]) != ctx->i32)
3743 args[0] = LLVMBuildBitCast(gallivm->builder, args[0], ctx->i32, "");
3744
3745 return lp_build_intrinsic(gallivm->builder,
3746 "llvm.amdgcn.icmp.i32",
3747 ctx->i64, args, 3,
3748 LP_FUNC_ATTR_NOUNWIND |
3749 LP_FUNC_ATTR_READNONE |
3750 LP_FUNC_ATTR_CONVERGENT);
3751 }
3752
3753 static void vote_all_emit(
3754 const struct lp_build_tgsi_action *action,
3755 struct lp_build_tgsi_context *bld_base,
3756 struct lp_build_emit_data *emit_data)
3757 {
3758 struct si_shader_context *ctx = si_shader_context(bld_base);
3759 struct gallivm_state *gallivm = &ctx->gallivm;
3760 LLVMValueRef active_set, vote_set;
3761 LLVMValueRef tmp;
3762
3763 active_set = si_emit_ballot(ctx, ctx->i32_1);
3764 vote_set = si_emit_ballot(ctx, emit_data->args[0]);
3765
3766 tmp = LLVMBuildICmp(gallivm->builder, LLVMIntEQ, vote_set, active_set, "");
3767 emit_data->output[emit_data->chan] =
3768 LLVMBuildSExt(gallivm->builder, tmp, ctx->i32, "");
3769 }
3770
3771 static void vote_any_emit(
3772 const struct lp_build_tgsi_action *action,
3773 struct lp_build_tgsi_context *bld_base,
3774 struct lp_build_emit_data *emit_data)
3775 {
3776 struct si_shader_context *ctx = si_shader_context(bld_base);
3777 struct gallivm_state *gallivm = &ctx->gallivm;
3778 LLVMValueRef vote_set;
3779 LLVMValueRef tmp;
3780
3781 vote_set = si_emit_ballot(ctx, emit_data->args[0]);
3782
3783 tmp = LLVMBuildICmp(gallivm->builder, LLVMIntNE,
3784 vote_set, LLVMConstInt(ctx->i64, 0, 0), "");
3785 emit_data->output[emit_data->chan] =
3786 LLVMBuildSExt(gallivm->builder, tmp, ctx->i32, "");
3787 }
3788
3789 static void vote_eq_emit(
3790 const struct lp_build_tgsi_action *action,
3791 struct lp_build_tgsi_context *bld_base,
3792 struct lp_build_emit_data *emit_data)
3793 {
3794 struct si_shader_context *ctx = si_shader_context(bld_base);
3795 struct gallivm_state *gallivm = &ctx->gallivm;
3796 LLVMValueRef active_set, vote_set;
3797 LLVMValueRef all, none, tmp;
3798
3799 active_set = si_emit_ballot(ctx, ctx->i32_1);
3800 vote_set = si_emit_ballot(ctx, emit_data->args[0]);
3801
3802 all = LLVMBuildICmp(gallivm->builder, LLVMIntEQ, vote_set, active_set, "");
3803 none = LLVMBuildICmp(gallivm->builder, LLVMIntEQ,
3804 vote_set, LLVMConstInt(ctx->i64, 0, 0), "");
3805 tmp = LLVMBuildOr(gallivm->builder, all, none, "");
3806 emit_data->output[emit_data->chan] =
3807 LLVMBuildSExt(gallivm->builder, tmp, ctx->i32, "");
3808 }
3809
3810 static void ballot_emit(
3811 const struct lp_build_tgsi_action *action,
3812 struct lp_build_tgsi_context *bld_base,
3813 struct lp_build_emit_data *emit_data)
3814 {
3815 struct si_shader_context *ctx = si_shader_context(bld_base);
3816 LLVMBuilderRef builder = ctx->gallivm.builder;
3817 LLVMValueRef tmp;
3818
3819 tmp = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_X);
3820 tmp = si_emit_ballot(ctx, tmp);
3821 tmp = LLVMBuildBitCast(builder, tmp, ctx->v2i32, "");
3822
3823 emit_data->output[0] = LLVMBuildExtractElement(builder, tmp, ctx->i32_0, "");
3824 emit_data->output[1] = LLVMBuildExtractElement(builder, tmp, ctx->i32_1, "");
3825 }
3826
3827 static void read_invoc_fetch_args(
3828 struct lp_build_tgsi_context *bld_base,
3829 struct lp_build_emit_data *emit_data)
3830 {
3831 emit_data->args[0] = lp_build_emit_fetch(bld_base, emit_data->inst,
3832 0, emit_data->src_chan);
3833
3834 /* Always read the source invocation (= lane) from the X channel. */
3835 emit_data->args[1] = lp_build_emit_fetch(bld_base, emit_data->inst,
3836 1, TGSI_CHAN_X);
3837 emit_data->arg_count = 2;
3838 }
3839
3840 static void read_lane_emit(
3841 const struct lp_build_tgsi_action *action,
3842 struct lp_build_tgsi_context *bld_base,
3843 struct lp_build_emit_data *emit_data)
3844 {
3845 struct si_shader_context *ctx = si_shader_context(bld_base);
3846 LLVMBuilderRef builder = ctx->gallivm.builder;
3847
3848 /* We currently have no other way to prevent LLVM from lifting the icmp
3849 * calls to a dominating basic block.
3850 */
3851 emit_optimization_barrier(ctx, &emit_data->args[0]);
3852
3853 for (unsigned i = 0; i < emit_data->arg_count; ++i) {
3854 emit_data->args[i] = LLVMBuildBitCast(builder, emit_data->args[i],
3855 ctx->i32, "");
3856 }
3857
3858 emit_data->output[emit_data->chan] =
3859 ac_build_intrinsic(&ctx->ac, action->intr_name,
3860 ctx->i32, emit_data->args, emit_data->arg_count,
3861 AC_FUNC_ATTR_READNONE |
3862 AC_FUNC_ATTR_CONVERGENT);
3863 }
3864
3865 static unsigned si_llvm_get_stream(struct lp_build_tgsi_context *bld_base,
3866 struct lp_build_emit_data *emit_data)
3867 {
3868 struct si_shader_context *ctx = si_shader_context(bld_base);
3869 struct tgsi_src_register src0 = emit_data->inst->Src[0].Register;
3870 LLVMValueRef imm;
3871 unsigned stream;
3872
3873 assert(src0.File == TGSI_FILE_IMMEDIATE);
3874
3875 imm = ctx->imms[src0.Index * TGSI_NUM_CHANNELS + src0.SwizzleX];
3876 stream = LLVMConstIntGetZExtValue(imm) & 0x3;
3877 return stream;
3878 }
3879
3880 /* Emit one vertex from the geometry shader */
3881 static void si_llvm_emit_vertex(
3882 const struct lp_build_tgsi_action *action,
3883 struct lp_build_tgsi_context *bld_base,
3884 struct lp_build_emit_data *emit_data)
3885 {
3886 struct si_shader_context *ctx = si_shader_context(bld_base);
3887 struct lp_build_context *uint = &bld_base->uint_bld;
3888 struct si_shader *shader = ctx->shader;
3889 struct tgsi_shader_info *info = &shader->selector->info;
3890 struct gallivm_state *gallivm = &ctx->gallivm;
3891 struct lp_build_if_state if_state;
3892 LLVMValueRef soffset = LLVMGetParam(ctx->main_fn,
3893 ctx->param_gs2vs_offset);
3894 LLVMValueRef gs_next_vertex;
3895 LLVMValueRef can_emit, kill;
3896 unsigned chan, offset;
3897 int i;
3898 unsigned stream;
3899
3900 stream = si_llvm_get_stream(bld_base, emit_data);
3901
3902 /* Write vertex attribute values to GSVS ring */
3903 gs_next_vertex = LLVMBuildLoad(gallivm->builder,
3904 ctx->gs_next_vertex[stream],
3905 "");
3906
3907 /* If this thread has already emitted the declared maximum number of
3908 * vertices, skip the write: excessive vertex emissions are not
3909 * supposed to have any effect.
3910 *
3911 * If the shader has no writes to memory, kill it instead. This skips
3912 * further memory loads and may allow LLVM to skip to the end
3913 * altogether.
3914 */
3915 can_emit = LLVMBuildICmp(gallivm->builder, LLVMIntULT, gs_next_vertex,
3916 LLVMConstInt(ctx->i32,
3917 shader->selector->gs_max_out_vertices, 0), "");
3918
3919 bool use_kill = !info->writes_memory;
3920 if (use_kill) {
3921 kill = lp_build_select(&bld_base->base, can_emit,
3922 LLVMConstReal(ctx->f32, 1.0f),
3923 LLVMConstReal(ctx->f32, -1.0f));
3924
3925 ac_build_kill(&ctx->ac, kill);
3926 } else {
3927 lp_build_if(&if_state, gallivm, can_emit);
3928 }
3929
3930 offset = 0;
3931 for (i = 0; i < info->num_outputs; i++) {
3932 LLVMValueRef *out_ptr = ctx->outputs[i];
3933
3934 for (chan = 0; chan < 4; chan++) {
3935 if (!(info->output_usagemask[i] & (1 << chan)) ||
3936 ((info->output_streams[i] >> (2 * chan)) & 3) != stream)
3937 continue;
3938
3939 LLVMValueRef out_val = LLVMBuildLoad(gallivm->builder, out_ptr[chan], "");
3940 LLVMValueRef voffset =
3941 LLVMConstInt(ctx->i32, offset *
3942 shader->selector->gs_max_out_vertices, 0);
3943 offset++;
3944
3945 voffset = lp_build_add(uint, voffset, gs_next_vertex);
3946 voffset = lp_build_mul_imm(uint, voffset, 4);
3947
3948 out_val = LLVMBuildBitCast(gallivm->builder, out_val, ctx->i32, "");
3949
3950 ac_build_buffer_store_dword(&ctx->ac,
3951 ctx->gsvs_ring[stream],
3952 out_val, 1,
3953 voffset, soffset, 0,
3954 1, 1, true, true);
3955 }
3956 }
3957
3958 gs_next_vertex = lp_build_add(uint, gs_next_vertex,
3959 ctx->i32_1);
3960
3961 LLVMBuildStore(gallivm->builder, gs_next_vertex, ctx->gs_next_vertex[stream]);
3962
3963 /* Signal vertex emission */
3964 ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_EMIT | AC_SENDMSG_GS | (stream << 8),
3965 si_get_gs_wave_id(ctx));
3966 if (!use_kill)
3967 lp_build_endif(&if_state);
3968 }
3969
3970 /* Cut one primitive from the geometry shader */
3971 static void si_llvm_emit_primitive(
3972 const struct lp_build_tgsi_action *action,
3973 struct lp_build_tgsi_context *bld_base,
3974 struct lp_build_emit_data *emit_data)
3975 {
3976 struct si_shader_context *ctx = si_shader_context(bld_base);
3977 unsigned stream;
3978
3979 /* Signal primitive cut */
3980 stream = si_llvm_get_stream(bld_base, emit_data);
3981 ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_CUT | AC_SENDMSG_GS | (stream << 8),
3982 si_get_gs_wave_id(ctx));
3983 }
3984
3985 static void si_llvm_emit_barrier(const struct lp_build_tgsi_action *action,
3986 struct lp_build_tgsi_context *bld_base,
3987 struct lp_build_emit_data *emit_data)
3988 {
3989 struct si_shader_context *ctx = si_shader_context(bld_base);
3990 struct gallivm_state *gallivm = &ctx->gallivm;
3991
3992 /* SI only (thanks to a hw bug workaround):
3993 * The real barrier instruction isn’t needed, because an entire patch
3994 * always fits into a single wave.
3995 */
3996 if (ctx->screen->b.chip_class == SI &&
3997 ctx->type == PIPE_SHADER_TESS_CTRL) {
3998 si_emit_waitcnt(ctx, LGKM_CNT & VM_CNT);
3999 return;
4000 }
4001
4002 lp_build_intrinsic(gallivm->builder,
4003 "llvm.amdgcn.s.barrier",
4004 ctx->voidt, NULL, 0, LP_FUNC_ATTR_CONVERGENT);
4005 }
4006
4007 static const struct lp_build_tgsi_action interp_action = {
4008 .fetch_args = interp_fetch_args,
4009 .emit = build_interp_intrinsic,
4010 };
4011
4012 static void si_create_function(struct si_shader_context *ctx,
4013 const char *name,
4014 LLVMTypeRef *returns, unsigned num_returns,
4015 struct si_function_info *fninfo,
4016 unsigned max_workgroup_size)
4017 {
4018 int i;
4019
4020 si_llvm_create_func(ctx, name, returns, num_returns,
4021 fninfo->types, fninfo->num_params);
4022 ctx->return_value = LLVMGetUndef(ctx->return_type);
4023
4024 for (i = 0; i < fninfo->num_sgpr_params; ++i) {
4025 LLVMValueRef P = LLVMGetParam(ctx->main_fn, i);
4026
4027 /* The combination of:
4028 * - ByVal
4029 * - dereferenceable
4030 * - invariant.load
4031 * allows the optimization passes to move loads and reduces
4032 * SGPR spilling significantly.
4033 */
4034 if (LLVMGetTypeKind(LLVMTypeOf(P)) == LLVMPointerTypeKind) {
4035 lp_add_function_attr(ctx->main_fn, i + 1, LP_FUNC_ATTR_BYVAL);
4036 lp_add_function_attr(ctx->main_fn, i + 1, LP_FUNC_ATTR_NOALIAS);
4037 ac_add_attr_dereferenceable(P, UINT64_MAX);
4038 } else
4039 lp_add_function_attr(ctx->main_fn, i + 1, LP_FUNC_ATTR_INREG);
4040 }
4041
4042 for (i = 0; i < fninfo->num_params; ++i) {
4043 if (fninfo->assign[i])
4044 *fninfo->assign[i] = LLVMGetParam(ctx->main_fn, i);
4045 }
4046
4047 if (max_workgroup_size) {
4048 si_llvm_add_attribute(ctx->main_fn, "amdgpu-max-work-group-size",
4049 max_workgroup_size);
4050 }
4051 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
4052 "no-signed-zeros-fp-math",
4053 "true");
4054
4055 if (ctx->screen->b.debug_flags & DBG_UNSAFE_MATH) {
4056 /* These were copied from some LLVM test. */
4057 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
4058 "less-precise-fpmad",
4059 "true");
4060 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
4061 "no-infs-fp-math",
4062 "true");
4063 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
4064 "no-nans-fp-math",
4065 "true");
4066 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
4067 "unsafe-fp-math",
4068 "true");
4069 }
4070 }
4071
4072 static void declare_streamout_params(struct si_shader_context *ctx,
4073 struct pipe_stream_output_info *so,
4074 struct si_function_info *fninfo)
4075 {
4076 int i;
4077
4078 /* Streamout SGPRs. */
4079 if (so->num_outputs) {
4080 if (ctx->type != PIPE_SHADER_TESS_EVAL)
4081 ctx->param_streamout_config = add_arg(fninfo, ARG_SGPR, ctx->ac.i32);
4082 else
4083 ctx->param_streamout_config = fninfo->num_params - 1;
4084
4085 ctx->param_streamout_write_index = add_arg(fninfo, ARG_SGPR, ctx->ac.i32);
4086 }
4087 /* A streamout buffer offset is loaded if the stride is non-zero. */
4088 for (i = 0; i < 4; i++) {
4089 if (!so->stride[i])
4090 continue;
4091
4092 ctx->param_streamout_offset[i] = add_arg(fninfo, ARG_SGPR, ctx->ac.i32);
4093 }
4094 }
4095
4096 static unsigned llvm_get_type_size(LLVMTypeRef type)
4097 {
4098 LLVMTypeKind kind = LLVMGetTypeKind(type);
4099
4100 switch (kind) {
4101 case LLVMIntegerTypeKind:
4102 return LLVMGetIntTypeWidth(type) / 8;
4103 case LLVMFloatTypeKind:
4104 return 4;
4105 case LLVMPointerTypeKind:
4106 return 8;
4107 case LLVMVectorTypeKind:
4108 return LLVMGetVectorSize(type) *
4109 llvm_get_type_size(LLVMGetElementType(type));
4110 case LLVMArrayTypeKind:
4111 return LLVMGetArrayLength(type) *
4112 llvm_get_type_size(LLVMGetElementType(type));
4113 default:
4114 assert(0);
4115 return 0;
4116 }
4117 }
4118
4119 static void declare_lds_as_pointer(struct si_shader_context *ctx)
4120 {
4121 struct gallivm_state *gallivm = &ctx->gallivm;
4122
4123 unsigned lds_size = ctx->screen->b.chip_class >= CIK ? 65536 : 32768;
4124 ctx->lds = LLVMBuildIntToPtr(gallivm->builder, ctx->i32_0,
4125 LLVMPointerType(LLVMArrayType(ctx->i32, lds_size / 4), LOCAL_ADDR_SPACE),
4126 "lds");
4127 }
4128
4129 static unsigned si_get_max_workgroup_size(const struct si_shader *shader)
4130 {
4131 switch (shader->selector->type) {
4132 case PIPE_SHADER_TESS_CTRL:
4133 /* Return this so that LLVM doesn't remove s_barrier
4134 * instructions on chips where we use s_barrier. */
4135 return shader->selector->screen->b.chip_class >= CIK ? 128 : 64;
4136
4137 case PIPE_SHADER_GEOMETRY:
4138 return shader->selector->screen->b.chip_class >= GFX9 ? 128 : 64;
4139
4140 case PIPE_SHADER_COMPUTE:
4141 break; /* see below */
4142
4143 default:
4144 return 0;
4145 }
4146
4147 const unsigned *properties = shader->selector->info.properties;
4148 unsigned max_work_group_size =
4149 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH] *
4150 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT] *
4151 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH];
4152
4153 if (!max_work_group_size) {
4154 /* This is a variable group size compute shader,
4155 * compile it for the maximum possible group size.
4156 */
4157 max_work_group_size = SI_MAX_VARIABLE_THREADS_PER_BLOCK;
4158 }
4159 return max_work_group_size;
4160 }
4161
4162 static void declare_per_stage_desc_pointers(struct si_shader_context *ctx,
4163 struct si_function_info *fninfo,
4164 bool assign_params)
4165 {
4166 unsigned const_and_shader_buffers =
4167 add_arg(fninfo, ARG_SGPR,
4168 si_const_array(ctx->v4i32,
4169 SI_NUM_SHADER_BUFFERS + SI_NUM_CONST_BUFFERS));
4170 unsigned samplers_and_images =
4171 add_arg(fninfo, ARG_SGPR,
4172 si_const_array(ctx->v8i32,
4173 SI_NUM_IMAGES + SI_NUM_SAMPLERS * 2));
4174
4175 if (assign_params) {
4176 ctx->param_const_and_shader_buffers = const_and_shader_buffers;
4177 ctx->param_samplers_and_images = samplers_and_images;
4178 }
4179 }
4180
4181 static void declare_default_desc_pointers(struct si_shader_context *ctx,
4182 struct si_function_info *fninfo)
4183 {
4184 ctx->param_rw_buffers = add_arg(fninfo, ARG_SGPR,
4185 si_const_array(ctx->v4i32, SI_NUM_RW_BUFFERS));
4186 declare_per_stage_desc_pointers(ctx, fninfo, true);
4187 }
4188
4189 static void declare_vs_specific_input_sgprs(struct si_shader_context *ctx,
4190 struct si_function_info *fninfo)
4191 {
4192 ctx->param_vertex_buffers = add_arg(fninfo, ARG_SGPR,
4193 si_const_array(ctx->v4i32, SI_NUM_VERTEX_BUFFERS));
4194 add_arg_assign(fninfo, ARG_SGPR, ctx->i32, &ctx->abi.base_vertex);
4195 add_arg_assign(fninfo, ARG_SGPR, ctx->i32, &ctx->abi.start_instance);
4196 add_arg_assign(fninfo, ARG_SGPR, ctx->i32, &ctx->abi.draw_id);
4197 ctx->param_vs_state_bits = add_arg(fninfo, ARG_SGPR, ctx->i32);
4198 }
4199
4200 static void declare_vs_input_vgprs(struct si_shader_context *ctx,
4201 struct si_function_info *fninfo,
4202 unsigned *num_prolog_vgprs)
4203 {
4204 struct si_shader *shader = ctx->shader;
4205
4206 add_arg_assign(fninfo, ARG_VGPR, ctx->i32, &ctx->abi.vertex_id);
4207 if (shader->key.as_ls) {
4208 ctx->param_rel_auto_id = add_arg(fninfo, ARG_VGPR, ctx->i32);
4209 add_arg_assign(fninfo, ARG_VGPR, ctx->i32, &ctx->abi.instance_id);
4210 } else {
4211 add_arg_assign(fninfo, ARG_VGPR, ctx->i32, &ctx->abi.instance_id);
4212 ctx->param_vs_prim_id = add_arg(fninfo, ARG_VGPR, ctx->i32);
4213 }
4214 add_arg(fninfo, ARG_VGPR, ctx->i32); /* unused */
4215
4216 if (!shader->is_gs_copy_shader) {
4217 /* Vertex load indices. */
4218 ctx->param_vertex_index0 = fninfo->num_params;
4219 for (unsigned i = 0; i < shader->selector->info.num_inputs; i++)
4220 add_arg(fninfo, ARG_VGPR, ctx->i32);
4221 *num_prolog_vgprs += shader->selector->info.num_inputs;
4222 }
4223 }
4224
4225 static void declare_tes_input_vgprs(struct si_shader_context *ctx,
4226 struct si_function_info *fninfo)
4227 {
4228 ctx->param_tes_u = add_arg(fninfo, ARG_VGPR, ctx->f32);
4229 ctx->param_tes_v = add_arg(fninfo, ARG_VGPR, ctx->f32);
4230 ctx->param_tes_rel_patch_id = add_arg(fninfo, ARG_VGPR, ctx->i32);
4231 ctx->param_tes_patch_id = add_arg(fninfo, ARG_VGPR, ctx->i32);
4232 }
4233
4234 enum {
4235 /* Convenient merged shader definitions. */
4236 SI_SHADER_MERGED_VERTEX_TESSCTRL = PIPE_SHADER_TYPES,
4237 SI_SHADER_MERGED_VERTEX_OR_TESSEVAL_GEOMETRY,
4238 };
4239
4240 static void create_function(struct si_shader_context *ctx)
4241 {
4242 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
4243 struct gallivm_state *gallivm = &ctx->gallivm;
4244 struct si_shader *shader = ctx->shader;
4245 struct si_function_info fninfo;
4246 LLVMTypeRef returns[16+32*4];
4247 unsigned i, num_return_sgprs;
4248 unsigned num_returns = 0;
4249 unsigned num_prolog_vgprs = 0;
4250 unsigned type = ctx->type;
4251
4252 si_init_function_info(&fninfo);
4253
4254 /* Set MERGED shaders. */
4255 if (ctx->screen->b.chip_class >= GFX9) {
4256 if (shader->key.as_ls || type == PIPE_SHADER_TESS_CTRL)
4257 type = SI_SHADER_MERGED_VERTEX_TESSCTRL; /* LS or HS */
4258 else if (shader->key.as_es || type == PIPE_SHADER_GEOMETRY)
4259 type = SI_SHADER_MERGED_VERTEX_OR_TESSEVAL_GEOMETRY;
4260 }
4261
4262 LLVMTypeRef v3i32 = LLVMVectorType(ctx->i32, 3);
4263
4264 switch (type) {
4265 case PIPE_SHADER_VERTEX:
4266 declare_default_desc_pointers(ctx, &fninfo);
4267 declare_vs_specific_input_sgprs(ctx, &fninfo);
4268
4269 if (shader->key.as_es) {
4270 ctx->param_es2gs_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4271 } else if (shader->key.as_ls) {
4272 /* no extra parameters */
4273 } else {
4274 if (shader->is_gs_copy_shader) {
4275 fninfo.num_params = ctx->param_rw_buffers + 1;
4276 fninfo.num_sgpr_params = fninfo.num_params;
4277 }
4278
4279 /* The locations of the other parameters are assigned dynamically. */
4280 declare_streamout_params(ctx, &shader->selector->so,
4281 &fninfo);
4282 }
4283
4284 /* VGPRs */
4285 declare_vs_input_vgprs(ctx, &fninfo, &num_prolog_vgprs);
4286 break;
4287
4288 case PIPE_SHADER_TESS_CTRL: /* SI-CI-VI */
4289 declare_default_desc_pointers(ctx, &fninfo);
4290 ctx->param_tcs_offchip_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4291 ctx->param_tcs_out_lds_offsets = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4292 ctx->param_tcs_out_lds_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4293 ctx->param_vs_state_bits = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4294 ctx->param_tcs_offchip_addr_base64k = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4295 ctx->param_tcs_factor_addr_base64k = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4296 ctx->param_tcs_offchip_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4297 ctx->param_tcs_factor_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4298
4299 /* VGPRs */
4300 ctx->param_tcs_patch_id = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4301 ctx->param_tcs_rel_ids = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4302
4303 /* param_tcs_offchip_offset and param_tcs_factor_offset are
4304 * placed after the user SGPRs.
4305 */
4306 for (i = 0; i < GFX6_TCS_NUM_USER_SGPR + 2; i++)
4307 returns[num_returns++] = ctx->i32; /* SGPRs */
4308 for (i = 0; i < 5; i++)
4309 returns[num_returns++] = ctx->f32; /* VGPRs */
4310 break;
4311
4312 case SI_SHADER_MERGED_VERTEX_TESSCTRL:
4313 /* Merged stages have 8 system SGPRs at the beginning. */
4314 ctx->param_rw_buffers = /* SPI_SHADER_USER_DATA_ADDR_LO_HS */
4315 add_arg(&fninfo, ARG_SGPR, si_const_array(ctx->v4i32, SI_NUM_RW_BUFFERS));
4316 ctx->param_tcs_offchip_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4317 ctx->param_merged_wave_info = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4318 ctx->param_tcs_factor_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4319 ctx->param_merged_scratch_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4320 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4321 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4322
4323 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4324 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4325 declare_per_stage_desc_pointers(ctx, &fninfo,
4326 ctx->type == PIPE_SHADER_VERTEX);
4327 declare_vs_specific_input_sgprs(ctx, &fninfo);
4328
4329 ctx->param_tcs_offchip_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4330 ctx->param_tcs_out_lds_offsets = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4331 ctx->param_tcs_out_lds_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4332 ctx->param_tcs_offchip_addr_base64k = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4333 ctx->param_tcs_factor_addr_base64k = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4334 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4335
4336 declare_per_stage_desc_pointers(ctx, &fninfo,
4337 ctx->type == PIPE_SHADER_TESS_CTRL);
4338
4339 /* VGPRs (first TCS, then VS) */
4340 ctx->param_tcs_patch_id = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4341 ctx->param_tcs_rel_ids = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4342
4343 if (ctx->type == PIPE_SHADER_VERTEX) {
4344 declare_vs_input_vgprs(ctx, &fninfo,
4345 &num_prolog_vgprs);
4346
4347 /* LS return values are inputs to the TCS main shader part. */
4348 for (i = 0; i < 8 + GFX9_TCS_NUM_USER_SGPR; i++)
4349 returns[num_returns++] = ctx->i32; /* SGPRs */
4350 for (i = 0; i < 2; i++)
4351 returns[num_returns++] = ctx->f32; /* VGPRs */
4352 } else {
4353 /* TCS return values are inputs to the TCS epilog.
4354 *
4355 * param_tcs_offchip_offset, param_tcs_factor_offset,
4356 * param_tcs_offchip_layout, and param_rw_buffers
4357 * should be passed to the epilog.
4358 */
4359 for (i = 0; i <= 8 + GFX9_SGPR_TCS_FACTOR_ADDR_BASE64K; i++)
4360 returns[num_returns++] = ctx->i32; /* SGPRs */
4361 for (i = 0; i < 5; i++)
4362 returns[num_returns++] = ctx->f32; /* VGPRs */
4363 }
4364 break;
4365
4366 case SI_SHADER_MERGED_VERTEX_OR_TESSEVAL_GEOMETRY:
4367 /* Merged stages have 8 system SGPRs at the beginning. */
4368 ctx->param_rw_buffers = /* SPI_SHADER_USER_DATA_ADDR_LO_GS */
4369 add_arg(&fninfo, ARG_SGPR, si_const_array(ctx->v4i32, SI_NUM_RW_BUFFERS));
4370 ctx->param_gs2vs_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4371 ctx->param_merged_wave_info = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4372 ctx->param_tcs_offchip_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4373 ctx->param_merged_scratch_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4374 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused (SPI_SHADER_PGM_LO/HI_GS << 8) */
4375 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused (SPI_SHADER_PGM_LO/HI_GS >> 24) */
4376
4377 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4378 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4379 declare_per_stage_desc_pointers(ctx, &fninfo,
4380 (ctx->type == PIPE_SHADER_VERTEX ||
4381 ctx->type == PIPE_SHADER_TESS_EVAL));
4382 if (ctx->type == PIPE_SHADER_VERTEX) {
4383 declare_vs_specific_input_sgprs(ctx, &fninfo);
4384 } else {
4385 /* TESS_EVAL (and also GEOMETRY):
4386 * Declare as many input SGPRs as the VS has. */
4387 ctx->param_tcs_offchip_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4388 ctx->param_tcs_offchip_addr_base64k = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4389 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4390 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4391 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4392 ctx->param_vs_state_bits = add_arg(&fninfo, ARG_SGPR, ctx->i32); /* unused */
4393 }
4394
4395 declare_per_stage_desc_pointers(ctx, &fninfo,
4396 ctx->type == PIPE_SHADER_GEOMETRY);
4397
4398 /* VGPRs (first GS, then VS/TES) */
4399 ctx->param_gs_vtx01_offset = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4400 ctx->param_gs_vtx23_offset = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4401 ctx->param_gs_prim_id = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4402 ctx->param_gs_instance_id = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4403 ctx->param_gs_vtx45_offset = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4404
4405 if (ctx->type == PIPE_SHADER_VERTEX) {
4406 declare_vs_input_vgprs(ctx, &fninfo,
4407 &num_prolog_vgprs);
4408 } else if (ctx->type == PIPE_SHADER_TESS_EVAL) {
4409 declare_tes_input_vgprs(ctx, &fninfo);
4410 }
4411
4412 if (ctx->type == PIPE_SHADER_VERTEX ||
4413 ctx->type == PIPE_SHADER_TESS_EVAL) {
4414 /* ES return values are inputs to GS. */
4415 for (i = 0; i < 8 + GFX9_GS_NUM_USER_SGPR; i++)
4416 returns[num_returns++] = ctx->i32; /* SGPRs */
4417 for (i = 0; i < 5; i++)
4418 returns[num_returns++] = ctx->f32; /* VGPRs */
4419 }
4420 break;
4421
4422 case PIPE_SHADER_TESS_EVAL:
4423 declare_default_desc_pointers(ctx, &fninfo);
4424 ctx->param_tcs_offchip_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4425 ctx->param_tcs_offchip_addr_base64k = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4426
4427 if (shader->key.as_es) {
4428 ctx->param_tcs_offchip_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4429 add_arg(&fninfo, ARG_SGPR, ctx->i32);
4430 ctx->param_es2gs_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4431 } else {
4432 add_arg(&fninfo, ARG_SGPR, ctx->i32);
4433 declare_streamout_params(ctx, &shader->selector->so,
4434 &fninfo);
4435 ctx->param_tcs_offchip_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4436 }
4437
4438 /* VGPRs */
4439 declare_tes_input_vgprs(ctx, &fninfo);
4440 break;
4441
4442 case PIPE_SHADER_GEOMETRY:
4443 declare_default_desc_pointers(ctx, &fninfo);
4444 ctx->param_gs2vs_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4445 ctx->param_gs_wave_id = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4446
4447 /* VGPRs */
4448 ctx->param_gs_vtx0_offset = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4449 ctx->param_gs_vtx1_offset = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4450 ctx->param_gs_prim_id = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4451 ctx->param_gs_vtx2_offset = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4452 ctx->param_gs_vtx3_offset = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4453 ctx->param_gs_vtx4_offset = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4454 ctx->param_gs_vtx5_offset = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4455 ctx->param_gs_instance_id = add_arg(&fninfo, ARG_VGPR, ctx->i32);
4456 break;
4457
4458 case PIPE_SHADER_FRAGMENT:
4459 declare_default_desc_pointers(ctx, &fninfo);
4460 add_arg_checked(&fninfo, ARG_SGPR, ctx->f32, SI_PARAM_ALPHA_REF);
4461 add_arg_checked(&fninfo, ARG_SGPR, ctx->i32, SI_PARAM_PRIM_MASK);
4462
4463 add_arg_checked(&fninfo, ARG_VGPR, ctx->v2i32, SI_PARAM_PERSP_SAMPLE);
4464 add_arg_checked(&fninfo, ARG_VGPR, ctx->v2i32, SI_PARAM_PERSP_CENTER);
4465 add_arg_checked(&fninfo, ARG_VGPR, ctx->v2i32, SI_PARAM_PERSP_CENTROID);
4466 add_arg_checked(&fninfo, ARG_VGPR, v3i32, SI_PARAM_PERSP_PULL_MODEL);
4467 add_arg_checked(&fninfo, ARG_VGPR, ctx->v2i32, SI_PARAM_LINEAR_SAMPLE);
4468 add_arg_checked(&fninfo, ARG_VGPR, ctx->v2i32, SI_PARAM_LINEAR_CENTER);
4469 add_arg_checked(&fninfo, ARG_VGPR, ctx->v2i32, SI_PARAM_LINEAR_CENTROID);
4470 add_arg_checked(&fninfo, ARG_VGPR, ctx->f32, SI_PARAM_LINE_STIPPLE_TEX);
4471 add_arg_checked(&fninfo, ARG_VGPR, ctx->f32, SI_PARAM_POS_X_FLOAT);
4472 add_arg_checked(&fninfo, ARG_VGPR, ctx->f32, SI_PARAM_POS_Y_FLOAT);
4473 add_arg_checked(&fninfo, ARG_VGPR, ctx->f32, SI_PARAM_POS_Z_FLOAT);
4474 add_arg_checked(&fninfo, ARG_VGPR, ctx->f32, SI_PARAM_POS_W_FLOAT);
4475 add_arg_checked(&fninfo, ARG_VGPR, ctx->i32, SI_PARAM_FRONT_FACE);
4476 shader->info.face_vgpr_index = 20;
4477 add_arg_checked(&fninfo, ARG_VGPR, ctx->i32, SI_PARAM_ANCILLARY);
4478 add_arg_checked(&fninfo, ARG_VGPR, ctx->f32, SI_PARAM_SAMPLE_COVERAGE);
4479 add_arg_checked(&fninfo, ARG_VGPR, ctx->i32, SI_PARAM_POS_FIXED_PT);
4480
4481 /* Color inputs from the prolog. */
4482 if (shader->selector->info.colors_read) {
4483 unsigned num_color_elements =
4484 util_bitcount(shader->selector->info.colors_read);
4485
4486 assert(fninfo.num_params + num_color_elements <= ARRAY_SIZE(fninfo.types));
4487 for (i = 0; i < num_color_elements; i++)
4488 add_arg(&fninfo, ARG_VGPR, ctx->f32);
4489
4490 num_prolog_vgprs += num_color_elements;
4491 }
4492
4493 /* Outputs for the epilog. */
4494 num_return_sgprs = SI_SGPR_ALPHA_REF + 1;
4495 num_returns =
4496 num_return_sgprs +
4497 util_bitcount(shader->selector->info.colors_written) * 4 +
4498 shader->selector->info.writes_z +
4499 shader->selector->info.writes_stencil +
4500 shader->selector->info.writes_samplemask +
4501 1 /* SampleMaskIn */;
4502
4503 num_returns = MAX2(num_returns,
4504 num_return_sgprs +
4505 PS_EPILOG_SAMPLEMASK_MIN_LOC + 1);
4506
4507 for (i = 0; i < num_return_sgprs; i++)
4508 returns[i] = ctx->i32;
4509 for (; i < num_returns; i++)
4510 returns[i] = ctx->f32;
4511 break;
4512
4513 case PIPE_SHADER_COMPUTE:
4514 declare_default_desc_pointers(ctx, &fninfo);
4515 if (shader->selector->info.uses_grid_size)
4516 ctx->param_grid_size = add_arg(&fninfo, ARG_SGPR, v3i32);
4517 if (shader->selector->info.uses_block_size)
4518 ctx->param_block_size = add_arg(&fninfo, ARG_SGPR, v3i32);
4519
4520 for (i = 0; i < 3; i++) {
4521 ctx->param_block_id[i] = -1;
4522 if (shader->selector->info.uses_block_id[i])
4523 ctx->param_block_id[i] = add_arg(&fninfo, ARG_SGPR, ctx->i32);
4524 }
4525
4526 ctx->param_thread_id = add_arg(&fninfo, ARG_VGPR, v3i32);
4527 break;
4528 default:
4529 assert(0 && "unimplemented shader");
4530 return;
4531 }
4532
4533 si_create_function(ctx, "main", returns, num_returns, &fninfo,
4534 si_get_max_workgroup_size(shader));
4535
4536 /* Reserve register locations for VGPR inputs the PS prolog may need. */
4537 if (ctx->type == PIPE_SHADER_FRAGMENT &&
4538 ctx->separate_prolog) {
4539 si_llvm_add_attribute(ctx->main_fn,
4540 "InitialPSInputAddr",
4541 S_0286D0_PERSP_SAMPLE_ENA(1) |
4542 S_0286D0_PERSP_CENTER_ENA(1) |
4543 S_0286D0_PERSP_CENTROID_ENA(1) |
4544 S_0286D0_LINEAR_SAMPLE_ENA(1) |
4545 S_0286D0_LINEAR_CENTER_ENA(1) |
4546 S_0286D0_LINEAR_CENTROID_ENA(1) |
4547 S_0286D0_FRONT_FACE_ENA(1) |
4548 S_0286D0_POS_FIXED_PT_ENA(1));
4549 }
4550
4551 shader->info.num_input_sgprs = 0;
4552 shader->info.num_input_vgprs = 0;
4553
4554 for (i = 0; i < fninfo.num_sgpr_params; ++i)
4555 shader->info.num_input_sgprs += llvm_get_type_size(fninfo.types[i]) / 4;
4556
4557 for (; i < fninfo.num_params; ++i)
4558 shader->info.num_input_vgprs += llvm_get_type_size(fninfo.types[i]) / 4;
4559
4560 assert(shader->info.num_input_vgprs >= num_prolog_vgprs);
4561 shader->info.num_input_vgprs -= num_prolog_vgprs;
4562
4563 if (!ctx->screen->has_ds_bpermute &&
4564 bld_base->info &&
4565 (bld_base->info->opcode_count[TGSI_OPCODE_DDX] > 0 ||
4566 bld_base->info->opcode_count[TGSI_OPCODE_DDY] > 0 ||
4567 bld_base->info->opcode_count[TGSI_OPCODE_DDX_FINE] > 0 ||
4568 bld_base->info->opcode_count[TGSI_OPCODE_DDY_FINE] > 0 ||
4569 bld_base->info->opcode_count[TGSI_OPCODE_INTERP_OFFSET] > 0 ||
4570 bld_base->info->opcode_count[TGSI_OPCODE_INTERP_SAMPLE] > 0))
4571 ctx->lds =
4572 LLVMAddGlobalInAddressSpace(gallivm->module,
4573 LLVMArrayType(ctx->i32, 64),
4574 "ddxy_lds",
4575 LOCAL_ADDR_SPACE);
4576
4577 if (shader->key.as_ls ||
4578 ctx->type == PIPE_SHADER_TESS_CTRL ||
4579 /* GFX9 has the ESGS ring buffer in LDS. */
4580 (ctx->screen->b.chip_class >= GFX9 &&
4581 (shader->key.as_es ||
4582 ctx->type == PIPE_SHADER_GEOMETRY)))
4583 declare_lds_as_pointer(ctx);
4584 }
4585
4586 /**
4587 * Load ESGS and GSVS ring buffer resource descriptors and save the variables
4588 * for later use.
4589 */
4590 static void preload_ring_buffers(struct si_shader_context *ctx)
4591 {
4592 struct gallivm_state *gallivm = &ctx->gallivm;
4593 LLVMBuilderRef builder = gallivm->builder;
4594
4595 LLVMValueRef buf_ptr = LLVMGetParam(ctx->main_fn,
4596 ctx->param_rw_buffers);
4597
4598 if (ctx->screen->b.chip_class <= VI &&
4599 (ctx->shader->key.as_es || ctx->type == PIPE_SHADER_GEOMETRY)) {
4600 unsigned ring =
4601 ctx->type == PIPE_SHADER_GEOMETRY ? SI_GS_RING_ESGS
4602 : SI_ES_RING_ESGS;
4603 LLVMValueRef offset = LLVMConstInt(ctx->i32, ring, 0);
4604
4605 ctx->esgs_ring =
4606 ac_build_indexed_load_const(&ctx->ac, buf_ptr, offset);
4607 }
4608
4609 if (ctx->shader->is_gs_copy_shader) {
4610 LLVMValueRef offset = LLVMConstInt(ctx->i32, SI_RING_GSVS, 0);
4611
4612 ctx->gsvs_ring[0] =
4613 ac_build_indexed_load_const(&ctx->ac, buf_ptr, offset);
4614 } else if (ctx->type == PIPE_SHADER_GEOMETRY) {
4615 const struct si_shader_selector *sel = ctx->shader->selector;
4616 LLVMValueRef offset = LLVMConstInt(ctx->i32, SI_RING_GSVS, 0);
4617 LLVMValueRef base_ring;
4618
4619 base_ring = ac_build_indexed_load_const(&ctx->ac, buf_ptr, offset);
4620
4621 /* The conceptual layout of the GSVS ring is
4622 * v0c0 .. vLv0 v0c1 .. vLc1 ..
4623 * but the real memory layout is swizzled across
4624 * threads:
4625 * t0v0c0 .. t15v0c0 t0v1c0 .. t15v1c0 ... t15vLcL
4626 * t16v0c0 ..
4627 * Override the buffer descriptor accordingly.
4628 */
4629 LLVMTypeRef v2i64 = LLVMVectorType(ctx->i64, 2);
4630 uint64_t stream_offset = 0;
4631
4632 for (unsigned stream = 0; stream < 4; ++stream) {
4633 unsigned num_components;
4634 unsigned stride;
4635 unsigned num_records;
4636 LLVMValueRef ring, tmp;
4637
4638 num_components = sel->info.num_stream_output_components[stream];
4639 if (!num_components)
4640 continue;
4641
4642 stride = 4 * num_components * sel->gs_max_out_vertices;
4643
4644 /* Limit on the stride field for <= CIK. */
4645 assert(stride < (1 << 14));
4646
4647 num_records = 64;
4648
4649 ring = LLVMBuildBitCast(builder, base_ring, v2i64, "");
4650 tmp = LLVMBuildExtractElement(builder, ring, ctx->i32_0, "");
4651 tmp = LLVMBuildAdd(builder, tmp,
4652 LLVMConstInt(ctx->i64,
4653 stream_offset, 0), "");
4654 stream_offset += stride * 64;
4655
4656 ring = LLVMBuildInsertElement(builder, ring, tmp, ctx->i32_0, "");
4657 ring = LLVMBuildBitCast(builder, ring, ctx->v4i32, "");
4658 tmp = LLVMBuildExtractElement(builder, ring, ctx->i32_1, "");
4659 tmp = LLVMBuildOr(builder, tmp,
4660 LLVMConstInt(ctx->i32,
4661 S_008F04_STRIDE(stride) |
4662 S_008F04_SWIZZLE_ENABLE(1), 0), "");
4663 ring = LLVMBuildInsertElement(builder, ring, tmp, ctx->i32_1, "");
4664 ring = LLVMBuildInsertElement(builder, ring,
4665 LLVMConstInt(ctx->i32, num_records, 0),
4666 LLVMConstInt(ctx->i32, 2, 0), "");
4667 ring = LLVMBuildInsertElement(builder, ring,
4668 LLVMConstInt(ctx->i32,
4669 S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
4670 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
4671 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
4672 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
4673 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
4674 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) |
4675 S_008F0C_ELEMENT_SIZE(1) | /* element_size = 4 (bytes) */
4676 S_008F0C_INDEX_STRIDE(1) | /* index_stride = 16 (elements) */
4677 S_008F0C_ADD_TID_ENABLE(1),
4678 0),
4679 LLVMConstInt(ctx->i32, 3, 0), "");
4680
4681 ctx->gsvs_ring[stream] = ring;
4682 }
4683 }
4684 }
4685
4686 static void si_llvm_emit_polygon_stipple(struct si_shader_context *ctx,
4687 LLVMValueRef param_rw_buffers,
4688 unsigned param_pos_fixed_pt)
4689 {
4690 struct gallivm_state *gallivm = &ctx->gallivm;
4691 LLVMBuilderRef builder = gallivm->builder;
4692 LLVMValueRef slot, desc, offset, row, bit, address[2];
4693
4694 /* Use the fixed-point gl_FragCoord input.
4695 * Since the stipple pattern is 32x32 and it repeats, just get 5 bits
4696 * per coordinate to get the repeating effect.
4697 */
4698 address[0] = unpack_param(ctx, param_pos_fixed_pt, 0, 5);
4699 address[1] = unpack_param(ctx, param_pos_fixed_pt, 16, 5);
4700
4701 /* Load the buffer descriptor. */
4702 slot = LLVMConstInt(ctx->i32, SI_PS_CONST_POLY_STIPPLE, 0);
4703 desc = ac_build_indexed_load_const(&ctx->ac, param_rw_buffers, slot);
4704
4705 /* The stipple pattern is 32x32, each row has 32 bits. */
4706 offset = LLVMBuildMul(builder, address[1],
4707 LLVMConstInt(ctx->i32, 4, 0), "");
4708 row = buffer_load_const(ctx, desc, offset);
4709 row = LLVMBuildBitCast(builder, row, ctx->i32, "");
4710 bit = LLVMBuildLShr(builder, row, address[0], "");
4711 bit = LLVMBuildTrunc(builder, bit, ctx->i1, "");
4712
4713 /* The intrinsic kills the thread if arg < 0. */
4714 bit = LLVMBuildSelect(builder, bit, LLVMConstReal(ctx->f32, 0),
4715 LLVMConstReal(ctx->f32, -1), "");
4716 ac_build_kill(&ctx->ac, bit);
4717 }
4718
4719 void si_shader_binary_read_config(struct ac_shader_binary *binary,
4720 struct si_shader_config *conf,
4721 unsigned symbol_offset)
4722 {
4723 unsigned i;
4724 const unsigned char *config =
4725 ac_shader_binary_config_start(binary, symbol_offset);
4726 bool really_needs_scratch = false;
4727
4728 /* LLVM adds SGPR spills to the scratch size.
4729 * Find out if we really need the scratch buffer.
4730 */
4731 for (i = 0; i < binary->reloc_count; i++) {
4732 const struct ac_shader_reloc *reloc = &binary->relocs[i];
4733
4734 if (!strcmp(scratch_rsrc_dword0_symbol, reloc->name) ||
4735 !strcmp(scratch_rsrc_dword1_symbol, reloc->name)) {
4736 really_needs_scratch = true;
4737 break;
4738 }
4739 }
4740
4741 /* XXX: We may be able to emit some of these values directly rather than
4742 * extracting fields to be emitted later.
4743 */
4744
4745 for (i = 0; i < binary->config_size_per_symbol; i+= 8) {
4746 unsigned reg = util_le32_to_cpu(*(uint32_t*)(config + i));
4747 unsigned value = util_le32_to_cpu(*(uint32_t*)(config + i + 4));
4748 switch (reg) {
4749 case R_00B028_SPI_SHADER_PGM_RSRC1_PS:
4750 case R_00B128_SPI_SHADER_PGM_RSRC1_VS:
4751 case R_00B228_SPI_SHADER_PGM_RSRC1_GS:
4752 case R_00B428_SPI_SHADER_PGM_RSRC1_HS:
4753 case R_00B848_COMPUTE_PGM_RSRC1:
4754 conf->num_sgprs = MAX2(conf->num_sgprs, (G_00B028_SGPRS(value) + 1) * 8);
4755 conf->num_vgprs = MAX2(conf->num_vgprs, (G_00B028_VGPRS(value) + 1) * 4);
4756 conf->float_mode = G_00B028_FLOAT_MODE(value);
4757 conf->rsrc1 = value;
4758 break;
4759 case R_00B02C_SPI_SHADER_PGM_RSRC2_PS:
4760 conf->lds_size = MAX2(conf->lds_size, G_00B02C_EXTRA_LDS_SIZE(value));
4761 break;
4762 case R_00B84C_COMPUTE_PGM_RSRC2:
4763 conf->lds_size = MAX2(conf->lds_size, G_00B84C_LDS_SIZE(value));
4764 conf->rsrc2 = value;
4765 break;
4766 case R_0286CC_SPI_PS_INPUT_ENA:
4767 conf->spi_ps_input_ena = value;
4768 break;
4769 case R_0286D0_SPI_PS_INPUT_ADDR:
4770 conf->spi_ps_input_addr = value;
4771 break;
4772 case R_0286E8_SPI_TMPRING_SIZE:
4773 case R_00B860_COMPUTE_TMPRING_SIZE:
4774 /* WAVESIZE is in units of 256 dwords. */
4775 if (really_needs_scratch)
4776 conf->scratch_bytes_per_wave =
4777 G_00B860_WAVESIZE(value) * 256 * 4;
4778 break;
4779 case 0x4: /* SPILLED_SGPRS */
4780 conf->spilled_sgprs = value;
4781 break;
4782 case 0x8: /* SPILLED_VGPRS */
4783 conf->spilled_vgprs = value;
4784 break;
4785 default:
4786 {
4787 static bool printed;
4788
4789 if (!printed) {
4790 fprintf(stderr, "Warning: LLVM emitted unknown "
4791 "config register: 0x%x\n", reg);
4792 printed = true;
4793 }
4794 }
4795 break;
4796 }
4797 }
4798
4799 if (!conf->spi_ps_input_addr)
4800 conf->spi_ps_input_addr = conf->spi_ps_input_ena;
4801 }
4802
4803 void si_shader_apply_scratch_relocs(struct si_shader *shader,
4804 uint64_t scratch_va)
4805 {
4806 unsigned i;
4807 uint32_t scratch_rsrc_dword0 = scratch_va;
4808 uint32_t scratch_rsrc_dword1 =
4809 S_008F04_BASE_ADDRESS_HI(scratch_va >> 32);
4810
4811 /* Enable scratch coalescing. */
4812 scratch_rsrc_dword1 |= S_008F04_SWIZZLE_ENABLE(1);
4813
4814 for (i = 0 ; i < shader->binary.reloc_count; i++) {
4815 const struct ac_shader_reloc *reloc =
4816 &shader->binary.relocs[i];
4817 if (!strcmp(scratch_rsrc_dword0_symbol, reloc->name)) {
4818 util_memcpy_cpu_to_le32(shader->binary.code + reloc->offset,
4819 &scratch_rsrc_dword0, 4);
4820 } else if (!strcmp(scratch_rsrc_dword1_symbol, reloc->name)) {
4821 util_memcpy_cpu_to_le32(shader->binary.code + reloc->offset,
4822 &scratch_rsrc_dword1, 4);
4823 }
4824 }
4825 }
4826
4827 static unsigned si_get_shader_binary_size(const struct si_shader *shader)
4828 {
4829 unsigned size = shader->binary.code_size;
4830
4831 if (shader->prolog)
4832 size += shader->prolog->binary.code_size;
4833 if (shader->previous_stage)
4834 size += shader->previous_stage->binary.code_size;
4835 if (shader->prolog2)
4836 size += shader->prolog2->binary.code_size;
4837 if (shader->epilog)
4838 size += shader->epilog->binary.code_size;
4839 return size;
4840 }
4841
4842 int si_shader_binary_upload(struct si_screen *sscreen, struct si_shader *shader)
4843 {
4844 const struct ac_shader_binary *prolog =
4845 shader->prolog ? &shader->prolog->binary : NULL;
4846 const struct ac_shader_binary *previous_stage =
4847 shader->previous_stage ? &shader->previous_stage->binary : NULL;
4848 const struct ac_shader_binary *prolog2 =
4849 shader->prolog2 ? &shader->prolog2->binary : NULL;
4850 const struct ac_shader_binary *epilog =
4851 shader->epilog ? &shader->epilog->binary : NULL;
4852 const struct ac_shader_binary *mainb = &shader->binary;
4853 unsigned bo_size = si_get_shader_binary_size(shader) +
4854 (!epilog ? mainb->rodata_size : 0);
4855 unsigned char *ptr;
4856
4857 assert(!prolog || !prolog->rodata_size);
4858 assert(!previous_stage || !previous_stage->rodata_size);
4859 assert(!prolog2 || !prolog2->rodata_size);
4860 assert((!prolog && !previous_stage && !prolog2 && !epilog) ||
4861 !mainb->rodata_size);
4862 assert(!epilog || !epilog->rodata_size);
4863
4864 r600_resource_reference(&shader->bo, NULL);
4865 shader->bo = (struct r600_resource*)
4866 pipe_buffer_create(&sscreen->b.b, 0,
4867 PIPE_USAGE_IMMUTABLE,
4868 align(bo_size, SI_CPDMA_ALIGNMENT));
4869 if (!shader->bo)
4870 return -ENOMEM;
4871
4872 /* Upload. */
4873 ptr = sscreen->b.ws->buffer_map(shader->bo->buf, NULL,
4874 PIPE_TRANSFER_READ_WRITE |
4875 PIPE_TRANSFER_UNSYNCHRONIZED);
4876
4877 /* Don't use util_memcpy_cpu_to_le32. LLVM binaries are
4878 * endian-independent. */
4879 if (prolog) {
4880 memcpy(ptr, prolog->code, prolog->code_size);
4881 ptr += prolog->code_size;
4882 }
4883 if (previous_stage) {
4884 memcpy(ptr, previous_stage->code, previous_stage->code_size);
4885 ptr += previous_stage->code_size;
4886 }
4887 if (prolog2) {
4888 memcpy(ptr, prolog2->code, prolog2->code_size);
4889 ptr += prolog2->code_size;
4890 }
4891
4892 memcpy(ptr, mainb->code, mainb->code_size);
4893 ptr += mainb->code_size;
4894
4895 if (epilog)
4896 memcpy(ptr, epilog->code, epilog->code_size);
4897 else if (mainb->rodata_size > 0)
4898 memcpy(ptr, mainb->rodata, mainb->rodata_size);
4899
4900 sscreen->b.ws->buffer_unmap(shader->bo->buf);
4901 return 0;
4902 }
4903
4904 static void si_shader_dump_disassembly(const struct ac_shader_binary *binary,
4905 struct pipe_debug_callback *debug,
4906 const char *name, FILE *file)
4907 {
4908 char *line, *p;
4909 unsigned i, count;
4910
4911 if (binary->disasm_string) {
4912 fprintf(file, "Shader %s disassembly:\n", name);
4913 fprintf(file, "%s", binary->disasm_string);
4914
4915 if (debug && debug->debug_message) {
4916 /* Very long debug messages are cut off, so send the
4917 * disassembly one line at a time. This causes more
4918 * overhead, but on the plus side it simplifies
4919 * parsing of resulting logs.
4920 */
4921 pipe_debug_message(debug, SHADER_INFO,
4922 "Shader Disassembly Begin");
4923
4924 line = binary->disasm_string;
4925 while (*line) {
4926 p = util_strchrnul(line, '\n');
4927 count = p - line;
4928
4929 if (count) {
4930 pipe_debug_message(debug, SHADER_INFO,
4931 "%.*s", count, line);
4932 }
4933
4934 if (!*p)
4935 break;
4936 line = p + 1;
4937 }
4938
4939 pipe_debug_message(debug, SHADER_INFO,
4940 "Shader Disassembly End");
4941 }
4942 } else {
4943 fprintf(file, "Shader %s binary:\n", name);
4944 for (i = 0; i < binary->code_size; i += 4) {
4945 fprintf(file, "@0x%x: %02x%02x%02x%02x\n", i,
4946 binary->code[i + 3], binary->code[i + 2],
4947 binary->code[i + 1], binary->code[i]);
4948 }
4949 }
4950 }
4951
4952 static void si_shader_dump_stats(struct si_screen *sscreen,
4953 const struct si_shader *shader,
4954 struct pipe_debug_callback *debug,
4955 unsigned processor,
4956 FILE *file,
4957 bool check_debug_option)
4958 {
4959 const struct si_shader_config *conf = &shader->config;
4960 unsigned num_inputs = shader->selector ? shader->selector->info.num_inputs : 0;
4961 unsigned code_size = si_get_shader_binary_size(shader);
4962 unsigned lds_increment = sscreen->b.chip_class >= CIK ? 512 : 256;
4963 unsigned lds_per_wave = 0;
4964 unsigned max_simd_waves = 10;
4965
4966 /* Compute LDS usage for PS. */
4967 switch (processor) {
4968 case PIPE_SHADER_FRAGMENT:
4969 /* The minimum usage per wave is (num_inputs * 48). The maximum
4970 * usage is (num_inputs * 48 * 16).
4971 * We can get anything in between and it varies between waves.
4972 *
4973 * The 48 bytes per input for a single primitive is equal to
4974 * 4 bytes/component * 4 components/input * 3 points.
4975 *
4976 * Other stages don't know the size at compile time or don't
4977 * allocate LDS per wave, but instead they do it per thread group.
4978 */
4979 lds_per_wave = conf->lds_size * lds_increment +
4980 align(num_inputs * 48, lds_increment);
4981 break;
4982 case PIPE_SHADER_COMPUTE:
4983 if (shader->selector) {
4984 unsigned max_workgroup_size =
4985 si_get_max_workgroup_size(shader);
4986 lds_per_wave = (conf->lds_size * lds_increment) /
4987 DIV_ROUND_UP(max_workgroup_size, 64);
4988 }
4989 break;
4990 }
4991
4992 /* Compute the per-SIMD wave counts. */
4993 if (conf->num_sgprs) {
4994 if (sscreen->b.chip_class >= VI)
4995 max_simd_waves = MIN2(max_simd_waves, 800 / conf->num_sgprs);
4996 else
4997 max_simd_waves = MIN2(max_simd_waves, 512 / conf->num_sgprs);
4998 }
4999
5000 if (conf->num_vgprs)
5001 max_simd_waves = MIN2(max_simd_waves, 256 / conf->num_vgprs);
5002
5003 /* LDS is 64KB per CU (4 SIMDs), which is 16KB per SIMD (usage above
5004 * 16KB makes some SIMDs unoccupied). */
5005 if (lds_per_wave)
5006 max_simd_waves = MIN2(max_simd_waves, 16384 / lds_per_wave);
5007
5008 if (!check_debug_option ||
5009 r600_can_dump_shader(&sscreen->b, processor)) {
5010 if (processor == PIPE_SHADER_FRAGMENT) {
5011 fprintf(file, "*** SHADER CONFIG ***\n"
5012 "SPI_PS_INPUT_ADDR = 0x%04x\n"
5013 "SPI_PS_INPUT_ENA = 0x%04x\n",
5014 conf->spi_ps_input_addr, conf->spi_ps_input_ena);
5015 }
5016
5017 fprintf(file, "*** SHADER STATS ***\n"
5018 "SGPRS: %d\n"
5019 "VGPRS: %d\n"
5020 "Spilled SGPRs: %d\n"
5021 "Spilled VGPRs: %d\n"
5022 "Private memory VGPRs: %d\n"
5023 "Code Size: %d bytes\n"
5024 "LDS: %d blocks\n"
5025 "Scratch: %d bytes per wave\n"
5026 "Max Waves: %d\n"
5027 "********************\n\n\n",
5028 conf->num_sgprs, conf->num_vgprs,
5029 conf->spilled_sgprs, conf->spilled_vgprs,
5030 conf->private_mem_vgprs, code_size,
5031 conf->lds_size, conf->scratch_bytes_per_wave,
5032 max_simd_waves);
5033 }
5034
5035 pipe_debug_message(debug, SHADER_INFO,
5036 "Shader Stats: SGPRS: %d VGPRS: %d Code Size: %d "
5037 "LDS: %d Scratch: %d Max Waves: %d Spilled SGPRs: %d "
5038 "Spilled VGPRs: %d PrivMem VGPRs: %d",
5039 conf->num_sgprs, conf->num_vgprs, code_size,
5040 conf->lds_size, conf->scratch_bytes_per_wave,
5041 max_simd_waves, conf->spilled_sgprs,
5042 conf->spilled_vgprs, conf->private_mem_vgprs);
5043 }
5044
5045 const char *si_get_shader_name(const struct si_shader *shader, unsigned processor)
5046 {
5047 switch (processor) {
5048 case PIPE_SHADER_VERTEX:
5049 if (shader->key.as_es)
5050 return "Vertex Shader as ES";
5051 else if (shader->key.as_ls)
5052 return "Vertex Shader as LS";
5053 else
5054 return "Vertex Shader as VS";
5055 case PIPE_SHADER_TESS_CTRL:
5056 return "Tessellation Control Shader";
5057 case PIPE_SHADER_TESS_EVAL:
5058 if (shader->key.as_es)
5059 return "Tessellation Evaluation Shader as ES";
5060 else
5061 return "Tessellation Evaluation Shader as VS";
5062 case PIPE_SHADER_GEOMETRY:
5063 if (shader->is_gs_copy_shader)
5064 return "GS Copy Shader as VS";
5065 else
5066 return "Geometry Shader";
5067 case PIPE_SHADER_FRAGMENT:
5068 return "Pixel Shader";
5069 case PIPE_SHADER_COMPUTE:
5070 return "Compute Shader";
5071 default:
5072 return "Unknown Shader";
5073 }
5074 }
5075
5076 void si_shader_dump(struct si_screen *sscreen, const struct si_shader *shader,
5077 struct pipe_debug_callback *debug, unsigned processor,
5078 FILE *file, bool check_debug_option)
5079 {
5080 if (!check_debug_option ||
5081 r600_can_dump_shader(&sscreen->b, processor))
5082 si_dump_shader_key(processor, shader, file);
5083
5084 if (!check_debug_option && shader->binary.llvm_ir_string) {
5085 if (shader->previous_stage &&
5086 shader->previous_stage->binary.llvm_ir_string) {
5087 fprintf(file, "\n%s - previous stage - LLVM IR:\n\n",
5088 si_get_shader_name(shader, processor));
5089 fprintf(file, "%s\n", shader->previous_stage->binary.llvm_ir_string);
5090 }
5091
5092 fprintf(file, "\n%s - main shader part - LLVM IR:\n\n",
5093 si_get_shader_name(shader, processor));
5094 fprintf(file, "%s\n", shader->binary.llvm_ir_string);
5095 }
5096
5097 if (!check_debug_option ||
5098 (r600_can_dump_shader(&sscreen->b, processor) &&
5099 !(sscreen->b.debug_flags & DBG_NO_ASM))) {
5100 fprintf(file, "\n%s:\n", si_get_shader_name(shader, processor));
5101
5102 if (shader->prolog)
5103 si_shader_dump_disassembly(&shader->prolog->binary,
5104 debug, "prolog", file);
5105 if (shader->previous_stage)
5106 si_shader_dump_disassembly(&shader->previous_stage->binary,
5107 debug, "previous stage", file);
5108 if (shader->prolog2)
5109 si_shader_dump_disassembly(&shader->prolog2->binary,
5110 debug, "prolog2", file);
5111
5112 si_shader_dump_disassembly(&shader->binary, debug, "main", file);
5113
5114 if (shader->epilog)
5115 si_shader_dump_disassembly(&shader->epilog->binary,
5116 debug, "epilog", file);
5117 fprintf(file, "\n");
5118 }
5119
5120 si_shader_dump_stats(sscreen, shader, debug, processor, file,
5121 check_debug_option);
5122 }
5123
5124 static int si_compile_llvm(struct si_screen *sscreen,
5125 struct ac_shader_binary *binary,
5126 struct si_shader_config *conf,
5127 LLVMTargetMachineRef tm,
5128 LLVMModuleRef mod,
5129 struct pipe_debug_callback *debug,
5130 unsigned processor,
5131 const char *name)
5132 {
5133 int r = 0;
5134 unsigned count = p_atomic_inc_return(&sscreen->b.num_compilations);
5135
5136 if (r600_can_dump_shader(&sscreen->b, processor)) {
5137 fprintf(stderr, "radeonsi: Compiling shader %d\n", count);
5138
5139 if (!(sscreen->b.debug_flags & (DBG_NO_IR | DBG_PREOPT_IR))) {
5140 fprintf(stderr, "%s LLVM IR:\n\n", name);
5141 ac_dump_module(mod);
5142 fprintf(stderr, "\n");
5143 }
5144 }
5145
5146 if (sscreen->record_llvm_ir) {
5147 char *ir = LLVMPrintModuleToString(mod);
5148 binary->llvm_ir_string = strdup(ir);
5149 LLVMDisposeMessage(ir);
5150 }
5151
5152 if (!si_replace_shader(count, binary)) {
5153 r = si_llvm_compile(mod, binary, tm, debug);
5154 if (r)
5155 return r;
5156 }
5157
5158 si_shader_binary_read_config(binary, conf, 0);
5159
5160 /* Enable 64-bit and 16-bit denormals, because there is no performance
5161 * cost.
5162 *
5163 * If denormals are enabled, all floating-point output modifiers are
5164 * ignored.
5165 *
5166 * Don't enable denormals for 32-bit floats, because:
5167 * - Floating-point output modifiers would be ignored by the hw.
5168 * - Some opcodes don't support denormals, such as v_mad_f32. We would
5169 * have to stop using those.
5170 * - SI & CI would be very slow.
5171 */
5172 conf->float_mode |= V_00B028_FP_64_DENORMS;
5173
5174 FREE(binary->config);
5175 FREE(binary->global_symbol_offsets);
5176 binary->config = NULL;
5177 binary->global_symbol_offsets = NULL;
5178
5179 /* Some shaders can't have rodata because their binaries can be
5180 * concatenated.
5181 */
5182 if (binary->rodata_size &&
5183 (processor == PIPE_SHADER_VERTEX ||
5184 processor == PIPE_SHADER_TESS_CTRL ||
5185 processor == PIPE_SHADER_TESS_EVAL ||
5186 processor == PIPE_SHADER_FRAGMENT)) {
5187 fprintf(stderr, "radeonsi: The shader can't have rodata.");
5188 return -EINVAL;
5189 }
5190
5191 return r;
5192 }
5193
5194 static void si_llvm_build_ret(struct si_shader_context *ctx, LLVMValueRef ret)
5195 {
5196 if (LLVMGetTypeKind(LLVMTypeOf(ret)) == LLVMVoidTypeKind)
5197 LLVMBuildRetVoid(ctx->gallivm.builder);
5198 else
5199 LLVMBuildRet(ctx->gallivm.builder, ret);
5200 }
5201
5202 /* Generate code for the hardware VS shader stage to go with a geometry shader */
5203 struct si_shader *
5204 si_generate_gs_copy_shader(struct si_screen *sscreen,
5205 LLVMTargetMachineRef tm,
5206 struct si_shader_selector *gs_selector,
5207 struct pipe_debug_callback *debug)
5208 {
5209 struct si_shader_context ctx;
5210 struct si_shader *shader;
5211 struct gallivm_state *gallivm = &ctx.gallivm;
5212 LLVMBuilderRef builder;
5213 struct lp_build_tgsi_context *bld_base = &ctx.bld_base;
5214 struct lp_build_context *uint = &bld_base->uint_bld;
5215 struct si_shader_output_values *outputs;
5216 struct tgsi_shader_info *gsinfo = &gs_selector->info;
5217 int i, r;
5218
5219 outputs = MALLOC(gsinfo->num_outputs * sizeof(outputs[0]));
5220
5221 if (!outputs)
5222 return NULL;
5223
5224 shader = CALLOC_STRUCT(si_shader);
5225 if (!shader) {
5226 FREE(outputs);
5227 return NULL;
5228 }
5229
5230
5231 shader->selector = gs_selector;
5232 shader->is_gs_copy_shader = true;
5233
5234 si_init_shader_ctx(&ctx, sscreen, tm);
5235 ctx.shader = shader;
5236 ctx.type = PIPE_SHADER_VERTEX;
5237
5238 builder = gallivm->builder;
5239
5240 create_function(&ctx);
5241 preload_ring_buffers(&ctx);
5242
5243 LLVMValueRef voffset =
5244 lp_build_mul_imm(uint, ctx.abi.vertex_id, 4);
5245
5246 /* Fetch the vertex stream ID.*/
5247 LLVMValueRef stream_id;
5248
5249 if (gs_selector->so.num_outputs)
5250 stream_id = unpack_param(&ctx, ctx.param_streamout_config, 24, 2);
5251 else
5252 stream_id = ctx.i32_0;
5253
5254 /* Fill in output information. */
5255 for (i = 0; i < gsinfo->num_outputs; ++i) {
5256 outputs[i].semantic_name = gsinfo->output_semantic_name[i];
5257 outputs[i].semantic_index = gsinfo->output_semantic_index[i];
5258
5259 for (int chan = 0; chan < 4; chan++) {
5260 outputs[i].vertex_stream[chan] =
5261 (gsinfo->output_streams[i] >> (2 * chan)) & 3;
5262 }
5263 }
5264
5265 LLVMBasicBlockRef end_bb;
5266 LLVMValueRef switch_inst;
5267
5268 end_bb = LLVMAppendBasicBlockInContext(gallivm->context, ctx.main_fn, "end");
5269 switch_inst = LLVMBuildSwitch(builder, stream_id, end_bb, 4);
5270
5271 for (int stream = 0; stream < 4; stream++) {
5272 LLVMBasicBlockRef bb;
5273 unsigned offset;
5274
5275 if (!gsinfo->num_stream_output_components[stream])
5276 continue;
5277
5278 if (stream > 0 && !gs_selector->so.num_outputs)
5279 continue;
5280
5281 bb = LLVMInsertBasicBlockInContext(gallivm->context, end_bb, "out");
5282 LLVMAddCase(switch_inst, LLVMConstInt(ctx.i32, stream, 0), bb);
5283 LLVMPositionBuilderAtEnd(builder, bb);
5284
5285 /* Fetch vertex data from GSVS ring */
5286 offset = 0;
5287 for (i = 0; i < gsinfo->num_outputs; ++i) {
5288 for (unsigned chan = 0; chan < 4; chan++) {
5289 if (!(gsinfo->output_usagemask[i] & (1 << chan)) ||
5290 outputs[i].vertex_stream[chan] != stream) {
5291 outputs[i].values[chan] = ctx.bld_base.base.undef;
5292 continue;
5293 }
5294
5295 LLVMValueRef soffset = LLVMConstInt(ctx.i32,
5296 offset * gs_selector->gs_max_out_vertices * 16 * 4, 0);
5297 offset++;
5298
5299 outputs[i].values[chan] =
5300 ac_build_buffer_load(&ctx.ac,
5301 ctx.gsvs_ring[0], 1,
5302 ctx.i32_0, voffset,
5303 soffset, 0, 1, 1,
5304 true, false);
5305 }
5306 }
5307
5308 /* Streamout and exports. */
5309 if (gs_selector->so.num_outputs) {
5310 si_llvm_emit_streamout(&ctx, outputs,
5311 gsinfo->num_outputs,
5312 stream);
5313 }
5314
5315 if (stream == 0)
5316 si_llvm_export_vs(bld_base, outputs, gsinfo->num_outputs);
5317
5318 LLVMBuildBr(builder, end_bb);
5319 }
5320
5321 LLVMPositionBuilderAtEnd(builder, end_bb);
5322
5323 LLVMBuildRetVoid(gallivm->builder);
5324
5325 ctx.type = PIPE_SHADER_GEOMETRY; /* override for shader dumping */
5326 si_llvm_optimize_module(&ctx);
5327
5328 r = si_compile_llvm(sscreen, &ctx.shader->binary,
5329 &ctx.shader->config, ctx.tm,
5330 ctx.gallivm.module,
5331 debug, PIPE_SHADER_GEOMETRY,
5332 "GS Copy Shader");
5333 if (!r) {
5334 if (r600_can_dump_shader(&sscreen->b, PIPE_SHADER_GEOMETRY))
5335 fprintf(stderr, "GS Copy Shader:\n");
5336 si_shader_dump(sscreen, ctx.shader, debug,
5337 PIPE_SHADER_GEOMETRY, stderr, true);
5338 r = si_shader_binary_upload(sscreen, ctx.shader);
5339 }
5340
5341 si_llvm_dispose(&ctx);
5342
5343 FREE(outputs);
5344
5345 if (r != 0) {
5346 FREE(shader);
5347 shader = NULL;
5348 }
5349 return shader;
5350 }
5351
5352 static void si_dump_shader_key_vs(const struct si_shader_key *key,
5353 const struct si_vs_prolog_bits *prolog,
5354 const char *prefix, FILE *f)
5355 {
5356 fprintf(f, " %s.instance_divisor_is_one = %u\n",
5357 prefix, prolog->instance_divisor_is_one);
5358 fprintf(f, " %s.instance_divisor_is_fetched = %u\n",
5359 prefix, prolog->instance_divisor_is_fetched);
5360
5361 fprintf(f, " mono.vs.fix_fetch = {");
5362 for (int i = 0; i < SI_MAX_ATTRIBS; i++)
5363 fprintf(f, !i ? "%u" : ", %u", key->mono.vs_fix_fetch[i]);
5364 fprintf(f, "}\n");
5365 }
5366
5367 static void si_dump_shader_key(unsigned processor, const struct si_shader *shader,
5368 FILE *f)
5369 {
5370 const struct si_shader_key *key = &shader->key;
5371
5372 fprintf(f, "SHADER KEY\n");
5373
5374 switch (processor) {
5375 case PIPE_SHADER_VERTEX:
5376 si_dump_shader_key_vs(key, &key->part.vs.prolog,
5377 "part.vs.prolog", f);
5378 fprintf(f, " as_es = %u\n", key->as_es);
5379 fprintf(f, " as_ls = %u\n", key->as_ls);
5380 fprintf(f, " mono.u.vs_export_prim_id = %u\n",
5381 key->mono.u.vs_export_prim_id);
5382 break;
5383
5384 case PIPE_SHADER_TESS_CTRL:
5385 if (shader->selector->screen->b.chip_class >= GFX9) {
5386 si_dump_shader_key_vs(key, &key->part.tcs.ls_prolog,
5387 "part.tcs.ls_prolog", f);
5388 }
5389 fprintf(f, " part.tcs.epilog.prim_mode = %u\n", key->part.tcs.epilog.prim_mode);
5390 fprintf(f, " mono.u.ff_tcs_inputs_to_copy = 0x%"PRIx64"\n", key->mono.u.ff_tcs_inputs_to_copy);
5391 break;
5392
5393 case PIPE_SHADER_TESS_EVAL:
5394 fprintf(f, " as_es = %u\n", key->as_es);
5395 fprintf(f, " mono.u.vs_export_prim_id = %u\n",
5396 key->mono.u.vs_export_prim_id);
5397 break;
5398
5399 case PIPE_SHADER_GEOMETRY:
5400 if (shader->is_gs_copy_shader)
5401 break;
5402
5403 if (shader->selector->screen->b.chip_class >= GFX9 &&
5404 key->part.gs.es->type == PIPE_SHADER_VERTEX) {
5405 si_dump_shader_key_vs(key, &key->part.gs.vs_prolog,
5406 "part.gs.vs_prolog", f);
5407 }
5408 fprintf(f, " part.gs.prolog.tri_strip_adj_fix = %u\n", key->part.gs.prolog.tri_strip_adj_fix);
5409 break;
5410
5411 case PIPE_SHADER_COMPUTE:
5412 break;
5413
5414 case PIPE_SHADER_FRAGMENT:
5415 fprintf(f, " part.ps.prolog.color_two_side = %u\n", key->part.ps.prolog.color_two_side);
5416 fprintf(f, " part.ps.prolog.flatshade_colors = %u\n", key->part.ps.prolog.flatshade_colors);
5417 fprintf(f, " part.ps.prolog.poly_stipple = %u\n", key->part.ps.prolog.poly_stipple);
5418 fprintf(f, " part.ps.prolog.force_persp_sample_interp = %u\n", key->part.ps.prolog.force_persp_sample_interp);
5419 fprintf(f, " part.ps.prolog.force_linear_sample_interp = %u\n", key->part.ps.prolog.force_linear_sample_interp);
5420 fprintf(f, " part.ps.prolog.force_persp_center_interp = %u\n", key->part.ps.prolog.force_persp_center_interp);
5421 fprintf(f, " part.ps.prolog.force_linear_center_interp = %u\n", key->part.ps.prolog.force_linear_center_interp);
5422 fprintf(f, " part.ps.prolog.bc_optimize_for_persp = %u\n", key->part.ps.prolog.bc_optimize_for_persp);
5423 fprintf(f, " part.ps.prolog.bc_optimize_for_linear = %u\n", key->part.ps.prolog.bc_optimize_for_linear);
5424 fprintf(f, " part.ps.epilog.spi_shader_col_format = 0x%x\n", key->part.ps.epilog.spi_shader_col_format);
5425 fprintf(f, " part.ps.epilog.color_is_int8 = 0x%X\n", key->part.ps.epilog.color_is_int8);
5426 fprintf(f, " part.ps.epilog.color_is_int10 = 0x%X\n", key->part.ps.epilog.color_is_int10);
5427 fprintf(f, " part.ps.epilog.last_cbuf = %u\n", key->part.ps.epilog.last_cbuf);
5428 fprintf(f, " part.ps.epilog.alpha_func = %u\n", key->part.ps.epilog.alpha_func);
5429 fprintf(f, " part.ps.epilog.alpha_to_one = %u\n", key->part.ps.epilog.alpha_to_one);
5430 fprintf(f, " part.ps.epilog.poly_line_smoothing = %u\n", key->part.ps.epilog.poly_line_smoothing);
5431 fprintf(f, " part.ps.epilog.clamp_color = %u\n", key->part.ps.epilog.clamp_color);
5432 break;
5433
5434 default:
5435 assert(0);
5436 }
5437
5438 if ((processor == PIPE_SHADER_GEOMETRY ||
5439 processor == PIPE_SHADER_TESS_EVAL ||
5440 processor == PIPE_SHADER_VERTEX) &&
5441 !key->as_es && !key->as_ls) {
5442 fprintf(f, " opt.kill_outputs = 0x%"PRIx64"\n", key->opt.kill_outputs);
5443 fprintf(f, " opt.clip_disable = %u\n", key->opt.clip_disable);
5444 }
5445 }
5446
5447 static void si_init_shader_ctx(struct si_shader_context *ctx,
5448 struct si_screen *sscreen,
5449 LLVMTargetMachineRef tm)
5450 {
5451 struct lp_build_tgsi_context *bld_base;
5452
5453 si_llvm_context_init(ctx, sscreen, tm);
5454
5455 bld_base = &ctx->bld_base;
5456 bld_base->emit_fetch_funcs[TGSI_FILE_CONSTANT] = fetch_constant;
5457
5458 bld_base->op_actions[TGSI_OPCODE_INTERP_CENTROID] = interp_action;
5459 bld_base->op_actions[TGSI_OPCODE_INTERP_SAMPLE] = interp_action;
5460 bld_base->op_actions[TGSI_OPCODE_INTERP_OFFSET] = interp_action;
5461
5462 bld_base->op_actions[TGSI_OPCODE_MEMBAR].emit = membar_emit;
5463
5464 bld_base->op_actions[TGSI_OPCODE_CLOCK].emit = clock_emit;
5465
5466 bld_base->op_actions[TGSI_OPCODE_DDX].emit = si_llvm_emit_ddxy;
5467 bld_base->op_actions[TGSI_OPCODE_DDY].emit = si_llvm_emit_ddxy;
5468 bld_base->op_actions[TGSI_OPCODE_DDX_FINE].emit = si_llvm_emit_ddxy;
5469 bld_base->op_actions[TGSI_OPCODE_DDY_FINE].emit = si_llvm_emit_ddxy;
5470
5471 bld_base->op_actions[TGSI_OPCODE_VOTE_ALL].emit = vote_all_emit;
5472 bld_base->op_actions[TGSI_OPCODE_VOTE_ANY].emit = vote_any_emit;
5473 bld_base->op_actions[TGSI_OPCODE_VOTE_EQ].emit = vote_eq_emit;
5474 bld_base->op_actions[TGSI_OPCODE_BALLOT].emit = ballot_emit;
5475 bld_base->op_actions[TGSI_OPCODE_READ_FIRST].intr_name = "llvm.amdgcn.readfirstlane";
5476 bld_base->op_actions[TGSI_OPCODE_READ_FIRST].emit = read_lane_emit;
5477 bld_base->op_actions[TGSI_OPCODE_READ_INVOC].intr_name = "llvm.amdgcn.readlane";
5478 bld_base->op_actions[TGSI_OPCODE_READ_INVOC].fetch_args = read_invoc_fetch_args;
5479 bld_base->op_actions[TGSI_OPCODE_READ_INVOC].emit = read_lane_emit;
5480
5481 bld_base->op_actions[TGSI_OPCODE_EMIT].emit = si_llvm_emit_vertex;
5482 bld_base->op_actions[TGSI_OPCODE_ENDPRIM].emit = si_llvm_emit_primitive;
5483 bld_base->op_actions[TGSI_OPCODE_BARRIER].emit = si_llvm_emit_barrier;
5484 }
5485
5486 static void si_optimize_vs_outputs(struct si_shader_context *ctx)
5487 {
5488 struct si_shader *shader = ctx->shader;
5489 struct tgsi_shader_info *info = &shader->selector->info;
5490
5491 if ((ctx->type != PIPE_SHADER_VERTEX &&
5492 ctx->type != PIPE_SHADER_TESS_EVAL) ||
5493 shader->key.as_ls ||
5494 shader->key.as_es)
5495 return;
5496
5497 ac_optimize_vs_outputs(&ctx->ac,
5498 ctx->main_fn,
5499 shader->info.vs_output_param_offset,
5500 info->num_outputs,
5501 &shader->info.nr_param_exports);
5502 }
5503
5504 static void si_count_scratch_private_memory(struct si_shader_context *ctx)
5505 {
5506 ctx->shader->config.private_mem_vgprs = 0;
5507
5508 /* Process all LLVM instructions. */
5509 LLVMBasicBlockRef bb = LLVMGetFirstBasicBlock(ctx->main_fn);
5510 while (bb) {
5511 LLVMValueRef next = LLVMGetFirstInstruction(bb);
5512
5513 while (next) {
5514 LLVMValueRef inst = next;
5515 next = LLVMGetNextInstruction(next);
5516
5517 if (LLVMGetInstructionOpcode(inst) != LLVMAlloca)
5518 continue;
5519
5520 LLVMTypeRef type = LLVMGetElementType(LLVMTypeOf(inst));
5521 /* No idea why LLVM aligns allocas to 4 elements. */
5522 unsigned alignment = LLVMGetAlignment(inst);
5523 unsigned dw_size = align(llvm_get_type_size(type) / 4, alignment);
5524 ctx->shader->config.private_mem_vgprs += dw_size;
5525 }
5526 bb = LLVMGetNextBasicBlock(bb);
5527 }
5528 }
5529
5530 static void si_init_exec_full_mask(struct si_shader_context *ctx)
5531 {
5532 LLVMValueRef full_mask = LLVMConstInt(ctx->i64, ~0ull, 0);
5533 lp_build_intrinsic(ctx->gallivm.builder,
5534 "llvm.amdgcn.init.exec", ctx->voidt,
5535 &full_mask, 1, LP_FUNC_ATTR_CONVERGENT);
5536 }
5537
5538 static void si_init_exec_from_input(struct si_shader_context *ctx,
5539 unsigned param, unsigned bitoffset)
5540 {
5541 LLVMValueRef args[] = {
5542 LLVMGetParam(ctx->main_fn, param),
5543 LLVMConstInt(ctx->i32, bitoffset, 0),
5544 };
5545 lp_build_intrinsic(ctx->gallivm.builder,
5546 "llvm.amdgcn.init.exec.from.input",
5547 ctx->voidt, args, 2, LP_FUNC_ATTR_CONVERGENT);
5548 }
5549
5550 static bool si_compile_tgsi_main(struct si_shader_context *ctx,
5551 bool is_monolithic)
5552 {
5553 struct si_shader *shader = ctx->shader;
5554 struct si_shader_selector *sel = shader->selector;
5555 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
5556
5557 switch (ctx->type) {
5558 case PIPE_SHADER_VERTEX:
5559 ctx->load_input = declare_input_vs;
5560 if (shader->key.as_ls)
5561 bld_base->emit_epilogue = si_llvm_emit_ls_epilogue;
5562 else if (shader->key.as_es)
5563 bld_base->emit_epilogue = si_llvm_emit_es_epilogue;
5564 else
5565 bld_base->emit_epilogue = si_llvm_emit_vs_epilogue;
5566 break;
5567 case PIPE_SHADER_TESS_CTRL:
5568 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_tcs;
5569 bld_base->emit_fetch_funcs[TGSI_FILE_OUTPUT] = fetch_output_tcs;
5570 bld_base->emit_store = store_output_tcs;
5571 bld_base->emit_epilogue = si_llvm_emit_tcs_epilogue;
5572 break;
5573 case PIPE_SHADER_TESS_EVAL:
5574 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_tes;
5575 if (shader->key.as_es)
5576 bld_base->emit_epilogue = si_llvm_emit_es_epilogue;
5577 else
5578 bld_base->emit_epilogue = si_llvm_emit_vs_epilogue;
5579 break;
5580 case PIPE_SHADER_GEOMETRY:
5581 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_gs;
5582 bld_base->emit_epilogue = si_llvm_emit_gs_epilogue;
5583 break;
5584 case PIPE_SHADER_FRAGMENT:
5585 ctx->load_input = declare_input_fs;
5586 bld_base->emit_epilogue = si_llvm_return_fs_outputs;
5587 break;
5588 case PIPE_SHADER_COMPUTE:
5589 ctx->declare_memory_region = declare_compute_memory;
5590 break;
5591 default:
5592 assert(!"Unsupported shader type");
5593 return false;
5594 }
5595
5596 create_function(ctx);
5597 preload_ring_buffers(ctx);
5598
5599 /* For GFX9 merged shaders:
5600 * - Set EXEC for the first shader. If the prolog is present, set
5601 * EXEC there instead.
5602 * - Add a barrier before the second shader.
5603 * - In the second shader, reset EXEC to ~0 and wrap the main part in
5604 * an if-statement. This is required for correctness in geometry
5605 * shaders, to ensure that empty GS waves do not send GS_EMIT and
5606 * GS_CUT messages.
5607 *
5608 * For monolithic merged shaders, the first shader is wrapped in an
5609 * if-block together with its prolog in si_build_wrapper_function.
5610 */
5611 if (ctx->screen->b.chip_class >= GFX9) {
5612 if (!is_monolithic &&
5613 sel->info.num_instructions > 1 && /* not empty shader */
5614 (shader->key.as_es || shader->key.as_ls) &&
5615 (ctx->type == PIPE_SHADER_TESS_EVAL ||
5616 (ctx->type == PIPE_SHADER_VERTEX &&
5617 !sel->vs_needs_prolog))) {
5618 si_init_exec_from_input(ctx,
5619 ctx->param_merged_wave_info, 0);
5620 } else if (ctx->type == PIPE_SHADER_TESS_CTRL ||
5621 ctx->type == PIPE_SHADER_GEOMETRY) {
5622 if (!is_monolithic)
5623 si_init_exec_full_mask(ctx);
5624
5625 /* The barrier must execute for all shaders in a
5626 * threadgroup.
5627 */
5628 si_llvm_emit_barrier(NULL, bld_base, NULL);
5629
5630 LLVMValueRef num_threads = unpack_param(ctx, ctx->param_merged_wave_info, 8, 8);
5631 LLVMValueRef ena =
5632 LLVMBuildICmp(ctx->ac.builder, LLVMIntULT,
5633 ac_get_thread_id(&ctx->ac), num_threads, "");
5634 lp_build_if(&ctx->merged_wrap_if_state, &ctx->gallivm, ena);
5635 }
5636 }
5637
5638 if (ctx->type == PIPE_SHADER_GEOMETRY) {
5639 int i;
5640 for (i = 0; i < 4; i++) {
5641 ctx->gs_next_vertex[i] =
5642 lp_build_alloca(&ctx->gallivm,
5643 ctx->i32, "");
5644 }
5645 }
5646
5647 if (ctx->type == PIPE_SHADER_FRAGMENT && sel->info.uses_kill &&
5648 ctx->screen->b.debug_flags & DBG_FS_CORRECT_DERIVS_AFTER_KILL) {
5649 /* This is initialized to 0.0 = not kill. */
5650 ctx->postponed_kill = lp_build_alloca(&ctx->gallivm, ctx->f32, "");
5651 }
5652
5653 if (!lp_build_tgsi_llvm(bld_base, sel->tokens)) {
5654 fprintf(stderr, "Failed to translate shader from TGSI to LLVM\n");
5655 return false;
5656 }
5657
5658 si_llvm_build_ret(ctx, ctx->return_value);
5659 return true;
5660 }
5661
5662 /**
5663 * Compute the VS prolog key, which contains all the information needed to
5664 * build the VS prolog function, and set shader->info bits where needed.
5665 *
5666 * \param info Shader info of the vertex shader.
5667 * \param num_input_sgprs Number of input SGPRs for the vertex shader.
5668 * \param prolog_key Key of the VS prolog
5669 * \param shader_out The vertex shader, or the next shader if merging LS+HS or ES+GS.
5670 * \param key Output shader part key.
5671 */
5672 static void si_get_vs_prolog_key(const struct tgsi_shader_info *info,
5673 unsigned num_input_sgprs,
5674 const struct si_vs_prolog_bits *prolog_key,
5675 struct si_shader *shader_out,
5676 union si_shader_part_key *key)
5677 {
5678 memset(key, 0, sizeof(*key));
5679 key->vs_prolog.states = *prolog_key;
5680 key->vs_prolog.num_input_sgprs = num_input_sgprs;
5681 key->vs_prolog.last_input = MAX2(1, info->num_inputs) - 1;
5682 key->vs_prolog.as_ls = shader_out->key.as_ls;
5683
5684 if (shader_out->selector->type == PIPE_SHADER_TESS_CTRL) {
5685 key->vs_prolog.as_ls = 1;
5686 key->vs_prolog.num_merged_next_stage_vgprs = 2;
5687 } else if (shader_out->selector->type == PIPE_SHADER_GEOMETRY) {
5688 key->vs_prolog.num_merged_next_stage_vgprs = 5;
5689 }
5690
5691 /* Enable loading the InstanceID VGPR. */
5692 uint16_t input_mask = u_bit_consecutive(0, info->num_inputs);
5693
5694 if ((key->vs_prolog.states.instance_divisor_is_one |
5695 key->vs_prolog.states.instance_divisor_is_fetched) & input_mask)
5696 shader_out->info.uses_instanceid = true;
5697 }
5698
5699 /**
5700 * Compute the PS prolog key, which contains all the information needed to
5701 * build the PS prolog function, and set related bits in shader->config.
5702 */
5703 static void si_get_ps_prolog_key(struct si_shader *shader,
5704 union si_shader_part_key *key,
5705 bool separate_prolog)
5706 {
5707 struct tgsi_shader_info *info = &shader->selector->info;
5708
5709 memset(key, 0, sizeof(*key));
5710 key->ps_prolog.states = shader->key.part.ps.prolog;
5711 key->ps_prolog.colors_read = info->colors_read;
5712 key->ps_prolog.num_input_sgprs = shader->info.num_input_sgprs;
5713 key->ps_prolog.num_input_vgprs = shader->info.num_input_vgprs;
5714 key->ps_prolog.wqm = info->uses_derivatives &&
5715 (key->ps_prolog.colors_read ||
5716 key->ps_prolog.states.force_persp_sample_interp ||
5717 key->ps_prolog.states.force_linear_sample_interp ||
5718 key->ps_prolog.states.force_persp_center_interp ||
5719 key->ps_prolog.states.force_linear_center_interp ||
5720 key->ps_prolog.states.bc_optimize_for_persp ||
5721 key->ps_prolog.states.bc_optimize_for_linear);
5722
5723 if (info->colors_read) {
5724 unsigned *color = shader->selector->color_attr_index;
5725
5726 if (shader->key.part.ps.prolog.color_two_side) {
5727 /* BCOLORs are stored after the last input. */
5728 key->ps_prolog.num_interp_inputs = info->num_inputs;
5729 key->ps_prolog.face_vgpr_index = shader->info.face_vgpr_index;
5730 shader->config.spi_ps_input_ena |= S_0286CC_FRONT_FACE_ENA(1);
5731 }
5732
5733 for (unsigned i = 0; i < 2; i++) {
5734 unsigned interp = info->input_interpolate[color[i]];
5735 unsigned location = info->input_interpolate_loc[color[i]];
5736
5737 if (!(info->colors_read & (0xf << i*4)))
5738 continue;
5739
5740 key->ps_prolog.color_attr_index[i] = color[i];
5741
5742 if (shader->key.part.ps.prolog.flatshade_colors &&
5743 interp == TGSI_INTERPOLATE_COLOR)
5744 interp = TGSI_INTERPOLATE_CONSTANT;
5745
5746 switch (interp) {
5747 case TGSI_INTERPOLATE_CONSTANT:
5748 key->ps_prolog.color_interp_vgpr_index[i] = -1;
5749 break;
5750 case TGSI_INTERPOLATE_PERSPECTIVE:
5751 case TGSI_INTERPOLATE_COLOR:
5752 /* Force the interpolation location for colors here. */
5753 if (shader->key.part.ps.prolog.force_persp_sample_interp)
5754 location = TGSI_INTERPOLATE_LOC_SAMPLE;
5755 if (shader->key.part.ps.prolog.force_persp_center_interp)
5756 location = TGSI_INTERPOLATE_LOC_CENTER;
5757
5758 switch (location) {
5759 case TGSI_INTERPOLATE_LOC_SAMPLE:
5760 key->ps_prolog.color_interp_vgpr_index[i] = 0;
5761 shader->config.spi_ps_input_ena |=
5762 S_0286CC_PERSP_SAMPLE_ENA(1);
5763 break;
5764 case TGSI_INTERPOLATE_LOC_CENTER:
5765 key->ps_prolog.color_interp_vgpr_index[i] = 2;
5766 shader->config.spi_ps_input_ena |=
5767 S_0286CC_PERSP_CENTER_ENA(1);
5768 break;
5769 case TGSI_INTERPOLATE_LOC_CENTROID:
5770 key->ps_prolog.color_interp_vgpr_index[i] = 4;
5771 shader->config.spi_ps_input_ena |=
5772 S_0286CC_PERSP_CENTROID_ENA(1);
5773 break;
5774 default:
5775 assert(0);
5776 }
5777 break;
5778 case TGSI_INTERPOLATE_LINEAR:
5779 /* Force the interpolation location for colors here. */
5780 if (shader->key.part.ps.prolog.force_linear_sample_interp)
5781 location = TGSI_INTERPOLATE_LOC_SAMPLE;
5782 if (shader->key.part.ps.prolog.force_linear_center_interp)
5783 location = TGSI_INTERPOLATE_LOC_CENTER;
5784
5785 /* The VGPR assignment for non-monolithic shaders
5786 * works because InitialPSInputAddr is set on the
5787 * main shader and PERSP_PULL_MODEL is never used.
5788 */
5789 switch (location) {
5790 case TGSI_INTERPOLATE_LOC_SAMPLE:
5791 key->ps_prolog.color_interp_vgpr_index[i] =
5792 separate_prolog ? 6 : 9;
5793 shader->config.spi_ps_input_ena |=
5794 S_0286CC_LINEAR_SAMPLE_ENA(1);
5795 break;
5796 case TGSI_INTERPOLATE_LOC_CENTER:
5797 key->ps_prolog.color_interp_vgpr_index[i] =
5798 separate_prolog ? 8 : 11;
5799 shader->config.spi_ps_input_ena |=
5800 S_0286CC_LINEAR_CENTER_ENA(1);
5801 break;
5802 case TGSI_INTERPOLATE_LOC_CENTROID:
5803 key->ps_prolog.color_interp_vgpr_index[i] =
5804 separate_prolog ? 10 : 13;
5805 shader->config.spi_ps_input_ena |=
5806 S_0286CC_LINEAR_CENTROID_ENA(1);
5807 break;
5808 default:
5809 assert(0);
5810 }
5811 break;
5812 default:
5813 assert(0);
5814 }
5815 }
5816 }
5817 }
5818
5819 /**
5820 * Check whether a PS prolog is required based on the key.
5821 */
5822 static bool si_need_ps_prolog(const union si_shader_part_key *key)
5823 {
5824 return key->ps_prolog.colors_read ||
5825 key->ps_prolog.states.force_persp_sample_interp ||
5826 key->ps_prolog.states.force_linear_sample_interp ||
5827 key->ps_prolog.states.force_persp_center_interp ||
5828 key->ps_prolog.states.force_linear_center_interp ||
5829 key->ps_prolog.states.bc_optimize_for_persp ||
5830 key->ps_prolog.states.bc_optimize_for_linear ||
5831 key->ps_prolog.states.poly_stipple;
5832 }
5833
5834 /**
5835 * Compute the PS epilog key, which contains all the information needed to
5836 * build the PS epilog function.
5837 */
5838 static void si_get_ps_epilog_key(struct si_shader *shader,
5839 union si_shader_part_key *key)
5840 {
5841 struct tgsi_shader_info *info = &shader->selector->info;
5842 memset(key, 0, sizeof(*key));
5843 key->ps_epilog.colors_written = info->colors_written;
5844 key->ps_epilog.writes_z = info->writes_z;
5845 key->ps_epilog.writes_stencil = info->writes_stencil;
5846 key->ps_epilog.writes_samplemask = info->writes_samplemask;
5847 key->ps_epilog.states = shader->key.part.ps.epilog;
5848 }
5849
5850 /**
5851 * Build the GS prolog function. Rotate the input vertices for triangle strips
5852 * with adjacency.
5853 */
5854 static void si_build_gs_prolog_function(struct si_shader_context *ctx,
5855 union si_shader_part_key *key)
5856 {
5857 unsigned num_sgprs, num_vgprs;
5858 struct gallivm_state *gallivm = &ctx->gallivm;
5859 struct si_function_info fninfo;
5860 LLVMBuilderRef builder = gallivm->builder;
5861 LLVMTypeRef returns[48];
5862 LLVMValueRef func, ret;
5863
5864 si_init_function_info(&fninfo);
5865
5866 if (ctx->screen->b.chip_class >= GFX9) {
5867 num_sgprs = 8 + GFX9_GS_NUM_USER_SGPR;
5868 num_vgprs = 5; /* ES inputs are not needed by GS */
5869 } else {
5870 num_sgprs = GFX6_GS_NUM_USER_SGPR + 2;
5871 num_vgprs = 8;
5872 }
5873
5874 for (unsigned i = 0; i < num_sgprs; ++i) {
5875 add_arg(&fninfo, ARG_SGPR, ctx->i32);
5876 returns[i] = ctx->i32;
5877 }
5878
5879 for (unsigned i = 0; i < num_vgprs; ++i) {
5880 add_arg(&fninfo, ARG_VGPR, ctx->i32);
5881 returns[num_sgprs + i] = ctx->f32;
5882 }
5883
5884 /* Create the function. */
5885 si_create_function(ctx, "gs_prolog", returns, num_sgprs + num_vgprs,
5886 &fninfo, 0);
5887 func = ctx->main_fn;
5888
5889 /* Set the full EXEC mask for the prolog, because we are only fiddling
5890 * with registers here. The main shader part will set the correct EXEC
5891 * mask.
5892 */
5893 if (ctx->screen->b.chip_class >= GFX9 && !key->gs_prolog.is_monolithic)
5894 si_init_exec_full_mask(ctx);
5895
5896 /* Copy inputs to outputs. This should be no-op, as the registers match,
5897 * but it will prevent the compiler from overwriting them unintentionally.
5898 */
5899 ret = ctx->return_value;
5900 for (unsigned i = 0; i < num_sgprs; i++) {
5901 LLVMValueRef p = LLVMGetParam(func, i);
5902 ret = LLVMBuildInsertValue(builder, ret, p, i, "");
5903 }
5904 for (unsigned i = 0; i < num_vgprs; i++) {
5905 LLVMValueRef p = LLVMGetParam(func, num_sgprs + i);
5906 p = LLVMBuildBitCast(builder, p, ctx->f32, "");
5907 ret = LLVMBuildInsertValue(builder, ret, p, num_sgprs + i, "");
5908 }
5909
5910 if (key->gs_prolog.states.tri_strip_adj_fix) {
5911 /* Remap the input vertices for every other primitive. */
5912 const unsigned gfx6_vtx_params[6] = {
5913 num_sgprs,
5914 num_sgprs + 1,
5915 num_sgprs + 3,
5916 num_sgprs + 4,
5917 num_sgprs + 5,
5918 num_sgprs + 6
5919 };
5920 const unsigned gfx9_vtx_params[3] = {
5921 num_sgprs,
5922 num_sgprs + 1,
5923 num_sgprs + 4,
5924 };
5925 LLVMValueRef vtx_in[6], vtx_out[6];
5926 LLVMValueRef prim_id, rotate;
5927
5928 if (ctx->screen->b.chip_class >= GFX9) {
5929 for (unsigned i = 0; i < 3; i++) {
5930 vtx_in[i*2] = unpack_param(ctx, gfx9_vtx_params[i], 0, 16);
5931 vtx_in[i*2+1] = unpack_param(ctx, gfx9_vtx_params[i], 16, 16);
5932 }
5933 } else {
5934 for (unsigned i = 0; i < 6; i++)
5935 vtx_in[i] = LLVMGetParam(func, gfx6_vtx_params[i]);
5936 }
5937
5938 prim_id = LLVMGetParam(func, num_sgprs + 2);
5939 rotate = LLVMBuildTrunc(builder, prim_id, ctx->i1, "");
5940
5941 for (unsigned i = 0; i < 6; ++i) {
5942 LLVMValueRef base, rotated;
5943 base = vtx_in[i];
5944 rotated = vtx_in[(i + 4) % 6];
5945 vtx_out[i] = LLVMBuildSelect(builder, rotate, rotated, base, "");
5946 }
5947
5948 if (ctx->screen->b.chip_class >= GFX9) {
5949 for (unsigned i = 0; i < 3; i++) {
5950 LLVMValueRef hi, out;
5951
5952 hi = LLVMBuildShl(builder, vtx_out[i*2+1],
5953 LLVMConstInt(ctx->i32, 16, 0), "");
5954 out = LLVMBuildOr(builder, vtx_out[i*2], hi, "");
5955 out = LLVMBuildBitCast(builder, out, ctx->f32, "");
5956 ret = LLVMBuildInsertValue(builder, ret, out,
5957 gfx9_vtx_params[i], "");
5958 }
5959 } else {
5960 for (unsigned i = 0; i < 6; i++) {
5961 LLVMValueRef out;
5962
5963 out = LLVMBuildBitCast(builder, vtx_out[i], ctx->f32, "");
5964 ret = LLVMBuildInsertValue(builder, ret, out,
5965 gfx6_vtx_params[i], "");
5966 }
5967 }
5968 }
5969
5970 LLVMBuildRet(builder, ret);
5971 }
5972
5973 /**
5974 * Given a list of shader part functions, build a wrapper function that
5975 * runs them in sequence to form a monolithic shader.
5976 */
5977 static void si_build_wrapper_function(struct si_shader_context *ctx,
5978 LLVMValueRef *parts,
5979 unsigned num_parts,
5980 unsigned main_part,
5981 unsigned next_shader_first_part)
5982 {
5983 struct gallivm_state *gallivm = &ctx->gallivm;
5984 LLVMBuilderRef builder = ctx->gallivm.builder;
5985 /* PS epilog has one arg per color component; gfx9 merged shader
5986 * prologs need to forward 32 user SGPRs.
5987 */
5988 struct si_function_info fninfo;
5989 LLVMValueRef initial[64], out[64];
5990 LLVMTypeRef function_type;
5991 unsigned num_first_params;
5992 unsigned num_out, initial_num_out;
5993 MAYBE_UNUSED unsigned num_out_sgpr; /* used in debug checks */
5994 MAYBE_UNUSED unsigned initial_num_out_sgpr; /* used in debug checks */
5995 unsigned num_sgprs, num_vgprs;
5996 unsigned gprs;
5997 struct lp_build_if_state if_state;
5998
5999 si_init_function_info(&fninfo);
6000
6001 for (unsigned i = 0; i < num_parts; ++i) {
6002 lp_add_function_attr(parts[i], -1, LP_FUNC_ATTR_ALWAYSINLINE);
6003 LLVMSetLinkage(parts[i], LLVMPrivateLinkage);
6004 }
6005
6006 /* The parameters of the wrapper function correspond to those of the
6007 * first part in terms of SGPRs and VGPRs, but we use the types of the
6008 * main part to get the right types. This is relevant for the
6009 * dereferenceable attribute on descriptor table pointers.
6010 */
6011 num_sgprs = 0;
6012 num_vgprs = 0;
6013
6014 function_type = LLVMGetElementType(LLVMTypeOf(parts[0]));
6015 num_first_params = LLVMCountParamTypes(function_type);
6016
6017 for (unsigned i = 0; i < num_first_params; ++i) {
6018 LLVMValueRef param = LLVMGetParam(parts[0], i);
6019
6020 if (ac_is_sgpr_param(param)) {
6021 assert(num_vgprs == 0);
6022 num_sgprs += llvm_get_type_size(LLVMTypeOf(param)) / 4;
6023 } else {
6024 num_vgprs += llvm_get_type_size(LLVMTypeOf(param)) / 4;
6025 }
6026 }
6027
6028 gprs = 0;
6029 while (gprs < num_sgprs + num_vgprs) {
6030 LLVMValueRef param = LLVMGetParam(parts[main_part], fninfo.num_params);
6031 LLVMTypeRef type = LLVMTypeOf(param);
6032 unsigned size = llvm_get_type_size(type) / 4;
6033
6034 add_arg(&fninfo, gprs < num_sgprs ? ARG_SGPR : ARG_VGPR, type);
6035
6036 assert(ac_is_sgpr_param(param) == (gprs < num_sgprs));
6037 assert(gprs + size <= num_sgprs + num_vgprs &&
6038 (gprs >= num_sgprs || gprs + size <= num_sgprs));
6039
6040 gprs += size;
6041 }
6042
6043 si_create_function(ctx, "wrapper", NULL, 0, &fninfo,
6044 si_get_max_workgroup_size(ctx->shader));
6045
6046 if (is_merged_shader(ctx->shader))
6047 si_init_exec_full_mask(ctx);
6048
6049 /* Record the arguments of the function as if they were an output of
6050 * a previous part.
6051 */
6052 num_out = 0;
6053 num_out_sgpr = 0;
6054
6055 for (unsigned i = 0; i < fninfo.num_params; ++i) {
6056 LLVMValueRef param = LLVMGetParam(ctx->main_fn, i);
6057 LLVMTypeRef param_type = LLVMTypeOf(param);
6058 LLVMTypeRef out_type = i < fninfo.num_sgpr_params ? ctx->i32 : ctx->f32;
6059 unsigned size = llvm_get_type_size(param_type) / 4;
6060
6061 if (size == 1) {
6062 if (param_type != out_type)
6063 param = LLVMBuildBitCast(builder, param, out_type, "");
6064 out[num_out++] = param;
6065 } else {
6066 LLVMTypeRef vector_type = LLVMVectorType(out_type, size);
6067
6068 if (LLVMGetTypeKind(param_type) == LLVMPointerTypeKind) {
6069 param = LLVMBuildPtrToInt(builder, param, ctx->i64, "");
6070 param_type = ctx->i64;
6071 }
6072
6073 if (param_type != vector_type)
6074 param = LLVMBuildBitCast(builder, param, vector_type, "");
6075
6076 for (unsigned j = 0; j < size; ++j)
6077 out[num_out++] = LLVMBuildExtractElement(
6078 builder, param, LLVMConstInt(ctx->i32, j, 0), "");
6079 }
6080
6081 if (i < fninfo.num_sgpr_params)
6082 num_out_sgpr = num_out;
6083 }
6084
6085 memcpy(initial, out, sizeof(out));
6086 initial_num_out = num_out;
6087 initial_num_out_sgpr = num_out_sgpr;
6088
6089 /* Now chain the parts. */
6090 for (unsigned part = 0; part < num_parts; ++part) {
6091 LLVMValueRef in[48];
6092 LLVMValueRef ret;
6093 LLVMTypeRef ret_type;
6094 unsigned out_idx = 0;
6095 unsigned num_params = LLVMCountParams(parts[part]);
6096
6097 /* Merged shaders are executed conditionally depending
6098 * on the number of enabled threads passed in the input SGPRs. */
6099 if (is_merged_shader(ctx->shader) && part == 0) {
6100 LLVMValueRef ena, count = initial[3];
6101
6102 count = LLVMBuildAnd(builder, count,
6103 LLVMConstInt(ctx->i32, 0x7f, 0), "");
6104 ena = LLVMBuildICmp(builder, LLVMIntULT,
6105 ac_get_thread_id(&ctx->ac), count, "");
6106 lp_build_if(&if_state, &ctx->gallivm, ena);
6107 }
6108
6109 /* Derive arguments for the next part from outputs of the
6110 * previous one.
6111 */
6112 for (unsigned param_idx = 0; param_idx < num_params; ++param_idx) {
6113 LLVMValueRef param;
6114 LLVMTypeRef param_type;
6115 bool is_sgpr;
6116 unsigned param_size;
6117 LLVMValueRef arg = NULL;
6118
6119 param = LLVMGetParam(parts[part], param_idx);
6120 param_type = LLVMTypeOf(param);
6121 param_size = llvm_get_type_size(param_type) / 4;
6122 is_sgpr = ac_is_sgpr_param(param);
6123
6124 if (is_sgpr) {
6125 #if HAVE_LLVM < 0x0400
6126 LLVMRemoveAttribute(param, LLVMByValAttribute);
6127 #else
6128 unsigned kind_id = LLVMGetEnumAttributeKindForName("byval", 5);
6129 LLVMRemoveEnumAttributeAtIndex(parts[part], param_idx + 1, kind_id);
6130 #endif
6131 lp_add_function_attr(parts[part], param_idx + 1, LP_FUNC_ATTR_INREG);
6132 }
6133
6134 assert(out_idx + param_size <= (is_sgpr ? num_out_sgpr : num_out));
6135 assert(is_sgpr || out_idx >= num_out_sgpr);
6136
6137 if (param_size == 1)
6138 arg = out[out_idx];
6139 else
6140 arg = lp_build_gather_values(gallivm, &out[out_idx], param_size);
6141
6142 if (LLVMTypeOf(arg) != param_type) {
6143 if (LLVMGetTypeKind(param_type) == LLVMPointerTypeKind) {
6144 arg = LLVMBuildBitCast(builder, arg, ctx->i64, "");
6145 arg = LLVMBuildIntToPtr(builder, arg, param_type, "");
6146 } else {
6147 arg = LLVMBuildBitCast(builder, arg, param_type, "");
6148 }
6149 }
6150
6151 in[param_idx] = arg;
6152 out_idx += param_size;
6153 }
6154
6155 ret = LLVMBuildCall(builder, parts[part], in, num_params, "");
6156
6157 if (is_merged_shader(ctx->shader) &&
6158 part + 1 == next_shader_first_part) {
6159 lp_build_endif(&if_state);
6160
6161 /* The second half of the merged shader should use
6162 * the inputs from the toplevel (wrapper) function,
6163 * not the return value from the last call.
6164 *
6165 * That's because the last call was executed condi-
6166 * tionally, so we can't consume it in the main
6167 * block.
6168 */
6169 memcpy(out, initial, sizeof(initial));
6170 num_out = initial_num_out;
6171 num_out_sgpr = initial_num_out_sgpr;
6172 continue;
6173 }
6174
6175 /* Extract the returned GPRs. */
6176 ret_type = LLVMTypeOf(ret);
6177 num_out = 0;
6178 num_out_sgpr = 0;
6179
6180 if (LLVMGetTypeKind(ret_type) != LLVMVoidTypeKind) {
6181 assert(LLVMGetTypeKind(ret_type) == LLVMStructTypeKind);
6182
6183 unsigned ret_size = LLVMCountStructElementTypes(ret_type);
6184
6185 for (unsigned i = 0; i < ret_size; ++i) {
6186 LLVMValueRef val =
6187 LLVMBuildExtractValue(builder, ret, i, "");
6188
6189 assert(num_out < ARRAY_SIZE(out));
6190 out[num_out++] = val;
6191
6192 if (LLVMTypeOf(val) == ctx->i32) {
6193 assert(num_out_sgpr + 1 == num_out);
6194 num_out_sgpr = num_out;
6195 }
6196 }
6197 }
6198 }
6199
6200 LLVMBuildRetVoid(builder);
6201 }
6202
6203 int si_compile_tgsi_shader(struct si_screen *sscreen,
6204 LLVMTargetMachineRef tm,
6205 struct si_shader *shader,
6206 bool is_monolithic,
6207 struct pipe_debug_callback *debug)
6208 {
6209 struct si_shader_selector *sel = shader->selector;
6210 struct si_shader_context ctx;
6211 int r = -1;
6212
6213 /* Dump TGSI code before doing TGSI->LLVM conversion in case the
6214 * conversion fails. */
6215 if (r600_can_dump_shader(&sscreen->b, sel->info.processor) &&
6216 !(sscreen->b.debug_flags & DBG_NO_TGSI)) {
6217 tgsi_dump(sel->tokens, 0);
6218 si_dump_streamout(&sel->so);
6219 }
6220
6221 si_init_shader_ctx(&ctx, sscreen, tm);
6222 si_llvm_context_set_tgsi(&ctx, shader);
6223 ctx.separate_prolog = !is_monolithic;
6224
6225 memset(shader->info.vs_output_param_offset, AC_EXP_PARAM_UNDEFINED,
6226 sizeof(shader->info.vs_output_param_offset));
6227
6228 shader->info.uses_instanceid = sel->info.uses_instanceid;
6229
6230 ctx.load_system_value = declare_system_value;
6231
6232 if (!si_compile_tgsi_main(&ctx, is_monolithic)) {
6233 si_llvm_dispose(&ctx);
6234 return -1;
6235 }
6236
6237 if (is_monolithic && ctx.type == PIPE_SHADER_VERTEX) {
6238 LLVMValueRef parts[2];
6239 bool need_prolog = sel->vs_needs_prolog;
6240
6241 parts[1] = ctx.main_fn;
6242
6243 if (need_prolog) {
6244 union si_shader_part_key prolog_key;
6245 si_get_vs_prolog_key(&sel->info,
6246 shader->info.num_input_sgprs,
6247 &shader->key.part.vs.prolog,
6248 shader, &prolog_key);
6249 si_build_vs_prolog_function(&ctx, &prolog_key);
6250 parts[0] = ctx.main_fn;
6251 }
6252
6253 si_build_wrapper_function(&ctx, parts + !need_prolog,
6254 1 + need_prolog, need_prolog, 0);
6255 } else if (is_monolithic && ctx.type == PIPE_SHADER_TESS_CTRL) {
6256 if (sscreen->b.chip_class >= GFX9) {
6257 struct si_shader_selector *ls = shader->key.part.tcs.ls;
6258 LLVMValueRef parts[4];
6259
6260 /* TCS main part */
6261 parts[2] = ctx.main_fn;
6262
6263 /* TCS epilog */
6264 union si_shader_part_key tcs_epilog_key;
6265 memset(&tcs_epilog_key, 0, sizeof(tcs_epilog_key));
6266 tcs_epilog_key.tcs_epilog.states = shader->key.part.tcs.epilog;
6267 si_build_tcs_epilog_function(&ctx, &tcs_epilog_key);
6268 parts[3] = ctx.main_fn;
6269
6270 /* VS prolog */
6271 if (ls->vs_needs_prolog) {
6272 union si_shader_part_key vs_prolog_key;
6273 si_get_vs_prolog_key(&ls->info,
6274 shader->info.num_input_sgprs,
6275 &shader->key.part.tcs.ls_prolog,
6276 shader, &vs_prolog_key);
6277 vs_prolog_key.vs_prolog.is_monolithic = true;
6278 si_build_vs_prolog_function(&ctx, &vs_prolog_key);
6279 parts[0] = ctx.main_fn;
6280 }
6281
6282 /* VS as LS main part */
6283 struct si_shader shader_ls = {};
6284 shader_ls.selector = ls;
6285 shader_ls.key.as_ls = 1;
6286 shader_ls.key.mono = shader->key.mono;
6287 shader_ls.key.opt = shader->key.opt;
6288 si_llvm_context_set_tgsi(&ctx, &shader_ls);
6289
6290 if (!si_compile_tgsi_main(&ctx, true)) {
6291 si_llvm_dispose(&ctx);
6292 return -1;
6293 }
6294 shader->info.uses_instanceid |= ls->info.uses_instanceid;
6295 parts[1] = ctx.main_fn;
6296
6297 /* Reset the shader context. */
6298 ctx.shader = shader;
6299 ctx.type = PIPE_SHADER_TESS_CTRL;
6300
6301 si_build_wrapper_function(&ctx,
6302 parts + !ls->vs_needs_prolog,
6303 4 - !ls->vs_needs_prolog, 0,
6304 ls->vs_needs_prolog ? 2 : 1);
6305 } else {
6306 LLVMValueRef parts[2];
6307 union si_shader_part_key epilog_key;
6308
6309 parts[0] = ctx.main_fn;
6310
6311 memset(&epilog_key, 0, sizeof(epilog_key));
6312 epilog_key.tcs_epilog.states = shader->key.part.tcs.epilog;
6313 si_build_tcs_epilog_function(&ctx, &epilog_key);
6314 parts[1] = ctx.main_fn;
6315
6316 si_build_wrapper_function(&ctx, parts, 2, 0, 0);
6317 }
6318 } else if (is_monolithic && ctx.type == PIPE_SHADER_GEOMETRY) {
6319 if (ctx.screen->b.chip_class >= GFX9) {
6320 struct si_shader_selector *es = shader->key.part.gs.es;
6321 LLVMValueRef es_prolog = NULL;
6322 LLVMValueRef es_main = NULL;
6323 LLVMValueRef gs_prolog = NULL;
6324 LLVMValueRef gs_main = ctx.main_fn;
6325
6326 /* GS prolog */
6327 union si_shader_part_key gs_prolog_key;
6328 memset(&gs_prolog_key, 0, sizeof(gs_prolog_key));
6329 gs_prolog_key.gs_prolog.states = shader->key.part.gs.prolog;
6330 gs_prolog_key.gs_prolog.is_monolithic = true;
6331 si_build_gs_prolog_function(&ctx, &gs_prolog_key);
6332 gs_prolog = ctx.main_fn;
6333
6334 /* ES prolog */
6335 if (es->vs_needs_prolog) {
6336 union si_shader_part_key vs_prolog_key;
6337 si_get_vs_prolog_key(&es->info,
6338 shader->info.num_input_sgprs,
6339 &shader->key.part.tcs.ls_prolog,
6340 shader, &vs_prolog_key);
6341 vs_prolog_key.vs_prolog.is_monolithic = true;
6342 si_build_vs_prolog_function(&ctx, &vs_prolog_key);
6343 es_prolog = ctx.main_fn;
6344 }
6345
6346 /* ES main part */
6347 struct si_shader shader_es = {};
6348 shader_es.selector = es;
6349 shader_es.key.as_es = 1;
6350 shader_es.key.mono = shader->key.mono;
6351 shader_es.key.opt = shader->key.opt;
6352 si_llvm_context_set_tgsi(&ctx, &shader_es);
6353
6354 if (!si_compile_tgsi_main(&ctx, true)) {
6355 si_llvm_dispose(&ctx);
6356 return -1;
6357 }
6358 shader->info.uses_instanceid |= es->info.uses_instanceid;
6359 es_main = ctx.main_fn;
6360
6361 /* Reset the shader context. */
6362 ctx.shader = shader;
6363 ctx.type = PIPE_SHADER_GEOMETRY;
6364
6365 /* Prepare the array of shader parts. */
6366 LLVMValueRef parts[4];
6367 unsigned num_parts = 0, main_part, next_first_part;
6368
6369 if (es_prolog)
6370 parts[num_parts++] = es_prolog;
6371
6372 parts[main_part = num_parts++] = es_main;
6373 parts[next_first_part = num_parts++] = gs_prolog;
6374 parts[num_parts++] = gs_main;
6375
6376 si_build_wrapper_function(&ctx, parts, num_parts,
6377 main_part, next_first_part);
6378 } else {
6379 LLVMValueRef parts[2];
6380 union si_shader_part_key prolog_key;
6381
6382 parts[1] = ctx.main_fn;
6383
6384 memset(&prolog_key, 0, sizeof(prolog_key));
6385 prolog_key.gs_prolog.states = shader->key.part.gs.prolog;
6386 si_build_gs_prolog_function(&ctx, &prolog_key);
6387 parts[0] = ctx.main_fn;
6388
6389 si_build_wrapper_function(&ctx, parts, 2, 1, 0);
6390 }
6391 } else if (is_monolithic && ctx.type == PIPE_SHADER_FRAGMENT) {
6392 LLVMValueRef parts[3];
6393 union si_shader_part_key prolog_key;
6394 union si_shader_part_key epilog_key;
6395 bool need_prolog;
6396
6397 si_get_ps_prolog_key(shader, &prolog_key, false);
6398 need_prolog = si_need_ps_prolog(&prolog_key);
6399
6400 parts[need_prolog ? 1 : 0] = ctx.main_fn;
6401
6402 if (need_prolog) {
6403 si_build_ps_prolog_function(&ctx, &prolog_key);
6404 parts[0] = ctx.main_fn;
6405 }
6406
6407 si_get_ps_epilog_key(shader, &epilog_key);
6408 si_build_ps_epilog_function(&ctx, &epilog_key);
6409 parts[need_prolog ? 2 : 1] = ctx.main_fn;
6410
6411 si_build_wrapper_function(&ctx, parts, need_prolog ? 3 : 2,
6412 need_prolog ? 1 : 0, 0);
6413 }
6414
6415 si_llvm_optimize_module(&ctx);
6416
6417 /* Post-optimization transformations and analysis. */
6418 si_optimize_vs_outputs(&ctx);
6419
6420 if ((debug && debug->debug_message) ||
6421 r600_can_dump_shader(&sscreen->b, ctx.type))
6422 si_count_scratch_private_memory(&ctx);
6423
6424 /* Compile to bytecode. */
6425 r = si_compile_llvm(sscreen, &shader->binary, &shader->config, tm,
6426 ctx.gallivm.module, debug, ctx.type, "TGSI shader");
6427 si_llvm_dispose(&ctx);
6428 if (r) {
6429 fprintf(stderr, "LLVM failed to compile shader\n");
6430 return r;
6431 }
6432
6433 /* Validate SGPR and VGPR usage for compute to detect compiler bugs.
6434 * LLVM 3.9svn has this bug.
6435 */
6436 if (sel->type == PIPE_SHADER_COMPUTE) {
6437 unsigned wave_size = 64;
6438 unsigned max_vgprs = 256;
6439 unsigned max_sgprs = sscreen->b.chip_class >= VI ? 800 : 512;
6440 unsigned max_sgprs_per_wave = 128;
6441 unsigned max_block_threads = si_get_max_workgroup_size(shader);
6442 unsigned min_waves_per_cu = DIV_ROUND_UP(max_block_threads, wave_size);
6443 unsigned min_waves_per_simd = DIV_ROUND_UP(min_waves_per_cu, 4);
6444
6445 max_vgprs = max_vgprs / min_waves_per_simd;
6446 max_sgprs = MIN2(max_sgprs / min_waves_per_simd, max_sgprs_per_wave);
6447
6448 if (shader->config.num_sgprs > max_sgprs ||
6449 shader->config.num_vgprs > max_vgprs) {
6450 fprintf(stderr, "LLVM failed to compile a shader correctly: "
6451 "SGPR:VGPR usage is %u:%u, but the hw limit is %u:%u\n",
6452 shader->config.num_sgprs, shader->config.num_vgprs,
6453 max_sgprs, max_vgprs);
6454
6455 /* Just terminate the process, because dependent
6456 * shaders can hang due to bad input data, but use
6457 * the env var to allow shader-db to work.
6458 */
6459 if (!debug_get_bool_option("SI_PASS_BAD_SHADERS", false))
6460 abort();
6461 }
6462 }
6463
6464 /* Add the scratch offset to input SGPRs. */
6465 if (shader->config.scratch_bytes_per_wave && !is_merged_shader(shader))
6466 shader->info.num_input_sgprs += 1; /* scratch byte offset */
6467
6468 /* Calculate the number of fragment input VGPRs. */
6469 if (ctx.type == PIPE_SHADER_FRAGMENT) {
6470 shader->info.num_input_vgprs = 0;
6471 shader->info.face_vgpr_index = -1;
6472
6473 if (G_0286CC_PERSP_SAMPLE_ENA(shader->config.spi_ps_input_addr))
6474 shader->info.num_input_vgprs += 2;
6475 if (G_0286CC_PERSP_CENTER_ENA(shader->config.spi_ps_input_addr))
6476 shader->info.num_input_vgprs += 2;
6477 if (G_0286CC_PERSP_CENTROID_ENA(shader->config.spi_ps_input_addr))
6478 shader->info.num_input_vgprs += 2;
6479 if (G_0286CC_PERSP_PULL_MODEL_ENA(shader->config.spi_ps_input_addr))
6480 shader->info.num_input_vgprs += 3;
6481 if (G_0286CC_LINEAR_SAMPLE_ENA(shader->config.spi_ps_input_addr))
6482 shader->info.num_input_vgprs += 2;
6483 if (G_0286CC_LINEAR_CENTER_ENA(shader->config.spi_ps_input_addr))
6484 shader->info.num_input_vgprs += 2;
6485 if (G_0286CC_LINEAR_CENTROID_ENA(shader->config.spi_ps_input_addr))
6486 shader->info.num_input_vgprs += 2;
6487 if (G_0286CC_LINE_STIPPLE_TEX_ENA(shader->config.spi_ps_input_addr))
6488 shader->info.num_input_vgprs += 1;
6489 if (G_0286CC_POS_X_FLOAT_ENA(shader->config.spi_ps_input_addr))
6490 shader->info.num_input_vgprs += 1;
6491 if (G_0286CC_POS_Y_FLOAT_ENA(shader->config.spi_ps_input_addr))
6492 shader->info.num_input_vgprs += 1;
6493 if (G_0286CC_POS_Z_FLOAT_ENA(shader->config.spi_ps_input_addr))
6494 shader->info.num_input_vgprs += 1;
6495 if (G_0286CC_POS_W_FLOAT_ENA(shader->config.spi_ps_input_addr))
6496 shader->info.num_input_vgprs += 1;
6497 if (G_0286CC_FRONT_FACE_ENA(shader->config.spi_ps_input_addr)) {
6498 shader->info.face_vgpr_index = shader->info.num_input_vgprs;
6499 shader->info.num_input_vgprs += 1;
6500 }
6501 if (G_0286CC_ANCILLARY_ENA(shader->config.spi_ps_input_addr))
6502 shader->info.num_input_vgprs += 1;
6503 if (G_0286CC_SAMPLE_COVERAGE_ENA(shader->config.spi_ps_input_addr))
6504 shader->info.num_input_vgprs += 1;
6505 if (G_0286CC_POS_FIXED_PT_ENA(shader->config.spi_ps_input_addr))
6506 shader->info.num_input_vgprs += 1;
6507 }
6508
6509 return 0;
6510 }
6511
6512 /**
6513 * Create, compile and return a shader part (prolog or epilog).
6514 *
6515 * \param sscreen screen
6516 * \param list list of shader parts of the same category
6517 * \param type shader type
6518 * \param key shader part key
6519 * \param prolog whether the part being requested is a prolog
6520 * \param tm LLVM target machine
6521 * \param debug debug callback
6522 * \param build the callback responsible for building the main function
6523 * \return non-NULL on success
6524 */
6525 static struct si_shader_part *
6526 si_get_shader_part(struct si_screen *sscreen,
6527 struct si_shader_part **list,
6528 enum pipe_shader_type type,
6529 bool prolog,
6530 union si_shader_part_key *key,
6531 LLVMTargetMachineRef tm,
6532 struct pipe_debug_callback *debug,
6533 void (*build)(struct si_shader_context *,
6534 union si_shader_part_key *),
6535 const char *name)
6536 {
6537 struct si_shader_part *result;
6538
6539 mtx_lock(&sscreen->shader_parts_mutex);
6540
6541 /* Find existing. */
6542 for (result = *list; result; result = result->next) {
6543 if (memcmp(&result->key, key, sizeof(*key)) == 0) {
6544 mtx_unlock(&sscreen->shader_parts_mutex);
6545 return result;
6546 }
6547 }
6548
6549 /* Compile a new one. */
6550 result = CALLOC_STRUCT(si_shader_part);
6551 result->key = *key;
6552
6553 struct si_shader shader = {};
6554 struct si_shader_context ctx;
6555 struct gallivm_state *gallivm = &ctx.gallivm;
6556
6557 si_init_shader_ctx(&ctx, sscreen, tm);
6558 ctx.shader = &shader;
6559 ctx.type = type;
6560
6561 switch (type) {
6562 case PIPE_SHADER_VERTEX:
6563 break;
6564 case PIPE_SHADER_TESS_CTRL:
6565 assert(!prolog);
6566 shader.key.part.tcs.epilog = key->tcs_epilog.states;
6567 break;
6568 case PIPE_SHADER_GEOMETRY:
6569 assert(prolog);
6570 break;
6571 case PIPE_SHADER_FRAGMENT:
6572 if (prolog)
6573 shader.key.part.ps.prolog = key->ps_prolog.states;
6574 else
6575 shader.key.part.ps.epilog = key->ps_epilog.states;
6576 break;
6577 default:
6578 unreachable("bad shader part");
6579 }
6580
6581 build(&ctx, key);
6582
6583 /* Compile. */
6584 si_llvm_optimize_module(&ctx);
6585
6586 if (si_compile_llvm(sscreen, &result->binary, &result->config, tm,
6587 gallivm->module, debug, ctx.type, name)) {
6588 FREE(result);
6589 result = NULL;
6590 goto out;
6591 }
6592
6593 result->next = *list;
6594 *list = result;
6595
6596 out:
6597 si_llvm_dispose(&ctx);
6598 mtx_unlock(&sscreen->shader_parts_mutex);
6599 return result;
6600 }
6601
6602 static LLVMValueRef si_prolog_get_rw_buffers(struct si_shader_context *ctx)
6603 {
6604 struct gallivm_state *gallivm = &ctx->gallivm;
6605 LLVMValueRef ptr[2], list;
6606
6607 /* Get the pointer to rw buffers. */
6608 ptr[0] = LLVMGetParam(ctx->main_fn, SI_SGPR_RW_BUFFERS);
6609 ptr[1] = LLVMGetParam(ctx->main_fn, SI_SGPR_RW_BUFFERS_HI);
6610 list = lp_build_gather_values(gallivm, ptr, 2);
6611 list = LLVMBuildBitCast(gallivm->builder, list, ctx->i64, "");
6612 list = LLVMBuildIntToPtr(gallivm->builder, list,
6613 si_const_array(ctx->v4i32, SI_NUM_RW_BUFFERS), "");
6614 return list;
6615 }
6616
6617 /**
6618 * Build the vertex shader prolog function.
6619 *
6620 * The inputs are the same as VS (a lot of SGPRs and 4 VGPR system values).
6621 * All inputs are returned unmodified. The vertex load indices are
6622 * stored after them, which will be used by the API VS for fetching inputs.
6623 *
6624 * For example, the expected outputs for instance_divisors[] = {0, 1, 2} are:
6625 * input_v0,
6626 * input_v1,
6627 * input_v2,
6628 * input_v3,
6629 * (VertexID + BaseVertex),
6630 * (InstanceID + StartInstance),
6631 * (InstanceID / 2 + StartInstance)
6632 */
6633 static void si_build_vs_prolog_function(struct si_shader_context *ctx,
6634 union si_shader_part_key *key)
6635 {
6636 struct gallivm_state *gallivm = &ctx->gallivm;
6637 struct si_function_info fninfo;
6638 LLVMTypeRef *returns;
6639 LLVMValueRef ret, func;
6640 int num_returns, i;
6641 unsigned first_vs_vgpr = key->vs_prolog.num_input_sgprs +
6642 key->vs_prolog.num_merged_next_stage_vgprs;
6643 unsigned num_input_vgprs = key->vs_prolog.num_merged_next_stage_vgprs + 4;
6644 unsigned num_all_input_regs = key->vs_prolog.num_input_sgprs +
6645 num_input_vgprs;
6646 unsigned user_sgpr_base = key->vs_prolog.num_merged_next_stage_vgprs ? 8 : 0;
6647
6648 si_init_function_info(&fninfo);
6649
6650 /* 4 preloaded VGPRs + vertex load indices as prolog outputs */
6651 returns = alloca((num_all_input_regs + key->vs_prolog.last_input + 1) *
6652 sizeof(LLVMTypeRef));
6653 num_returns = 0;
6654
6655 /* Declare input and output SGPRs. */
6656 for (i = 0; i < key->vs_prolog.num_input_sgprs; i++) {
6657 add_arg(&fninfo, ARG_SGPR, ctx->i32);
6658 returns[num_returns++] = ctx->i32;
6659 }
6660
6661 /* Preloaded VGPRs (outputs must be floats) */
6662 for (i = 0; i < num_input_vgprs; i++) {
6663 add_arg(&fninfo, ARG_VGPR, ctx->i32);
6664 returns[num_returns++] = ctx->f32;
6665 }
6666
6667 fninfo.assign[first_vs_vgpr] = &ctx->abi.vertex_id;
6668 fninfo.assign[first_vs_vgpr + (key->vs_prolog.as_ls ? 2 : 1)] = &ctx->abi.instance_id;
6669
6670 /* Vertex load indices. */
6671 for (i = 0; i <= key->vs_prolog.last_input; i++)
6672 returns[num_returns++] = ctx->f32;
6673
6674 /* Create the function. */
6675 si_create_function(ctx, "vs_prolog", returns, num_returns, &fninfo, 0);
6676 func = ctx->main_fn;
6677
6678 if (key->vs_prolog.num_merged_next_stage_vgprs &&
6679 !key->vs_prolog.is_monolithic)
6680 si_init_exec_from_input(ctx, 3, 0);
6681
6682 /* Copy inputs to outputs. This should be no-op, as the registers match,
6683 * but it will prevent the compiler from overwriting them unintentionally.
6684 */
6685 ret = ctx->return_value;
6686 for (i = 0; i < key->vs_prolog.num_input_sgprs; i++) {
6687 LLVMValueRef p = LLVMGetParam(func, i);
6688 ret = LLVMBuildInsertValue(gallivm->builder, ret, p, i, "");
6689 }
6690 for (; i < fninfo.num_params; i++) {
6691 LLVMValueRef p = LLVMGetParam(func, i);
6692 p = LLVMBuildBitCast(gallivm->builder, p, ctx->f32, "");
6693 ret = LLVMBuildInsertValue(gallivm->builder, ret, p, i, "");
6694 }
6695
6696 /* Compute vertex load indices from instance divisors. */
6697 LLVMValueRef instance_divisor_constbuf = NULL;
6698
6699 if (key->vs_prolog.states.instance_divisor_is_fetched) {
6700 LLVMValueRef list = si_prolog_get_rw_buffers(ctx);
6701 LLVMValueRef buf_index =
6702 LLVMConstInt(ctx->i32, SI_VS_CONST_INSTANCE_DIVISORS, 0);
6703 instance_divisor_constbuf =
6704 ac_build_indexed_load_const(&ctx->ac, list, buf_index);
6705 }
6706
6707 for (i = 0; i <= key->vs_prolog.last_input; i++) {
6708 bool divisor_is_one =
6709 key->vs_prolog.states.instance_divisor_is_one & (1u << i);
6710 bool divisor_is_fetched =
6711 key->vs_prolog.states.instance_divisor_is_fetched & (1u << i);
6712 LLVMValueRef index;
6713
6714 if (divisor_is_one || divisor_is_fetched) {
6715 LLVMValueRef divisor = ctx->i32_1;
6716
6717 if (divisor_is_fetched) {
6718 divisor = buffer_load_const(ctx, instance_divisor_constbuf,
6719 LLVMConstInt(ctx->i32, i * 4, 0));
6720 divisor = LLVMBuildBitCast(gallivm->builder, divisor,
6721 ctx->i32, "");
6722 }
6723
6724 /* InstanceID / Divisor + StartInstance */
6725 index = get_instance_index_for_fetch(ctx,
6726 user_sgpr_base +
6727 SI_SGPR_START_INSTANCE,
6728 divisor);
6729 } else {
6730 /* VertexID + BaseVertex */
6731 index = LLVMBuildAdd(gallivm->builder,
6732 ctx->abi.vertex_id,
6733 LLVMGetParam(func, user_sgpr_base +
6734 SI_SGPR_BASE_VERTEX), "");
6735 }
6736
6737 index = LLVMBuildBitCast(gallivm->builder, index, ctx->f32, "");
6738 ret = LLVMBuildInsertValue(gallivm->builder, ret, index,
6739 fninfo.num_params + i, "");
6740 }
6741
6742 si_llvm_build_ret(ctx, ret);
6743 }
6744
6745 static bool si_get_vs_prolog(struct si_screen *sscreen,
6746 LLVMTargetMachineRef tm,
6747 struct si_shader *shader,
6748 struct pipe_debug_callback *debug,
6749 struct si_shader *main_part,
6750 const struct si_vs_prolog_bits *key)
6751 {
6752 struct si_shader_selector *vs = main_part->selector;
6753
6754 /* The prolog is a no-op if there are no inputs. */
6755 if (!vs->vs_needs_prolog)
6756 return true;
6757
6758 /* Get the prolog. */
6759 union si_shader_part_key prolog_key;
6760 si_get_vs_prolog_key(&vs->info, main_part->info.num_input_sgprs,
6761 key, shader, &prolog_key);
6762
6763 shader->prolog =
6764 si_get_shader_part(sscreen, &sscreen->vs_prologs,
6765 PIPE_SHADER_VERTEX, true, &prolog_key, tm,
6766 debug, si_build_vs_prolog_function,
6767 "Vertex Shader Prolog");
6768 return shader->prolog != NULL;
6769 }
6770
6771 /**
6772 * Select and compile (or reuse) vertex shader parts (prolog & epilog).
6773 */
6774 static bool si_shader_select_vs_parts(struct si_screen *sscreen,
6775 LLVMTargetMachineRef tm,
6776 struct si_shader *shader,
6777 struct pipe_debug_callback *debug)
6778 {
6779 return si_get_vs_prolog(sscreen, tm, shader, debug, shader,
6780 &shader->key.part.vs.prolog);
6781 }
6782
6783 /**
6784 * Compile the TCS epilog function. This writes tesselation factors to memory
6785 * based on the output primitive type of the tesselator (determined by TES).
6786 */
6787 static void si_build_tcs_epilog_function(struct si_shader_context *ctx,
6788 union si_shader_part_key *key)
6789 {
6790 struct gallivm_state *gallivm = &ctx->gallivm;
6791 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
6792 struct si_function_info fninfo;
6793 LLVMValueRef func;
6794
6795 si_init_function_info(&fninfo);
6796
6797 if (ctx->screen->b.chip_class >= GFX9) {
6798 add_arg(&fninfo, ARG_SGPR, ctx->i64);
6799 ctx->param_tcs_offchip_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
6800 add_arg(&fninfo, ARG_SGPR, ctx->i32); /* wave info */
6801 ctx->param_tcs_factor_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
6802 add_arg(&fninfo, ARG_SGPR, ctx->i32);
6803 add_arg(&fninfo, ARG_SGPR, ctx->i32);
6804 add_arg(&fninfo, ARG_SGPR, ctx->i32);
6805 add_arg(&fninfo, ARG_SGPR, ctx->i64);
6806 add_arg(&fninfo, ARG_SGPR, ctx->i64);
6807 add_arg(&fninfo, ARG_SGPR, ctx->i64);
6808 add_arg(&fninfo, ARG_SGPR, ctx->i64);
6809 add_arg(&fninfo, ARG_SGPR, ctx->i32);
6810 add_arg(&fninfo, ARG_SGPR, ctx->i32);
6811 add_arg(&fninfo, ARG_SGPR, ctx->i32);
6812 add_arg(&fninfo, ARG_SGPR, ctx->i32);
6813 ctx->param_tcs_offchip_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
6814 add_arg(&fninfo, ARG_SGPR, ctx->i32);
6815 add_arg(&fninfo, ARG_SGPR, ctx->i32);
6816 ctx->param_tcs_offchip_addr_base64k = add_arg(&fninfo, ARG_SGPR, ctx->i32);
6817 ctx->param_tcs_factor_addr_base64k = add_arg(&fninfo, ARG_SGPR, ctx->i32);
6818 } else {
6819 add_arg(&fninfo, ARG_SGPR, ctx->i64);
6820 add_arg(&fninfo, ARG_SGPR, ctx->i64);
6821 add_arg(&fninfo, ARG_SGPR, ctx->i64);
6822 ctx->param_tcs_offchip_layout = add_arg(&fninfo, ARG_SGPR, ctx->i32);
6823 add_arg(&fninfo, ARG_SGPR, ctx->i32);
6824 add_arg(&fninfo, ARG_SGPR, ctx->i32);
6825 add_arg(&fninfo, ARG_SGPR, ctx->i32);
6826 ctx->param_tcs_offchip_addr_base64k = add_arg(&fninfo, ARG_SGPR, ctx->i32);
6827 ctx->param_tcs_factor_addr_base64k = add_arg(&fninfo, ARG_SGPR, ctx->i32);
6828 ctx->param_tcs_offchip_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
6829 ctx->param_tcs_factor_offset = add_arg(&fninfo, ARG_SGPR, ctx->i32);
6830 }
6831
6832 add_arg(&fninfo, ARG_VGPR, ctx->i32); /* VGPR gap */
6833 add_arg(&fninfo, ARG_VGPR, ctx->i32); /* VGPR gap */
6834 unsigned tess_factors_idx =
6835 add_arg(&fninfo, ARG_VGPR, ctx->i32); /* patch index within the wave (REL_PATCH_ID) */
6836 add_arg(&fninfo, ARG_VGPR, ctx->i32); /* invocation ID within the patch */
6837 add_arg(&fninfo, ARG_VGPR, ctx->i32); /* LDS offset where tess factors should be loaded from */
6838
6839 /* Create the function. */
6840 si_create_function(ctx, "tcs_epilog", NULL, 0, &fninfo,
6841 ctx->screen->b.chip_class >= CIK ? 128 : 64);
6842 declare_lds_as_pointer(ctx);
6843 func = ctx->main_fn;
6844
6845 si_write_tess_factors(bld_base,
6846 LLVMGetParam(func, tess_factors_idx),
6847 LLVMGetParam(func, tess_factors_idx + 1),
6848 LLVMGetParam(func, tess_factors_idx + 2));
6849
6850 LLVMBuildRetVoid(gallivm->builder);
6851 }
6852
6853 /**
6854 * Select and compile (or reuse) TCS parts (epilog).
6855 */
6856 static bool si_shader_select_tcs_parts(struct si_screen *sscreen,
6857 LLVMTargetMachineRef tm,
6858 struct si_shader *shader,
6859 struct pipe_debug_callback *debug)
6860 {
6861 if (sscreen->b.chip_class >= GFX9) {
6862 struct si_shader *ls_main_part =
6863 shader->key.part.tcs.ls->main_shader_part_ls;
6864
6865 if (!si_get_vs_prolog(sscreen, tm, shader, debug, ls_main_part,
6866 &shader->key.part.tcs.ls_prolog))
6867 return false;
6868
6869 shader->previous_stage = ls_main_part;
6870 }
6871
6872 /* Get the epilog. */
6873 union si_shader_part_key epilog_key;
6874 memset(&epilog_key, 0, sizeof(epilog_key));
6875 epilog_key.tcs_epilog.states = shader->key.part.tcs.epilog;
6876
6877 shader->epilog = si_get_shader_part(sscreen, &sscreen->tcs_epilogs,
6878 PIPE_SHADER_TESS_CTRL, false,
6879 &epilog_key, tm, debug,
6880 si_build_tcs_epilog_function,
6881 "Tessellation Control Shader Epilog");
6882 return shader->epilog != NULL;
6883 }
6884
6885 /**
6886 * Select and compile (or reuse) GS parts (prolog).
6887 */
6888 static bool si_shader_select_gs_parts(struct si_screen *sscreen,
6889 LLVMTargetMachineRef tm,
6890 struct si_shader *shader,
6891 struct pipe_debug_callback *debug)
6892 {
6893 if (sscreen->b.chip_class >= GFX9) {
6894 struct si_shader *es_main_part =
6895 shader->key.part.gs.es->main_shader_part_es;
6896
6897 if (shader->key.part.gs.es->type == PIPE_SHADER_VERTEX &&
6898 !si_get_vs_prolog(sscreen, tm, shader, debug, es_main_part,
6899 &shader->key.part.gs.vs_prolog))
6900 return false;
6901
6902 shader->previous_stage = es_main_part;
6903 }
6904
6905 if (!shader->key.part.gs.prolog.tri_strip_adj_fix)
6906 return true;
6907
6908 union si_shader_part_key prolog_key;
6909 memset(&prolog_key, 0, sizeof(prolog_key));
6910 prolog_key.gs_prolog.states = shader->key.part.gs.prolog;
6911
6912 shader->prolog2 = si_get_shader_part(sscreen, &sscreen->gs_prologs,
6913 PIPE_SHADER_GEOMETRY, true,
6914 &prolog_key, tm, debug,
6915 si_build_gs_prolog_function,
6916 "Geometry Shader Prolog");
6917 return shader->prolog2 != NULL;
6918 }
6919
6920 /**
6921 * Build the pixel shader prolog function. This handles:
6922 * - two-side color selection and interpolation
6923 * - overriding interpolation parameters for the API PS
6924 * - polygon stippling
6925 *
6926 * All preloaded SGPRs and VGPRs are passed through unmodified unless they are
6927 * overriden by other states. (e.g. per-sample interpolation)
6928 * Interpolated colors are stored after the preloaded VGPRs.
6929 */
6930 static void si_build_ps_prolog_function(struct si_shader_context *ctx,
6931 union si_shader_part_key *key)
6932 {
6933 struct gallivm_state *gallivm = &ctx->gallivm;
6934 struct si_function_info fninfo;
6935 LLVMValueRef ret, func;
6936 int num_returns, i, num_color_channels;
6937
6938 assert(si_need_ps_prolog(key));
6939
6940 si_init_function_info(&fninfo);
6941
6942 /* Declare inputs. */
6943 for (i = 0; i < key->ps_prolog.num_input_sgprs; i++)
6944 add_arg(&fninfo, ARG_SGPR, ctx->i32);
6945
6946 for (i = 0; i < key->ps_prolog.num_input_vgprs; i++)
6947 add_arg(&fninfo, ARG_VGPR, ctx->f32);
6948
6949 /* Declare outputs (same as inputs + add colors if needed) */
6950 num_returns = fninfo.num_params;
6951 num_color_channels = util_bitcount(key->ps_prolog.colors_read);
6952 for (i = 0; i < num_color_channels; i++)
6953 fninfo.types[num_returns++] = ctx->f32;
6954
6955 /* Create the function. */
6956 si_create_function(ctx, "ps_prolog", fninfo.types, num_returns,
6957 &fninfo, 0);
6958 func = ctx->main_fn;
6959
6960 /* Copy inputs to outputs. This should be no-op, as the registers match,
6961 * but it will prevent the compiler from overwriting them unintentionally.
6962 */
6963 ret = ctx->return_value;
6964 for (i = 0; i < fninfo.num_params; i++) {
6965 LLVMValueRef p = LLVMGetParam(func, i);
6966 ret = LLVMBuildInsertValue(gallivm->builder, ret, p, i, "");
6967 }
6968
6969 /* Polygon stippling. */
6970 if (key->ps_prolog.states.poly_stipple) {
6971 /* POS_FIXED_PT is always last. */
6972 unsigned pos = key->ps_prolog.num_input_sgprs +
6973 key->ps_prolog.num_input_vgprs - 1;
6974 LLVMValueRef list = si_prolog_get_rw_buffers(ctx);
6975
6976 si_llvm_emit_polygon_stipple(ctx, list, pos);
6977 }
6978
6979 if (key->ps_prolog.states.bc_optimize_for_persp ||
6980 key->ps_prolog.states.bc_optimize_for_linear) {
6981 unsigned i, base = key->ps_prolog.num_input_sgprs;
6982 LLVMValueRef center[2], centroid[2], tmp, bc_optimize;
6983
6984 /* The shader should do: if (PRIM_MASK[31]) CENTROID = CENTER;
6985 * The hw doesn't compute CENTROID if the whole wave only
6986 * contains fully-covered quads.
6987 *
6988 * PRIM_MASK is after user SGPRs.
6989 */
6990 bc_optimize = LLVMGetParam(func, SI_PS_NUM_USER_SGPR);
6991 bc_optimize = LLVMBuildLShr(gallivm->builder, bc_optimize,
6992 LLVMConstInt(ctx->i32, 31, 0), "");
6993 bc_optimize = LLVMBuildTrunc(gallivm->builder, bc_optimize,
6994 ctx->i1, "");
6995
6996 if (key->ps_prolog.states.bc_optimize_for_persp) {
6997 /* Read PERSP_CENTER. */
6998 for (i = 0; i < 2; i++)
6999 center[i] = LLVMGetParam(func, base + 2 + i);
7000 /* Read PERSP_CENTROID. */
7001 for (i = 0; i < 2; i++)
7002 centroid[i] = LLVMGetParam(func, base + 4 + i);
7003 /* Select PERSP_CENTROID. */
7004 for (i = 0; i < 2; i++) {
7005 tmp = LLVMBuildSelect(gallivm->builder, bc_optimize,
7006 center[i], centroid[i], "");
7007 ret = LLVMBuildInsertValue(gallivm->builder, ret,
7008 tmp, base + 4 + i, "");
7009 }
7010 }
7011 if (key->ps_prolog.states.bc_optimize_for_linear) {
7012 /* Read LINEAR_CENTER. */
7013 for (i = 0; i < 2; i++)
7014 center[i] = LLVMGetParam(func, base + 8 + i);
7015 /* Read LINEAR_CENTROID. */
7016 for (i = 0; i < 2; i++)
7017 centroid[i] = LLVMGetParam(func, base + 10 + i);
7018 /* Select LINEAR_CENTROID. */
7019 for (i = 0; i < 2; i++) {
7020 tmp = LLVMBuildSelect(gallivm->builder, bc_optimize,
7021 center[i], centroid[i], "");
7022 ret = LLVMBuildInsertValue(gallivm->builder, ret,
7023 tmp, base + 10 + i, "");
7024 }
7025 }
7026 }
7027
7028 /* Force per-sample interpolation. */
7029 if (key->ps_prolog.states.force_persp_sample_interp) {
7030 unsigned i, base = key->ps_prolog.num_input_sgprs;
7031 LLVMValueRef persp_sample[2];
7032
7033 /* Read PERSP_SAMPLE. */
7034 for (i = 0; i < 2; i++)
7035 persp_sample[i] = LLVMGetParam(func, base + i);
7036 /* Overwrite PERSP_CENTER. */
7037 for (i = 0; i < 2; i++)
7038 ret = LLVMBuildInsertValue(gallivm->builder, ret,
7039 persp_sample[i], base + 2 + i, "");
7040 /* Overwrite PERSP_CENTROID. */
7041 for (i = 0; i < 2; i++)
7042 ret = LLVMBuildInsertValue(gallivm->builder, ret,
7043 persp_sample[i], base + 4 + i, "");
7044 }
7045 if (key->ps_prolog.states.force_linear_sample_interp) {
7046 unsigned i, base = key->ps_prolog.num_input_sgprs;
7047 LLVMValueRef linear_sample[2];
7048
7049 /* Read LINEAR_SAMPLE. */
7050 for (i = 0; i < 2; i++)
7051 linear_sample[i] = LLVMGetParam(func, base + 6 + i);
7052 /* Overwrite LINEAR_CENTER. */
7053 for (i = 0; i < 2; i++)
7054 ret = LLVMBuildInsertValue(gallivm->builder, ret,
7055 linear_sample[i], base + 8 + i, "");
7056 /* Overwrite LINEAR_CENTROID. */
7057 for (i = 0; i < 2; i++)
7058 ret = LLVMBuildInsertValue(gallivm->builder, ret,
7059 linear_sample[i], base + 10 + i, "");
7060 }
7061
7062 /* Force center interpolation. */
7063 if (key->ps_prolog.states.force_persp_center_interp) {
7064 unsigned i, base = key->ps_prolog.num_input_sgprs;
7065 LLVMValueRef persp_center[2];
7066
7067 /* Read PERSP_CENTER. */
7068 for (i = 0; i < 2; i++)
7069 persp_center[i] = LLVMGetParam(func, base + 2 + i);
7070 /* Overwrite PERSP_SAMPLE. */
7071 for (i = 0; i < 2; i++)
7072 ret = LLVMBuildInsertValue(gallivm->builder, ret,
7073 persp_center[i], base + i, "");
7074 /* Overwrite PERSP_CENTROID. */
7075 for (i = 0; i < 2; i++)
7076 ret = LLVMBuildInsertValue(gallivm->builder, ret,
7077 persp_center[i], base + 4 + i, "");
7078 }
7079 if (key->ps_prolog.states.force_linear_center_interp) {
7080 unsigned i, base = key->ps_prolog.num_input_sgprs;
7081 LLVMValueRef linear_center[2];
7082
7083 /* Read LINEAR_CENTER. */
7084 for (i = 0; i < 2; i++)
7085 linear_center[i] = LLVMGetParam(func, base + 8 + i);
7086 /* Overwrite LINEAR_SAMPLE. */
7087 for (i = 0; i < 2; i++)
7088 ret = LLVMBuildInsertValue(gallivm->builder, ret,
7089 linear_center[i], base + 6 + i, "");
7090 /* Overwrite LINEAR_CENTROID. */
7091 for (i = 0; i < 2; i++)
7092 ret = LLVMBuildInsertValue(gallivm->builder, ret,
7093 linear_center[i], base + 10 + i, "");
7094 }
7095
7096 /* Interpolate colors. */
7097 unsigned color_out_idx = 0;
7098 for (i = 0; i < 2; i++) {
7099 unsigned writemask = (key->ps_prolog.colors_read >> (i * 4)) & 0xf;
7100 unsigned face_vgpr = key->ps_prolog.num_input_sgprs +
7101 key->ps_prolog.face_vgpr_index;
7102 LLVMValueRef interp[2], color[4];
7103 LLVMValueRef interp_ij = NULL, prim_mask = NULL, face = NULL;
7104
7105 if (!writemask)
7106 continue;
7107
7108 /* If the interpolation qualifier is not CONSTANT (-1). */
7109 if (key->ps_prolog.color_interp_vgpr_index[i] != -1) {
7110 unsigned interp_vgpr = key->ps_prolog.num_input_sgprs +
7111 key->ps_prolog.color_interp_vgpr_index[i];
7112
7113 /* Get the (i,j) updated by bc_optimize handling. */
7114 interp[0] = LLVMBuildExtractValue(gallivm->builder, ret,
7115 interp_vgpr, "");
7116 interp[1] = LLVMBuildExtractValue(gallivm->builder, ret,
7117 interp_vgpr + 1, "");
7118 interp_ij = lp_build_gather_values(gallivm, interp, 2);
7119 }
7120
7121 /* Use the absolute location of the input. */
7122 prim_mask = LLVMGetParam(func, SI_PS_NUM_USER_SGPR);
7123
7124 if (key->ps_prolog.states.color_two_side) {
7125 face = LLVMGetParam(func, face_vgpr);
7126 face = LLVMBuildBitCast(gallivm->builder, face, ctx->i32, "");
7127 }
7128
7129 interp_fs_input(ctx,
7130 key->ps_prolog.color_attr_index[i],
7131 TGSI_SEMANTIC_COLOR, i,
7132 key->ps_prolog.num_interp_inputs,
7133 key->ps_prolog.colors_read, interp_ij,
7134 prim_mask, face, color);
7135
7136 while (writemask) {
7137 unsigned chan = u_bit_scan(&writemask);
7138 ret = LLVMBuildInsertValue(gallivm->builder, ret, color[chan],
7139 fninfo.num_params + color_out_idx++, "");
7140 }
7141 }
7142
7143 /* Tell LLVM to insert WQM instruction sequence when needed. */
7144 if (key->ps_prolog.wqm) {
7145 LLVMAddTargetDependentFunctionAttr(func,
7146 "amdgpu-ps-wqm-outputs", "");
7147 }
7148
7149 si_llvm_build_ret(ctx, ret);
7150 }
7151
7152 /**
7153 * Build the pixel shader epilog function. This handles everything that must be
7154 * emulated for pixel shader exports. (alpha-test, format conversions, etc)
7155 */
7156 static void si_build_ps_epilog_function(struct si_shader_context *ctx,
7157 union si_shader_part_key *key)
7158 {
7159 struct gallivm_state *gallivm = &ctx->gallivm;
7160 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
7161 struct si_function_info fninfo;
7162 LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
7163 int i;
7164 struct si_ps_exports exp = {};
7165
7166 si_init_function_info(&fninfo);
7167
7168 /* Declare input SGPRs. */
7169 ctx->param_rw_buffers = add_arg(&fninfo, ARG_SGPR, ctx->i64);
7170 ctx->param_const_and_shader_buffers = add_arg(&fninfo, ARG_SGPR, ctx->i64);
7171 ctx->param_samplers_and_images = add_arg(&fninfo, ARG_SGPR, ctx->i64);
7172 add_arg_checked(&fninfo, ARG_SGPR, ctx->f32, SI_PARAM_ALPHA_REF);
7173
7174 /* Declare input VGPRs. */
7175 unsigned required_num_params =
7176 fninfo.num_sgpr_params +
7177 util_bitcount(key->ps_epilog.colors_written) * 4 +
7178 key->ps_epilog.writes_z +
7179 key->ps_epilog.writes_stencil +
7180 key->ps_epilog.writes_samplemask;
7181
7182 required_num_params = MAX2(required_num_params,
7183 fninfo.num_sgpr_params + PS_EPILOG_SAMPLEMASK_MIN_LOC + 1);
7184
7185 while (fninfo.num_params < required_num_params)
7186 add_arg(&fninfo, ARG_VGPR, ctx->f32);
7187
7188 /* Create the function. */
7189 si_create_function(ctx, "ps_epilog", NULL, 0, &fninfo, 0);
7190 /* Disable elimination of unused inputs. */
7191 si_llvm_add_attribute(ctx->main_fn,
7192 "InitialPSInputAddr", 0xffffff);
7193
7194 /* Process colors. */
7195 unsigned vgpr = fninfo.num_sgpr_params;
7196 unsigned colors_written = key->ps_epilog.colors_written;
7197 int last_color_export = -1;
7198
7199 /* Find the last color export. */
7200 if (!key->ps_epilog.writes_z &&
7201 !key->ps_epilog.writes_stencil &&
7202 !key->ps_epilog.writes_samplemask) {
7203 unsigned spi_format = key->ps_epilog.states.spi_shader_col_format;
7204
7205 /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
7206 if (colors_written == 0x1 && key->ps_epilog.states.last_cbuf > 0) {
7207 /* Just set this if any of the colorbuffers are enabled. */
7208 if (spi_format &
7209 ((1ull << (4 * (key->ps_epilog.states.last_cbuf + 1))) - 1))
7210 last_color_export = 0;
7211 } else {
7212 for (i = 0; i < 8; i++)
7213 if (colors_written & (1 << i) &&
7214 (spi_format >> (i * 4)) & 0xf)
7215 last_color_export = i;
7216 }
7217 }
7218
7219 while (colors_written) {
7220 LLVMValueRef color[4];
7221 int mrt = u_bit_scan(&colors_written);
7222
7223 for (i = 0; i < 4; i++)
7224 color[i] = LLVMGetParam(ctx->main_fn, vgpr++);
7225
7226 si_export_mrt_color(bld_base, color, mrt,
7227 fninfo.num_params - 1,
7228 mrt == last_color_export, &exp);
7229 }
7230
7231 /* Process depth, stencil, samplemask. */
7232 if (key->ps_epilog.writes_z)
7233 depth = LLVMGetParam(ctx->main_fn, vgpr++);
7234 if (key->ps_epilog.writes_stencil)
7235 stencil = LLVMGetParam(ctx->main_fn, vgpr++);
7236 if (key->ps_epilog.writes_samplemask)
7237 samplemask = LLVMGetParam(ctx->main_fn, vgpr++);
7238
7239 if (depth || stencil || samplemask)
7240 si_export_mrt_z(bld_base, depth, stencil, samplemask, &exp);
7241 else if (last_color_export == -1)
7242 si_export_null(bld_base);
7243
7244 if (exp.num)
7245 si_emit_ps_exports(ctx, &exp);
7246
7247 /* Compile. */
7248 LLVMBuildRetVoid(gallivm->builder);
7249 }
7250
7251 /**
7252 * Select and compile (or reuse) pixel shader parts (prolog & epilog).
7253 */
7254 static bool si_shader_select_ps_parts(struct si_screen *sscreen,
7255 LLVMTargetMachineRef tm,
7256 struct si_shader *shader,
7257 struct pipe_debug_callback *debug)
7258 {
7259 union si_shader_part_key prolog_key;
7260 union si_shader_part_key epilog_key;
7261
7262 /* Get the prolog. */
7263 si_get_ps_prolog_key(shader, &prolog_key, true);
7264
7265 /* The prolog is a no-op if these aren't set. */
7266 if (si_need_ps_prolog(&prolog_key)) {
7267 shader->prolog =
7268 si_get_shader_part(sscreen, &sscreen->ps_prologs,
7269 PIPE_SHADER_FRAGMENT, true,
7270 &prolog_key, tm, debug,
7271 si_build_ps_prolog_function,
7272 "Fragment Shader Prolog");
7273 if (!shader->prolog)
7274 return false;
7275 }
7276
7277 /* Get the epilog. */
7278 si_get_ps_epilog_key(shader, &epilog_key);
7279
7280 shader->epilog =
7281 si_get_shader_part(sscreen, &sscreen->ps_epilogs,
7282 PIPE_SHADER_FRAGMENT, false,
7283 &epilog_key, tm, debug,
7284 si_build_ps_epilog_function,
7285 "Fragment Shader Epilog");
7286 if (!shader->epilog)
7287 return false;
7288
7289 /* Enable POS_FIXED_PT if polygon stippling is enabled. */
7290 if (shader->key.part.ps.prolog.poly_stipple) {
7291 shader->config.spi_ps_input_ena |= S_0286CC_POS_FIXED_PT_ENA(1);
7292 assert(G_0286CC_POS_FIXED_PT_ENA(shader->config.spi_ps_input_addr));
7293 }
7294
7295 /* Set up the enable bits for per-sample shading if needed. */
7296 if (shader->key.part.ps.prolog.force_persp_sample_interp &&
7297 (G_0286CC_PERSP_CENTER_ENA(shader->config.spi_ps_input_ena) ||
7298 G_0286CC_PERSP_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
7299 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_CENTER_ENA;
7300 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_CENTROID_ENA;
7301 shader->config.spi_ps_input_ena |= S_0286CC_PERSP_SAMPLE_ENA(1);
7302 }
7303 if (shader->key.part.ps.prolog.force_linear_sample_interp &&
7304 (G_0286CC_LINEAR_CENTER_ENA(shader->config.spi_ps_input_ena) ||
7305 G_0286CC_LINEAR_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
7306 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_CENTER_ENA;
7307 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_CENTROID_ENA;
7308 shader->config.spi_ps_input_ena |= S_0286CC_LINEAR_SAMPLE_ENA(1);
7309 }
7310 if (shader->key.part.ps.prolog.force_persp_center_interp &&
7311 (G_0286CC_PERSP_SAMPLE_ENA(shader->config.spi_ps_input_ena) ||
7312 G_0286CC_PERSP_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
7313 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_SAMPLE_ENA;
7314 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_CENTROID_ENA;
7315 shader->config.spi_ps_input_ena |= S_0286CC_PERSP_CENTER_ENA(1);
7316 }
7317 if (shader->key.part.ps.prolog.force_linear_center_interp &&
7318 (G_0286CC_LINEAR_SAMPLE_ENA(shader->config.spi_ps_input_ena) ||
7319 G_0286CC_LINEAR_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
7320 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_SAMPLE_ENA;
7321 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_CENTROID_ENA;
7322 shader->config.spi_ps_input_ena |= S_0286CC_LINEAR_CENTER_ENA(1);
7323 }
7324
7325 /* POW_W_FLOAT requires that one of the perspective weights is enabled. */
7326 if (G_0286CC_POS_W_FLOAT_ENA(shader->config.spi_ps_input_ena) &&
7327 !(shader->config.spi_ps_input_ena & 0xf)) {
7328 shader->config.spi_ps_input_ena |= S_0286CC_PERSP_CENTER_ENA(1);
7329 assert(G_0286CC_PERSP_CENTER_ENA(shader->config.spi_ps_input_addr));
7330 }
7331
7332 /* At least one pair of interpolation weights must be enabled. */
7333 if (!(shader->config.spi_ps_input_ena & 0x7f)) {
7334 shader->config.spi_ps_input_ena |= S_0286CC_LINEAR_CENTER_ENA(1);
7335 assert(G_0286CC_LINEAR_CENTER_ENA(shader->config.spi_ps_input_addr));
7336 }
7337
7338 /* The sample mask input is always enabled, because the API shader always
7339 * passes it through to the epilog. Disable it here if it's unused.
7340 */
7341 if (!shader->key.part.ps.epilog.poly_line_smoothing &&
7342 !shader->selector->info.reads_samplemask)
7343 shader->config.spi_ps_input_ena &= C_0286CC_SAMPLE_COVERAGE_ENA;
7344
7345 return true;
7346 }
7347
7348 void si_multiwave_lds_size_workaround(struct si_screen *sscreen,
7349 unsigned *lds_size)
7350 {
7351 /* SPI barrier management bug:
7352 * Make sure we have at least 4k of LDS in use to avoid the bug.
7353 * It applies to workgroup sizes of more than one wavefront.
7354 */
7355 if (sscreen->b.family == CHIP_BONAIRE ||
7356 sscreen->b.family == CHIP_KABINI ||
7357 sscreen->b.family == CHIP_MULLINS)
7358 *lds_size = MAX2(*lds_size, 8);
7359 }
7360
7361 static void si_fix_resource_usage(struct si_screen *sscreen,
7362 struct si_shader *shader)
7363 {
7364 unsigned min_sgprs = shader->info.num_input_sgprs + 2; /* VCC */
7365
7366 shader->config.num_sgprs = MAX2(shader->config.num_sgprs, min_sgprs);
7367
7368 if (shader->selector->type == PIPE_SHADER_COMPUTE &&
7369 si_get_max_workgroup_size(shader) > 64) {
7370 si_multiwave_lds_size_workaround(sscreen,
7371 &shader->config.lds_size);
7372 }
7373 }
7374
7375 int si_shader_create(struct si_screen *sscreen, LLVMTargetMachineRef tm,
7376 struct si_shader *shader,
7377 struct pipe_debug_callback *debug)
7378 {
7379 struct si_shader_selector *sel = shader->selector;
7380 struct si_shader *mainp = *si_get_main_shader_part(sel, &shader->key);
7381 int r;
7382
7383 /* LS, ES, VS are compiled on demand if the main part hasn't been
7384 * compiled for that stage.
7385 *
7386 * Vertex shaders are compiled on demand when a vertex fetch
7387 * workaround must be applied.
7388 */
7389 if (shader->is_monolithic) {
7390 /* Monolithic shader (compiled as a whole, has many variants,
7391 * may take a long time to compile).
7392 */
7393 r = si_compile_tgsi_shader(sscreen, tm, shader, true, debug);
7394 if (r)
7395 return r;
7396 } else {
7397 /* The shader consists of several parts:
7398 *
7399 * - the middle part is the user shader, it has 1 variant only
7400 * and it was compiled during the creation of the shader
7401 * selector
7402 * - the prolog part is inserted at the beginning
7403 * - the epilog part is inserted at the end
7404 *
7405 * The prolog and epilog have many (but simple) variants.
7406 *
7407 * Starting with gfx9, geometry and tessellation control
7408 * shaders also contain the prolog and user shader parts of
7409 * the previous shader stage.
7410 */
7411
7412 if (!mainp)
7413 return -1;
7414
7415 /* Copy the compiled TGSI shader data over. */
7416 shader->is_binary_shared = true;
7417 shader->binary = mainp->binary;
7418 shader->config = mainp->config;
7419 shader->info.num_input_sgprs = mainp->info.num_input_sgprs;
7420 shader->info.num_input_vgprs = mainp->info.num_input_vgprs;
7421 shader->info.face_vgpr_index = mainp->info.face_vgpr_index;
7422 memcpy(shader->info.vs_output_param_offset,
7423 mainp->info.vs_output_param_offset,
7424 sizeof(mainp->info.vs_output_param_offset));
7425 shader->info.uses_instanceid = mainp->info.uses_instanceid;
7426 shader->info.nr_pos_exports = mainp->info.nr_pos_exports;
7427 shader->info.nr_param_exports = mainp->info.nr_param_exports;
7428
7429 /* Select prologs and/or epilogs. */
7430 switch (sel->type) {
7431 case PIPE_SHADER_VERTEX:
7432 if (!si_shader_select_vs_parts(sscreen, tm, shader, debug))
7433 return -1;
7434 break;
7435 case PIPE_SHADER_TESS_CTRL:
7436 if (!si_shader_select_tcs_parts(sscreen, tm, shader, debug))
7437 return -1;
7438 break;
7439 case PIPE_SHADER_TESS_EVAL:
7440 break;
7441 case PIPE_SHADER_GEOMETRY:
7442 if (!si_shader_select_gs_parts(sscreen, tm, shader, debug))
7443 return -1;
7444 break;
7445 case PIPE_SHADER_FRAGMENT:
7446 if (!si_shader_select_ps_parts(sscreen, tm, shader, debug))
7447 return -1;
7448
7449 /* Make sure we have at least as many VGPRs as there
7450 * are allocated inputs.
7451 */
7452 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
7453 shader->info.num_input_vgprs);
7454 break;
7455 }
7456
7457 /* Update SGPR and VGPR counts. */
7458 if (shader->prolog) {
7459 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
7460 shader->prolog->config.num_sgprs);
7461 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
7462 shader->prolog->config.num_vgprs);
7463 }
7464 if (shader->previous_stage) {
7465 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
7466 shader->previous_stage->config.num_sgprs);
7467 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
7468 shader->previous_stage->config.num_vgprs);
7469 shader->config.spilled_sgprs =
7470 MAX2(shader->config.spilled_sgprs,
7471 shader->previous_stage->config.spilled_sgprs);
7472 shader->config.spilled_vgprs =
7473 MAX2(shader->config.spilled_vgprs,
7474 shader->previous_stage->config.spilled_vgprs);
7475 shader->config.private_mem_vgprs =
7476 MAX2(shader->config.private_mem_vgprs,
7477 shader->previous_stage->config.private_mem_vgprs);
7478 shader->config.scratch_bytes_per_wave =
7479 MAX2(shader->config.scratch_bytes_per_wave,
7480 shader->previous_stage->config.scratch_bytes_per_wave);
7481 shader->info.uses_instanceid |=
7482 shader->previous_stage->info.uses_instanceid;
7483 }
7484 if (shader->prolog2) {
7485 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
7486 shader->prolog2->config.num_sgprs);
7487 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
7488 shader->prolog2->config.num_vgprs);
7489 }
7490 if (shader->epilog) {
7491 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
7492 shader->epilog->config.num_sgprs);
7493 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
7494 shader->epilog->config.num_vgprs);
7495 }
7496 }
7497
7498 si_fix_resource_usage(sscreen, shader);
7499 si_shader_dump(sscreen, shader, debug, sel->info.processor,
7500 stderr, true);
7501
7502 /* Upload. */
7503 r = si_shader_binary_upload(sscreen, shader);
7504 if (r) {
7505 fprintf(stderr, "LLVM failed to upload shader\n");
7506 return r;
7507 }
7508
7509 return 0;
7510 }
7511
7512 void si_shader_destroy(struct si_shader *shader)
7513 {
7514 if (shader->scratch_bo)
7515 r600_resource_reference(&shader->scratch_bo, NULL);
7516
7517 r600_resource_reference(&shader->bo, NULL);
7518
7519 if (!shader->is_binary_shared)
7520 radeon_shader_binary_clean(&shader->binary);
7521
7522 free(shader->shader_log);
7523 }