projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
radeonsi: rename readonly_memory -> can_speculate
[mesa.git] / src / gallium / drivers / radeonsi / si_shader.c
1 /*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 *
23 * Authors:
24 * Tom Stellard <thomas.stellard@amd.com>
25 * Michel Dänzer <michel.daenzer@amd.com>
26 * Christian König <christian.koenig@amd.com>
27 */
28
29 #include "gallivm/lp_bld_const.h"
30 #include "gallivm/lp_bld_gather.h"
31 #include "gallivm/lp_bld_intr.h"
32 #include "gallivm/lp_bld_logic.h"
33 #include "gallivm/lp_bld_arit.h"
34 #include "gallivm/lp_bld_flow.h"
35 #include "gallivm/lp_bld_misc.h"
36 #include "util/u_memory.h"
37 #include "util/u_string.h"
38 #include "tgsi/tgsi_build.h"
39 #include "tgsi/tgsi_util.h"
40 #include "tgsi/tgsi_dump.h"
41
42 #include "ac_binary.h"
43 #include "ac_llvm_util.h"
44 #include "ac_exp_param.h"
45 #include "si_shader_internal.h"
46 #include "si_pipe.h"
47 #include "sid.h"
48
49
50 static const char *scratch_rsrc_dword0_symbol =
51 "SCRATCH_RSRC_DWORD0";
52
53 static const char *scratch_rsrc_dword1_symbol =
54 "SCRATCH_RSRC_DWORD1";
55
56 struct si_shader_output_values
57 {
58 LLVMValueRef values[4];
59 unsigned semantic_name;
60 unsigned semantic_index;
61 ubyte vertex_stream[4];
62 };
63
64 static void si_init_shader_ctx(struct si_shader_context *ctx,
65 struct si_screen *sscreen,
66 LLVMTargetMachineRef tm);
67
68 static void si_llvm_emit_barrier(const struct lp_build_tgsi_action *action,
69 struct lp_build_tgsi_context *bld_base,
70 struct lp_build_emit_data *emit_data);
71
72 static void si_dump_shader_key(unsigned processor, const struct si_shader *shader,
73 FILE *f);
74
75 static unsigned llvm_get_type_size(LLVMTypeRef type);
76
77 static void si_build_vs_prolog_function(struct si_shader_context *ctx,
78 union si_shader_part_key *key);
79 static void si_build_tcs_epilog_function(struct si_shader_context *ctx,
80 union si_shader_part_key *key);
81 static void si_build_ps_prolog_function(struct si_shader_context *ctx,
82 union si_shader_part_key *key);
83 static void si_build_ps_epilog_function(struct si_shader_context *ctx,
84 union si_shader_part_key *key);
85
86 /* Ideally pass the sample mask input to the PS epilog as v13, which
87 * is its usual location, so that the shader doesn't have to add v_mov.
88 */
89 #define PS_EPILOG_SAMPLEMASK_MIN_LOC 13
90
91 enum {
92 CONST_ADDR_SPACE = 2,
93 LOCAL_ADDR_SPACE = 3,
94 };
95
96 static bool is_merged_shader(struct si_shader *shader)
97 {
98 if (shader->selector->screen->b.chip_class <= VI)
99 return false;
100
101 return shader->key.as_ls ||
102 shader->key.as_es ||
103 shader->selector->type == PIPE_SHADER_TESS_CTRL ||
104 shader->selector->type == PIPE_SHADER_GEOMETRY;
105 }
106
107 /**
108 * Returns a unique index for a per-patch semantic name and index. The index
109 * must be less than 32, so that a 32-bit bitmask of used inputs or outputs
110 * can be calculated.
111 */
112 unsigned si_shader_io_get_unique_index_patch(unsigned semantic_name, unsigned index)
113 {
114 switch (semantic_name) {
115 case TGSI_SEMANTIC_TESSOUTER:
116 return 0;
117 case TGSI_SEMANTIC_TESSINNER:
118 return 1;
119 case TGSI_SEMANTIC_PATCH:
120 assert(index < 30);
121 return 2 + index;
122
123 default:
124 assert(!"invalid semantic name");
125 return 0;
126 }
127 }
128
129 /**
130 * Returns a unique index for a semantic name and index. The index must be
131 * less than 64, so that a 64-bit bitmask of used inputs or outputs can be
132 * calculated.
133 */
134 unsigned si_shader_io_get_unique_index(unsigned semantic_name, unsigned index)
135 {
136 switch (semantic_name) {
137 case TGSI_SEMANTIC_POSITION:
138 return 0;
139 case TGSI_SEMANTIC_PSIZE:
140 return 1;
141 case TGSI_SEMANTIC_CLIPDIST:
142 assert(index <= 1);
143 return 2 + index;
144 case TGSI_SEMANTIC_GENERIC:
145 if (index < SI_MAX_IO_GENERIC)
146 return 4 + index;
147
148 assert(!"invalid generic index");
149 return 0;
150
151 case TGSI_SEMANTIC_FOG:
152 return SI_MAX_IO_GENERIC + 4;
153 case TGSI_SEMANTIC_LAYER:
154 return SI_MAX_IO_GENERIC + 5;
155 case TGSI_SEMANTIC_VIEWPORT_INDEX:
156 return SI_MAX_IO_GENERIC + 6;
157 case TGSI_SEMANTIC_PRIMID:
158 return SI_MAX_IO_GENERIC + 7;
159 case TGSI_SEMANTIC_COLOR: /* these alias */
160 case TGSI_SEMANTIC_BCOLOR:
161 assert(index < 2);
162 return SI_MAX_IO_GENERIC + 8 + index;
163 case TGSI_SEMANTIC_TEXCOORD:
164 assert(index < 8);
165 assert(SI_MAX_IO_GENERIC + 10 + index < 64);
166 return SI_MAX_IO_GENERIC + 10 + index;
167 default:
168 assert(!"invalid semantic name");
169 return 0;
170 }
171 }
172
173 /**
174 * Get the value of a shader input parameter and extract a bitfield.
175 */
176 static LLVMValueRef unpack_param(struct si_shader_context *ctx,
177 unsigned param, unsigned rshift,
178 unsigned bitwidth)
179 {
180 struct gallivm_state *gallivm = &ctx->gallivm;
181 LLVMValueRef value = LLVMGetParam(ctx->main_fn,
182 param);
183
184 if (LLVMGetTypeKind(LLVMTypeOf(value)) == LLVMFloatTypeKind)
185 value = bitcast(&ctx->bld_base,
186 TGSI_TYPE_UNSIGNED, value);
187
188 if (rshift)
189 value = LLVMBuildLShr(gallivm->builder, value,
190 LLVMConstInt(ctx->i32, rshift, 0), "");
191
192 if (rshift + bitwidth < 32) {
193 unsigned mask = (1 << bitwidth) - 1;
194 value = LLVMBuildAnd(gallivm->builder, value,
195 LLVMConstInt(ctx->i32, mask, 0), "");
196 }
197
198 return value;
199 }
200
201 static LLVMValueRef get_rel_patch_id(struct si_shader_context *ctx)
202 {
203 switch (ctx->type) {
204 case PIPE_SHADER_TESS_CTRL:
205 return unpack_param(ctx, ctx->param_tcs_rel_ids, 0, 8);
206
207 case PIPE_SHADER_TESS_EVAL:
208 return LLVMGetParam(ctx->main_fn,
209 ctx->param_tes_rel_patch_id);
210
211 default:
212 assert(0);
213 return NULL;
214 }
215 }
216
217 /* Tessellation shaders pass outputs to the next shader using LDS.
218 *
219 * LS outputs = TCS inputs
220 * TCS outputs = TES inputs
221 *
222 * The LDS layout is:
223 * - TCS inputs for patch 0
224 * - TCS inputs for patch 1
225 * - TCS inputs for patch 2 = get_tcs_in_current_patch_offset (if RelPatchID==2)
226 * - ...
227 * - TCS outputs for patch 0 = get_tcs_out_patch0_offset
228 * - Per-patch TCS outputs for patch 0 = get_tcs_out_patch0_patch_data_offset
229 * - TCS outputs for patch 1
230 * - Per-patch TCS outputs for patch 1
231 * - TCS outputs for patch 2 = get_tcs_out_current_patch_offset (if RelPatchID==2)
232 * - Per-patch TCS outputs for patch 2 = get_tcs_out_current_patch_data_offset (if RelPatchID==2)
233 * - ...
234 *
235 * All three shaders VS(LS), TCS, TES share the same LDS space.
236 */
237
238 static LLVMValueRef
239 get_tcs_in_patch_stride(struct si_shader_context *ctx)
240 {
241 return unpack_param(ctx, ctx->param_vs_state_bits, 8, 13);
242 }
243
244 static LLVMValueRef
245 get_tcs_out_patch_stride(struct si_shader_context *ctx)
246 {
247 return unpack_param(ctx, ctx->param_tcs_out_lds_layout, 0, 13);
248 }
249
250 static LLVMValueRef
251 get_tcs_out_patch0_offset(struct si_shader_context *ctx)
252 {
253 return lp_build_mul_imm(&ctx->bld_base.uint_bld,
254 unpack_param(ctx,
255 ctx->param_tcs_out_lds_offsets,
256 0, 16),
257 4);
258 }
259
260 static LLVMValueRef
261 get_tcs_out_patch0_patch_data_offset(struct si_shader_context *ctx)
262 {
263 return lp_build_mul_imm(&ctx->bld_base.uint_bld,
264 unpack_param(ctx,
265 ctx->param_tcs_out_lds_offsets,
266 16, 16),
267 4);
268 }
269
270 static LLVMValueRef
271 get_tcs_in_current_patch_offset(struct si_shader_context *ctx)
272 {
273 struct gallivm_state *gallivm = &ctx->gallivm;
274 LLVMValueRef patch_stride = get_tcs_in_patch_stride(ctx);
275 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
276
277 return LLVMBuildMul(gallivm->builder, patch_stride, rel_patch_id, "");
278 }
279
280 static LLVMValueRef
281 get_tcs_out_current_patch_offset(struct si_shader_context *ctx)
282 {
283 struct gallivm_state *gallivm = &ctx->gallivm;
284 LLVMValueRef patch0_offset = get_tcs_out_patch0_offset(ctx);
285 LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
286 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
287
288 return LLVMBuildAdd(gallivm->builder, patch0_offset,
289 LLVMBuildMul(gallivm->builder, patch_stride,
290 rel_patch_id, ""),
291 "");
292 }
293
294 static LLVMValueRef
295 get_tcs_out_current_patch_data_offset(struct si_shader_context *ctx)
296 {
297 struct gallivm_state *gallivm = &ctx->gallivm;
298 LLVMValueRef patch0_patch_data_offset =
299 get_tcs_out_patch0_patch_data_offset(ctx);
300 LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
301 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
302
303 return LLVMBuildAdd(gallivm->builder, patch0_patch_data_offset,
304 LLVMBuildMul(gallivm->builder, patch_stride,
305 rel_patch_id, ""),
306 "");
307 }
308
309 static LLVMValueRef get_instance_index_for_fetch(
310 struct si_shader_context *ctx,
311 unsigned param_start_instance, unsigned divisor)
312 {
313 struct gallivm_state *gallivm = &ctx->gallivm;
314
315 LLVMValueRef result = LLVMGetParam(ctx->main_fn,
316 ctx->param_instance_id);
317
318 /* The division must be done before START_INSTANCE is added. */
319 if (divisor > 1)
320 result = LLVMBuildUDiv(gallivm->builder, result,
321 LLVMConstInt(ctx->i32, divisor, 0), "");
322
323 return LLVMBuildAdd(gallivm->builder, result,
324 LLVMGetParam(ctx->main_fn, param_start_instance), "");
325 }
326
327 /* Bitcast <4 x float> to <2 x double>, extract the component, and convert
328 * to float. */
329 static LLVMValueRef extract_double_to_float(struct si_shader_context *ctx,
330 LLVMValueRef vec4,
331 unsigned double_index)
332 {
333 LLVMBuilderRef builder = ctx->gallivm.builder;
334 LLVMTypeRef f64 = LLVMDoubleTypeInContext(ctx->gallivm.context);
335 LLVMValueRef dvec2 = LLVMBuildBitCast(builder, vec4,
336 LLVMVectorType(f64, 2), "");
337 LLVMValueRef index = LLVMConstInt(ctx->i32, double_index, 0);
338 LLVMValueRef value = LLVMBuildExtractElement(builder, dvec2, index, "");
339 return LLVMBuildFPTrunc(builder, value, ctx->f32, "");
340 }
341
342 static void declare_input_vs(
343 struct si_shader_context *ctx,
344 unsigned input_index,
345 const struct tgsi_full_declaration *decl,
346 LLVMValueRef out[4])
347 {
348 struct gallivm_state *gallivm = &ctx->gallivm;
349
350 unsigned chan;
351 unsigned fix_fetch;
352 unsigned num_fetches;
353 unsigned fetch_stride;
354
355 LLVMValueRef t_list_ptr;
356 LLVMValueRef t_offset;
357 LLVMValueRef t_list;
358 LLVMValueRef vertex_index;
359 LLVMValueRef input[3];
360
361 /* Load the T list */
362 t_list_ptr = LLVMGetParam(ctx->main_fn, ctx->param_vertex_buffers);
363
364 t_offset = LLVMConstInt(ctx->i32, input_index, 0);
365
366 t_list = ac_build_indexed_load_const(&ctx->ac, t_list_ptr, t_offset);
367
368 vertex_index = LLVMGetParam(ctx->main_fn,
369 ctx->param_vertex_index0 +
370 input_index);
371
372 fix_fetch = ctx->shader->key.mono.vs_fix_fetch[input_index];
373
374 /* Do multiple loads for special formats. */
375 switch (fix_fetch) {
376 case SI_FIX_FETCH_RGB_64_FLOAT:
377 num_fetches = 3; /* 3 2-dword loads */
378 fetch_stride = 8;
379 break;
380 case SI_FIX_FETCH_RGBA_64_FLOAT:
381 num_fetches = 2; /* 2 4-dword loads */
382 fetch_stride = 16;
383 break;
384 case SI_FIX_FETCH_RGB_8:
385 case SI_FIX_FETCH_RGB_8_INT:
386 num_fetches = 3;
387 fetch_stride = 1;
388 break;
389 case SI_FIX_FETCH_RGB_16:
390 case SI_FIX_FETCH_RGB_16_INT:
391 num_fetches = 3;
392 fetch_stride = 2;
393 break;
394 default:
395 num_fetches = 1;
396 fetch_stride = 0;
397 }
398
399 for (unsigned i = 0; i < num_fetches; i++) {
400 LLVMValueRef voffset = LLVMConstInt(ctx->i32, fetch_stride * i, 0);
401
402 input[i] = ac_build_buffer_load_format(&ctx->ac, t_list,
403 vertex_index, voffset,
404 true);
405 }
406
407 /* Break up the vec4 into individual components */
408 for (chan = 0; chan < 4; chan++) {
409 LLVMValueRef llvm_chan = LLVMConstInt(ctx->i32, chan, 0);
410 out[chan] = LLVMBuildExtractElement(gallivm->builder,
411 input[0], llvm_chan, "");
412 }
413
414 switch (fix_fetch) {
415 case SI_FIX_FETCH_A2_SNORM:
416 case SI_FIX_FETCH_A2_SSCALED:
417 case SI_FIX_FETCH_A2_SINT: {
418 /* The hardware returns an unsigned value; convert it to a
419 * signed one.
420 */
421 LLVMValueRef tmp = out[3];
422 LLVMValueRef c30 = LLVMConstInt(ctx->i32, 30, 0);
423
424 /* First, recover the sign-extended signed integer value. */
425 if (fix_fetch == SI_FIX_FETCH_A2_SSCALED)
426 tmp = LLVMBuildFPToUI(gallivm->builder, tmp, ctx->i32, "");
427 else
428 tmp = LLVMBuildBitCast(gallivm->builder, tmp, ctx->i32, "");
429
430 /* For the integer-like cases, do a natural sign extension.
431 *
432 * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
433 * and happen to contain 0, 1, 2, 3 as the two LSBs of the
434 * exponent.
435 */
436 tmp = LLVMBuildShl(gallivm->builder, tmp,
437 fix_fetch == SI_FIX_FETCH_A2_SNORM ?
438 LLVMConstInt(ctx->i32, 7, 0) : c30, "");
439 tmp = LLVMBuildAShr(gallivm->builder, tmp, c30, "");
440
441 /* Convert back to the right type. */
442 if (fix_fetch == SI_FIX_FETCH_A2_SNORM) {
443 LLVMValueRef clamp;
444 LLVMValueRef neg_one = LLVMConstReal(ctx->f32, -1.0);
445 tmp = LLVMBuildSIToFP(gallivm->builder, tmp, ctx->f32, "");
446 clamp = LLVMBuildFCmp(gallivm->builder, LLVMRealULT, tmp, neg_one, "");
447 tmp = LLVMBuildSelect(gallivm->builder, clamp, neg_one, tmp, "");
448 } else if (fix_fetch == SI_FIX_FETCH_A2_SSCALED) {
449 tmp = LLVMBuildSIToFP(gallivm->builder, tmp, ctx->f32, "");
450 }
451
452 out[3] = tmp;
453 break;
454 }
455 case SI_FIX_FETCH_RGBA_32_UNORM:
456 case SI_FIX_FETCH_RGBX_32_UNORM:
457 for (chan = 0; chan < 4; chan++) {
458 out[chan] = LLVMBuildBitCast(gallivm->builder, out[chan],
459 ctx->i32, "");
460 out[chan] = LLVMBuildUIToFP(gallivm->builder,
461 out[chan], ctx->f32, "");
462 out[chan] = LLVMBuildFMul(gallivm->builder, out[chan],
463 LLVMConstReal(ctx->f32, 1.0 / UINT_MAX), "");
464 }
465 /* RGBX UINT returns 1 in alpha, which would be rounded to 0 by normalizing. */
466 if (fix_fetch == SI_FIX_FETCH_RGBX_32_UNORM)
467 out[3] = LLVMConstReal(ctx->f32, 1);
468 break;
469 case SI_FIX_FETCH_RGBA_32_SNORM:
470 case SI_FIX_FETCH_RGBX_32_SNORM:
471 case SI_FIX_FETCH_RGBA_32_FIXED:
472 case SI_FIX_FETCH_RGBX_32_FIXED: {
473 double scale;
474 if (fix_fetch >= SI_FIX_FETCH_RGBA_32_FIXED)
475 scale = 1.0 / 0x10000;
476 else
477 scale = 1.0 / INT_MAX;
478
479 for (chan = 0; chan < 4; chan++) {
480 out[chan] = LLVMBuildBitCast(gallivm->builder, out[chan],
481 ctx->i32, "");
482 out[chan] = LLVMBuildSIToFP(gallivm->builder,
483 out[chan], ctx->f32, "");
484 out[chan] = LLVMBuildFMul(gallivm->builder, out[chan],
485 LLVMConstReal(ctx->f32, scale), "");
486 }
487 /* RGBX SINT returns 1 in alpha, which would be rounded to 0 by normalizing. */
488 if (fix_fetch == SI_FIX_FETCH_RGBX_32_SNORM ||
489 fix_fetch == SI_FIX_FETCH_RGBX_32_FIXED)
490 out[3] = LLVMConstReal(ctx->f32, 1);
491 break;
492 }
493 case SI_FIX_FETCH_RGBA_32_USCALED:
494 for (chan = 0; chan < 4; chan++) {
495 out[chan] = LLVMBuildBitCast(gallivm->builder, out[chan],
496 ctx->i32, "");
497 out[chan] = LLVMBuildUIToFP(gallivm->builder,
498 out[chan], ctx->f32, "");
499 }
500 break;
501 case SI_FIX_FETCH_RGBA_32_SSCALED:
502 for (chan = 0; chan < 4; chan++) {
503 out[chan] = LLVMBuildBitCast(gallivm->builder, out[chan],
504 ctx->i32, "");
505 out[chan] = LLVMBuildSIToFP(gallivm->builder,
506 out[chan], ctx->f32, "");
507 }
508 break;
509 case SI_FIX_FETCH_RG_64_FLOAT:
510 for (chan = 0; chan < 2; chan++)
511 out[chan] = extract_double_to_float(ctx, input[0], chan);
512
513 out[2] = LLVMConstReal(ctx->f32, 0);
514 out[3] = LLVMConstReal(ctx->f32, 1);
515 break;
516 case SI_FIX_FETCH_RGB_64_FLOAT:
517 for (chan = 0; chan < 3; chan++)
518 out[chan] = extract_double_to_float(ctx, input[chan], 0);
519
520 out[3] = LLVMConstReal(ctx->f32, 1);
521 break;
522 case SI_FIX_FETCH_RGBA_64_FLOAT:
523 for (chan = 0; chan < 4; chan++) {
524 out[chan] = extract_double_to_float(ctx, input[chan / 2],
525 chan % 2);
526 }
527 break;
528 case SI_FIX_FETCH_RGB_8:
529 case SI_FIX_FETCH_RGB_8_INT:
530 case SI_FIX_FETCH_RGB_16:
531 case SI_FIX_FETCH_RGB_16_INT:
532 for (chan = 0; chan < 3; chan++) {
533 out[chan] = LLVMBuildExtractElement(gallivm->builder,
534 input[chan],
535 ctx->i32_0, "");
536 }
537 if (fix_fetch == SI_FIX_FETCH_RGB_8 ||
538 fix_fetch == SI_FIX_FETCH_RGB_16) {
539 out[3] = LLVMConstReal(ctx->f32, 1);
540 } else {
541 out[3] = LLVMBuildBitCast(gallivm->builder, ctx->i32_1,
542 ctx->f32, "");
543 }
544 break;
545 }
546 }
547
548 static LLVMValueRef get_primitive_id(struct lp_build_tgsi_context *bld_base,
549 unsigned swizzle)
550 {
551 struct si_shader_context *ctx = si_shader_context(bld_base);
552
553 if (swizzle > 0)
554 return ctx->i32_0;
555
556 switch (ctx->type) {
557 case PIPE_SHADER_VERTEX:
558 return LLVMGetParam(ctx->main_fn,
559 ctx->param_vs_prim_id);
560 case PIPE_SHADER_TESS_CTRL:
561 return LLVMGetParam(ctx->main_fn,
562 ctx->param_tcs_patch_id);
563 case PIPE_SHADER_TESS_EVAL:
564 return LLVMGetParam(ctx->main_fn,
565 ctx->param_tes_patch_id);
566 case PIPE_SHADER_GEOMETRY:
567 return LLVMGetParam(ctx->main_fn,
568 ctx->param_gs_prim_id);
569 default:
570 assert(0);
571 return ctx->i32_0;
572 }
573 }
574
575 /**
576 * Return the value of tgsi_ind_register for indexing.
577 * This is the indirect index with the constant offset added to it.
578 */
579 static LLVMValueRef get_indirect_index(struct si_shader_context *ctx,
580 const struct tgsi_ind_register *ind,
581 int rel_index)
582 {
583 struct gallivm_state *gallivm = &ctx->gallivm;
584 LLVMValueRef result;
585
586 result = ctx->addrs[ind->Index][ind->Swizzle];
587 result = LLVMBuildLoad(gallivm->builder, result, "");
588 result = LLVMBuildAdd(gallivm->builder, result,
589 LLVMConstInt(ctx->i32, rel_index, 0), "");
590 return result;
591 }
592
593 /**
594 * Like get_indirect_index, but restricts the return value to a (possibly
595 * undefined) value inside [0..num).
596 */
597 LLVMValueRef si_get_bounded_indirect_index(struct si_shader_context *ctx,
598 const struct tgsi_ind_register *ind,
599 int rel_index, unsigned num)
600 {
601 LLVMValueRef result = get_indirect_index(ctx, ind, rel_index);
602
603 return si_llvm_bound_index(ctx, result, num);
604 }
605
606
607 /**
608 * Calculate a dword address given an input or output register and a stride.
609 */
610 static LLVMValueRef get_dw_address(struct si_shader_context *ctx,
611 const struct tgsi_full_dst_register *dst,
612 const struct tgsi_full_src_register *src,
613 LLVMValueRef vertex_dw_stride,
614 LLVMValueRef base_addr)
615 {
616 struct gallivm_state *gallivm = &ctx->gallivm;
617 struct tgsi_shader_info *info = &ctx->shader->selector->info;
618 ubyte *name, *index, *array_first;
619 int first, param;
620 struct tgsi_full_dst_register reg;
621
622 /* Set the register description. The address computation is the same
623 * for sources and destinations. */
624 if (src) {
625 reg.Register.File = src->Register.File;
626 reg.Register.Index = src->Register.Index;
627 reg.Register.Indirect = src->Register.Indirect;
628 reg.Register.Dimension = src->Register.Dimension;
629 reg.Indirect = src->Indirect;
630 reg.Dimension = src->Dimension;
631 reg.DimIndirect = src->DimIndirect;
632 } else
633 reg = *dst;
634
635 /* If the register is 2-dimensional (e.g. an array of vertices
636 * in a primitive), calculate the base address of the vertex. */
637 if (reg.Register.Dimension) {
638 LLVMValueRef index;
639
640 if (reg.Dimension.Indirect)
641 index = get_indirect_index(ctx, &reg.DimIndirect,
642 reg.Dimension.Index);
643 else
644 index = LLVMConstInt(ctx->i32, reg.Dimension.Index, 0);
645
646 base_addr = LLVMBuildAdd(gallivm->builder, base_addr,
647 LLVMBuildMul(gallivm->builder, index,
648 vertex_dw_stride, ""), "");
649 }
650
651 /* Get information about the register. */
652 if (reg.Register.File == TGSI_FILE_INPUT) {
653 name = info->input_semantic_name;
654 index = info->input_semantic_index;
655 array_first = info->input_array_first;
656 } else if (reg.Register.File == TGSI_FILE_OUTPUT) {
657 name = info->output_semantic_name;
658 index = info->output_semantic_index;
659 array_first = info->output_array_first;
660 } else {
661 assert(0);
662 return NULL;
663 }
664
665 if (reg.Register.Indirect) {
666 /* Add the relative address of the element. */
667 LLVMValueRef ind_index;
668
669 if (reg.Indirect.ArrayID)
670 first = array_first[reg.Indirect.ArrayID];
671 else
672 first = reg.Register.Index;
673
674 ind_index = get_indirect_index(ctx, &reg.Indirect,
675 reg.Register.Index - first);
676
677 base_addr = LLVMBuildAdd(gallivm->builder, base_addr,
678 LLVMBuildMul(gallivm->builder, ind_index,
679 LLVMConstInt(ctx->i32, 4, 0), ""), "");
680
681 param = reg.Register.Dimension ?
682 si_shader_io_get_unique_index(name[first], index[first]) :
683 si_shader_io_get_unique_index_patch(name[first], index[first]);
684 } else {
685 param = reg.Register.Dimension ?
686 si_shader_io_get_unique_index(name[reg.Register.Index],
687 index[reg.Register.Index]) :
688 si_shader_io_get_unique_index_patch(name[reg.Register.Index],
689 index[reg.Register.Index]);
690 }
691
692 /* Add the base address of the element. */
693 return LLVMBuildAdd(gallivm->builder, base_addr,
694 LLVMConstInt(ctx->i32, param * 4, 0), "");
695 }
696
697 /* The offchip buffer layout for TCS->TES is
698 *
699 * - attribute 0 of patch 0 vertex 0
700 * - attribute 0 of patch 0 vertex 1
701 * - attribute 0 of patch 0 vertex 2
702 * ...
703 * - attribute 0 of patch 1 vertex 0
704 * - attribute 0 of patch 1 vertex 1
705 * ...
706 * - attribute 1 of patch 0 vertex 0
707 * - attribute 1 of patch 0 vertex 1
708 * ...
709 * - per patch attribute 0 of patch 0
710 * - per patch attribute 0 of patch 1
711 * ...
712 *
713 * Note that every attribute has 4 components.
714 */
715 static LLVMValueRef get_tcs_tes_buffer_address(struct si_shader_context *ctx,
716 LLVMValueRef rel_patch_id,
717 LLVMValueRef vertex_index,
718 LLVMValueRef param_index)
719 {
720 struct gallivm_state *gallivm = &ctx->gallivm;
721 LLVMValueRef base_addr, vertices_per_patch, num_patches, total_vertices;
722 LLVMValueRef param_stride, constant16;
723
724 vertices_per_patch = unpack_param(ctx, ctx->param_tcs_offchip_layout, 6, 6);
725 num_patches = unpack_param(ctx, ctx->param_tcs_offchip_layout, 0, 6);
726 total_vertices = LLVMBuildMul(gallivm->builder, vertices_per_patch,
727 num_patches, "");
728
729 constant16 = LLVMConstInt(ctx->i32, 16, 0);
730 if (vertex_index) {
731 base_addr = LLVMBuildMul(gallivm->builder, rel_patch_id,
732 vertices_per_patch, "");
733
734 base_addr = LLVMBuildAdd(gallivm->builder, base_addr,
735 vertex_index, "");
736
737 param_stride = total_vertices;
738 } else {
739 base_addr = rel_patch_id;
740 param_stride = num_patches;
741 }
742
743 base_addr = LLVMBuildAdd(gallivm->builder, base_addr,
744 LLVMBuildMul(gallivm->builder, param_index,
745 param_stride, ""), "");
746
747 base_addr = LLVMBuildMul(gallivm->builder, base_addr, constant16, "");
748
749 if (!vertex_index) {
750 LLVMValueRef patch_data_offset =
751 unpack_param(ctx, ctx->param_tcs_offchip_layout, 12, 20);
752
753 base_addr = LLVMBuildAdd(gallivm->builder, base_addr,
754 patch_data_offset, "");
755 }
756 return base_addr;
757 }
758
759 static LLVMValueRef get_tcs_tes_buffer_address_from_reg(
760 struct si_shader_context *ctx,
761 const struct tgsi_full_dst_register *dst,
762 const struct tgsi_full_src_register *src)
763 {
764 struct gallivm_state *gallivm = &ctx->gallivm;
765 struct tgsi_shader_info *info = &ctx->shader->selector->info;
766 ubyte *name, *index, *array_first;
767 struct tgsi_full_src_register reg;
768 LLVMValueRef vertex_index = NULL;
769 LLVMValueRef param_index = NULL;
770 unsigned param_index_base, param_base;
771
772 reg = src ? *src : tgsi_full_src_register_from_dst(dst);
773
774 if (reg.Register.Dimension) {
775
776 if (reg.Dimension.Indirect)
777 vertex_index = get_indirect_index(ctx, &reg.DimIndirect,
778 reg.Dimension.Index);
779 else
780 vertex_index = LLVMConstInt(ctx->i32, reg.Dimension.Index, 0);
781 }
782
783 /* Get information about the register. */
784 if (reg.Register.File == TGSI_FILE_INPUT) {
785 name = info->input_semantic_name;
786 index = info->input_semantic_index;
787 array_first = info->input_array_first;
788 } else if (reg.Register.File == TGSI_FILE_OUTPUT) {
789 name = info->output_semantic_name;
790 index = info->output_semantic_index;
791 array_first = info->output_array_first;
792 } else {
793 assert(0);
794 return NULL;
795 }
796
797 if (reg.Register.Indirect) {
798 if (reg.Indirect.ArrayID)
799 param_base = array_first[reg.Indirect.ArrayID];
800 else
801 param_base = reg.Register.Index;
802
803 param_index = get_indirect_index(ctx, &reg.Indirect,
804 reg.Register.Index - param_base);
805
806 } else {
807 param_base = reg.Register.Index;
808 param_index = ctx->i32_0;
809 }
810
811 param_index_base = reg.Register.Dimension ?
812 si_shader_io_get_unique_index(name[param_base], index[param_base]) :
813 si_shader_io_get_unique_index_patch(name[param_base], index[param_base]);
814
815 param_index = LLVMBuildAdd(gallivm->builder, param_index,
816 LLVMConstInt(ctx->i32, param_index_base, 0),
817 "");
818
819 return get_tcs_tes_buffer_address(ctx, get_rel_patch_id(ctx),
820 vertex_index, param_index);
821 }
822
823 static LLVMValueRef buffer_load(struct lp_build_tgsi_context *bld_base,
824 enum tgsi_opcode_type type, unsigned swizzle,
825 LLVMValueRef buffer, LLVMValueRef offset,
826 LLVMValueRef base, bool can_speculate)
827 {
828 struct si_shader_context *ctx = si_shader_context(bld_base);
829 struct gallivm_state *gallivm = &ctx->gallivm;
830 LLVMValueRef value, value2;
831 LLVMTypeRef llvm_type = tgsi2llvmtype(bld_base, type);
832 LLVMTypeRef vec_type = LLVMVectorType(llvm_type, 4);
833
834 if (swizzle == ~0) {
835 value = ac_build_buffer_load(&ctx->ac, buffer, 4, NULL, base, offset,
836 0, 1, 0, can_speculate);
837
838 return LLVMBuildBitCast(gallivm->builder, value, vec_type, "");
839 }
840
841 if (!tgsi_type_is_64bit(type)) {
842 value = ac_build_buffer_load(&ctx->ac, buffer, 4, NULL, base, offset,
843 0, 1, 0, can_speculate);
844
845 value = LLVMBuildBitCast(gallivm->builder, value, vec_type, "");
846 return LLVMBuildExtractElement(gallivm->builder, value,
847 LLVMConstInt(ctx->i32, swizzle, 0), "");
848 }
849
850 value = ac_build_buffer_load(&ctx->ac, buffer, 1, NULL, base, offset,
851 swizzle * 4, 1, 0, can_speculate);
852
853 value2 = ac_build_buffer_load(&ctx->ac, buffer, 1, NULL, base, offset,
854 swizzle * 4 + 4, 1, 0, can_speculate);
855
856 return si_llvm_emit_fetch_64bit(bld_base, type, value, value2);
857 }
858
859 /**
860 * Load from LDS.
861 *
862 * \param type output value type
863 * \param swizzle offset (typically 0..3); it can be ~0, which loads a vec4
864 * \param dw_addr address in dwords
865 */
866 static LLVMValueRef lds_load(struct lp_build_tgsi_context *bld_base,
867 enum tgsi_opcode_type type, unsigned swizzle,
868 LLVMValueRef dw_addr)
869 {
870 struct si_shader_context *ctx = si_shader_context(bld_base);
871 struct gallivm_state *gallivm = &ctx->gallivm;
872 LLVMValueRef value;
873
874 if (swizzle == ~0) {
875 LLVMValueRef values[TGSI_NUM_CHANNELS];
876
877 for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; chan++)
878 values[chan] = lds_load(bld_base, type, chan, dw_addr);
879
880 return lp_build_gather_values(gallivm, values,
881 TGSI_NUM_CHANNELS);
882 }
883
884 dw_addr = lp_build_add(&bld_base->uint_bld, dw_addr,
885 LLVMConstInt(ctx->i32, swizzle, 0));
886
887 value = ac_build_indexed_load(&ctx->ac, ctx->lds, dw_addr, false);
888 if (tgsi_type_is_64bit(type)) {
889 LLVMValueRef value2;
890 dw_addr = lp_build_add(&bld_base->uint_bld, dw_addr,
891 ctx->i32_1);
892 value2 = ac_build_indexed_load(&ctx->ac, ctx->lds, dw_addr, false);
893 return si_llvm_emit_fetch_64bit(bld_base, type, value, value2);
894 }
895
896 return LLVMBuildBitCast(gallivm->builder, value,
897 tgsi2llvmtype(bld_base, type), "");
898 }
899
900 /**
901 * Store to LDS.
902 *
903 * \param swizzle offset (typically 0..3)
904 * \param dw_addr address in dwords
905 * \param value value to store
906 */
907 static void lds_store(struct lp_build_tgsi_context *bld_base,
908 unsigned dw_offset_imm, LLVMValueRef dw_addr,
909 LLVMValueRef value)
910 {
911 struct si_shader_context *ctx = si_shader_context(bld_base);
912 struct gallivm_state *gallivm = &ctx->gallivm;
913
914 dw_addr = lp_build_add(&bld_base->uint_bld, dw_addr,
915 LLVMConstInt(ctx->i32, dw_offset_imm, 0));
916
917 value = LLVMBuildBitCast(gallivm->builder, value, ctx->i32, "");
918 ac_build_indexed_store(&ctx->ac, ctx->lds,
919 dw_addr, value);
920 }
921
922 static LLVMValueRef desc_from_addr_base64k(struct si_shader_context *ctx,
923 unsigned param)
924 {
925 LLVMBuilderRef builder = ctx->gallivm.builder;
926
927 LLVMValueRef addr = LLVMGetParam(ctx->main_fn, param);
928 addr = LLVMBuildZExt(builder, addr, ctx->i64, "");
929 addr = LLVMBuildShl(builder, addr, LLVMConstInt(ctx->i64, 16, 0), "");
930
931 uint64_t desc2 = 0xffffffff;
932 uint64_t desc3 = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
933 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
934 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
935 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
936 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
937 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
938 LLVMValueRef hi = LLVMConstInt(ctx->i64, desc2 | (desc3 << 32), 0);
939
940 LLVMValueRef desc = LLVMGetUndef(LLVMVectorType(ctx->i64, 2));
941 desc = LLVMBuildInsertElement(builder, desc, addr, ctx->i32_0, "");
942 desc = LLVMBuildInsertElement(builder, desc, hi, ctx->i32_1, "");
943 return LLVMBuildBitCast(builder, desc, ctx->v4i32, "");
944 }
945
946 static LLVMValueRef fetch_input_tcs(
947 struct lp_build_tgsi_context *bld_base,
948 const struct tgsi_full_src_register *reg,
949 enum tgsi_opcode_type type, unsigned swizzle)
950 {
951 struct si_shader_context *ctx = si_shader_context(bld_base);
952 LLVMValueRef dw_addr, stride;
953
954 stride = unpack_param(ctx, ctx->param_vs_state_bits, 24, 8);
955 dw_addr = get_tcs_in_current_patch_offset(ctx);
956 dw_addr = get_dw_address(ctx, NULL, reg, stride, dw_addr);
957
958 return lds_load(bld_base, type, swizzle, dw_addr);
959 }
960
961 static LLVMValueRef fetch_output_tcs(
962 struct lp_build_tgsi_context *bld_base,
963 const struct tgsi_full_src_register *reg,
964 enum tgsi_opcode_type type, unsigned swizzle)
965 {
966 struct si_shader_context *ctx = si_shader_context(bld_base);
967 LLVMValueRef dw_addr, stride;
968
969 if (reg->Register.Dimension) {
970 stride = unpack_param(ctx, ctx->param_tcs_out_lds_layout, 13, 8);
971 dw_addr = get_tcs_out_current_patch_offset(ctx);
972 dw_addr = get_dw_address(ctx, NULL, reg, stride, dw_addr);
973 } else {
974 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
975 dw_addr = get_dw_address(ctx, NULL, reg, NULL, dw_addr);
976 }
977
978 return lds_load(bld_base, type, swizzle, dw_addr);
979 }
980
981 static LLVMValueRef fetch_input_tes(
982 struct lp_build_tgsi_context *bld_base,
983 const struct tgsi_full_src_register *reg,
984 enum tgsi_opcode_type type, unsigned swizzle)
985 {
986 struct si_shader_context *ctx = si_shader_context(bld_base);
987 LLVMValueRef buffer, base, addr;
988
989 buffer = desc_from_addr_base64k(ctx, ctx->param_tcs_offchip_addr_base64k);
990
991 base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
992 addr = get_tcs_tes_buffer_address_from_reg(ctx, NULL, reg);
993
994 return buffer_load(bld_base, type, swizzle, buffer, base, addr, true);
995 }
996
997 static void store_output_tcs(struct lp_build_tgsi_context *bld_base,
998 const struct tgsi_full_instruction *inst,
999 const struct tgsi_opcode_info *info,
1000 LLVMValueRef dst[4])
1001 {
1002 struct si_shader_context *ctx = si_shader_context(bld_base);
1003 struct gallivm_state *gallivm = &ctx->gallivm;
1004 const struct tgsi_full_dst_register *reg = &inst->Dst[0];
1005 const struct tgsi_shader_info *sh_info = &ctx->shader->selector->info;
1006 unsigned chan_index;
1007 LLVMValueRef dw_addr, stride;
1008 LLVMValueRef buffer, base, buf_addr;
1009 LLVMValueRef values[4];
1010 bool skip_lds_store;
1011 bool is_tess_factor = false;
1012
1013 /* Only handle per-patch and per-vertex outputs here.
1014 * Vectors will be lowered to scalars and this function will be called again.
1015 */
1016 if (reg->Register.File != TGSI_FILE_OUTPUT ||
1017 (dst[0] && LLVMGetTypeKind(LLVMTypeOf(dst[0])) == LLVMVectorTypeKind)) {
1018 si_llvm_emit_store(bld_base, inst, info, dst);
1019 return;
1020 }
1021
1022 if (reg->Register.Dimension) {
1023 stride = unpack_param(ctx, ctx->param_tcs_out_lds_layout, 13, 8);
1024 dw_addr = get_tcs_out_current_patch_offset(ctx);
1025 dw_addr = get_dw_address(ctx, reg, NULL, stride, dw_addr);
1026 skip_lds_store = !sh_info->reads_pervertex_outputs;
1027 } else {
1028 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1029 dw_addr = get_dw_address(ctx, reg, NULL, NULL, dw_addr);
1030 skip_lds_store = !sh_info->reads_perpatch_outputs;
1031
1032 if (!reg->Register.Indirect) {
1033 int name = sh_info->output_semantic_name[reg->Register.Index];
1034
1035 /* Always write tess factors into LDS for the TCS epilog. */
1036 if (name == TGSI_SEMANTIC_TESSINNER ||
1037 name == TGSI_SEMANTIC_TESSOUTER) {
1038 skip_lds_store = false;
1039 is_tess_factor = true;
1040 }
1041 }
1042 }
1043
1044 buffer = desc_from_addr_base64k(ctx, ctx->param_tcs_offchip_addr_base64k);
1045
1046 base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
1047 buf_addr = get_tcs_tes_buffer_address_from_reg(ctx, reg, NULL);
1048
1049
1050 TGSI_FOR_EACH_DST0_ENABLED_CHANNEL(inst, chan_index) {
1051 LLVMValueRef value = dst[chan_index];
1052
1053 if (inst->Instruction.Saturate)
1054 value = ac_build_clamp(&ctx->ac, value);
1055
1056 /* Skip LDS stores if there is no LDS read of this output. */
1057 if (!skip_lds_store)
1058 lds_store(bld_base, chan_index, dw_addr, value);
1059
1060 value = LLVMBuildBitCast(gallivm->builder, value, ctx->i32, "");
1061 values[chan_index] = value;
1062
1063 if (inst->Dst[0].Register.WriteMask != 0xF && !is_tess_factor) {
1064 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 1,
1065 buf_addr, base,
1066 4 * chan_index, 1, 0, true, false);
1067 }
1068 }
1069
1070 if (inst->Dst[0].Register.WriteMask == 0xF && !is_tess_factor) {
1071 LLVMValueRef value = lp_build_gather_values(gallivm,
1072 values, 4);
1073 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, buf_addr,
1074 base, 0, 1, 0, true, false);
1075 }
1076 }
1077
1078 static LLVMValueRef fetch_input_gs(
1079 struct lp_build_tgsi_context *bld_base,
1080 const struct tgsi_full_src_register *reg,
1081 enum tgsi_opcode_type type,
1082 unsigned swizzle)
1083 {
1084 struct si_shader_context *ctx = si_shader_context(bld_base);
1085 struct si_shader *shader = ctx->shader;
1086 struct lp_build_context *uint = &ctx->bld_base.uint_bld;
1087 struct gallivm_state *gallivm = &ctx->gallivm;
1088 LLVMValueRef vtx_offset, soffset;
1089 struct tgsi_shader_info *info = &shader->selector->info;
1090 unsigned semantic_name = info->input_semantic_name[reg->Register.Index];
1091 unsigned semantic_index = info->input_semantic_index[reg->Register.Index];
1092 unsigned param;
1093 LLVMValueRef value;
1094
1095 if (swizzle != ~0 && semantic_name == TGSI_SEMANTIC_PRIMID)
1096 return get_primitive_id(bld_base, swizzle);
1097
1098 if (!reg->Register.Dimension)
1099 return NULL;
1100
1101 param = si_shader_io_get_unique_index(semantic_name, semantic_index);
1102
1103 /* GFX9 has the ESGS ring in LDS. */
1104 if (ctx->screen->b.chip_class >= GFX9) {
1105 unsigned index = reg->Dimension.Index;
1106
1107 switch (index / 2) {
1108 case 0:
1109 vtx_offset = unpack_param(ctx, ctx->param_gs_vtx01_offset,
1110 index % 2 ? 16 : 0, 16);
1111 break;
1112 case 1:
1113 vtx_offset = unpack_param(ctx, ctx->param_gs_vtx23_offset,
1114 index % 2 ? 16 : 0, 16);
1115 break;
1116 case 2:
1117 vtx_offset = unpack_param(ctx, ctx->param_gs_vtx45_offset,
1118 index % 2 ? 16 : 0, 16);
1119 break;
1120 default:
1121 assert(0);
1122 return NULL;
1123 }
1124
1125 vtx_offset = LLVMBuildAdd(gallivm->builder, vtx_offset,
1126 LLVMConstInt(ctx->i32, param * 4, 0), "");
1127 return lds_load(bld_base, type, swizzle, vtx_offset);
1128 }
1129
1130 /* GFX6: input load from the ESGS ring in memory. */
1131 if (swizzle == ~0) {
1132 LLVMValueRef values[TGSI_NUM_CHANNELS];
1133 unsigned chan;
1134 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1135 values[chan] = fetch_input_gs(bld_base, reg, type, chan);
1136 }
1137 return lp_build_gather_values(gallivm, values,
1138 TGSI_NUM_CHANNELS);
1139 }
1140
1141 /* Get the vertex offset parameter on GFX6. */
1142 unsigned vtx_offset_param = reg->Dimension.Index;
1143 if (vtx_offset_param < 2) {
1144 vtx_offset_param += ctx->param_gs_vtx0_offset;
1145 } else {
1146 assert(vtx_offset_param < 6);
1147 vtx_offset_param += ctx->param_gs_vtx2_offset - 2;
1148 }
1149 vtx_offset = lp_build_mul_imm(uint,
1150 LLVMGetParam(ctx->main_fn,
1151 vtx_offset_param),
1152 4);
1153
1154 soffset = LLVMConstInt(ctx->i32, (param * 4 + swizzle) * 256, 0);
1155
1156 value = ac_build_buffer_load(&ctx->ac, ctx->esgs_ring, 1, ctx->i32_0,
1157 vtx_offset, soffset, 0, 1, 0, true);
1158 if (tgsi_type_is_64bit(type)) {
1159 LLVMValueRef value2;
1160 soffset = LLVMConstInt(ctx->i32, (param * 4 + swizzle + 1) * 256, 0);
1161
1162 value2 = ac_build_buffer_load(&ctx->ac, ctx->esgs_ring, 1,
1163 ctx->i32_0, vtx_offset, soffset,
1164 0, 1, 0, true);
1165 return si_llvm_emit_fetch_64bit(bld_base, type,
1166 value, value2);
1167 }
1168 return LLVMBuildBitCast(gallivm->builder,
1169 value,
1170 tgsi2llvmtype(bld_base, type), "");
1171 }
1172
1173 static int lookup_interp_param_index(unsigned interpolate, unsigned location)
1174 {
1175 switch (interpolate) {
1176 case TGSI_INTERPOLATE_CONSTANT:
1177 return 0;
1178
1179 case TGSI_INTERPOLATE_LINEAR:
1180 if (location == TGSI_INTERPOLATE_LOC_SAMPLE)
1181 return SI_PARAM_LINEAR_SAMPLE;
1182 else if (location == TGSI_INTERPOLATE_LOC_CENTROID)
1183 return SI_PARAM_LINEAR_CENTROID;
1184 else
1185 return SI_PARAM_LINEAR_CENTER;
1186 break;
1187 case TGSI_INTERPOLATE_COLOR:
1188 case TGSI_INTERPOLATE_PERSPECTIVE:
1189 if (location == TGSI_INTERPOLATE_LOC_SAMPLE)
1190 return SI_PARAM_PERSP_SAMPLE;
1191 else if (location == TGSI_INTERPOLATE_LOC_CENTROID)
1192 return SI_PARAM_PERSP_CENTROID;
1193 else
1194 return SI_PARAM_PERSP_CENTER;
1195 break;
1196 default:
1197 fprintf(stderr, "Warning: Unhandled interpolation mode.\n");
1198 return -1;
1199 }
1200 }
1201
1202 /**
1203 * Interpolate a fragment shader input.
1204 *
1205 * @param ctx context
1206 * @param input_index index of the input in hardware
1207 * @param semantic_name TGSI_SEMANTIC_*
1208 * @param semantic_index semantic index
1209 * @param num_interp_inputs number of all interpolated inputs (= BCOLOR offset)
1210 * @param colors_read_mask color components read (4 bits for each color, 8 bits in total)
1211 * @param interp_param interpolation weights (i,j)
1212 * @param prim_mask SI_PARAM_PRIM_MASK
1213 * @param face SI_PARAM_FRONT_FACE
1214 * @param result the return value (4 components)
1215 */
1216 static void interp_fs_input(struct si_shader_context *ctx,
1217 unsigned input_index,
1218 unsigned semantic_name,
1219 unsigned semantic_index,
1220 unsigned num_interp_inputs,
1221 unsigned colors_read_mask,
1222 LLVMValueRef interp_param,
1223 LLVMValueRef prim_mask,
1224 LLVMValueRef face,
1225 LLVMValueRef result[4])
1226 {
1227 struct gallivm_state *gallivm = &ctx->gallivm;
1228 LLVMValueRef attr_number;
1229 LLVMValueRef i, j;
1230
1231 unsigned chan;
1232
1233 /* fs.constant returns the param from the middle vertex, so it's not
1234 * really useful for flat shading. It's meant to be used for custom
1235 * interpolation (but the intrinsic can't fetch from the other two
1236 * vertices).
1237 *
1238 * Luckily, it doesn't matter, because we rely on the FLAT_SHADE state
1239 * to do the right thing. The only reason we use fs.constant is that
1240 * fs.interp cannot be used on integers, because they can be equal
1241 * to NaN.
1242 *
1243 * When interp is false we will use fs.constant or for newer llvm,
1244 * amdgcn.interp.mov.
1245 */
1246 bool interp = interp_param != NULL;
1247
1248 attr_number = LLVMConstInt(ctx->i32, input_index, 0);
1249
1250 if (interp) {
1251 interp_param = LLVMBuildBitCast(gallivm->builder, interp_param,
1252 LLVMVectorType(ctx->f32, 2), "");
1253
1254 i = LLVMBuildExtractElement(gallivm->builder, interp_param,
1255 ctx->i32_0, "");
1256 j = LLVMBuildExtractElement(gallivm->builder, interp_param,
1257 ctx->i32_1, "");
1258 }
1259
1260 if (semantic_name == TGSI_SEMANTIC_COLOR &&
1261 ctx->shader->key.part.ps.prolog.color_two_side) {
1262 LLVMValueRef is_face_positive;
1263 LLVMValueRef back_attr_number;
1264
1265 /* If BCOLOR0 is used, BCOLOR1 is at offset "num_inputs + 1",
1266 * otherwise it's at offset "num_inputs".
1267 */
1268 unsigned back_attr_offset = num_interp_inputs;
1269 if (semantic_index == 1 && colors_read_mask & 0xf)
1270 back_attr_offset += 1;
1271
1272 back_attr_number = LLVMConstInt(ctx->i32, back_attr_offset, 0);
1273
1274 is_face_positive = LLVMBuildICmp(gallivm->builder, LLVMIntNE,
1275 face, ctx->i32_0, "");
1276
1277 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1278 LLVMValueRef llvm_chan = LLVMConstInt(ctx->i32, chan, 0);
1279 LLVMValueRef front, back;
1280
1281 if (interp) {
1282 front = ac_build_fs_interp(&ctx->ac, llvm_chan,
1283 attr_number, prim_mask,
1284 i, j);
1285 back = ac_build_fs_interp(&ctx->ac, llvm_chan,
1286 back_attr_number, prim_mask,
1287 i, j);
1288 } else {
1289 front = ac_build_fs_interp_mov(&ctx->ac,
1290 LLVMConstInt(ctx->i32, 2, 0), /* P0 */
1291 llvm_chan, attr_number, prim_mask);
1292 back = ac_build_fs_interp_mov(&ctx->ac,
1293 LLVMConstInt(ctx->i32, 2, 0), /* P0 */
1294 llvm_chan, back_attr_number, prim_mask);
1295 }
1296
1297 result[chan] = LLVMBuildSelect(gallivm->builder,
1298 is_face_positive,
1299 front,
1300 back,
1301 "");
1302 }
1303 } else if (semantic_name == TGSI_SEMANTIC_FOG) {
1304 if (interp) {
1305 result[0] = ac_build_fs_interp(&ctx->ac, ctx->i32_0,
1306 attr_number, prim_mask, i, j);
1307 } else {
1308 result[0] = ac_build_fs_interp_mov(&ctx->ac, ctx->i32_0,
1309 LLVMConstInt(ctx->i32, 2, 0), /* P0 */
1310 attr_number, prim_mask);
1311 }
1312 result[1] =
1313 result[2] = LLVMConstReal(ctx->f32, 0.0f);
1314 result[3] = LLVMConstReal(ctx->f32, 1.0f);
1315 } else {
1316 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1317 LLVMValueRef llvm_chan = LLVMConstInt(ctx->i32, chan, 0);
1318
1319 if (interp) {
1320 result[chan] = ac_build_fs_interp(&ctx->ac,
1321 llvm_chan, attr_number, prim_mask, i, j);
1322 } else {
1323 result[chan] = ac_build_fs_interp_mov(&ctx->ac,
1324 LLVMConstInt(ctx->i32, 2, 0), /* P0 */
1325 llvm_chan, attr_number, prim_mask);
1326 }
1327 }
1328 }
1329 }
1330
1331 static void declare_input_fs(
1332 struct si_shader_context *ctx,
1333 unsigned input_index,
1334 const struct tgsi_full_declaration *decl,
1335 LLVMValueRef out[4])
1336 {
1337 struct lp_build_context *base = &ctx->bld_base.base;
1338 struct si_shader *shader = ctx->shader;
1339 LLVMValueRef main_fn = ctx->main_fn;
1340 LLVMValueRef interp_param = NULL;
1341 int interp_param_idx;
1342
1343 /* Get colors from input VGPRs (set by the prolog). */
1344 if (decl->Semantic.Name == TGSI_SEMANTIC_COLOR) {
1345 unsigned i = decl->Semantic.Index;
1346 unsigned colors_read = shader->selector->info.colors_read;
1347 unsigned mask = colors_read >> (i * 4);
1348 unsigned offset = SI_PARAM_POS_FIXED_PT + 1 +
1349 (i ? util_bitcount(colors_read & 0xf) : 0);
1350
1351 out[0] = mask & 0x1 ? LLVMGetParam(main_fn, offset++) : base->undef;
1352 out[1] = mask & 0x2 ? LLVMGetParam(main_fn, offset++) : base->undef;
1353 out[2] = mask & 0x4 ? LLVMGetParam(main_fn, offset++) : base->undef;
1354 out[3] = mask & 0x8 ? LLVMGetParam(main_fn, offset++) : base->undef;
1355 return;
1356 }
1357
1358 interp_param_idx = lookup_interp_param_index(decl->Interp.Interpolate,
1359 decl->Interp.Location);
1360 if (interp_param_idx == -1)
1361 return;
1362 else if (interp_param_idx) {
1363 interp_param = LLVMGetParam(ctx->main_fn, interp_param_idx);
1364 }
1365
1366 if (decl->Semantic.Name == TGSI_SEMANTIC_COLOR &&
1367 decl->Interp.Interpolate == TGSI_INTERPOLATE_COLOR &&
1368 ctx->shader->key.part.ps.prolog.flatshade_colors)
1369 interp_param = NULL; /* load the constant color */
1370
1371 interp_fs_input(ctx, input_index, decl->Semantic.Name,
1372 decl->Semantic.Index, shader->selector->info.num_inputs,
1373 shader->selector->info.colors_read, interp_param,
1374 LLVMGetParam(main_fn, SI_PARAM_PRIM_MASK),
1375 LLVMGetParam(main_fn, SI_PARAM_FRONT_FACE),
1376 &out[0]);
1377 }
1378
1379 static LLVMValueRef get_sample_id(struct si_shader_context *ctx)
1380 {
1381 return unpack_param(ctx, SI_PARAM_ANCILLARY, 8, 4);
1382 }
1383
1384
1385 /**
1386 * Load a dword from a constant buffer.
1387 */
1388 static LLVMValueRef buffer_load_const(struct si_shader_context *ctx,
1389 LLVMValueRef resource,
1390 LLVMValueRef offset)
1391 {
1392 LLVMBuilderRef builder = ctx->gallivm.builder;
1393 LLVMValueRef args[2] = {resource, offset};
1394
1395 return lp_build_intrinsic(builder, "llvm.SI.load.const.v4i32", ctx->f32, args, 2,
1396 LP_FUNC_ATTR_READNONE |
1397 LP_FUNC_ATTR_LEGACY);
1398 }
1399
1400 static LLVMValueRef load_sample_position(struct si_shader_context *ctx, LLVMValueRef sample_id)
1401 {
1402 struct lp_build_context *uint_bld = &ctx->bld_base.uint_bld;
1403 struct gallivm_state *gallivm = &ctx->gallivm;
1404 LLVMBuilderRef builder = gallivm->builder;
1405 LLVMValueRef desc = LLVMGetParam(ctx->main_fn, ctx->param_rw_buffers);
1406 LLVMValueRef buf_index = LLVMConstInt(ctx->i32, SI_PS_CONST_SAMPLE_POSITIONS, 0);
1407 LLVMValueRef resource = ac_build_indexed_load_const(&ctx->ac, desc, buf_index);
1408
1409 /* offset = sample_id * 8 (8 = 2 floats containing samplepos.xy) */
1410 LLVMValueRef offset0 = lp_build_mul_imm(uint_bld, sample_id, 8);
1411 LLVMValueRef offset1 = LLVMBuildAdd(builder, offset0, LLVMConstInt(ctx->i32, 4, 0), "");
1412
1413 LLVMValueRef pos[4] = {
1414 buffer_load_const(ctx, resource, offset0),
1415 buffer_load_const(ctx, resource, offset1),
1416 LLVMConstReal(ctx->f32, 0),
1417 LLVMConstReal(ctx->f32, 0)
1418 };
1419
1420 return lp_build_gather_values(gallivm, pos, 4);
1421 }
1422
1423 static void declare_system_value(struct si_shader_context *ctx,
1424 unsigned index,
1425 const struct tgsi_full_declaration *decl)
1426 {
1427 struct lp_build_context *bld = &ctx->bld_base.base;
1428 struct gallivm_state *gallivm = &ctx->gallivm;
1429 LLVMValueRef value = 0;
1430
1431 assert(index < RADEON_LLVM_MAX_SYSTEM_VALUES);
1432
1433 switch (decl->Semantic.Name) {
1434 case TGSI_SEMANTIC_INSTANCEID:
1435 value = LLVMGetParam(ctx->main_fn,
1436 ctx->param_instance_id);
1437 break;
1438
1439 case TGSI_SEMANTIC_VERTEXID:
1440 value = LLVMBuildAdd(gallivm->builder,
1441 LLVMGetParam(ctx->main_fn,
1442 ctx->param_vertex_id),
1443 LLVMGetParam(ctx->main_fn,
1444 ctx->param_base_vertex), "");
1445 break;
1446
1447 case TGSI_SEMANTIC_VERTEXID_NOBASE:
1448 /* Unused. Clarify the meaning in indexed vs. non-indexed
1449 * draws if this is ever used again. */
1450 assert(false);
1451 break;
1452
1453 case TGSI_SEMANTIC_BASEVERTEX:
1454 {
1455 /* For non-indexed draws, the base vertex set by the driver
1456 * (for direct draws) or the CP (for indirect draws) is the
1457 * first vertex ID, but GLSL expects 0 to be returned.
1458 */
1459 LLVMValueRef vs_state = LLVMGetParam(ctx->main_fn, ctx->param_vs_state_bits);
1460 LLVMValueRef indexed;
1461
1462 indexed = LLVMBuildLShr(gallivm->builder, vs_state, ctx->i32_1, "");
1463 indexed = LLVMBuildTrunc(gallivm->builder, indexed, ctx->i1, "");
1464
1465 value = LLVMBuildSelect(gallivm->builder, indexed,
1466 LLVMGetParam(ctx->main_fn, ctx->param_base_vertex),
1467 ctx->i32_0, "");
1468 break;
1469 }
1470
1471 case TGSI_SEMANTIC_BASEINSTANCE:
1472 value = LLVMGetParam(ctx->main_fn, ctx->param_start_instance);
1473 break;
1474
1475 case TGSI_SEMANTIC_DRAWID:
1476 value = LLVMGetParam(ctx->main_fn, ctx->param_draw_id);
1477 break;
1478
1479 case TGSI_SEMANTIC_INVOCATIONID:
1480 if (ctx->type == PIPE_SHADER_TESS_CTRL)
1481 value = unpack_param(ctx, ctx->param_tcs_rel_ids, 8, 5);
1482 else if (ctx->type == PIPE_SHADER_GEOMETRY)
1483 value = LLVMGetParam(ctx->main_fn,
1484 ctx->param_gs_instance_id);
1485 else
1486 assert(!"INVOCATIONID not implemented");
1487 break;
1488
1489 case TGSI_SEMANTIC_POSITION:
1490 {
1491 LLVMValueRef pos[4] = {
1492 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_X_FLOAT),
1493 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Y_FLOAT),
1494 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Z_FLOAT),
1495 lp_build_emit_llvm_unary(&ctx->bld_base, TGSI_OPCODE_RCP,
1496 LLVMGetParam(ctx->main_fn,
1497 SI_PARAM_POS_W_FLOAT)),
1498 };
1499 value = lp_build_gather_values(gallivm, pos, 4);
1500 break;
1501 }
1502
1503 case TGSI_SEMANTIC_FACE:
1504 value = LLVMGetParam(ctx->main_fn, SI_PARAM_FRONT_FACE);
1505 break;
1506
1507 case TGSI_SEMANTIC_SAMPLEID:
1508 value = get_sample_id(ctx);
1509 break;
1510
1511 case TGSI_SEMANTIC_SAMPLEPOS: {
1512 LLVMValueRef pos[4] = {
1513 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_X_FLOAT),
1514 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Y_FLOAT),
1515 LLVMConstReal(ctx->f32, 0),
1516 LLVMConstReal(ctx->f32, 0)
1517 };
1518 pos[0] = lp_build_emit_llvm_unary(&ctx->bld_base,
1519 TGSI_OPCODE_FRC, pos[0]);
1520 pos[1] = lp_build_emit_llvm_unary(&ctx->bld_base,
1521 TGSI_OPCODE_FRC, pos[1]);
1522 value = lp_build_gather_values(gallivm, pos, 4);
1523 break;
1524 }
1525
1526 case TGSI_SEMANTIC_SAMPLEMASK:
1527 /* This can only occur with the OpenGL Core profile, which
1528 * doesn't support smoothing.
1529 */
1530 value = LLVMGetParam(ctx->main_fn, SI_PARAM_SAMPLE_COVERAGE);
1531 break;
1532
1533 case TGSI_SEMANTIC_TESSCOORD:
1534 {
1535 LLVMValueRef coord[4] = {
1536 LLVMGetParam(ctx->main_fn, ctx->param_tes_u),
1537 LLVMGetParam(ctx->main_fn, ctx->param_tes_v),
1538 bld->zero,
1539 bld->zero
1540 };
1541
1542 /* For triangles, the vector should be (u, v, 1-u-v). */
1543 if (ctx->shader->selector->info.properties[TGSI_PROPERTY_TES_PRIM_MODE] ==
1544 PIPE_PRIM_TRIANGLES)
1545 coord[2] = lp_build_sub(bld, bld->one,
1546 lp_build_add(bld, coord[0], coord[1]));
1547
1548 value = lp_build_gather_values(gallivm, coord, 4);
1549 break;
1550 }
1551
1552 case TGSI_SEMANTIC_VERTICESIN:
1553 if (ctx->type == PIPE_SHADER_TESS_CTRL)
1554 value = unpack_param(ctx, ctx->param_tcs_out_lds_layout, 26, 6);
1555 else if (ctx->type == PIPE_SHADER_TESS_EVAL)
1556 value = unpack_param(ctx, ctx->param_tcs_offchip_layout, 6, 6);
1557 else
1558 assert(!"invalid shader stage for TGSI_SEMANTIC_VERTICESIN");
1559 break;
1560
1561 case TGSI_SEMANTIC_TESSINNER:
1562 case TGSI_SEMANTIC_TESSOUTER:
1563 {
1564 LLVMValueRef buffer, base, addr;
1565 int param = si_shader_io_get_unique_index_patch(decl->Semantic.Name, 0);
1566
1567 buffer = desc_from_addr_base64k(ctx, ctx->param_tcs_offchip_addr_base64k);
1568
1569 base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
1570 addr = get_tcs_tes_buffer_address(ctx, get_rel_patch_id(ctx), NULL,
1571 LLVMConstInt(ctx->i32, param, 0));
1572
1573 value = buffer_load(&ctx->bld_base, TGSI_TYPE_FLOAT,
1574 ~0, buffer, base, addr, true);
1575
1576 break;
1577 }
1578
1579 case TGSI_SEMANTIC_DEFAULT_TESSOUTER_SI:
1580 case TGSI_SEMANTIC_DEFAULT_TESSINNER_SI:
1581 {
1582 LLVMValueRef buf, slot, val[4];
1583 int i, offset;
1584
1585 slot = LLVMConstInt(ctx->i32, SI_HS_CONST_DEFAULT_TESS_LEVELS, 0);
1586 buf = LLVMGetParam(ctx->main_fn, ctx->param_rw_buffers);
1587 buf = ac_build_indexed_load_const(&ctx->ac, buf, slot);
1588 offset = decl->Semantic.Name == TGSI_SEMANTIC_DEFAULT_TESSINNER_SI ? 4 : 0;
1589
1590 for (i = 0; i < 4; i++)
1591 val[i] = buffer_load_const(ctx, buf,
1592 LLVMConstInt(ctx->i32, (offset + i) * 4, 0));
1593 value = lp_build_gather_values(gallivm, val, 4);
1594 break;
1595 }
1596
1597 case TGSI_SEMANTIC_PRIMID:
1598 value = get_primitive_id(&ctx->bld_base, 0);
1599 break;
1600
1601 case TGSI_SEMANTIC_GRID_SIZE:
1602 value = LLVMGetParam(ctx->main_fn, ctx->param_grid_size);
1603 break;
1604
1605 case TGSI_SEMANTIC_BLOCK_SIZE:
1606 {
1607 LLVMValueRef values[3];
1608 unsigned i;
1609 unsigned *properties = ctx->shader->selector->info.properties;
1610
1611 if (properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH] != 0) {
1612 unsigned sizes[3] = {
1613 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH],
1614 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT],
1615 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH]
1616 };
1617
1618 for (i = 0; i < 3; ++i)
1619 values[i] = LLVMConstInt(ctx->i32, sizes[i], 0);
1620
1621 value = lp_build_gather_values(gallivm, values, 3);
1622 } else {
1623 value = LLVMGetParam(ctx->main_fn, ctx->param_block_size);
1624 }
1625 break;
1626 }
1627
1628 case TGSI_SEMANTIC_BLOCK_ID:
1629 {
1630 LLVMValueRef values[3];
1631
1632 for (int i = 0; i < 3; i++) {
1633 values[i] = ctx->i32_0;
1634 if (ctx->param_block_id[i] >= 0) {
1635 values[i] = LLVMGetParam(ctx->main_fn,
1636 ctx->param_block_id[i]);
1637 }
1638 }
1639 value = lp_build_gather_values(gallivm, values, 3);
1640 break;
1641 }
1642
1643 case TGSI_SEMANTIC_THREAD_ID:
1644 value = LLVMGetParam(ctx->main_fn, ctx->param_thread_id);
1645 break;
1646
1647 case TGSI_SEMANTIC_HELPER_INVOCATION:
1648 value = lp_build_intrinsic(gallivm->builder,
1649 "llvm.amdgcn.ps.live",
1650 ctx->i1, NULL, 0,
1651 LP_FUNC_ATTR_READNONE);
1652 value = LLVMBuildNot(gallivm->builder, value, "");
1653 value = LLVMBuildSExt(gallivm->builder, value, ctx->i32, "");
1654 break;
1655
1656 case TGSI_SEMANTIC_SUBGROUP_SIZE:
1657 value = LLVMConstInt(ctx->i32, 64, 0);
1658 break;
1659
1660 case TGSI_SEMANTIC_SUBGROUP_INVOCATION:
1661 value = ac_get_thread_id(&ctx->ac);
1662 break;
1663
1664 case TGSI_SEMANTIC_SUBGROUP_EQ_MASK:
1665 {
1666 LLVMValueRef id = ac_get_thread_id(&ctx->ac);
1667 id = LLVMBuildZExt(gallivm->builder, id, ctx->i64, "");
1668 value = LLVMBuildShl(gallivm->builder, LLVMConstInt(ctx->i64, 1, 0), id, "");
1669 value = LLVMBuildBitCast(gallivm->builder, value, ctx->v2i32, "");
1670 break;
1671 }
1672
1673 case TGSI_SEMANTIC_SUBGROUP_GE_MASK:
1674 case TGSI_SEMANTIC_SUBGROUP_GT_MASK:
1675 case TGSI_SEMANTIC_SUBGROUP_LE_MASK:
1676 case TGSI_SEMANTIC_SUBGROUP_LT_MASK:
1677 {
1678 LLVMValueRef id = ac_get_thread_id(&ctx->ac);
1679 if (decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_GT_MASK ||
1680 decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_LE_MASK) {
1681 /* All bits set except LSB */
1682 value = LLVMConstInt(ctx->i64, -2, 0);
1683 } else {
1684 /* All bits set */
1685 value = LLVMConstInt(ctx->i64, -1, 0);
1686 }
1687 id = LLVMBuildZExt(gallivm->builder, id, ctx->i64, "");
1688 value = LLVMBuildShl(gallivm->builder, value, id, "");
1689 if (decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_LE_MASK ||
1690 decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_LT_MASK)
1691 value = LLVMBuildNot(gallivm->builder, value, "");
1692 value = LLVMBuildBitCast(gallivm->builder, value, ctx->v2i32, "");
1693 break;
1694 }
1695
1696 default:
1697 assert(!"unknown system value");
1698 return;
1699 }
1700
1701 ctx->system_values[index] = value;
1702 }
1703
1704 static void declare_compute_memory(struct si_shader_context *ctx,
1705 const struct tgsi_full_declaration *decl)
1706 {
1707 struct si_shader_selector *sel = ctx->shader->selector;
1708 struct gallivm_state *gallivm = &ctx->gallivm;
1709
1710 LLVMTypeRef i8p = LLVMPointerType(ctx->i8, LOCAL_ADDR_SPACE);
1711 LLVMValueRef var;
1712
1713 assert(decl->Declaration.MemType == TGSI_MEMORY_TYPE_SHARED);
1714 assert(decl->Range.First == decl->Range.Last);
1715 assert(!ctx->shared_memory);
1716
1717 var = LLVMAddGlobalInAddressSpace(gallivm->module,
1718 LLVMArrayType(ctx->i8, sel->local_size),
1719 "compute_lds",
1720 LOCAL_ADDR_SPACE);
1721 LLVMSetAlignment(var, 4);
1722
1723 ctx->shared_memory = LLVMBuildBitCast(gallivm->builder, var, i8p, "");
1724 }
1725
1726 static LLVMValueRef load_const_buffer_desc(struct si_shader_context *ctx, int i)
1727 {
1728 LLVMValueRef list_ptr = LLVMGetParam(ctx->main_fn,
1729 ctx->param_const_and_shader_buffers);
1730
1731 return ac_build_indexed_load_const(&ctx->ac, list_ptr,
1732 LLVMConstInt(ctx->i32, si_get_constbuf_slot(i), 0));
1733 }
1734
1735 static LLVMValueRef fetch_constant(
1736 struct lp_build_tgsi_context *bld_base,
1737 const struct tgsi_full_src_register *reg,
1738 enum tgsi_opcode_type type,
1739 unsigned swizzle)
1740 {
1741 struct si_shader_context *ctx = si_shader_context(bld_base);
1742 struct lp_build_context *base = &bld_base->base;
1743 const struct tgsi_ind_register *ireg = &reg->Indirect;
1744 unsigned buf, idx;
1745
1746 LLVMValueRef addr, bufp;
1747 LLVMValueRef result;
1748
1749 if (swizzle == LP_CHAN_ALL) {
1750 unsigned chan;
1751 LLVMValueRef values[4];
1752 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan)
1753 values[chan] = fetch_constant(bld_base, reg, type, chan);
1754
1755 return lp_build_gather_values(&ctx->gallivm, values, 4);
1756 }
1757
1758 buf = reg->Register.Dimension ? reg->Dimension.Index : 0;
1759 idx = reg->Register.Index * 4 + swizzle;
1760
1761 if (reg->Register.Dimension && reg->Dimension.Indirect) {
1762 LLVMValueRef ptr = LLVMGetParam(ctx->main_fn, ctx->param_const_and_shader_buffers);
1763 LLVMValueRef index;
1764 index = si_get_bounded_indirect_index(ctx, &reg->DimIndirect,
1765 reg->Dimension.Index,
1766 ctx->num_const_buffers);
1767 index = LLVMBuildAdd(ctx->gallivm.builder, index,
1768 LLVMConstInt(ctx->i32, SI_NUM_SHADER_BUFFERS, 0), "");
1769 bufp = ac_build_indexed_load_const(&ctx->ac, ptr, index);
1770 } else
1771 bufp = load_const_buffer_desc(ctx, buf);
1772
1773 if (reg->Register.Indirect) {
1774 addr = ctx->addrs[ireg->Index][ireg->Swizzle];
1775 addr = LLVMBuildLoad(base->gallivm->builder, addr, "load addr reg");
1776 addr = lp_build_mul_imm(&bld_base->uint_bld, addr, 16);
1777 addr = lp_build_add(&bld_base->uint_bld, addr,
1778 LLVMConstInt(ctx->i32, idx * 4, 0));
1779 } else {
1780 addr = LLVMConstInt(ctx->i32, idx * 4, 0);
1781 }
1782
1783 result = buffer_load_const(ctx, bufp, addr);
1784
1785 if (!tgsi_type_is_64bit(type))
1786 result = bitcast(bld_base, type, result);
1787 else {
1788 LLVMValueRef addr2, result2;
1789
1790 addr2 = lp_build_add(&bld_base->uint_bld, addr,
1791 LLVMConstInt(ctx->i32, 4, 0));
1792 result2 = buffer_load_const(ctx, bufp, addr2);
1793
1794 result = si_llvm_emit_fetch_64bit(bld_base, type,
1795 result, result2);
1796 }
1797 return result;
1798 }
1799
1800 /* Upper 16 bits must be zero. */
1801 static LLVMValueRef si_llvm_pack_two_int16(struct si_shader_context *ctx,
1802 LLVMValueRef val[2])
1803 {
1804 return LLVMBuildOr(ctx->gallivm.builder, val[0],
1805 LLVMBuildShl(ctx->gallivm.builder, val[1],
1806 LLVMConstInt(ctx->i32, 16, 0),
1807 ""), "");
1808 }
1809
1810 /* Upper 16 bits are ignored and will be dropped. */
1811 static LLVMValueRef si_llvm_pack_two_int32_as_int16(struct si_shader_context *ctx,
1812 LLVMValueRef val[2])
1813 {
1814 LLVMValueRef v[2] = {
1815 LLVMBuildAnd(ctx->gallivm.builder, val[0],
1816 LLVMConstInt(ctx->i32, 0xffff, 0), ""),
1817 val[1],
1818 };
1819 return si_llvm_pack_two_int16(ctx, v);
1820 }
1821
1822 /* Initialize arguments for the shader export intrinsic */
1823 static void si_llvm_init_export_args(struct lp_build_tgsi_context *bld_base,
1824 LLVMValueRef *values,
1825 unsigned target,
1826 struct ac_export_args *args)
1827 {
1828 struct si_shader_context *ctx = si_shader_context(bld_base);
1829 struct lp_build_context *base = &bld_base->base;
1830 LLVMBuilderRef builder = ctx->gallivm.builder;
1831 LLVMValueRef val[4];
1832 unsigned spi_shader_col_format = V_028714_SPI_SHADER_32_ABGR;
1833 unsigned chan;
1834 bool is_int8, is_int10;
1835
1836 /* Default is 0xf. Adjusted below depending on the format. */
1837 args->enabled_channels = 0xf; /* writemask */
1838
1839 /* Specify whether the EXEC mask represents the valid mask */
1840 args->valid_mask = 0;
1841
1842 /* Specify whether this is the last export */
1843 args->done = 0;
1844
1845 /* Specify the target we are exporting */
1846 args->target = target;
1847
1848 if (ctx->type == PIPE_SHADER_FRAGMENT) {
1849 const struct si_shader_key *key = &ctx->shader->key;
1850 unsigned col_formats = key->part.ps.epilog.spi_shader_col_format;
1851 int cbuf = target - V_008DFC_SQ_EXP_MRT;
1852
1853 assert(cbuf >= 0 && cbuf < 8);
1854 spi_shader_col_format = (col_formats >> (cbuf * 4)) & 0xf;
1855 is_int8 = (key->part.ps.epilog.color_is_int8 >> cbuf) & 0x1;
1856 is_int10 = (key->part.ps.epilog.color_is_int10 >> cbuf) & 0x1;
1857 }
1858
1859 args->compr = false;
1860 args->out[0] = base->undef;
1861 args->out[1] = base->undef;
1862 args->out[2] = base->undef;
1863 args->out[3] = base->undef;
1864
1865 switch (spi_shader_col_format) {
1866 case V_028714_SPI_SHADER_ZERO:
1867 args->enabled_channels = 0; /* writemask */
1868 args->target = V_008DFC_SQ_EXP_NULL;
1869 break;
1870
1871 case V_028714_SPI_SHADER_32_R:
1872 args->enabled_channels = 1; /* writemask */
1873 args->out[0] = values[0];
1874 break;
1875
1876 case V_028714_SPI_SHADER_32_GR:
1877 args->enabled_channels = 0x3; /* writemask */
1878 args->out[0] = values[0];
1879 args->out[1] = values[1];
1880 break;
1881
1882 case V_028714_SPI_SHADER_32_AR:
1883 args->enabled_channels = 0x9; /* writemask */
1884 args->out[0] = values[0];
1885 args->out[3] = values[3];
1886 break;
1887
1888 case V_028714_SPI_SHADER_FP16_ABGR:
1889 args->compr = 1; /* COMPR flag */
1890
1891 for (chan = 0; chan < 2; chan++) {
1892 LLVMValueRef pack_args[2] = {
1893 values[2 * chan],
1894 values[2 * chan + 1]
1895 };
1896 LLVMValueRef packed;
1897
1898 packed = ac_build_cvt_pkrtz_f16(&ctx->ac, pack_args);
1899 args->out[chan] =
1900 LLVMBuildBitCast(ctx->gallivm.builder,
1901 packed, ctx->f32, "");
1902 }
1903 break;
1904
1905 case V_028714_SPI_SHADER_UNORM16_ABGR:
1906 for (chan = 0; chan < 4; chan++) {
1907 val[chan] = ac_build_clamp(&ctx->ac, values[chan]);
1908 val[chan] = LLVMBuildFMul(builder, val[chan],
1909 LLVMConstReal(ctx->f32, 65535), "");
1910 val[chan] = LLVMBuildFAdd(builder, val[chan],
1911 LLVMConstReal(ctx->f32, 0.5), "");
1912 val[chan] = LLVMBuildFPToUI(builder, val[chan],
1913 ctx->i32, "");
1914 }
1915
1916 args->compr = 1; /* COMPR flag */
1917 args->out[0] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1918 si_llvm_pack_two_int16(ctx, val));
1919 args->out[1] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1920 si_llvm_pack_two_int16(ctx, val+2));
1921 break;
1922
1923 case V_028714_SPI_SHADER_SNORM16_ABGR:
1924 for (chan = 0; chan < 4; chan++) {
1925 /* Clamp between [-1, 1]. */
1926 val[chan] = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MIN,
1927 values[chan],
1928 LLVMConstReal(ctx->f32, 1));
1929 val[chan] = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MAX,
1930 val[chan],
1931 LLVMConstReal(ctx->f32, -1));
1932 /* Convert to a signed integer in [-32767, 32767]. */
1933 val[chan] = LLVMBuildFMul(builder, val[chan],
1934 LLVMConstReal(ctx->f32, 32767), "");
1935 /* If positive, add 0.5, else add -0.5. */
1936 val[chan] = LLVMBuildFAdd(builder, val[chan],
1937 LLVMBuildSelect(builder,
1938 LLVMBuildFCmp(builder, LLVMRealOGE,
1939 val[chan], base->zero, ""),
1940 LLVMConstReal(ctx->f32, 0.5),
1941 LLVMConstReal(ctx->f32, -0.5), ""), "");
1942 val[chan] = LLVMBuildFPToSI(builder, val[chan], ctx->i32, "");
1943 }
1944
1945 args->compr = 1; /* COMPR flag */
1946 args->out[0] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1947 si_llvm_pack_two_int32_as_int16(ctx, val));
1948 args->out[1] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1949 si_llvm_pack_two_int32_as_int16(ctx, val+2));
1950 break;
1951
1952 case V_028714_SPI_SHADER_UINT16_ABGR: {
1953 LLVMValueRef max_rgb = LLVMConstInt(ctx->i32,
1954 is_int8 ? 255 : is_int10 ? 1023 : 65535, 0);
1955 LLVMValueRef max_alpha =
1956 !is_int10 ? max_rgb : LLVMConstInt(ctx->i32, 3, 0);
1957
1958 /* Clamp. */
1959 for (chan = 0; chan < 4; chan++) {
1960 val[chan] = bitcast(bld_base, TGSI_TYPE_UNSIGNED, values[chan]);
1961 val[chan] = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_UMIN,
1962 val[chan],
1963 chan == 3 ? max_alpha : max_rgb);
1964 }
1965
1966 args->compr = 1; /* COMPR flag */
1967 args->out[0] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1968 si_llvm_pack_two_int16(ctx, val));
1969 args->out[1] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1970 si_llvm_pack_two_int16(ctx, val+2));
1971 break;
1972 }
1973
1974 case V_028714_SPI_SHADER_SINT16_ABGR: {
1975 LLVMValueRef max_rgb = LLVMConstInt(ctx->i32,
1976 is_int8 ? 127 : is_int10 ? 511 : 32767, 0);
1977 LLVMValueRef min_rgb = LLVMConstInt(ctx->i32,
1978 is_int8 ? -128 : is_int10 ? -512 : -32768, 0);
1979 LLVMValueRef max_alpha =
1980 !is_int10 ? max_rgb : ctx->i32_1;
1981 LLVMValueRef min_alpha =
1982 !is_int10 ? min_rgb : LLVMConstInt(ctx->i32, -2, 0);
1983
1984 /* Clamp. */
1985 for (chan = 0; chan < 4; chan++) {
1986 val[chan] = bitcast(bld_base, TGSI_TYPE_UNSIGNED, values[chan]);
1987 val[chan] = lp_build_emit_llvm_binary(bld_base,
1988 TGSI_OPCODE_IMIN,
1989 val[chan], chan == 3 ? max_alpha : max_rgb);
1990 val[chan] = lp_build_emit_llvm_binary(bld_base,
1991 TGSI_OPCODE_IMAX,
1992 val[chan], chan == 3 ? min_alpha : min_rgb);
1993 }
1994
1995 args->compr = 1; /* COMPR flag */
1996 args->out[0] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1997 si_llvm_pack_two_int32_as_int16(ctx, val));
1998 args->out[1] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1999 si_llvm_pack_two_int32_as_int16(ctx, val+2));
2000 break;
2001 }
2002
2003 case V_028714_SPI_SHADER_32_ABGR:
2004 memcpy(&args->out[0], values, sizeof(values[0]) * 4);
2005 break;
2006 }
2007 }
2008
2009 static void si_alpha_test(struct lp_build_tgsi_context *bld_base,
2010 LLVMValueRef alpha)
2011 {
2012 struct si_shader_context *ctx = si_shader_context(bld_base);
2013
2014 if (ctx->shader->key.part.ps.epilog.alpha_func != PIPE_FUNC_NEVER) {
2015 LLVMValueRef alpha_ref = LLVMGetParam(ctx->main_fn,
2016 SI_PARAM_ALPHA_REF);
2017
2018 LLVMValueRef alpha_pass =
2019 lp_build_cmp(&bld_base->base,
2020 ctx->shader->key.part.ps.epilog.alpha_func,
2021 alpha, alpha_ref);
2022 LLVMValueRef arg =
2023 lp_build_select(&bld_base->base,
2024 alpha_pass,
2025 LLVMConstReal(ctx->f32, 1.0f),
2026 LLVMConstReal(ctx->f32, -1.0f));
2027
2028 ac_build_kill(&ctx->ac, arg);
2029 } else {
2030 ac_build_kill(&ctx->ac, NULL);
2031 }
2032 }
2033
2034 static LLVMValueRef si_scale_alpha_by_sample_mask(struct lp_build_tgsi_context *bld_base,
2035 LLVMValueRef alpha,
2036 unsigned samplemask_param)
2037 {
2038 struct si_shader_context *ctx = si_shader_context(bld_base);
2039 struct gallivm_state *gallivm = &ctx->gallivm;
2040 LLVMValueRef coverage;
2041
2042 /* alpha = alpha * popcount(coverage) / SI_NUM_SMOOTH_AA_SAMPLES */
2043 coverage = LLVMGetParam(ctx->main_fn,
2044 samplemask_param);
2045 coverage = bitcast(bld_base, TGSI_TYPE_SIGNED, coverage);
2046
2047 coverage = lp_build_intrinsic(gallivm->builder, "llvm.ctpop.i32",
2048 ctx->i32,
2049 &coverage, 1, LP_FUNC_ATTR_READNONE);
2050
2051 coverage = LLVMBuildUIToFP(gallivm->builder, coverage,
2052 ctx->f32, "");
2053
2054 coverage = LLVMBuildFMul(gallivm->builder, coverage,
2055 LLVMConstReal(ctx->f32,
2056 1.0 / SI_NUM_SMOOTH_AA_SAMPLES), "");
2057
2058 return LLVMBuildFMul(gallivm->builder, alpha, coverage, "");
2059 }
2060
2061 static void si_llvm_emit_clipvertex(struct lp_build_tgsi_context *bld_base,
2062 struct ac_export_args *pos, LLVMValueRef *out_elts)
2063 {
2064 struct si_shader_context *ctx = si_shader_context(bld_base);
2065 struct lp_build_context *base = &bld_base->base;
2066 unsigned reg_index;
2067 unsigned chan;
2068 unsigned const_chan;
2069 LLVMValueRef base_elt;
2070 LLVMValueRef ptr = LLVMGetParam(ctx->main_fn, ctx->param_rw_buffers);
2071 LLVMValueRef constbuf_index = LLVMConstInt(ctx->i32,
2072 SI_VS_CONST_CLIP_PLANES, 0);
2073 LLVMValueRef const_resource = ac_build_indexed_load_const(&ctx->ac, ptr, constbuf_index);
2074
2075 for (reg_index = 0; reg_index < 2; reg_index ++) {
2076 struct ac_export_args *args = &pos[2 + reg_index];
2077
2078 args->out[0] =
2079 args->out[1] =
2080 args->out[2] =
2081 args->out[3] = LLVMConstReal(ctx->f32, 0.0f);
2082
2083 /* Compute dot products of position and user clip plane vectors */
2084 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
2085 for (const_chan = 0; const_chan < TGSI_NUM_CHANNELS; const_chan++) {
2086 LLVMValueRef addr =
2087 LLVMConstInt(ctx->i32, ((reg_index * 4 + chan) * 4 +
2088 const_chan) * 4, 0);
2089 base_elt = buffer_load_const(ctx, const_resource,
2090 addr);
2091 args->out[chan] =
2092 lp_build_add(base, args->out[chan],
2093 lp_build_mul(base, base_elt,
2094 out_elts[const_chan]));
2095 }
2096 }
2097
2098 args->enabled_channels = 0xf;
2099 args->valid_mask = 0;
2100 args->done = 0;
2101 args->target = V_008DFC_SQ_EXP_POS + 2 + reg_index;
2102 args->compr = 0;
2103 }
2104 }
2105
2106 static void si_dump_streamout(struct pipe_stream_output_info *so)
2107 {
2108 unsigned i;
2109
2110 if (so->num_outputs)
2111 fprintf(stderr, "STREAMOUT\n");
2112
2113 for (i = 0; i < so->num_outputs; i++) {
2114 unsigned mask = ((1 << so->output[i].num_components) - 1) <<
2115 so->output[i].start_component;
2116 fprintf(stderr, " %i: BUF%i[%i..%i] <- OUT[%i].%s%s%s%s\n",
2117 i, so->output[i].output_buffer,
2118 so->output[i].dst_offset, so->output[i].dst_offset + so->output[i].num_components - 1,
2119 so->output[i].register_index,
2120 mask & 1 ? "x" : "",
2121 mask & 2 ? "y" : "",
2122 mask & 4 ? "z" : "",
2123 mask & 8 ? "w" : "");
2124 }
2125 }
2126
2127 static void emit_streamout_output(struct si_shader_context *ctx,
2128 LLVMValueRef const *so_buffers,
2129 LLVMValueRef const *so_write_offsets,
2130 struct pipe_stream_output *stream_out,
2131 struct si_shader_output_values *shader_out)
2132 {
2133 struct gallivm_state *gallivm = &ctx->gallivm;
2134 LLVMBuilderRef builder = gallivm->builder;
2135 unsigned buf_idx = stream_out->output_buffer;
2136 unsigned start = stream_out->start_component;
2137 unsigned num_comps = stream_out->num_components;
2138 LLVMValueRef out[4];
2139
2140 assert(num_comps && num_comps <= 4);
2141 if (!num_comps || num_comps > 4)
2142 return;
2143
2144 /* Load the output as int. */
2145 for (int j = 0; j < num_comps; j++) {
2146 assert(stream_out->stream == shader_out->vertex_stream[start + j]);
2147
2148 out[j] = LLVMBuildBitCast(builder,
2149 shader_out->values[start + j],
2150 ctx->i32, "");
2151 }
2152
2153 /* Pack the output. */
2154 LLVMValueRef vdata = NULL;
2155
2156 switch (num_comps) {
2157 case 1: /* as i32 */
2158 vdata = out[0];
2159 break;
2160 case 2: /* as v2i32 */
2161 case 3: /* as v4i32 (aligned to 4) */
2162 case 4: /* as v4i32 */
2163 vdata = LLVMGetUndef(LLVMVectorType(ctx->i32, util_next_power_of_two(num_comps)));
2164 for (int j = 0; j < num_comps; j++) {
2165 vdata = LLVMBuildInsertElement(builder, vdata, out[j],
2166 LLVMConstInt(ctx->i32, j, 0), "");
2167 }
2168 break;
2169 }
2170
2171 ac_build_buffer_store_dword(&ctx->ac, so_buffers[buf_idx],
2172 vdata, num_comps,
2173 so_write_offsets[buf_idx],
2174 ctx->i32_0,
2175 stream_out->dst_offset * 4, 1, 1, true, false);
2176 }
2177
2178 /**
2179 * Write streamout data to buffers for vertex stream @p stream (different
2180 * vertex streams can occur for GS copy shaders).
2181 */
2182 static void si_llvm_emit_streamout(struct si_shader_context *ctx,
2183 struct si_shader_output_values *outputs,
2184 unsigned noutput, unsigned stream)
2185 {
2186 struct si_shader_selector *sel = ctx->shader->selector;
2187 struct pipe_stream_output_info *so = &sel->so;
2188 struct gallivm_state *gallivm = &ctx->gallivm;
2189 LLVMBuilderRef builder = gallivm->builder;
2190 int i;
2191 struct lp_build_if_state if_ctx;
2192
2193 /* Get bits [22:16], i.e. (so_param >> 16) & 127; */
2194 LLVMValueRef so_vtx_count =
2195 unpack_param(ctx, ctx->param_streamout_config, 16, 7);
2196
2197 LLVMValueRef tid = ac_get_thread_id(&ctx->ac);
2198
2199 /* can_emit = tid < so_vtx_count; */
2200 LLVMValueRef can_emit =
2201 LLVMBuildICmp(builder, LLVMIntULT, tid, so_vtx_count, "");
2202
2203 /* Emit the streamout code conditionally. This actually avoids
2204 * out-of-bounds buffer access. The hw tells us via the SGPR
2205 * (so_vtx_count) which threads are allowed to emit streamout data. */
2206 lp_build_if(&if_ctx, gallivm, can_emit);
2207 {
2208 /* The buffer offset is computed as follows:
2209 * ByteOffset = streamout_offset[buffer_id]*4 +
2210 * (streamout_write_index + thread_id)*stride[buffer_id] +
2211 * attrib_offset
2212 */
2213
2214 LLVMValueRef so_write_index =
2215 LLVMGetParam(ctx->main_fn,
2216 ctx->param_streamout_write_index);
2217
2218 /* Compute (streamout_write_index + thread_id). */
2219 so_write_index = LLVMBuildAdd(builder, so_write_index, tid, "");
2220
2221 /* Load the descriptor and compute the write offset for each
2222 * enabled buffer. */
2223 LLVMValueRef so_write_offset[4] = {};
2224 LLVMValueRef so_buffers[4];
2225 LLVMValueRef buf_ptr = LLVMGetParam(ctx->main_fn,
2226 ctx->param_rw_buffers);
2227
2228 for (i = 0; i < 4; i++) {
2229 if (!so->stride[i])
2230 continue;
2231
2232 LLVMValueRef offset = LLVMConstInt(ctx->i32,
2233 SI_VS_STREAMOUT_BUF0 + i, 0);
2234
2235 so_buffers[i] = ac_build_indexed_load_const(&ctx->ac, buf_ptr, offset);
2236
2237 LLVMValueRef so_offset = LLVMGetParam(ctx->main_fn,
2238 ctx->param_streamout_offset[i]);
2239 so_offset = LLVMBuildMul(builder, so_offset, LLVMConstInt(ctx->i32, 4, 0), "");
2240
2241 so_write_offset[i] = LLVMBuildMul(builder, so_write_index,
2242 LLVMConstInt(ctx->i32, so->stride[i]*4, 0), "");
2243 so_write_offset[i] = LLVMBuildAdd(builder, so_write_offset[i], so_offset, "");
2244 }
2245
2246 /* Write streamout data. */
2247 for (i = 0; i < so->num_outputs; i++) {
2248 unsigned reg = so->output[i].register_index;
2249
2250 if (reg >= noutput)
2251 continue;
2252
2253 if (stream != so->output[i].stream)
2254 continue;
2255
2256 emit_streamout_output(ctx, so_buffers, so_write_offset,
2257 &so->output[i], &outputs[reg]);
2258 }
2259 }
2260 lp_build_endif(&if_ctx);
2261 }
2262
2263
2264 /* Generate export instructions for hardware VS shader stage */
2265 static void si_llvm_export_vs(struct lp_build_tgsi_context *bld_base,
2266 struct si_shader_output_values *outputs,
2267 unsigned noutput)
2268 {
2269 struct si_shader_context *ctx = si_shader_context(bld_base);
2270 struct si_shader *shader = ctx->shader;
2271 struct lp_build_context *base = &bld_base->base;
2272 struct ac_export_args args, pos_args[4] = {};
2273 LLVMValueRef psize_value = NULL, edgeflag_value = NULL, layer_value = NULL, viewport_index_value = NULL;
2274 unsigned semantic_name, semantic_index;
2275 unsigned target;
2276 unsigned param_count = 0;
2277 unsigned pos_idx;
2278 int i;
2279
2280 for (i = 0; i < noutput; i++) {
2281 semantic_name = outputs[i].semantic_name;
2282 semantic_index = outputs[i].semantic_index;
2283 bool export_param = true;
2284
2285 switch (semantic_name) {
2286 case TGSI_SEMANTIC_POSITION: /* ignore these */
2287 case TGSI_SEMANTIC_PSIZE:
2288 case TGSI_SEMANTIC_CLIPVERTEX:
2289 case TGSI_SEMANTIC_EDGEFLAG:
2290 break;
2291 case TGSI_SEMANTIC_GENERIC:
2292 /* don't process indices the function can't handle */
2293 if (semantic_index >= SI_MAX_IO_GENERIC)
2294 break;
2295 /* fall through */
2296 default:
2297 if (shader->key.opt.hw_vs.kill_outputs &
2298 (1ull << si_shader_io_get_unique_index(semantic_name, semantic_index)))
2299 export_param = false;
2300 }
2301
2302 if (outputs[i].vertex_stream[0] != 0 &&
2303 outputs[i].vertex_stream[1] != 0 &&
2304 outputs[i].vertex_stream[2] != 0 &&
2305 outputs[i].vertex_stream[3] != 0)
2306 export_param = false;
2307
2308 handle_semantic:
2309 /* Select the correct target */
2310 switch(semantic_name) {
2311 case TGSI_SEMANTIC_PSIZE:
2312 psize_value = outputs[i].values[0];
2313 continue;
2314 case TGSI_SEMANTIC_EDGEFLAG:
2315 edgeflag_value = outputs[i].values[0];
2316 continue;
2317 case TGSI_SEMANTIC_LAYER:
2318 layer_value = outputs[i].values[0];
2319 semantic_name = TGSI_SEMANTIC_GENERIC;
2320 goto handle_semantic;
2321 case TGSI_SEMANTIC_VIEWPORT_INDEX:
2322 viewport_index_value = outputs[i].values[0];
2323 semantic_name = TGSI_SEMANTIC_GENERIC;
2324 goto handle_semantic;
2325 case TGSI_SEMANTIC_POSITION:
2326 target = V_008DFC_SQ_EXP_POS;
2327 break;
2328 case TGSI_SEMANTIC_CLIPDIST:
2329 if (shader->key.opt.hw_vs.clip_disable) {
2330 semantic_name = TGSI_SEMANTIC_GENERIC;
2331 goto handle_semantic;
2332 }
2333 target = V_008DFC_SQ_EXP_POS + 2 + semantic_index;
2334 break;
2335 case TGSI_SEMANTIC_CLIPVERTEX:
2336 if (shader->key.opt.hw_vs.clip_disable)
2337 continue;
2338 si_llvm_emit_clipvertex(bld_base, pos_args, outputs[i].values);
2339 continue;
2340 case TGSI_SEMANTIC_COLOR:
2341 case TGSI_SEMANTIC_BCOLOR:
2342 case TGSI_SEMANTIC_PRIMID:
2343 case TGSI_SEMANTIC_FOG:
2344 case TGSI_SEMANTIC_TEXCOORD:
2345 case TGSI_SEMANTIC_GENERIC:
2346 if (!export_param)
2347 continue;
2348 target = V_008DFC_SQ_EXP_PARAM + param_count;
2349 assert(i < ARRAY_SIZE(shader->info.vs_output_param_offset));
2350 shader->info.vs_output_param_offset[i] = param_count;
2351 param_count++;
2352 break;
2353 default:
2354 target = 0;
2355 fprintf(stderr,
2356 "Warning: SI unhandled vs output type:%d\n",
2357 semantic_name);
2358 }
2359
2360 si_llvm_init_export_args(bld_base, outputs[i].values, target, &args);
2361
2362 if (target >= V_008DFC_SQ_EXP_POS &&
2363 target <= (V_008DFC_SQ_EXP_POS + 3)) {
2364 memcpy(&pos_args[target - V_008DFC_SQ_EXP_POS],
2365 &args, sizeof(args));
2366 } else {
2367 ac_build_export(&ctx->ac, &args);
2368 }
2369
2370 if (semantic_name == TGSI_SEMANTIC_CLIPDIST) {
2371 semantic_name = TGSI_SEMANTIC_GENERIC;
2372 goto handle_semantic;
2373 }
2374 }
2375
2376 shader->info.nr_param_exports = param_count;
2377
2378 /* We need to add the position output manually if it's missing. */
2379 if (!pos_args[0].out[0]) {
2380 pos_args[0].enabled_channels = 0xf; /* writemask */
2381 pos_args[0].valid_mask = 0; /* EXEC mask */
2382 pos_args[0].done = 0; /* last export? */
2383 pos_args[0].target = V_008DFC_SQ_EXP_POS;
2384 pos_args[0].compr = 0; /* COMPR flag */
2385 pos_args[0].out[0] = base->zero; /* X */
2386 pos_args[0].out[1] = base->zero; /* Y */
2387 pos_args[0].out[2] = base->zero; /* Z */
2388 pos_args[0].out[3] = base->one; /* W */
2389 }
2390
2391 /* Write the misc vector (point size, edgeflag, layer, viewport). */
2392 if (shader->selector->info.writes_psize ||
2393 shader->selector->info.writes_edgeflag ||
2394 shader->selector->info.writes_viewport_index ||
2395 shader->selector->info.writes_layer) {
2396 pos_args[1].enabled_channels = shader->selector->info.writes_psize |
2397 (shader->selector->info.writes_edgeflag << 1) |
2398 (shader->selector->info.writes_layer << 2);
2399
2400 pos_args[1].valid_mask = 0; /* EXEC mask */
2401 pos_args[1].done = 0; /* last export? */
2402 pos_args[1].target = V_008DFC_SQ_EXP_POS + 1;
2403 pos_args[1].compr = 0; /* COMPR flag */
2404 pos_args[1].out[0] = base->zero; /* X */
2405 pos_args[1].out[1] = base->zero; /* Y */
2406 pos_args[1].out[2] = base->zero; /* Z */
2407 pos_args[1].out[3] = base->zero; /* W */
2408
2409 if (shader->selector->info.writes_psize)
2410 pos_args[1].out[0] = psize_value;
2411
2412 if (shader->selector->info.writes_edgeflag) {
2413 /* The output is a float, but the hw expects an integer
2414 * with the first bit containing the edge flag. */
2415 edgeflag_value = LLVMBuildFPToUI(ctx->gallivm.builder,
2416 edgeflag_value,
2417 ctx->i32, "");
2418 edgeflag_value = lp_build_min(&bld_base->int_bld,
2419 edgeflag_value,
2420 ctx->i32_1);
2421
2422 /* The LLVM intrinsic expects a float. */
2423 pos_args[1].out[1] = LLVMBuildBitCast(ctx->gallivm.builder,
2424 edgeflag_value,
2425 ctx->f32, "");
2426 }
2427
2428 if (ctx->screen->b.chip_class >= GFX9) {
2429 /* GFX9 has the layer in out.z[10:0] and the viewport
2430 * index in out.z[19:16].
2431 */
2432 if (shader->selector->info.writes_layer)
2433 pos_args[1].out[2] = layer_value;
2434
2435 if (shader->selector->info.writes_viewport_index) {
2436 LLVMValueRef v = viewport_index_value;
2437
2438 v = bitcast(bld_base, TGSI_TYPE_UNSIGNED, v);
2439 v = LLVMBuildShl(ctx->gallivm.builder, v,
2440 LLVMConstInt(ctx->i32, 16, 0), "");
2441 v = LLVMBuildOr(ctx->gallivm.builder, v,
2442 bitcast(bld_base, TGSI_TYPE_UNSIGNED,
2443 pos_args[1].out[2]), "");
2444 pos_args[1].out[2] = bitcast(bld_base, TGSI_TYPE_FLOAT, v);
2445 pos_args[1].enabled_channels |= 1 << 2;
2446 }
2447 } else {
2448 if (shader->selector->info.writes_layer)
2449 pos_args[1].out[2] = layer_value;
2450
2451 if (shader->selector->info.writes_viewport_index) {
2452 pos_args[1].out[3] = viewport_index_value;
2453 pos_args[1].enabled_channels |= 1 << 3;
2454 }
2455 }
2456 }
2457
2458 for (i = 0; i < 4; i++)
2459 if (pos_args[i].out[0])
2460 shader->info.nr_pos_exports++;
2461
2462 pos_idx = 0;
2463 for (i = 0; i < 4; i++) {
2464 if (!pos_args[i].out[0])
2465 continue;
2466
2467 /* Specify the target we are exporting */
2468 pos_args[i].target = V_008DFC_SQ_EXP_POS + pos_idx++;
2469
2470 if (pos_idx == shader->info.nr_pos_exports)
2471 /* Specify that this is the last export */
2472 pos_args[i].done = 1;
2473
2474 ac_build_export(&ctx->ac, &pos_args[i]);
2475 }
2476 }
2477
2478 /**
2479 * Forward all outputs from the vertex shader to the TES. This is only used
2480 * for the fixed function TCS.
2481 */
2482 static void si_copy_tcs_inputs(struct lp_build_tgsi_context *bld_base)
2483 {
2484 struct si_shader_context *ctx = si_shader_context(bld_base);
2485 struct gallivm_state *gallivm = &ctx->gallivm;
2486 LLVMValueRef invocation_id, buffer, buffer_offset;
2487 LLVMValueRef lds_vertex_stride, lds_vertex_offset, lds_base;
2488 uint64_t inputs;
2489
2490 invocation_id = unpack_param(ctx, ctx->param_tcs_rel_ids, 8, 5);
2491 buffer = desc_from_addr_base64k(ctx, ctx->param_tcs_offchip_addr_base64k);
2492 buffer_offset = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
2493
2494 lds_vertex_stride = unpack_param(ctx, ctx->param_vs_state_bits, 24, 8);
2495 lds_vertex_offset = LLVMBuildMul(gallivm->builder, invocation_id,
2496 lds_vertex_stride, "");
2497 lds_base = get_tcs_in_current_patch_offset(ctx);
2498 lds_base = LLVMBuildAdd(gallivm->builder, lds_base, lds_vertex_offset, "");
2499
2500 inputs = ctx->shader->key.mono.ff_tcs_inputs_to_copy;
2501 while (inputs) {
2502 unsigned i = u_bit_scan64(&inputs);
2503
2504 LLVMValueRef lds_ptr = LLVMBuildAdd(gallivm->builder, lds_base,
2505 LLVMConstInt(ctx->i32, 4 * i, 0),
2506 "");
2507
2508 LLVMValueRef buffer_addr = get_tcs_tes_buffer_address(ctx,
2509 get_rel_patch_id(ctx),
2510 invocation_id,
2511 LLVMConstInt(ctx->i32, i, 0));
2512
2513 LLVMValueRef value = lds_load(bld_base, TGSI_TYPE_SIGNED, ~0,
2514 lds_ptr);
2515
2516 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, buffer_addr,
2517 buffer_offset, 0, 1, 0, true, false);
2518 }
2519 }
2520
2521 static void si_write_tess_factors(struct lp_build_tgsi_context *bld_base,
2522 LLVMValueRef rel_patch_id,
2523 LLVMValueRef invocation_id,
2524 LLVMValueRef tcs_out_current_patch_data_offset)
2525 {
2526 struct si_shader_context *ctx = si_shader_context(bld_base);
2527 struct gallivm_state *gallivm = &ctx->gallivm;
2528 struct si_shader *shader = ctx->shader;
2529 unsigned tess_inner_index, tess_outer_index;
2530 LLVMValueRef lds_base, lds_inner, lds_outer, byteoffset, buffer;
2531 LLVMValueRef out[6], vec0, vec1, tf_base, inner[4], outer[4];
2532 unsigned stride, outer_comps, inner_comps, i, offset;
2533 struct lp_build_if_state if_ctx, inner_if_ctx;
2534
2535 si_llvm_emit_barrier(NULL, bld_base, NULL);
2536
2537 /* Do this only for invocation 0, because the tess levels are per-patch,
2538 * not per-vertex.
2539 *
2540 * This can't jump, because invocation 0 executes this. It should
2541 * at least mask out the loads and stores for other invocations.
2542 */
2543 lp_build_if(&if_ctx, gallivm,
2544 LLVMBuildICmp(gallivm->builder, LLVMIntEQ,
2545 invocation_id, ctx->i32_0, ""));
2546
2547 /* Determine the layout of one tess factor element in the buffer. */
2548 switch (shader->key.part.tcs.epilog.prim_mode) {
2549 case PIPE_PRIM_LINES:
2550 stride = 2; /* 2 dwords, 1 vec2 store */
2551 outer_comps = 2;
2552 inner_comps = 0;
2553 break;
2554 case PIPE_PRIM_TRIANGLES:
2555 stride = 4; /* 4 dwords, 1 vec4 store */
2556 outer_comps = 3;
2557 inner_comps = 1;
2558 break;
2559 case PIPE_PRIM_QUADS:
2560 stride = 6; /* 6 dwords, 2 stores (vec4 + vec2) */
2561 outer_comps = 4;
2562 inner_comps = 2;
2563 break;
2564 default:
2565 assert(0);
2566 return;
2567 }
2568
2569 /* Load tess_inner and tess_outer from LDS.
2570 * Any invocation can write them, so we can't get them from a temporary.
2571 */
2572 tess_inner_index = si_shader_io_get_unique_index_patch(TGSI_SEMANTIC_TESSINNER, 0);
2573 tess_outer_index = si_shader_io_get_unique_index_patch(TGSI_SEMANTIC_TESSOUTER, 0);
2574
2575 lds_base = tcs_out_current_patch_data_offset;
2576 lds_inner = LLVMBuildAdd(gallivm->builder, lds_base,
2577 LLVMConstInt(ctx->i32,
2578 tess_inner_index * 4, 0), "");
2579 lds_outer = LLVMBuildAdd(gallivm->builder, lds_base,
2580 LLVMConstInt(ctx->i32,
2581 tess_outer_index * 4, 0), "");
2582
2583 for (i = 0; i < 4; i++) {
2584 inner[i] = LLVMGetUndef(ctx->i32);
2585 outer[i] = LLVMGetUndef(ctx->i32);
2586 }
2587
2588 if (shader->key.part.tcs.epilog.prim_mode == PIPE_PRIM_LINES) {
2589 /* For isolines, the hardware expects tess factors in the
2590 * reverse order from what GLSL / TGSI specify.
2591 */
2592 outer[0] = out[1] = lds_load(bld_base, TGSI_TYPE_SIGNED, 0, lds_outer);
2593 outer[1] = out[0] = lds_load(bld_base, TGSI_TYPE_SIGNED, 1, lds_outer);
2594 } else {
2595 for (i = 0; i < outer_comps; i++) {
2596 outer[i] = out[i] =
2597 lds_load(bld_base, TGSI_TYPE_SIGNED, i, lds_outer);
2598 }
2599 for (i = 0; i < inner_comps; i++) {
2600 inner[i] = out[outer_comps+i] =
2601 lds_load(bld_base, TGSI_TYPE_SIGNED, i, lds_inner);
2602 }
2603 }
2604
2605 /* Convert the outputs to vectors for stores. */
2606 vec0 = lp_build_gather_values(gallivm, out, MIN2(stride, 4));
2607 vec1 = NULL;
2608
2609 if (stride > 4)
2610 vec1 = lp_build_gather_values(gallivm, out+4, stride - 4);
2611
2612 /* Get the buffer. */
2613 buffer = desc_from_addr_base64k(ctx, ctx->param_tcs_factor_addr_base64k);
2614
2615 /* Get the offset. */
2616 tf_base = LLVMGetParam(ctx->main_fn,
2617 ctx->param_tcs_factor_offset);
2618 byteoffset = LLVMBuildMul(gallivm->builder, rel_patch_id,
2619 LLVMConstInt(ctx->i32, 4 * stride, 0), "");
2620
2621 lp_build_if(&inner_if_ctx, gallivm,
2622 LLVMBuildICmp(gallivm->builder, LLVMIntEQ,
2623 rel_patch_id, ctx->i32_0, ""));
2624
2625 /* Store the dynamic HS control word. */
2626 offset = 0;
2627 if (ctx->screen->b.chip_class <= VI) {
2628 ac_build_buffer_store_dword(&ctx->ac, buffer,
2629 LLVMConstInt(ctx->i32, 0x80000000, 0),
2630 1, ctx->i32_0, tf_base,
2631 offset, 1, 0, true, false);
2632 offset += 4;
2633 }
2634
2635 lp_build_endif(&inner_if_ctx);
2636
2637 /* Store the tessellation factors. */
2638 ac_build_buffer_store_dword(&ctx->ac, buffer, vec0,
2639 MIN2(stride, 4), byteoffset, tf_base,
2640 offset, 1, 0, true, false);
2641 offset += 16;
2642 if (vec1)
2643 ac_build_buffer_store_dword(&ctx->ac, buffer, vec1,
2644 stride - 4, byteoffset, tf_base,
2645 offset, 1, 0, true, false);
2646
2647 /* Store the tess factors into the offchip buffer if TES reads them. */
2648 if (shader->key.part.tcs.epilog.tes_reads_tess_factors) {
2649 LLVMValueRef buf, base, inner_vec, outer_vec, tf_outer_offset;
2650 LLVMValueRef tf_inner_offset;
2651 unsigned param_outer, param_inner;
2652
2653 buf = desc_from_addr_base64k(ctx, ctx->param_tcs_offchip_addr_base64k);
2654 base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
2655
2656 param_outer = si_shader_io_get_unique_index_patch(
2657 TGSI_SEMANTIC_TESSOUTER, 0);
2658 tf_outer_offset = get_tcs_tes_buffer_address(ctx, rel_patch_id, NULL,
2659 LLVMConstInt(ctx->i32, param_outer, 0));
2660
2661 outer_vec = lp_build_gather_values(gallivm, outer,
2662 util_next_power_of_two(outer_comps));
2663
2664 ac_build_buffer_store_dword(&ctx->ac, buf, outer_vec,
2665 outer_comps, tf_outer_offset,
2666 base, 0, 1, 0, true, false);
2667 if (inner_comps) {
2668 param_inner = si_shader_io_get_unique_index_patch(
2669 TGSI_SEMANTIC_TESSINNER, 0);
2670 tf_inner_offset = get_tcs_tes_buffer_address(ctx, rel_patch_id, NULL,
2671 LLVMConstInt(ctx->i32, param_inner, 0));
2672
2673 inner_vec = inner_comps == 1 ? inner[0] :
2674 lp_build_gather_values(gallivm, inner, inner_comps);
2675 ac_build_buffer_store_dword(&ctx->ac, buf, inner_vec,
2676 inner_comps, tf_inner_offset,
2677 base, 0, 1, 0, true, false);
2678 }
2679 }
2680
2681 lp_build_endif(&if_ctx);
2682 }
2683
2684 static LLVMValueRef
2685 si_insert_input_ret(struct si_shader_context *ctx, LLVMValueRef ret,
2686 unsigned param, unsigned return_index)
2687 {
2688 return LLVMBuildInsertValue(ctx->gallivm.builder, ret,
2689 LLVMGetParam(ctx->main_fn, param),
2690 return_index, "");
2691 }
2692
2693 static LLVMValueRef
2694 si_insert_input_ret_float(struct si_shader_context *ctx, LLVMValueRef ret,
2695 unsigned param, unsigned return_index)
2696 {
2697 LLVMBuilderRef builder = ctx->gallivm.builder;
2698 LLVMValueRef p = LLVMGetParam(ctx->main_fn, param);
2699
2700 return LLVMBuildInsertValue(builder, ret,
2701 LLVMBuildBitCast(builder, p, ctx->f32, ""),
2702 return_index, "");
2703 }
2704
2705 static LLVMValueRef
2706 si_insert_input_ptr_as_2xi32(struct si_shader_context *ctx, LLVMValueRef ret,
2707 unsigned param, unsigned return_index)
2708 {
2709 LLVMBuilderRef builder = ctx->gallivm.builder;
2710 LLVMValueRef ptr, lo, hi;
2711
2712 ptr = LLVMGetParam(ctx->main_fn, param);
2713 ptr = LLVMBuildPtrToInt(builder, ptr, ctx->i64, "");
2714 ptr = LLVMBuildBitCast(builder, ptr, ctx->v2i32, "");
2715 lo = LLVMBuildExtractElement(builder, ptr, ctx->i32_0, "");
2716 hi = LLVMBuildExtractElement(builder, ptr, ctx->i32_1, "");
2717 ret = LLVMBuildInsertValue(builder, ret, lo, return_index, "");
2718 return LLVMBuildInsertValue(builder, ret, hi, return_index + 1, "");
2719 }
2720
2721 /* This only writes the tessellation factor levels. */
2722 static void si_llvm_emit_tcs_epilogue(struct lp_build_tgsi_context *bld_base)
2723 {
2724 struct si_shader_context *ctx = si_shader_context(bld_base);
2725 LLVMValueRef rel_patch_id, invocation_id, tf_lds_offset;
2726
2727 si_copy_tcs_inputs(bld_base);
2728
2729 rel_patch_id = get_rel_patch_id(ctx);
2730 invocation_id = unpack_param(ctx, ctx->param_tcs_rel_ids, 8, 5);
2731 tf_lds_offset = get_tcs_out_current_patch_data_offset(ctx);
2732
2733 /* Return epilog parameters from this function. */
2734 LLVMBuilderRef builder = ctx->gallivm.builder;
2735 LLVMValueRef ret = ctx->return_value;
2736 unsigned vgpr;
2737
2738 if (ctx->screen->b.chip_class >= GFX9) {
2739 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_layout,
2740 8 + GFX9_SGPR_TCS_OFFCHIP_LAYOUT);
2741 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_addr_base64k,
2742 8 + GFX9_SGPR_TCS_OFFCHIP_ADDR_BASE64K);
2743 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_addr_base64k,
2744 8 + GFX9_SGPR_TCS_FACTOR_ADDR_BASE64K);
2745 /* Tess offchip and tess factor offsets are at the beginning. */
2746 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_offset, 2);
2747 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_offset, 4);
2748 vgpr = 8 + GFX9_SGPR_TCS_FACTOR_ADDR_BASE64K + 1;
2749 } else {
2750 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_layout,
2751 GFX6_SGPR_TCS_OFFCHIP_LAYOUT);
2752 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_addr_base64k,
2753 GFX6_SGPR_TCS_OFFCHIP_ADDR_BASE64K);
2754 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_addr_base64k,
2755 GFX6_SGPR_TCS_FACTOR_ADDR_BASE64K);
2756 /* Tess offchip and tess factor offsets are after user SGPRs. */
2757 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_offset,
2758 GFX6_TCS_NUM_USER_SGPR);
2759 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_offset,
2760 GFX6_TCS_NUM_USER_SGPR + 1);
2761 vgpr = GFX6_TCS_NUM_USER_SGPR + 2;
2762 }
2763
2764 /* VGPRs */
2765 rel_patch_id = bitcast(bld_base, TGSI_TYPE_FLOAT, rel_patch_id);
2766 invocation_id = bitcast(bld_base, TGSI_TYPE_FLOAT, invocation_id);
2767 tf_lds_offset = bitcast(bld_base, TGSI_TYPE_FLOAT, tf_lds_offset);
2768
2769 ret = LLVMBuildInsertValue(builder, ret, rel_patch_id, vgpr++, "");
2770 ret = LLVMBuildInsertValue(builder, ret, invocation_id, vgpr++, "");
2771 ret = LLVMBuildInsertValue(builder, ret, tf_lds_offset, vgpr++, "");
2772 ctx->return_value = ret;
2773 }
2774
2775 /* Pass TCS inputs from LS to TCS on GFX9. */
2776 static void si_set_ls_return_value_for_tcs(struct si_shader_context *ctx)
2777 {
2778 LLVMValueRef ret = ctx->return_value;
2779
2780 ret = si_insert_input_ptr_as_2xi32(ctx, ret, ctx->param_rw_buffers, 0);
2781 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_offset, 2);
2782 ret = si_insert_input_ret(ctx, ret, ctx->param_merged_wave_info, 3);
2783 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_offset, 4);
2784 ret = si_insert_input_ret(ctx, ret, ctx->param_merged_scratch_offset, 5);
2785
2786 ret = si_insert_input_ret(ctx, ret, ctx->param_vs_state_bits,
2787 8 + SI_SGPR_VS_STATE_BITS);
2788 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_layout,
2789 8 + GFX9_SGPR_TCS_OFFCHIP_LAYOUT);
2790 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_out_lds_offsets,
2791 8 + GFX9_SGPR_TCS_OUT_OFFSETS);
2792 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_out_lds_layout,
2793 8 + GFX9_SGPR_TCS_OUT_LAYOUT);
2794 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_addr_base64k,
2795 8 + GFX9_SGPR_TCS_OFFCHIP_ADDR_BASE64K);
2796 ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_addr_base64k,
2797 8 + GFX9_SGPR_TCS_FACTOR_ADDR_BASE64K);
2798
2799 unsigned desc_param = ctx->param_tcs_factor_addr_base64k + 2;
2800 ret = si_insert_input_ptr_as_2xi32(ctx, ret, desc_param,
2801 8 + GFX9_SGPR_TCS_CONST_AND_SHADER_BUFFERS);
2802 ret = si_insert_input_ptr_as_2xi32(ctx, ret, desc_param + 1,
2803 8 + GFX9_SGPR_TCS_SAMPLERS_AND_IMAGES);
2804
2805 unsigned vgpr = 8 + GFX9_TCS_NUM_USER_SGPR;
2806 ret = si_insert_input_ret_float(ctx, ret,
2807 ctx->param_tcs_patch_id, vgpr++);
2808 ret = si_insert_input_ret_float(ctx, ret,
2809 ctx->param_tcs_rel_ids, vgpr++);
2810 ctx->return_value = ret;
2811 }
2812
2813 /* Pass GS inputs from ES to GS on GFX9. */
2814 static void si_set_es_return_value_for_gs(struct si_shader_context *ctx)
2815 {
2816 LLVMValueRef ret = ctx->return_value;
2817
2818 ret = si_insert_input_ptr_as_2xi32(ctx, ret, ctx->param_rw_buffers, 0);
2819 ret = si_insert_input_ret(ctx, ret, ctx->param_gs2vs_offset, 2);
2820 ret = si_insert_input_ret(ctx, ret, ctx->param_merged_wave_info, 3);
2821
2822 ret = si_insert_input_ret(ctx, ret, ctx->param_merged_scratch_offset, 5);
2823
2824 unsigned desc_param = ctx->param_vs_state_bits + 1;
2825 ret = si_insert_input_ptr_as_2xi32(ctx, ret, desc_param,
2826 8 + GFX9_SGPR_GS_CONST_AND_SHADER_BUFFERS);
2827 ret = si_insert_input_ptr_as_2xi32(ctx, ret, desc_param + 1,
2828 8 + GFX9_SGPR_GS_SAMPLERS_AND_IMAGES);
2829
2830 unsigned vgpr = 8 + GFX9_GS_NUM_USER_SGPR;
2831 for (unsigned i = 0; i < 5; i++) {
2832 unsigned param = ctx->param_gs_vtx01_offset + i;
2833 ret = si_insert_input_ret_float(ctx, ret, param, vgpr++);
2834 }
2835 ctx->return_value = ret;
2836 }
2837
2838 static void si_llvm_emit_ls_epilogue(struct lp_build_tgsi_context *bld_base)
2839 {
2840 struct si_shader_context *ctx = si_shader_context(bld_base);
2841 struct si_shader *shader = ctx->shader;
2842 struct tgsi_shader_info *info = &shader->selector->info;
2843 struct gallivm_state *gallivm = &ctx->gallivm;
2844 unsigned i, chan;
2845 LLVMValueRef vertex_id = LLVMGetParam(ctx->main_fn,
2846 ctx->param_rel_auto_id);
2847 LLVMValueRef vertex_dw_stride =
2848 unpack_param(ctx, ctx->param_vs_state_bits, 24, 8);
2849 LLVMValueRef base_dw_addr = LLVMBuildMul(gallivm->builder, vertex_id,
2850 vertex_dw_stride, "");
2851
2852 /* Write outputs to LDS. The next shader (TCS aka HS) will read
2853 * its inputs from it. */
2854 for (i = 0; i < info->num_outputs; i++) {
2855 LLVMValueRef *out_ptr = ctx->outputs[i];
2856 unsigned name = info->output_semantic_name[i];
2857 unsigned index = info->output_semantic_index[i];
2858
2859 /* The ARB_shader_viewport_layer_array spec contains the
2860 * following issue:
2861 *
2862 * 2) What happens if gl_ViewportIndex or gl_Layer is
2863 * written in the vertex shader and a geometry shader is
2864 * present?
2865 *
2866 * RESOLVED: The value written by the last vertex processing
2867 * stage is used. If the last vertex processing stage
2868 * (vertex, tessellation evaluation or geometry) does not
2869 * statically assign to gl_ViewportIndex or gl_Layer, index
2870 * or layer zero is assumed.
2871 *
2872 * So writes to those outputs in VS-as-LS are simply ignored.
2873 */
2874 if (name == TGSI_SEMANTIC_LAYER ||
2875 name == TGSI_SEMANTIC_VIEWPORT_INDEX)
2876 continue;
2877
2878 int param = si_shader_io_get_unique_index(name, index);
2879 LLVMValueRef dw_addr = LLVMBuildAdd(gallivm->builder, base_dw_addr,
2880 LLVMConstInt(ctx->i32, param * 4, 0), "");
2881
2882 for (chan = 0; chan < 4; chan++) {
2883 lds_store(bld_base, chan, dw_addr,
2884 LLVMBuildLoad(gallivm->builder, out_ptr[chan], ""));
2885 }
2886 }
2887
2888 if (ctx->screen->b.chip_class >= GFX9)
2889 si_set_ls_return_value_for_tcs(ctx);
2890 }
2891
2892 static void si_llvm_emit_es_epilogue(struct lp_build_tgsi_context *bld_base)
2893 {
2894 struct si_shader_context *ctx = si_shader_context(bld_base);
2895 struct gallivm_state *gallivm = &ctx->gallivm;
2896 struct si_shader *es = ctx->shader;
2897 struct tgsi_shader_info *info = &es->selector->info;
2898 LLVMValueRef soffset = LLVMGetParam(ctx->main_fn,
2899 ctx->param_es2gs_offset);
2900 LLVMValueRef lds_base = NULL;
2901 unsigned chan;
2902 int i;
2903
2904 if (ctx->screen->b.chip_class >= GFX9 && info->num_outputs) {
2905 unsigned itemsize_dw = es->selector->esgs_itemsize / 4;
2906 lds_base = LLVMBuildMul(gallivm->builder, ac_get_thread_id(&ctx->ac),
2907 LLVMConstInt(ctx->i32, itemsize_dw, 0), "");
2908 }
2909
2910 for (i = 0; i < info->num_outputs; i++) {
2911 LLVMValueRef *out_ptr = ctx->outputs[i];
2912 int param;
2913
2914 if (info->output_semantic_name[i] == TGSI_SEMANTIC_VIEWPORT_INDEX ||
2915 info->output_semantic_name[i] == TGSI_SEMANTIC_LAYER)
2916 continue;
2917
2918 param = si_shader_io_get_unique_index(info->output_semantic_name[i],
2919 info->output_semantic_index[i]);
2920
2921 for (chan = 0; chan < 4; chan++) {
2922 LLVMValueRef out_val = LLVMBuildLoad(gallivm->builder, out_ptr[chan], "");
2923 out_val = LLVMBuildBitCast(gallivm->builder, out_val, ctx->i32, "");
2924
2925 /* GFX9 has the ESGS ring in LDS. */
2926 if (ctx->screen->b.chip_class >= GFX9) {
2927 lds_store(bld_base, param * 4 + chan, lds_base, out_val);
2928 continue;
2929 }
2930
2931 ac_build_buffer_store_dword(&ctx->ac,
2932 ctx->esgs_ring,
2933 out_val, 1, NULL, soffset,
2934 (4 * param + chan) * 4,
2935 1, 1, true, true);
2936 }
2937 }
2938
2939 if (ctx->screen->b.chip_class >= GFX9)
2940 si_set_es_return_value_for_gs(ctx);
2941 }
2942
2943 static LLVMValueRef si_get_gs_wave_id(struct si_shader_context *ctx)
2944 {
2945 if (ctx->screen->b.chip_class >= GFX9)
2946 return unpack_param(ctx, ctx->param_merged_wave_info, 16, 8);
2947 else
2948 return LLVMGetParam(ctx->main_fn, ctx->param_gs_wave_id);
2949 }
2950
2951 static void si_llvm_emit_gs_epilogue(struct lp_build_tgsi_context *bld_base)
2952 {
2953 struct si_shader_context *ctx = si_shader_context(bld_base);
2954
2955 ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_NOP | AC_SENDMSG_GS_DONE,
2956 si_get_gs_wave_id(ctx));
2957 }
2958
2959 static void si_llvm_emit_vs_epilogue(struct lp_build_tgsi_context *bld_base)
2960 {
2961 struct si_shader_context *ctx = si_shader_context(bld_base);
2962 struct gallivm_state *gallivm = &ctx->gallivm;
2963 struct tgsi_shader_info *info = &ctx->shader->selector->info;
2964 struct si_shader_output_values *outputs = NULL;
2965 int i,j;
2966
2967 assert(!ctx->shader->is_gs_copy_shader);
2968
2969 outputs = MALLOC((info->num_outputs + 1) * sizeof(outputs[0]));
2970
2971 /* Vertex color clamping.
2972 *
2973 * This uses a state constant loaded in a user data SGPR and
2974 * an IF statement is added that clamps all colors if the constant
2975 * is true.
2976 */
2977 if (ctx->type == PIPE_SHADER_VERTEX) {
2978 struct lp_build_if_state if_ctx;
2979 LLVMValueRef cond = NULL;
2980 LLVMValueRef addr, val;
2981
2982 for (i = 0; i < info->num_outputs; i++) {
2983 if (info->output_semantic_name[i] != TGSI_SEMANTIC_COLOR &&
2984 info->output_semantic_name[i] != TGSI_SEMANTIC_BCOLOR)
2985 continue;
2986
2987 /* We've found a color. */
2988 if (!cond) {
2989 /* The state is in the first bit of the user SGPR. */
2990 cond = LLVMGetParam(ctx->main_fn,
2991 ctx->param_vs_state_bits);
2992 cond = LLVMBuildTrunc(gallivm->builder, cond,
2993 ctx->i1, "");
2994 lp_build_if(&if_ctx, gallivm, cond);
2995 }
2996
2997 for (j = 0; j < 4; j++) {
2998 addr = ctx->outputs[i][j];
2999 val = LLVMBuildLoad(gallivm->builder, addr, "");
3000 val = ac_build_clamp(&ctx->ac, val);
3001 LLVMBuildStore(gallivm->builder, val, addr);
3002 }
3003 }
3004
3005 if (cond)
3006 lp_build_endif(&if_ctx);
3007 }
3008
3009 for (i = 0; i < info->num_outputs; i++) {
3010 outputs[i].semantic_name = info->output_semantic_name[i];
3011 outputs[i].semantic_index = info->output_semantic_index[i];
3012
3013 for (j = 0; j < 4; j++) {
3014 outputs[i].values[j] =
3015 LLVMBuildLoad(gallivm->builder,
3016 ctx->outputs[i][j],
3017 "");
3018 outputs[i].vertex_stream[j] =
3019 (info->output_streams[i] >> (2 * j)) & 3;
3020 }
3021 }
3022
3023 if (ctx->shader->selector->so.num_outputs)
3024 si_llvm_emit_streamout(ctx, outputs, i, 0);
3025
3026 /* Export PrimitiveID. */
3027 if (ctx->shader->key.mono.vs_export_prim_id) {
3028 outputs[i].semantic_name = TGSI_SEMANTIC_PRIMID;
3029 outputs[i].semantic_index = 0;
3030 outputs[i].values[0] = bitcast(bld_base, TGSI_TYPE_FLOAT,
3031 get_primitive_id(bld_base, 0));
3032 for (j = 1; j < 4; j++)
3033 outputs[i].values[j] = LLVMConstReal(ctx->f32, 0);
3034
3035 memset(outputs[i].vertex_stream, 0,
3036 sizeof(outputs[i].vertex_stream));
3037 i++;
3038 }
3039
3040 si_llvm_export_vs(bld_base, outputs, i);
3041 FREE(outputs);
3042 }
3043
3044 struct si_ps_exports {
3045 unsigned num;
3046 struct ac_export_args args[10];
3047 };
3048
3049 unsigned si_get_spi_shader_z_format(bool writes_z, bool writes_stencil,
3050 bool writes_samplemask)
3051 {
3052 if (writes_z) {
3053 /* Z needs 32 bits. */
3054 if (writes_samplemask)
3055 return V_028710_SPI_SHADER_32_ABGR;
3056 else if (writes_stencil)
3057 return V_028710_SPI_SHADER_32_GR;
3058 else
3059 return V_028710_SPI_SHADER_32_R;
3060 } else if (writes_stencil || writes_samplemask) {
3061 /* Both stencil and sample mask need only 16 bits. */
3062 return V_028710_SPI_SHADER_UINT16_ABGR;
3063 } else {
3064 return V_028710_SPI_SHADER_ZERO;
3065 }
3066 }
3067
3068 static void si_export_mrt_z(struct lp_build_tgsi_context *bld_base,
3069 LLVMValueRef depth, LLVMValueRef stencil,
3070 LLVMValueRef samplemask, struct si_ps_exports *exp)
3071 {
3072 struct si_shader_context *ctx = si_shader_context(bld_base);
3073 struct lp_build_context *base = &bld_base->base;
3074 struct ac_export_args args;
3075 unsigned mask = 0;
3076 unsigned format = si_get_spi_shader_z_format(depth != NULL,
3077 stencil != NULL,
3078 samplemask != NULL);
3079
3080 assert(depth || stencil || samplemask);
3081
3082 args.valid_mask = 1; /* whether the EXEC mask is valid */
3083 args.done = 1; /* DONE bit */
3084
3085 /* Specify the target we are exporting */
3086 args.target = V_008DFC_SQ_EXP_MRTZ;
3087
3088 args.compr = 0; /* COMP flag */
3089 args.out[0] = base->undef; /* R, depth */
3090 args.out[1] = base->undef; /* G, stencil test value[0:7], stencil op value[8:15] */
3091 args.out[2] = base->undef; /* B, sample mask */
3092 args.out[3] = base->undef; /* A, alpha to mask */
3093
3094 if (format == V_028710_SPI_SHADER_UINT16_ABGR) {
3095 assert(!depth);
3096 args.compr = 1; /* COMPR flag */
3097
3098 if (stencil) {
3099 /* Stencil should be in X[23:16]. */
3100 stencil = bitcast(bld_base, TGSI_TYPE_UNSIGNED, stencil);
3101 stencil = LLVMBuildShl(ctx->gallivm.builder, stencil,
3102 LLVMConstInt(ctx->i32, 16, 0), "");
3103 args.out[0] = bitcast(bld_base, TGSI_TYPE_FLOAT, stencil);
3104 mask |= 0x3;
3105 }
3106 if (samplemask) {
3107 /* SampleMask should be in Y[15:0]. */
3108 args.out[1] = samplemask;
3109 mask |= 0xc;
3110 }
3111 } else {
3112 if (depth) {
3113 args.out[0] = depth;
3114 mask |= 0x1;
3115 }
3116 if (stencil) {
3117 args.out[1] = stencil;
3118 mask |= 0x2;
3119 }
3120 if (samplemask) {
3121 args.out[2] = samplemask;
3122 mask |= 0x4;
3123 }
3124 }
3125
3126 /* SI (except OLAND and HAINAN) has a bug that it only looks
3127 * at the X writemask component. */
3128 if (ctx->screen->b.chip_class == SI &&
3129 ctx->screen->b.family != CHIP_OLAND &&
3130 ctx->screen->b.family != CHIP_HAINAN)
3131 mask |= 0x1;
3132
3133 /* Specify which components to enable */
3134 args.enabled_channels = mask;
3135
3136 memcpy(&exp->args[exp->num++], &args, sizeof(args));
3137 }
3138
3139 static void si_export_mrt_color(struct lp_build_tgsi_context *bld_base,
3140 LLVMValueRef *color, unsigned index,
3141 unsigned samplemask_param,
3142 bool is_last, struct si_ps_exports *exp)
3143 {
3144 struct si_shader_context *ctx = si_shader_context(bld_base);
3145 struct lp_build_context *base = &bld_base->base;
3146 int i;
3147
3148 /* Clamp color */
3149 if (ctx->shader->key.part.ps.epilog.clamp_color)
3150 for (i = 0; i < 4; i++)
3151 color[i] = ac_build_clamp(&ctx->ac, color[i]);
3152
3153 /* Alpha to one */
3154 if (ctx->shader->key.part.ps.epilog.alpha_to_one)
3155 color[3] = base->one;
3156
3157 /* Alpha test */
3158 if (index == 0 &&
3159 ctx->shader->key.part.ps.epilog.alpha_func != PIPE_FUNC_ALWAYS)
3160 si_alpha_test(bld_base, color[3]);
3161
3162 /* Line & polygon smoothing */
3163 if (ctx->shader->key.part.ps.epilog.poly_line_smoothing)
3164 color[3] = si_scale_alpha_by_sample_mask(bld_base, color[3],
3165 samplemask_param);
3166
3167 /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
3168 if (ctx->shader->key.part.ps.epilog.last_cbuf > 0) {
3169 struct ac_export_args args[8];
3170 int c, last = -1;
3171
3172 /* Get the export arguments, also find out what the last one is. */
3173 for (c = 0; c <= ctx->shader->key.part.ps.epilog.last_cbuf; c++) {
3174 si_llvm_init_export_args(bld_base, color,
3175 V_008DFC_SQ_EXP_MRT + c, &args[c]);
3176 if (args[c].enabled_channels)
3177 last = c;
3178 }
3179
3180 /* Emit all exports. */
3181 for (c = 0; c <= ctx->shader->key.part.ps.epilog.last_cbuf; c++) {
3182 if (is_last && last == c) {
3183 args[c].valid_mask = 1; /* whether the EXEC mask is valid */
3184 args[c].done = 1; /* DONE bit */
3185 } else if (!args[c].enabled_channels)
3186 continue; /* unnecessary NULL export */
3187
3188 memcpy(&exp->args[exp->num++], &args[c], sizeof(args[c]));
3189 }
3190 } else {
3191 struct ac_export_args args;
3192
3193 /* Export */
3194 si_llvm_init_export_args(bld_base, color, V_008DFC_SQ_EXP_MRT + index,
3195 &args);
3196 if (is_last) {
3197 args.valid_mask = 1; /* whether the EXEC mask is valid */
3198 args.done = 1; /* DONE bit */
3199 } else if (!args.enabled_channels)
3200 return; /* unnecessary NULL export */
3201
3202 memcpy(&exp->args[exp->num++], &args, sizeof(args));
3203 }
3204 }
3205
3206 static void si_emit_ps_exports(struct si_shader_context *ctx,
3207 struct si_ps_exports *exp)
3208 {
3209 for (unsigned i = 0; i < exp->num; i++)
3210 ac_build_export(&ctx->ac, &exp->args[i]);
3211 }
3212
3213 static void si_export_null(struct lp_build_tgsi_context *bld_base)
3214 {
3215 struct si_shader_context *ctx = si_shader_context(bld_base);
3216 struct lp_build_context *base = &bld_base->base;
3217 struct ac_export_args args;
3218
3219 args.enabled_channels = 0x0; /* enabled channels */
3220 args.valid_mask = 1; /* whether the EXEC mask is valid */
3221 args.done = 1; /* DONE bit */
3222 args.target = V_008DFC_SQ_EXP_NULL;
3223 args.compr = 0; /* COMPR flag (0 = 32-bit export) */
3224 args.out[0] = base->undef; /* R */
3225 args.out[1] = base->undef; /* G */
3226 args.out[2] = base->undef; /* B */
3227 args.out[3] = base->undef; /* A */
3228
3229 ac_build_export(&ctx->ac, &args);
3230 }
3231
3232 /**
3233 * Return PS outputs in this order:
3234 *
3235 * v[0:3] = color0.xyzw
3236 * v[4:7] = color1.xyzw
3237 * ...
3238 * vN+0 = Depth
3239 * vN+1 = Stencil
3240 * vN+2 = SampleMask
3241 * vN+3 = SampleMaskIn (used for OpenGL smoothing)
3242 *
3243 * The alpha-ref SGPR is returned via its original location.
3244 */
3245 static void si_llvm_return_fs_outputs(struct lp_build_tgsi_context *bld_base)
3246 {
3247 struct si_shader_context *ctx = si_shader_context(bld_base);
3248 struct si_shader *shader = ctx->shader;
3249 struct tgsi_shader_info *info = &shader->selector->info;
3250 LLVMBuilderRef builder = ctx->gallivm.builder;
3251 unsigned i, j, first_vgpr, vgpr;
3252
3253 LLVMValueRef color[8][4] = {};
3254 LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
3255 LLVMValueRef ret;
3256
3257 /* Read the output values. */
3258 for (i = 0; i < info->num_outputs; i++) {
3259 unsigned semantic_name = info->output_semantic_name[i];
3260 unsigned semantic_index = info->output_semantic_index[i];
3261
3262 switch (semantic_name) {
3263 case TGSI_SEMANTIC_COLOR:
3264 assert(semantic_index < 8);
3265 for (j = 0; j < 4; j++) {
3266 LLVMValueRef ptr = ctx->outputs[i][j];
3267 LLVMValueRef result = LLVMBuildLoad(builder, ptr, "");
3268 color[semantic_index][j] = result;
3269 }
3270 break;
3271 case TGSI_SEMANTIC_POSITION:
3272 depth = LLVMBuildLoad(builder,
3273 ctx->outputs[i][2], "");
3274 break;
3275 case TGSI_SEMANTIC_STENCIL:
3276 stencil = LLVMBuildLoad(builder,
3277 ctx->outputs[i][1], "");
3278 break;
3279 case TGSI_SEMANTIC_SAMPLEMASK:
3280 samplemask = LLVMBuildLoad(builder,
3281 ctx->outputs[i][0], "");
3282 break;
3283 default:
3284 fprintf(stderr, "Warning: SI unhandled fs output type:%d\n",
3285 semantic_name);
3286 }
3287 }
3288
3289 /* Fill the return structure. */
3290 ret = ctx->return_value;
3291
3292 /* Set SGPRs. */
3293 ret = LLVMBuildInsertValue(builder, ret,
3294 bitcast(bld_base, TGSI_TYPE_SIGNED,
3295 LLVMGetParam(ctx->main_fn,
3296 SI_PARAM_ALPHA_REF)),
3297 SI_SGPR_ALPHA_REF, "");
3298
3299 /* Set VGPRs */
3300 first_vgpr = vgpr = SI_SGPR_ALPHA_REF + 1;
3301 for (i = 0; i < ARRAY_SIZE(color); i++) {
3302 if (!color[i][0])
3303 continue;
3304
3305 for (j = 0; j < 4; j++)
3306 ret = LLVMBuildInsertValue(builder, ret, color[i][j], vgpr++, "");
3307 }
3308 if (depth)
3309 ret = LLVMBuildInsertValue(builder, ret, depth, vgpr++, "");
3310 if (stencil)
3311 ret = LLVMBuildInsertValue(builder, ret, stencil, vgpr++, "");
3312 if (samplemask)
3313 ret = LLVMBuildInsertValue(builder, ret, samplemask, vgpr++, "");
3314
3315 /* Add the input sample mask for smoothing at the end. */
3316 if (vgpr < first_vgpr + PS_EPILOG_SAMPLEMASK_MIN_LOC)
3317 vgpr = first_vgpr + PS_EPILOG_SAMPLEMASK_MIN_LOC;
3318 ret = LLVMBuildInsertValue(builder, ret,
3319 LLVMGetParam(ctx->main_fn,
3320 SI_PARAM_SAMPLE_COVERAGE), vgpr++, "");
3321
3322 ctx->return_value = ret;
3323 }
3324
3325 /* Prevent optimizations (at least of memory accesses) across the current
3326 * point in the program by emitting empty inline assembly that is marked as
3327 * having side effects.
3328 *
3329 * Optionally, a value can be passed through the inline assembly to prevent
3330 * LLVM from hoisting calls to ReadNone functions.
3331 */
3332 static void emit_optimization_barrier(struct si_shader_context *ctx,
3333 LLVMValueRef *pvgpr)
3334 {
3335 static int counter = 0;
3336
3337 LLVMBuilderRef builder = ctx->gallivm.builder;
3338 char code[16];
3339
3340 snprintf(code, sizeof(code), "; %d", p_atomic_inc_return(&counter));
3341
3342 if (!pvgpr) {
3343 LLVMTypeRef ftype = LLVMFunctionType(ctx->voidt, NULL, 0, false);
3344 LLVMValueRef inlineasm = LLVMConstInlineAsm(ftype, code, "", true, false);
3345 LLVMBuildCall(builder, inlineasm, NULL, 0, "");
3346 } else {
3347 LLVMTypeRef ftype = LLVMFunctionType(ctx->i32, &ctx->i32, 1, false);
3348 LLVMValueRef inlineasm = LLVMConstInlineAsm(ftype, code, "=v,0", true, false);
3349 LLVMValueRef vgpr = *pvgpr;
3350 LLVMTypeRef vgpr_type = LLVMTypeOf(vgpr);
3351 unsigned vgpr_size = llvm_get_type_size(vgpr_type);
3352 LLVMValueRef vgpr0;
3353
3354 assert(vgpr_size % 4 == 0);
3355
3356 vgpr = LLVMBuildBitCast(builder, vgpr, LLVMVectorType(ctx->i32, vgpr_size / 4), "");
3357 vgpr0 = LLVMBuildExtractElement(builder, vgpr, ctx->i32_0, "");
3358 vgpr0 = LLVMBuildCall(builder, inlineasm, &vgpr0, 1, "");
3359 vgpr = LLVMBuildInsertElement(builder, vgpr, vgpr0, ctx->i32_0, "");
3360 vgpr = LLVMBuildBitCast(builder, vgpr, vgpr_type, "");
3361
3362 *pvgpr = vgpr;
3363 }
3364 }
3365
3366 void si_emit_waitcnt(struct si_shader_context *ctx, unsigned simm16)
3367 {
3368 struct gallivm_state *gallivm = &ctx->gallivm;
3369 LLVMBuilderRef builder = gallivm->builder;
3370 LLVMValueRef args[1] = {
3371 LLVMConstInt(ctx->i32, simm16, 0)
3372 };
3373 lp_build_intrinsic(builder, "llvm.amdgcn.s.waitcnt",
3374 ctx->voidt, args, 1, 0);
3375 }
3376
3377 static void membar_emit(
3378 const struct lp_build_tgsi_action *action,
3379 struct lp_build_tgsi_context *bld_base,
3380 struct lp_build_emit_data *emit_data)
3381 {
3382 struct si_shader_context *ctx = si_shader_context(bld_base);
3383 LLVMValueRef src0 = lp_build_emit_fetch(bld_base, emit_data->inst, 0, 0);
3384 unsigned flags = LLVMConstIntGetZExtValue(src0);
3385 unsigned waitcnt = NOOP_WAITCNT;
3386
3387 if (flags & TGSI_MEMBAR_THREAD_GROUP)
3388 waitcnt &= VM_CNT & LGKM_CNT;
3389
3390 if (flags & (TGSI_MEMBAR_ATOMIC_BUFFER |
3391 TGSI_MEMBAR_SHADER_BUFFER |
3392 TGSI_MEMBAR_SHADER_IMAGE))
3393 waitcnt &= VM_CNT;
3394
3395 if (flags & TGSI_MEMBAR_SHARED)
3396 waitcnt &= LGKM_CNT;
3397
3398 if (waitcnt != NOOP_WAITCNT)
3399 si_emit_waitcnt(ctx, waitcnt);
3400 }
3401
3402 static void clock_emit(
3403 const struct lp_build_tgsi_action *action,
3404 struct lp_build_tgsi_context *bld_base,
3405 struct lp_build_emit_data *emit_data)
3406 {
3407 struct si_shader_context *ctx = si_shader_context(bld_base);
3408 struct gallivm_state *gallivm = &ctx->gallivm;
3409 LLVMValueRef tmp;
3410
3411 tmp = lp_build_intrinsic(gallivm->builder, "llvm.readcyclecounter",
3412 ctx->i64, NULL, 0, 0);
3413 tmp = LLVMBuildBitCast(gallivm->builder, tmp, ctx->v2i32, "");
3414
3415 emit_data->output[0] =
3416 LLVMBuildExtractElement(gallivm->builder, tmp, ctx->i32_0, "");
3417 emit_data->output[1] =
3418 LLVMBuildExtractElement(gallivm->builder, tmp, ctx->i32_1, "");
3419 }
3420
3421 LLVMTypeRef si_const_array(LLVMTypeRef elem_type, int num_elements)
3422 {
3423 return LLVMPointerType(LLVMArrayType(elem_type, num_elements),
3424 CONST_ADDR_SPACE);
3425 }
3426
3427 static void si_llvm_emit_ddxy(
3428 const struct lp_build_tgsi_action *action,
3429 struct lp_build_tgsi_context *bld_base,
3430 struct lp_build_emit_data *emit_data)
3431 {
3432 struct si_shader_context *ctx = si_shader_context(bld_base);
3433 struct gallivm_state *gallivm = &ctx->gallivm;
3434 unsigned opcode = emit_data->info->opcode;
3435 LLVMValueRef val;
3436 int idx;
3437 unsigned mask;
3438
3439 if (opcode == TGSI_OPCODE_DDX_FINE)
3440 mask = AC_TID_MASK_LEFT;
3441 else if (opcode == TGSI_OPCODE_DDY_FINE)
3442 mask = AC_TID_MASK_TOP;
3443 else
3444 mask = AC_TID_MASK_TOP_LEFT;
3445
3446 /* for DDX we want to next X pixel, DDY next Y pixel. */
3447 idx = (opcode == TGSI_OPCODE_DDX || opcode == TGSI_OPCODE_DDX_FINE) ? 1 : 2;
3448
3449 val = LLVMBuildBitCast(gallivm->builder, emit_data->args[0], ctx->i32, "");
3450 val = ac_build_ddxy(&ctx->ac, ctx->screen->has_ds_bpermute,
3451 mask, idx, ctx->lds, val);
3452 emit_data->output[emit_data->chan] = val;
3453 }
3454
3455 /*
3456 * this takes an I,J coordinate pair,
3457 * and works out the X and Y derivatives.
3458 * it returns DDX(I), DDX(J), DDY(I), DDY(J).
3459 */
3460 static LLVMValueRef si_llvm_emit_ddxy_interp(
3461 struct lp_build_tgsi_context *bld_base,
3462 LLVMValueRef interp_ij)
3463 {
3464 struct si_shader_context *ctx = si_shader_context(bld_base);
3465 struct gallivm_state *gallivm = &ctx->gallivm;
3466 LLVMValueRef result[4], a;
3467 unsigned i;
3468
3469 for (i = 0; i < 2; i++) {
3470 a = LLVMBuildExtractElement(gallivm->builder, interp_ij,
3471 LLVMConstInt(ctx->i32, i, 0), "");
3472 result[i] = lp_build_emit_llvm_unary(bld_base, TGSI_OPCODE_DDX, a);
3473 result[2+i] = lp_build_emit_llvm_unary(bld_base, TGSI_OPCODE_DDY, a);
3474 }
3475
3476 return lp_build_gather_values(gallivm, result, 4);
3477 }
3478
3479 static void interp_fetch_args(
3480 struct lp_build_tgsi_context *bld_base,
3481 struct lp_build_emit_data *emit_data)
3482 {
3483 struct si_shader_context *ctx = si_shader_context(bld_base);
3484 struct gallivm_state *gallivm = &ctx->gallivm;
3485 const struct tgsi_full_instruction *inst = emit_data->inst;
3486
3487 if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET) {
3488 /* offset is in second src, first two channels */
3489 emit_data->args[0] = lp_build_emit_fetch(bld_base,
3490 emit_data->inst, 1,
3491 TGSI_CHAN_X);
3492 emit_data->args[1] = lp_build_emit_fetch(bld_base,
3493 emit_data->inst, 1,
3494 TGSI_CHAN_Y);
3495 emit_data->arg_count = 2;
3496 } else if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE) {
3497 LLVMValueRef sample_position;
3498 LLVMValueRef sample_id;
3499 LLVMValueRef halfval = LLVMConstReal(ctx->f32, 0.5f);
3500
3501 /* fetch sample ID, then fetch its sample position,
3502 * and place into first two channels.
3503 */
3504 sample_id = lp_build_emit_fetch(bld_base,
3505 emit_data->inst, 1, TGSI_CHAN_X);
3506 sample_id = LLVMBuildBitCast(gallivm->builder, sample_id,
3507 ctx->i32, "");
3508 sample_position = load_sample_position(ctx, sample_id);
3509
3510 emit_data->args[0] = LLVMBuildExtractElement(gallivm->builder,
3511 sample_position,
3512 ctx->i32_0, "");
3513
3514 emit_data->args[0] = LLVMBuildFSub(gallivm->builder, emit_data->args[0], halfval, "");
3515 emit_data->args[1] = LLVMBuildExtractElement(gallivm->builder,
3516 sample_position,
3517 ctx->i32_1, "");
3518 emit_data->args[1] = LLVMBuildFSub(gallivm->builder, emit_data->args[1], halfval, "");
3519 emit_data->arg_count = 2;
3520 }
3521 }
3522
3523 static void build_interp_intrinsic(const struct lp_build_tgsi_action *action,
3524 struct lp_build_tgsi_context *bld_base,
3525 struct lp_build_emit_data *emit_data)
3526 {
3527 struct si_shader_context *ctx = si_shader_context(bld_base);
3528 struct si_shader *shader = ctx->shader;
3529 struct gallivm_state *gallivm = &ctx->gallivm;
3530 LLVMValueRef interp_param;
3531 const struct tgsi_full_instruction *inst = emit_data->inst;
3532 int input_index = inst->Src[0].Register.Index;
3533 int chan;
3534 int i;
3535 LLVMValueRef attr_number;
3536 LLVMValueRef params = LLVMGetParam(ctx->main_fn, SI_PARAM_PRIM_MASK);
3537 int interp_param_idx;
3538 unsigned interp = shader->selector->info.input_interpolate[input_index];
3539 unsigned location;
3540
3541 assert(inst->Src[0].Register.File == TGSI_FILE_INPUT);
3542
3543 if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET ||
3544 inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE)
3545 location = TGSI_INTERPOLATE_LOC_CENTER;
3546 else
3547 location = TGSI_INTERPOLATE_LOC_CENTROID;
3548
3549 interp_param_idx = lookup_interp_param_index(interp, location);
3550 if (interp_param_idx == -1)
3551 return;
3552 else if (interp_param_idx)
3553 interp_param = LLVMGetParam(ctx->main_fn, interp_param_idx);
3554 else
3555 interp_param = NULL;
3556
3557 attr_number = LLVMConstInt(ctx->i32, input_index, 0);
3558
3559 if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET ||
3560 inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE) {
3561 LLVMValueRef ij_out[2];
3562 LLVMValueRef ddxy_out = si_llvm_emit_ddxy_interp(bld_base, interp_param);
3563
3564 /*
3565 * take the I then J parameters, and the DDX/Y for it, and
3566 * calculate the IJ inputs for the interpolator.
3567 * temp1 = ddx * offset/sample.x + I;
3568 * interp_param.I = ddy * offset/sample.y + temp1;
3569 * temp1 = ddx * offset/sample.x + J;
3570 * interp_param.J = ddy * offset/sample.y + temp1;
3571 */
3572 for (i = 0; i < 2; i++) {
3573 LLVMValueRef ix_ll = LLVMConstInt(ctx->i32, i, 0);
3574 LLVMValueRef iy_ll = LLVMConstInt(ctx->i32, i + 2, 0);
3575 LLVMValueRef ddx_el = LLVMBuildExtractElement(gallivm->builder,
3576 ddxy_out, ix_ll, "");
3577 LLVMValueRef ddy_el = LLVMBuildExtractElement(gallivm->builder,
3578 ddxy_out, iy_ll, "");
3579 LLVMValueRef interp_el = LLVMBuildExtractElement(gallivm->builder,
3580 interp_param, ix_ll, "");
3581 LLVMValueRef temp1, temp2;
3582
3583 interp_el = LLVMBuildBitCast(gallivm->builder, interp_el,
3584 ctx->f32, "");
3585
3586 temp1 = LLVMBuildFMul(gallivm->builder, ddx_el, emit_data->args[0], "");
3587
3588 temp1 = LLVMBuildFAdd(gallivm->builder, temp1, interp_el, "");
3589
3590 temp2 = LLVMBuildFMul(gallivm->builder, ddy_el, emit_data->args[1], "");
3591
3592 ij_out[i] = LLVMBuildFAdd(gallivm->builder, temp2, temp1, "");
3593 }
3594 interp_param = lp_build_gather_values(gallivm, ij_out, 2);
3595 }
3596
3597 for (chan = 0; chan < 4; chan++) {
3598 LLVMValueRef llvm_chan;
3599 unsigned schan;
3600
3601 schan = tgsi_util_get_full_src_register_swizzle(&inst->Src[0], chan);
3602 llvm_chan = LLVMConstInt(ctx->i32, schan, 0);
3603
3604 if (interp_param) {
3605 interp_param = LLVMBuildBitCast(gallivm->builder,
3606 interp_param, LLVMVectorType(ctx->f32, 2), "");
3607 LLVMValueRef i = LLVMBuildExtractElement(
3608 gallivm->builder, interp_param, ctx->i32_0, "");
3609 LLVMValueRef j = LLVMBuildExtractElement(
3610 gallivm->builder, interp_param, ctx->i32_1, "");
3611 emit_data->output[chan] = ac_build_fs_interp(&ctx->ac,
3612 llvm_chan, attr_number, params,
3613 i, j);
3614 } else {
3615 emit_data->output[chan] = ac_build_fs_interp_mov(&ctx->ac,
3616 LLVMConstInt(ctx->i32, 2, 0), /* P0 */
3617 llvm_chan, attr_number, params);
3618 }
3619 }
3620 }
3621
3622 static LLVMValueRef si_emit_ballot(struct si_shader_context *ctx,
3623 LLVMValueRef value)
3624 {
3625 struct gallivm_state *gallivm = &ctx->gallivm;
3626 LLVMValueRef args[3] = {
3627 value,
3628 ctx->i32_0,
3629 LLVMConstInt(ctx->i32, LLVMIntNE, 0)
3630 };
3631
3632 /* We currently have no other way to prevent LLVM from lifting the icmp
3633 * calls to a dominating basic block.
3634 */
3635 emit_optimization_barrier(ctx, &args[0]);
3636
3637 if (LLVMTypeOf(args[0]) != ctx->i32)
3638 args[0] = LLVMBuildBitCast(gallivm->builder, args[0], ctx->i32, "");
3639
3640 return lp_build_intrinsic(gallivm->builder,
3641 "llvm.amdgcn.icmp.i32",
3642 ctx->i64, args, 3,
3643 LP_FUNC_ATTR_NOUNWIND |
3644 LP_FUNC_ATTR_READNONE |
3645 LP_FUNC_ATTR_CONVERGENT);
3646 }
3647
3648 static void vote_all_emit(
3649 const struct lp_build_tgsi_action *action,
3650 struct lp_build_tgsi_context *bld_base,
3651 struct lp_build_emit_data *emit_data)
3652 {
3653 struct si_shader_context *ctx = si_shader_context(bld_base);
3654 struct gallivm_state *gallivm = &ctx->gallivm;
3655 LLVMValueRef active_set, vote_set;
3656 LLVMValueRef tmp;
3657
3658 active_set = si_emit_ballot(ctx, ctx->i32_1);
3659 vote_set = si_emit_ballot(ctx, emit_data->args[0]);
3660
3661 tmp = LLVMBuildICmp(gallivm->builder, LLVMIntEQ, vote_set, active_set, "");
3662 emit_data->output[emit_data->chan] =
3663 LLVMBuildSExt(gallivm->builder, tmp, ctx->i32, "");
3664 }
3665
3666 static void vote_any_emit(
3667 const struct lp_build_tgsi_action *action,
3668 struct lp_build_tgsi_context *bld_base,
3669 struct lp_build_emit_data *emit_data)
3670 {
3671 struct si_shader_context *ctx = si_shader_context(bld_base);
3672 struct gallivm_state *gallivm = &ctx->gallivm;
3673 LLVMValueRef vote_set;
3674 LLVMValueRef tmp;
3675
3676 vote_set = si_emit_ballot(ctx, emit_data->args[0]);
3677
3678 tmp = LLVMBuildICmp(gallivm->builder, LLVMIntNE,
3679 vote_set, LLVMConstInt(ctx->i64, 0, 0), "");
3680 emit_data->output[emit_data->chan] =
3681 LLVMBuildSExt(gallivm->builder, tmp, ctx->i32, "");
3682 }
3683
3684 static void vote_eq_emit(
3685 const struct lp_build_tgsi_action *action,
3686 struct lp_build_tgsi_context *bld_base,
3687 struct lp_build_emit_data *emit_data)
3688 {
3689 struct si_shader_context *ctx = si_shader_context(bld_base);
3690 struct gallivm_state *gallivm = &ctx->gallivm;
3691 LLVMValueRef active_set, vote_set;
3692 LLVMValueRef all, none, tmp;
3693
3694 active_set = si_emit_ballot(ctx, ctx->i32_1);
3695 vote_set = si_emit_ballot(ctx, emit_data->args[0]);
3696
3697 all = LLVMBuildICmp(gallivm->builder, LLVMIntEQ, vote_set, active_set, "");
3698 none = LLVMBuildICmp(gallivm->builder, LLVMIntEQ,
3699 vote_set, LLVMConstInt(ctx->i64, 0, 0), "");
3700 tmp = LLVMBuildOr(gallivm->builder, all, none, "");
3701 emit_data->output[emit_data->chan] =
3702 LLVMBuildSExt(gallivm->builder, tmp, ctx->i32, "");
3703 }
3704
3705 static void ballot_emit(
3706 const struct lp_build_tgsi_action *action,
3707 struct lp_build_tgsi_context *bld_base,
3708 struct lp_build_emit_data *emit_data)
3709 {
3710 struct si_shader_context *ctx = si_shader_context(bld_base);
3711 LLVMBuilderRef builder = ctx->gallivm.builder;
3712 LLVMValueRef tmp;
3713
3714 tmp = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_X);
3715 tmp = si_emit_ballot(ctx, tmp);
3716 tmp = LLVMBuildBitCast(builder, tmp, ctx->v2i32, "");
3717
3718 emit_data->output[0] = LLVMBuildExtractElement(builder, tmp, ctx->i32_0, "");
3719 emit_data->output[1] = LLVMBuildExtractElement(builder, tmp, ctx->i32_1, "");
3720 }
3721
3722 static void read_invoc_fetch_args(
3723 struct lp_build_tgsi_context *bld_base,
3724 struct lp_build_emit_data *emit_data)
3725 {
3726 emit_data->args[0] = lp_build_emit_fetch(bld_base, emit_data->inst,
3727 0, emit_data->src_chan);
3728
3729 /* Always read the source invocation (= lane) from the X channel. */
3730 emit_data->args[1] = lp_build_emit_fetch(bld_base, emit_data->inst,
3731 1, TGSI_CHAN_X);
3732 emit_data->arg_count = 2;
3733 }
3734
3735 static void read_lane_emit(
3736 const struct lp_build_tgsi_action *action,
3737 struct lp_build_tgsi_context *bld_base,
3738 struct lp_build_emit_data *emit_data)
3739 {
3740 struct si_shader_context *ctx = si_shader_context(bld_base);
3741 LLVMBuilderRef builder = ctx->gallivm.builder;
3742
3743 /* We currently have no other way to prevent LLVM from lifting the icmp
3744 * calls to a dominating basic block.
3745 */
3746 emit_optimization_barrier(ctx, &emit_data->args[0]);
3747
3748 for (unsigned i = 0; i < emit_data->arg_count; ++i) {
3749 emit_data->args[i] = LLVMBuildBitCast(builder, emit_data->args[i],
3750 ctx->i32, "");
3751 }
3752
3753 emit_data->output[emit_data->chan] =
3754 ac_build_intrinsic(&ctx->ac, action->intr_name,
3755 ctx->i32, emit_data->args, emit_data->arg_count,
3756 AC_FUNC_ATTR_READNONE |
3757 AC_FUNC_ATTR_CONVERGENT);
3758 }
3759
3760 static unsigned si_llvm_get_stream(struct lp_build_tgsi_context *bld_base,
3761 struct lp_build_emit_data *emit_data)
3762 {
3763 struct si_shader_context *ctx = si_shader_context(bld_base);
3764 struct tgsi_src_register src0 = emit_data->inst->Src[0].Register;
3765 LLVMValueRef imm;
3766 unsigned stream;
3767
3768 assert(src0.File == TGSI_FILE_IMMEDIATE);
3769
3770 imm = ctx->imms[src0.Index * TGSI_NUM_CHANNELS + src0.SwizzleX];
3771 stream = LLVMConstIntGetZExtValue(imm) & 0x3;
3772 return stream;
3773 }
3774
3775 /* Emit one vertex from the geometry shader */
3776 static void si_llvm_emit_vertex(
3777 const struct lp_build_tgsi_action *action,
3778 struct lp_build_tgsi_context *bld_base,
3779 struct lp_build_emit_data *emit_data)
3780 {
3781 struct si_shader_context *ctx = si_shader_context(bld_base);
3782 struct lp_build_context *uint = &bld_base->uint_bld;
3783 struct si_shader *shader = ctx->shader;
3784 struct tgsi_shader_info *info = &shader->selector->info;
3785 struct gallivm_state *gallivm = &ctx->gallivm;
3786 struct lp_build_if_state if_state;
3787 LLVMValueRef soffset = LLVMGetParam(ctx->main_fn,
3788 ctx->param_gs2vs_offset);
3789 LLVMValueRef gs_next_vertex;
3790 LLVMValueRef can_emit, kill;
3791 unsigned chan, offset;
3792 int i;
3793 unsigned stream;
3794
3795 stream = si_llvm_get_stream(bld_base, emit_data);
3796
3797 /* Write vertex attribute values to GSVS ring */
3798 gs_next_vertex = LLVMBuildLoad(gallivm->builder,
3799 ctx->gs_next_vertex[stream],
3800 "");
3801
3802 /* If this thread has already emitted the declared maximum number of
3803 * vertices, skip the write: excessive vertex emissions are not
3804 * supposed to have any effect.
3805 *
3806 * If the shader has no writes to memory, kill it instead. This skips
3807 * further memory loads and may allow LLVM to skip to the end
3808 * altogether.
3809 */
3810 can_emit = LLVMBuildICmp(gallivm->builder, LLVMIntULT, gs_next_vertex,
3811 LLVMConstInt(ctx->i32,
3812 shader->selector->gs_max_out_vertices, 0), "");
3813
3814 bool use_kill = !info->writes_memory;
3815 if (use_kill) {
3816 kill = lp_build_select(&bld_base->base, can_emit,
3817 LLVMConstReal(ctx->f32, 1.0f),
3818 LLVMConstReal(ctx->f32, -1.0f));
3819
3820 ac_build_kill(&ctx->ac, kill);
3821 } else {
3822 lp_build_if(&if_state, gallivm, can_emit);
3823 }
3824
3825 offset = 0;
3826 for (i = 0; i < info->num_outputs; i++) {
3827 LLVMValueRef *out_ptr = ctx->outputs[i];
3828
3829 for (chan = 0; chan < 4; chan++) {
3830 if (!(info->output_usagemask[i] & (1 << chan)) ||
3831 ((info->output_streams[i] >> (2 * chan)) & 3) != stream)
3832 continue;
3833
3834 LLVMValueRef out_val = LLVMBuildLoad(gallivm->builder, out_ptr[chan], "");
3835 LLVMValueRef voffset =
3836 LLVMConstInt(ctx->i32, offset *
3837 shader->selector->gs_max_out_vertices, 0);
3838 offset++;
3839
3840 voffset = lp_build_add(uint, voffset, gs_next_vertex);
3841 voffset = lp_build_mul_imm(uint, voffset, 4);
3842
3843 out_val = LLVMBuildBitCast(gallivm->builder, out_val, ctx->i32, "");
3844
3845 ac_build_buffer_store_dword(&ctx->ac,
3846 ctx->gsvs_ring[stream],
3847 out_val, 1,
3848 voffset, soffset, 0,
3849 1, 1, true, true);
3850 }
3851 }
3852
3853 gs_next_vertex = lp_build_add(uint, gs_next_vertex,
3854 ctx->i32_1);
3855
3856 LLVMBuildStore(gallivm->builder, gs_next_vertex, ctx->gs_next_vertex[stream]);
3857
3858 /* Signal vertex emission */
3859 ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_EMIT | AC_SENDMSG_GS | (stream << 8),
3860 si_get_gs_wave_id(ctx));
3861 if (!use_kill)
3862 lp_build_endif(&if_state);
3863 }
3864
3865 /* Cut one primitive from the geometry shader */
3866 static void si_llvm_emit_primitive(
3867 const struct lp_build_tgsi_action *action,
3868 struct lp_build_tgsi_context *bld_base,
3869 struct lp_build_emit_data *emit_data)
3870 {
3871 struct si_shader_context *ctx = si_shader_context(bld_base);
3872 unsigned stream;
3873
3874 /* Signal primitive cut */
3875 stream = si_llvm_get_stream(bld_base, emit_data);
3876 ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_CUT | AC_SENDMSG_GS | (stream << 8),
3877 si_get_gs_wave_id(ctx));
3878 }
3879
3880 static void si_llvm_emit_barrier(const struct lp_build_tgsi_action *action,
3881 struct lp_build_tgsi_context *bld_base,
3882 struct lp_build_emit_data *emit_data)
3883 {
3884 struct si_shader_context *ctx = si_shader_context(bld_base);
3885 struct gallivm_state *gallivm = &ctx->gallivm;
3886
3887 /* SI only (thanks to a hw bug workaround):
3888 * The real barrier instruction isn’t needed, because an entire patch
3889 * always fits into a single wave.
3890 */
3891 if (ctx->screen->b.chip_class == SI &&
3892 ctx->type == PIPE_SHADER_TESS_CTRL) {
3893 si_emit_waitcnt(ctx, LGKM_CNT & VM_CNT);
3894 return;
3895 }
3896
3897 lp_build_intrinsic(gallivm->builder,
3898 "llvm.amdgcn.s.barrier",
3899 ctx->voidt, NULL, 0, LP_FUNC_ATTR_CONVERGENT);
3900 }
3901
3902 static const struct lp_build_tgsi_action interp_action = {
3903 .fetch_args = interp_fetch_args,
3904 .emit = build_interp_intrinsic,
3905 };
3906
3907 static void si_create_function(struct si_shader_context *ctx,
3908 const char *name,
3909 LLVMTypeRef *returns, unsigned num_returns,
3910 LLVMTypeRef *params, unsigned num_params,
3911 int last_sgpr, unsigned max_workgroup_size)
3912 {
3913 int i;
3914
3915 si_llvm_create_func(ctx, name, returns, num_returns,
3916 params, num_params);
3917 ctx->return_value = LLVMGetUndef(ctx->return_type);
3918
3919 for (i = 0; i <= last_sgpr; ++i) {
3920 LLVMValueRef P = LLVMGetParam(ctx->main_fn, i);
3921
3922 /* The combination of:
3923 * - ByVal
3924 * - dereferenceable
3925 * - invariant.load
3926 * allows the optimization passes to move loads and reduces
3927 * SGPR spilling significantly.
3928 */
3929 if (LLVMGetTypeKind(LLVMTypeOf(P)) == LLVMPointerTypeKind) {
3930 lp_add_function_attr(ctx->main_fn, i + 1, LP_FUNC_ATTR_BYVAL);
3931 lp_add_function_attr(ctx->main_fn, i + 1, LP_FUNC_ATTR_NOALIAS);
3932 ac_add_attr_dereferenceable(P, UINT64_MAX);
3933 } else
3934 lp_add_function_attr(ctx->main_fn, i + 1, LP_FUNC_ATTR_INREG);
3935 }
3936
3937 if (max_workgroup_size) {
3938 si_llvm_add_attribute(ctx->main_fn, "amdgpu-max-work-group-size",
3939 max_workgroup_size);
3940 }
3941 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
3942 "no-signed-zeros-fp-math",
3943 "true");
3944
3945 if (ctx->screen->b.debug_flags & DBG_UNSAFE_MATH) {
3946 /* These were copied from some LLVM test. */
3947 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
3948 "less-precise-fpmad",
3949 "true");
3950 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
3951 "no-infs-fp-math",
3952 "true");
3953 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
3954 "no-nans-fp-math",
3955 "true");
3956 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
3957 "unsafe-fp-math",
3958 "true");
3959 }
3960 }
3961
3962 static void declare_streamout_params(struct si_shader_context *ctx,
3963 struct pipe_stream_output_info *so,
3964 LLVMTypeRef *params, LLVMTypeRef i32,
3965 unsigned *num_params)
3966 {
3967 int i;
3968
3969 /* Streamout SGPRs. */
3970 if (so->num_outputs) {
3971 if (ctx->type != PIPE_SHADER_TESS_EVAL)
3972 params[ctx->param_streamout_config = (*num_params)++] = i32;
3973 else
3974 ctx->param_streamout_config = *num_params - 1;
3975
3976 params[ctx->param_streamout_write_index = (*num_params)++] = i32;
3977 }
3978 /* A streamout buffer offset is loaded if the stride is non-zero. */
3979 for (i = 0; i < 4; i++) {
3980 if (!so->stride[i])
3981 continue;
3982
3983 params[ctx->param_streamout_offset[i] = (*num_params)++] = i32;
3984 }
3985 }
3986
3987 static unsigned llvm_get_type_size(LLVMTypeRef type)
3988 {
3989 LLVMTypeKind kind = LLVMGetTypeKind(type);
3990
3991 switch (kind) {
3992 case LLVMIntegerTypeKind:
3993 return LLVMGetIntTypeWidth(type) / 8;
3994 case LLVMFloatTypeKind:
3995 return 4;
3996 case LLVMPointerTypeKind:
3997 return 8;
3998 case LLVMVectorTypeKind:
3999 return LLVMGetVectorSize(type) *
4000 llvm_get_type_size(LLVMGetElementType(type));
4001 case LLVMArrayTypeKind:
4002 return LLVMGetArrayLength(type) *
4003 llvm_get_type_size(LLVMGetElementType(type));
4004 default:
4005 assert(0);
4006 return 0;
4007 }
4008 }
4009
4010 static void declare_lds_as_pointer(struct si_shader_context *ctx)
4011 {
4012 struct gallivm_state *gallivm = &ctx->gallivm;
4013
4014 unsigned lds_size = ctx->screen->b.chip_class >= CIK ? 65536 : 32768;
4015 ctx->lds = LLVMBuildIntToPtr(gallivm->builder, ctx->i32_0,
4016 LLVMPointerType(LLVMArrayType(ctx->i32, lds_size / 4), LOCAL_ADDR_SPACE),
4017 "lds");
4018 }
4019
4020 static unsigned si_get_max_workgroup_size(const struct si_shader *shader)
4021 {
4022 switch (shader->selector->type) {
4023 case PIPE_SHADER_TESS_CTRL:
4024 /* Return this so that LLVM doesn't remove s_barrier
4025 * instructions on chips where we use s_barrier. */
4026 return shader->selector->screen->b.chip_class >= CIK ? 128 : 64;
4027
4028 case PIPE_SHADER_GEOMETRY:
4029 return shader->selector->screen->b.chip_class >= GFX9 ? 128 : 64;
4030
4031 case PIPE_SHADER_COMPUTE:
4032 break; /* see below */
4033
4034 default:
4035 return 0;
4036 }
4037
4038 const unsigned *properties = shader->selector->info.properties;
4039 unsigned max_work_group_size =
4040 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH] *
4041 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT] *
4042 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH];
4043
4044 if (!max_work_group_size) {
4045 /* This is a variable group size compute shader,
4046 * compile it for the maximum possible group size.
4047 */
4048 max_work_group_size = SI_MAX_VARIABLE_THREADS_PER_BLOCK;
4049 }
4050 return max_work_group_size;
4051 }
4052
4053 static void declare_per_stage_desc_pointers(struct si_shader_context *ctx,
4054 LLVMTypeRef *params,
4055 unsigned *num_params,
4056 bool assign_params)
4057 {
4058 params[(*num_params)++] = si_const_array(ctx->v4i32,
4059 SI_NUM_SHADER_BUFFERS + SI_NUM_CONST_BUFFERS);
4060 params[(*num_params)++] = si_const_array(ctx->v8i32,
4061 SI_NUM_IMAGES + SI_NUM_SAMPLERS * 2);
4062
4063 if (assign_params) {
4064 ctx->param_const_and_shader_buffers = *num_params - 2;
4065 ctx->param_samplers_and_images = *num_params - 1;
4066 }
4067 }
4068
4069 static void declare_default_desc_pointers(struct si_shader_context *ctx,
4070 LLVMTypeRef *params,
4071 unsigned *num_params)
4072 {
4073 params[ctx->param_rw_buffers = (*num_params)++] =
4074 si_const_array(ctx->v4i32, SI_NUM_RW_BUFFERS);
4075 declare_per_stage_desc_pointers(ctx, params, num_params, true);
4076 }
4077
4078 static void declare_vs_specific_input_sgprs(struct si_shader_context *ctx,
4079 LLVMTypeRef *params,
4080 unsigned *num_params)
4081 {
4082 params[ctx->param_vertex_buffers = (*num_params)++] =
4083 si_const_array(ctx->v4i32, SI_NUM_VERTEX_BUFFERS);
4084 params[ctx->param_base_vertex = (*num_params)++] = ctx->i32;
4085 params[ctx->param_start_instance = (*num_params)++] = ctx->i32;
4086 params[ctx->param_draw_id = (*num_params)++] = ctx->i32;
4087 params[ctx->param_vs_state_bits = (*num_params)++] = ctx->i32;
4088 }
4089
4090 static void declare_vs_input_vgprs(struct si_shader_context *ctx,
4091 LLVMTypeRef *params, unsigned *num_params,
4092 unsigned *num_prolog_vgprs)
4093 {
4094 struct si_shader *shader = ctx->shader;
4095
4096 params[ctx->param_vertex_id = (*num_params)++] = ctx->i32;
4097 if (shader->key.as_ls) {
4098 params[ctx->param_rel_auto_id = (*num_params)++] = ctx->i32;
4099 params[ctx->param_instance_id = (*num_params)++] = ctx->i32;
4100 } else {
4101 params[ctx->param_instance_id = (*num_params)++] = ctx->i32;
4102 params[ctx->param_vs_prim_id = (*num_params)++] = ctx->i32;
4103 }
4104 params[(*num_params)++] = ctx->i32; /* unused */
4105
4106 if (!shader->is_gs_copy_shader) {
4107 /* Vertex load indices. */
4108 ctx->param_vertex_index0 = (*num_params);
4109 for (unsigned i = 0; i < shader->selector->info.num_inputs; i++)
4110 params[(*num_params)++] = ctx->i32;
4111 *num_prolog_vgprs += shader->selector->info.num_inputs;
4112 }
4113 }
4114
4115 static void declare_tes_input_vgprs(struct si_shader_context *ctx,
4116 LLVMTypeRef *params, unsigned *num_params)
4117 {
4118 params[ctx->param_tes_u = (*num_params)++] = ctx->f32;
4119 params[ctx->param_tes_v = (*num_params)++] = ctx->f32;
4120 params[ctx->param_tes_rel_patch_id = (*num_params)++] = ctx->i32;
4121 params[ctx->param_tes_patch_id = (*num_params)++] = ctx->i32;
4122 }
4123
4124 enum {
4125 /* Convenient merged shader definitions. */
4126 SI_SHADER_MERGED_VERTEX_TESSCTRL = PIPE_SHADER_TYPES,
4127 SI_SHADER_MERGED_VERTEX_OR_TESSEVAL_GEOMETRY,
4128 };
4129
4130 static void create_function(struct si_shader_context *ctx)
4131 {
4132 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
4133 struct gallivm_state *gallivm = &ctx->gallivm;
4134 struct si_shader *shader = ctx->shader;
4135 LLVMTypeRef params[100]; /* just make it large enough */
4136 LLVMTypeRef returns[16+32*4];
4137 unsigned i, last_sgpr, num_params = 0, num_return_sgprs;
4138 unsigned num_returns = 0;
4139 unsigned num_prolog_vgprs = 0;
4140 unsigned type = ctx->type;
4141
4142 /* Set MERGED shaders. */
4143 if (ctx->screen->b.chip_class >= GFX9) {
4144 if (shader->key.as_ls || type == PIPE_SHADER_TESS_CTRL)
4145 type = SI_SHADER_MERGED_VERTEX_TESSCTRL; /* LS or HS */
4146 else if (shader->key.as_es || type == PIPE_SHADER_GEOMETRY)
4147 type = SI_SHADER_MERGED_VERTEX_OR_TESSEVAL_GEOMETRY;
4148 }
4149
4150 LLVMTypeRef v3i32 = LLVMVectorType(ctx->i32, 3);
4151
4152 switch (type) {
4153 case PIPE_SHADER_VERTEX:
4154 declare_default_desc_pointers(ctx, params, &num_params);
4155 declare_vs_specific_input_sgprs(ctx, params, &num_params);
4156
4157 if (shader->key.as_es) {
4158 params[ctx->param_es2gs_offset = num_params++] = ctx->i32;
4159 } else if (shader->key.as_ls) {
4160 /* no extra parameters */
4161 } else {
4162 if (shader->is_gs_copy_shader)
4163 num_params = ctx->param_rw_buffers + 1;
4164
4165 /* The locations of the other parameters are assigned dynamically. */
4166 declare_streamout_params(ctx, &shader->selector->so,
4167 params, ctx->i32, &num_params);
4168 }
4169
4170 last_sgpr = num_params-1;
4171
4172 /* VGPRs */
4173 declare_vs_input_vgprs(ctx, params, &num_params,
4174 &num_prolog_vgprs);
4175 break;
4176
4177 case PIPE_SHADER_TESS_CTRL: /* SI-CI-VI */
4178 declare_default_desc_pointers(ctx, params, &num_params);
4179 params[ctx->param_tcs_offchip_layout = num_params++] = ctx->i32;
4180 params[ctx->param_tcs_out_lds_offsets = num_params++] = ctx->i32;
4181 params[ctx->param_tcs_out_lds_layout = num_params++] = ctx->i32;
4182 params[ctx->param_vs_state_bits = num_params++] = ctx->i32;
4183 params[ctx->param_tcs_offchip_addr_base64k = num_params++] = ctx->i32;
4184 params[ctx->param_tcs_factor_addr_base64k = num_params++] = ctx->i32;
4185 params[ctx->param_tcs_offchip_offset = num_params++] = ctx->i32;
4186 params[ctx->param_tcs_factor_offset = num_params++] = ctx->i32;
4187 last_sgpr = num_params - 1;
4188
4189 /* VGPRs */
4190 params[ctx->param_tcs_patch_id = num_params++] = ctx->i32;
4191 params[ctx->param_tcs_rel_ids = num_params++] = ctx->i32;
4192
4193 /* param_tcs_offchip_offset and param_tcs_factor_offset are
4194 * placed after the user SGPRs.
4195 */
4196 for (i = 0; i < GFX6_TCS_NUM_USER_SGPR + 2; i++)
4197 returns[num_returns++] = ctx->i32; /* SGPRs */
4198 for (i = 0; i < 3; i++)
4199 returns[num_returns++] = ctx->f32; /* VGPRs */
4200 break;
4201
4202 case SI_SHADER_MERGED_VERTEX_TESSCTRL:
4203 /* Merged stages have 8 system SGPRs at the beginning. */
4204 params[ctx->param_rw_buffers = num_params++] = /* SPI_SHADER_USER_DATA_ADDR_LO_HS */
4205 si_const_array(ctx->v4i32, SI_NUM_RW_BUFFERS);
4206 params[ctx->param_tcs_offchip_offset = num_params++] = ctx->i32;
4207 params[ctx->param_merged_wave_info = num_params++] = ctx->i32;
4208 params[ctx->param_tcs_factor_offset = num_params++] = ctx->i32;
4209 params[ctx->param_merged_scratch_offset = num_params++] = ctx->i32;
4210 params[num_params++] = ctx->i32; /* unused */
4211 params[num_params++] = ctx->i32; /* unused */
4212
4213 params[num_params++] = ctx->i32; /* unused */
4214 params[num_params++] = ctx->i32; /* unused */
4215 declare_per_stage_desc_pointers(ctx, params, &num_params,
4216 ctx->type == PIPE_SHADER_VERTEX);
4217 declare_vs_specific_input_sgprs(ctx, params, &num_params);
4218
4219 params[ctx->param_tcs_offchip_layout = num_params++] = ctx->i32;
4220 params[ctx->param_tcs_out_lds_offsets = num_params++] = ctx->i32;
4221 params[ctx->param_tcs_out_lds_layout = num_params++] = ctx->i32;
4222 params[ctx->param_tcs_offchip_addr_base64k = num_params++] = ctx->i32;
4223 params[ctx->param_tcs_factor_addr_base64k = num_params++] = ctx->i32;
4224 params[num_params++] = ctx->i32; /* unused */
4225
4226 declare_per_stage_desc_pointers(ctx, params, &num_params,
4227 ctx->type == PIPE_SHADER_TESS_CTRL);
4228 last_sgpr = num_params - 1;
4229
4230 /* VGPRs (first TCS, then VS) */
4231 params[ctx->param_tcs_patch_id = num_params++] = ctx->i32;
4232 params[ctx->param_tcs_rel_ids = num_params++] = ctx->i32;
4233
4234 if (ctx->type == PIPE_SHADER_VERTEX) {
4235 declare_vs_input_vgprs(ctx, params, &num_params,
4236 &num_prolog_vgprs);
4237
4238 /* LS return values are inputs to the TCS main shader part. */
4239 for (i = 0; i < 8 + GFX9_TCS_NUM_USER_SGPR; i++)
4240 returns[num_returns++] = ctx->i32; /* SGPRs */
4241 for (i = 0; i < 2; i++)
4242 returns[num_returns++] = ctx->f32; /* VGPRs */
4243 } else {
4244 /* TCS return values are inputs to the TCS epilog.
4245 *
4246 * param_tcs_offchip_offset, param_tcs_factor_offset,
4247 * param_tcs_offchip_layout, and param_rw_buffers
4248 * should be passed to the epilog.
4249 */
4250 for (i = 0; i <= 8 + GFX9_SGPR_TCS_FACTOR_ADDR_BASE64K; i++)
4251 returns[num_returns++] = ctx->i32; /* SGPRs */
4252 for (i = 0; i < 3; i++)
4253 returns[num_returns++] = ctx->f32; /* VGPRs */
4254 }
4255 break;
4256
4257 case SI_SHADER_MERGED_VERTEX_OR_TESSEVAL_GEOMETRY:
4258 /* Merged stages have 8 system SGPRs at the beginning. */
4259 params[ctx->param_rw_buffers = num_params++] = /* SPI_SHADER_USER_DATA_ADDR_LO_GS */
4260 si_const_array(ctx->v4i32, SI_NUM_RW_BUFFERS);
4261 params[ctx->param_gs2vs_offset = num_params++] = ctx->i32;
4262 params[ctx->param_merged_wave_info = num_params++] = ctx->i32;
4263 params[ctx->param_tcs_offchip_offset = num_params++] = ctx->i32;
4264 params[ctx->param_merged_scratch_offset = num_params++] = ctx->i32;
4265 params[num_params++] = ctx->i32; /* unused (SPI_SHADER_PGM_LO/HI_GS << 8) */
4266 params[num_params++] = ctx->i32; /* unused (SPI_SHADER_PGM_LO/HI_GS >> 24) */
4267
4268 params[num_params++] = ctx->i32; /* unused */
4269 params[num_params++] = ctx->i32; /* unused */
4270 declare_per_stage_desc_pointers(ctx, params, &num_params,
4271 (ctx->type == PIPE_SHADER_VERTEX ||
4272 ctx->type == PIPE_SHADER_TESS_EVAL));
4273 if (ctx->type == PIPE_SHADER_VERTEX) {
4274 declare_vs_specific_input_sgprs(ctx, params, &num_params);
4275 } else {
4276 /* TESS_EVAL (and also GEOMETRY):
4277 * Declare as many input SGPRs as the VS has. */
4278 params[ctx->param_tcs_offchip_layout = num_params++] = ctx->i32;
4279 params[ctx->param_tcs_offchip_addr_base64k = num_params++] = ctx->i32;
4280 params[num_params++] = ctx->i32; /* unused */
4281 params[num_params++] = ctx->i32; /* unused */
4282 params[num_params++] = ctx->i32; /* unused */
4283 params[ctx->param_vs_state_bits = num_params++] = ctx->i32; /* unused */
4284 }
4285
4286 declare_per_stage_desc_pointers(ctx, params, &num_params,
4287 ctx->type == PIPE_SHADER_GEOMETRY);
4288 last_sgpr = num_params - 1;
4289
4290 /* VGPRs (first GS, then VS/TES) */
4291 params[ctx->param_gs_vtx01_offset = num_params++] = ctx->i32;
4292 params[ctx->param_gs_vtx23_offset = num_params++] = ctx->i32;
4293 params[ctx->param_gs_prim_id = num_params++] = ctx->i32;
4294 params[ctx->param_gs_instance_id = num_params++] = ctx->i32;
4295 params[ctx->param_gs_vtx45_offset = num_params++] = ctx->i32;
4296
4297 if (ctx->type == PIPE_SHADER_VERTEX) {
4298 declare_vs_input_vgprs(ctx, params, &num_params,
4299 &num_prolog_vgprs);
4300 } else if (ctx->type == PIPE_SHADER_TESS_EVAL) {
4301 declare_tes_input_vgprs(ctx, params, &num_params);
4302 }
4303
4304 if (ctx->type == PIPE_SHADER_VERTEX ||
4305 ctx->type == PIPE_SHADER_TESS_EVAL) {
4306 /* ES return values are inputs to GS. */
4307 for (i = 0; i < 8 + GFX9_GS_NUM_USER_SGPR; i++)
4308 returns[num_returns++] = ctx->i32; /* SGPRs */
4309 for (i = 0; i < 5; i++)
4310 returns[num_returns++] = ctx->f32; /* VGPRs */
4311 }
4312 break;
4313
4314 case PIPE_SHADER_TESS_EVAL:
4315 declare_default_desc_pointers(ctx, params, &num_params);
4316 params[ctx->param_tcs_offchip_layout = num_params++] = ctx->i32;
4317 params[ctx->param_tcs_offchip_addr_base64k = num_params++] = ctx->i32;
4318
4319 if (shader->key.as_es) {
4320 params[ctx->param_tcs_offchip_offset = num_params++] = ctx->i32;
4321 params[num_params++] = ctx->i32;
4322 params[ctx->param_es2gs_offset = num_params++] = ctx->i32;
4323 } else {
4324 params[num_params++] = ctx->i32;
4325 declare_streamout_params(ctx, &shader->selector->so,
4326 params, ctx->i32, &num_params);
4327 params[ctx->param_tcs_offchip_offset = num_params++] = ctx->i32;
4328 }
4329 last_sgpr = num_params - 1;
4330
4331 /* VGPRs */
4332 declare_tes_input_vgprs(ctx, params, &num_params);
4333 break;
4334
4335 case PIPE_SHADER_GEOMETRY:
4336 declare_default_desc_pointers(ctx, params, &num_params);
4337 params[ctx->param_gs2vs_offset = num_params++] = ctx->i32;
4338 params[ctx->param_gs_wave_id = num_params++] = ctx->i32;
4339 last_sgpr = num_params - 1;
4340
4341 /* VGPRs */
4342 params[ctx->param_gs_vtx0_offset = num_params++] = ctx->i32;
4343 params[ctx->param_gs_vtx1_offset = num_params++] = ctx->i32;
4344 params[ctx->param_gs_prim_id = num_params++] = ctx->i32;
4345 params[ctx->param_gs_vtx2_offset = num_params++] = ctx->i32;
4346 params[ctx->param_gs_vtx3_offset = num_params++] = ctx->i32;
4347 params[ctx->param_gs_vtx4_offset = num_params++] = ctx->i32;
4348 params[ctx->param_gs_vtx5_offset = num_params++] = ctx->i32;
4349 params[ctx->param_gs_instance_id = num_params++] = ctx->i32;
4350 break;
4351
4352 case PIPE_SHADER_FRAGMENT:
4353 declare_default_desc_pointers(ctx, params, &num_params);
4354 params[SI_PARAM_ALPHA_REF] = ctx->f32;
4355 params[SI_PARAM_PRIM_MASK] = ctx->i32;
4356 last_sgpr = SI_PARAM_PRIM_MASK;
4357 params[SI_PARAM_PERSP_SAMPLE] = ctx->v2i32;
4358 params[SI_PARAM_PERSP_CENTER] = ctx->v2i32;
4359 params[SI_PARAM_PERSP_CENTROID] = ctx->v2i32;
4360 params[SI_PARAM_PERSP_PULL_MODEL] = v3i32;
4361 params[SI_PARAM_LINEAR_SAMPLE] = ctx->v2i32;
4362 params[SI_PARAM_LINEAR_CENTER] = ctx->v2i32;
4363 params[SI_PARAM_LINEAR_CENTROID] = ctx->v2i32;
4364 params[SI_PARAM_LINE_STIPPLE_TEX] = ctx->f32;
4365 params[SI_PARAM_POS_X_FLOAT] = ctx->f32;
4366 params[SI_PARAM_POS_Y_FLOAT] = ctx->f32;
4367 params[SI_PARAM_POS_Z_FLOAT] = ctx->f32;
4368 params[SI_PARAM_POS_W_FLOAT] = ctx->f32;
4369 params[SI_PARAM_FRONT_FACE] = ctx->i32;
4370 shader->info.face_vgpr_index = 20;
4371 params[SI_PARAM_ANCILLARY] = ctx->i32;
4372 params[SI_PARAM_SAMPLE_COVERAGE] = ctx->f32;
4373 params[SI_PARAM_POS_FIXED_PT] = ctx->i32;
4374 num_params = SI_PARAM_POS_FIXED_PT+1;
4375
4376 /* Color inputs from the prolog. */
4377 if (shader->selector->info.colors_read) {
4378 unsigned num_color_elements =
4379 util_bitcount(shader->selector->info.colors_read);
4380
4381 assert(num_params + num_color_elements <= ARRAY_SIZE(params));
4382 for (i = 0; i < num_color_elements; i++)
4383 params[num_params++] = ctx->f32;
4384
4385 num_prolog_vgprs += num_color_elements;
4386 }
4387
4388 /* Outputs for the epilog. */
4389 num_return_sgprs = SI_SGPR_ALPHA_REF + 1;
4390 num_returns =
4391 num_return_sgprs +
4392 util_bitcount(shader->selector->info.colors_written) * 4 +
4393 shader->selector->info.writes_z +
4394 shader->selector->info.writes_stencil +
4395 shader->selector->info.writes_samplemask +
4396 1 /* SampleMaskIn */;
4397
4398 num_returns = MAX2(num_returns,
4399 num_return_sgprs +
4400 PS_EPILOG_SAMPLEMASK_MIN_LOC + 1);
4401
4402 for (i = 0; i < num_return_sgprs; i++)
4403 returns[i] = ctx->i32;
4404 for (; i < num_returns; i++)
4405 returns[i] = ctx->f32;
4406 break;
4407
4408 case PIPE_SHADER_COMPUTE:
4409 declare_default_desc_pointers(ctx, params, &num_params);
4410 if (shader->selector->info.uses_grid_size)
4411 params[ctx->param_grid_size = num_params++] = v3i32;
4412 if (shader->selector->info.uses_block_size)
4413 params[ctx->param_block_size = num_params++] = v3i32;
4414
4415 for (i = 0; i < 3; i++) {
4416 ctx->param_block_id[i] = -1;
4417 if (shader->selector->info.uses_block_id[i])
4418 params[ctx->param_block_id[i] = num_params++] = ctx->i32;
4419 }
4420 last_sgpr = num_params - 1;
4421
4422 params[ctx->param_thread_id = num_params++] = v3i32;
4423 break;
4424 default:
4425 assert(0 && "unimplemented shader");
4426 return;
4427 }
4428
4429 assert(num_params <= ARRAY_SIZE(params));
4430
4431 si_create_function(ctx, "main", returns, num_returns, params,
4432 num_params, last_sgpr,
4433 si_get_max_workgroup_size(shader));
4434
4435 /* Reserve register locations for VGPR inputs the PS prolog may need. */
4436 if (ctx->type == PIPE_SHADER_FRAGMENT &&
4437 ctx->separate_prolog) {
4438 si_llvm_add_attribute(ctx->main_fn,
4439 "InitialPSInputAddr",
4440 S_0286D0_PERSP_SAMPLE_ENA(1) |
4441 S_0286D0_PERSP_CENTER_ENA(1) |
4442 S_0286D0_PERSP_CENTROID_ENA(1) |
4443 S_0286D0_LINEAR_SAMPLE_ENA(1) |
4444 S_0286D0_LINEAR_CENTER_ENA(1) |
4445 S_0286D0_LINEAR_CENTROID_ENA(1) |
4446 S_0286D0_FRONT_FACE_ENA(1) |
4447 S_0286D0_POS_FIXED_PT_ENA(1));
4448 }
4449
4450 shader->info.num_input_sgprs = 0;
4451 shader->info.num_input_vgprs = 0;
4452
4453 for (i = 0; i <= last_sgpr; ++i)
4454 shader->info.num_input_sgprs += llvm_get_type_size(params[i]) / 4;
4455
4456 for (; i < num_params; ++i)
4457 shader->info.num_input_vgprs += llvm_get_type_size(params[i]) / 4;
4458
4459 assert(shader->info.num_input_vgprs >= num_prolog_vgprs);
4460 shader->info.num_input_vgprs -= num_prolog_vgprs;
4461
4462 if (!ctx->screen->has_ds_bpermute &&
4463 bld_base->info &&
4464 (bld_base->info->opcode_count[TGSI_OPCODE_DDX] > 0 ||
4465 bld_base->info->opcode_count[TGSI_OPCODE_DDY] > 0 ||
4466 bld_base->info->opcode_count[TGSI_OPCODE_DDX_FINE] > 0 ||
4467 bld_base->info->opcode_count[TGSI_OPCODE_DDY_FINE] > 0 ||
4468 bld_base->info->opcode_count[TGSI_OPCODE_INTERP_OFFSET] > 0 ||
4469 bld_base->info->opcode_count[TGSI_OPCODE_INTERP_SAMPLE] > 0))
4470 ctx->lds =
4471 LLVMAddGlobalInAddressSpace(gallivm->module,
4472 LLVMArrayType(ctx->i32, 64),
4473 "ddxy_lds",
4474 LOCAL_ADDR_SPACE);
4475
4476 if (shader->key.as_ls ||
4477 ctx->type == PIPE_SHADER_TESS_CTRL ||
4478 /* GFX9 has the ESGS ring buffer in LDS. */
4479 (ctx->screen->b.chip_class >= GFX9 &&
4480 (shader->key.as_es ||
4481 ctx->type == PIPE_SHADER_GEOMETRY)))
4482 declare_lds_as_pointer(ctx);
4483 }
4484
4485 /**
4486 * Load ESGS and GSVS ring buffer resource descriptors and save the variables
4487 * for later use.
4488 */
4489 static void preload_ring_buffers(struct si_shader_context *ctx)
4490 {
4491 struct gallivm_state *gallivm = &ctx->gallivm;
4492 LLVMBuilderRef builder = gallivm->builder;
4493
4494 LLVMValueRef buf_ptr = LLVMGetParam(ctx->main_fn,
4495 ctx->param_rw_buffers);
4496
4497 if (ctx->screen->b.chip_class <= VI &&
4498 (ctx->shader->key.as_es || ctx->type == PIPE_SHADER_GEOMETRY)) {
4499 unsigned ring =
4500 ctx->type == PIPE_SHADER_GEOMETRY ? SI_GS_RING_ESGS
4501 : SI_ES_RING_ESGS;
4502 LLVMValueRef offset = LLVMConstInt(ctx->i32, ring, 0);
4503
4504 ctx->esgs_ring =
4505 ac_build_indexed_load_const(&ctx->ac, buf_ptr, offset);
4506 }
4507
4508 if (ctx->shader->is_gs_copy_shader) {
4509 LLVMValueRef offset = LLVMConstInt(ctx->i32, SI_RING_GSVS, 0);
4510
4511 ctx->gsvs_ring[0] =
4512 ac_build_indexed_load_const(&ctx->ac, buf_ptr, offset);
4513 } else if (ctx->type == PIPE_SHADER_GEOMETRY) {
4514 const struct si_shader_selector *sel = ctx->shader->selector;
4515 LLVMValueRef offset = LLVMConstInt(ctx->i32, SI_RING_GSVS, 0);
4516 LLVMValueRef base_ring;
4517
4518 base_ring = ac_build_indexed_load_const(&ctx->ac, buf_ptr, offset);
4519
4520 /* The conceptual layout of the GSVS ring is
4521 * v0c0 .. vLv0 v0c1 .. vLc1 ..
4522 * but the real memory layout is swizzled across
4523 * threads:
4524 * t0v0c0 .. t15v0c0 t0v1c0 .. t15v1c0 ... t15vLcL
4525 * t16v0c0 ..
4526 * Override the buffer descriptor accordingly.
4527 */
4528 LLVMTypeRef v2i64 = LLVMVectorType(ctx->i64, 2);
4529 uint64_t stream_offset = 0;
4530
4531 for (unsigned stream = 0; stream < 4; ++stream) {
4532 unsigned num_components;
4533 unsigned stride;
4534 unsigned num_records;
4535 LLVMValueRef ring, tmp;
4536
4537 num_components = sel->info.num_stream_output_components[stream];
4538 if (!num_components)
4539 continue;
4540
4541 stride = 4 * num_components * sel->gs_max_out_vertices;
4542
4543 /* Limit on the stride field for <= CIK. */
4544 assert(stride < (1 << 14));
4545
4546 num_records = 64;
4547
4548 ring = LLVMBuildBitCast(builder, base_ring, v2i64, "");
4549 tmp = LLVMBuildExtractElement(builder, ring, ctx->i32_0, "");
4550 tmp = LLVMBuildAdd(builder, tmp,
4551 LLVMConstInt(ctx->i64,
4552 stream_offset, 0), "");
4553 stream_offset += stride * 64;
4554
4555 ring = LLVMBuildInsertElement(builder, ring, tmp, ctx->i32_0, "");
4556 ring = LLVMBuildBitCast(builder, ring, ctx->v4i32, "");
4557 tmp = LLVMBuildExtractElement(builder, ring, ctx->i32_1, "");
4558 tmp = LLVMBuildOr(builder, tmp,
4559 LLVMConstInt(ctx->i32,
4560 S_008F04_STRIDE(stride) |
4561 S_008F04_SWIZZLE_ENABLE(1), 0), "");
4562 ring = LLVMBuildInsertElement(builder, ring, tmp, ctx->i32_1, "");
4563 ring = LLVMBuildInsertElement(builder, ring,
4564 LLVMConstInt(ctx->i32, num_records, 0),
4565 LLVMConstInt(ctx->i32, 2, 0), "");
4566 ring = LLVMBuildInsertElement(builder, ring,
4567 LLVMConstInt(ctx->i32,
4568 S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
4569 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
4570 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
4571 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
4572 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
4573 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) |
4574 S_008F0C_ELEMENT_SIZE(1) | /* element_size = 4 (bytes) */
4575 S_008F0C_INDEX_STRIDE(1) | /* index_stride = 16 (elements) */
4576 S_008F0C_ADD_TID_ENABLE(1),
4577 0),
4578 LLVMConstInt(ctx->i32, 3, 0), "");
4579
4580 ctx->gsvs_ring[stream] = ring;
4581 }
4582 }
4583 }
4584
4585 static void si_llvm_emit_polygon_stipple(struct si_shader_context *ctx,
4586 LLVMValueRef param_rw_buffers,
4587 unsigned param_pos_fixed_pt)
4588 {
4589 struct gallivm_state *gallivm = &ctx->gallivm;
4590 LLVMBuilderRef builder = gallivm->builder;
4591 LLVMValueRef slot, desc, offset, row, bit, address[2];
4592
4593 /* Use the fixed-point gl_FragCoord input.
4594 * Since the stipple pattern is 32x32 and it repeats, just get 5 bits
4595 * per coordinate to get the repeating effect.
4596 */
4597 address[0] = unpack_param(ctx, param_pos_fixed_pt, 0, 5);
4598 address[1] = unpack_param(ctx, param_pos_fixed_pt, 16, 5);
4599
4600 /* Load the buffer descriptor. */
4601 slot = LLVMConstInt(ctx->i32, SI_PS_CONST_POLY_STIPPLE, 0);
4602 desc = ac_build_indexed_load_const(&ctx->ac, param_rw_buffers, slot);
4603
4604 /* The stipple pattern is 32x32, each row has 32 bits. */
4605 offset = LLVMBuildMul(builder, address[1],
4606 LLVMConstInt(ctx->i32, 4, 0), "");
4607 row = buffer_load_const(ctx, desc, offset);
4608 row = LLVMBuildBitCast(builder, row, ctx->i32, "");
4609 bit = LLVMBuildLShr(builder, row, address[0], "");
4610 bit = LLVMBuildTrunc(builder, bit, ctx->i1, "");
4611
4612 /* The intrinsic kills the thread if arg < 0. */
4613 bit = LLVMBuildSelect(builder, bit, LLVMConstReal(ctx->f32, 0),
4614 LLVMConstReal(ctx->f32, -1), "");
4615 ac_build_kill(&ctx->ac, bit);
4616 }
4617
4618 void si_shader_binary_read_config(struct ac_shader_binary *binary,
4619 struct si_shader_config *conf,
4620 unsigned symbol_offset)
4621 {
4622 unsigned i;
4623 const unsigned char *config =
4624 ac_shader_binary_config_start(binary, symbol_offset);
4625 bool really_needs_scratch = false;
4626
4627 /* LLVM adds SGPR spills to the scratch size.
4628 * Find out if we really need the scratch buffer.
4629 */
4630 for (i = 0; i < binary->reloc_count; i++) {
4631 const struct ac_shader_reloc *reloc = &binary->relocs[i];
4632
4633 if (!strcmp(scratch_rsrc_dword0_symbol, reloc->name) ||
4634 !strcmp(scratch_rsrc_dword1_symbol, reloc->name)) {
4635 really_needs_scratch = true;
4636 break;
4637 }
4638 }
4639
4640 /* XXX: We may be able to emit some of these values directly rather than
4641 * extracting fields to be emitted later.
4642 */
4643
4644 for (i = 0; i < binary->config_size_per_symbol; i+= 8) {
4645 unsigned reg = util_le32_to_cpu(*(uint32_t*)(config + i));
4646 unsigned value = util_le32_to_cpu(*(uint32_t*)(config + i + 4));
4647 switch (reg) {
4648 case R_00B028_SPI_SHADER_PGM_RSRC1_PS:
4649 case R_00B128_SPI_SHADER_PGM_RSRC1_VS:
4650 case R_00B228_SPI_SHADER_PGM_RSRC1_GS:
4651 case R_00B428_SPI_SHADER_PGM_RSRC1_HS:
4652 case R_00B848_COMPUTE_PGM_RSRC1:
4653 conf->num_sgprs = MAX2(conf->num_sgprs, (G_00B028_SGPRS(value) + 1) * 8);
4654 conf->num_vgprs = MAX2(conf->num_vgprs, (G_00B028_VGPRS(value) + 1) * 4);
4655 conf->float_mode = G_00B028_FLOAT_MODE(value);
4656 conf->rsrc1 = value;
4657 break;
4658 case R_00B02C_SPI_SHADER_PGM_RSRC2_PS:
4659 conf->lds_size = MAX2(conf->lds_size, G_00B02C_EXTRA_LDS_SIZE(value));
4660 break;
4661 case R_00B84C_COMPUTE_PGM_RSRC2:
4662 conf->lds_size = MAX2(conf->lds_size, G_00B84C_LDS_SIZE(value));
4663 conf->rsrc2 = value;
4664 break;
4665 case R_0286CC_SPI_PS_INPUT_ENA:
4666 conf->spi_ps_input_ena = value;
4667 break;
4668 case R_0286D0_SPI_PS_INPUT_ADDR:
4669 conf->spi_ps_input_addr = value;
4670 break;
4671 case R_0286E8_SPI_TMPRING_SIZE:
4672 case R_00B860_COMPUTE_TMPRING_SIZE:
4673 /* WAVESIZE is in units of 256 dwords. */
4674 if (really_needs_scratch)
4675 conf->scratch_bytes_per_wave =
4676 G_00B860_WAVESIZE(value) * 256 * 4;
4677 break;
4678 case 0x4: /* SPILLED_SGPRS */
4679 conf->spilled_sgprs = value;
4680 break;
4681 case 0x8: /* SPILLED_VGPRS */
4682 conf->spilled_vgprs = value;
4683 break;
4684 default:
4685 {
4686 static bool printed;
4687
4688 if (!printed) {
4689 fprintf(stderr, "Warning: LLVM emitted unknown "
4690 "config register: 0x%x\n", reg);
4691 printed = true;
4692 }
4693 }
4694 break;
4695 }
4696 }
4697
4698 if (!conf->spi_ps_input_addr)
4699 conf->spi_ps_input_addr = conf->spi_ps_input_ena;
4700 }
4701
4702 void si_shader_apply_scratch_relocs(struct si_shader *shader,
4703 uint64_t scratch_va)
4704 {
4705 unsigned i;
4706 uint32_t scratch_rsrc_dword0 = scratch_va;
4707 uint32_t scratch_rsrc_dword1 =
4708 S_008F04_BASE_ADDRESS_HI(scratch_va >> 32);
4709
4710 /* Enable scratch coalescing. */
4711 scratch_rsrc_dword1 |= S_008F04_SWIZZLE_ENABLE(1);
4712
4713 for (i = 0 ; i < shader->binary.reloc_count; i++) {
4714 const struct ac_shader_reloc *reloc =
4715 &shader->binary.relocs[i];
4716 if (!strcmp(scratch_rsrc_dword0_symbol, reloc->name)) {
4717 util_memcpy_cpu_to_le32(shader->binary.code + reloc->offset,
4718 &scratch_rsrc_dword0, 4);
4719 } else if (!strcmp(scratch_rsrc_dword1_symbol, reloc->name)) {
4720 util_memcpy_cpu_to_le32(shader->binary.code + reloc->offset,
4721 &scratch_rsrc_dword1, 4);
4722 }
4723 }
4724 }
4725
4726 static unsigned si_get_shader_binary_size(const struct si_shader *shader)
4727 {
4728 unsigned size = shader->binary.code_size;
4729
4730 if (shader->prolog)
4731 size += shader->prolog->binary.code_size;
4732 if (shader->previous_stage)
4733 size += shader->previous_stage->binary.code_size;
4734 if (shader->prolog2)
4735 size += shader->prolog2->binary.code_size;
4736 if (shader->epilog)
4737 size += shader->epilog->binary.code_size;
4738 return size;
4739 }
4740
4741 int si_shader_binary_upload(struct si_screen *sscreen, struct si_shader *shader)
4742 {
4743 const struct ac_shader_binary *prolog =
4744 shader->prolog ? &shader->prolog->binary : NULL;
4745 const struct ac_shader_binary *previous_stage =
4746 shader->previous_stage ? &shader->previous_stage->binary : NULL;
4747 const struct ac_shader_binary *prolog2 =
4748 shader->prolog2 ? &shader->prolog2->binary : NULL;
4749 const struct ac_shader_binary *epilog =
4750 shader->epilog ? &shader->epilog->binary : NULL;
4751 const struct ac_shader_binary *mainb = &shader->binary;
4752 unsigned bo_size = si_get_shader_binary_size(shader) +
4753 (!epilog ? mainb->rodata_size : 0);
4754 unsigned char *ptr;
4755
4756 assert(!prolog || !prolog->rodata_size);
4757 assert(!previous_stage || !previous_stage->rodata_size);
4758 assert(!prolog2 || !prolog2->rodata_size);
4759 assert((!prolog && !previous_stage && !prolog2 && !epilog) ||
4760 !mainb->rodata_size);
4761 assert(!epilog || !epilog->rodata_size);
4762
4763 /* GFX9 can fetch at most 128 bytes past the end of the shader.
4764 * Prevent VM faults.
4765 */
4766 if (sscreen->b.chip_class >= GFX9)
4767 bo_size += 128;
4768
4769 r600_resource_reference(&shader->bo, NULL);
4770 shader->bo = (struct r600_resource*)
4771 pipe_buffer_create(&sscreen->b.b, 0,
4772 PIPE_USAGE_IMMUTABLE,
4773 align(bo_size, SI_CPDMA_ALIGNMENT));
4774 if (!shader->bo)
4775 return -ENOMEM;
4776
4777 /* Upload. */
4778 ptr = sscreen->b.ws->buffer_map(shader->bo->buf, NULL,
4779 PIPE_TRANSFER_READ_WRITE |
4780 PIPE_TRANSFER_UNSYNCHRONIZED);
4781
4782 /* Don't use util_memcpy_cpu_to_le32. LLVM binaries are
4783 * endian-independent. */
4784 if (prolog) {
4785 memcpy(ptr, prolog->code, prolog->code_size);
4786 ptr += prolog->code_size;
4787 }
4788 if (previous_stage) {
4789 memcpy(ptr, previous_stage->code, previous_stage->code_size);
4790 ptr += previous_stage->code_size;
4791 }
4792 if (prolog2) {
4793 memcpy(ptr, prolog2->code, prolog2->code_size);
4794 ptr += prolog2->code_size;
4795 }
4796
4797 memcpy(ptr, mainb->code, mainb->code_size);
4798 ptr += mainb->code_size;
4799
4800 if (epilog)
4801 memcpy(ptr, epilog->code, epilog->code_size);
4802 else if (mainb->rodata_size > 0)
4803 memcpy(ptr, mainb->rodata, mainb->rodata_size);
4804
4805 sscreen->b.ws->buffer_unmap(shader->bo->buf);
4806 return 0;
4807 }
4808
4809 static void si_shader_dump_disassembly(const struct ac_shader_binary *binary,
4810 struct pipe_debug_callback *debug,
4811 const char *name, FILE *file)
4812 {
4813 char *line, *p;
4814 unsigned i, count;
4815
4816 if (binary->disasm_string) {
4817 fprintf(file, "Shader %s disassembly:\n", name);
4818 fprintf(file, "%s", binary->disasm_string);
4819
4820 if (debug && debug->debug_message) {
4821 /* Very long debug messages are cut off, so send the
4822 * disassembly one line at a time. This causes more
4823 * overhead, but on the plus side it simplifies
4824 * parsing of resulting logs.
4825 */
4826 pipe_debug_message(debug, SHADER_INFO,
4827 "Shader Disassembly Begin");
4828
4829 line = binary->disasm_string;
4830 while (*line) {
4831 p = util_strchrnul(line, '\n');
4832 count = p - line;
4833
4834 if (count) {
4835 pipe_debug_message(debug, SHADER_INFO,
4836 "%.*s", count, line);
4837 }
4838
4839 if (!*p)
4840 break;
4841 line = p + 1;
4842 }
4843
4844 pipe_debug_message(debug, SHADER_INFO,
4845 "Shader Disassembly End");
4846 }
4847 } else {
4848 fprintf(file, "Shader %s binary:\n", name);
4849 for (i = 0; i < binary->code_size; i += 4) {
4850 fprintf(file, "@0x%x: %02x%02x%02x%02x\n", i,
4851 binary->code[i + 3], binary->code[i + 2],
4852 binary->code[i + 1], binary->code[i]);
4853 }
4854 }
4855 }
4856
4857 static void si_shader_dump_stats(struct si_screen *sscreen,
4858 const struct si_shader *shader,
4859 struct pipe_debug_callback *debug,
4860 unsigned processor,
4861 FILE *file,
4862 bool check_debug_option)
4863 {
4864 const struct si_shader_config *conf = &shader->config;
4865 unsigned num_inputs = shader->selector ? shader->selector->info.num_inputs : 0;
4866 unsigned code_size = si_get_shader_binary_size(shader);
4867 unsigned lds_increment = sscreen->b.chip_class >= CIK ? 512 : 256;
4868 unsigned lds_per_wave = 0;
4869 unsigned max_simd_waves = 10;
4870
4871 /* Compute LDS usage for PS. */
4872 switch (processor) {
4873 case PIPE_SHADER_FRAGMENT:
4874 /* The minimum usage per wave is (num_inputs * 48). The maximum
4875 * usage is (num_inputs * 48 * 16).
4876 * We can get anything in between and it varies between waves.
4877 *
4878 * The 48 bytes per input for a single primitive is equal to
4879 * 4 bytes/component * 4 components/input * 3 points.
4880 *
4881 * Other stages don't know the size at compile time or don't
4882 * allocate LDS per wave, but instead they do it per thread group.
4883 */
4884 lds_per_wave = conf->lds_size * lds_increment +
4885 align(num_inputs * 48, lds_increment);
4886 break;
4887 case PIPE_SHADER_COMPUTE:
4888 if (shader->selector) {
4889 unsigned max_workgroup_size =
4890 si_get_max_workgroup_size(shader);
4891 lds_per_wave = (conf->lds_size * lds_increment) /
4892 DIV_ROUND_UP(max_workgroup_size, 64);
4893 }
4894 break;
4895 }
4896
4897 /* Compute the per-SIMD wave counts. */
4898 if (conf->num_sgprs) {
4899 if (sscreen->b.chip_class >= VI)
4900 max_simd_waves = MIN2(max_simd_waves, 800 / conf->num_sgprs);
4901 else
4902 max_simd_waves = MIN2(max_simd_waves, 512 / conf->num_sgprs);
4903 }
4904
4905 if (conf->num_vgprs)
4906 max_simd_waves = MIN2(max_simd_waves, 256 / conf->num_vgprs);
4907
4908 /* LDS is 64KB per CU (4 SIMDs), which is 16KB per SIMD (usage above
4909 * 16KB makes some SIMDs unoccupied). */
4910 if (lds_per_wave)
4911 max_simd_waves = MIN2(max_simd_waves, 16384 / lds_per_wave);
4912
4913 if (!check_debug_option ||
4914 r600_can_dump_shader(&sscreen->b, processor)) {
4915 if (processor == PIPE_SHADER_FRAGMENT) {
4916 fprintf(file, "*** SHADER CONFIG ***\n"
4917 "SPI_PS_INPUT_ADDR = 0x%04x\n"
4918 "SPI_PS_INPUT_ENA = 0x%04x\n",
4919 conf->spi_ps_input_addr, conf->spi_ps_input_ena);
4920 }
4921
4922 fprintf(file, "*** SHADER STATS ***\n"
4923 "SGPRS: %d\n"
4924 "VGPRS: %d\n"
4925 "Spilled SGPRs: %d\n"
4926 "Spilled VGPRs: %d\n"
4927 "Private memory VGPRs: %d\n"
4928 "Code Size: %d bytes\n"
4929 "LDS: %d blocks\n"
4930 "Scratch: %d bytes per wave\n"
4931 "Max Waves: %d\n"
4932 "********************\n\n\n",
4933 conf->num_sgprs, conf->num_vgprs,
4934 conf->spilled_sgprs, conf->spilled_vgprs,
4935 conf->private_mem_vgprs, code_size,
4936 conf->lds_size, conf->scratch_bytes_per_wave,
4937 max_simd_waves);
4938 }
4939
4940 pipe_debug_message(debug, SHADER_INFO,
4941 "Shader Stats: SGPRS: %d VGPRS: %d Code Size: %d "
4942 "LDS: %d Scratch: %d Max Waves: %d Spilled SGPRs: %d "
4943 "Spilled VGPRs: %d PrivMem VGPRs: %d",
4944 conf->num_sgprs, conf->num_vgprs, code_size,
4945 conf->lds_size, conf->scratch_bytes_per_wave,
4946 max_simd_waves, conf->spilled_sgprs,
4947 conf->spilled_vgprs, conf->private_mem_vgprs);
4948 }
4949
4950 const char *si_get_shader_name(const struct si_shader *shader, unsigned processor)
4951 {
4952 switch (processor) {
4953 case PIPE_SHADER_VERTEX:
4954 if (shader->key.as_es)
4955 return "Vertex Shader as ES";
4956 else if (shader->key.as_ls)
4957 return "Vertex Shader as LS";
4958 else
4959 return "Vertex Shader as VS";
4960 case PIPE_SHADER_TESS_CTRL:
4961 return "Tessellation Control Shader";
4962 case PIPE_SHADER_TESS_EVAL:
4963 if (shader->key.as_es)
4964 return "Tessellation Evaluation Shader as ES";
4965 else
4966 return "Tessellation Evaluation Shader as VS";
4967 case PIPE_SHADER_GEOMETRY:
4968 if (shader->is_gs_copy_shader)
4969 return "GS Copy Shader as VS";
4970 else
4971 return "Geometry Shader";
4972 case PIPE_SHADER_FRAGMENT:
4973 return "Pixel Shader";
4974 case PIPE_SHADER_COMPUTE:
4975 return "Compute Shader";
4976 default:
4977 return "Unknown Shader";
4978 }
4979 }
4980
4981 void si_shader_dump(struct si_screen *sscreen, const struct si_shader *shader,
4982 struct pipe_debug_callback *debug, unsigned processor,
4983 FILE *file, bool check_debug_option)
4984 {
4985 if (!check_debug_option ||
4986 r600_can_dump_shader(&sscreen->b, processor))
4987 si_dump_shader_key(processor, shader, file);
4988
4989 if (!check_debug_option && shader->binary.llvm_ir_string) {
4990 fprintf(file, "\n%s - main shader part - LLVM IR:\n\n",
4991 si_get_shader_name(shader, processor));
4992 fprintf(file, "%s\n", shader->binary.llvm_ir_string);
4993 }
4994
4995 if (!check_debug_option ||
4996 (r600_can_dump_shader(&sscreen->b, processor) &&
4997 !(sscreen->b.debug_flags & DBG_NO_ASM))) {
4998 fprintf(file, "\n%s:\n", si_get_shader_name(shader, processor));
4999
5000 if (shader->prolog)
5001 si_shader_dump_disassembly(&shader->prolog->binary,
5002 debug, "prolog", file);
5003 if (shader->previous_stage)
5004 si_shader_dump_disassembly(&shader->previous_stage->binary,
5005 debug, "previous stage", file);
5006 if (shader->prolog2)
5007 si_shader_dump_disassembly(&shader->prolog2->binary,
5008 debug, "prolog2", file);
5009
5010 si_shader_dump_disassembly(&shader->binary, debug, "main", file);
5011
5012 if (shader->epilog)
5013 si_shader_dump_disassembly(&shader->epilog->binary,
5014 debug, "epilog", file);
5015 fprintf(file, "\n");
5016 }
5017
5018 si_shader_dump_stats(sscreen, shader, debug, processor, file,
5019 check_debug_option);
5020 }
5021
5022 static int si_compile_llvm(struct si_screen *sscreen,
5023 struct ac_shader_binary *binary,
5024 struct si_shader_config *conf,
5025 LLVMTargetMachineRef tm,
5026 LLVMModuleRef mod,
5027 struct pipe_debug_callback *debug,
5028 unsigned processor,
5029 const char *name)
5030 {
5031 int r = 0;
5032 unsigned count = p_atomic_inc_return(&sscreen->b.num_compilations);
5033
5034 if (r600_can_dump_shader(&sscreen->b, processor)) {
5035 fprintf(stderr, "radeonsi: Compiling shader %d\n", count);
5036
5037 if (!(sscreen->b.debug_flags & (DBG_NO_IR | DBG_PREOPT_IR))) {
5038 fprintf(stderr, "%s LLVM IR:\n\n", name);
5039 ac_dump_module(mod);
5040 fprintf(stderr, "\n");
5041 }
5042 }
5043
5044 if (sscreen->record_llvm_ir) {
5045 char *ir = LLVMPrintModuleToString(mod);
5046 binary->llvm_ir_string = strdup(ir);
5047 LLVMDisposeMessage(ir);
5048 }
5049
5050 if (!si_replace_shader(count, binary)) {
5051 r = si_llvm_compile(mod, binary, tm, debug);
5052 if (r)
5053 return r;
5054 }
5055
5056 si_shader_binary_read_config(binary, conf, 0);
5057
5058 /* Enable 64-bit and 16-bit denormals, because there is no performance
5059 * cost.
5060 *
5061 * If denormals are enabled, all floating-point output modifiers are
5062 * ignored.
5063 *
5064 * Don't enable denormals for 32-bit floats, because:
5065 * - Floating-point output modifiers would be ignored by the hw.
5066 * - Some opcodes don't support denormals, such as v_mad_f32. We would
5067 * have to stop using those.
5068 * - SI & CI would be very slow.
5069 */
5070 conf->float_mode |= V_00B028_FP_64_DENORMS;
5071
5072 FREE(binary->config);
5073 FREE(binary->global_symbol_offsets);
5074 binary->config = NULL;
5075 binary->global_symbol_offsets = NULL;
5076
5077 /* Some shaders can't have rodata because their binaries can be
5078 * concatenated.
5079 */
5080 if (binary->rodata_size &&
5081 (processor == PIPE_SHADER_VERTEX ||
5082 processor == PIPE_SHADER_TESS_CTRL ||
5083 processor == PIPE_SHADER_TESS_EVAL ||
5084 processor == PIPE_SHADER_FRAGMENT)) {
5085 fprintf(stderr, "radeonsi: The shader can't have rodata.");
5086 return -EINVAL;
5087 }
5088
5089 return r;
5090 }
5091
5092 static void si_llvm_build_ret(struct si_shader_context *ctx, LLVMValueRef ret)
5093 {
5094 if (LLVMGetTypeKind(LLVMTypeOf(ret)) == LLVMVoidTypeKind)
5095 LLVMBuildRetVoid(ctx->gallivm.builder);
5096 else
5097 LLVMBuildRet(ctx->gallivm.builder, ret);
5098 }
5099
5100 /* Generate code for the hardware VS shader stage to go with a geometry shader */
5101 struct si_shader *
5102 si_generate_gs_copy_shader(struct si_screen *sscreen,
5103 LLVMTargetMachineRef tm,
5104 struct si_shader_selector *gs_selector,
5105 struct pipe_debug_callback *debug)
5106 {
5107 struct si_shader_context ctx;
5108 struct si_shader *shader;
5109 struct gallivm_state *gallivm = &ctx.gallivm;
5110 LLVMBuilderRef builder;
5111 struct lp_build_tgsi_context *bld_base = &ctx.bld_base;
5112 struct lp_build_context *uint = &bld_base->uint_bld;
5113 struct si_shader_output_values *outputs;
5114 struct tgsi_shader_info *gsinfo = &gs_selector->info;
5115 int i, r;
5116
5117 outputs = MALLOC(gsinfo->num_outputs * sizeof(outputs[0]));
5118
5119 if (!outputs)
5120 return NULL;
5121
5122 shader = CALLOC_STRUCT(si_shader);
5123 if (!shader) {
5124 FREE(outputs);
5125 return NULL;
5126 }
5127
5128
5129 shader->selector = gs_selector;
5130 shader->is_gs_copy_shader = true;
5131
5132 si_init_shader_ctx(&ctx, sscreen, tm);
5133 ctx.shader = shader;
5134 ctx.type = PIPE_SHADER_VERTEX;
5135
5136 builder = gallivm->builder;
5137
5138 create_function(&ctx);
5139 preload_ring_buffers(&ctx);
5140
5141 LLVMValueRef voffset =
5142 lp_build_mul_imm(uint, LLVMGetParam(ctx.main_fn,
5143 ctx.param_vertex_id), 4);
5144
5145 /* Fetch the vertex stream ID.*/
5146 LLVMValueRef stream_id;
5147
5148 if (gs_selector->so.num_outputs)
5149 stream_id = unpack_param(&ctx, ctx.param_streamout_config, 24, 2);
5150 else
5151 stream_id = ctx.i32_0;
5152
5153 /* Fill in output information. */
5154 for (i = 0; i < gsinfo->num_outputs; ++i) {
5155 outputs[i].semantic_name = gsinfo->output_semantic_name[i];
5156 outputs[i].semantic_index = gsinfo->output_semantic_index[i];
5157
5158 for (int chan = 0; chan < 4; chan++) {
5159 outputs[i].vertex_stream[chan] =
5160 (gsinfo->output_streams[i] >> (2 * chan)) & 3;
5161 }
5162 }
5163
5164 LLVMBasicBlockRef end_bb;
5165 LLVMValueRef switch_inst;
5166
5167 end_bb = LLVMAppendBasicBlockInContext(gallivm->context, ctx.main_fn, "end");
5168 switch_inst = LLVMBuildSwitch(builder, stream_id, end_bb, 4);
5169
5170 for (int stream = 0; stream < 4; stream++) {
5171 LLVMBasicBlockRef bb;
5172 unsigned offset;
5173
5174 if (!gsinfo->num_stream_output_components[stream])
5175 continue;
5176
5177 if (stream > 0 && !gs_selector->so.num_outputs)
5178 continue;
5179
5180 bb = LLVMInsertBasicBlockInContext(gallivm->context, end_bb, "out");
5181 LLVMAddCase(switch_inst, LLVMConstInt(ctx.i32, stream, 0), bb);
5182 LLVMPositionBuilderAtEnd(builder, bb);
5183
5184 /* Fetch vertex data from GSVS ring */
5185 offset = 0;
5186 for (i = 0; i < gsinfo->num_outputs; ++i) {
5187 for (unsigned chan = 0; chan < 4; chan++) {
5188 if (!(gsinfo->output_usagemask[i] & (1 << chan)) ||
5189 outputs[i].vertex_stream[chan] != stream) {
5190 outputs[i].values[chan] = ctx.bld_base.base.undef;
5191 continue;
5192 }
5193
5194 LLVMValueRef soffset = LLVMConstInt(ctx.i32,
5195 offset * gs_selector->gs_max_out_vertices * 16 * 4, 0);
5196 offset++;
5197
5198 outputs[i].values[chan] =
5199 ac_build_buffer_load(&ctx.ac,
5200 ctx.gsvs_ring[0], 1,
5201 ctx.i32_0, voffset,
5202 soffset, 0, 1, 1, true);
5203 }
5204 }
5205
5206 /* Streamout and exports. */
5207 if (gs_selector->so.num_outputs) {
5208 si_llvm_emit_streamout(&ctx, outputs,
5209 gsinfo->num_outputs,
5210 stream);
5211 }
5212
5213 if (stream == 0)
5214 si_llvm_export_vs(bld_base, outputs, gsinfo->num_outputs);
5215
5216 LLVMBuildBr(builder, end_bb);
5217 }
5218
5219 LLVMPositionBuilderAtEnd(builder, end_bb);
5220
5221 LLVMBuildRetVoid(gallivm->builder);
5222
5223 ctx.type = PIPE_SHADER_GEOMETRY; /* override for shader dumping */
5224 si_llvm_optimize_module(&ctx);
5225
5226 r = si_compile_llvm(sscreen, &ctx.shader->binary,
5227 &ctx.shader->config, ctx.tm,
5228 ctx.gallivm.module,
5229 debug, PIPE_SHADER_GEOMETRY,
5230 "GS Copy Shader");
5231 if (!r) {
5232 if (r600_can_dump_shader(&sscreen->b, PIPE_SHADER_GEOMETRY))
5233 fprintf(stderr, "GS Copy Shader:\n");
5234 si_shader_dump(sscreen, ctx.shader, debug,
5235 PIPE_SHADER_GEOMETRY, stderr, true);
5236 r = si_shader_binary_upload(sscreen, ctx.shader);
5237 }
5238
5239 si_llvm_dispose(&ctx);
5240
5241 FREE(outputs);
5242
5243 if (r != 0) {
5244 FREE(shader);
5245 shader = NULL;
5246 }
5247 return shader;
5248 }
5249
5250 static void si_dump_shader_key_vs(const struct si_shader_key *key,
5251 const struct si_vs_prolog_bits *prolog,
5252 const char *prefix, FILE *f)
5253 {
5254 fprintf(f, " %s.instance_divisors = {", prefix);
5255 for (int i = 0; i < ARRAY_SIZE(prolog->instance_divisors); i++) {
5256 fprintf(f, !i ? "%u" : ", %u",
5257 prolog->instance_divisors[i]);
5258 }
5259 fprintf(f, "}\n");
5260
5261 fprintf(f, " mono.vs.fix_fetch = {");
5262 for (int i = 0; i < SI_MAX_ATTRIBS; i++)
5263 fprintf(f, !i ? "%u" : ", %u", key->mono.vs_fix_fetch[i]);
5264 fprintf(f, "}\n");
5265 }
5266
5267 static void si_dump_shader_key(unsigned processor, const struct si_shader *shader,
5268 FILE *f)
5269 {
5270 const struct si_shader_key *key = &shader->key;
5271
5272 fprintf(f, "SHADER KEY\n");
5273
5274 switch (processor) {
5275 case PIPE_SHADER_VERTEX:
5276 si_dump_shader_key_vs(key, &key->part.vs.prolog,
5277 "part.vs.prolog", f);
5278 fprintf(f, " as_es = %u\n", key->as_es);
5279 fprintf(f, " as_ls = %u\n", key->as_ls);
5280 fprintf(f, " mono.vs_export_prim_id = %u\n",
5281 key->mono.vs_export_prim_id);
5282 break;
5283
5284 case PIPE_SHADER_TESS_CTRL:
5285 if (shader->selector->screen->b.chip_class >= GFX9) {
5286 si_dump_shader_key_vs(key, &key->part.tcs.ls_prolog,
5287 "part.tcs.ls_prolog", f);
5288 }
5289 fprintf(f, " part.tcs.epilog.prim_mode = %u\n", key->part.tcs.epilog.prim_mode);
5290 fprintf(f, " mono.ff_tcs_inputs_to_copy = 0x%"PRIx64"\n", key->mono.ff_tcs_inputs_to_copy);
5291 break;
5292
5293 case PIPE_SHADER_TESS_EVAL:
5294 fprintf(f, " as_es = %u\n", key->as_es);
5295 fprintf(f, " mono.vs_export_prim_id = %u\n",
5296 key->mono.vs_export_prim_id);
5297 break;
5298
5299 case PIPE_SHADER_GEOMETRY:
5300 if (shader->is_gs_copy_shader)
5301 break;
5302
5303 if (shader->selector->screen->b.chip_class >= GFX9 &&
5304 key->part.gs.es->type == PIPE_SHADER_VERTEX) {
5305 si_dump_shader_key_vs(key, &key->part.gs.vs_prolog,
5306 "part.gs.vs_prolog", f);
5307 }
5308 fprintf(f, " part.gs.prolog.tri_strip_adj_fix = %u\n", key->part.gs.prolog.tri_strip_adj_fix);
5309 break;
5310
5311 case PIPE_SHADER_COMPUTE:
5312 break;
5313
5314 case PIPE_SHADER_FRAGMENT:
5315 fprintf(f, " part.ps.prolog.color_two_side = %u\n", key->part.ps.prolog.color_two_side);
5316 fprintf(f, " part.ps.prolog.flatshade_colors = %u\n", key->part.ps.prolog.flatshade_colors);
5317 fprintf(f, " part.ps.prolog.poly_stipple = %u\n", key->part.ps.prolog.poly_stipple);
5318 fprintf(f, " part.ps.prolog.force_persp_sample_interp = %u\n", key->part.ps.prolog.force_persp_sample_interp);
5319 fprintf(f, " part.ps.prolog.force_linear_sample_interp = %u\n", key->part.ps.prolog.force_linear_sample_interp);
5320 fprintf(f, " part.ps.prolog.force_persp_center_interp = %u\n", key->part.ps.prolog.force_persp_center_interp);
5321 fprintf(f, " part.ps.prolog.force_linear_center_interp = %u\n", key->part.ps.prolog.force_linear_center_interp);
5322 fprintf(f, " part.ps.prolog.bc_optimize_for_persp = %u\n", key->part.ps.prolog.bc_optimize_for_persp);
5323 fprintf(f, " part.ps.prolog.bc_optimize_for_linear = %u\n", key->part.ps.prolog.bc_optimize_for_linear);
5324 fprintf(f, " part.ps.epilog.spi_shader_col_format = 0x%x\n", key->part.ps.epilog.spi_shader_col_format);
5325 fprintf(f, " part.ps.epilog.color_is_int8 = 0x%X\n", key->part.ps.epilog.color_is_int8);
5326 fprintf(f, " part.ps.epilog.color_is_int10 = 0x%X\n", key->part.ps.epilog.color_is_int10);
5327 fprintf(f, " part.ps.epilog.last_cbuf = %u\n", key->part.ps.epilog.last_cbuf);
5328 fprintf(f, " part.ps.epilog.alpha_func = %u\n", key->part.ps.epilog.alpha_func);
5329 fprintf(f, " part.ps.epilog.alpha_to_one = %u\n", key->part.ps.epilog.alpha_to_one);
5330 fprintf(f, " part.ps.epilog.poly_line_smoothing = %u\n", key->part.ps.epilog.poly_line_smoothing);
5331 fprintf(f, " part.ps.epilog.clamp_color = %u\n", key->part.ps.epilog.clamp_color);
5332 break;
5333
5334 default:
5335 assert(0);
5336 }
5337
5338 if ((processor == PIPE_SHADER_GEOMETRY ||
5339 processor == PIPE_SHADER_TESS_EVAL ||
5340 processor == PIPE_SHADER_VERTEX) &&
5341 !key->as_es && !key->as_ls) {
5342 fprintf(f, " opt.hw_vs.kill_outputs = 0x%"PRIx64"\n", key->opt.hw_vs.kill_outputs);
5343 fprintf(f, " opt.hw_vs.clip_disable = %u\n", key->opt.hw_vs.clip_disable);
5344 }
5345 }
5346
5347 static void si_init_shader_ctx(struct si_shader_context *ctx,
5348 struct si_screen *sscreen,
5349 LLVMTargetMachineRef tm)
5350 {
5351 struct lp_build_tgsi_context *bld_base;
5352
5353 si_llvm_context_init(ctx, sscreen, tm);
5354
5355 bld_base = &ctx->bld_base;
5356 bld_base->emit_fetch_funcs[TGSI_FILE_CONSTANT] = fetch_constant;
5357
5358 bld_base->op_actions[TGSI_OPCODE_INTERP_CENTROID] = interp_action;
5359 bld_base->op_actions[TGSI_OPCODE_INTERP_SAMPLE] = interp_action;
5360 bld_base->op_actions[TGSI_OPCODE_INTERP_OFFSET] = interp_action;
5361
5362 bld_base->op_actions[TGSI_OPCODE_MEMBAR].emit = membar_emit;
5363
5364 bld_base->op_actions[TGSI_OPCODE_CLOCK].emit = clock_emit;
5365
5366 bld_base->op_actions[TGSI_OPCODE_DDX].emit = si_llvm_emit_ddxy;
5367 bld_base->op_actions[TGSI_OPCODE_DDY].emit = si_llvm_emit_ddxy;
5368 bld_base->op_actions[TGSI_OPCODE_DDX_FINE].emit = si_llvm_emit_ddxy;
5369 bld_base->op_actions[TGSI_OPCODE_DDY_FINE].emit = si_llvm_emit_ddxy;
5370
5371 bld_base->op_actions[TGSI_OPCODE_VOTE_ALL].emit = vote_all_emit;
5372 bld_base->op_actions[TGSI_OPCODE_VOTE_ANY].emit = vote_any_emit;
5373 bld_base->op_actions[TGSI_OPCODE_VOTE_EQ].emit = vote_eq_emit;
5374 bld_base->op_actions[TGSI_OPCODE_BALLOT].emit = ballot_emit;
5375 bld_base->op_actions[TGSI_OPCODE_READ_FIRST].intr_name = "llvm.amdgcn.readfirstlane";
5376 bld_base->op_actions[TGSI_OPCODE_READ_FIRST].emit = read_lane_emit;
5377 bld_base->op_actions[TGSI_OPCODE_READ_INVOC].intr_name = "llvm.amdgcn.readlane";
5378 bld_base->op_actions[TGSI_OPCODE_READ_INVOC].fetch_args = read_invoc_fetch_args;
5379 bld_base->op_actions[TGSI_OPCODE_READ_INVOC].emit = read_lane_emit;
5380
5381 bld_base->op_actions[TGSI_OPCODE_EMIT].emit = si_llvm_emit_vertex;
5382 bld_base->op_actions[TGSI_OPCODE_ENDPRIM].emit = si_llvm_emit_primitive;
5383 bld_base->op_actions[TGSI_OPCODE_BARRIER].emit = si_llvm_emit_barrier;
5384 }
5385
5386 static void si_optimize_vs_outputs(struct si_shader_context *ctx)
5387 {
5388 struct si_shader *shader = ctx->shader;
5389 struct tgsi_shader_info *info = &shader->selector->info;
5390
5391 if ((ctx->type != PIPE_SHADER_VERTEX &&
5392 ctx->type != PIPE_SHADER_TESS_EVAL) ||
5393 shader->key.as_ls ||
5394 shader->key.as_es)
5395 return;
5396
5397 ac_optimize_vs_outputs(&ctx->ac,
5398 ctx->main_fn,
5399 shader->info.vs_output_param_offset,
5400 info->num_outputs,
5401 &shader->info.nr_param_exports);
5402 }
5403
5404 static void si_count_scratch_private_memory(struct si_shader_context *ctx)
5405 {
5406 ctx->shader->config.private_mem_vgprs = 0;
5407
5408 /* Process all LLVM instructions. */
5409 LLVMBasicBlockRef bb = LLVMGetFirstBasicBlock(ctx->main_fn);
5410 while (bb) {
5411 LLVMValueRef next = LLVMGetFirstInstruction(bb);
5412
5413 while (next) {
5414 LLVMValueRef inst = next;
5415 next = LLVMGetNextInstruction(next);
5416
5417 if (LLVMGetInstructionOpcode(inst) != LLVMAlloca)
5418 continue;
5419
5420 LLVMTypeRef type = LLVMGetElementType(LLVMTypeOf(inst));
5421 /* No idea why LLVM aligns allocas to 4 elements. */
5422 unsigned alignment = LLVMGetAlignment(inst);
5423 unsigned dw_size = align(llvm_get_type_size(type) / 4, alignment);
5424 ctx->shader->config.private_mem_vgprs += dw_size;
5425 }
5426 bb = LLVMGetNextBasicBlock(bb);
5427 }
5428 }
5429
5430 static void si_init_exec_full_mask(struct si_shader_context *ctx)
5431 {
5432 LLVMValueRef full_mask = LLVMConstInt(ctx->i64, ~0ull, 0);
5433 lp_build_intrinsic(ctx->gallivm.builder,
5434 "llvm.amdgcn.init.exec", ctx->voidt,
5435 &full_mask, 1, LP_FUNC_ATTR_CONVERGENT);
5436 }
5437
5438 static void si_init_exec_from_input(struct si_shader_context *ctx,
5439 unsigned param, unsigned bitoffset)
5440 {
5441 LLVMValueRef args[] = {
5442 LLVMGetParam(ctx->main_fn, param),
5443 LLVMConstInt(ctx->i32, bitoffset, 0),
5444 };
5445 lp_build_intrinsic(ctx->gallivm.builder,
5446 "llvm.amdgcn.init.exec.from.input",
5447 ctx->voidt, args, 2, LP_FUNC_ATTR_CONVERGENT);
5448 }
5449
5450 static bool si_compile_tgsi_main(struct si_shader_context *ctx,
5451 bool is_monolithic)
5452 {
5453 struct si_shader *shader = ctx->shader;
5454 struct si_shader_selector *sel = shader->selector;
5455 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
5456
5457 switch (ctx->type) {
5458 case PIPE_SHADER_VERTEX:
5459 ctx->load_input = declare_input_vs;
5460 if (shader->key.as_ls)
5461 bld_base->emit_epilogue = si_llvm_emit_ls_epilogue;
5462 else if (shader->key.as_es)
5463 bld_base->emit_epilogue = si_llvm_emit_es_epilogue;
5464 else
5465 bld_base->emit_epilogue = si_llvm_emit_vs_epilogue;
5466 break;
5467 case PIPE_SHADER_TESS_CTRL:
5468 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_tcs;
5469 bld_base->emit_fetch_funcs[TGSI_FILE_OUTPUT] = fetch_output_tcs;
5470 bld_base->emit_store = store_output_tcs;
5471 bld_base->emit_epilogue = si_llvm_emit_tcs_epilogue;
5472 break;
5473 case PIPE_SHADER_TESS_EVAL:
5474 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_tes;
5475 if (shader->key.as_es)
5476 bld_base->emit_epilogue = si_llvm_emit_es_epilogue;
5477 else
5478 bld_base->emit_epilogue = si_llvm_emit_vs_epilogue;
5479 break;
5480 case PIPE_SHADER_GEOMETRY:
5481 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_gs;
5482 bld_base->emit_epilogue = si_llvm_emit_gs_epilogue;
5483 break;
5484 case PIPE_SHADER_FRAGMENT:
5485 ctx->load_input = declare_input_fs;
5486 bld_base->emit_epilogue = si_llvm_return_fs_outputs;
5487 break;
5488 case PIPE_SHADER_COMPUTE:
5489 ctx->declare_memory_region = declare_compute_memory;
5490 break;
5491 default:
5492 assert(!"Unsupported shader type");
5493 return false;
5494 }
5495
5496 create_function(ctx);
5497 preload_ring_buffers(ctx);
5498
5499 /* For GFX9 merged shaders:
5500 * - Set EXEC. If the prolog is present, set EXEC there instead.
5501 * - Add a barrier before the second shader.
5502 *
5503 * The same thing for monolithic shaders is done in
5504 * si_build_wrapper_function.
5505 */
5506 if (ctx->screen->b.chip_class >= GFX9 && !is_monolithic) {
5507 if (sel->info.num_instructions > 1 && /* not empty shader */
5508 (shader->key.as_es || shader->key.as_ls) &&
5509 (ctx->type == PIPE_SHADER_TESS_EVAL ||
5510 (ctx->type == PIPE_SHADER_VERTEX &&
5511 !sel->vs_needs_prolog))) {
5512 si_init_exec_from_input(ctx,
5513 ctx->param_merged_wave_info, 0);
5514 } else if (ctx->type == PIPE_SHADER_TESS_CTRL ||
5515 ctx->type == PIPE_SHADER_GEOMETRY) {
5516 si_init_exec_from_input(ctx,
5517 ctx->param_merged_wave_info, 8);
5518 si_llvm_emit_barrier(NULL, bld_base, NULL);
5519 }
5520 }
5521
5522 if (ctx->type == PIPE_SHADER_GEOMETRY) {
5523 int i;
5524 for (i = 0; i < 4; i++) {
5525 ctx->gs_next_vertex[i] =
5526 lp_build_alloca(&ctx->gallivm,
5527 ctx->i32, "");
5528 }
5529 }
5530
5531 if (!lp_build_tgsi_llvm(bld_base, sel->tokens)) {
5532 fprintf(stderr, "Failed to translate shader from TGSI to LLVM\n");
5533 return false;
5534 }
5535
5536 si_llvm_build_ret(ctx, ctx->return_value);
5537 return true;
5538 }
5539
5540 /**
5541 * Compute the VS prolog key, which contains all the information needed to
5542 * build the VS prolog function, and set shader->info bits where needed.
5543 *
5544 * \param info Shader info of the vertex shader.
5545 * \param num_input_sgprs Number of input SGPRs for the vertex shader.
5546 * \param prolog_key Key of the VS prolog
5547 * \param shader_out The vertex shader, or the next shader if merging LS+HS or ES+GS.
5548 * \param key Output shader part key.
5549 */
5550 static void si_get_vs_prolog_key(const struct tgsi_shader_info *info,
5551 unsigned num_input_sgprs,
5552 const struct si_vs_prolog_bits *prolog_key,
5553 struct si_shader *shader_out,
5554 union si_shader_part_key *key)
5555 {
5556 memset(key, 0, sizeof(*key));
5557 key->vs_prolog.states = *prolog_key;
5558 key->vs_prolog.num_input_sgprs = num_input_sgprs;
5559 key->vs_prolog.last_input = MAX2(1, info->num_inputs) - 1;
5560 key->vs_prolog.as_ls = shader_out->key.as_ls;
5561
5562 if (shader_out->selector->type == PIPE_SHADER_TESS_CTRL) {
5563 key->vs_prolog.as_ls = 1;
5564 key->vs_prolog.num_merged_next_stage_vgprs = 2;
5565 } else if (shader_out->selector->type == PIPE_SHADER_GEOMETRY) {
5566 key->vs_prolog.num_merged_next_stage_vgprs = 5;
5567 }
5568
5569 /* Set the instanceID flag. */
5570 for (unsigned i = 0; i < info->num_inputs; i++)
5571 if (key->vs_prolog.states.instance_divisors[i])
5572 shader_out->info.uses_instanceid = true;
5573 }
5574
5575 /**
5576 * Compute the PS prolog key, which contains all the information needed to
5577 * build the PS prolog function, and set related bits in shader->config.
5578 */
5579 static void si_get_ps_prolog_key(struct si_shader *shader,
5580 union si_shader_part_key *key,
5581 bool separate_prolog)
5582 {
5583 struct tgsi_shader_info *info = &shader->selector->info;
5584
5585 memset(key, 0, sizeof(*key));
5586 key->ps_prolog.states = shader->key.part.ps.prolog;
5587 key->ps_prolog.colors_read = info->colors_read;
5588 key->ps_prolog.num_input_sgprs = shader->info.num_input_sgprs;
5589 key->ps_prolog.num_input_vgprs = shader->info.num_input_vgprs;
5590 key->ps_prolog.wqm = info->uses_derivatives &&
5591 (key->ps_prolog.colors_read ||
5592 key->ps_prolog.states.force_persp_sample_interp ||
5593 key->ps_prolog.states.force_linear_sample_interp ||
5594 key->ps_prolog.states.force_persp_center_interp ||
5595 key->ps_prolog.states.force_linear_center_interp ||
5596 key->ps_prolog.states.bc_optimize_for_persp ||
5597 key->ps_prolog.states.bc_optimize_for_linear);
5598
5599 if (info->colors_read) {
5600 unsigned *color = shader->selector->color_attr_index;
5601
5602 if (shader->key.part.ps.prolog.color_two_side) {
5603 /* BCOLORs are stored after the last input. */
5604 key->ps_prolog.num_interp_inputs = info->num_inputs;
5605 key->ps_prolog.face_vgpr_index = shader->info.face_vgpr_index;
5606 shader->config.spi_ps_input_ena |= S_0286CC_FRONT_FACE_ENA(1);
5607 }
5608
5609 for (unsigned i = 0; i < 2; i++) {
5610 unsigned interp = info->input_interpolate[color[i]];
5611 unsigned location = info->input_interpolate_loc[color[i]];
5612
5613 if (!(info->colors_read & (0xf << i*4)))
5614 continue;
5615
5616 key->ps_prolog.color_attr_index[i] = color[i];
5617
5618 if (shader->key.part.ps.prolog.flatshade_colors &&
5619 interp == TGSI_INTERPOLATE_COLOR)
5620 interp = TGSI_INTERPOLATE_CONSTANT;
5621
5622 switch (interp) {
5623 case TGSI_INTERPOLATE_CONSTANT:
5624 key->ps_prolog.color_interp_vgpr_index[i] = -1;
5625 break;
5626 case TGSI_INTERPOLATE_PERSPECTIVE:
5627 case TGSI_INTERPOLATE_COLOR:
5628 /* Force the interpolation location for colors here. */
5629 if (shader->key.part.ps.prolog.force_persp_sample_interp)
5630 location = TGSI_INTERPOLATE_LOC_SAMPLE;
5631 if (shader->key.part.ps.prolog.force_persp_center_interp)
5632 location = TGSI_INTERPOLATE_LOC_CENTER;
5633
5634 switch (location) {
5635 case TGSI_INTERPOLATE_LOC_SAMPLE:
5636 key->ps_prolog.color_interp_vgpr_index[i] = 0;
5637 shader->config.spi_ps_input_ena |=
5638 S_0286CC_PERSP_SAMPLE_ENA(1);
5639 break;
5640 case TGSI_INTERPOLATE_LOC_CENTER:
5641 key->ps_prolog.color_interp_vgpr_index[i] = 2;
5642 shader->config.spi_ps_input_ena |=
5643 S_0286CC_PERSP_CENTER_ENA(1);
5644 break;
5645 case TGSI_INTERPOLATE_LOC_CENTROID:
5646 key->ps_prolog.color_interp_vgpr_index[i] = 4;
5647 shader->config.spi_ps_input_ena |=
5648 S_0286CC_PERSP_CENTROID_ENA(1);
5649 break;
5650 default:
5651 assert(0);
5652 }
5653 break;
5654 case TGSI_INTERPOLATE_LINEAR:
5655 /* Force the interpolation location for colors here. */
5656 if (shader->key.part.ps.prolog.force_linear_sample_interp)
5657 location = TGSI_INTERPOLATE_LOC_SAMPLE;
5658 if (shader->key.part.ps.prolog.force_linear_center_interp)
5659 location = TGSI_INTERPOLATE_LOC_CENTER;
5660
5661 /* The VGPR assignment for non-monolithic shaders
5662 * works because InitialPSInputAddr is set on the
5663 * main shader and PERSP_PULL_MODEL is never used.
5664 */
5665 switch (location) {
5666 case TGSI_INTERPOLATE_LOC_SAMPLE:
5667 key->ps_prolog.color_interp_vgpr_index[i] =
5668 separate_prolog ? 6 : 9;
5669 shader->config.spi_ps_input_ena |=
5670 S_0286CC_LINEAR_SAMPLE_ENA(1);
5671 break;
5672 case TGSI_INTERPOLATE_LOC_CENTER:
5673 key->ps_prolog.color_interp_vgpr_index[i] =
5674 separate_prolog ? 8 : 11;
5675 shader->config.spi_ps_input_ena |=
5676 S_0286CC_LINEAR_CENTER_ENA(1);
5677 break;
5678 case TGSI_INTERPOLATE_LOC_CENTROID:
5679 key->ps_prolog.color_interp_vgpr_index[i] =
5680 separate_prolog ? 10 : 13;
5681 shader->config.spi_ps_input_ena |=
5682 S_0286CC_LINEAR_CENTROID_ENA(1);
5683 break;
5684 default:
5685 assert(0);
5686 }
5687 break;
5688 default:
5689 assert(0);
5690 }
5691 }
5692 }
5693 }
5694
5695 /**
5696 * Check whether a PS prolog is required based on the key.
5697 */
5698 static bool si_need_ps_prolog(const union si_shader_part_key *key)
5699 {
5700 return key->ps_prolog.colors_read ||
5701 key->ps_prolog.states.force_persp_sample_interp ||
5702 key->ps_prolog.states.force_linear_sample_interp ||
5703 key->ps_prolog.states.force_persp_center_interp ||
5704 key->ps_prolog.states.force_linear_center_interp ||
5705 key->ps_prolog.states.bc_optimize_for_persp ||
5706 key->ps_prolog.states.bc_optimize_for_linear ||
5707 key->ps_prolog.states.poly_stipple;
5708 }
5709
5710 /**
5711 * Compute the PS epilog key, which contains all the information needed to
5712 * build the PS epilog function.
5713 */
5714 static void si_get_ps_epilog_key(struct si_shader *shader,
5715 union si_shader_part_key *key)
5716 {
5717 struct tgsi_shader_info *info = &shader->selector->info;
5718 memset(key, 0, sizeof(*key));
5719 key->ps_epilog.colors_written = info->colors_written;
5720 key->ps_epilog.writes_z = info->writes_z;
5721 key->ps_epilog.writes_stencil = info->writes_stencil;
5722 key->ps_epilog.writes_samplemask = info->writes_samplemask;
5723 key->ps_epilog.states = shader->key.part.ps.epilog;
5724 }
5725
5726 /**
5727 * Build the GS prolog function. Rotate the input vertices for triangle strips
5728 * with adjacency.
5729 */
5730 static void si_build_gs_prolog_function(struct si_shader_context *ctx,
5731 union si_shader_part_key *key)
5732 {
5733 unsigned num_sgprs, num_vgprs;
5734 struct gallivm_state *gallivm = &ctx->gallivm;
5735 LLVMBuilderRef builder = gallivm->builder;
5736 LLVMTypeRef params[48]; /* 40 SGPRs (maximum) + some VGPRs */
5737 LLVMTypeRef returns[48];
5738 LLVMValueRef func, ret;
5739
5740 if (ctx->screen->b.chip_class >= GFX9) {
5741 num_sgprs = 8 + GFX9_GS_NUM_USER_SGPR;
5742 num_vgprs = 5; /* ES inputs are not needed by GS */
5743 } else {
5744 num_sgprs = GFX6_GS_NUM_USER_SGPR + 2;
5745 num_vgprs = 8;
5746 }
5747
5748 for (unsigned i = 0; i < num_sgprs; ++i) {
5749 params[i] = ctx->i32;
5750 returns[i] = ctx->i32;
5751 }
5752
5753 for (unsigned i = 0; i < num_vgprs; ++i) {
5754 params[num_sgprs + i] = ctx->i32;
5755 returns[num_sgprs + i] = ctx->f32;
5756 }
5757
5758 /* Create the function. */
5759 si_create_function(ctx, "gs_prolog", returns, num_sgprs + num_vgprs,
5760 params, num_sgprs + num_vgprs, num_sgprs - 1, 0);
5761 func = ctx->main_fn;
5762
5763 /* Set the full EXEC mask for the prolog, because we are only fiddling
5764 * with registers here. The main shader part will set the correct EXEC
5765 * mask.
5766 */
5767 if (ctx->screen->b.chip_class >= GFX9 && !key->gs_prolog.is_monolithic)
5768 si_init_exec_full_mask(ctx);
5769
5770 /* Copy inputs to outputs. This should be no-op, as the registers match,
5771 * but it will prevent the compiler from overwriting them unintentionally.
5772 */
5773 ret = ctx->return_value;
5774 for (unsigned i = 0; i < num_sgprs; i++) {
5775 LLVMValueRef p = LLVMGetParam(func, i);
5776 ret = LLVMBuildInsertValue(builder, ret, p, i, "");
5777 }
5778 for (unsigned i = 0; i < num_vgprs; i++) {
5779 LLVMValueRef p = LLVMGetParam(func, num_sgprs + i);
5780 p = LLVMBuildBitCast(builder, p, ctx->f32, "");
5781 ret = LLVMBuildInsertValue(builder, ret, p, num_sgprs + i, "");
5782 }
5783
5784 if (key->gs_prolog.states.tri_strip_adj_fix) {
5785 /* Remap the input vertices for every other primitive. */
5786 const unsigned gfx6_vtx_params[6] = {
5787 num_sgprs,
5788 num_sgprs + 1,
5789 num_sgprs + 3,
5790 num_sgprs + 4,
5791 num_sgprs + 5,
5792 num_sgprs + 6
5793 };
5794 const unsigned gfx9_vtx_params[3] = {
5795 num_sgprs,
5796 num_sgprs + 1,
5797 num_sgprs + 4,
5798 };
5799 LLVMValueRef vtx_in[6], vtx_out[6];
5800 LLVMValueRef prim_id, rotate;
5801
5802 if (ctx->screen->b.chip_class >= GFX9) {
5803 for (unsigned i = 0; i < 3; i++) {
5804 vtx_in[i*2] = unpack_param(ctx, gfx9_vtx_params[i], 0, 16);
5805 vtx_in[i*2+1] = unpack_param(ctx, gfx9_vtx_params[i], 16, 16);
5806 }
5807 } else {
5808 for (unsigned i = 0; i < 6; i++)
5809 vtx_in[i] = LLVMGetParam(func, gfx6_vtx_params[i]);
5810 }
5811
5812 prim_id = LLVMGetParam(func, num_sgprs + 2);
5813 rotate = LLVMBuildTrunc(builder, prim_id, ctx->i1, "");
5814
5815 for (unsigned i = 0; i < 6; ++i) {
5816 LLVMValueRef base, rotated;
5817 base = vtx_in[i];
5818 rotated = vtx_in[(i + 4) % 6];
5819 vtx_out[i] = LLVMBuildSelect(builder, rotate, rotated, base, "");
5820 }
5821
5822 if (ctx->screen->b.chip_class >= GFX9) {
5823 for (unsigned i = 0; i < 3; i++) {
5824 LLVMValueRef hi, out;
5825
5826 hi = LLVMBuildShl(builder, vtx_out[i*2+1],
5827 LLVMConstInt(ctx->i32, 16, 0), "");
5828 out = LLVMBuildOr(builder, vtx_out[i*2], hi, "");
5829 out = LLVMBuildBitCast(builder, out, ctx->f32, "");
5830 ret = LLVMBuildInsertValue(builder, ret, out,
5831 gfx9_vtx_params[i], "");
5832 }
5833 } else {
5834 for (unsigned i = 0; i < 6; i++) {
5835 LLVMValueRef out;
5836
5837 out = LLVMBuildBitCast(builder, vtx_out[i], ctx->f32, "");
5838 ret = LLVMBuildInsertValue(builder, ret, out,
5839 gfx6_vtx_params[i], "");
5840 }
5841 }
5842 }
5843
5844 LLVMBuildRet(builder, ret);
5845 }
5846
5847 /**
5848 * Given a list of shader part functions, build a wrapper function that
5849 * runs them in sequence to form a monolithic shader.
5850 */
5851 static void si_build_wrapper_function(struct si_shader_context *ctx,
5852 LLVMValueRef *parts,
5853 unsigned num_parts,
5854 unsigned main_part,
5855 unsigned next_shader_first_part)
5856 {
5857 struct gallivm_state *gallivm = &ctx->gallivm;
5858 LLVMBuilderRef builder = ctx->gallivm.builder;
5859 /* PS epilog has one arg per color component */
5860 LLVMTypeRef param_types[48];
5861 LLVMValueRef initial[48], out[48];
5862 LLVMTypeRef function_type;
5863 unsigned num_params;
5864 unsigned num_out, initial_num_out;
5865 MAYBE_UNUSED unsigned num_out_sgpr; /* used in debug checks */
5866 MAYBE_UNUSED unsigned initial_num_out_sgpr; /* used in debug checks */
5867 unsigned num_sgprs, num_vgprs;
5868 unsigned last_sgpr_param;
5869 unsigned gprs;
5870 struct lp_build_if_state if_state;
5871
5872 for (unsigned i = 0; i < num_parts; ++i) {
5873 lp_add_function_attr(parts[i], -1, LP_FUNC_ATTR_ALWAYSINLINE);
5874 LLVMSetLinkage(parts[i], LLVMPrivateLinkage);
5875 }
5876
5877 /* The parameters of the wrapper function correspond to those of the
5878 * first part in terms of SGPRs and VGPRs, but we use the types of the
5879 * main part to get the right types. This is relevant for the
5880 * dereferenceable attribute on descriptor table pointers.
5881 */
5882 num_sgprs = 0;
5883 num_vgprs = 0;
5884
5885 function_type = LLVMGetElementType(LLVMTypeOf(parts[0]));
5886 num_params = LLVMCountParamTypes(function_type);
5887
5888 for (unsigned i = 0; i < num_params; ++i) {
5889 LLVMValueRef param = LLVMGetParam(parts[0], i);
5890
5891 if (ac_is_sgpr_param(param)) {
5892 assert(num_vgprs == 0);
5893 num_sgprs += llvm_get_type_size(LLVMTypeOf(param)) / 4;
5894 } else {
5895 num_vgprs += llvm_get_type_size(LLVMTypeOf(param)) / 4;
5896 }
5897 }
5898 assert(num_vgprs + num_sgprs <= ARRAY_SIZE(param_types));
5899
5900 num_params = 0;
5901 last_sgpr_param = 0;
5902 gprs = 0;
5903 while (gprs < num_sgprs + num_vgprs) {
5904 LLVMValueRef param = LLVMGetParam(parts[main_part], num_params);
5905 unsigned size;
5906
5907 param_types[num_params] = LLVMTypeOf(param);
5908 if (gprs < num_sgprs)
5909 last_sgpr_param = num_params;
5910 size = llvm_get_type_size(param_types[num_params]) / 4;
5911 num_params++;
5912
5913 assert(ac_is_sgpr_param(param) == (gprs < num_sgprs));
5914 assert(gprs + size <= num_sgprs + num_vgprs &&
5915 (gprs >= num_sgprs || gprs + size <= num_sgprs));
5916
5917 gprs += size;
5918 }
5919
5920 si_create_function(ctx, "wrapper", NULL, 0, param_types, num_params,
5921 last_sgpr_param,
5922 si_get_max_workgroup_size(ctx->shader));
5923
5924 if (is_merged_shader(ctx->shader))
5925 si_init_exec_full_mask(ctx);
5926
5927 /* Record the arguments of the function as if they were an output of
5928 * a previous part.
5929 */
5930 num_out = 0;
5931 num_out_sgpr = 0;
5932
5933 for (unsigned i = 0; i < num_params; ++i) {
5934 LLVMValueRef param = LLVMGetParam(ctx->main_fn, i);
5935 LLVMTypeRef param_type = LLVMTypeOf(param);
5936 LLVMTypeRef out_type = i <= last_sgpr_param ? ctx->i32 : ctx->f32;
5937 unsigned size = llvm_get_type_size(param_type) / 4;
5938
5939 if (size == 1) {
5940 if (param_type != out_type)
5941 param = LLVMBuildBitCast(builder, param, out_type, "");
5942 out[num_out++] = param;
5943 } else {
5944 LLVMTypeRef vector_type = LLVMVectorType(out_type, size);
5945
5946 if (LLVMGetTypeKind(param_type) == LLVMPointerTypeKind) {
5947 param = LLVMBuildPtrToInt(builder, param, ctx->i64, "");
5948 param_type = ctx->i64;
5949 }
5950
5951 if (param_type != vector_type)
5952 param = LLVMBuildBitCast(builder, param, vector_type, "");
5953
5954 for (unsigned j = 0; j < size; ++j)
5955 out[num_out++] = LLVMBuildExtractElement(
5956 builder, param, LLVMConstInt(ctx->i32, j, 0), "");
5957 }
5958
5959 if (i <= last_sgpr_param)
5960 num_out_sgpr = num_out;
5961 }
5962
5963 memcpy(initial, out, sizeof(out));
5964 initial_num_out = num_out;
5965 initial_num_out_sgpr = num_out_sgpr;
5966
5967 /* Now chain the parts. */
5968 for (unsigned part = 0; part < num_parts; ++part) {
5969 LLVMValueRef in[48];
5970 LLVMValueRef ret;
5971 LLVMTypeRef ret_type;
5972 unsigned out_idx = 0;
5973
5974 num_params = LLVMCountParams(parts[part]);
5975 assert(num_params <= ARRAY_SIZE(param_types));
5976
5977 /* Merged shaders are executed conditionally depending
5978 * on the number of enabled threads passed in the input SGPRs. */
5979 if (is_merged_shader(ctx->shader) &&
5980 (part == 0 || part == next_shader_first_part)) {
5981 LLVMValueRef ena, count = initial[3];
5982
5983 /* The thread count for the 2nd shader is at bit-offset 8. */
5984 if (part == next_shader_first_part) {
5985 count = LLVMBuildLShr(builder, count,
5986 LLVMConstInt(ctx->i32, 8, 0), "");
5987 }
5988 count = LLVMBuildAnd(builder, count,
5989 LLVMConstInt(ctx->i32, 0x7f, 0), "");
5990 ena = LLVMBuildICmp(builder, LLVMIntULT,
5991 ac_get_thread_id(&ctx->ac), count, "");
5992 lp_build_if(&if_state, &ctx->gallivm, ena);
5993 }
5994
5995 /* Derive arguments for the next part from outputs of the
5996 * previous one.
5997 */
5998 for (unsigned param_idx = 0; param_idx < num_params; ++param_idx) {
5999 LLVMValueRef param;
6000 LLVMTypeRef param_type;
6001 bool is_sgpr;
6002 unsigned param_size;
6003 LLVMValueRef arg = NULL;
6004
6005 param = LLVMGetParam(parts[part], param_idx);
6006 param_type = LLVMTypeOf(param);
6007 param_size = llvm_get_type_size(param_type) / 4;
6008 is_sgpr = ac_is_sgpr_param(param);
6009
6010 if (is_sgpr) {
6011 #if HAVE_LLVM < 0x0400
6012 LLVMRemoveAttribute(param, LLVMByValAttribute);
6013 #else
6014 unsigned kind_id = LLVMGetEnumAttributeKindForName("byval", 5);
6015 LLVMRemoveEnumAttributeAtIndex(parts[part], param_idx + 1, kind_id);
6016 #endif
6017 lp_add_function_attr(parts[part], param_idx + 1, LP_FUNC_ATTR_INREG);
6018 }
6019
6020 assert(out_idx + param_size <= (is_sgpr ? num_out_sgpr : num_out));
6021 assert(is_sgpr || out_idx >= num_out_sgpr);
6022
6023 if (param_size == 1)
6024 arg = out[out_idx];
6025 else
6026 arg = lp_build_gather_values(gallivm, &out[out_idx], param_size);
6027
6028 if (LLVMTypeOf(arg) != param_type) {
6029 if (LLVMGetTypeKind(param_type) == LLVMPointerTypeKind) {
6030 arg = LLVMBuildBitCast(builder, arg, ctx->i64, "");
6031 arg = LLVMBuildIntToPtr(builder, arg, param_type, "");
6032 } else {
6033 arg = LLVMBuildBitCast(builder, arg, param_type, "");
6034 }
6035 }
6036
6037 in[param_idx] = arg;
6038 out_idx += param_size;
6039 }
6040
6041 ret = LLVMBuildCall(builder, parts[part], in, num_params, "");
6042
6043 if (is_merged_shader(ctx->shader) &&
6044 (part + 1 == next_shader_first_part ||
6045 part + 1 == num_parts)) {
6046 lp_build_endif(&if_state);
6047
6048 if (part + 1 == next_shader_first_part) {
6049 /* A barrier is required between 2 merged shaders. */
6050 si_llvm_emit_barrier(NULL, &ctx->bld_base, NULL);
6051
6052 /* The second half of the merged shader should use
6053 * the inputs from the toplevel (wrapper) function,
6054 * not the return value from the last call.
6055 *
6056 * That's because the last call was executed condi-
6057 * tionally, so we can't consume it in the main
6058 * block.
6059 */
6060 memcpy(out, initial, sizeof(initial));
6061 num_out = initial_num_out;
6062 num_out_sgpr = initial_num_out_sgpr;
6063 }
6064 continue;
6065 }
6066
6067 /* Extract the returned GPRs. */
6068 ret_type = LLVMTypeOf(ret);
6069 num_out = 0;
6070 num_out_sgpr = 0;
6071
6072 if (LLVMGetTypeKind(ret_type) != LLVMVoidTypeKind) {
6073 assert(LLVMGetTypeKind(ret_type) == LLVMStructTypeKind);
6074
6075 unsigned ret_size = LLVMCountStructElementTypes(ret_type);
6076
6077 for (unsigned i = 0; i < ret_size; ++i) {
6078 LLVMValueRef val =
6079 LLVMBuildExtractValue(builder, ret, i, "");
6080
6081 out[num_out++] = val;
6082
6083 if (LLVMTypeOf(val) == ctx->i32) {
6084 assert(num_out_sgpr + 1 == num_out);
6085 num_out_sgpr = num_out;
6086 }
6087 }
6088 }
6089 }
6090
6091 LLVMBuildRetVoid(builder);
6092 }
6093
6094 int si_compile_tgsi_shader(struct si_screen *sscreen,
6095 LLVMTargetMachineRef tm,
6096 struct si_shader *shader,
6097 bool is_monolithic,
6098 struct pipe_debug_callback *debug)
6099 {
6100 struct si_shader_selector *sel = shader->selector;
6101 struct si_shader_context ctx;
6102 int r = -1;
6103
6104 /* Dump TGSI code before doing TGSI->LLVM conversion in case the
6105 * conversion fails. */
6106 if (r600_can_dump_shader(&sscreen->b, sel->info.processor) &&
6107 !(sscreen->b.debug_flags & DBG_NO_TGSI)) {
6108 tgsi_dump(sel->tokens, 0);
6109 si_dump_streamout(&sel->so);
6110 }
6111
6112 si_init_shader_ctx(&ctx, sscreen, tm);
6113 si_llvm_context_set_tgsi(&ctx, shader);
6114 ctx.separate_prolog = !is_monolithic;
6115
6116 memset(shader->info.vs_output_param_offset, AC_EXP_PARAM_UNDEFINED,
6117 sizeof(shader->info.vs_output_param_offset));
6118
6119 shader->info.uses_instanceid = sel->info.uses_instanceid;
6120
6121 ctx.load_system_value = declare_system_value;
6122
6123 if (!si_compile_tgsi_main(&ctx, is_monolithic)) {
6124 si_llvm_dispose(&ctx);
6125 return -1;
6126 }
6127
6128 if (is_monolithic && ctx.type == PIPE_SHADER_VERTEX) {
6129 LLVMValueRef parts[2];
6130 bool need_prolog = sel->vs_needs_prolog;
6131
6132 parts[1] = ctx.main_fn;
6133
6134 if (need_prolog) {
6135 union si_shader_part_key prolog_key;
6136 si_get_vs_prolog_key(&sel->info,
6137 shader->info.num_input_sgprs,
6138 &shader->key.part.vs.prolog,
6139 shader, &prolog_key);
6140 si_build_vs_prolog_function(&ctx, &prolog_key);
6141 parts[0] = ctx.main_fn;
6142 }
6143
6144 si_build_wrapper_function(&ctx, parts + !need_prolog,
6145 1 + need_prolog, need_prolog, 0);
6146 } else if (is_monolithic && ctx.type == PIPE_SHADER_TESS_CTRL) {
6147 if (sscreen->b.chip_class >= GFX9) {
6148 struct si_shader_selector *ls = shader->key.part.tcs.ls;
6149 LLVMValueRef parts[4];
6150
6151 /* TCS main part */
6152 parts[2] = ctx.main_fn;
6153
6154 /* TCS epilog */
6155 union si_shader_part_key tcs_epilog_key;
6156 memset(&tcs_epilog_key, 0, sizeof(tcs_epilog_key));
6157 tcs_epilog_key.tcs_epilog.states = shader->key.part.tcs.epilog;
6158 si_build_tcs_epilog_function(&ctx, &tcs_epilog_key);
6159 parts[3] = ctx.main_fn;
6160
6161 /* VS prolog */
6162 if (ls->vs_needs_prolog) {
6163 union si_shader_part_key vs_prolog_key;
6164 si_get_vs_prolog_key(&ls->info,
6165 shader->info.num_input_sgprs,
6166 &shader->key.part.tcs.ls_prolog,
6167 shader, &vs_prolog_key);
6168 vs_prolog_key.vs_prolog.is_monolithic = true;
6169 si_build_vs_prolog_function(&ctx, &vs_prolog_key);
6170 parts[0] = ctx.main_fn;
6171 }
6172
6173 /* VS as LS main part */
6174 struct si_shader shader_ls = {};
6175 shader_ls.selector = ls;
6176 shader_ls.key.as_ls = 1;
6177 shader_ls.key.mono = shader->key.mono;
6178 shader_ls.key.opt = shader->key.opt;
6179 si_llvm_context_set_tgsi(&ctx, &shader_ls);
6180
6181 if (!si_compile_tgsi_main(&ctx, true)) {
6182 si_llvm_dispose(&ctx);
6183 return -1;
6184 }
6185 shader->info.uses_instanceid |= ls->info.uses_instanceid;
6186 parts[1] = ctx.main_fn;
6187
6188 /* Reset the shader context. */
6189 ctx.shader = shader;
6190 ctx.type = PIPE_SHADER_TESS_CTRL;
6191
6192 si_build_wrapper_function(&ctx,
6193 parts + !ls->vs_needs_prolog,
6194 4 - !ls->vs_needs_prolog, 0,
6195 ls->vs_needs_prolog ? 2 : 1);
6196 } else {
6197 LLVMValueRef parts[2];
6198 union si_shader_part_key epilog_key;
6199
6200 parts[0] = ctx.main_fn;
6201
6202 memset(&epilog_key, 0, sizeof(epilog_key));
6203 epilog_key.tcs_epilog.states = shader->key.part.tcs.epilog;
6204 si_build_tcs_epilog_function(&ctx, &epilog_key);
6205 parts[1] = ctx.main_fn;
6206
6207 si_build_wrapper_function(&ctx, parts, 2, 0, 0);
6208 }
6209 } else if (is_monolithic && ctx.type == PIPE_SHADER_GEOMETRY) {
6210 if (ctx.screen->b.chip_class >= GFX9) {
6211 struct si_shader_selector *es = shader->key.part.gs.es;
6212 LLVMValueRef es_prolog = NULL;
6213 LLVMValueRef es_main = NULL;
6214 LLVMValueRef gs_prolog = NULL;
6215 LLVMValueRef gs_main = ctx.main_fn;
6216
6217 /* GS prolog */
6218 union si_shader_part_key gs_prolog_key;
6219 memset(&gs_prolog_key, 0, sizeof(gs_prolog_key));
6220 gs_prolog_key.gs_prolog.states = shader->key.part.gs.prolog;
6221 gs_prolog_key.gs_prolog.is_monolithic = true;
6222 si_build_gs_prolog_function(&ctx, &gs_prolog_key);
6223 gs_prolog = ctx.main_fn;
6224
6225 /* ES prolog */
6226 if (es->vs_needs_prolog) {
6227 union si_shader_part_key vs_prolog_key;
6228 si_get_vs_prolog_key(&es->info,
6229 shader->info.num_input_sgprs,
6230 &shader->key.part.tcs.ls_prolog,
6231 shader, &vs_prolog_key);
6232 vs_prolog_key.vs_prolog.is_monolithic = true;
6233 si_build_vs_prolog_function(&ctx, &vs_prolog_key);
6234 es_prolog = ctx.main_fn;
6235 }
6236
6237 /* ES main part */
6238 struct si_shader shader_es = {};
6239 shader_es.selector = es;
6240 shader_es.key.as_es = 1;
6241 shader_es.key.mono = shader->key.mono;
6242 shader_es.key.opt = shader->key.opt;
6243 si_llvm_context_set_tgsi(&ctx, &shader_es);
6244
6245 if (!si_compile_tgsi_main(&ctx, true)) {
6246 si_llvm_dispose(&ctx);
6247 return -1;
6248 }
6249 shader->info.uses_instanceid |= es->info.uses_instanceid;
6250 es_main = ctx.main_fn;
6251
6252 /* Reset the shader context. */
6253 ctx.shader = shader;
6254 ctx.type = PIPE_SHADER_GEOMETRY;
6255
6256 /* Prepare the array of shader parts. */
6257 LLVMValueRef parts[4];
6258 unsigned num_parts = 0, main_part, next_first_part;
6259
6260 if (es_prolog)
6261 parts[num_parts++] = es_prolog;
6262
6263 parts[main_part = num_parts++] = es_main;
6264 parts[next_first_part = num_parts++] = gs_prolog;
6265 parts[num_parts++] = gs_main;
6266
6267 si_build_wrapper_function(&ctx, parts, num_parts,
6268 main_part, next_first_part);
6269 } else {
6270 LLVMValueRef parts[2];
6271 union si_shader_part_key prolog_key;
6272
6273 parts[1] = ctx.main_fn;
6274
6275 memset(&prolog_key, 0, sizeof(prolog_key));
6276 prolog_key.gs_prolog.states = shader->key.part.gs.prolog;
6277 si_build_gs_prolog_function(&ctx, &prolog_key);
6278 parts[0] = ctx.main_fn;
6279
6280 si_build_wrapper_function(&ctx, parts, 2, 1, 0);
6281 }
6282 } else if (is_monolithic && ctx.type == PIPE_SHADER_FRAGMENT) {
6283 LLVMValueRef parts[3];
6284 union si_shader_part_key prolog_key;
6285 union si_shader_part_key epilog_key;
6286 bool need_prolog;
6287
6288 si_get_ps_prolog_key(shader, &prolog_key, false);
6289 need_prolog = si_need_ps_prolog(&prolog_key);
6290
6291 parts[need_prolog ? 1 : 0] = ctx.main_fn;
6292
6293 if (need_prolog) {
6294 si_build_ps_prolog_function(&ctx, &prolog_key);
6295 parts[0] = ctx.main_fn;
6296 }
6297
6298 si_get_ps_epilog_key(shader, &epilog_key);
6299 si_build_ps_epilog_function(&ctx, &epilog_key);
6300 parts[need_prolog ? 2 : 1] = ctx.main_fn;
6301
6302 si_build_wrapper_function(&ctx, parts, need_prolog ? 3 : 2,
6303 need_prolog ? 1 : 0, 0);
6304 }
6305
6306 si_llvm_optimize_module(&ctx);
6307
6308 /* Post-optimization transformations and analysis. */
6309 si_optimize_vs_outputs(&ctx);
6310
6311 if ((debug && debug->debug_message) ||
6312 r600_can_dump_shader(&sscreen->b, ctx.type))
6313 si_count_scratch_private_memory(&ctx);
6314
6315 /* Compile to bytecode. */
6316 r = si_compile_llvm(sscreen, &shader->binary, &shader->config, tm,
6317 ctx.gallivm.module, debug, ctx.type, "TGSI shader");
6318 si_llvm_dispose(&ctx);
6319 if (r) {
6320 fprintf(stderr, "LLVM failed to compile shader\n");
6321 return r;
6322 }
6323
6324 /* Validate SGPR and VGPR usage for compute to detect compiler bugs.
6325 * LLVM 3.9svn has this bug.
6326 */
6327 if (sel->type == PIPE_SHADER_COMPUTE) {
6328 unsigned wave_size = 64;
6329 unsigned max_vgprs = 256;
6330 unsigned max_sgprs = sscreen->b.chip_class >= VI ? 800 : 512;
6331 unsigned max_sgprs_per_wave = 128;
6332 unsigned max_block_threads = si_get_max_workgroup_size(shader);
6333 unsigned min_waves_per_cu = DIV_ROUND_UP(max_block_threads, wave_size);
6334 unsigned min_waves_per_simd = DIV_ROUND_UP(min_waves_per_cu, 4);
6335
6336 max_vgprs = max_vgprs / min_waves_per_simd;
6337 max_sgprs = MIN2(max_sgprs / min_waves_per_simd, max_sgprs_per_wave);
6338
6339 if (shader->config.num_sgprs > max_sgprs ||
6340 shader->config.num_vgprs > max_vgprs) {
6341 fprintf(stderr, "LLVM failed to compile a shader correctly: "
6342 "SGPR:VGPR usage is %u:%u, but the hw limit is %u:%u\n",
6343 shader->config.num_sgprs, shader->config.num_vgprs,
6344 max_sgprs, max_vgprs);
6345
6346 /* Just terminate the process, because dependent
6347 * shaders can hang due to bad input data, but use
6348 * the env var to allow shader-db to work.
6349 */
6350 if (!debug_get_bool_option("SI_PASS_BAD_SHADERS", false))
6351 abort();
6352 }
6353 }
6354
6355 /* Add the scratch offset to input SGPRs. */
6356 if (shader->config.scratch_bytes_per_wave && !is_merged_shader(shader))
6357 shader->info.num_input_sgprs += 1; /* scratch byte offset */
6358
6359 /* Calculate the number of fragment input VGPRs. */
6360 if (ctx.type == PIPE_SHADER_FRAGMENT) {
6361 shader->info.num_input_vgprs = 0;
6362 shader->info.face_vgpr_index = -1;
6363
6364 if (G_0286CC_PERSP_SAMPLE_ENA(shader->config.spi_ps_input_addr))
6365 shader->info.num_input_vgprs += 2;
6366 if (G_0286CC_PERSP_CENTER_ENA(shader->config.spi_ps_input_addr))
6367 shader->info.num_input_vgprs += 2;
6368 if (G_0286CC_PERSP_CENTROID_ENA(shader->config.spi_ps_input_addr))
6369 shader->info.num_input_vgprs += 2;
6370 if (G_0286CC_PERSP_PULL_MODEL_ENA(shader->config.spi_ps_input_addr))
6371 shader->info.num_input_vgprs += 3;
6372 if (G_0286CC_LINEAR_SAMPLE_ENA(shader->config.spi_ps_input_addr))
6373 shader->info.num_input_vgprs += 2;
6374 if (G_0286CC_LINEAR_CENTER_ENA(shader->config.spi_ps_input_addr))
6375 shader->info.num_input_vgprs += 2;
6376 if (G_0286CC_LINEAR_CENTROID_ENA(shader->config.spi_ps_input_addr))
6377 shader->info.num_input_vgprs += 2;
6378 if (G_0286CC_LINE_STIPPLE_TEX_ENA(shader->config.spi_ps_input_addr))
6379 shader->info.num_input_vgprs += 1;
6380 if (G_0286CC_POS_X_FLOAT_ENA(shader->config.spi_ps_input_addr))
6381 shader->info.num_input_vgprs += 1;
6382 if (G_0286CC_POS_Y_FLOAT_ENA(shader->config.spi_ps_input_addr))
6383 shader->info.num_input_vgprs += 1;
6384 if (G_0286CC_POS_Z_FLOAT_ENA(shader->config.spi_ps_input_addr))
6385 shader->info.num_input_vgprs += 1;
6386 if (G_0286CC_POS_W_FLOAT_ENA(shader->config.spi_ps_input_addr))
6387 shader->info.num_input_vgprs += 1;
6388 if (G_0286CC_FRONT_FACE_ENA(shader->config.spi_ps_input_addr)) {
6389 shader->info.face_vgpr_index = shader->info.num_input_vgprs;
6390 shader->info.num_input_vgprs += 1;
6391 }
6392 if (G_0286CC_ANCILLARY_ENA(shader->config.spi_ps_input_addr))
6393 shader->info.num_input_vgprs += 1;
6394 if (G_0286CC_SAMPLE_COVERAGE_ENA(shader->config.spi_ps_input_addr))
6395 shader->info.num_input_vgprs += 1;
6396 if (G_0286CC_POS_FIXED_PT_ENA(shader->config.spi_ps_input_addr))
6397 shader->info.num_input_vgprs += 1;
6398 }
6399
6400 return 0;
6401 }
6402
6403 /**
6404 * Create, compile and return a shader part (prolog or epilog).
6405 *
6406 * \param sscreen screen
6407 * \param list list of shader parts of the same category
6408 * \param type shader type
6409 * \param key shader part key
6410 * \param prolog whether the part being requested is a prolog
6411 * \param tm LLVM target machine
6412 * \param debug debug callback
6413 * \param build the callback responsible for building the main function
6414 * \return non-NULL on success
6415 */
6416 static struct si_shader_part *
6417 si_get_shader_part(struct si_screen *sscreen,
6418 struct si_shader_part **list,
6419 enum pipe_shader_type type,
6420 bool prolog,
6421 union si_shader_part_key *key,
6422 LLVMTargetMachineRef tm,
6423 struct pipe_debug_callback *debug,
6424 void (*build)(struct si_shader_context *,
6425 union si_shader_part_key *),
6426 const char *name)
6427 {
6428 struct si_shader_part *result;
6429
6430 mtx_lock(&sscreen->shader_parts_mutex);
6431
6432 /* Find existing. */
6433 for (result = *list; result; result = result->next) {
6434 if (memcmp(&result->key, key, sizeof(*key)) == 0) {
6435 mtx_unlock(&sscreen->shader_parts_mutex);
6436 return result;
6437 }
6438 }
6439
6440 /* Compile a new one. */
6441 result = CALLOC_STRUCT(si_shader_part);
6442 result->key = *key;
6443
6444 struct si_shader shader = {};
6445 struct si_shader_context ctx;
6446 struct gallivm_state *gallivm = &ctx.gallivm;
6447
6448 si_init_shader_ctx(&ctx, sscreen, tm);
6449 ctx.shader = &shader;
6450 ctx.type = type;
6451
6452 switch (type) {
6453 case PIPE_SHADER_VERTEX:
6454 break;
6455 case PIPE_SHADER_TESS_CTRL:
6456 assert(!prolog);
6457 shader.key.part.tcs.epilog = key->tcs_epilog.states;
6458 break;
6459 case PIPE_SHADER_GEOMETRY:
6460 assert(prolog);
6461 break;
6462 case PIPE_SHADER_FRAGMENT:
6463 if (prolog)
6464 shader.key.part.ps.prolog = key->ps_prolog.states;
6465 else
6466 shader.key.part.ps.epilog = key->ps_epilog.states;
6467 break;
6468 default:
6469 unreachable("bad shader part");
6470 }
6471
6472 build(&ctx, key);
6473
6474 /* Compile. */
6475 si_llvm_optimize_module(&ctx);
6476
6477 if (si_compile_llvm(sscreen, &result->binary, &result->config, tm,
6478 gallivm->module, debug, ctx.type, name)) {
6479 FREE(result);
6480 result = NULL;
6481 goto out;
6482 }
6483
6484 result->next = *list;
6485 *list = result;
6486
6487 out:
6488 si_llvm_dispose(&ctx);
6489 mtx_unlock(&sscreen->shader_parts_mutex);
6490 return result;
6491 }
6492
6493 /**
6494 * Build the vertex shader prolog function.
6495 *
6496 * The inputs are the same as VS (a lot of SGPRs and 4 VGPR system values).
6497 * All inputs are returned unmodified. The vertex load indices are
6498 * stored after them, which will be used by the API VS for fetching inputs.
6499 *
6500 * For example, the expected outputs for instance_divisors[] = {0, 1, 2} are:
6501 * input_v0,
6502 * input_v1,
6503 * input_v2,
6504 * input_v3,
6505 * (VertexID + BaseVertex),
6506 * (InstanceID + StartInstance),
6507 * (InstanceID / 2 + StartInstance)
6508 */
6509 static void si_build_vs_prolog_function(struct si_shader_context *ctx,
6510 union si_shader_part_key *key)
6511 {
6512 struct gallivm_state *gallivm = &ctx->gallivm;
6513 LLVMTypeRef *params, *returns;
6514 LLVMValueRef ret, func;
6515 int last_sgpr, num_params, num_returns, i;
6516 unsigned first_vs_vgpr = key->vs_prolog.num_input_sgprs +
6517 key->vs_prolog.num_merged_next_stage_vgprs;
6518 unsigned num_input_vgprs = key->vs_prolog.num_merged_next_stage_vgprs + 4;
6519 unsigned num_all_input_regs = key->vs_prolog.num_input_sgprs +
6520 num_input_vgprs;
6521 unsigned user_sgpr_base = key->vs_prolog.num_merged_next_stage_vgprs ? 8 : 0;
6522
6523 ctx->param_vertex_id = first_vs_vgpr;
6524 ctx->param_instance_id = first_vs_vgpr + (key->vs_prolog.as_ls ? 2 : 1);
6525
6526 /* 4 preloaded VGPRs + vertex load indices as prolog outputs */
6527 params = alloca(num_all_input_regs * sizeof(LLVMTypeRef));
6528 returns = alloca((num_all_input_regs + key->vs_prolog.last_input + 1) *
6529 sizeof(LLVMTypeRef));
6530 num_params = 0;
6531 num_returns = 0;
6532
6533 /* Declare input and output SGPRs. */
6534 num_params = 0;
6535 for (i = 0; i < key->vs_prolog.num_input_sgprs; i++) {
6536 params[num_params++] = ctx->i32;
6537 returns[num_returns++] = ctx->i32;
6538 }
6539 last_sgpr = num_params - 1;
6540
6541 /* Preloaded VGPRs (outputs must be floats) */
6542 for (i = 0; i < num_input_vgprs; i++) {
6543 params[num_params++] = ctx->i32;
6544 returns[num_returns++] = ctx->f32;
6545 }
6546
6547 /* Vertex load indices. */
6548 for (i = 0; i <= key->vs_prolog.last_input; i++)
6549 returns[num_returns++] = ctx->f32;
6550
6551 /* Create the function. */
6552 si_create_function(ctx, "vs_prolog", returns, num_returns, params,
6553 num_params, last_sgpr, 0);
6554 func = ctx->main_fn;
6555
6556 if (key->vs_prolog.num_merged_next_stage_vgprs &&
6557 !key->vs_prolog.is_monolithic)
6558 si_init_exec_from_input(ctx, 3, 0);
6559
6560 /* Copy inputs to outputs. This should be no-op, as the registers match,
6561 * but it will prevent the compiler from overwriting them unintentionally.
6562 */
6563 ret = ctx->return_value;
6564 for (i = 0; i < key->vs_prolog.num_input_sgprs; i++) {
6565 LLVMValueRef p = LLVMGetParam(func, i);
6566 ret = LLVMBuildInsertValue(gallivm->builder, ret, p, i, "");
6567 }
6568 for (; i < num_params; i++) {
6569 LLVMValueRef p = LLVMGetParam(func, i);
6570 p = LLVMBuildBitCast(gallivm->builder, p, ctx->f32, "");
6571 ret = LLVMBuildInsertValue(gallivm->builder, ret, p, i, "");
6572 }
6573
6574 /* Compute vertex load indices from instance divisors. */
6575 for (i = 0; i <= key->vs_prolog.last_input; i++) {
6576 unsigned divisor = key->vs_prolog.states.instance_divisors[i];
6577 LLVMValueRef index;
6578
6579 if (divisor) {
6580 /* InstanceID / Divisor + StartInstance */
6581 index = get_instance_index_for_fetch(ctx,
6582 user_sgpr_base +
6583 SI_SGPR_START_INSTANCE,
6584 divisor);
6585 } else {
6586 /* VertexID + BaseVertex */
6587 index = LLVMBuildAdd(gallivm->builder,
6588 LLVMGetParam(func, ctx->param_vertex_id),
6589 LLVMGetParam(func, user_sgpr_base +
6590 SI_SGPR_BASE_VERTEX), "");
6591 }
6592
6593 index = LLVMBuildBitCast(gallivm->builder, index, ctx->f32, "");
6594 ret = LLVMBuildInsertValue(gallivm->builder, ret, index,
6595 num_params++, "");
6596 }
6597
6598 si_llvm_build_ret(ctx, ret);
6599 }
6600
6601 static bool si_get_vs_prolog(struct si_screen *sscreen,
6602 LLVMTargetMachineRef tm,
6603 struct si_shader *shader,
6604 struct pipe_debug_callback *debug,
6605 struct si_shader *main_part,
6606 const struct si_vs_prolog_bits *key)
6607 {
6608 struct si_shader_selector *vs = main_part->selector;
6609
6610 /* The prolog is a no-op if there are no inputs. */
6611 if (!vs->vs_needs_prolog)
6612 return true;
6613
6614 /* Get the prolog. */
6615 union si_shader_part_key prolog_key;
6616 si_get_vs_prolog_key(&vs->info, main_part->info.num_input_sgprs,
6617 key, shader, &prolog_key);
6618
6619 shader->prolog =
6620 si_get_shader_part(sscreen, &sscreen->vs_prologs,
6621 PIPE_SHADER_VERTEX, true, &prolog_key, tm,
6622 debug, si_build_vs_prolog_function,
6623 "Vertex Shader Prolog");
6624 return shader->prolog != NULL;
6625 }
6626
6627 /**
6628 * Select and compile (or reuse) vertex shader parts (prolog & epilog).
6629 */
6630 static bool si_shader_select_vs_parts(struct si_screen *sscreen,
6631 LLVMTargetMachineRef tm,
6632 struct si_shader *shader,
6633 struct pipe_debug_callback *debug)
6634 {
6635 return si_get_vs_prolog(sscreen, tm, shader, debug, shader,
6636 &shader->key.part.vs.prolog);
6637 }
6638
6639 /**
6640 * Compile the TCS epilog function. This writes tesselation factors to memory
6641 * based on the output primitive type of the tesselator (determined by TES).
6642 */
6643 static void si_build_tcs_epilog_function(struct si_shader_context *ctx,
6644 union si_shader_part_key *key)
6645 {
6646 struct gallivm_state *gallivm = &ctx->gallivm;
6647 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
6648 LLVMTypeRef params[32];
6649 LLVMValueRef func;
6650 int last_sgpr, num_params = 0;
6651
6652 if (ctx->screen->b.chip_class >= GFX9) {
6653 params[num_params++] = ctx->i64;
6654 params[ctx->param_tcs_offchip_offset = num_params++] = ctx->i32;
6655 params[num_params++] = ctx->i32; /* wave info */
6656 params[ctx->param_tcs_factor_offset = num_params++] = ctx->i32;
6657 params[num_params++] = ctx->i32;
6658 params[num_params++] = ctx->i32;
6659 params[num_params++] = ctx->i32;
6660 params[num_params++] = ctx->i64;
6661 params[num_params++] = ctx->i64;
6662 params[num_params++] = ctx->i64;
6663 params[num_params++] = ctx->i64;
6664 params[num_params++] = ctx->i32;
6665 params[num_params++] = ctx->i32;
6666 params[num_params++] = ctx->i32;
6667 params[num_params++] = ctx->i32;
6668 params[ctx->param_tcs_offchip_layout = num_params++] = ctx->i32;
6669 params[num_params++] = ctx->i32;
6670 params[num_params++] = ctx->i32;
6671 params[ctx->param_tcs_offchip_addr_base64k = num_params++] = ctx->i32;
6672 params[ctx->param_tcs_factor_addr_base64k = num_params++] = ctx->i32;
6673 } else {
6674 params[num_params++] = ctx->i64;
6675 params[num_params++] = ctx->i64;
6676 params[num_params++] = ctx->i64;
6677 params[ctx->param_tcs_offchip_layout = num_params++] = ctx->i32;
6678 params[num_params++] = ctx->i32;
6679 params[num_params++] = ctx->i32;
6680 params[num_params++] = ctx->i32;
6681 params[ctx->param_tcs_offchip_addr_base64k = num_params++] = ctx->i32;
6682 params[ctx->param_tcs_factor_addr_base64k = num_params++] = ctx->i32;
6683 params[ctx->param_tcs_offchip_offset = num_params++] = ctx->i32;
6684 params[ctx->param_tcs_factor_offset = num_params++] = ctx->i32;
6685 }
6686 last_sgpr = num_params - 1;
6687
6688 params[num_params++] = ctx->i32; /* patch index within the wave (REL_PATCH_ID) */
6689 params[num_params++] = ctx->i32; /* invocation ID within the patch */
6690 params[num_params++] = ctx->i32; /* LDS offset where tess factors should be loaded from */
6691
6692 /* Create the function. */
6693 si_create_function(ctx, "tcs_epilog", NULL, 0, params, num_params, last_sgpr,
6694 ctx->screen->b.chip_class >= CIK ? 128 : 64);
6695 declare_lds_as_pointer(ctx);
6696 func = ctx->main_fn;
6697
6698 si_write_tess_factors(bld_base,
6699 LLVMGetParam(func, last_sgpr + 1),
6700 LLVMGetParam(func, last_sgpr + 2),
6701 LLVMGetParam(func, last_sgpr + 3));
6702
6703 LLVMBuildRetVoid(gallivm->builder);
6704 }
6705
6706 /**
6707 * Select and compile (or reuse) TCS parts (epilog).
6708 */
6709 static bool si_shader_select_tcs_parts(struct si_screen *sscreen,
6710 LLVMTargetMachineRef tm,
6711 struct si_shader *shader,
6712 struct pipe_debug_callback *debug)
6713 {
6714 if (sscreen->b.chip_class >= GFX9) {
6715 struct si_shader *ls_main_part =
6716 shader->key.part.tcs.ls->main_shader_part_ls;
6717
6718 if (!si_get_vs_prolog(sscreen, tm, shader, debug, ls_main_part,
6719 &shader->key.part.tcs.ls_prolog))
6720 return false;
6721
6722 shader->previous_stage = ls_main_part;
6723 }
6724
6725 /* Get the epilog. */
6726 union si_shader_part_key epilog_key;
6727 memset(&epilog_key, 0, sizeof(epilog_key));
6728 epilog_key.tcs_epilog.states = shader->key.part.tcs.epilog;
6729
6730 shader->epilog = si_get_shader_part(sscreen, &sscreen->tcs_epilogs,
6731 PIPE_SHADER_TESS_CTRL, false,
6732 &epilog_key, tm, debug,
6733 si_build_tcs_epilog_function,
6734 "Tessellation Control Shader Epilog");
6735 return shader->epilog != NULL;
6736 }
6737
6738 /**
6739 * Select and compile (or reuse) GS parts (prolog).
6740 */
6741 static bool si_shader_select_gs_parts(struct si_screen *sscreen,
6742 LLVMTargetMachineRef tm,
6743 struct si_shader *shader,
6744 struct pipe_debug_callback *debug)
6745 {
6746 if (sscreen->b.chip_class >= GFX9) {
6747 struct si_shader *es_main_part =
6748 shader->key.part.gs.es->main_shader_part_es;
6749
6750 if (shader->key.part.gs.es->type == PIPE_SHADER_VERTEX &&
6751 !si_get_vs_prolog(sscreen, tm, shader, debug, es_main_part,
6752 &shader->key.part.gs.vs_prolog))
6753 return false;
6754
6755 shader->previous_stage = es_main_part;
6756 }
6757
6758 if (!shader->key.part.gs.prolog.tri_strip_adj_fix)
6759 return true;
6760
6761 union si_shader_part_key prolog_key;
6762 memset(&prolog_key, 0, sizeof(prolog_key));
6763 prolog_key.gs_prolog.states = shader->key.part.gs.prolog;
6764
6765 shader->prolog2 = si_get_shader_part(sscreen, &sscreen->gs_prologs,
6766 PIPE_SHADER_GEOMETRY, true,
6767 &prolog_key, tm, debug,
6768 si_build_gs_prolog_function,
6769 "Geometry Shader Prolog");
6770 return shader->prolog2 != NULL;
6771 }
6772
6773 /**
6774 * Build the pixel shader prolog function. This handles:
6775 * - two-side color selection and interpolation
6776 * - overriding interpolation parameters for the API PS
6777 * - polygon stippling
6778 *
6779 * All preloaded SGPRs and VGPRs are passed through unmodified unless they are
6780 * overriden by other states. (e.g. per-sample interpolation)
6781 * Interpolated colors are stored after the preloaded VGPRs.
6782 */
6783 static void si_build_ps_prolog_function(struct si_shader_context *ctx,
6784 union si_shader_part_key *key)
6785 {
6786 struct gallivm_state *gallivm = &ctx->gallivm;
6787 LLVMTypeRef *params;
6788 LLVMValueRef ret, func;
6789 int last_sgpr, num_params, num_returns, i, num_color_channels;
6790
6791 assert(si_need_ps_prolog(key));
6792
6793 /* Number of inputs + 8 color elements. */
6794 params = alloca((key->ps_prolog.num_input_sgprs +
6795 key->ps_prolog.num_input_vgprs + 8) *
6796 sizeof(LLVMTypeRef));
6797
6798 /* Declare inputs. */
6799 num_params = 0;
6800 for (i = 0; i < key->ps_prolog.num_input_sgprs; i++)
6801 params[num_params++] = ctx->i32;
6802 last_sgpr = num_params - 1;
6803
6804 for (i = 0; i < key->ps_prolog.num_input_vgprs; i++)
6805 params[num_params++] = ctx->f32;
6806
6807 /* Declare outputs (same as inputs + add colors if needed) */
6808 num_returns = num_params;
6809 num_color_channels = util_bitcount(key->ps_prolog.colors_read);
6810 for (i = 0; i < num_color_channels; i++)
6811 params[num_returns++] = ctx->f32;
6812
6813 /* Create the function. */
6814 si_create_function(ctx, "ps_prolog", params, num_returns, params,
6815 num_params, last_sgpr, 0);
6816 func = ctx->main_fn;
6817
6818 /* Copy inputs to outputs. This should be no-op, as the registers match,
6819 * but it will prevent the compiler from overwriting them unintentionally.
6820 */
6821 ret = ctx->return_value;
6822 for (i = 0; i < num_params; i++) {
6823 LLVMValueRef p = LLVMGetParam(func, i);
6824 ret = LLVMBuildInsertValue(gallivm->builder, ret, p, i, "");
6825 }
6826
6827 /* Polygon stippling. */
6828 if (key->ps_prolog.states.poly_stipple) {
6829 /* POS_FIXED_PT is always last. */
6830 unsigned pos = key->ps_prolog.num_input_sgprs +
6831 key->ps_prolog.num_input_vgprs - 1;
6832 LLVMValueRef ptr[2], list;
6833
6834 /* Get the pointer to rw buffers. */
6835 ptr[0] = LLVMGetParam(func, SI_SGPR_RW_BUFFERS);
6836 ptr[1] = LLVMGetParam(func, SI_SGPR_RW_BUFFERS_HI);
6837 list = lp_build_gather_values(gallivm, ptr, 2);
6838 list = LLVMBuildBitCast(gallivm->builder, list, ctx->i64, "");
6839 list = LLVMBuildIntToPtr(gallivm->builder, list,
6840 si_const_array(ctx->v4i32, SI_NUM_RW_BUFFERS), "");
6841
6842 si_llvm_emit_polygon_stipple(ctx, list, pos);
6843 }
6844
6845 if (key->ps_prolog.states.bc_optimize_for_persp ||
6846 key->ps_prolog.states.bc_optimize_for_linear) {
6847 unsigned i, base = key->ps_prolog.num_input_sgprs;
6848 LLVMValueRef center[2], centroid[2], tmp, bc_optimize;
6849
6850 /* The shader should do: if (PRIM_MASK[31]) CENTROID = CENTER;
6851 * The hw doesn't compute CENTROID if the whole wave only
6852 * contains fully-covered quads.
6853 *
6854 * PRIM_MASK is after user SGPRs.
6855 */
6856 bc_optimize = LLVMGetParam(func, SI_PS_NUM_USER_SGPR);
6857 bc_optimize = LLVMBuildLShr(gallivm->builder, bc_optimize,
6858 LLVMConstInt(ctx->i32, 31, 0), "");
6859 bc_optimize = LLVMBuildTrunc(gallivm->builder, bc_optimize,
6860 ctx->i1, "");
6861
6862 if (key->ps_prolog.states.bc_optimize_for_persp) {
6863 /* Read PERSP_CENTER. */
6864 for (i = 0; i < 2; i++)
6865 center[i] = LLVMGetParam(func, base + 2 + i);
6866 /* Read PERSP_CENTROID. */
6867 for (i = 0; i < 2; i++)
6868 centroid[i] = LLVMGetParam(func, base + 4 + i);
6869 /* Select PERSP_CENTROID. */
6870 for (i = 0; i < 2; i++) {
6871 tmp = LLVMBuildSelect(gallivm->builder, bc_optimize,
6872 center[i], centroid[i], "");
6873 ret = LLVMBuildInsertValue(gallivm->builder, ret,
6874 tmp, base + 4 + i, "");
6875 }
6876 }
6877 if (key->ps_prolog.states.bc_optimize_for_linear) {
6878 /* Read LINEAR_CENTER. */
6879 for (i = 0; i < 2; i++)
6880 center[i] = LLVMGetParam(func, base + 8 + i);
6881 /* Read LINEAR_CENTROID. */
6882 for (i = 0; i < 2; i++)
6883 centroid[i] = LLVMGetParam(func, base + 10 + i);
6884 /* Select LINEAR_CENTROID. */
6885 for (i = 0; i < 2; i++) {
6886 tmp = LLVMBuildSelect(gallivm->builder, bc_optimize,
6887 center[i], centroid[i], "");
6888 ret = LLVMBuildInsertValue(gallivm->builder, ret,
6889 tmp, base + 10 + i, "");
6890 }
6891 }
6892 }
6893
6894 /* Force per-sample interpolation. */
6895 if (key->ps_prolog.states.force_persp_sample_interp) {
6896 unsigned i, base = key->ps_prolog.num_input_sgprs;
6897 LLVMValueRef persp_sample[2];
6898
6899 /* Read PERSP_SAMPLE. */
6900 for (i = 0; i < 2; i++)
6901 persp_sample[i] = LLVMGetParam(func, base + i);
6902 /* Overwrite PERSP_CENTER. */
6903 for (i = 0; i < 2; i++)
6904 ret = LLVMBuildInsertValue(gallivm->builder, ret,
6905 persp_sample[i], base + 2 + i, "");
6906 /* Overwrite PERSP_CENTROID. */
6907 for (i = 0; i < 2; i++)
6908 ret = LLVMBuildInsertValue(gallivm->builder, ret,
6909 persp_sample[i], base + 4 + i, "");
6910 }
6911 if (key->ps_prolog.states.force_linear_sample_interp) {
6912 unsigned i, base = key->ps_prolog.num_input_sgprs;
6913 LLVMValueRef linear_sample[2];
6914
6915 /* Read LINEAR_SAMPLE. */
6916 for (i = 0; i < 2; i++)
6917 linear_sample[i] = LLVMGetParam(func, base + 6 + i);
6918 /* Overwrite LINEAR_CENTER. */
6919 for (i = 0; i < 2; i++)
6920 ret = LLVMBuildInsertValue(gallivm->builder, ret,
6921 linear_sample[i], base + 8 + i, "");
6922 /* Overwrite LINEAR_CENTROID. */
6923 for (i = 0; i < 2; i++)
6924 ret = LLVMBuildInsertValue(gallivm->builder, ret,
6925 linear_sample[i], base + 10 + i, "");
6926 }
6927
6928 /* Force center interpolation. */
6929 if (key->ps_prolog.states.force_persp_center_interp) {
6930 unsigned i, base = key->ps_prolog.num_input_sgprs;
6931 LLVMValueRef persp_center[2];
6932
6933 /* Read PERSP_CENTER. */
6934 for (i = 0; i < 2; i++)
6935 persp_center[i] = LLVMGetParam(func, base + 2 + i);
6936 /* Overwrite PERSP_SAMPLE. */
6937 for (i = 0; i < 2; i++)
6938 ret = LLVMBuildInsertValue(gallivm->builder, ret,
6939 persp_center[i], base + i, "");
6940 /* Overwrite PERSP_CENTROID. */
6941 for (i = 0; i < 2; i++)
6942 ret = LLVMBuildInsertValue(gallivm->builder, ret,
6943 persp_center[i], base + 4 + i, "");
6944 }
6945 if (key->ps_prolog.states.force_linear_center_interp) {
6946 unsigned i, base = key->ps_prolog.num_input_sgprs;
6947 LLVMValueRef linear_center[2];
6948
6949 /* Read LINEAR_CENTER. */
6950 for (i = 0; i < 2; i++)
6951 linear_center[i] = LLVMGetParam(func, base + 8 + i);
6952 /* Overwrite LINEAR_SAMPLE. */
6953 for (i = 0; i < 2; i++)
6954 ret = LLVMBuildInsertValue(gallivm->builder, ret,
6955 linear_center[i], base + 6 + i, "");
6956 /* Overwrite LINEAR_CENTROID. */
6957 for (i = 0; i < 2; i++)
6958 ret = LLVMBuildInsertValue(gallivm->builder, ret,
6959 linear_center[i], base + 10 + i, "");
6960 }
6961
6962 /* Interpolate colors. */
6963 for (i = 0; i < 2; i++) {
6964 unsigned writemask = (key->ps_prolog.colors_read >> (i * 4)) & 0xf;
6965 unsigned face_vgpr = key->ps_prolog.num_input_sgprs +
6966 key->ps_prolog.face_vgpr_index;
6967 LLVMValueRef interp[2], color[4];
6968 LLVMValueRef interp_ij = NULL, prim_mask = NULL, face = NULL;
6969
6970 if (!writemask)
6971 continue;
6972
6973 /* If the interpolation qualifier is not CONSTANT (-1). */
6974 if (key->ps_prolog.color_interp_vgpr_index[i] != -1) {
6975 unsigned interp_vgpr = key->ps_prolog.num_input_sgprs +
6976 key->ps_prolog.color_interp_vgpr_index[i];
6977
6978 /* Get the (i,j) updated by bc_optimize handling. */
6979 interp[0] = LLVMBuildExtractValue(gallivm->builder, ret,
6980 interp_vgpr, "");
6981 interp[1] = LLVMBuildExtractValue(gallivm->builder, ret,
6982 interp_vgpr + 1, "");
6983 interp_ij = lp_build_gather_values(gallivm, interp, 2);
6984 }
6985
6986 /* Use the absolute location of the input. */
6987 prim_mask = LLVMGetParam(func, SI_PS_NUM_USER_SGPR);
6988
6989 if (key->ps_prolog.states.color_two_side) {
6990 face = LLVMGetParam(func, face_vgpr);
6991 face = LLVMBuildBitCast(gallivm->builder, face, ctx->i32, "");
6992 }
6993
6994 interp_fs_input(ctx,
6995 key->ps_prolog.color_attr_index[i],
6996 TGSI_SEMANTIC_COLOR, i,
6997 key->ps_prolog.num_interp_inputs,
6998 key->ps_prolog.colors_read, interp_ij,
6999 prim_mask, face, color);
7000
7001 while (writemask) {
7002 unsigned chan = u_bit_scan(&writemask);
7003 ret = LLVMBuildInsertValue(gallivm->builder, ret, color[chan],
7004 num_params++, "");
7005 }
7006 }
7007
7008 /* Tell LLVM to insert WQM instruction sequence when needed. */
7009 if (key->ps_prolog.wqm) {
7010 LLVMAddTargetDependentFunctionAttr(func,
7011 "amdgpu-ps-wqm-outputs", "");
7012 }
7013
7014 si_llvm_build_ret(ctx, ret);
7015 }
7016
7017 /**
7018 * Build the pixel shader epilog function. This handles everything that must be
7019 * emulated for pixel shader exports. (alpha-test, format conversions, etc)
7020 */
7021 static void si_build_ps_epilog_function(struct si_shader_context *ctx,
7022 union si_shader_part_key *key)
7023 {
7024 struct gallivm_state *gallivm = &ctx->gallivm;
7025 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
7026 LLVMTypeRef params[16+8*4+3];
7027 LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
7028 int last_sgpr, num_params = 0, i;
7029 struct si_ps_exports exp = {};
7030
7031 /* Declare input SGPRs. */
7032 params[ctx->param_rw_buffers = num_params++] = ctx->i64;
7033 params[ctx->param_const_and_shader_buffers = num_params++] = ctx->i64;
7034 params[ctx->param_samplers_and_images = num_params++] = ctx->i64;
7035 assert(num_params == SI_PARAM_ALPHA_REF);
7036 params[SI_PARAM_ALPHA_REF] = ctx->f32;
7037 last_sgpr = SI_PARAM_ALPHA_REF;
7038
7039 /* Declare input VGPRs. */
7040 num_params = (last_sgpr + 1) +
7041 util_bitcount(key->ps_epilog.colors_written) * 4 +
7042 key->ps_epilog.writes_z +
7043 key->ps_epilog.writes_stencil +
7044 key->ps_epilog.writes_samplemask;
7045
7046 num_params = MAX2(num_params,
7047 last_sgpr + 1 + PS_EPILOG_SAMPLEMASK_MIN_LOC + 1);
7048
7049 assert(num_params <= ARRAY_SIZE(params));
7050
7051 for (i = last_sgpr + 1; i < num_params; i++)
7052 params[i] = ctx->f32;
7053
7054 /* Create the function. */
7055 si_create_function(ctx, "ps_epilog", NULL, 0, params, num_params,
7056 last_sgpr, 0);
7057 /* Disable elimination of unused inputs. */
7058 si_llvm_add_attribute(ctx->main_fn,
7059 "InitialPSInputAddr", 0xffffff);
7060
7061 /* Process colors. */
7062 unsigned vgpr = last_sgpr + 1;
7063 unsigned colors_written = key->ps_epilog.colors_written;
7064 int last_color_export = -1;
7065
7066 /* Find the last color export. */
7067 if (!key->ps_epilog.writes_z &&
7068 !key->ps_epilog.writes_stencil &&
7069 !key->ps_epilog.writes_samplemask) {
7070 unsigned spi_format = key->ps_epilog.states.spi_shader_col_format;
7071
7072 /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
7073 if (colors_written == 0x1 && key->ps_epilog.states.last_cbuf > 0) {
7074 /* Just set this if any of the colorbuffers are enabled. */
7075 if (spi_format &
7076 ((1llu << (4 * (key->ps_epilog.states.last_cbuf + 1))) - 1))
7077 last_color_export = 0;
7078 } else {
7079 for (i = 0; i < 8; i++)
7080 if (colors_written & (1 << i) &&
7081 (spi_format >> (i * 4)) & 0xf)
7082 last_color_export = i;
7083 }
7084 }
7085
7086 while (colors_written) {
7087 LLVMValueRef color[4];
7088 int mrt = u_bit_scan(&colors_written);
7089
7090 for (i = 0; i < 4; i++)
7091 color[i] = LLVMGetParam(ctx->main_fn, vgpr++);
7092
7093 si_export_mrt_color(bld_base, color, mrt,
7094 num_params - 1,
7095 mrt == last_color_export, &exp);
7096 }
7097
7098 /* Process depth, stencil, samplemask. */
7099 if (key->ps_epilog.writes_z)
7100 depth = LLVMGetParam(ctx->main_fn, vgpr++);
7101 if (key->ps_epilog.writes_stencil)
7102 stencil = LLVMGetParam(ctx->main_fn, vgpr++);
7103 if (key->ps_epilog.writes_samplemask)
7104 samplemask = LLVMGetParam(ctx->main_fn, vgpr++);
7105
7106 if (depth || stencil || samplemask)
7107 si_export_mrt_z(bld_base, depth, stencil, samplemask, &exp);
7108 else if (last_color_export == -1)
7109 si_export_null(bld_base);
7110
7111 if (exp.num)
7112 si_emit_ps_exports(ctx, &exp);
7113
7114 /* Compile. */
7115 LLVMBuildRetVoid(gallivm->builder);
7116 }
7117
7118 /**
7119 * Select and compile (or reuse) pixel shader parts (prolog & epilog).
7120 */
7121 static bool si_shader_select_ps_parts(struct si_screen *sscreen,
7122 LLVMTargetMachineRef tm,
7123 struct si_shader *shader,
7124 struct pipe_debug_callback *debug)
7125 {
7126 union si_shader_part_key prolog_key;
7127 union si_shader_part_key epilog_key;
7128
7129 /* Get the prolog. */
7130 si_get_ps_prolog_key(shader, &prolog_key, true);
7131
7132 /* The prolog is a no-op if these aren't set. */
7133 if (si_need_ps_prolog(&prolog_key)) {
7134 shader->prolog =
7135 si_get_shader_part(sscreen, &sscreen->ps_prologs,
7136 PIPE_SHADER_FRAGMENT, true,
7137 &prolog_key, tm, debug,
7138 si_build_ps_prolog_function,
7139 "Fragment Shader Prolog");
7140 if (!shader->prolog)
7141 return false;
7142 }
7143
7144 /* Get the epilog. */
7145 si_get_ps_epilog_key(shader, &epilog_key);
7146
7147 shader->epilog =
7148 si_get_shader_part(sscreen, &sscreen->ps_epilogs,
7149 PIPE_SHADER_FRAGMENT, false,
7150 &epilog_key, tm, debug,
7151 si_build_ps_epilog_function,
7152 "Fragment Shader Epilog");
7153 if (!shader->epilog)
7154 return false;
7155
7156 /* Enable POS_FIXED_PT if polygon stippling is enabled. */
7157 if (shader->key.part.ps.prolog.poly_stipple) {
7158 shader->config.spi_ps_input_ena |= S_0286CC_POS_FIXED_PT_ENA(1);
7159 assert(G_0286CC_POS_FIXED_PT_ENA(shader->config.spi_ps_input_addr));
7160 }
7161
7162 /* Set up the enable bits for per-sample shading if needed. */
7163 if (shader->key.part.ps.prolog.force_persp_sample_interp &&
7164 (G_0286CC_PERSP_CENTER_ENA(shader->config.spi_ps_input_ena) ||
7165 G_0286CC_PERSP_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
7166 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_CENTER_ENA;
7167 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_CENTROID_ENA;
7168 shader->config.spi_ps_input_ena |= S_0286CC_PERSP_SAMPLE_ENA(1);
7169 }
7170 if (shader->key.part.ps.prolog.force_linear_sample_interp &&
7171 (G_0286CC_LINEAR_CENTER_ENA(shader->config.spi_ps_input_ena) ||
7172 G_0286CC_LINEAR_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
7173 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_CENTER_ENA;
7174 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_CENTROID_ENA;
7175 shader->config.spi_ps_input_ena |= S_0286CC_LINEAR_SAMPLE_ENA(1);
7176 }
7177 if (shader->key.part.ps.prolog.force_persp_center_interp &&
7178 (G_0286CC_PERSP_SAMPLE_ENA(shader->config.spi_ps_input_ena) ||
7179 G_0286CC_PERSP_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
7180 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_SAMPLE_ENA;
7181 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_CENTROID_ENA;
7182 shader->config.spi_ps_input_ena |= S_0286CC_PERSP_CENTER_ENA(1);
7183 }
7184 if (shader->key.part.ps.prolog.force_linear_center_interp &&
7185 (G_0286CC_LINEAR_SAMPLE_ENA(shader->config.spi_ps_input_ena) ||
7186 G_0286CC_LINEAR_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
7187 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_SAMPLE_ENA;
7188 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_CENTROID_ENA;
7189 shader->config.spi_ps_input_ena |= S_0286CC_LINEAR_CENTER_ENA(1);
7190 }
7191
7192 /* POW_W_FLOAT requires that one of the perspective weights is enabled. */
7193 if (G_0286CC_POS_W_FLOAT_ENA(shader->config.spi_ps_input_ena) &&
7194 !(shader->config.spi_ps_input_ena & 0xf)) {
7195 shader->config.spi_ps_input_ena |= S_0286CC_PERSP_CENTER_ENA(1);
7196 assert(G_0286CC_PERSP_CENTER_ENA(shader->config.spi_ps_input_addr));
7197 }
7198
7199 /* At least one pair of interpolation weights must be enabled. */
7200 if (!(shader->config.spi_ps_input_ena & 0x7f)) {
7201 shader->config.spi_ps_input_ena |= S_0286CC_LINEAR_CENTER_ENA(1);
7202 assert(G_0286CC_LINEAR_CENTER_ENA(shader->config.spi_ps_input_addr));
7203 }
7204
7205 /* The sample mask input is always enabled, because the API shader always
7206 * passes it through to the epilog. Disable it here if it's unused.
7207 */
7208 if (!shader->key.part.ps.epilog.poly_line_smoothing &&
7209 !shader->selector->info.reads_samplemask)
7210 shader->config.spi_ps_input_ena &= C_0286CC_SAMPLE_COVERAGE_ENA;
7211
7212 return true;
7213 }
7214
7215 void si_multiwave_lds_size_workaround(struct si_screen *sscreen,
7216 unsigned *lds_size)
7217 {
7218 /* SPI barrier management bug:
7219 * Make sure we have at least 4k of LDS in use to avoid the bug.
7220 * It applies to workgroup sizes of more than one wavefront.
7221 */
7222 if (sscreen->b.family == CHIP_BONAIRE ||
7223 sscreen->b.family == CHIP_KABINI ||
7224 sscreen->b.family == CHIP_MULLINS)
7225 *lds_size = MAX2(*lds_size, 8);
7226 }
7227
7228 static void si_fix_resource_usage(struct si_screen *sscreen,
7229 struct si_shader *shader)
7230 {
7231 unsigned min_sgprs = shader->info.num_input_sgprs + 2; /* VCC */
7232
7233 shader->config.num_sgprs = MAX2(shader->config.num_sgprs, min_sgprs);
7234
7235 if (shader->selector->type == PIPE_SHADER_COMPUTE &&
7236 si_get_max_workgroup_size(shader) > 64) {
7237 si_multiwave_lds_size_workaround(sscreen,
7238 &shader->config.lds_size);
7239 }
7240 }
7241
7242 int si_shader_create(struct si_screen *sscreen, LLVMTargetMachineRef tm,
7243 struct si_shader *shader,
7244 struct pipe_debug_callback *debug)
7245 {
7246 struct si_shader_selector *sel = shader->selector;
7247 struct si_shader *mainp = *si_get_main_shader_part(sel, &shader->key);
7248 int r;
7249
7250 /* LS, ES, VS are compiled on demand if the main part hasn't been
7251 * compiled for that stage.
7252 *
7253 * Vertex shaders are compiled on demand when a vertex fetch
7254 * workaround must be applied.
7255 */
7256 if (shader->is_monolithic) {
7257 /* Monolithic shader (compiled as a whole, has many variants,
7258 * may take a long time to compile).
7259 */
7260 r = si_compile_tgsi_shader(sscreen, tm, shader, true, debug);
7261 if (r)
7262 return r;
7263 } else {
7264 /* The shader consists of 2-3 parts:
7265 *
7266 * - the middle part is the user shader, it has 1 variant only
7267 * and it was compiled during the creation of the shader
7268 * selector
7269 * - the prolog part is inserted at the beginning
7270 * - the epilog part is inserted at the end
7271 *
7272 * The prolog and epilog have many (but simple) variants.
7273 */
7274
7275 /* Copy the compiled TGSI shader data over. */
7276 shader->is_binary_shared = true;
7277 shader->binary = mainp->binary;
7278 shader->config = mainp->config;
7279 shader->info.num_input_sgprs = mainp->info.num_input_sgprs;
7280 shader->info.num_input_vgprs = mainp->info.num_input_vgprs;
7281 shader->info.face_vgpr_index = mainp->info.face_vgpr_index;
7282 memcpy(shader->info.vs_output_param_offset,
7283 mainp->info.vs_output_param_offset,
7284 sizeof(mainp->info.vs_output_param_offset));
7285 shader->info.uses_instanceid = mainp->info.uses_instanceid;
7286 shader->info.nr_pos_exports = mainp->info.nr_pos_exports;
7287 shader->info.nr_param_exports = mainp->info.nr_param_exports;
7288
7289 /* Select prologs and/or epilogs. */
7290 switch (sel->type) {
7291 case PIPE_SHADER_VERTEX:
7292 if (!si_shader_select_vs_parts(sscreen, tm, shader, debug))
7293 return -1;
7294 break;
7295 case PIPE_SHADER_TESS_CTRL:
7296 if (!si_shader_select_tcs_parts(sscreen, tm, shader, debug))
7297 return -1;
7298 break;
7299 case PIPE_SHADER_TESS_EVAL:
7300 break;
7301 case PIPE_SHADER_GEOMETRY:
7302 if (!si_shader_select_gs_parts(sscreen, tm, shader, debug))
7303 return -1;
7304 break;
7305 case PIPE_SHADER_FRAGMENT:
7306 if (!si_shader_select_ps_parts(sscreen, tm, shader, debug))
7307 return -1;
7308
7309 /* Make sure we have at least as many VGPRs as there
7310 * are allocated inputs.
7311 */
7312 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
7313 shader->info.num_input_vgprs);
7314 break;
7315 }
7316
7317 /* Update SGPR and VGPR counts. */
7318 if (shader->prolog) {
7319 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
7320 shader->prolog->config.num_sgprs);
7321 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
7322 shader->prolog->config.num_vgprs);
7323 }
7324 if (shader->previous_stage) {
7325 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
7326 shader->previous_stage->config.num_sgprs);
7327 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
7328 shader->previous_stage->config.num_vgprs);
7329 shader->config.spilled_sgprs =
7330 MAX2(shader->config.spilled_sgprs,
7331 shader->previous_stage->config.spilled_sgprs);
7332 shader->config.spilled_vgprs =
7333 MAX2(shader->config.spilled_vgprs,
7334 shader->previous_stage->config.spilled_vgprs);
7335 shader->config.private_mem_vgprs =
7336 MAX2(shader->config.private_mem_vgprs,
7337 shader->previous_stage->config.private_mem_vgprs);
7338 shader->config.scratch_bytes_per_wave =
7339 MAX2(shader->config.scratch_bytes_per_wave,
7340 shader->previous_stage->config.scratch_bytes_per_wave);
7341 shader->info.uses_instanceid |=
7342 shader->previous_stage->info.uses_instanceid;
7343 }
7344 if (shader->prolog2) {
7345 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
7346 shader->prolog2->config.num_sgprs);
7347 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
7348 shader->prolog2->config.num_vgprs);
7349 }
7350 if (shader->epilog) {
7351 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
7352 shader->epilog->config.num_sgprs);
7353 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
7354 shader->epilog->config.num_vgprs);
7355 }
7356 }
7357
7358 si_fix_resource_usage(sscreen, shader);
7359 si_shader_dump(sscreen, shader, debug, sel->info.processor,
7360 stderr, true);
7361
7362 /* Upload. */
7363 r = si_shader_binary_upload(sscreen, shader);
7364 if (r) {
7365 fprintf(stderr, "LLVM failed to upload shader\n");
7366 return r;
7367 }
7368
7369 return 0;
7370 }
7371
7372 void si_shader_destroy(struct si_shader *shader)
7373 {
7374 if (shader->scratch_bo)
7375 r600_resource_reference(&shader->scratch_bo, NULL);
7376
7377 r600_resource_reference(&shader->bo, NULL);
7378
7379 if (!shader->is_binary_shared)
7380 radeon_shader_binary_clean(&shader->binary);
7381
7382 free(shader->shader_log);
7383 }