projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
radeonsi: clean up 'radeon_bld' references
[mesa.git] / src / gallium / drivers / radeonsi / si_shader.c
1 /*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 *
23 * Authors:
24 * Tom Stellard <thomas.stellard@amd.com>
25 * Michel Dänzer <michel.daenzer@amd.com>
26 * Christian König <christian.koenig@amd.com>
27 */
28
29 #include "gallivm/lp_bld_const.h"
30 #include "gallivm/lp_bld_gather.h"
31 #include "gallivm/lp_bld_intr.h"
32 #include "gallivm/lp_bld_logic.h"
33 #include "gallivm/lp_bld_arit.h"
34 #include "gallivm/lp_bld_flow.h"
35 #include "gallivm/lp_bld_misc.h"
36 #include "util/u_memory.h"
37 #include "util/u_string.h"
38 #include "tgsi/tgsi_build.h"
39 #include "tgsi/tgsi_util.h"
40 #include "tgsi/tgsi_dump.h"
41
42 #include "ac_binary.h"
43 #include "ac_llvm_util.h"
44 #include "si_shader_internal.h"
45 #include "si_pipe.h"
46 #include "sid.h"
47
48
49 static const char *scratch_rsrc_dword0_symbol =
50 "SCRATCH_RSRC_DWORD0";
51
52 static const char *scratch_rsrc_dword1_symbol =
53 "SCRATCH_RSRC_DWORD1";
54
55 struct si_shader_output_values
56 {
57 LLVMValueRef values[4];
58 unsigned semantic_name;
59 unsigned semantic_index;
60 ubyte vertex_stream[4];
61 };
62
63 static void si_init_shader_ctx(struct si_shader_context *ctx,
64 struct si_screen *sscreen,
65 struct si_shader *shader,
66 LLVMTargetMachineRef tm);
67
68 static void si_llvm_emit_barrier(const struct lp_build_tgsi_action *action,
69 struct lp_build_tgsi_context *bld_base,
70 struct lp_build_emit_data *emit_data);
71
72 static void si_dump_shader_key(unsigned shader, struct si_shader_key *key,
73 FILE *f);
74
75 static void si_build_vs_prolog_function(struct si_shader_context *ctx,
76 union si_shader_part_key *key);
77 static void si_build_vs_epilog_function(struct si_shader_context *ctx,
78 union si_shader_part_key *key);
79 static void si_build_tcs_epilog_function(struct si_shader_context *ctx,
80 union si_shader_part_key *key);
81 static void si_build_ps_prolog_function(struct si_shader_context *ctx,
82 union si_shader_part_key *key);
83 static void si_build_ps_epilog_function(struct si_shader_context *ctx,
84 union si_shader_part_key *key);
85
86 /* Ideally pass the sample mask input to the PS epilog as v13, which
87 * is its usual location, so that the shader doesn't have to add v_mov.
88 */
89 #define PS_EPILOG_SAMPLEMASK_MIN_LOC 13
90
91 /* The VS location of the PrimitiveID input is the same in the epilog,
92 * so that the main shader part doesn't have to move it.
93 */
94 #define VS_EPILOG_PRIMID_LOC 2
95
96 enum {
97 CONST_ADDR_SPACE = 2,
98 LOCAL_ADDR_SPACE = 3,
99 };
100
101 /**
102 * Returns a unique index for a semantic name and index. The index must be
103 * less than 64, so that a 64-bit bitmask of used inputs or outputs can be
104 * calculated.
105 */
106 unsigned si_shader_io_get_unique_index(unsigned semantic_name, unsigned index)
107 {
108 switch (semantic_name) {
109 case TGSI_SEMANTIC_POSITION:
110 return 0;
111 case TGSI_SEMANTIC_PSIZE:
112 return 1;
113 case TGSI_SEMANTIC_CLIPDIST:
114 assert(index <= 1);
115 return 2 + index;
116 case TGSI_SEMANTIC_GENERIC:
117 if (index <= 63-4)
118 return 4 + index;
119
120 assert(!"invalid generic index");
121 return 0;
122
123 /* patch indices are completely separate and thus start from 0 */
124 case TGSI_SEMANTIC_TESSOUTER:
125 return 0;
126 case TGSI_SEMANTIC_TESSINNER:
127 return 1;
128 case TGSI_SEMANTIC_PATCH:
129 return 2 + index;
130
131 default:
132 assert(!"invalid semantic name");
133 return 0;
134 }
135 }
136
137 unsigned si_shader_io_get_unique_index2(unsigned name, unsigned index)
138 {
139 switch (name) {
140 case TGSI_SEMANTIC_FOG:
141 return 0;
142 case TGSI_SEMANTIC_LAYER:
143 return 1;
144 case TGSI_SEMANTIC_VIEWPORT_INDEX:
145 return 2;
146 case TGSI_SEMANTIC_PRIMID:
147 return 3;
148 case TGSI_SEMANTIC_COLOR: /* these alias */
149 case TGSI_SEMANTIC_BCOLOR:
150 return 4 + index;
151 case TGSI_SEMANTIC_TEXCOORD:
152 return 6 + index;
153 default:
154 assert(!"invalid semantic name");
155 return 0;
156 }
157 }
158
159 /**
160 * Get the value of a shader input parameter and extract a bitfield.
161 */
162 static LLVMValueRef unpack_param(struct si_shader_context *ctx,
163 unsigned param, unsigned rshift,
164 unsigned bitwidth)
165 {
166 struct gallivm_state *gallivm = &ctx->gallivm;
167 LLVMValueRef value = LLVMGetParam(ctx->main_fn,
168 param);
169
170 if (LLVMGetTypeKind(LLVMTypeOf(value)) == LLVMFloatTypeKind)
171 value = bitcast(&ctx->bld_base,
172 TGSI_TYPE_UNSIGNED, value);
173
174 if (rshift)
175 value = LLVMBuildLShr(gallivm->builder, value,
176 lp_build_const_int32(gallivm, rshift), "");
177
178 if (rshift + bitwidth < 32) {
179 unsigned mask = (1 << bitwidth) - 1;
180 value = LLVMBuildAnd(gallivm->builder, value,
181 lp_build_const_int32(gallivm, mask), "");
182 }
183
184 return value;
185 }
186
187 static LLVMValueRef get_rel_patch_id(struct si_shader_context *ctx)
188 {
189 switch (ctx->type) {
190 case PIPE_SHADER_TESS_CTRL:
191 return unpack_param(ctx, SI_PARAM_REL_IDS, 0, 8);
192
193 case PIPE_SHADER_TESS_EVAL:
194 return LLVMGetParam(ctx->main_fn,
195 ctx->param_tes_rel_patch_id);
196
197 default:
198 assert(0);
199 return NULL;
200 }
201 }
202
203 /* Tessellation shaders pass outputs to the next shader using LDS.
204 *
205 * LS outputs = TCS inputs
206 * TCS outputs = TES inputs
207 *
208 * The LDS layout is:
209 * - TCS inputs for patch 0
210 * - TCS inputs for patch 1
211 * - TCS inputs for patch 2 = get_tcs_in_current_patch_offset (if RelPatchID==2)
212 * - ...
213 * - TCS outputs for patch 0 = get_tcs_out_patch0_offset
214 * - Per-patch TCS outputs for patch 0 = get_tcs_out_patch0_patch_data_offset
215 * - TCS outputs for patch 1
216 * - Per-patch TCS outputs for patch 1
217 * - TCS outputs for patch 2 = get_tcs_out_current_patch_offset (if RelPatchID==2)
218 * - Per-patch TCS outputs for patch 2 = get_tcs_out_current_patch_data_offset (if RelPatchID==2)
219 * - ...
220 *
221 * All three shaders VS(LS), TCS, TES share the same LDS space.
222 */
223
224 static LLVMValueRef
225 get_tcs_in_patch_stride(struct si_shader_context *ctx)
226 {
227 if (ctx->type == PIPE_SHADER_VERTEX)
228 return unpack_param(ctx, SI_PARAM_LS_OUT_LAYOUT, 0, 13);
229 else if (ctx->type == PIPE_SHADER_TESS_CTRL)
230 return unpack_param(ctx, SI_PARAM_TCS_IN_LAYOUT, 0, 13);
231 else {
232 assert(0);
233 return NULL;
234 }
235 }
236
237 static LLVMValueRef
238 get_tcs_out_patch_stride(struct si_shader_context *ctx)
239 {
240 return unpack_param(ctx, SI_PARAM_TCS_OUT_LAYOUT, 0, 13);
241 }
242
243 static LLVMValueRef
244 get_tcs_out_patch0_offset(struct si_shader_context *ctx)
245 {
246 return lp_build_mul_imm(&ctx->bld_base.uint_bld,
247 unpack_param(ctx,
248 SI_PARAM_TCS_OUT_OFFSETS,
249 0, 16),
250 4);
251 }
252
253 static LLVMValueRef
254 get_tcs_out_patch0_patch_data_offset(struct si_shader_context *ctx)
255 {
256 return lp_build_mul_imm(&ctx->bld_base.uint_bld,
257 unpack_param(ctx,
258 SI_PARAM_TCS_OUT_OFFSETS,
259 16, 16),
260 4);
261 }
262
263 static LLVMValueRef
264 get_tcs_in_current_patch_offset(struct si_shader_context *ctx)
265 {
266 struct gallivm_state *gallivm = &ctx->gallivm;
267 LLVMValueRef patch_stride = get_tcs_in_patch_stride(ctx);
268 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
269
270 return LLVMBuildMul(gallivm->builder, patch_stride, rel_patch_id, "");
271 }
272
273 static LLVMValueRef
274 get_tcs_out_current_patch_offset(struct si_shader_context *ctx)
275 {
276 struct gallivm_state *gallivm = &ctx->gallivm;
277 LLVMValueRef patch0_offset = get_tcs_out_patch0_offset(ctx);
278 LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
279 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
280
281 return LLVMBuildAdd(gallivm->builder, patch0_offset,
282 LLVMBuildMul(gallivm->builder, patch_stride,
283 rel_patch_id, ""),
284 "");
285 }
286
287 static LLVMValueRef
288 get_tcs_out_current_patch_data_offset(struct si_shader_context *ctx)
289 {
290 struct gallivm_state *gallivm = &ctx->gallivm;
291 LLVMValueRef patch0_patch_data_offset =
292 get_tcs_out_patch0_patch_data_offset(ctx);
293 LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
294 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
295
296 return LLVMBuildAdd(gallivm->builder, patch0_patch_data_offset,
297 LLVMBuildMul(gallivm->builder, patch_stride,
298 rel_patch_id, ""),
299 "");
300 }
301
302 static LLVMValueRef get_instance_index_for_fetch(
303 struct si_shader_context *ctx,
304 unsigned param_start_instance, unsigned divisor)
305 {
306 struct gallivm_state *gallivm = ctx->bld_base.base.gallivm;
307
308 LLVMValueRef result = LLVMGetParam(ctx->main_fn,
309 ctx->param_instance_id);
310
311 /* The division must be done before START_INSTANCE is added. */
312 if (divisor > 1)
313 result = LLVMBuildUDiv(gallivm->builder, result,
314 lp_build_const_int32(gallivm, divisor), "");
315
316 return LLVMBuildAdd(gallivm->builder, result,
317 LLVMGetParam(ctx->main_fn, param_start_instance), "");
318 }
319
320 /* Bitcast <4 x float> to <2 x double>, extract the component, and convert
321 * to float. */
322 static LLVMValueRef extract_double_to_float(struct si_shader_context *ctx,
323 LLVMValueRef vec4,
324 unsigned double_index)
325 {
326 LLVMBuilderRef builder = ctx->gallivm.builder;
327 LLVMTypeRef f64 = LLVMDoubleTypeInContext(ctx->gallivm.context);
328 LLVMValueRef dvec2 = LLVMBuildBitCast(builder, vec4,
329 LLVMVectorType(f64, 2), "");
330 LLVMValueRef index = LLVMConstInt(ctx->i32, double_index, 0);
331 LLVMValueRef value = LLVMBuildExtractElement(builder, dvec2, index, "");
332 return LLVMBuildFPTrunc(builder, value, ctx->f32, "");
333 }
334
335 static void declare_input_vs(
336 struct si_shader_context *ctx,
337 unsigned input_index,
338 const struct tgsi_full_declaration *decl,
339 LLVMValueRef out[4])
340 {
341 struct lp_build_context *base = &ctx->bld_base.base;
342 struct gallivm_state *gallivm = base->gallivm;
343
344 unsigned chan;
345 unsigned fix_fetch;
346 unsigned num_fetches;
347 unsigned fetch_stride;
348
349 LLVMValueRef t_list_ptr;
350 LLVMValueRef t_offset;
351 LLVMValueRef t_list;
352 LLVMValueRef vertex_index;
353 LLVMValueRef input[3];
354
355 /* Load the T list */
356 t_list_ptr = LLVMGetParam(ctx->main_fn, SI_PARAM_VERTEX_BUFFERS);
357
358 t_offset = lp_build_const_int32(gallivm, input_index);
359
360 t_list = ac_build_indexed_load_const(&ctx->ac, t_list_ptr, t_offset);
361
362 vertex_index = LLVMGetParam(ctx->main_fn,
363 ctx->param_vertex_index0 +
364 input_index);
365
366 fix_fetch = ctx->shader->key.mono.vs.fix_fetch[input_index];
367
368 /* Do multiple loads for special formats. */
369 switch (fix_fetch) {
370 case SI_FIX_FETCH_RGB_64_FLOAT:
371 num_fetches = 3; /* 3 2-dword loads */
372 fetch_stride = 8;
373 break;
374 case SI_FIX_FETCH_RGBA_64_FLOAT:
375 num_fetches = 2; /* 2 4-dword loads */
376 fetch_stride = 16;
377 break;
378 case SI_FIX_FETCH_RGB_8:
379 case SI_FIX_FETCH_RGB_8_INT:
380 num_fetches = 3;
381 fetch_stride = 1;
382 break;
383 case SI_FIX_FETCH_RGB_16:
384 case SI_FIX_FETCH_RGB_16_INT:
385 num_fetches = 3;
386 fetch_stride = 2;
387 break;
388 default:
389 num_fetches = 1;
390 fetch_stride = 0;
391 }
392
393 for (unsigned i = 0; i < num_fetches; i++) {
394 LLVMValueRef voffset = LLVMConstInt(ctx->i32, fetch_stride * i, 0);
395
396 input[i] = ac_build_buffer_load_format(&ctx->ac, t_list,
397 vertex_index, voffset,
398 true);
399 }
400
401 /* Break up the vec4 into individual components */
402 for (chan = 0; chan < 4; chan++) {
403 LLVMValueRef llvm_chan = lp_build_const_int32(gallivm, chan);
404 out[chan] = LLVMBuildExtractElement(gallivm->builder,
405 input[0], llvm_chan, "");
406 }
407
408 switch (fix_fetch) {
409 case SI_FIX_FETCH_A2_SNORM:
410 case SI_FIX_FETCH_A2_SSCALED:
411 case SI_FIX_FETCH_A2_SINT: {
412 /* The hardware returns an unsigned value; convert it to a
413 * signed one.
414 */
415 LLVMValueRef tmp = out[3];
416 LLVMValueRef c30 = LLVMConstInt(ctx->i32, 30, 0);
417
418 /* First, recover the sign-extended signed integer value. */
419 if (fix_fetch == SI_FIX_FETCH_A2_SSCALED)
420 tmp = LLVMBuildFPToUI(gallivm->builder, tmp, ctx->i32, "");
421 else
422 tmp = LLVMBuildBitCast(gallivm->builder, tmp, ctx->i32, "");
423
424 /* For the integer-like cases, do a natural sign extension.
425 *
426 * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
427 * and happen to contain 0, 1, 2, 3 as the two LSBs of the
428 * exponent.
429 */
430 tmp = LLVMBuildShl(gallivm->builder, tmp,
431 fix_fetch == SI_FIX_FETCH_A2_SNORM ?
432 LLVMConstInt(ctx->i32, 7, 0) : c30, "");
433 tmp = LLVMBuildAShr(gallivm->builder, tmp, c30, "");
434
435 /* Convert back to the right type. */
436 if (fix_fetch == SI_FIX_FETCH_A2_SNORM) {
437 LLVMValueRef clamp;
438 LLVMValueRef neg_one = LLVMConstReal(ctx->f32, -1.0);
439 tmp = LLVMBuildSIToFP(gallivm->builder, tmp, ctx->f32, "");
440 clamp = LLVMBuildFCmp(gallivm->builder, LLVMRealULT, tmp, neg_one, "");
441 tmp = LLVMBuildSelect(gallivm->builder, clamp, neg_one, tmp, "");
442 } else if (fix_fetch == SI_FIX_FETCH_A2_SSCALED) {
443 tmp = LLVMBuildSIToFP(gallivm->builder, tmp, ctx->f32, "");
444 }
445
446 out[3] = tmp;
447 break;
448 }
449 case SI_FIX_FETCH_RGBA_32_UNORM:
450 case SI_FIX_FETCH_RGBX_32_UNORM:
451 for (chan = 0; chan < 4; chan++) {
452 out[chan] = LLVMBuildBitCast(gallivm->builder, out[chan],
453 ctx->i32, "");
454 out[chan] = LLVMBuildUIToFP(gallivm->builder,
455 out[chan], ctx->f32, "");
456 out[chan] = LLVMBuildFMul(gallivm->builder, out[chan],
457 LLVMConstReal(ctx->f32, 1.0 / UINT_MAX), "");
458 }
459 /* RGBX UINT returns 1 in alpha, which would be rounded to 0 by normalizing. */
460 if (fix_fetch == SI_FIX_FETCH_RGBX_32_UNORM)
461 out[3] = LLVMConstReal(ctx->f32, 1);
462 break;
463 case SI_FIX_FETCH_RGBA_32_SNORM:
464 case SI_FIX_FETCH_RGBX_32_SNORM:
465 case SI_FIX_FETCH_RGBA_32_FIXED:
466 case SI_FIX_FETCH_RGBX_32_FIXED: {
467 double scale;
468 if (fix_fetch >= SI_FIX_FETCH_RGBA_32_FIXED)
469 scale = 1.0 / 0x10000;
470 else
471 scale = 1.0 / INT_MAX;
472
473 for (chan = 0; chan < 4; chan++) {
474 out[chan] = LLVMBuildBitCast(gallivm->builder, out[chan],
475 ctx->i32, "");
476 out[chan] = LLVMBuildSIToFP(gallivm->builder,
477 out[chan], ctx->f32, "");
478 out[chan] = LLVMBuildFMul(gallivm->builder, out[chan],
479 LLVMConstReal(ctx->f32, scale), "");
480 }
481 /* RGBX SINT returns 1 in alpha, which would be rounded to 0 by normalizing. */
482 if (fix_fetch == SI_FIX_FETCH_RGBX_32_SNORM ||
483 fix_fetch == SI_FIX_FETCH_RGBX_32_FIXED)
484 out[3] = LLVMConstReal(ctx->f32, 1);
485 break;
486 }
487 case SI_FIX_FETCH_RGBA_32_USCALED:
488 for (chan = 0; chan < 4; chan++) {
489 out[chan] = LLVMBuildBitCast(gallivm->builder, out[chan],
490 ctx->i32, "");
491 out[chan] = LLVMBuildUIToFP(gallivm->builder,
492 out[chan], ctx->f32, "");
493 }
494 break;
495 case SI_FIX_FETCH_RGBA_32_SSCALED:
496 for (chan = 0; chan < 4; chan++) {
497 out[chan] = LLVMBuildBitCast(gallivm->builder, out[chan],
498 ctx->i32, "");
499 out[chan] = LLVMBuildSIToFP(gallivm->builder,
500 out[chan], ctx->f32, "");
501 }
502 break;
503 case SI_FIX_FETCH_RG_64_FLOAT:
504 for (chan = 0; chan < 2; chan++)
505 out[chan] = extract_double_to_float(ctx, input[0], chan);
506
507 out[2] = LLVMConstReal(ctx->f32, 0);
508 out[3] = LLVMConstReal(ctx->f32, 1);
509 break;
510 case SI_FIX_FETCH_RGB_64_FLOAT:
511 for (chan = 0; chan < 3; chan++)
512 out[chan] = extract_double_to_float(ctx, input[chan], 0);
513
514 out[3] = LLVMConstReal(ctx->f32, 1);
515 break;
516 case SI_FIX_FETCH_RGBA_64_FLOAT:
517 for (chan = 0; chan < 4; chan++) {
518 out[chan] = extract_double_to_float(ctx, input[chan / 2],
519 chan % 2);
520 }
521 break;
522 case SI_FIX_FETCH_RGB_8:
523 case SI_FIX_FETCH_RGB_8_INT:
524 case SI_FIX_FETCH_RGB_16:
525 case SI_FIX_FETCH_RGB_16_INT:
526 for (chan = 0; chan < 3; chan++) {
527 out[chan] = LLVMBuildExtractElement(gallivm->builder,
528 input[chan],
529 ctx->i32_0, "");
530 }
531 if (fix_fetch == SI_FIX_FETCH_RGB_8 ||
532 fix_fetch == SI_FIX_FETCH_RGB_16) {
533 out[3] = LLVMConstReal(ctx->f32, 1);
534 } else {
535 out[3] = LLVMBuildBitCast(gallivm->builder, ctx->i32_1,
536 ctx->f32, "");
537 }
538 break;
539 }
540 }
541
542 static LLVMValueRef get_primitive_id(struct lp_build_tgsi_context *bld_base,
543 unsigned swizzle)
544 {
545 struct si_shader_context *ctx = si_shader_context(bld_base);
546
547 if (swizzle > 0)
548 return bld_base->uint_bld.zero;
549
550 switch (ctx->type) {
551 case PIPE_SHADER_VERTEX:
552 return LLVMGetParam(ctx->main_fn,
553 ctx->param_vs_prim_id);
554 case PIPE_SHADER_TESS_CTRL:
555 return LLVMGetParam(ctx->main_fn,
556 SI_PARAM_PATCH_ID);
557 case PIPE_SHADER_TESS_EVAL:
558 return LLVMGetParam(ctx->main_fn,
559 ctx->param_tes_patch_id);
560 case PIPE_SHADER_GEOMETRY:
561 return LLVMGetParam(ctx->main_fn,
562 SI_PARAM_PRIMITIVE_ID);
563 default:
564 assert(0);
565 return bld_base->uint_bld.zero;
566 }
567 }
568
569 /**
570 * Return the value of tgsi_ind_register for indexing.
571 * This is the indirect index with the constant offset added to it.
572 */
573 static LLVMValueRef get_indirect_index(struct si_shader_context *ctx,
574 const struct tgsi_ind_register *ind,
575 int rel_index)
576 {
577 struct gallivm_state *gallivm = ctx->bld_base.base.gallivm;
578 LLVMValueRef result;
579
580 result = ctx->addrs[ind->Index][ind->Swizzle];
581 result = LLVMBuildLoad(gallivm->builder, result, "");
582 result = LLVMBuildAdd(gallivm->builder, result,
583 lp_build_const_int32(gallivm, rel_index), "");
584 return result;
585 }
586
587 /**
588 * Like get_indirect_index, but restricts the return value to a (possibly
589 * undefined) value inside [0..num).
590 */
591 static LLVMValueRef get_bounded_indirect_index(struct si_shader_context *ctx,
592 const struct tgsi_ind_register *ind,
593 int rel_index, unsigned num)
594 {
595 LLVMValueRef result = get_indirect_index(ctx, ind, rel_index);
596
597 /* LLVM 3.8: If indirect resource indexing is used:
598 * - SI & CIK hang
599 * - VI crashes
600 */
601 if (HAVE_LLVM == 0x0308)
602 return LLVMGetUndef(ctx->i32);
603
604 return si_llvm_bound_index(ctx, result, num);
605 }
606
607
608 /**
609 * Calculate a dword address given an input or output register and a stride.
610 */
611 static LLVMValueRef get_dw_address(struct si_shader_context *ctx,
612 const struct tgsi_full_dst_register *dst,
613 const struct tgsi_full_src_register *src,
614 LLVMValueRef vertex_dw_stride,
615 LLVMValueRef base_addr)
616 {
617 struct gallivm_state *gallivm = ctx->bld_base.base.gallivm;
618 struct tgsi_shader_info *info = &ctx->shader->selector->info;
619 ubyte *name, *index, *array_first;
620 int first, param;
621 struct tgsi_full_dst_register reg;
622
623 /* Set the register description. The address computation is the same
624 * for sources and destinations. */
625 if (src) {
626 reg.Register.File = src->Register.File;
627 reg.Register.Index = src->Register.Index;
628 reg.Register.Indirect = src->Register.Indirect;
629 reg.Register.Dimension = src->Register.Dimension;
630 reg.Indirect = src->Indirect;
631 reg.Dimension = src->Dimension;
632 reg.DimIndirect = src->DimIndirect;
633 } else
634 reg = *dst;
635
636 /* If the register is 2-dimensional (e.g. an array of vertices
637 * in a primitive), calculate the base address of the vertex. */
638 if (reg.Register.Dimension) {
639 LLVMValueRef index;
640
641 if (reg.Dimension.Indirect)
642 index = get_indirect_index(ctx, &reg.DimIndirect,
643 reg.Dimension.Index);
644 else
645 index = lp_build_const_int32(gallivm, reg.Dimension.Index);
646
647 base_addr = LLVMBuildAdd(gallivm->builder, base_addr,
648 LLVMBuildMul(gallivm->builder, index,
649 vertex_dw_stride, ""), "");
650 }
651
652 /* Get information about the register. */
653 if (reg.Register.File == TGSI_FILE_INPUT) {
654 name = info->input_semantic_name;
655 index = info->input_semantic_index;
656 array_first = info->input_array_first;
657 } else if (reg.Register.File == TGSI_FILE_OUTPUT) {
658 name = info->output_semantic_name;
659 index = info->output_semantic_index;
660 array_first = info->output_array_first;
661 } else {
662 assert(0);
663 return NULL;
664 }
665
666 if (reg.Register.Indirect) {
667 /* Add the relative address of the element. */
668 LLVMValueRef ind_index;
669
670 if (reg.Indirect.ArrayID)
671 first = array_first[reg.Indirect.ArrayID];
672 else
673 first = reg.Register.Index;
674
675 ind_index = get_indirect_index(ctx, &reg.Indirect,
676 reg.Register.Index - first);
677
678 base_addr = LLVMBuildAdd(gallivm->builder, base_addr,
679 LLVMBuildMul(gallivm->builder, ind_index,
680 lp_build_const_int32(gallivm, 4), ""), "");
681
682 param = si_shader_io_get_unique_index(name[first], index[first]);
683 } else {
684 param = si_shader_io_get_unique_index(name[reg.Register.Index],
685 index[reg.Register.Index]);
686 }
687
688 /* Add the base address of the element. */
689 return LLVMBuildAdd(gallivm->builder, base_addr,
690 lp_build_const_int32(gallivm, param * 4), "");
691 }
692
693 /* The offchip buffer layout for TCS->TES is
694 *
695 * - attribute 0 of patch 0 vertex 0
696 * - attribute 0 of patch 0 vertex 1
697 * - attribute 0 of patch 0 vertex 2
698 * ...
699 * - attribute 0 of patch 1 vertex 0
700 * - attribute 0 of patch 1 vertex 1
701 * ...
702 * - attribute 1 of patch 0 vertex 0
703 * - attribute 1 of patch 0 vertex 1
704 * ...
705 * - per patch attribute 0 of patch 0
706 * - per patch attribute 0 of patch 1
707 * ...
708 *
709 * Note that every attribute has 4 components.
710 */
711 static LLVMValueRef get_tcs_tes_buffer_address(struct si_shader_context *ctx,
712 LLVMValueRef rel_patch_id,
713 LLVMValueRef vertex_index,
714 LLVMValueRef param_index)
715 {
716 struct gallivm_state *gallivm = ctx->bld_base.base.gallivm;
717 LLVMValueRef base_addr, vertices_per_patch, num_patches, total_vertices;
718 LLVMValueRef param_stride, constant16;
719
720 vertices_per_patch = unpack_param(ctx, SI_PARAM_TCS_OFFCHIP_LAYOUT, 9, 6);
721 num_patches = unpack_param(ctx, SI_PARAM_TCS_OFFCHIP_LAYOUT, 0, 9);
722 total_vertices = LLVMBuildMul(gallivm->builder, vertices_per_patch,
723 num_patches, "");
724
725 constant16 = lp_build_const_int32(gallivm, 16);
726 if (vertex_index) {
727 base_addr = LLVMBuildMul(gallivm->builder, rel_patch_id,
728 vertices_per_patch, "");
729
730 base_addr = LLVMBuildAdd(gallivm->builder, base_addr,
731 vertex_index, "");
732
733 param_stride = total_vertices;
734 } else {
735 base_addr = rel_patch_id;
736 param_stride = num_patches;
737 }
738
739 base_addr = LLVMBuildAdd(gallivm->builder, base_addr,
740 LLVMBuildMul(gallivm->builder, param_index,
741 param_stride, ""), "");
742
743 base_addr = LLVMBuildMul(gallivm->builder, base_addr, constant16, "");
744
745 if (!vertex_index) {
746 LLVMValueRef patch_data_offset =
747 unpack_param(ctx, SI_PARAM_TCS_OFFCHIP_LAYOUT, 16, 16);
748
749 base_addr = LLVMBuildAdd(gallivm->builder, base_addr,
750 patch_data_offset, "");
751 }
752 return base_addr;
753 }
754
755 static LLVMValueRef get_tcs_tes_buffer_address_from_reg(
756 struct si_shader_context *ctx,
757 const struct tgsi_full_dst_register *dst,
758 const struct tgsi_full_src_register *src)
759 {
760 struct gallivm_state *gallivm = ctx->bld_base.base.gallivm;
761 struct tgsi_shader_info *info = &ctx->shader->selector->info;
762 ubyte *name, *index, *array_first;
763 struct tgsi_full_src_register reg;
764 LLVMValueRef vertex_index = NULL;
765 LLVMValueRef param_index = NULL;
766 unsigned param_index_base, param_base;
767
768 reg = src ? *src : tgsi_full_src_register_from_dst(dst);
769
770 if (reg.Register.Dimension) {
771
772 if (reg.Dimension.Indirect)
773 vertex_index = get_indirect_index(ctx, &reg.DimIndirect,
774 reg.Dimension.Index);
775 else
776 vertex_index = lp_build_const_int32(gallivm,
777 reg.Dimension.Index);
778 }
779
780 /* Get information about the register. */
781 if (reg.Register.File == TGSI_FILE_INPUT) {
782 name = info->input_semantic_name;
783 index = info->input_semantic_index;
784 array_first = info->input_array_first;
785 } else if (reg.Register.File == TGSI_FILE_OUTPUT) {
786 name = info->output_semantic_name;
787 index = info->output_semantic_index;
788 array_first = info->output_array_first;
789 } else {
790 assert(0);
791 return NULL;
792 }
793
794 if (reg.Register.Indirect) {
795 if (reg.Indirect.ArrayID)
796 param_base = array_first[reg.Indirect.ArrayID];
797 else
798 param_base = reg.Register.Index;
799
800 param_index = get_indirect_index(ctx, &reg.Indirect,
801 reg.Register.Index - param_base);
802
803 } else {
804 param_base = reg.Register.Index;
805 param_index = lp_build_const_int32(gallivm, 0);
806 }
807
808 param_index_base = si_shader_io_get_unique_index(name[param_base],
809 index[param_base]);
810
811 param_index = LLVMBuildAdd(gallivm->builder, param_index,
812 lp_build_const_int32(gallivm, param_index_base),
813 "");
814
815 return get_tcs_tes_buffer_address(ctx, get_rel_patch_id(ctx),
816 vertex_index, param_index);
817 }
818
819 static LLVMValueRef buffer_load(struct lp_build_tgsi_context *bld_base,
820 enum tgsi_opcode_type type, unsigned swizzle,
821 LLVMValueRef buffer, LLVMValueRef offset,
822 LLVMValueRef base, bool readonly_memory)
823 {
824 struct si_shader_context *ctx = si_shader_context(bld_base);
825 struct gallivm_state *gallivm = bld_base->base.gallivm;
826 LLVMValueRef value, value2;
827 LLVMTypeRef llvm_type = tgsi2llvmtype(bld_base, type);
828 LLVMTypeRef vec_type = LLVMVectorType(llvm_type, 4);
829
830 if (swizzle == ~0) {
831 value = ac_build_buffer_load(&ctx->ac, buffer, 4, NULL, base, offset,
832 0, 1, 0, readonly_memory);
833
834 return LLVMBuildBitCast(gallivm->builder, value, vec_type, "");
835 }
836
837 if (!tgsi_type_is_64bit(type)) {
838 value = ac_build_buffer_load(&ctx->ac, buffer, 4, NULL, base, offset,
839 0, 1, 0, readonly_memory);
840
841 value = LLVMBuildBitCast(gallivm->builder, value, vec_type, "");
842 return LLVMBuildExtractElement(gallivm->builder, value,
843 lp_build_const_int32(gallivm, swizzle), "");
844 }
845
846 value = ac_build_buffer_load(&ctx->ac, buffer, 1, NULL, base, offset,
847 swizzle * 4, 1, 0, readonly_memory);
848
849 value2 = ac_build_buffer_load(&ctx->ac, buffer, 1, NULL, base, offset,
850 swizzle * 4 + 4, 1, 0, readonly_memory);
851
852 return si_llvm_emit_fetch_64bit(bld_base, type, value, value2);
853 }
854
855 /**
856 * Load from LDS.
857 *
858 * \param type output value type
859 * \param swizzle offset (typically 0..3); it can be ~0, which loads a vec4
860 * \param dw_addr address in dwords
861 */
862 static LLVMValueRef lds_load(struct lp_build_tgsi_context *bld_base,
863 enum tgsi_opcode_type type, unsigned swizzle,
864 LLVMValueRef dw_addr)
865 {
866 struct si_shader_context *ctx = si_shader_context(bld_base);
867 struct gallivm_state *gallivm = bld_base->base.gallivm;
868 LLVMValueRef value;
869
870 if (swizzle == ~0) {
871 LLVMValueRef values[TGSI_NUM_CHANNELS];
872
873 for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; chan++)
874 values[chan] = lds_load(bld_base, type, chan, dw_addr);
875
876 return lp_build_gather_values(bld_base->base.gallivm, values,
877 TGSI_NUM_CHANNELS);
878 }
879
880 dw_addr = lp_build_add(&bld_base->uint_bld, dw_addr,
881 lp_build_const_int32(gallivm, swizzle));
882
883 value = ac_build_indexed_load(&ctx->ac, ctx->lds, dw_addr, false);
884 if (tgsi_type_is_64bit(type)) {
885 LLVMValueRef value2;
886 dw_addr = lp_build_add(&bld_base->uint_bld, dw_addr,
887 lp_build_const_int32(gallivm, 1));
888 value2 = ac_build_indexed_load(&ctx->ac, ctx->lds, dw_addr, false);
889 return si_llvm_emit_fetch_64bit(bld_base, type, value, value2);
890 }
891
892 return LLVMBuildBitCast(gallivm->builder, value,
893 tgsi2llvmtype(bld_base, type), "");
894 }
895
896 /**
897 * Store to LDS.
898 *
899 * \param swizzle offset (typically 0..3)
900 * \param dw_addr address in dwords
901 * \param value value to store
902 */
903 static void lds_store(struct lp_build_tgsi_context *bld_base,
904 unsigned swizzle, LLVMValueRef dw_addr,
905 LLVMValueRef value)
906 {
907 struct si_shader_context *ctx = si_shader_context(bld_base);
908 struct gallivm_state *gallivm = bld_base->base.gallivm;
909
910 dw_addr = lp_build_add(&bld_base->uint_bld, dw_addr,
911 lp_build_const_int32(gallivm, swizzle));
912
913 value = LLVMBuildBitCast(gallivm->builder, value, ctx->i32, "");
914 ac_build_indexed_store(&ctx->ac, ctx->lds,
915 dw_addr, value);
916 }
917
918 static LLVMValueRef fetch_input_tcs(
919 struct lp_build_tgsi_context *bld_base,
920 const struct tgsi_full_src_register *reg,
921 enum tgsi_opcode_type type, unsigned swizzle)
922 {
923 struct si_shader_context *ctx = si_shader_context(bld_base);
924 LLVMValueRef dw_addr, stride;
925
926 stride = unpack_param(ctx, SI_PARAM_TCS_IN_LAYOUT, 13, 8);
927 dw_addr = get_tcs_in_current_patch_offset(ctx);
928 dw_addr = get_dw_address(ctx, NULL, reg, stride, dw_addr);
929
930 return lds_load(bld_base, type, swizzle, dw_addr);
931 }
932
933 static LLVMValueRef fetch_output_tcs(
934 struct lp_build_tgsi_context *bld_base,
935 const struct tgsi_full_src_register *reg,
936 enum tgsi_opcode_type type, unsigned swizzle)
937 {
938 struct si_shader_context *ctx = si_shader_context(bld_base);
939 LLVMValueRef dw_addr, stride;
940
941 if (reg->Register.Dimension) {
942 stride = unpack_param(ctx, SI_PARAM_TCS_OUT_LAYOUT, 13, 8);
943 dw_addr = get_tcs_out_current_patch_offset(ctx);
944 dw_addr = get_dw_address(ctx, NULL, reg, stride, dw_addr);
945 } else {
946 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
947 dw_addr = get_dw_address(ctx, NULL, reg, NULL, dw_addr);
948 }
949
950 return lds_load(bld_base, type, swizzle, dw_addr);
951 }
952
953 static LLVMValueRef fetch_input_tes(
954 struct lp_build_tgsi_context *bld_base,
955 const struct tgsi_full_src_register *reg,
956 enum tgsi_opcode_type type, unsigned swizzle)
957 {
958 struct si_shader_context *ctx = si_shader_context(bld_base);
959 struct gallivm_state *gallivm = bld_base->base.gallivm;
960 LLVMValueRef rw_buffers, buffer, base, addr;
961
962 rw_buffers = LLVMGetParam(ctx->main_fn,
963 SI_PARAM_RW_BUFFERS);
964 buffer = ac_build_indexed_load_const(&ctx->ac, rw_buffers,
965 lp_build_const_int32(gallivm, SI_HS_RING_TESS_OFFCHIP));
966
967 base = LLVMGetParam(ctx->main_fn, ctx->param_oc_lds);
968 addr = get_tcs_tes_buffer_address_from_reg(ctx, NULL, reg);
969
970 return buffer_load(bld_base, type, swizzle, buffer, base, addr, true);
971 }
972
973 static void store_output_tcs(struct lp_build_tgsi_context *bld_base,
974 const struct tgsi_full_instruction *inst,
975 const struct tgsi_opcode_info *info,
976 LLVMValueRef dst[4])
977 {
978 struct si_shader_context *ctx = si_shader_context(bld_base);
979 struct gallivm_state *gallivm = bld_base->base.gallivm;
980 const struct tgsi_full_dst_register *reg = &inst->Dst[0];
981 const struct tgsi_shader_info *sh_info = &ctx->shader->selector->info;
982 unsigned chan_index;
983 LLVMValueRef dw_addr, stride;
984 LLVMValueRef rw_buffers, buffer, base, buf_addr;
985 LLVMValueRef values[4];
986 bool skip_lds_store;
987 bool is_tess_factor = false;
988
989 /* Only handle per-patch and per-vertex outputs here.
990 * Vectors will be lowered to scalars and this function will be called again.
991 */
992 if (reg->Register.File != TGSI_FILE_OUTPUT ||
993 (dst[0] && LLVMGetTypeKind(LLVMTypeOf(dst[0])) == LLVMVectorTypeKind)) {
994 si_llvm_emit_store(bld_base, inst, info, dst);
995 return;
996 }
997
998 if (reg->Register.Dimension) {
999 stride = unpack_param(ctx, SI_PARAM_TCS_OUT_LAYOUT, 13, 8);
1000 dw_addr = get_tcs_out_current_patch_offset(ctx);
1001 dw_addr = get_dw_address(ctx, reg, NULL, stride, dw_addr);
1002 skip_lds_store = !sh_info->reads_pervertex_outputs;
1003 } else {
1004 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1005 dw_addr = get_dw_address(ctx, reg, NULL, NULL, dw_addr);
1006 skip_lds_store = !sh_info->reads_perpatch_outputs;
1007
1008 if (!reg->Register.Indirect) {
1009 int name = sh_info->output_semantic_name[reg->Register.Index];
1010
1011 /* Always write tess factors into LDS for the TCS epilog. */
1012 if (name == TGSI_SEMANTIC_TESSINNER ||
1013 name == TGSI_SEMANTIC_TESSOUTER) {
1014 skip_lds_store = false;
1015 is_tess_factor = true;
1016 }
1017 }
1018 }
1019
1020 rw_buffers = LLVMGetParam(ctx->main_fn,
1021 SI_PARAM_RW_BUFFERS);
1022 buffer = ac_build_indexed_load_const(&ctx->ac, rw_buffers,
1023 lp_build_const_int32(gallivm, SI_HS_RING_TESS_OFFCHIP));
1024
1025 base = LLVMGetParam(ctx->main_fn, ctx->param_oc_lds);
1026 buf_addr = get_tcs_tes_buffer_address_from_reg(ctx, reg, NULL);
1027
1028
1029 TGSI_FOR_EACH_DST0_ENABLED_CHANNEL(inst, chan_index) {
1030 LLVMValueRef value = dst[chan_index];
1031
1032 if (inst->Instruction.Saturate)
1033 value = ac_build_clamp(&ctx->ac, value);
1034
1035 /* Skip LDS stores if there is no LDS read of this output. */
1036 if (!skip_lds_store)
1037 lds_store(bld_base, chan_index, dw_addr, value);
1038
1039 value = LLVMBuildBitCast(gallivm->builder, value, ctx->i32, "");
1040 values[chan_index] = value;
1041
1042 if (inst->Dst[0].Register.WriteMask != 0xF && !is_tess_factor) {
1043 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 1,
1044 buf_addr, base,
1045 4 * chan_index, 1, 0, true, false);
1046 }
1047 }
1048
1049 if (inst->Dst[0].Register.WriteMask == 0xF && !is_tess_factor) {
1050 LLVMValueRef value = lp_build_gather_values(bld_base->base.gallivm,
1051 values, 4);
1052 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, buf_addr,
1053 base, 0, 1, 0, true, false);
1054 }
1055 }
1056
1057 static LLVMValueRef fetch_input_gs(
1058 struct lp_build_tgsi_context *bld_base,
1059 const struct tgsi_full_src_register *reg,
1060 enum tgsi_opcode_type type,
1061 unsigned swizzle)
1062 {
1063 struct lp_build_context *base = &bld_base->base;
1064 struct si_shader_context *ctx = si_shader_context(bld_base);
1065 struct si_shader *shader = ctx->shader;
1066 struct lp_build_context *uint = &ctx->bld_base.uint_bld;
1067 struct gallivm_state *gallivm = base->gallivm;
1068 LLVMValueRef vtx_offset, soffset;
1069 unsigned vtx_offset_param;
1070 struct tgsi_shader_info *info = &shader->selector->info;
1071 unsigned semantic_name = info->input_semantic_name[reg->Register.Index];
1072 unsigned semantic_index = info->input_semantic_index[reg->Register.Index];
1073 unsigned param;
1074 LLVMValueRef value;
1075
1076 if (swizzle != ~0 && semantic_name == TGSI_SEMANTIC_PRIMID)
1077 return get_primitive_id(bld_base, swizzle);
1078
1079 if (!reg->Register.Dimension)
1080 return NULL;
1081
1082 if (swizzle == ~0) {
1083 LLVMValueRef values[TGSI_NUM_CHANNELS];
1084 unsigned chan;
1085 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1086 values[chan] = fetch_input_gs(bld_base, reg, type, chan);
1087 }
1088 return lp_build_gather_values(bld_base->base.gallivm, values,
1089 TGSI_NUM_CHANNELS);
1090 }
1091
1092 /* Get the vertex offset parameter */
1093 vtx_offset_param = reg->Dimension.Index;
1094 if (vtx_offset_param < 2) {
1095 vtx_offset_param += SI_PARAM_VTX0_OFFSET;
1096 } else {
1097 assert(vtx_offset_param < 6);
1098 vtx_offset_param += SI_PARAM_VTX2_OFFSET - 2;
1099 }
1100 vtx_offset = lp_build_mul_imm(uint,
1101 LLVMGetParam(ctx->main_fn,
1102 vtx_offset_param),
1103 4);
1104
1105 param = si_shader_io_get_unique_index(semantic_name, semantic_index);
1106 soffset = LLVMConstInt(ctx->i32, (param * 4 + swizzle) * 256, 0);
1107
1108 value = ac_build_buffer_load(&ctx->ac, ctx->esgs_ring, 1, uint->zero,
1109 vtx_offset, soffset, 0, 1, 0, true);
1110 if (tgsi_type_is_64bit(type)) {
1111 LLVMValueRef value2;
1112 soffset = LLVMConstInt(ctx->i32, (param * 4 + swizzle + 1) * 256, 0);
1113
1114 value2 = ac_build_buffer_load(&ctx->ac, ctx->esgs_ring, 1,
1115 uint->zero, vtx_offset, soffset,
1116 0, 1, 0, true);
1117 return si_llvm_emit_fetch_64bit(bld_base, type,
1118 value, value2);
1119 }
1120 return LLVMBuildBitCast(gallivm->builder,
1121 value,
1122 tgsi2llvmtype(bld_base, type), "");
1123 }
1124
1125 static int lookup_interp_param_index(unsigned interpolate, unsigned location)
1126 {
1127 switch (interpolate) {
1128 case TGSI_INTERPOLATE_CONSTANT:
1129 return 0;
1130
1131 case TGSI_INTERPOLATE_LINEAR:
1132 if (location == TGSI_INTERPOLATE_LOC_SAMPLE)
1133 return SI_PARAM_LINEAR_SAMPLE;
1134 else if (location == TGSI_INTERPOLATE_LOC_CENTROID)
1135 return SI_PARAM_LINEAR_CENTROID;
1136 else
1137 return SI_PARAM_LINEAR_CENTER;
1138 break;
1139 case TGSI_INTERPOLATE_COLOR:
1140 case TGSI_INTERPOLATE_PERSPECTIVE:
1141 if (location == TGSI_INTERPOLATE_LOC_SAMPLE)
1142 return SI_PARAM_PERSP_SAMPLE;
1143 else if (location == TGSI_INTERPOLATE_LOC_CENTROID)
1144 return SI_PARAM_PERSP_CENTROID;
1145 else
1146 return SI_PARAM_PERSP_CENTER;
1147 break;
1148 default:
1149 fprintf(stderr, "Warning: Unhandled interpolation mode.\n");
1150 return -1;
1151 }
1152 }
1153
1154 /**
1155 * Interpolate a fragment shader input.
1156 *
1157 * @param ctx context
1158 * @param input_index index of the input in hardware
1159 * @param semantic_name TGSI_SEMANTIC_*
1160 * @param semantic_index semantic index
1161 * @param num_interp_inputs number of all interpolated inputs (= BCOLOR offset)
1162 * @param colors_read_mask color components read (4 bits for each color, 8 bits in total)
1163 * @param interp_param interpolation weights (i,j)
1164 * @param prim_mask SI_PARAM_PRIM_MASK
1165 * @param face SI_PARAM_FRONT_FACE
1166 * @param result the return value (4 components)
1167 */
1168 static void interp_fs_input(struct si_shader_context *ctx,
1169 unsigned input_index,
1170 unsigned semantic_name,
1171 unsigned semantic_index,
1172 unsigned num_interp_inputs,
1173 unsigned colors_read_mask,
1174 LLVMValueRef interp_param,
1175 LLVMValueRef prim_mask,
1176 LLVMValueRef face,
1177 LLVMValueRef result[4])
1178 {
1179 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
1180 struct lp_build_context *base = &bld_base->base;
1181 struct lp_build_context *uint = &bld_base->uint_bld;
1182 struct gallivm_state *gallivm = base->gallivm;
1183 LLVMValueRef attr_number;
1184 LLVMValueRef i, j;
1185
1186 unsigned chan;
1187
1188 /* fs.constant returns the param from the middle vertex, so it's not
1189 * really useful for flat shading. It's meant to be used for custom
1190 * interpolation (but the intrinsic can't fetch from the other two
1191 * vertices).
1192 *
1193 * Luckily, it doesn't matter, because we rely on the FLAT_SHADE state
1194 * to do the right thing. The only reason we use fs.constant is that
1195 * fs.interp cannot be used on integers, because they can be equal
1196 * to NaN.
1197 *
1198 * When interp is false we will use fs.constant or for newer llvm,
1199 * amdgcn.interp.mov.
1200 */
1201 bool interp = interp_param != NULL;
1202
1203 attr_number = lp_build_const_int32(gallivm, input_index);
1204
1205 if (interp) {
1206 interp_param = LLVMBuildBitCast(gallivm->builder, interp_param,
1207 LLVMVectorType(ctx->f32, 2), "");
1208
1209 i = LLVMBuildExtractElement(gallivm->builder, interp_param,
1210 uint->zero, "");
1211 j = LLVMBuildExtractElement(gallivm->builder, interp_param,
1212 uint->one, "");
1213 }
1214
1215 if (semantic_name == TGSI_SEMANTIC_COLOR &&
1216 ctx->shader->key.part.ps.prolog.color_two_side) {
1217 LLVMValueRef is_face_positive;
1218 LLVMValueRef back_attr_number;
1219
1220 /* If BCOLOR0 is used, BCOLOR1 is at offset "num_inputs + 1",
1221 * otherwise it's at offset "num_inputs".
1222 */
1223 unsigned back_attr_offset = num_interp_inputs;
1224 if (semantic_index == 1 && colors_read_mask & 0xf)
1225 back_attr_offset += 1;
1226
1227 back_attr_number = lp_build_const_int32(gallivm, back_attr_offset);
1228
1229 is_face_positive = LLVMBuildICmp(gallivm->builder, LLVMIntNE,
1230 face, uint->zero, "");
1231
1232 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1233 LLVMValueRef llvm_chan = lp_build_const_int32(gallivm, chan);
1234 LLVMValueRef front, back;
1235
1236 if (interp) {
1237 front = ac_build_fs_interp(&ctx->ac, llvm_chan,
1238 attr_number, prim_mask,
1239 i, j);
1240 back = ac_build_fs_interp(&ctx->ac, llvm_chan,
1241 back_attr_number, prim_mask,
1242 i, j);
1243 } else {
1244 front = ac_build_fs_interp_mov(&ctx->ac,
1245 lp_build_const_int32(gallivm, 2), /* P0 */
1246 llvm_chan, attr_number, prim_mask);
1247 back = ac_build_fs_interp_mov(&ctx->ac,
1248 lp_build_const_int32(gallivm, 2), /* P0 */
1249 llvm_chan, back_attr_number, prim_mask);
1250 }
1251
1252 result[chan] = LLVMBuildSelect(gallivm->builder,
1253 is_face_positive,
1254 front,
1255 back,
1256 "");
1257 }
1258 } else if (semantic_name == TGSI_SEMANTIC_FOG) {
1259 if (interp) {
1260 result[0] = ac_build_fs_interp(&ctx->ac, uint->zero,
1261 attr_number, prim_mask, i, j);
1262 } else {
1263 result[0] = ac_build_fs_interp_mov(&ctx->ac, uint->zero,
1264 lp_build_const_int32(gallivm, 2), /* P0 */
1265 attr_number, prim_mask);
1266 }
1267 result[1] =
1268 result[2] = lp_build_const_float(gallivm, 0.0f);
1269 result[3] = lp_build_const_float(gallivm, 1.0f);
1270 } else {
1271 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1272 LLVMValueRef llvm_chan = lp_build_const_int32(gallivm, chan);
1273
1274 if (interp) {
1275 result[chan] = ac_build_fs_interp(&ctx->ac,
1276 llvm_chan, attr_number, prim_mask, i, j);
1277 } else {
1278 result[chan] = ac_build_fs_interp_mov(&ctx->ac,
1279 lp_build_const_int32(gallivm, 2), /* P0 */
1280 llvm_chan, attr_number, prim_mask);
1281 }
1282 }
1283 }
1284 }
1285
1286 static void declare_input_fs(
1287 struct si_shader_context *ctx,
1288 unsigned input_index,
1289 const struct tgsi_full_declaration *decl,
1290 LLVMValueRef out[4])
1291 {
1292 struct lp_build_context *base = &ctx->bld_base.base;
1293 struct si_shader *shader = ctx->shader;
1294 LLVMValueRef main_fn = ctx->main_fn;
1295 LLVMValueRef interp_param = NULL;
1296 int interp_param_idx;
1297
1298 /* Get colors from input VGPRs (set by the prolog). */
1299 if (decl->Semantic.Name == TGSI_SEMANTIC_COLOR) {
1300 unsigned i = decl->Semantic.Index;
1301 unsigned colors_read = shader->selector->info.colors_read;
1302 unsigned mask = colors_read >> (i * 4);
1303 unsigned offset = SI_PARAM_POS_FIXED_PT + 1 +
1304 (i ? util_bitcount(colors_read & 0xf) : 0);
1305
1306 out[0] = mask & 0x1 ? LLVMGetParam(main_fn, offset++) : base->undef;
1307 out[1] = mask & 0x2 ? LLVMGetParam(main_fn, offset++) : base->undef;
1308 out[2] = mask & 0x4 ? LLVMGetParam(main_fn, offset++) : base->undef;
1309 out[3] = mask & 0x8 ? LLVMGetParam(main_fn, offset++) : base->undef;
1310 return;
1311 }
1312
1313 interp_param_idx = lookup_interp_param_index(decl->Interp.Interpolate,
1314 decl->Interp.Location);
1315 if (interp_param_idx == -1)
1316 return;
1317 else if (interp_param_idx) {
1318 interp_param = LLVMGetParam(ctx->main_fn, interp_param_idx);
1319 }
1320
1321 if (decl->Semantic.Name == TGSI_SEMANTIC_COLOR &&
1322 decl->Interp.Interpolate == TGSI_INTERPOLATE_COLOR &&
1323 ctx->shader->key.part.ps.prolog.flatshade_colors)
1324 interp_param = NULL; /* load the constant color */
1325
1326 interp_fs_input(ctx, input_index, decl->Semantic.Name,
1327 decl->Semantic.Index, shader->selector->info.num_inputs,
1328 shader->selector->info.colors_read, interp_param,
1329 LLVMGetParam(main_fn, SI_PARAM_PRIM_MASK),
1330 LLVMGetParam(main_fn, SI_PARAM_FRONT_FACE),
1331 &out[0]);
1332 }
1333
1334 static LLVMValueRef get_sample_id(struct si_shader_context *ctx)
1335 {
1336 return unpack_param(ctx, SI_PARAM_ANCILLARY, 8, 4);
1337 }
1338
1339
1340 /**
1341 * Load a dword from a constant buffer.
1342 */
1343 static LLVMValueRef buffer_load_const(struct si_shader_context *ctx,
1344 LLVMValueRef resource,
1345 LLVMValueRef offset)
1346 {
1347 LLVMBuilderRef builder = ctx->gallivm.builder;
1348 LLVMValueRef args[2] = {resource, offset};
1349
1350 return lp_build_intrinsic(builder, "llvm.SI.load.const", ctx->f32, args, 2,
1351 LP_FUNC_ATTR_READNONE |
1352 LP_FUNC_ATTR_LEGACY);
1353 }
1354
1355 static LLVMValueRef load_sample_position(struct si_shader_context *ctx, LLVMValueRef sample_id)
1356 {
1357 struct lp_build_context *uint_bld = &ctx->bld_base.uint_bld;
1358 struct gallivm_state *gallivm = &ctx->gallivm;
1359 LLVMBuilderRef builder = gallivm->builder;
1360 LLVMValueRef desc = LLVMGetParam(ctx->main_fn, SI_PARAM_RW_BUFFERS);
1361 LLVMValueRef buf_index = lp_build_const_int32(gallivm, SI_PS_CONST_SAMPLE_POSITIONS);
1362 LLVMValueRef resource = ac_build_indexed_load_const(&ctx->ac, desc, buf_index);
1363
1364 /* offset = sample_id * 8 (8 = 2 floats containing samplepos.xy) */
1365 LLVMValueRef offset0 = lp_build_mul_imm(uint_bld, sample_id, 8);
1366 LLVMValueRef offset1 = LLVMBuildAdd(builder, offset0, lp_build_const_int32(gallivm, 4), "");
1367
1368 LLVMValueRef pos[4] = {
1369 buffer_load_const(ctx, resource, offset0),
1370 buffer_load_const(ctx, resource, offset1),
1371 lp_build_const_float(gallivm, 0),
1372 lp_build_const_float(gallivm, 0)
1373 };
1374
1375 return lp_build_gather_values(gallivm, pos, 4);
1376 }
1377
1378 static void declare_system_value(struct si_shader_context *ctx,
1379 unsigned index,
1380 const struct tgsi_full_declaration *decl)
1381 {
1382 struct lp_build_context *bld = &ctx->bld_base.base;
1383 struct gallivm_state *gallivm = &ctx->gallivm;
1384 LLVMValueRef value = 0;
1385
1386 switch (decl->Semantic.Name) {
1387 case TGSI_SEMANTIC_INSTANCEID:
1388 value = LLVMGetParam(ctx->main_fn,
1389 ctx->param_instance_id);
1390 break;
1391
1392 case TGSI_SEMANTIC_VERTEXID:
1393 value = LLVMBuildAdd(gallivm->builder,
1394 LLVMGetParam(ctx->main_fn,
1395 ctx->param_vertex_id),
1396 LLVMGetParam(ctx->main_fn,
1397 SI_PARAM_BASE_VERTEX), "");
1398 break;
1399
1400 case TGSI_SEMANTIC_VERTEXID_NOBASE:
1401 value = LLVMGetParam(ctx->main_fn,
1402 ctx->param_vertex_id);
1403 break;
1404
1405 case TGSI_SEMANTIC_BASEVERTEX:
1406 value = LLVMGetParam(ctx->main_fn,
1407 SI_PARAM_BASE_VERTEX);
1408 break;
1409
1410 case TGSI_SEMANTIC_BASEINSTANCE:
1411 value = LLVMGetParam(ctx->main_fn,
1412 SI_PARAM_START_INSTANCE);
1413 break;
1414
1415 case TGSI_SEMANTIC_DRAWID:
1416 value = LLVMGetParam(ctx->main_fn,
1417 SI_PARAM_DRAWID);
1418 break;
1419
1420 case TGSI_SEMANTIC_INVOCATIONID:
1421 if (ctx->type == PIPE_SHADER_TESS_CTRL)
1422 value = unpack_param(ctx, SI_PARAM_REL_IDS, 8, 5);
1423 else if (ctx->type == PIPE_SHADER_GEOMETRY)
1424 value = LLVMGetParam(ctx->main_fn,
1425 SI_PARAM_GS_INSTANCE_ID);
1426 else
1427 assert(!"INVOCATIONID not implemented");
1428 break;
1429
1430 case TGSI_SEMANTIC_POSITION:
1431 {
1432 LLVMValueRef pos[4] = {
1433 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_X_FLOAT),
1434 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Y_FLOAT),
1435 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Z_FLOAT),
1436 lp_build_emit_llvm_unary(&ctx->bld_base, TGSI_OPCODE_RCP,
1437 LLVMGetParam(ctx->main_fn,
1438 SI_PARAM_POS_W_FLOAT)),
1439 };
1440 value = lp_build_gather_values(gallivm, pos, 4);
1441 break;
1442 }
1443
1444 case TGSI_SEMANTIC_FACE:
1445 value = LLVMGetParam(ctx->main_fn, SI_PARAM_FRONT_FACE);
1446 break;
1447
1448 case TGSI_SEMANTIC_SAMPLEID:
1449 value = get_sample_id(ctx);
1450 break;
1451
1452 case TGSI_SEMANTIC_SAMPLEPOS: {
1453 LLVMValueRef pos[4] = {
1454 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_X_FLOAT),
1455 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Y_FLOAT),
1456 lp_build_const_float(gallivm, 0),
1457 lp_build_const_float(gallivm, 0)
1458 };
1459 pos[0] = lp_build_emit_llvm_unary(&ctx->bld_base,
1460 TGSI_OPCODE_FRC, pos[0]);
1461 pos[1] = lp_build_emit_llvm_unary(&ctx->bld_base,
1462 TGSI_OPCODE_FRC, pos[1]);
1463 value = lp_build_gather_values(gallivm, pos, 4);
1464 break;
1465 }
1466
1467 case TGSI_SEMANTIC_SAMPLEMASK:
1468 /* This can only occur with the OpenGL Core profile, which
1469 * doesn't support smoothing.
1470 */
1471 value = LLVMGetParam(ctx->main_fn, SI_PARAM_SAMPLE_COVERAGE);
1472 break;
1473
1474 case TGSI_SEMANTIC_TESSCOORD:
1475 {
1476 LLVMValueRef coord[4] = {
1477 LLVMGetParam(ctx->main_fn, ctx->param_tes_u),
1478 LLVMGetParam(ctx->main_fn, ctx->param_tes_v),
1479 bld->zero,
1480 bld->zero
1481 };
1482
1483 /* For triangles, the vector should be (u, v, 1-u-v). */
1484 if (ctx->shader->selector->info.properties[TGSI_PROPERTY_TES_PRIM_MODE] ==
1485 PIPE_PRIM_TRIANGLES)
1486 coord[2] = lp_build_sub(bld, bld->one,
1487 lp_build_add(bld, coord[0], coord[1]));
1488
1489 value = lp_build_gather_values(gallivm, coord, 4);
1490 break;
1491 }
1492
1493 case TGSI_SEMANTIC_VERTICESIN:
1494 if (ctx->type == PIPE_SHADER_TESS_CTRL)
1495 value = unpack_param(ctx, SI_PARAM_TCS_OUT_LAYOUT, 26, 6);
1496 else if (ctx->type == PIPE_SHADER_TESS_EVAL)
1497 value = unpack_param(ctx, SI_PARAM_TCS_OFFCHIP_LAYOUT, 9, 7);
1498 else
1499 assert(!"invalid shader stage for TGSI_SEMANTIC_VERTICESIN");
1500 break;
1501
1502 case TGSI_SEMANTIC_TESSINNER:
1503 case TGSI_SEMANTIC_TESSOUTER:
1504 {
1505 LLVMValueRef rw_buffers, buffer, base, addr;
1506 int param = si_shader_io_get_unique_index(decl->Semantic.Name, 0);
1507
1508 rw_buffers = LLVMGetParam(ctx->main_fn,
1509 SI_PARAM_RW_BUFFERS);
1510 buffer = ac_build_indexed_load_const(&ctx->ac, rw_buffers,
1511 lp_build_const_int32(gallivm, SI_HS_RING_TESS_OFFCHIP));
1512
1513 base = LLVMGetParam(ctx->main_fn, ctx->param_oc_lds);
1514 addr = get_tcs_tes_buffer_address(ctx, get_rel_patch_id(ctx), NULL,
1515 lp_build_const_int32(gallivm, param));
1516
1517 value = buffer_load(&ctx->bld_base, TGSI_TYPE_FLOAT,
1518 ~0, buffer, base, addr, true);
1519
1520 break;
1521 }
1522
1523 case TGSI_SEMANTIC_DEFAULT_TESSOUTER_SI:
1524 case TGSI_SEMANTIC_DEFAULT_TESSINNER_SI:
1525 {
1526 LLVMValueRef buf, slot, val[4];
1527 int i, offset;
1528
1529 slot = lp_build_const_int32(gallivm, SI_HS_CONST_DEFAULT_TESS_LEVELS);
1530 buf = LLVMGetParam(ctx->main_fn, SI_PARAM_RW_BUFFERS);
1531 buf = ac_build_indexed_load_const(&ctx->ac, buf, slot);
1532 offset = decl->Semantic.Name == TGSI_SEMANTIC_DEFAULT_TESSINNER_SI ? 4 : 0;
1533
1534 for (i = 0; i < 4; i++)
1535 val[i] = buffer_load_const(ctx, buf,
1536 lp_build_const_int32(gallivm, (offset + i) * 4));
1537 value = lp_build_gather_values(gallivm, val, 4);
1538 break;
1539 }
1540
1541 case TGSI_SEMANTIC_PRIMID:
1542 value = get_primitive_id(&ctx->bld_base, 0);
1543 break;
1544
1545 case TGSI_SEMANTIC_GRID_SIZE:
1546 value = LLVMGetParam(ctx->main_fn, SI_PARAM_GRID_SIZE);
1547 break;
1548
1549 case TGSI_SEMANTIC_BLOCK_SIZE:
1550 {
1551 LLVMValueRef values[3];
1552 unsigned i;
1553 unsigned *properties = ctx->shader->selector->info.properties;
1554
1555 if (properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH] != 0) {
1556 unsigned sizes[3] = {
1557 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH],
1558 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT],
1559 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH]
1560 };
1561
1562 for (i = 0; i < 3; ++i)
1563 values[i] = lp_build_const_int32(gallivm, sizes[i]);
1564
1565 value = lp_build_gather_values(gallivm, values, 3);
1566 } else {
1567 value = LLVMGetParam(ctx->main_fn, SI_PARAM_BLOCK_SIZE);
1568 }
1569 break;
1570 }
1571
1572 case TGSI_SEMANTIC_BLOCK_ID:
1573 value = LLVMGetParam(ctx->main_fn, SI_PARAM_BLOCK_ID);
1574 break;
1575
1576 case TGSI_SEMANTIC_THREAD_ID:
1577 value = LLVMGetParam(ctx->main_fn, SI_PARAM_THREAD_ID);
1578 break;
1579
1580 case TGSI_SEMANTIC_HELPER_INVOCATION:
1581 if (HAVE_LLVM >= 0x0309) {
1582 value = lp_build_intrinsic(gallivm->builder,
1583 "llvm.amdgcn.ps.live",
1584 ctx->i1, NULL, 0,
1585 LP_FUNC_ATTR_READNONE);
1586 value = LLVMBuildNot(gallivm->builder, value, "");
1587 value = LLVMBuildSExt(gallivm->builder, value, ctx->i32, "");
1588 } else {
1589 assert(!"TGSI_SEMANTIC_HELPER_INVOCATION unsupported");
1590 return;
1591 }
1592 break;
1593
1594 default:
1595 assert(!"unknown system value");
1596 return;
1597 }
1598
1599 ctx->system_values[index] = value;
1600 }
1601
1602 static void declare_compute_memory(struct si_shader_context *ctx,
1603 const struct tgsi_full_declaration *decl)
1604 {
1605 struct si_shader_selector *sel = ctx->shader->selector;
1606 struct gallivm_state *gallivm = &ctx->gallivm;
1607
1608 LLVMTypeRef i8p = LLVMPointerType(ctx->i8, LOCAL_ADDR_SPACE);
1609 LLVMValueRef var;
1610
1611 assert(decl->Declaration.MemType == TGSI_MEMORY_TYPE_SHARED);
1612 assert(decl->Range.First == decl->Range.Last);
1613 assert(!ctx->shared_memory);
1614
1615 var = LLVMAddGlobalInAddressSpace(gallivm->module,
1616 LLVMArrayType(ctx->i8, sel->local_size),
1617 "compute_lds",
1618 LOCAL_ADDR_SPACE);
1619 LLVMSetAlignment(var, 4);
1620
1621 ctx->shared_memory = LLVMBuildBitCast(gallivm->builder, var, i8p, "");
1622 }
1623
1624 static LLVMValueRef load_const_buffer_desc(struct si_shader_context *ctx, int i)
1625 {
1626 LLVMValueRef list_ptr = LLVMGetParam(ctx->main_fn,
1627 SI_PARAM_CONST_BUFFERS);
1628
1629 return ac_build_indexed_load_const(&ctx->ac, list_ptr,
1630 LLVMConstInt(ctx->i32, i, 0));
1631 }
1632
1633 static LLVMValueRef fetch_constant(
1634 struct lp_build_tgsi_context *bld_base,
1635 const struct tgsi_full_src_register *reg,
1636 enum tgsi_opcode_type type,
1637 unsigned swizzle)
1638 {
1639 struct si_shader_context *ctx = si_shader_context(bld_base);
1640 struct lp_build_context *base = &bld_base->base;
1641 const struct tgsi_ind_register *ireg = &reg->Indirect;
1642 unsigned buf, idx;
1643
1644 LLVMValueRef addr, bufp;
1645 LLVMValueRef result;
1646
1647 if (swizzle == LP_CHAN_ALL) {
1648 unsigned chan;
1649 LLVMValueRef values[4];
1650 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan)
1651 values[chan] = fetch_constant(bld_base, reg, type, chan);
1652
1653 return lp_build_gather_values(bld_base->base.gallivm, values, 4);
1654 }
1655
1656 buf = reg->Register.Dimension ? reg->Dimension.Index : 0;
1657 idx = reg->Register.Index * 4 + swizzle;
1658
1659 if (reg->Register.Dimension && reg->Dimension.Indirect) {
1660 LLVMValueRef ptr = LLVMGetParam(ctx->main_fn, SI_PARAM_CONST_BUFFERS);
1661 LLVMValueRef index;
1662 index = get_bounded_indirect_index(ctx, &reg->DimIndirect,
1663 reg->Dimension.Index,
1664 SI_NUM_CONST_BUFFERS);
1665 bufp = ac_build_indexed_load_const(&ctx->ac, ptr, index);
1666 } else
1667 bufp = load_const_buffer_desc(ctx, buf);
1668
1669 if (reg->Register.Indirect) {
1670 addr = ctx->addrs[ireg->Index][ireg->Swizzle];
1671 addr = LLVMBuildLoad(base->gallivm->builder, addr, "load addr reg");
1672 addr = lp_build_mul_imm(&bld_base->uint_bld, addr, 16);
1673 addr = lp_build_add(&bld_base->uint_bld, addr,
1674 lp_build_const_int32(base->gallivm, idx * 4));
1675 } else {
1676 addr = LLVMConstInt(ctx->i32, idx * 4, 0);
1677 }
1678
1679 result = buffer_load_const(ctx, bufp, addr);
1680
1681 if (!tgsi_type_is_64bit(type))
1682 result = bitcast(bld_base, type, result);
1683 else {
1684 LLVMValueRef addr2, result2;
1685
1686 addr2 = lp_build_add(&bld_base->uint_bld, addr,
1687 LLVMConstInt(ctx->i32, 4, 0));
1688 result2 = buffer_load_const(ctx, bufp, addr2);
1689
1690 result = si_llvm_emit_fetch_64bit(bld_base, type,
1691 result, result2);
1692 }
1693 return result;
1694 }
1695
1696 /* Upper 16 bits must be zero. */
1697 static LLVMValueRef si_llvm_pack_two_int16(struct gallivm_state *gallivm,
1698 LLVMValueRef val[2])
1699 {
1700 return LLVMBuildOr(gallivm->builder, val[0],
1701 LLVMBuildShl(gallivm->builder, val[1],
1702 lp_build_const_int32(gallivm, 16),
1703 ""), "");
1704 }
1705
1706 /* Upper 16 bits are ignored and will be dropped. */
1707 static LLVMValueRef si_llvm_pack_two_int32_as_int16(struct gallivm_state *gallivm,
1708 LLVMValueRef val[2])
1709 {
1710 LLVMValueRef v[2] = {
1711 LLVMBuildAnd(gallivm->builder, val[0],
1712 lp_build_const_int32(gallivm, 0xffff), ""),
1713 val[1],
1714 };
1715 return si_llvm_pack_two_int16(gallivm, v);
1716 }
1717
1718 /* Initialize arguments for the shader export intrinsic */
1719 static void si_llvm_init_export_args(struct lp_build_tgsi_context *bld_base,
1720 LLVMValueRef *values,
1721 unsigned target,
1722 struct ac_export_args *args)
1723 {
1724 struct si_shader_context *ctx = si_shader_context(bld_base);
1725 struct lp_build_context *base = &bld_base->base;
1726 struct gallivm_state *gallivm = base->gallivm;
1727 LLVMBuilderRef builder = base->gallivm->builder;
1728 LLVMValueRef val[4];
1729 unsigned spi_shader_col_format = V_028714_SPI_SHADER_32_ABGR;
1730 unsigned chan;
1731 bool is_int8, is_int10;
1732
1733 /* Default is 0xf. Adjusted below depending on the format. */
1734 args->enabled_channels = 0xf; /* writemask */
1735
1736 /* Specify whether the EXEC mask represents the valid mask */
1737 args->valid_mask = 0;
1738
1739 /* Specify whether this is the last export */
1740 args->done = 0;
1741
1742 /* Specify the target we are exporting */
1743 args->target = target;
1744
1745 if (ctx->type == PIPE_SHADER_FRAGMENT) {
1746 const struct si_shader_key *key = &ctx->shader->key;
1747 unsigned col_formats = key->part.ps.epilog.spi_shader_col_format;
1748 int cbuf = target - V_008DFC_SQ_EXP_MRT;
1749
1750 assert(cbuf >= 0 && cbuf < 8);
1751 spi_shader_col_format = (col_formats >> (cbuf * 4)) & 0xf;
1752 is_int8 = (key->part.ps.epilog.color_is_int8 >> cbuf) & 0x1;
1753 is_int10 = (key->part.ps.epilog.color_is_int10 >> cbuf) & 0x1;
1754 }
1755
1756 args->compr = false;
1757 args->out[0] = base->undef;
1758 args->out[1] = base->undef;
1759 args->out[2] = base->undef;
1760 args->out[3] = base->undef;
1761
1762 switch (spi_shader_col_format) {
1763 case V_028714_SPI_SHADER_ZERO:
1764 args->enabled_channels = 0; /* writemask */
1765 args->target = V_008DFC_SQ_EXP_NULL;
1766 break;
1767
1768 case V_028714_SPI_SHADER_32_R:
1769 args->enabled_channels = 1; /* writemask */
1770 args->out[0] = values[0];
1771 break;
1772
1773 case V_028714_SPI_SHADER_32_GR:
1774 args->enabled_channels = 0x3; /* writemask */
1775 args->out[0] = values[0];
1776 args->out[1] = values[1];
1777 break;
1778
1779 case V_028714_SPI_SHADER_32_AR:
1780 args->enabled_channels = 0x9; /* writemask */
1781 args->out[0] = values[0];
1782 args->out[3] = values[3];
1783 break;
1784
1785 case V_028714_SPI_SHADER_FP16_ABGR:
1786 args->compr = 1; /* COMPR flag */
1787
1788 for (chan = 0; chan < 2; chan++) {
1789 LLVMValueRef pack_args[2] = {
1790 values[2 * chan],
1791 values[2 * chan + 1]
1792 };
1793 LLVMValueRef packed;
1794
1795 packed = ac_build_cvt_pkrtz_f16(&ctx->ac, pack_args);
1796 args->out[chan] =
1797 LLVMBuildBitCast(base->gallivm->builder,
1798 packed, ctx->f32, "");
1799 }
1800 break;
1801
1802 case V_028714_SPI_SHADER_UNORM16_ABGR:
1803 for (chan = 0; chan < 4; chan++) {
1804 val[chan] = ac_build_clamp(&ctx->ac, values[chan]);
1805 val[chan] = LLVMBuildFMul(builder, val[chan],
1806 lp_build_const_float(gallivm, 65535), "");
1807 val[chan] = LLVMBuildFAdd(builder, val[chan],
1808 lp_build_const_float(gallivm, 0.5), "");
1809 val[chan] = LLVMBuildFPToUI(builder, val[chan],
1810 ctx->i32, "");
1811 }
1812
1813 args->compr = 1; /* COMPR flag */
1814 args->out[0] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1815 si_llvm_pack_two_int16(gallivm, val));
1816 args->out[1] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1817 si_llvm_pack_two_int16(gallivm, val+2));
1818 break;
1819
1820 case V_028714_SPI_SHADER_SNORM16_ABGR:
1821 for (chan = 0; chan < 4; chan++) {
1822 /* Clamp between [-1, 1]. */
1823 val[chan] = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MIN,
1824 values[chan],
1825 lp_build_const_float(gallivm, 1));
1826 val[chan] = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MAX,
1827 val[chan],
1828 lp_build_const_float(gallivm, -1));
1829 /* Convert to a signed integer in [-32767, 32767]. */
1830 val[chan] = LLVMBuildFMul(builder, val[chan],
1831 lp_build_const_float(gallivm, 32767), "");
1832 /* If positive, add 0.5, else add -0.5. */
1833 val[chan] = LLVMBuildFAdd(builder, val[chan],
1834 LLVMBuildSelect(builder,
1835 LLVMBuildFCmp(builder, LLVMRealOGE,
1836 val[chan], base->zero, ""),
1837 lp_build_const_float(gallivm, 0.5),
1838 lp_build_const_float(gallivm, -0.5), ""), "");
1839 val[chan] = LLVMBuildFPToSI(builder, val[chan], ctx->i32, "");
1840 }
1841
1842 args->compr = 1; /* COMPR flag */
1843 args->out[0] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1844 si_llvm_pack_two_int32_as_int16(gallivm, val));
1845 args->out[1] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1846 si_llvm_pack_two_int32_as_int16(gallivm, val+2));
1847 break;
1848
1849 case V_028714_SPI_SHADER_UINT16_ABGR: {
1850 LLVMValueRef max_rgb = lp_build_const_int32(gallivm,
1851 is_int8 ? 255 : is_int10 ? 1023 : 65535);
1852 LLVMValueRef max_alpha =
1853 !is_int10 ? max_rgb : lp_build_const_int32(gallivm, 3);
1854
1855 /* Clamp. */
1856 for (chan = 0; chan < 4; chan++) {
1857 val[chan] = bitcast(bld_base, TGSI_TYPE_UNSIGNED, values[chan]);
1858 val[chan] = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_UMIN,
1859 val[chan],
1860 chan == 3 ? max_alpha : max_rgb);
1861 }
1862
1863 args->compr = 1; /* COMPR flag */
1864 args->out[0] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1865 si_llvm_pack_two_int16(gallivm, val));
1866 args->out[1] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1867 si_llvm_pack_two_int16(gallivm, val+2));
1868 break;
1869 }
1870
1871 case V_028714_SPI_SHADER_SINT16_ABGR: {
1872 LLVMValueRef max_rgb = lp_build_const_int32(gallivm,
1873 is_int8 ? 127 : is_int10 ? 511 : 32767);
1874 LLVMValueRef min_rgb = lp_build_const_int32(gallivm,
1875 is_int8 ? -128 : is_int10 ? -512 : -32768);
1876 LLVMValueRef max_alpha =
1877 !is_int10 ? max_rgb : lp_build_const_int32(gallivm, 1);
1878 LLVMValueRef min_alpha =
1879 !is_int10 ? min_rgb : lp_build_const_int32(gallivm, -2);
1880
1881 /* Clamp. */
1882 for (chan = 0; chan < 4; chan++) {
1883 val[chan] = bitcast(bld_base, TGSI_TYPE_UNSIGNED, values[chan]);
1884 val[chan] = lp_build_emit_llvm_binary(bld_base,
1885 TGSI_OPCODE_IMIN,
1886 val[chan], chan == 3 ? max_alpha : max_rgb);
1887 val[chan] = lp_build_emit_llvm_binary(bld_base,
1888 TGSI_OPCODE_IMAX,
1889 val[chan], chan == 3 ? min_alpha : min_rgb);
1890 }
1891
1892 args->compr = 1; /* COMPR flag */
1893 args->out[0] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1894 si_llvm_pack_two_int32_as_int16(gallivm, val));
1895 args->out[1] = bitcast(bld_base, TGSI_TYPE_FLOAT,
1896 si_llvm_pack_two_int32_as_int16(gallivm, val+2));
1897 break;
1898 }
1899
1900 case V_028714_SPI_SHADER_32_ABGR:
1901 memcpy(&args->out[0], values, sizeof(values[0]) * 4);
1902 break;
1903 }
1904 }
1905
1906 static void si_alpha_test(struct lp_build_tgsi_context *bld_base,
1907 LLVMValueRef alpha)
1908 {
1909 struct si_shader_context *ctx = si_shader_context(bld_base);
1910 struct gallivm_state *gallivm = bld_base->base.gallivm;
1911
1912 if (ctx->shader->key.part.ps.epilog.alpha_func != PIPE_FUNC_NEVER) {
1913 LLVMValueRef alpha_ref = LLVMGetParam(ctx->main_fn,
1914 SI_PARAM_ALPHA_REF);
1915
1916 LLVMValueRef alpha_pass =
1917 lp_build_cmp(&bld_base->base,
1918 ctx->shader->key.part.ps.epilog.alpha_func,
1919 alpha, alpha_ref);
1920 LLVMValueRef arg =
1921 lp_build_select(&bld_base->base,
1922 alpha_pass,
1923 lp_build_const_float(gallivm, 1.0f),
1924 lp_build_const_float(gallivm, -1.0f));
1925
1926 ac_build_kill(&ctx->ac, arg);
1927 } else {
1928 ac_build_kill(&ctx->ac, NULL);
1929 }
1930 }
1931
1932 static LLVMValueRef si_scale_alpha_by_sample_mask(struct lp_build_tgsi_context *bld_base,
1933 LLVMValueRef alpha,
1934 unsigned samplemask_param)
1935 {
1936 struct si_shader_context *ctx = si_shader_context(bld_base);
1937 struct gallivm_state *gallivm = bld_base->base.gallivm;
1938 LLVMValueRef coverage;
1939
1940 /* alpha = alpha * popcount(coverage) / SI_NUM_SMOOTH_AA_SAMPLES */
1941 coverage = LLVMGetParam(ctx->main_fn,
1942 samplemask_param);
1943 coverage = bitcast(bld_base, TGSI_TYPE_SIGNED, coverage);
1944
1945 coverage = lp_build_intrinsic(gallivm->builder, "llvm.ctpop.i32",
1946 ctx->i32,
1947 &coverage, 1, LP_FUNC_ATTR_READNONE);
1948
1949 coverage = LLVMBuildUIToFP(gallivm->builder, coverage,
1950 ctx->f32, "");
1951
1952 coverage = LLVMBuildFMul(gallivm->builder, coverage,
1953 lp_build_const_float(gallivm,
1954 1.0 / SI_NUM_SMOOTH_AA_SAMPLES), "");
1955
1956 return LLVMBuildFMul(gallivm->builder, alpha, coverage, "");
1957 }
1958
1959 static void si_llvm_emit_clipvertex(struct lp_build_tgsi_context *bld_base,
1960 struct ac_export_args *pos, LLVMValueRef *out_elts)
1961 {
1962 struct si_shader_context *ctx = si_shader_context(bld_base);
1963 struct lp_build_context *base = &bld_base->base;
1964 unsigned reg_index;
1965 unsigned chan;
1966 unsigned const_chan;
1967 LLVMValueRef base_elt;
1968 LLVMValueRef ptr = LLVMGetParam(ctx->main_fn, SI_PARAM_RW_BUFFERS);
1969 LLVMValueRef constbuf_index = lp_build_const_int32(base->gallivm,
1970 SI_VS_CONST_CLIP_PLANES);
1971 LLVMValueRef const_resource = ac_build_indexed_load_const(&ctx->ac, ptr, constbuf_index);
1972
1973 for (reg_index = 0; reg_index < 2; reg_index ++) {
1974 struct ac_export_args *args = &pos[2 + reg_index];
1975
1976 args->out[0] =
1977 args->out[1] =
1978 args->out[2] =
1979 args->out[3] = lp_build_const_float(base->gallivm, 0.0f);
1980
1981 /* Compute dot products of position and user clip plane vectors */
1982 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1983 for (const_chan = 0; const_chan < TGSI_NUM_CHANNELS; const_chan++) {
1984 LLVMValueRef addr =
1985 LLVMConstInt(ctx->i32, ((reg_index * 4 + chan) * 4 +
1986 const_chan) * 4, 0);
1987 base_elt = buffer_load_const(ctx, const_resource,
1988 addr);
1989 args->out[chan] =
1990 lp_build_add(base, args->out[chan],
1991 lp_build_mul(base, base_elt,
1992 out_elts[const_chan]));
1993 }
1994 }
1995
1996 args->enabled_channels = 0xf;
1997 args->valid_mask = 0;
1998 args->done = 0;
1999 args->target = V_008DFC_SQ_EXP_POS + 2 + reg_index;
2000 args->compr = 0;
2001 }
2002 }
2003
2004 static void si_dump_streamout(struct pipe_stream_output_info *so)
2005 {
2006 unsigned i;
2007
2008 if (so->num_outputs)
2009 fprintf(stderr, "STREAMOUT\n");
2010
2011 for (i = 0; i < so->num_outputs; i++) {
2012 unsigned mask = ((1 << so->output[i].num_components) - 1) <<
2013 so->output[i].start_component;
2014 fprintf(stderr, " %i: BUF%i[%i..%i] <- OUT[%i].%s%s%s%s\n",
2015 i, so->output[i].output_buffer,
2016 so->output[i].dst_offset, so->output[i].dst_offset + so->output[i].num_components - 1,
2017 so->output[i].register_index,
2018 mask & 1 ? "x" : "",
2019 mask & 2 ? "y" : "",
2020 mask & 4 ? "z" : "",
2021 mask & 8 ? "w" : "");
2022 }
2023 }
2024
2025 static void emit_streamout_output(struct si_shader_context *ctx,
2026 LLVMValueRef const *so_buffers,
2027 LLVMValueRef const *so_write_offsets,
2028 struct pipe_stream_output *stream_out,
2029 struct si_shader_output_values *shader_out)
2030 {
2031 struct gallivm_state *gallivm = &ctx->gallivm;
2032 LLVMBuilderRef builder = gallivm->builder;
2033 unsigned buf_idx = stream_out->output_buffer;
2034 unsigned start = stream_out->start_component;
2035 unsigned num_comps = stream_out->num_components;
2036 LLVMValueRef out[4];
2037
2038 assert(num_comps && num_comps <= 4);
2039 if (!num_comps || num_comps > 4)
2040 return;
2041
2042 /* Load the output as int. */
2043 for (int j = 0; j < num_comps; j++) {
2044 assert(stream_out->stream == shader_out->vertex_stream[start + j]);
2045
2046 out[j] = LLVMBuildBitCast(builder,
2047 shader_out->values[start + j],
2048 ctx->i32, "");
2049 }
2050
2051 /* Pack the output. */
2052 LLVMValueRef vdata = NULL;
2053
2054 switch (num_comps) {
2055 case 1: /* as i32 */
2056 vdata = out[0];
2057 break;
2058 case 2: /* as v2i32 */
2059 case 3: /* as v4i32 (aligned to 4) */
2060 case 4: /* as v4i32 */
2061 vdata = LLVMGetUndef(LLVMVectorType(ctx->i32, util_next_power_of_two(num_comps)));
2062 for (int j = 0; j < num_comps; j++) {
2063 vdata = LLVMBuildInsertElement(builder, vdata, out[j],
2064 LLVMConstInt(ctx->i32, j, 0), "");
2065 }
2066 break;
2067 }
2068
2069 ac_build_buffer_store_dword(&ctx->ac, so_buffers[buf_idx],
2070 vdata, num_comps,
2071 so_write_offsets[buf_idx],
2072 LLVMConstInt(ctx->i32, 0, 0),
2073 stream_out->dst_offset * 4, 1, 1, true, false);
2074 }
2075
2076 /**
2077 * Write streamout data to buffers for vertex stream @p stream (different
2078 * vertex streams can occur for GS copy shaders).
2079 */
2080 static void si_llvm_emit_streamout(struct si_shader_context *ctx,
2081 struct si_shader_output_values *outputs,
2082 unsigned noutput, unsigned stream)
2083 {
2084 struct si_shader_selector *sel = ctx->shader->selector;
2085 struct pipe_stream_output_info *so = &sel->so;
2086 struct gallivm_state *gallivm = &ctx->gallivm;
2087 LLVMBuilderRef builder = gallivm->builder;
2088 int i;
2089 struct lp_build_if_state if_ctx;
2090
2091 /* Get bits [22:16], i.e. (so_param >> 16) & 127; */
2092 LLVMValueRef so_vtx_count =
2093 unpack_param(ctx, ctx->param_streamout_config, 16, 7);
2094
2095 LLVMValueRef tid = ac_get_thread_id(&ctx->ac);
2096
2097 /* can_emit = tid < so_vtx_count; */
2098 LLVMValueRef can_emit =
2099 LLVMBuildICmp(builder, LLVMIntULT, tid, so_vtx_count, "");
2100
2101 /* Emit the streamout code conditionally. This actually avoids
2102 * out-of-bounds buffer access. The hw tells us via the SGPR
2103 * (so_vtx_count) which threads are allowed to emit streamout data. */
2104 lp_build_if(&if_ctx, gallivm, can_emit);
2105 {
2106 /* The buffer offset is computed as follows:
2107 * ByteOffset = streamout_offset[buffer_id]*4 +
2108 * (streamout_write_index + thread_id)*stride[buffer_id] +
2109 * attrib_offset
2110 */
2111
2112 LLVMValueRef so_write_index =
2113 LLVMGetParam(ctx->main_fn,
2114 ctx->param_streamout_write_index);
2115
2116 /* Compute (streamout_write_index + thread_id). */
2117 so_write_index = LLVMBuildAdd(builder, so_write_index, tid, "");
2118
2119 /* Load the descriptor and compute the write offset for each
2120 * enabled buffer. */
2121 LLVMValueRef so_write_offset[4] = {};
2122 LLVMValueRef so_buffers[4];
2123 LLVMValueRef buf_ptr = LLVMGetParam(ctx->main_fn,
2124 SI_PARAM_RW_BUFFERS);
2125
2126 for (i = 0; i < 4; i++) {
2127 if (!so->stride[i])
2128 continue;
2129
2130 LLVMValueRef offset = lp_build_const_int32(gallivm,
2131 SI_VS_STREAMOUT_BUF0 + i);
2132
2133 so_buffers[i] = ac_build_indexed_load_const(&ctx->ac, buf_ptr, offset);
2134
2135 LLVMValueRef so_offset = LLVMGetParam(ctx->main_fn,
2136 ctx->param_streamout_offset[i]);
2137 so_offset = LLVMBuildMul(builder, so_offset, LLVMConstInt(ctx->i32, 4, 0), "");
2138
2139 so_write_offset[i] = LLVMBuildMul(builder, so_write_index,
2140 LLVMConstInt(ctx->i32, so->stride[i]*4, 0), "");
2141 so_write_offset[i] = LLVMBuildAdd(builder, so_write_offset[i], so_offset, "");
2142 }
2143
2144 /* Write streamout data. */
2145 for (i = 0; i < so->num_outputs; i++) {
2146 unsigned reg = so->output[i].register_index;
2147
2148 if (reg >= noutput)
2149 continue;
2150
2151 if (stream != so->output[i].stream)
2152 continue;
2153
2154 emit_streamout_output(ctx, so_buffers, so_write_offset,
2155 &so->output[i], &outputs[reg]);
2156 }
2157 }
2158 lp_build_endif(&if_ctx);
2159 }
2160
2161
2162 /* Generate export instructions for hardware VS shader stage */
2163 static void si_llvm_export_vs(struct lp_build_tgsi_context *bld_base,
2164 struct si_shader_output_values *outputs,
2165 unsigned noutput)
2166 {
2167 struct si_shader_context *ctx = si_shader_context(bld_base);
2168 struct si_shader *shader = ctx->shader;
2169 struct lp_build_context *base = &bld_base->base;
2170 struct ac_export_args args, pos_args[4] = {};
2171 LLVMValueRef psize_value = NULL, edgeflag_value = NULL, layer_value = NULL, viewport_index_value = NULL;
2172 unsigned semantic_name, semantic_index;
2173 unsigned target;
2174 unsigned param_count = 0;
2175 unsigned pos_idx;
2176 int i;
2177
2178 for (i = 0; i < noutput; i++) {
2179 semantic_name = outputs[i].semantic_name;
2180 semantic_index = outputs[i].semantic_index;
2181 bool export_param = true;
2182
2183 switch (semantic_name) {
2184 case TGSI_SEMANTIC_POSITION: /* ignore these */
2185 case TGSI_SEMANTIC_PSIZE:
2186 case TGSI_SEMANTIC_CLIPVERTEX:
2187 case TGSI_SEMANTIC_EDGEFLAG:
2188 break;
2189 case TGSI_SEMANTIC_GENERIC:
2190 case TGSI_SEMANTIC_CLIPDIST:
2191 if (shader->key.opt.hw_vs.kill_outputs &
2192 (1ull << si_shader_io_get_unique_index(semantic_name, semantic_index)))
2193 export_param = false;
2194 break;
2195 default:
2196 if (shader->key.opt.hw_vs.kill_outputs2 &
2197 (1u << si_shader_io_get_unique_index2(semantic_name, semantic_index)))
2198 export_param = false;
2199 break;
2200 }
2201
2202 if (outputs[i].vertex_stream[0] != 0 &&
2203 outputs[i].vertex_stream[1] != 0 &&
2204 outputs[i].vertex_stream[2] != 0 &&
2205 outputs[i].vertex_stream[3] != 0)
2206 export_param = false;
2207
2208 handle_semantic:
2209 /* Select the correct target */
2210 switch(semantic_name) {
2211 case TGSI_SEMANTIC_PSIZE:
2212 psize_value = outputs[i].values[0];
2213 continue;
2214 case TGSI_SEMANTIC_EDGEFLAG:
2215 edgeflag_value = outputs[i].values[0];
2216 continue;
2217 case TGSI_SEMANTIC_LAYER:
2218 layer_value = outputs[i].values[0];
2219 semantic_name = TGSI_SEMANTIC_GENERIC;
2220 goto handle_semantic;
2221 case TGSI_SEMANTIC_VIEWPORT_INDEX:
2222 viewport_index_value = outputs[i].values[0];
2223 semantic_name = TGSI_SEMANTIC_GENERIC;
2224 goto handle_semantic;
2225 case TGSI_SEMANTIC_POSITION:
2226 target = V_008DFC_SQ_EXP_POS;
2227 break;
2228 case TGSI_SEMANTIC_CLIPDIST:
2229 if (shader->key.opt.hw_vs.clip_disable) {
2230 semantic_name = TGSI_SEMANTIC_GENERIC;
2231 goto handle_semantic;
2232 }
2233 target = V_008DFC_SQ_EXP_POS + 2 + semantic_index;
2234 break;
2235 case TGSI_SEMANTIC_CLIPVERTEX:
2236 if (shader->key.opt.hw_vs.clip_disable)
2237 continue;
2238 si_llvm_emit_clipvertex(bld_base, pos_args, outputs[i].values);
2239 continue;
2240 case TGSI_SEMANTIC_COLOR:
2241 case TGSI_SEMANTIC_BCOLOR:
2242 case TGSI_SEMANTIC_PRIMID:
2243 case TGSI_SEMANTIC_FOG:
2244 case TGSI_SEMANTIC_TEXCOORD:
2245 case TGSI_SEMANTIC_GENERIC:
2246 if (!export_param)
2247 continue;
2248 target = V_008DFC_SQ_EXP_PARAM + param_count;
2249 assert(i < ARRAY_SIZE(shader->info.vs_output_param_offset));
2250 shader->info.vs_output_param_offset[i] = param_count;
2251 param_count++;
2252 break;
2253 default:
2254 target = 0;
2255 fprintf(stderr,
2256 "Warning: SI unhandled vs output type:%d\n",
2257 semantic_name);
2258 }
2259
2260 si_llvm_init_export_args(bld_base, outputs[i].values, target, &args);
2261
2262 if (target >= V_008DFC_SQ_EXP_POS &&
2263 target <= (V_008DFC_SQ_EXP_POS + 3)) {
2264 memcpy(&pos_args[target - V_008DFC_SQ_EXP_POS],
2265 &args, sizeof(args));
2266 } else {
2267 ac_build_export(&ctx->ac, &args);
2268 }
2269
2270 if (semantic_name == TGSI_SEMANTIC_CLIPDIST) {
2271 semantic_name = TGSI_SEMANTIC_GENERIC;
2272 goto handle_semantic;
2273 }
2274 }
2275
2276 shader->info.nr_param_exports = param_count;
2277
2278 /* We need to add the position output manually if it's missing. */
2279 if (!pos_args[0].out[0]) {
2280 pos_args[0].enabled_channels = 0xf; /* writemask */
2281 pos_args[0].valid_mask = 0; /* EXEC mask */
2282 pos_args[0].done = 0; /* last export? */
2283 pos_args[0].target = V_008DFC_SQ_EXP_POS;
2284 pos_args[0].compr = 0; /* COMPR flag */
2285 pos_args[0].out[0] = base->zero; /* X */
2286 pos_args[0].out[1] = base->zero; /* Y */
2287 pos_args[0].out[2] = base->zero; /* Z */
2288 pos_args[0].out[3] = base->one; /* W */
2289 }
2290
2291 /* Write the misc vector (point size, edgeflag, layer, viewport). */
2292 if (shader->selector->info.writes_psize ||
2293 shader->selector->info.writes_edgeflag ||
2294 shader->selector->info.writes_viewport_index ||
2295 shader->selector->info.writes_layer) {
2296 pos_args[1].enabled_channels = shader->selector->info.writes_psize |
2297 (shader->selector->info.writes_edgeflag << 1) |
2298 (shader->selector->info.writes_layer << 2) |
2299 (shader->selector->info.writes_viewport_index << 3);
2300 pos_args[1].valid_mask = 0; /* EXEC mask */
2301 pos_args[1].done = 0; /* last export? */
2302 pos_args[1].target = V_008DFC_SQ_EXP_POS + 1;
2303 pos_args[1].compr = 0; /* COMPR flag */
2304 pos_args[1].out[0] = base->zero; /* X */
2305 pos_args[1].out[1] = base->zero; /* Y */
2306 pos_args[1].out[2] = base->zero; /* Z */
2307 pos_args[1].out[3] = base->zero; /* W */
2308
2309 if (shader->selector->info.writes_psize)
2310 pos_args[1].out[0] = psize_value;
2311
2312 if (shader->selector->info.writes_edgeflag) {
2313 /* The output is a float, but the hw expects an integer
2314 * with the first bit containing the edge flag. */
2315 edgeflag_value = LLVMBuildFPToUI(base->gallivm->builder,
2316 edgeflag_value,
2317 ctx->i32, "");
2318 edgeflag_value = lp_build_min(&bld_base->int_bld,
2319 edgeflag_value,
2320 bld_base->int_bld.one);
2321
2322 /* The LLVM intrinsic expects a float. */
2323 pos_args[1].out[1] = LLVMBuildBitCast(base->gallivm->builder,
2324 edgeflag_value,
2325 ctx->f32, "");
2326 }
2327
2328 if (shader->selector->info.writes_layer)
2329 pos_args[1].out[2] = layer_value;
2330
2331 if (shader->selector->info.writes_viewport_index)
2332 pos_args[1].out[3] = viewport_index_value;
2333 }
2334
2335 for (i = 0; i < 4; i++)
2336 if (pos_args[i].out[0])
2337 shader->info.nr_pos_exports++;
2338
2339 pos_idx = 0;
2340 for (i = 0; i < 4; i++) {
2341 if (!pos_args[i].out[0])
2342 continue;
2343
2344 /* Specify the target we are exporting */
2345 pos_args[i].target = V_008DFC_SQ_EXP_POS + pos_idx++;
2346
2347 if (pos_idx == shader->info.nr_pos_exports)
2348 /* Specify that this is the last export */
2349 pos_args[i].done = 1;
2350
2351 ac_build_export(&ctx->ac, &pos_args[i]);
2352 }
2353 }
2354
2355 /**
2356 * Forward all outputs from the vertex shader to the TES. This is only used
2357 * for the fixed function TCS.
2358 */
2359 static void si_copy_tcs_inputs(struct lp_build_tgsi_context *bld_base)
2360 {
2361 struct si_shader_context *ctx = si_shader_context(bld_base);
2362 struct gallivm_state *gallivm = bld_base->base.gallivm;
2363 LLVMValueRef invocation_id, rw_buffers, buffer, buffer_offset;
2364 LLVMValueRef lds_vertex_stride, lds_vertex_offset, lds_base;
2365 uint64_t inputs;
2366
2367 invocation_id = unpack_param(ctx, SI_PARAM_REL_IDS, 8, 5);
2368
2369 rw_buffers = LLVMGetParam(ctx->main_fn, SI_PARAM_RW_BUFFERS);
2370 buffer = ac_build_indexed_load_const(&ctx->ac, rw_buffers,
2371 lp_build_const_int32(gallivm, SI_HS_RING_TESS_OFFCHIP));
2372
2373 buffer_offset = LLVMGetParam(ctx->main_fn, ctx->param_oc_lds);
2374
2375 lds_vertex_stride = unpack_param(ctx, SI_PARAM_TCS_IN_LAYOUT, 13, 8);
2376 lds_vertex_offset = LLVMBuildMul(gallivm->builder, invocation_id,
2377 lds_vertex_stride, "");
2378 lds_base = get_tcs_in_current_patch_offset(ctx);
2379 lds_base = LLVMBuildAdd(gallivm->builder, lds_base, lds_vertex_offset, "");
2380
2381 inputs = ctx->shader->key.mono.tcs.inputs_to_copy;
2382 while (inputs) {
2383 unsigned i = u_bit_scan64(&inputs);
2384
2385 LLVMValueRef lds_ptr = LLVMBuildAdd(gallivm->builder, lds_base,
2386 lp_build_const_int32(gallivm, 4 * i),
2387 "");
2388
2389 LLVMValueRef buffer_addr = get_tcs_tes_buffer_address(ctx,
2390 get_rel_patch_id(ctx),
2391 invocation_id,
2392 lp_build_const_int32(gallivm, i));
2393
2394 LLVMValueRef value = lds_load(bld_base, TGSI_TYPE_SIGNED, ~0,
2395 lds_ptr);
2396
2397 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, buffer_addr,
2398 buffer_offset, 0, 1, 0, true, false);
2399 }
2400 }
2401
2402 static void si_write_tess_factors(struct lp_build_tgsi_context *bld_base,
2403 LLVMValueRef rel_patch_id,
2404 LLVMValueRef invocation_id,
2405 LLVMValueRef tcs_out_current_patch_data_offset)
2406 {
2407 struct si_shader_context *ctx = si_shader_context(bld_base);
2408 struct gallivm_state *gallivm = bld_base->base.gallivm;
2409 struct si_shader *shader = ctx->shader;
2410 unsigned tess_inner_index, tess_outer_index;
2411 LLVMValueRef lds_base, lds_inner, lds_outer, byteoffset, buffer;
2412 LLVMValueRef out[6], vec0, vec1, rw_buffers, tf_base, inner[4], outer[4];
2413 unsigned stride, outer_comps, inner_comps, i;
2414 struct lp_build_if_state if_ctx, inner_if_ctx;
2415
2416 si_llvm_emit_barrier(NULL, bld_base, NULL);
2417
2418 /* Do this only for invocation 0, because the tess levels are per-patch,
2419 * not per-vertex.
2420 *
2421 * This can't jump, because invocation 0 executes this. It should
2422 * at least mask out the loads and stores for other invocations.
2423 */
2424 lp_build_if(&if_ctx, gallivm,
2425 LLVMBuildICmp(gallivm->builder, LLVMIntEQ,
2426 invocation_id, bld_base->uint_bld.zero, ""));
2427
2428 /* Determine the layout of one tess factor element in the buffer. */
2429 switch (shader->key.part.tcs.epilog.prim_mode) {
2430 case PIPE_PRIM_LINES:
2431 stride = 2; /* 2 dwords, 1 vec2 store */
2432 outer_comps = 2;
2433 inner_comps = 0;
2434 break;
2435 case PIPE_PRIM_TRIANGLES:
2436 stride = 4; /* 4 dwords, 1 vec4 store */
2437 outer_comps = 3;
2438 inner_comps = 1;
2439 break;
2440 case PIPE_PRIM_QUADS:
2441 stride = 6; /* 6 dwords, 2 stores (vec4 + vec2) */
2442 outer_comps = 4;
2443 inner_comps = 2;
2444 break;
2445 default:
2446 assert(0);
2447 return;
2448 }
2449
2450 /* Load tess_inner and tess_outer from LDS.
2451 * Any invocation can write them, so we can't get them from a temporary.
2452 */
2453 tess_inner_index = si_shader_io_get_unique_index(TGSI_SEMANTIC_TESSINNER, 0);
2454 tess_outer_index = si_shader_io_get_unique_index(TGSI_SEMANTIC_TESSOUTER, 0);
2455
2456 lds_base = tcs_out_current_patch_data_offset;
2457 lds_inner = LLVMBuildAdd(gallivm->builder, lds_base,
2458 lp_build_const_int32(gallivm,
2459 tess_inner_index * 4), "");
2460 lds_outer = LLVMBuildAdd(gallivm->builder, lds_base,
2461 lp_build_const_int32(gallivm,
2462 tess_outer_index * 4), "");
2463
2464 for (i = 0; i < 4; i++) {
2465 inner[i] = LLVMGetUndef(ctx->i32);
2466 outer[i] = LLVMGetUndef(ctx->i32);
2467 }
2468
2469 if (shader->key.part.tcs.epilog.prim_mode == PIPE_PRIM_LINES) {
2470 /* For isolines, the hardware expects tess factors in the
2471 * reverse order from what GLSL / TGSI specify.
2472 */
2473 outer[0] = out[1] = lds_load(bld_base, TGSI_TYPE_SIGNED, 0, lds_outer);
2474 outer[1] = out[0] = lds_load(bld_base, TGSI_TYPE_SIGNED, 1, lds_outer);
2475 } else {
2476 for (i = 0; i < outer_comps; i++) {
2477 outer[i] = out[i] =
2478 lds_load(bld_base, TGSI_TYPE_SIGNED, i, lds_outer);
2479 }
2480 for (i = 0; i < inner_comps; i++) {
2481 inner[i] = out[outer_comps+i] =
2482 lds_load(bld_base, TGSI_TYPE_SIGNED, i, lds_inner);
2483 }
2484 }
2485
2486 /* Convert the outputs to vectors for stores. */
2487 vec0 = lp_build_gather_values(gallivm, out, MIN2(stride, 4));
2488 vec1 = NULL;
2489
2490 if (stride > 4)
2491 vec1 = lp_build_gather_values(gallivm, out+4, stride - 4);
2492
2493 /* Get the buffer. */
2494 rw_buffers = LLVMGetParam(ctx->main_fn,
2495 SI_PARAM_RW_BUFFERS);
2496 buffer = ac_build_indexed_load_const(&ctx->ac, rw_buffers,
2497 lp_build_const_int32(gallivm, SI_HS_RING_TESS_FACTOR));
2498
2499 /* Get the offset. */
2500 tf_base = LLVMGetParam(ctx->main_fn,
2501 SI_PARAM_TESS_FACTOR_OFFSET);
2502 byteoffset = LLVMBuildMul(gallivm->builder, rel_patch_id,
2503 lp_build_const_int32(gallivm, 4 * stride), "");
2504
2505 lp_build_if(&inner_if_ctx, gallivm,
2506 LLVMBuildICmp(gallivm->builder, LLVMIntEQ,
2507 rel_patch_id, bld_base->uint_bld.zero, ""));
2508
2509 /* Store the dynamic HS control word. */
2510 ac_build_buffer_store_dword(&ctx->ac, buffer,
2511 lp_build_const_int32(gallivm, 0x80000000),
2512 1, lp_build_const_int32(gallivm, 0), tf_base,
2513 0, 1, 0, true, false);
2514
2515 lp_build_endif(&inner_if_ctx);
2516
2517 /* Store the tessellation factors. */
2518 ac_build_buffer_store_dword(&ctx->ac, buffer, vec0,
2519 MIN2(stride, 4), byteoffset, tf_base,
2520 4, 1, 0, true, false);
2521 if (vec1)
2522 ac_build_buffer_store_dword(&ctx->ac, buffer, vec1,
2523 stride - 4, byteoffset, tf_base,
2524 20, 1, 0, true, false);
2525
2526 /* Store the tess factors into the offchip buffer if TES reads them. */
2527 if (shader->key.part.tcs.epilog.tes_reads_tess_factors) {
2528 LLVMValueRef buf, base, inner_vec, outer_vec, tf_outer_offset;
2529 LLVMValueRef tf_inner_offset;
2530 unsigned param_outer, param_inner;
2531
2532 buf = ac_build_indexed_load_const(&ctx->ac, rw_buffers,
2533 LLVMConstInt(ctx->i32, SI_HS_RING_TESS_OFFCHIP, 0));
2534 base = LLVMGetParam(ctx->main_fn, ctx->param_oc_lds);
2535
2536 param_outer = si_shader_io_get_unique_index(
2537 TGSI_SEMANTIC_TESSOUTER, 0);
2538 tf_outer_offset = get_tcs_tes_buffer_address(ctx, rel_patch_id, NULL,
2539 LLVMConstInt(ctx->i32, param_outer, 0));
2540
2541 outer_vec = lp_build_gather_values(gallivm, outer,
2542 util_next_power_of_two(outer_comps));
2543
2544 ac_build_buffer_store_dword(&ctx->ac, buf, outer_vec,
2545 outer_comps, tf_outer_offset,
2546 base, 0, 1, 0, true, false);
2547 if (inner_comps) {
2548 param_inner = si_shader_io_get_unique_index(
2549 TGSI_SEMANTIC_TESSINNER, 0);
2550 tf_inner_offset = get_tcs_tes_buffer_address(ctx, rel_patch_id, NULL,
2551 LLVMConstInt(ctx->i32, param_inner, 0));
2552
2553 inner_vec = inner_comps == 1 ? inner[0] :
2554 lp_build_gather_values(gallivm, inner, inner_comps);
2555 ac_build_buffer_store_dword(&ctx->ac, buf, inner_vec,
2556 inner_comps, tf_inner_offset,
2557 base, 0, 1, 0, true, false);
2558 }
2559 }
2560
2561 lp_build_endif(&if_ctx);
2562 }
2563
2564 /* This only writes the tessellation factor levels. */
2565 static void si_llvm_emit_tcs_epilogue(struct lp_build_tgsi_context *bld_base)
2566 {
2567 struct si_shader_context *ctx = si_shader_context(bld_base);
2568 LLVMValueRef rel_patch_id, invocation_id, tf_lds_offset;
2569 LLVMValueRef offchip_soffset, offchip_layout;
2570
2571 si_copy_tcs_inputs(bld_base);
2572
2573 rel_patch_id = get_rel_patch_id(ctx);
2574 invocation_id = unpack_param(ctx, SI_PARAM_REL_IDS, 8, 5);
2575 tf_lds_offset = get_tcs_out_current_patch_data_offset(ctx);
2576
2577 /* Return epilog parameters from this function. */
2578 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
2579 LLVMValueRef ret = ctx->return_value;
2580 LLVMValueRef rw_buffers, rw0, rw1, tf_soffset;
2581 unsigned vgpr;
2582
2583 /* RW_BUFFERS pointer */
2584 rw_buffers = LLVMGetParam(ctx->main_fn,
2585 SI_PARAM_RW_BUFFERS);
2586 rw_buffers = LLVMBuildPtrToInt(builder, rw_buffers, ctx->i64, "");
2587 rw_buffers = LLVMBuildBitCast(builder, rw_buffers, ctx->v2i32, "");
2588 rw0 = LLVMBuildExtractElement(builder, rw_buffers,
2589 bld_base->uint_bld.zero, "");
2590 rw1 = LLVMBuildExtractElement(builder, rw_buffers,
2591 bld_base->uint_bld.one, "");
2592 ret = LLVMBuildInsertValue(builder, ret, rw0, 0, "");
2593 ret = LLVMBuildInsertValue(builder, ret, rw1, 1, "");
2594
2595 /* Tess offchip and factor buffer soffset are after user SGPRs. */
2596 offchip_layout = LLVMGetParam(ctx->main_fn,
2597 SI_PARAM_TCS_OFFCHIP_LAYOUT);
2598 offchip_soffset = LLVMGetParam(ctx->main_fn, ctx->param_oc_lds);
2599 tf_soffset = LLVMGetParam(ctx->main_fn,
2600 SI_PARAM_TESS_FACTOR_OFFSET);
2601 ret = LLVMBuildInsertValue(builder, ret, offchip_layout,
2602 SI_SGPR_TCS_OFFCHIP_LAYOUT, "");
2603 ret = LLVMBuildInsertValue(builder, ret, offchip_soffset,
2604 SI_TCS_NUM_USER_SGPR, "");
2605 ret = LLVMBuildInsertValue(builder, ret, tf_soffset,
2606 SI_TCS_NUM_USER_SGPR + 1, "");
2607
2608 /* VGPRs */
2609 rel_patch_id = bitcast(bld_base, TGSI_TYPE_FLOAT, rel_patch_id);
2610 invocation_id = bitcast(bld_base, TGSI_TYPE_FLOAT, invocation_id);
2611 tf_lds_offset = bitcast(bld_base, TGSI_TYPE_FLOAT, tf_lds_offset);
2612
2613 vgpr = SI_TCS_NUM_USER_SGPR + 2;
2614 ret = LLVMBuildInsertValue(builder, ret, rel_patch_id, vgpr++, "");
2615 ret = LLVMBuildInsertValue(builder, ret, invocation_id, vgpr++, "");
2616 ret = LLVMBuildInsertValue(builder, ret, tf_lds_offset, vgpr++, "");
2617 ctx->return_value = ret;
2618 }
2619
2620 static void si_llvm_emit_ls_epilogue(struct lp_build_tgsi_context *bld_base)
2621 {
2622 struct si_shader_context *ctx = si_shader_context(bld_base);
2623 struct si_shader *shader = ctx->shader;
2624 struct tgsi_shader_info *info = &shader->selector->info;
2625 struct gallivm_state *gallivm = bld_base->base.gallivm;
2626 unsigned i, chan;
2627 LLVMValueRef vertex_id = LLVMGetParam(ctx->main_fn,
2628 ctx->param_rel_auto_id);
2629 LLVMValueRef vertex_dw_stride =
2630 unpack_param(ctx, SI_PARAM_LS_OUT_LAYOUT, 13, 8);
2631 LLVMValueRef base_dw_addr = LLVMBuildMul(gallivm->builder, vertex_id,
2632 vertex_dw_stride, "");
2633
2634 /* Write outputs to LDS. The next shader (TCS aka HS) will read
2635 * its inputs from it. */
2636 for (i = 0; i < info->num_outputs; i++) {
2637 LLVMValueRef *out_ptr = ctx->outputs[i];
2638 unsigned name = info->output_semantic_name[i];
2639 unsigned index = info->output_semantic_index[i];
2640 int param = si_shader_io_get_unique_index(name, index);
2641 LLVMValueRef dw_addr = LLVMBuildAdd(gallivm->builder, base_dw_addr,
2642 lp_build_const_int32(gallivm, param * 4), "");
2643
2644 for (chan = 0; chan < 4; chan++) {
2645 lds_store(bld_base, chan, dw_addr,
2646 LLVMBuildLoad(gallivm->builder, out_ptr[chan], ""));
2647 }
2648 }
2649 }
2650
2651 static void si_llvm_emit_es_epilogue(struct lp_build_tgsi_context *bld_base)
2652 {
2653 struct si_shader_context *ctx = si_shader_context(bld_base);
2654 struct gallivm_state *gallivm = bld_base->base.gallivm;
2655 struct si_shader *es = ctx->shader;
2656 struct tgsi_shader_info *info = &es->selector->info;
2657 LLVMValueRef soffset = LLVMGetParam(ctx->main_fn,
2658 ctx->param_es2gs_offset);
2659 unsigned chan;
2660 int i;
2661
2662 for (i = 0; i < info->num_outputs; i++) {
2663 LLVMValueRef *out_ptr = ctx->outputs[i];
2664 int param_index;
2665
2666 if (info->output_semantic_name[i] == TGSI_SEMANTIC_VIEWPORT_INDEX ||
2667 info->output_semantic_name[i] == TGSI_SEMANTIC_LAYER)
2668 continue;
2669
2670 param_index = si_shader_io_get_unique_index(info->output_semantic_name[i],
2671 info->output_semantic_index[i]);
2672
2673 for (chan = 0; chan < 4; chan++) {
2674 LLVMValueRef out_val = LLVMBuildLoad(gallivm->builder, out_ptr[chan], "");
2675 out_val = LLVMBuildBitCast(gallivm->builder, out_val, ctx->i32, "");
2676
2677 ac_build_buffer_store_dword(&ctx->ac,
2678 ctx->esgs_ring,
2679 out_val, 1, NULL, soffset,
2680 (4 * param_index + chan) * 4,
2681 1, 1, true, true);
2682 }
2683 }
2684 }
2685
2686 static void si_llvm_emit_gs_epilogue(struct lp_build_tgsi_context *bld_base)
2687 {
2688 struct si_shader_context *ctx = si_shader_context(bld_base);
2689
2690 ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_NOP | AC_SENDMSG_GS_DONE,
2691 LLVMGetParam(ctx->main_fn, SI_PARAM_GS_WAVE_ID));
2692 }
2693
2694 static void si_llvm_emit_vs_epilogue(struct lp_build_tgsi_context *bld_base)
2695 {
2696 struct si_shader_context *ctx = si_shader_context(bld_base);
2697 struct gallivm_state *gallivm = bld_base->base.gallivm;
2698 struct tgsi_shader_info *info = &ctx->shader->selector->info;
2699 struct si_shader_output_values *outputs = NULL;
2700 int i,j;
2701
2702 assert(!ctx->shader->is_gs_copy_shader);
2703
2704 outputs = MALLOC((info->num_outputs + 1) * sizeof(outputs[0]));
2705
2706 /* Vertex color clamping.
2707 *
2708 * This uses a state constant loaded in a user data SGPR and
2709 * an IF statement is added that clamps all colors if the constant
2710 * is true.
2711 */
2712 if (ctx->type == PIPE_SHADER_VERTEX) {
2713 struct lp_build_if_state if_ctx;
2714 LLVMValueRef cond = NULL;
2715 LLVMValueRef addr, val;
2716
2717 for (i = 0; i < info->num_outputs; i++) {
2718 if (info->output_semantic_name[i] != TGSI_SEMANTIC_COLOR &&
2719 info->output_semantic_name[i] != TGSI_SEMANTIC_BCOLOR)
2720 continue;
2721
2722 /* We've found a color. */
2723 if (!cond) {
2724 /* The state is in the first bit of the user SGPR. */
2725 cond = LLVMGetParam(ctx->main_fn,
2726 SI_PARAM_VS_STATE_BITS);
2727 cond = LLVMBuildTrunc(gallivm->builder, cond,
2728 ctx->i1, "");
2729 lp_build_if(&if_ctx, gallivm, cond);
2730 }
2731
2732 for (j = 0; j < 4; j++) {
2733 addr = ctx->outputs[i][j];
2734 val = LLVMBuildLoad(gallivm->builder, addr, "");
2735 val = ac_build_clamp(&ctx->ac, val);
2736 LLVMBuildStore(gallivm->builder, val, addr);
2737 }
2738 }
2739
2740 if (cond)
2741 lp_build_endif(&if_ctx);
2742 }
2743
2744 for (i = 0; i < info->num_outputs; i++) {
2745 outputs[i].semantic_name = info->output_semantic_name[i];
2746 outputs[i].semantic_index = info->output_semantic_index[i];
2747
2748 for (j = 0; j < 4; j++) {
2749 outputs[i].values[j] =
2750 LLVMBuildLoad(gallivm->builder,
2751 ctx->outputs[i][j],
2752 "");
2753 outputs[i].vertex_stream[j] =
2754 (info->output_streams[i] >> (2 * j)) & 3;
2755 }
2756
2757 }
2758
2759 /* Return the primitive ID from the LLVM function. */
2760 ctx->return_value =
2761 LLVMBuildInsertValue(gallivm->builder,
2762 ctx->return_value,
2763 bitcast(bld_base, TGSI_TYPE_FLOAT,
2764 get_primitive_id(bld_base, 0)),
2765 VS_EPILOG_PRIMID_LOC, "");
2766
2767 if (ctx->shader->selector->so.num_outputs)
2768 si_llvm_emit_streamout(ctx, outputs, i, 0);
2769 si_llvm_export_vs(bld_base, outputs, i);
2770 FREE(outputs);
2771 }
2772
2773 struct si_ps_exports {
2774 unsigned num;
2775 struct ac_export_args args[10];
2776 };
2777
2778 unsigned si_get_spi_shader_z_format(bool writes_z, bool writes_stencil,
2779 bool writes_samplemask)
2780 {
2781 if (writes_z) {
2782 /* Z needs 32 bits. */
2783 if (writes_samplemask)
2784 return V_028710_SPI_SHADER_32_ABGR;
2785 else if (writes_stencil)
2786 return V_028710_SPI_SHADER_32_GR;
2787 else
2788 return V_028710_SPI_SHADER_32_R;
2789 } else if (writes_stencil || writes_samplemask) {
2790 /* Both stencil and sample mask need only 16 bits. */
2791 return V_028710_SPI_SHADER_UINT16_ABGR;
2792 } else {
2793 return V_028710_SPI_SHADER_ZERO;
2794 }
2795 }
2796
2797 static void si_export_mrt_z(struct lp_build_tgsi_context *bld_base,
2798 LLVMValueRef depth, LLVMValueRef stencil,
2799 LLVMValueRef samplemask, struct si_ps_exports *exp)
2800 {
2801 struct si_shader_context *ctx = si_shader_context(bld_base);
2802 struct lp_build_context *base = &bld_base->base;
2803 struct ac_export_args args;
2804 unsigned mask = 0;
2805 unsigned format = si_get_spi_shader_z_format(depth != NULL,
2806 stencil != NULL,
2807 samplemask != NULL);
2808
2809 assert(depth || stencil || samplemask);
2810
2811 args.valid_mask = 1; /* whether the EXEC mask is valid */
2812 args.done = 1; /* DONE bit */
2813
2814 /* Specify the target we are exporting */
2815 args.target = V_008DFC_SQ_EXP_MRTZ;
2816
2817 args.compr = 0; /* COMP flag */
2818 args.out[0] = base->undef; /* R, depth */
2819 args.out[1] = base->undef; /* G, stencil test value[0:7], stencil op value[8:15] */
2820 args.out[2] = base->undef; /* B, sample mask */
2821 args.out[3] = base->undef; /* A, alpha to mask */
2822
2823 if (format == V_028710_SPI_SHADER_UINT16_ABGR) {
2824 assert(!depth);
2825 args.compr = 1; /* COMPR flag */
2826
2827 if (stencil) {
2828 /* Stencil should be in X[23:16]. */
2829 stencil = bitcast(bld_base, TGSI_TYPE_UNSIGNED, stencil);
2830 stencil = LLVMBuildShl(base->gallivm->builder, stencil,
2831 LLVMConstInt(ctx->i32, 16, 0), "");
2832 args.out[0] = bitcast(bld_base, TGSI_TYPE_FLOAT, stencil);
2833 mask |= 0x3;
2834 }
2835 if (samplemask) {
2836 /* SampleMask should be in Y[15:0]. */
2837 args.out[1] = samplemask;
2838 mask |= 0xc;
2839 }
2840 } else {
2841 if (depth) {
2842 args.out[0] = depth;
2843 mask |= 0x1;
2844 }
2845 if (stencil) {
2846 args.out[1] = stencil;
2847 mask |= 0x2;
2848 }
2849 if (samplemask) {
2850 args.out[2] = samplemask;
2851 mask |= 0x4;
2852 }
2853 }
2854
2855 /* SI (except OLAND and HAINAN) has a bug that it only looks
2856 * at the X writemask component. */
2857 if (ctx->screen->b.chip_class == SI &&
2858 ctx->screen->b.family != CHIP_OLAND &&
2859 ctx->screen->b.family != CHIP_HAINAN)
2860 mask |= 0x1;
2861
2862 /* Specify which components to enable */
2863 args.enabled_channels = mask;
2864
2865 memcpy(&exp->args[exp->num++], &args, sizeof(args));
2866 }
2867
2868 static void si_export_mrt_color(struct lp_build_tgsi_context *bld_base,
2869 LLVMValueRef *color, unsigned index,
2870 unsigned samplemask_param,
2871 bool is_last, struct si_ps_exports *exp)
2872 {
2873 struct si_shader_context *ctx = si_shader_context(bld_base);
2874 struct lp_build_context *base = &bld_base->base;
2875 int i;
2876
2877 /* Clamp color */
2878 if (ctx->shader->key.part.ps.epilog.clamp_color)
2879 for (i = 0; i < 4; i++)
2880 color[i] = ac_build_clamp(&ctx->ac, color[i]);
2881
2882 /* Alpha to one */
2883 if (ctx->shader->key.part.ps.epilog.alpha_to_one)
2884 color[3] = base->one;
2885
2886 /* Alpha test */
2887 if (index == 0 &&
2888 ctx->shader->key.part.ps.epilog.alpha_func != PIPE_FUNC_ALWAYS)
2889 si_alpha_test(bld_base, color[3]);
2890
2891 /* Line & polygon smoothing */
2892 if (ctx->shader->key.part.ps.epilog.poly_line_smoothing)
2893 color[3] = si_scale_alpha_by_sample_mask(bld_base, color[3],
2894 samplemask_param);
2895
2896 /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
2897 if (ctx->shader->key.part.ps.epilog.last_cbuf > 0) {
2898 struct ac_export_args args[8];
2899 int c, last = -1;
2900
2901 /* Get the export arguments, also find out what the last one is. */
2902 for (c = 0; c <= ctx->shader->key.part.ps.epilog.last_cbuf; c++) {
2903 si_llvm_init_export_args(bld_base, color,
2904 V_008DFC_SQ_EXP_MRT + c, &args[c]);
2905 if (args[c].enabled_channels)
2906 last = c;
2907 }
2908
2909 /* Emit all exports. */
2910 for (c = 0; c <= ctx->shader->key.part.ps.epilog.last_cbuf; c++) {
2911 if (is_last && last == c) {
2912 args[c].valid_mask = 1; /* whether the EXEC mask is valid */
2913 args[c].done = 1; /* DONE bit */
2914 } else if (!args[c].enabled_channels)
2915 continue; /* unnecessary NULL export */
2916
2917 memcpy(&exp->args[exp->num++], &args[c], sizeof(args[c]));
2918 }
2919 } else {
2920 struct ac_export_args args;
2921
2922 /* Export */
2923 si_llvm_init_export_args(bld_base, color, V_008DFC_SQ_EXP_MRT + index,
2924 &args);
2925 if (is_last) {
2926 args.valid_mask = 1; /* whether the EXEC mask is valid */
2927 args.done = 1; /* DONE bit */
2928 } else if (!args.enabled_channels)
2929 return; /* unnecessary NULL export */
2930
2931 memcpy(&exp->args[exp->num++], &args, sizeof(args));
2932 }
2933 }
2934
2935 static void si_emit_ps_exports(struct si_shader_context *ctx,
2936 struct si_ps_exports *exp)
2937 {
2938 for (unsigned i = 0; i < exp->num; i++)
2939 ac_build_export(&ctx->ac, &exp->args[i]);
2940 }
2941
2942 static void si_export_null(struct lp_build_tgsi_context *bld_base)
2943 {
2944 struct si_shader_context *ctx = si_shader_context(bld_base);
2945 struct lp_build_context *base = &bld_base->base;
2946 struct ac_export_args args;
2947
2948 args.enabled_channels = 0x0; /* enabled channels */
2949 args.valid_mask = 1; /* whether the EXEC mask is valid */
2950 args.done = 1; /* DONE bit */
2951 args.target = V_008DFC_SQ_EXP_NULL;
2952 args.compr = 0; /* COMPR flag (0 = 32-bit export) */
2953 args.out[0] = base->undef; /* R */
2954 args.out[1] = base->undef; /* G */
2955 args.out[2] = base->undef; /* B */
2956 args.out[3] = base->undef; /* A */
2957
2958 ac_build_export(&ctx->ac, &args);
2959 }
2960
2961 /**
2962 * Return PS outputs in this order:
2963 *
2964 * v[0:3] = color0.xyzw
2965 * v[4:7] = color1.xyzw
2966 * ...
2967 * vN+0 = Depth
2968 * vN+1 = Stencil
2969 * vN+2 = SampleMask
2970 * vN+3 = SampleMaskIn (used for OpenGL smoothing)
2971 *
2972 * The alpha-ref SGPR is returned via its original location.
2973 */
2974 static void si_llvm_return_fs_outputs(struct lp_build_tgsi_context *bld_base)
2975 {
2976 struct si_shader_context *ctx = si_shader_context(bld_base);
2977 struct si_shader *shader = ctx->shader;
2978 struct lp_build_context *base = &bld_base->base;
2979 struct tgsi_shader_info *info = &shader->selector->info;
2980 LLVMBuilderRef builder = base->gallivm->builder;
2981 unsigned i, j, first_vgpr, vgpr;
2982
2983 LLVMValueRef color[8][4] = {};
2984 LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
2985 LLVMValueRef ret;
2986
2987 /* Read the output values. */
2988 for (i = 0; i < info->num_outputs; i++) {
2989 unsigned semantic_name = info->output_semantic_name[i];
2990 unsigned semantic_index = info->output_semantic_index[i];
2991
2992 switch (semantic_name) {
2993 case TGSI_SEMANTIC_COLOR:
2994 assert(semantic_index < 8);
2995 for (j = 0; j < 4; j++) {
2996 LLVMValueRef ptr = ctx->outputs[i][j];
2997 LLVMValueRef result = LLVMBuildLoad(builder, ptr, "");
2998 color[semantic_index][j] = result;
2999 }
3000 break;
3001 case TGSI_SEMANTIC_POSITION:
3002 depth = LLVMBuildLoad(builder,
3003 ctx->outputs[i][2], "");
3004 break;
3005 case TGSI_SEMANTIC_STENCIL:
3006 stencil = LLVMBuildLoad(builder,
3007 ctx->outputs[i][1], "");
3008 break;
3009 case TGSI_SEMANTIC_SAMPLEMASK:
3010 samplemask = LLVMBuildLoad(builder,
3011 ctx->outputs[i][0], "");
3012 break;
3013 default:
3014 fprintf(stderr, "Warning: SI unhandled fs output type:%d\n",
3015 semantic_name);
3016 }
3017 }
3018
3019 /* Fill the return structure. */
3020 ret = ctx->return_value;
3021
3022 /* Set SGPRs. */
3023 ret = LLVMBuildInsertValue(builder, ret,
3024 bitcast(bld_base, TGSI_TYPE_SIGNED,
3025 LLVMGetParam(ctx->main_fn,
3026 SI_PARAM_ALPHA_REF)),
3027 SI_SGPR_ALPHA_REF, "");
3028
3029 /* Set VGPRs */
3030 first_vgpr = vgpr = SI_SGPR_ALPHA_REF + 1;
3031 for (i = 0; i < ARRAY_SIZE(color); i++) {
3032 if (!color[i][0])
3033 continue;
3034
3035 for (j = 0; j < 4; j++)
3036 ret = LLVMBuildInsertValue(builder, ret, color[i][j], vgpr++, "");
3037 }
3038 if (depth)
3039 ret = LLVMBuildInsertValue(builder, ret, depth, vgpr++, "");
3040 if (stencil)
3041 ret = LLVMBuildInsertValue(builder, ret, stencil, vgpr++, "");
3042 if (samplemask)
3043 ret = LLVMBuildInsertValue(builder, ret, samplemask, vgpr++, "");
3044
3045 /* Add the input sample mask for smoothing at the end. */
3046 if (vgpr < first_vgpr + PS_EPILOG_SAMPLEMASK_MIN_LOC)
3047 vgpr = first_vgpr + PS_EPILOG_SAMPLEMASK_MIN_LOC;
3048 ret = LLVMBuildInsertValue(builder, ret,
3049 LLVMGetParam(ctx->main_fn,
3050 SI_PARAM_SAMPLE_COVERAGE), vgpr++, "");
3051
3052 ctx->return_value = ret;
3053 }
3054
3055 /**
3056 * Given a v8i32 resource descriptor for a buffer, extract the size of the
3057 * buffer in number of elements and return it as an i32.
3058 */
3059 static LLVMValueRef get_buffer_size(
3060 struct lp_build_tgsi_context *bld_base,
3061 LLVMValueRef descriptor)
3062 {
3063 struct si_shader_context *ctx = si_shader_context(bld_base);
3064 struct gallivm_state *gallivm = bld_base->base.gallivm;
3065 LLVMBuilderRef builder = gallivm->builder;
3066 LLVMValueRef size =
3067 LLVMBuildExtractElement(builder, descriptor,
3068 lp_build_const_int32(gallivm, 2), "");
3069
3070 if (ctx->screen->b.chip_class == VI) {
3071 /* On VI, the descriptor contains the size in bytes,
3072 * but TXQ must return the size in elements.
3073 * The stride is always non-zero for resources using TXQ.
3074 */
3075 LLVMValueRef stride =
3076 LLVMBuildExtractElement(builder, descriptor,
3077 lp_build_const_int32(gallivm, 1), "");
3078 stride = LLVMBuildLShr(builder, stride,
3079 lp_build_const_int32(gallivm, 16), "");
3080 stride = LLVMBuildAnd(builder, stride,
3081 lp_build_const_int32(gallivm, 0x3FFF), "");
3082
3083 size = LLVMBuildUDiv(builder, size, stride, "");
3084 }
3085
3086 return size;
3087 }
3088
3089 static void build_tex_intrinsic(const struct lp_build_tgsi_action *action,
3090 struct lp_build_tgsi_context *bld_base,
3091 struct lp_build_emit_data *emit_data);
3092
3093 /* Prevent optimizations (at least of memory accesses) across the current
3094 * point in the program by emitting empty inline assembly that is marked as
3095 * having side effects.
3096 */
3097 #if 0 /* unused currently */
3098 static void emit_optimization_barrier(struct si_shader_context *ctx)
3099 {
3100 LLVMBuilderRef builder = ctx->gallivm.builder;
3101 LLVMTypeRef ftype = LLVMFunctionType(ctx->voidt, NULL, 0, false);
3102 LLVMValueRef inlineasm = LLVMConstInlineAsm(ftype, "", "", true, false);
3103 LLVMBuildCall(builder, inlineasm, NULL, 0, "");
3104 }
3105 #endif
3106
3107 /* Combine these with & instead of |. */
3108 #define NOOP_WAITCNT 0xf7f
3109 #define LGKM_CNT 0x07f
3110 #define VM_CNT 0xf70
3111
3112 static void emit_waitcnt(struct si_shader_context *ctx, unsigned simm16)
3113 {
3114 struct gallivm_state *gallivm = &ctx->gallivm;
3115 LLVMBuilderRef builder = gallivm->builder;
3116 LLVMValueRef args[1] = {
3117 lp_build_const_int32(gallivm, simm16)
3118 };
3119 lp_build_intrinsic(builder, "llvm.amdgcn.s.waitcnt",
3120 ctx->voidt, args, 1, 0);
3121 }
3122
3123 static void membar_emit(
3124 const struct lp_build_tgsi_action *action,
3125 struct lp_build_tgsi_context *bld_base,
3126 struct lp_build_emit_data *emit_data)
3127 {
3128 struct si_shader_context *ctx = si_shader_context(bld_base);
3129 LLVMValueRef src0 = lp_build_emit_fetch(bld_base, emit_data->inst, 0, 0);
3130 unsigned flags = LLVMConstIntGetZExtValue(src0);
3131 unsigned waitcnt = NOOP_WAITCNT;
3132
3133 if (flags & TGSI_MEMBAR_THREAD_GROUP)
3134 waitcnt &= VM_CNT & LGKM_CNT;
3135
3136 if (flags & (TGSI_MEMBAR_ATOMIC_BUFFER |
3137 TGSI_MEMBAR_SHADER_BUFFER |
3138 TGSI_MEMBAR_SHADER_IMAGE))
3139 waitcnt &= VM_CNT;
3140
3141 if (flags & TGSI_MEMBAR_SHARED)
3142 waitcnt &= LGKM_CNT;
3143
3144 if (waitcnt != NOOP_WAITCNT)
3145 emit_waitcnt(ctx, waitcnt);
3146 }
3147
3148 static void clock_emit(
3149 const struct lp_build_tgsi_action *action,
3150 struct lp_build_tgsi_context *bld_base,
3151 struct lp_build_emit_data *emit_data)
3152 {
3153 struct si_shader_context *ctx = si_shader_context(bld_base);
3154 struct gallivm_state *gallivm = &ctx->gallivm;
3155 LLVMValueRef tmp;
3156
3157 tmp = lp_build_intrinsic(gallivm->builder, "llvm.readcyclecounter",
3158 ctx->i64, NULL, 0, 0);
3159 tmp = LLVMBuildBitCast(gallivm->builder, tmp, ctx->v2i32, "");
3160
3161 emit_data->output[0] =
3162 LLVMBuildExtractElement(gallivm->builder, tmp, ctx->i32_0, "");
3163 emit_data->output[1] =
3164 LLVMBuildExtractElement(gallivm->builder, tmp, ctx->i32_1, "");
3165 }
3166
3167 static LLVMValueRef
3168 shader_buffer_fetch_rsrc(struct si_shader_context *ctx,
3169 const struct tgsi_full_src_register *reg)
3170 {
3171 LLVMValueRef index;
3172 LLVMValueRef rsrc_ptr = LLVMGetParam(ctx->main_fn,
3173 SI_PARAM_SHADER_BUFFERS);
3174
3175 if (!reg->Register.Indirect)
3176 index = LLVMConstInt(ctx->i32, reg->Register.Index, 0);
3177 else
3178 index = get_bounded_indirect_index(ctx, &reg->Indirect,
3179 reg->Register.Index,
3180 SI_NUM_SHADER_BUFFERS);
3181
3182 return ac_build_indexed_load_const(&ctx->ac, rsrc_ptr, index);
3183 }
3184
3185 static bool tgsi_is_array_sampler(unsigned target)
3186 {
3187 return target == TGSI_TEXTURE_1D_ARRAY ||
3188 target == TGSI_TEXTURE_SHADOW1D_ARRAY ||
3189 target == TGSI_TEXTURE_2D_ARRAY ||
3190 target == TGSI_TEXTURE_SHADOW2D_ARRAY ||
3191 target == TGSI_TEXTURE_CUBE_ARRAY ||
3192 target == TGSI_TEXTURE_SHADOWCUBE_ARRAY ||
3193 target == TGSI_TEXTURE_2D_ARRAY_MSAA;
3194 }
3195
3196 static bool tgsi_is_array_image(unsigned target)
3197 {
3198 return target == TGSI_TEXTURE_3D ||
3199 target == TGSI_TEXTURE_CUBE ||
3200 target == TGSI_TEXTURE_1D_ARRAY ||
3201 target == TGSI_TEXTURE_2D_ARRAY ||
3202 target == TGSI_TEXTURE_CUBE_ARRAY ||
3203 target == TGSI_TEXTURE_2D_ARRAY_MSAA;
3204 }
3205
3206 /**
3207 * Given a 256-bit resource descriptor, force the DCC enable bit to off.
3208 *
3209 * At least on Tonga, executing image stores on images with DCC enabled and
3210 * non-trivial can eventually lead to lockups. This can occur when an
3211 * application binds an image as read-only but then uses a shader that writes
3212 * to it. The OpenGL spec allows almost arbitrarily bad behavior (including
3213 * program termination) in this case, but it doesn't cost much to be a bit
3214 * nicer: disabling DCC in the shader still leads to undefined results but
3215 * avoids the lockup.
3216 */
3217 static LLVMValueRef force_dcc_off(struct si_shader_context *ctx,
3218 LLVMValueRef rsrc)
3219 {
3220 if (ctx->screen->b.chip_class <= CIK) {
3221 return rsrc;
3222 } else {
3223 LLVMBuilderRef builder = ctx->gallivm.builder;
3224 LLVMValueRef i32_6 = LLVMConstInt(ctx->i32, 6, 0);
3225 LLVMValueRef i32_C = LLVMConstInt(ctx->i32, C_008F28_COMPRESSION_EN, 0);
3226 LLVMValueRef tmp;
3227
3228 tmp = LLVMBuildExtractElement(builder, rsrc, i32_6, "");
3229 tmp = LLVMBuildAnd(builder, tmp, i32_C, "");
3230 return LLVMBuildInsertElement(builder, rsrc, tmp, i32_6, "");
3231 }
3232 }
3233
3234 static LLVMTypeRef const_array(LLVMTypeRef elem_type, int num_elements)
3235 {
3236 return LLVMPointerType(LLVMArrayType(elem_type, num_elements),
3237 CONST_ADDR_SPACE);
3238 }
3239
3240 static LLVMValueRef load_image_desc(struct si_shader_context *ctx,
3241 LLVMValueRef list, LLVMValueRef index,
3242 unsigned target)
3243 {
3244 LLVMBuilderRef builder = ctx->gallivm.builder;
3245
3246 if (target == TGSI_TEXTURE_BUFFER) {
3247 index = LLVMBuildMul(builder, index,
3248 LLVMConstInt(ctx->i32, 2, 0), "");
3249 index = LLVMBuildAdd(builder, index,
3250 LLVMConstInt(ctx->i32, 1, 0), "");
3251 list = LLVMBuildPointerCast(builder, list,
3252 const_array(ctx->v4i32, 0), "");
3253 }
3254
3255 return ac_build_indexed_load_const(&ctx->ac, list, index);
3256 }
3257
3258 /**
3259 * Load the resource descriptor for \p image.
3260 */
3261 static void
3262 image_fetch_rsrc(
3263 struct lp_build_tgsi_context *bld_base,
3264 const struct tgsi_full_src_register *image,
3265 bool is_store, unsigned target,
3266 LLVMValueRef *rsrc)
3267 {
3268 struct si_shader_context *ctx = si_shader_context(bld_base);
3269 LLVMValueRef rsrc_ptr = LLVMGetParam(ctx->main_fn,
3270 SI_PARAM_IMAGES);
3271 LLVMValueRef index;
3272 bool dcc_off = is_store;
3273
3274 assert(image->Register.File == TGSI_FILE_IMAGE);
3275
3276 if (!image->Register.Indirect) {
3277 const struct tgsi_shader_info *info = bld_base->info;
3278 unsigned images_writemask = info->images_store |
3279 info->images_atomic;
3280
3281 index = LLVMConstInt(ctx->i32, image->Register.Index, 0);
3282
3283 if (images_writemask & (1 << image->Register.Index))
3284 dcc_off = true;
3285 } else {
3286 /* From the GL_ARB_shader_image_load_store extension spec:
3287 *
3288 * If a shader performs an image load, store, or atomic
3289 * operation using an image variable declared as an array,
3290 * and if the index used to select an individual element is
3291 * negative or greater than or equal to the size of the
3292 * array, the results of the operation are undefined but may
3293 * not lead to termination.
3294 */
3295 index = get_bounded_indirect_index(ctx, &image->Indirect,
3296 image->Register.Index,
3297 SI_NUM_IMAGES);
3298 }
3299
3300 *rsrc = load_image_desc(ctx, rsrc_ptr, index, target);
3301 if (dcc_off && target != TGSI_TEXTURE_BUFFER)
3302 *rsrc = force_dcc_off(ctx, *rsrc);
3303 }
3304
3305 static LLVMValueRef image_fetch_coords(
3306 struct lp_build_tgsi_context *bld_base,
3307 const struct tgsi_full_instruction *inst,
3308 unsigned src)
3309 {
3310 struct si_shader_context *ctx = si_shader_context(bld_base);
3311 struct gallivm_state *gallivm = bld_base->base.gallivm;
3312 LLVMBuilderRef builder = gallivm->builder;
3313 unsigned target = inst->Memory.Texture;
3314 unsigned num_coords = tgsi_util_get_texture_coord_dim(target);
3315 LLVMValueRef coords[4];
3316 LLVMValueRef tmp;
3317 int chan;
3318
3319 for (chan = 0; chan < num_coords; ++chan) {
3320 tmp = lp_build_emit_fetch(bld_base, inst, src, chan);
3321 tmp = LLVMBuildBitCast(builder, tmp, bld_base->uint_bld.elem_type, "");
3322 coords[chan] = tmp;
3323 }
3324
3325 /* 1D textures are allocated and used as 2D on GFX9. */
3326 if (ctx->screen->b.chip_class >= GFX9) {
3327 if (target == TGSI_TEXTURE_1D) {
3328 coords[1] = bld_base->uint_bld.zero;
3329 num_coords++;
3330 } else if (target == TGSI_TEXTURE_1D_ARRAY) {
3331 coords[2] = coords[1];
3332 coords[1] = bld_base->uint_bld.zero;
3333 }
3334 }
3335
3336 if (num_coords == 1)
3337 return coords[0];
3338
3339 if (num_coords == 3) {
3340 /* LLVM has difficulties lowering 3-element vectors. */
3341 coords[3] = bld_base->uint_bld.undef;
3342 num_coords = 4;
3343 }
3344
3345 return lp_build_gather_values(gallivm, coords, num_coords);
3346 }
3347
3348 /**
3349 * Append the extra mode bits that are used by image load and store.
3350 */
3351 static void image_append_args(
3352 struct si_shader_context *ctx,
3353 struct lp_build_emit_data * emit_data,
3354 unsigned target,
3355 bool atomic,
3356 bool force_glc)
3357 {
3358 const struct tgsi_full_instruction *inst = emit_data->inst;
3359 LLVMValueRef i1false = LLVMConstInt(ctx->i1, 0, 0);
3360 LLVMValueRef i1true = LLVMConstInt(ctx->i1, 1, 0);
3361 LLVMValueRef r128 = i1false;
3362 LLVMValueRef da = tgsi_is_array_image(target) ? i1true : i1false;
3363 LLVMValueRef glc =
3364 force_glc ||
3365 inst->Memory.Qualifier & (TGSI_MEMORY_COHERENT | TGSI_MEMORY_VOLATILE) ?
3366 i1true : i1false;
3367 LLVMValueRef slc = i1false;
3368 LLVMValueRef lwe = i1false;
3369
3370 if (atomic || (HAVE_LLVM <= 0x0309)) {
3371 emit_data->args[emit_data->arg_count++] = r128;
3372 emit_data->args[emit_data->arg_count++] = da;
3373 if (!atomic) {
3374 emit_data->args[emit_data->arg_count++] = glc;
3375 }
3376 emit_data->args[emit_data->arg_count++] = slc;
3377 return;
3378 }
3379
3380 /* HAVE_LLVM >= 0x0400 */
3381 emit_data->args[emit_data->arg_count++] = glc;
3382 emit_data->args[emit_data->arg_count++] = slc;
3383 emit_data->args[emit_data->arg_count++] = lwe;
3384 emit_data->args[emit_data->arg_count++] = da;
3385 }
3386
3387 /**
3388 * Append the resource and indexing arguments for buffer intrinsics.
3389 *
3390 * \param rsrc the v4i32 buffer resource
3391 * \param index index into the buffer (stride-based)
3392 * \param offset byte offset into the buffer
3393 */
3394 static void buffer_append_args(
3395 struct si_shader_context *ctx,
3396 struct lp_build_emit_data *emit_data,
3397 LLVMValueRef rsrc,
3398 LLVMValueRef index,
3399 LLVMValueRef offset,
3400 bool atomic,
3401 bool force_glc)
3402 {
3403 const struct tgsi_full_instruction *inst = emit_data->inst;
3404 LLVMValueRef i1false = LLVMConstInt(ctx->i1, 0, 0);
3405 LLVMValueRef i1true = LLVMConstInt(ctx->i1, 1, 0);
3406
3407 emit_data->args[emit_data->arg_count++] = rsrc;
3408 emit_data->args[emit_data->arg_count++] = index; /* vindex */
3409 emit_data->args[emit_data->arg_count++] = offset; /* voffset */
3410 if (!atomic) {
3411 emit_data->args[emit_data->arg_count++] =
3412 force_glc ||
3413 inst->Memory.Qualifier & (TGSI_MEMORY_COHERENT | TGSI_MEMORY_VOLATILE) ?
3414 i1true : i1false; /* glc */
3415 }
3416 emit_data->args[emit_data->arg_count++] = i1false; /* slc */
3417 }
3418
3419 static void load_fetch_args(
3420 struct lp_build_tgsi_context * bld_base,
3421 struct lp_build_emit_data * emit_data)
3422 {
3423 struct si_shader_context *ctx = si_shader_context(bld_base);
3424 struct gallivm_state *gallivm = bld_base->base.gallivm;
3425 const struct tgsi_full_instruction * inst = emit_data->inst;
3426 unsigned target = inst->Memory.Texture;
3427 LLVMValueRef rsrc;
3428
3429 emit_data->dst_type = LLVMVectorType(bld_base->base.elem_type, 4);
3430
3431 if (inst->Src[0].Register.File == TGSI_FILE_BUFFER) {
3432 LLVMBuilderRef builder = gallivm->builder;
3433 LLVMValueRef offset;
3434 LLVMValueRef tmp;
3435
3436 rsrc = shader_buffer_fetch_rsrc(ctx, &inst->Src[0]);
3437
3438 tmp = lp_build_emit_fetch(bld_base, inst, 1, 0);
3439 offset = LLVMBuildBitCast(builder, tmp, bld_base->uint_bld.elem_type, "");
3440
3441 buffer_append_args(ctx, emit_data, rsrc, bld_base->uint_bld.zero,
3442 offset, false, false);
3443 } else if (inst->Src[0].Register.File == TGSI_FILE_IMAGE) {
3444 LLVMValueRef coords;
3445
3446 image_fetch_rsrc(bld_base, &inst->Src[0], false, target, &rsrc);
3447 coords = image_fetch_coords(bld_base, inst, 1);
3448
3449 if (target == TGSI_TEXTURE_BUFFER) {
3450 buffer_append_args(ctx, emit_data, rsrc, coords,
3451 bld_base->uint_bld.zero, false, false);
3452 } else {
3453 emit_data->args[0] = coords;
3454 emit_data->args[1] = rsrc;
3455 emit_data->args[2] = lp_build_const_int32(gallivm, 15); /* dmask */
3456 emit_data->arg_count = 3;
3457
3458 image_append_args(ctx, emit_data, target, false, false);
3459 }
3460 }
3461 }
3462
3463 static unsigned get_load_intr_attribs(bool readonly_memory)
3464 {
3465 /* READNONE means writes can't affect it, while READONLY means that
3466 * writes can affect it. */
3467 return readonly_memory && HAVE_LLVM >= 0x0400 ?
3468 LP_FUNC_ATTR_READNONE :
3469 LP_FUNC_ATTR_READONLY;
3470 }
3471
3472 static unsigned get_store_intr_attribs(bool writeonly_memory)
3473 {
3474 return writeonly_memory && HAVE_LLVM >= 0x0400 ?
3475 LP_FUNC_ATTR_INACCESSIBLE_MEM_ONLY :
3476 LP_FUNC_ATTR_WRITEONLY;
3477 }
3478
3479 static void load_emit_buffer(struct si_shader_context *ctx,
3480 struct lp_build_emit_data *emit_data,
3481 bool readonly_memory)
3482 {
3483 const struct tgsi_full_instruction *inst = emit_data->inst;
3484 struct gallivm_state *gallivm = &ctx->gallivm;
3485 LLVMBuilderRef builder = gallivm->builder;
3486 uint writemask = inst->Dst[0].Register.WriteMask;
3487 uint count = util_last_bit(writemask);
3488 const char *intrinsic_name;
3489 LLVMTypeRef dst_type;
3490
3491 switch (count) {
3492 case 1:
3493 intrinsic_name = "llvm.amdgcn.buffer.load.f32";
3494 dst_type = ctx->f32;
3495 break;
3496 case 2:
3497 intrinsic_name = "llvm.amdgcn.buffer.load.v2f32";
3498 dst_type = LLVMVectorType(ctx->f32, 2);
3499 break;
3500 default: // 3 & 4
3501 intrinsic_name = "llvm.amdgcn.buffer.load.v4f32";
3502 dst_type = ctx->v4f32;
3503 count = 4;
3504 }
3505
3506 emit_data->output[emit_data->chan] = lp_build_intrinsic(
3507 builder, intrinsic_name, dst_type,
3508 emit_data->args, emit_data->arg_count,
3509 get_load_intr_attribs(readonly_memory));
3510 }
3511
3512 static LLVMValueRef get_memory_ptr(struct si_shader_context *ctx,
3513 const struct tgsi_full_instruction *inst,
3514 LLVMTypeRef type, int arg)
3515 {
3516 struct gallivm_state *gallivm = &ctx->gallivm;
3517 LLVMBuilderRef builder = gallivm->builder;
3518 LLVMValueRef offset, ptr;
3519 int addr_space;
3520
3521 offset = lp_build_emit_fetch(&ctx->bld_base, inst, arg, 0);
3522 offset = LLVMBuildBitCast(builder, offset, ctx->i32, "");
3523
3524 ptr = ctx->shared_memory;
3525 ptr = LLVMBuildGEP(builder, ptr, &offset, 1, "");
3526 addr_space = LLVMGetPointerAddressSpace(LLVMTypeOf(ptr));
3527 ptr = LLVMBuildBitCast(builder, ptr, LLVMPointerType(type, addr_space), "");
3528
3529 return ptr;
3530 }
3531
3532 static void load_emit_memory(
3533 struct si_shader_context *ctx,
3534 struct lp_build_emit_data *emit_data)
3535 {
3536 const struct tgsi_full_instruction *inst = emit_data->inst;
3537 struct lp_build_context *base = &ctx->bld_base.base;
3538 struct gallivm_state *gallivm = &ctx->gallivm;
3539 LLVMBuilderRef builder = gallivm->builder;
3540 unsigned writemask = inst->Dst[0].Register.WriteMask;
3541 LLVMValueRef channels[4], ptr, derived_ptr, index;
3542 int chan;
3543
3544 ptr = get_memory_ptr(ctx, inst, base->elem_type, 1);
3545
3546 for (chan = 0; chan < 4; ++chan) {
3547 if (!(writemask & (1 << chan))) {
3548 channels[chan] = LLVMGetUndef(base->elem_type);
3549 continue;
3550 }
3551
3552 index = lp_build_const_int32(gallivm, chan);
3553 derived_ptr = LLVMBuildGEP(builder, ptr, &index, 1, "");
3554 channels[chan] = LLVMBuildLoad(builder, derived_ptr, "");
3555 }
3556 emit_data->output[emit_data->chan] = lp_build_gather_values(gallivm, channels, 4);
3557 }
3558
3559 /**
3560 * Return true if the memory accessed by a LOAD or STORE instruction is
3561 * read-only or write-only, respectively.
3562 *
3563 * \param shader_buffers_reverse_access_mask
3564 * For LOAD, set this to (store | atomic) slot usage in the shader.
3565 * For STORE, set this to (load | atomic) slot usage in the shader.
3566 * \param images_reverse_access_mask Same as above, but for images.
3567 */
3568 static bool is_oneway_access_only(const struct tgsi_full_instruction *inst,
3569 const struct tgsi_shader_info *info,
3570 unsigned shader_buffers_reverse_access_mask,
3571 unsigned images_reverse_access_mask)
3572 {
3573 /* RESTRICT means NOALIAS.
3574 * If there are no writes, we can assume the accessed memory is read-only.
3575 * If there are no reads, we can assume the accessed memory is write-only.
3576 */
3577 if (inst->Memory.Qualifier & TGSI_MEMORY_RESTRICT) {
3578 unsigned reverse_access_mask;
3579
3580 if (inst->Src[0].Register.File == TGSI_FILE_BUFFER) {
3581 reverse_access_mask = shader_buffers_reverse_access_mask;
3582 } else if (inst->Memory.Texture == TGSI_TEXTURE_BUFFER) {
3583 reverse_access_mask = info->images_buffers &
3584 images_reverse_access_mask;
3585 } else {
3586 reverse_access_mask = ~info->images_buffers &
3587 images_reverse_access_mask;
3588 }
3589
3590 if (inst->Src[0].Register.Indirect) {
3591 if (!reverse_access_mask)
3592 return true;
3593 } else {
3594 if (!(reverse_access_mask &
3595 (1u << inst->Src[0].Register.Index)))
3596 return true;
3597 }
3598 }
3599
3600 /* If there are no buffer writes (for both shader buffers & image
3601 * buffers), it implies that buffer memory is read-only.
3602 * If there are no buffer reads (for both shader buffers & image
3603 * buffers), it implies that buffer memory is write-only.
3604 *
3605 * Same for the case when there are no writes/reads for non-buffer
3606 * images.
3607 */
3608 if (inst->Src[0].Register.File == TGSI_FILE_BUFFER ||
3609 (inst->Src[0].Register.File == TGSI_FILE_IMAGE &&
3610 inst->Memory.Texture == TGSI_TEXTURE_BUFFER)) {
3611 if (!shader_buffers_reverse_access_mask &&
3612 !(info->images_buffers & images_reverse_access_mask))
3613 return true;
3614 } else {
3615 if (!(~info->images_buffers & images_reverse_access_mask))
3616 return true;
3617 }
3618 return false;
3619 }
3620
3621 static void load_emit(
3622 const struct lp_build_tgsi_action *action,
3623 struct lp_build_tgsi_context *bld_base,
3624 struct lp_build_emit_data *emit_data)
3625 {
3626 struct si_shader_context *ctx = si_shader_context(bld_base);
3627 struct gallivm_state *gallivm = bld_base->base.gallivm;
3628 LLVMBuilderRef builder = gallivm->builder;
3629 const struct tgsi_full_instruction * inst = emit_data->inst;
3630 const struct tgsi_shader_info *info = &ctx->shader->selector->info;
3631 char intrinsic_name[64];
3632 bool readonly_memory = false;
3633
3634 if (inst->Src[0].Register.File == TGSI_FILE_MEMORY) {
3635 load_emit_memory(ctx, emit_data);
3636 return;
3637 }
3638
3639 if (inst->Memory.Qualifier & TGSI_MEMORY_VOLATILE)
3640 emit_waitcnt(ctx, VM_CNT);
3641
3642 readonly_memory = !(inst->Memory.Qualifier & TGSI_MEMORY_VOLATILE) &&
3643 is_oneway_access_only(inst, info,
3644 info->shader_buffers_store |
3645 info->shader_buffers_atomic,
3646 info->images_store |
3647 info->images_atomic);
3648
3649 if (inst->Src[0].Register.File == TGSI_FILE_BUFFER) {
3650 load_emit_buffer(ctx, emit_data, readonly_memory);
3651 return;
3652 }
3653
3654 if (inst->Memory.Texture == TGSI_TEXTURE_BUFFER) {
3655 emit_data->output[emit_data->chan] =
3656 lp_build_intrinsic(
3657 builder, "llvm.amdgcn.buffer.load.format.v4f32", emit_data->dst_type,
3658 emit_data->args, emit_data->arg_count,
3659 get_load_intr_attribs(readonly_memory));
3660 } else {
3661 ac_get_image_intr_name("llvm.amdgcn.image.load",
3662 emit_data->dst_type, /* vdata */
3663 LLVMTypeOf(emit_data->args[0]), /* coords */
3664 LLVMTypeOf(emit_data->args[1]), /* rsrc */
3665 intrinsic_name, sizeof(intrinsic_name));
3666
3667 emit_data->output[emit_data->chan] =
3668 lp_build_intrinsic(
3669 builder, intrinsic_name, emit_data->dst_type,
3670 emit_data->args, emit_data->arg_count,
3671 get_load_intr_attribs(readonly_memory));
3672 }
3673 }
3674
3675 static void store_fetch_args(
3676 struct lp_build_tgsi_context * bld_base,
3677 struct lp_build_emit_data * emit_data)
3678 {
3679 struct si_shader_context *ctx = si_shader_context(bld_base);
3680 struct gallivm_state *gallivm = bld_base->base.gallivm;
3681 LLVMBuilderRef builder = gallivm->builder;
3682 const struct tgsi_full_instruction * inst = emit_data->inst;
3683 struct tgsi_full_src_register memory;
3684 LLVMValueRef chans[4];
3685 LLVMValueRef data;
3686 LLVMValueRef rsrc;
3687 unsigned chan;
3688
3689 emit_data->dst_type = LLVMVoidTypeInContext(gallivm->context);
3690
3691 for (chan = 0; chan < 4; ++chan) {
3692 chans[chan] = lp_build_emit_fetch(bld_base, inst, 1, chan);
3693 }
3694 data = lp_build_gather_values(gallivm, chans, 4);
3695
3696 emit_data->args[emit_data->arg_count++] = data;
3697
3698 memory = tgsi_full_src_register_from_dst(&inst->Dst[0]);
3699
3700 if (inst->Dst[0].Register.File == TGSI_FILE_BUFFER) {
3701 LLVMValueRef offset;
3702 LLVMValueRef tmp;
3703
3704 rsrc = shader_buffer_fetch_rsrc(ctx, &memory);
3705
3706 tmp = lp_build_emit_fetch(bld_base, inst, 0, 0);
3707 offset = LLVMBuildBitCast(builder, tmp, bld_base->uint_bld.elem_type, "");
3708
3709 buffer_append_args(ctx, emit_data, rsrc, bld_base->uint_bld.zero,
3710 offset, false, false);
3711 } else if (inst->Dst[0].Register.File == TGSI_FILE_IMAGE) {
3712 unsigned target = inst->Memory.Texture;
3713 LLVMValueRef coords;
3714
3715 /* 8bit/16bit TC L1 write corruption bug on SI.
3716 * All store opcodes not aligned to a dword are affected.
3717 *
3718 * The only way to get unaligned stores in radeonsi is through
3719 * shader images.
3720 */
3721 bool force_glc = ctx->screen->b.chip_class == SI;
3722
3723 coords = image_fetch_coords(bld_base, inst, 0);
3724
3725 if (target == TGSI_TEXTURE_BUFFER) {
3726 image_fetch_rsrc(bld_base, &memory, true, target, &rsrc);
3727 buffer_append_args(ctx, emit_data, rsrc, coords,
3728 bld_base->uint_bld.zero, false, force_glc);
3729 } else {
3730 emit_data->args[1] = coords;
3731 image_fetch_rsrc(bld_base, &memory, true, target,
3732 &emit_data->args[2]);
3733 emit_data->args[3] = lp_build_const_int32(gallivm, 15); /* dmask */
3734 emit_data->arg_count = 4;
3735
3736 image_append_args(ctx, emit_data, target, false, force_glc);
3737 }
3738 }
3739 }
3740
3741 static void store_emit_buffer(
3742 struct si_shader_context *ctx,
3743 struct lp_build_emit_data *emit_data,
3744 bool writeonly_memory)
3745 {
3746 const struct tgsi_full_instruction *inst = emit_data->inst;
3747 struct gallivm_state *gallivm = &ctx->gallivm;
3748 LLVMBuilderRef builder = gallivm->builder;
3749 struct lp_build_context *uint_bld = &ctx->bld_base.uint_bld;
3750 LLVMValueRef base_data = emit_data->args[0];
3751 LLVMValueRef base_offset = emit_data->args[3];
3752 unsigned writemask = inst->Dst[0].Register.WriteMask;
3753
3754 while (writemask) {
3755 int start, count;
3756 const char *intrinsic_name;
3757 LLVMValueRef data;
3758 LLVMValueRef offset;
3759 LLVMValueRef tmp;
3760
3761 u_bit_scan_consecutive_range(&writemask, &start, &count);
3762
3763 /* Due to an LLVM limitation, split 3-element writes
3764 * into a 2-element and a 1-element write. */
3765 if (count == 3) {
3766 writemask |= 1 << (start + 2);
3767 count = 2;
3768 }
3769
3770 if (count == 4) {
3771 data = base_data;
3772 intrinsic_name = "llvm.amdgcn.buffer.store.v4f32";
3773 } else if (count == 2) {
3774 LLVMTypeRef v2f32 = LLVMVectorType(ctx->f32, 2);
3775
3776 tmp = LLVMBuildExtractElement(
3777 builder, base_data,
3778 lp_build_const_int32(gallivm, start), "");
3779 data = LLVMBuildInsertElement(
3780 builder, LLVMGetUndef(v2f32), tmp,
3781 uint_bld->zero, "");
3782
3783 tmp = LLVMBuildExtractElement(
3784 builder, base_data,
3785 lp_build_const_int32(gallivm, start + 1), "");
3786 data = LLVMBuildInsertElement(
3787 builder, data, tmp, uint_bld->one, "");
3788
3789 intrinsic_name = "llvm.amdgcn.buffer.store.v2f32";
3790 } else {
3791 assert(count == 1);
3792 data = LLVMBuildExtractElement(
3793 builder, base_data,
3794 lp_build_const_int32(gallivm, start), "");
3795 intrinsic_name = "llvm.amdgcn.buffer.store.f32";
3796 }
3797
3798 offset = base_offset;
3799 if (start != 0) {
3800 offset = LLVMBuildAdd(
3801 builder, offset,
3802 lp_build_const_int32(gallivm, start * 4), "");
3803 }
3804
3805 emit_data->args[0] = data;
3806 emit_data->args[3] = offset;
3807
3808 lp_build_intrinsic(
3809 builder, intrinsic_name, emit_data->dst_type,
3810 emit_data->args, emit_data->arg_count,
3811 get_store_intr_attribs(writeonly_memory));
3812 }
3813 }
3814
3815 static void store_emit_memory(
3816 struct si_shader_context *ctx,
3817 struct lp_build_emit_data *emit_data)
3818 {
3819 const struct tgsi_full_instruction *inst = emit_data->inst;
3820 struct gallivm_state *gallivm = &ctx->gallivm;
3821 struct lp_build_context *base = &ctx->bld_base.base;
3822 LLVMBuilderRef builder = gallivm->builder;
3823 unsigned writemask = inst->Dst[0].Register.WriteMask;
3824 LLVMValueRef ptr, derived_ptr, data, index;
3825 int chan;
3826
3827 ptr = get_memory_ptr(ctx, inst, base->elem_type, 0);
3828
3829 for (chan = 0; chan < 4; ++chan) {
3830 if (!(writemask & (1 << chan))) {
3831 continue;
3832 }
3833 data = lp_build_emit_fetch(&ctx->bld_base, inst, 1, chan);
3834 index = lp_build_const_int32(gallivm, chan);
3835 derived_ptr = LLVMBuildGEP(builder, ptr, &index, 1, "");
3836 LLVMBuildStore(builder, data, derived_ptr);
3837 }
3838 }
3839
3840 static void store_emit(
3841 const struct lp_build_tgsi_action *action,
3842 struct lp_build_tgsi_context *bld_base,
3843 struct lp_build_emit_data *emit_data)
3844 {
3845 struct si_shader_context *ctx = si_shader_context(bld_base);
3846 struct gallivm_state *gallivm = bld_base->base.gallivm;
3847 LLVMBuilderRef builder = gallivm->builder;
3848 const struct tgsi_full_instruction * inst = emit_data->inst;
3849 const struct tgsi_shader_info *info = &ctx->shader->selector->info;
3850 unsigned target = inst->Memory.Texture;
3851 char intrinsic_name[64];
3852 bool writeonly_memory = false;
3853
3854 if (inst->Dst[0].Register.File == TGSI_FILE_MEMORY) {
3855 store_emit_memory(ctx, emit_data);
3856 return;
3857 }
3858
3859 if (inst->Memory.Qualifier & TGSI_MEMORY_VOLATILE)
3860 emit_waitcnt(ctx, VM_CNT);
3861
3862 writeonly_memory = is_oneway_access_only(inst, info,
3863 info->shader_buffers_load |
3864 info->shader_buffers_atomic,
3865 info->images_load |
3866 info->images_atomic);
3867
3868 if (inst->Dst[0].Register.File == TGSI_FILE_BUFFER) {
3869 store_emit_buffer(ctx, emit_data, writeonly_memory);
3870 return;
3871 }
3872
3873 if (target == TGSI_TEXTURE_BUFFER) {
3874 emit_data->output[emit_data->chan] = lp_build_intrinsic(
3875 builder, "llvm.amdgcn.buffer.store.format.v4f32",
3876 emit_data->dst_type, emit_data->args,
3877 emit_data->arg_count,
3878 get_store_intr_attribs(writeonly_memory));
3879 } else {
3880 ac_get_image_intr_name("llvm.amdgcn.image.store",
3881 LLVMTypeOf(emit_data->args[0]), /* vdata */
3882 LLVMTypeOf(emit_data->args[1]), /* coords */
3883 LLVMTypeOf(emit_data->args[2]), /* rsrc */
3884 intrinsic_name, sizeof(intrinsic_name));
3885
3886 emit_data->output[emit_data->chan] =
3887 lp_build_intrinsic(
3888 builder, intrinsic_name, emit_data->dst_type,
3889 emit_data->args, emit_data->arg_count,
3890 get_store_intr_attribs(writeonly_memory));
3891 }
3892 }
3893
3894 static void atomic_fetch_args(
3895 struct lp_build_tgsi_context * bld_base,
3896 struct lp_build_emit_data * emit_data)
3897 {
3898 struct si_shader_context *ctx = si_shader_context(bld_base);
3899 struct gallivm_state *gallivm = bld_base->base.gallivm;
3900 LLVMBuilderRef builder = gallivm->builder;
3901 const struct tgsi_full_instruction * inst = emit_data->inst;
3902 LLVMValueRef data1, data2;
3903 LLVMValueRef rsrc;
3904 LLVMValueRef tmp;
3905
3906 emit_data->dst_type = bld_base->base.elem_type;
3907
3908 tmp = lp_build_emit_fetch(bld_base, inst, 2, 0);
3909 data1 = LLVMBuildBitCast(builder, tmp, bld_base->uint_bld.elem_type, "");
3910
3911 if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS) {
3912 tmp = lp_build_emit_fetch(bld_base, inst, 3, 0);
3913 data2 = LLVMBuildBitCast(builder, tmp, bld_base->uint_bld.elem_type, "");
3914 }
3915
3916 /* llvm.amdgcn.image/buffer.atomic.cmpswap reflect the hardware order
3917 * of arguments, which is reversed relative to TGSI (and GLSL)
3918 */
3919 if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS)
3920 emit_data->args[emit_data->arg_count++] = data2;
3921 emit_data->args[emit_data->arg_count++] = data1;
3922
3923 if (inst->Src[0].Register.File == TGSI_FILE_BUFFER) {
3924 LLVMValueRef offset;
3925
3926 rsrc = shader_buffer_fetch_rsrc(ctx, &inst->Src[0]);
3927
3928 tmp = lp_build_emit_fetch(bld_base, inst, 1, 0);
3929 offset = LLVMBuildBitCast(builder, tmp, bld_base->uint_bld.elem_type, "");
3930
3931 buffer_append_args(ctx, emit_data, rsrc, bld_base->uint_bld.zero,
3932 offset, true, false);
3933 } else if (inst->Src[0].Register.File == TGSI_FILE_IMAGE) {
3934 unsigned target = inst->Memory.Texture;
3935 LLVMValueRef coords;
3936
3937 image_fetch_rsrc(bld_base, &inst->Src[0], true, target, &rsrc);
3938 coords = image_fetch_coords(bld_base, inst, 1);
3939
3940 if (target == TGSI_TEXTURE_BUFFER) {
3941 buffer_append_args(ctx, emit_data, rsrc, coords,
3942 bld_base->uint_bld.zero, true, false);
3943 } else {
3944 emit_data->args[emit_data->arg_count++] = coords;
3945 emit_data->args[emit_data->arg_count++] = rsrc;
3946
3947 image_append_args(ctx, emit_data, target, true, false);
3948 }
3949 }
3950 }
3951
3952 static void atomic_emit_memory(struct si_shader_context *ctx,
3953 struct lp_build_emit_data *emit_data) {
3954 struct gallivm_state *gallivm = &ctx->gallivm;
3955 LLVMBuilderRef builder = gallivm->builder;
3956 const struct tgsi_full_instruction * inst = emit_data->inst;
3957 LLVMValueRef ptr, result, arg;
3958
3959 ptr = get_memory_ptr(ctx, inst, ctx->i32, 1);
3960
3961 arg = lp_build_emit_fetch(&ctx->bld_base, inst, 2, 0);
3962 arg = LLVMBuildBitCast(builder, arg, ctx->i32, "");
3963
3964 if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS) {
3965 LLVMValueRef new_data;
3966 new_data = lp_build_emit_fetch(&ctx->bld_base,
3967 inst, 3, 0);
3968
3969 new_data = LLVMBuildBitCast(builder, new_data, ctx->i32, "");
3970
3971 #if HAVE_LLVM >= 0x309
3972 result = LLVMBuildAtomicCmpXchg(builder, ptr, arg, new_data,
3973 LLVMAtomicOrderingSequentiallyConsistent,
3974 LLVMAtomicOrderingSequentiallyConsistent,
3975 false);
3976 #endif
3977
3978 result = LLVMBuildExtractValue(builder, result, 0, "");
3979 } else {
3980 LLVMAtomicRMWBinOp op;
3981
3982 switch(inst->Instruction.Opcode) {
3983 case TGSI_OPCODE_ATOMUADD:
3984 op = LLVMAtomicRMWBinOpAdd;
3985 break;
3986 case TGSI_OPCODE_ATOMXCHG:
3987 op = LLVMAtomicRMWBinOpXchg;
3988 break;
3989 case TGSI_OPCODE_ATOMAND:
3990 op = LLVMAtomicRMWBinOpAnd;
3991 break;
3992 case TGSI_OPCODE_ATOMOR:
3993 op = LLVMAtomicRMWBinOpOr;
3994 break;
3995 case TGSI_OPCODE_ATOMXOR:
3996 op = LLVMAtomicRMWBinOpXor;
3997 break;
3998 case TGSI_OPCODE_ATOMUMIN:
3999 op = LLVMAtomicRMWBinOpUMin;
4000 break;
4001 case TGSI_OPCODE_ATOMUMAX:
4002 op = LLVMAtomicRMWBinOpUMax;
4003 break;
4004 case TGSI_OPCODE_ATOMIMIN:
4005 op = LLVMAtomicRMWBinOpMin;
4006 break;
4007 case TGSI_OPCODE_ATOMIMAX:
4008 op = LLVMAtomicRMWBinOpMax;
4009 break;
4010 default:
4011 unreachable("unknown atomic opcode");
4012 }
4013
4014 result = LLVMBuildAtomicRMW(builder, op, ptr, arg,
4015 LLVMAtomicOrderingSequentiallyConsistent,
4016 false);
4017 }
4018 emit_data->output[emit_data->chan] = LLVMBuildBitCast(builder, result, emit_data->dst_type, "");
4019 }
4020
4021 static void atomic_emit(
4022 const struct lp_build_tgsi_action *action,
4023 struct lp_build_tgsi_context *bld_base,
4024 struct lp_build_emit_data *emit_data)
4025 {
4026 struct si_shader_context *ctx = si_shader_context(bld_base);
4027 struct gallivm_state *gallivm = bld_base->base.gallivm;
4028 LLVMBuilderRef builder = gallivm->builder;
4029 const struct tgsi_full_instruction * inst = emit_data->inst;
4030 char intrinsic_name[40];
4031 LLVMValueRef tmp;
4032
4033 if (inst->Src[0].Register.File == TGSI_FILE_MEMORY) {
4034 atomic_emit_memory(ctx, emit_data);
4035 return;
4036 }
4037
4038 if (inst->Src[0].Register.File == TGSI_FILE_BUFFER ||
4039 inst->Memory.Texture == TGSI_TEXTURE_BUFFER) {
4040 snprintf(intrinsic_name, sizeof(intrinsic_name),
4041 "llvm.amdgcn.buffer.atomic.%s", action->intr_name);
4042 } else {
4043 LLVMValueRef coords;
4044 char coords_type[8];
4045
4046 if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS)
4047 coords = emit_data->args[2];
4048 else
4049 coords = emit_data->args[1];
4050
4051 ac_build_type_name_for_intr(LLVMTypeOf(coords), coords_type, sizeof(coords_type));
4052 snprintf(intrinsic_name, sizeof(intrinsic_name),
4053 "llvm.amdgcn.image.atomic.%s.%s",
4054 action->intr_name, coords_type);
4055 }
4056
4057 tmp = lp_build_intrinsic(
4058 builder, intrinsic_name, bld_base->uint_bld.elem_type,
4059 emit_data->args, emit_data->arg_count, 0);
4060 emit_data->output[emit_data->chan] =
4061 LLVMBuildBitCast(builder, tmp, bld_base->base.elem_type, "");
4062 }
4063
4064 static void set_tex_fetch_args(struct si_shader_context *ctx,
4065 struct lp_build_emit_data *emit_data,
4066 unsigned target,
4067 LLVMValueRef res_ptr, LLVMValueRef samp_ptr,
4068 LLVMValueRef *param, unsigned count,
4069 unsigned dmask)
4070 {
4071 struct gallivm_state *gallivm = &ctx->gallivm;
4072 struct ac_image_args args = {};
4073
4074 /* Pad to power of two vector */
4075 while (count < util_next_power_of_two(count))
4076 param[count++] = LLVMGetUndef(ctx->i32);
4077
4078 if (count > 1)
4079 args.addr = lp_build_gather_values(gallivm, param, count);
4080 else
4081 args.addr = param[0];
4082
4083 args.resource = res_ptr;
4084 args.sampler = samp_ptr;
4085 args.dmask = dmask;
4086 args.unorm = target == TGSI_TEXTURE_RECT ||
4087 target == TGSI_TEXTURE_SHADOWRECT;
4088 args.da = tgsi_is_array_sampler(target);
4089
4090 /* Ugly, but we seem to have no other choice right now. */
4091 STATIC_ASSERT(sizeof(args) <= sizeof(emit_data->args));
4092 memcpy(emit_data->args, &args, sizeof(args));
4093 }
4094
4095 static LLVMValueRef fix_resinfo(struct si_shader_context *ctx,
4096 unsigned target, LLVMValueRef out)
4097 {
4098 LLVMBuilderRef builder = ctx->gallivm.builder;
4099
4100 /* 1D textures are allocated and used as 2D on GFX9. */
4101 if (ctx->screen->b.chip_class >= GFX9 &&
4102 (target == TGSI_TEXTURE_1D_ARRAY ||
4103 target == TGSI_TEXTURE_SHADOW1D_ARRAY)) {
4104 LLVMValueRef layers =
4105 LLVMBuildExtractElement(builder, out,
4106 LLVMConstInt(ctx->i32, 2, 0), "");
4107 out = LLVMBuildInsertElement(builder, out, layers,
4108 LLVMConstInt(ctx->i32, 1, 0), "");
4109 }
4110
4111 /* Divide the number of layers by 6 to get the number of cubes. */
4112 if (target == TGSI_TEXTURE_CUBE_ARRAY ||
4113 target == TGSI_TEXTURE_SHADOWCUBE_ARRAY) {
4114 LLVMValueRef imm2 = LLVMConstInt(ctx->i32, 2, 0);
4115
4116 LLVMValueRef z = LLVMBuildExtractElement(builder, out, imm2, "");
4117 z = LLVMBuildSDiv(builder, z, LLVMConstInt(ctx->i32, 6, 0), "");
4118
4119 out = LLVMBuildInsertElement(builder, out, z, imm2, "");
4120 }
4121 return out;
4122 }
4123
4124 static void resq_fetch_args(
4125 struct lp_build_tgsi_context * bld_base,
4126 struct lp_build_emit_data * emit_data)
4127 {
4128 struct si_shader_context *ctx = si_shader_context(bld_base);
4129 const struct tgsi_full_instruction *inst = emit_data->inst;
4130 const struct tgsi_full_src_register *reg = &inst->Src[0];
4131
4132 emit_data->dst_type = ctx->v4i32;
4133
4134 if (reg->Register.File == TGSI_FILE_BUFFER) {
4135 emit_data->args[0] = shader_buffer_fetch_rsrc(ctx, reg);
4136 emit_data->arg_count = 1;
4137 } else if (inst->Memory.Texture == TGSI_TEXTURE_BUFFER) {
4138 image_fetch_rsrc(bld_base, reg, false, inst->Memory.Texture,
4139 &emit_data->args[0]);
4140 emit_data->arg_count = 1;
4141 } else {
4142 LLVMValueRef res_ptr;
4143 unsigned image_target;
4144
4145 if (inst->Memory.Texture == TGSI_TEXTURE_3D)
4146 image_target = TGSI_TEXTURE_2D_ARRAY;
4147 else
4148 image_target = inst->Memory.Texture;
4149
4150 image_fetch_rsrc(bld_base, reg, false, inst->Memory.Texture,
4151 &res_ptr);
4152 set_tex_fetch_args(ctx, emit_data, image_target,
4153 res_ptr, NULL, &bld_base->uint_bld.zero, 1,
4154 0xf);
4155 }
4156 }
4157
4158 static void resq_emit(
4159 const struct lp_build_tgsi_action *action,
4160 struct lp_build_tgsi_context *bld_base,
4161 struct lp_build_emit_data *emit_data)
4162 {
4163 struct si_shader_context *ctx = si_shader_context(bld_base);
4164 struct gallivm_state *gallivm = bld_base->base.gallivm;
4165 LLVMBuilderRef builder = gallivm->builder;
4166 const struct tgsi_full_instruction *inst = emit_data->inst;
4167 LLVMValueRef out;
4168
4169 if (inst->Src[0].Register.File == TGSI_FILE_BUFFER) {
4170 out = LLVMBuildExtractElement(builder, emit_data->args[0],
4171 lp_build_const_int32(gallivm, 2), "");
4172 } else if (inst->Memory.Texture == TGSI_TEXTURE_BUFFER) {
4173 out = get_buffer_size(bld_base, emit_data->args[0]);
4174 } else {
4175 struct ac_image_args args;
4176
4177 memcpy(&args, emit_data->args, sizeof(args)); /* ugly */
4178 args.opcode = ac_image_get_resinfo;
4179 out = ac_build_image_opcode(&ctx->ac, &args);
4180
4181 out = fix_resinfo(ctx, inst->Memory.Texture, out);
4182 }
4183
4184 emit_data->output[emit_data->chan] = out;
4185 }
4186
4187 static const struct lp_build_tgsi_action tex_action;
4188
4189 enum desc_type {
4190 DESC_IMAGE,
4191 DESC_BUFFER,
4192 DESC_FMASK,
4193 DESC_SAMPLER,
4194 };
4195
4196 /**
4197 * Load an image view, fmask view. or sampler state descriptor.
4198 */
4199 static LLVMValueRef load_sampler_desc(struct si_shader_context *ctx,
4200 LLVMValueRef list, LLVMValueRef index,
4201 enum desc_type type)
4202 {
4203 struct gallivm_state *gallivm = &ctx->gallivm;
4204 LLVMBuilderRef builder = gallivm->builder;
4205
4206 switch (type) {
4207 case DESC_IMAGE:
4208 /* The image is at [0:7]. */
4209 index = LLVMBuildMul(builder, index, LLVMConstInt(ctx->i32, 2, 0), "");
4210 break;
4211 case DESC_BUFFER:
4212 /* The buffer is in [4:7]. */
4213 index = LLVMBuildMul(builder, index, LLVMConstInt(ctx->i32, 4, 0), "");
4214 index = LLVMBuildAdd(builder, index, LLVMConstInt(ctx->i32, 1, 0), "");
4215 list = LLVMBuildPointerCast(builder, list,
4216 const_array(ctx->v4i32, 0), "");
4217 break;
4218 case DESC_FMASK:
4219 /* The FMASK is at [8:15]. */
4220 index = LLVMBuildMul(builder, index, LLVMConstInt(ctx->i32, 2, 0), "");
4221 index = LLVMBuildAdd(builder, index, LLVMConstInt(ctx->i32, 1, 0), "");
4222 break;
4223 case DESC_SAMPLER:
4224 /* The sampler state is at [12:15]. */
4225 index = LLVMBuildMul(builder, index, LLVMConstInt(ctx->i32, 4, 0), "");
4226 index = LLVMBuildAdd(builder, index, LLVMConstInt(ctx->i32, 3, 0), "");
4227 list = LLVMBuildPointerCast(builder, list,
4228 const_array(ctx->v4i32, 0), "");
4229 break;
4230 }
4231
4232 return ac_build_indexed_load_const(&ctx->ac, list, index);
4233 }
4234
4235 /* Disable anisotropic filtering if BASE_LEVEL == LAST_LEVEL.
4236 *
4237 * SI-CI:
4238 * If BASE_LEVEL == LAST_LEVEL, the shader must disable anisotropic
4239 * filtering manually. The driver sets img7 to a mask clearing
4240 * MAX_ANISO_RATIO if BASE_LEVEL == LAST_LEVEL. The shader must do:
4241 * s_and_b32 samp0, samp0, img7
4242 *
4243 * VI:
4244 * The ANISO_OVERRIDE sampler field enables this fix in TA.
4245 */
4246 static LLVMValueRef sici_fix_sampler_aniso(struct si_shader_context *ctx,
4247 LLVMValueRef res, LLVMValueRef samp)
4248 {
4249 LLVMBuilderRef builder = ctx->gallivm.builder;
4250 LLVMValueRef img7, samp0;
4251
4252 if (ctx->screen->b.chip_class >= VI)
4253 return samp;
4254
4255 img7 = LLVMBuildExtractElement(builder, res,
4256 LLVMConstInt(ctx->i32, 7, 0), "");
4257 samp0 = LLVMBuildExtractElement(builder, samp,
4258 LLVMConstInt(ctx->i32, 0, 0), "");
4259 samp0 = LLVMBuildAnd(builder, samp0, img7, "");
4260 return LLVMBuildInsertElement(builder, samp, samp0,
4261 LLVMConstInt(ctx->i32, 0, 0), "");
4262 }
4263
4264 static void tex_fetch_ptrs(
4265 struct lp_build_tgsi_context *bld_base,
4266 struct lp_build_emit_data *emit_data,
4267 LLVMValueRef *res_ptr, LLVMValueRef *samp_ptr, LLVMValueRef *fmask_ptr)
4268 {
4269 struct si_shader_context *ctx = si_shader_context(bld_base);
4270 LLVMValueRef list = LLVMGetParam(ctx->main_fn, SI_PARAM_SAMPLERS);
4271 const struct tgsi_full_instruction *inst = emit_data->inst;
4272 const struct tgsi_full_src_register *reg;
4273 unsigned target = inst->Texture.Texture;
4274 unsigned sampler_src;
4275 LLVMValueRef index;
4276
4277 sampler_src = emit_data->inst->Instruction.NumSrcRegs - 1;
4278 reg = &emit_data->inst->Src[sampler_src];
4279
4280 if (reg->Register.Indirect) {
4281 index = get_bounded_indirect_index(ctx,
4282 &reg->Indirect,
4283 reg->Register.Index,
4284 SI_NUM_SAMPLERS);
4285 } else {
4286 index = LLVMConstInt(ctx->i32, reg->Register.Index, 0);
4287 }
4288
4289 if (target == TGSI_TEXTURE_BUFFER)
4290 *res_ptr = load_sampler_desc(ctx, list, index, DESC_BUFFER);
4291 else
4292 *res_ptr = load_sampler_desc(ctx, list, index, DESC_IMAGE);
4293
4294 if (samp_ptr)
4295 *samp_ptr = NULL;
4296 if (fmask_ptr)
4297 *fmask_ptr = NULL;
4298
4299 if (target == TGSI_TEXTURE_2D_MSAA ||
4300 target == TGSI_TEXTURE_2D_ARRAY_MSAA) {
4301 if (fmask_ptr)
4302 *fmask_ptr = load_sampler_desc(ctx, list, index,
4303 DESC_FMASK);
4304 } else if (target != TGSI_TEXTURE_BUFFER) {
4305 if (samp_ptr) {
4306 *samp_ptr = load_sampler_desc(ctx, list, index,
4307 DESC_SAMPLER);
4308 *samp_ptr = sici_fix_sampler_aniso(ctx, *res_ptr, *samp_ptr);
4309 }
4310 }
4311 }
4312
4313 static void txq_fetch_args(
4314 struct lp_build_tgsi_context *bld_base,
4315 struct lp_build_emit_data *emit_data)
4316 {
4317 struct si_shader_context *ctx = si_shader_context(bld_base);
4318 const struct tgsi_full_instruction *inst = emit_data->inst;
4319 unsigned target = inst->Texture.Texture;
4320 LLVMValueRef res_ptr;
4321 LLVMValueRef address;
4322
4323 tex_fetch_ptrs(bld_base, emit_data, &res_ptr, NULL, NULL);
4324
4325 if (target == TGSI_TEXTURE_BUFFER) {
4326 /* Read the size from the buffer descriptor directly. */
4327 emit_data->args[0] = get_buffer_size(bld_base, res_ptr);
4328 return;
4329 }
4330
4331 /* Textures - set the mip level. */
4332 address = lp_build_emit_fetch(bld_base, inst, 0, TGSI_CHAN_X);
4333
4334 set_tex_fetch_args(ctx, emit_data, target, res_ptr,
4335 NULL, &address, 1, 0xf);
4336 }
4337
4338 static void txq_emit(const struct lp_build_tgsi_action *action,
4339 struct lp_build_tgsi_context *bld_base,
4340 struct lp_build_emit_data *emit_data)
4341 {
4342 struct si_shader_context *ctx = si_shader_context(bld_base);
4343 struct ac_image_args args;
4344 unsigned target = emit_data->inst->Texture.Texture;
4345
4346 if (target == TGSI_TEXTURE_BUFFER) {
4347 /* Just return the buffer size. */
4348 emit_data->output[emit_data->chan] = emit_data->args[0];
4349 return;
4350 }
4351
4352 memcpy(&args, emit_data->args, sizeof(args)); /* ugly */
4353
4354 args.opcode = ac_image_get_resinfo;
4355 LLVMValueRef result = ac_build_image_opcode(&ctx->ac, &args);
4356
4357 emit_data->output[emit_data->chan] = fix_resinfo(ctx, target, result);
4358 }
4359
4360 static void tex_fetch_args(
4361 struct lp_build_tgsi_context *bld_base,
4362 struct lp_build_emit_data *emit_data)
4363 {
4364 struct si_shader_context *ctx = si_shader_context(bld_base);
4365 struct gallivm_state *gallivm = bld_base->base.gallivm;
4366 const struct tgsi_full_instruction *inst = emit_data->inst;
4367 unsigned opcode = inst->Instruction.Opcode;
4368 unsigned target = inst->Texture.Texture;
4369 LLVMValueRef coords[5], derivs[6];
4370 LLVMValueRef address[16];
4371 unsigned num_coords = tgsi_util_get_texture_coord_dim(target);
4372 int ref_pos = tgsi_util_get_shadow_ref_src_index(target);
4373 unsigned count = 0;
4374 unsigned chan;
4375 unsigned num_deriv_channels = 0;
4376 bool has_offset = inst->Texture.NumOffsets > 0;
4377 LLVMValueRef res_ptr, samp_ptr, fmask_ptr = NULL;
4378 unsigned dmask = 0xf;
4379
4380 tex_fetch_ptrs(bld_base, emit_data, &res_ptr, &samp_ptr, &fmask_ptr);
4381
4382 if (target == TGSI_TEXTURE_BUFFER) {
4383 emit_data->dst_type = ctx->v4f32;
4384 emit_data->args[0] = LLVMBuildBitCast(gallivm->builder, res_ptr,
4385 ctx->v16i8, "");
4386 emit_data->args[1] = bld_base->uint_bld.zero;
4387 emit_data->args[2] = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_X);
4388 emit_data->arg_count = 3;
4389 return;
4390 }
4391
4392 /* Fetch and project texture coordinates */
4393 coords[3] = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_W);
4394 for (chan = 0; chan < 3; chan++ ) {
4395 coords[chan] = lp_build_emit_fetch(bld_base,
4396 emit_data->inst, 0,
4397 chan);
4398 if (opcode == TGSI_OPCODE_TXP)
4399 coords[chan] = lp_build_emit_llvm_binary(bld_base,
4400 TGSI_OPCODE_DIV,
4401 coords[chan],
4402 coords[3]);
4403 }
4404
4405 if (opcode == TGSI_OPCODE_TXP)
4406 coords[3] = bld_base->base.one;
4407
4408 /* Pack offsets. */
4409 if (has_offset &&
4410 opcode != TGSI_OPCODE_TXF &&
4411 opcode != TGSI_OPCODE_TXF_LZ) {
4412 /* The offsets are six-bit signed integers packed like this:
4413 * X=[5:0], Y=[13:8], and Z=[21:16].
4414 */
4415 LLVMValueRef offset[3], pack;
4416
4417 assert(inst->Texture.NumOffsets == 1);
4418
4419 for (chan = 0; chan < 3; chan++) {
4420 offset[chan] = lp_build_emit_fetch_texoffset(bld_base,
4421 emit_data->inst, 0, chan);
4422 offset[chan] = LLVMBuildAnd(gallivm->builder, offset[chan],
4423 lp_build_const_int32(gallivm, 0x3f), "");
4424 if (chan)
4425 offset[chan] = LLVMBuildShl(gallivm->builder, offset[chan],
4426 lp_build_const_int32(gallivm, chan*8), "");
4427 }
4428
4429 pack = LLVMBuildOr(gallivm->builder, offset[0], offset[1], "");
4430 pack = LLVMBuildOr(gallivm->builder, pack, offset[2], "");
4431 address[count++] = pack;
4432 }
4433
4434 /* Pack LOD bias value */
4435 if (opcode == TGSI_OPCODE_TXB)
4436 address[count++] = coords[3];
4437 if (opcode == TGSI_OPCODE_TXB2)
4438 address[count++] = lp_build_emit_fetch(bld_base, inst, 1, TGSI_CHAN_X);
4439
4440 /* Pack depth comparison value */
4441 if (tgsi_is_shadow_target(target) && opcode != TGSI_OPCODE_LODQ) {
4442 LLVMValueRef z;
4443
4444 if (target == TGSI_TEXTURE_SHADOWCUBE_ARRAY) {
4445 z = lp_build_emit_fetch(bld_base, inst, 1, TGSI_CHAN_X);
4446 } else {
4447 assert(ref_pos >= 0);
4448 z = coords[ref_pos];
4449 }
4450
4451 /* TC-compatible HTILE promotes Z16 and Z24 to Z32_FLOAT,
4452 * so the depth comparison value isn't clamped for Z16 and
4453 * Z24 anymore. Do it manually here.
4454 *
4455 * It's unnecessary if the original texture format was
4456 * Z32_FLOAT, but we don't know that here.
4457 */
4458 if (ctx->screen->b.chip_class == VI)
4459 z = ac_build_clamp(&ctx->ac, z);
4460
4461 address[count++] = z;
4462 }
4463
4464 /* Pack user derivatives */
4465 if (opcode == TGSI_OPCODE_TXD) {
4466 int param, num_src_deriv_channels, num_dst_deriv_channels;
4467
4468 switch (target) {
4469 case TGSI_TEXTURE_3D:
4470 num_src_deriv_channels = 3;
4471 num_dst_deriv_channels = 3;
4472 num_deriv_channels = 3;
4473 break;
4474 case TGSI_TEXTURE_2D:
4475 case TGSI_TEXTURE_SHADOW2D:
4476 case TGSI_TEXTURE_RECT:
4477 case TGSI_TEXTURE_SHADOWRECT:
4478 case TGSI_TEXTURE_2D_ARRAY:
4479 case TGSI_TEXTURE_SHADOW2D_ARRAY:
4480 num_src_deriv_channels = 2;
4481 num_dst_deriv_channels = 2;
4482 num_deriv_channels = 2;
4483 break;
4484 case TGSI_TEXTURE_CUBE:
4485 case TGSI_TEXTURE_SHADOWCUBE:
4486 case TGSI_TEXTURE_CUBE_ARRAY:
4487 case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
4488 /* Cube derivatives will be converted to 2D. */
4489 num_src_deriv_channels = 3;
4490 num_dst_deriv_channels = 3;
4491 num_deriv_channels = 2;
4492 break;
4493 case TGSI_TEXTURE_1D:
4494 case TGSI_TEXTURE_SHADOW1D:
4495 case TGSI_TEXTURE_1D_ARRAY:
4496 case TGSI_TEXTURE_SHADOW1D_ARRAY:
4497 num_src_deriv_channels = 1;
4498
4499 /* 1D textures are allocated and used as 2D on GFX9. */
4500 if (ctx->screen->b.chip_class >= GFX9) {
4501 num_dst_deriv_channels = 2;
4502 num_deriv_channels = 2;
4503 } else {
4504 num_dst_deriv_channels = 1;
4505 num_deriv_channels = 1;
4506 }
4507 break;
4508 default:
4509 unreachable("invalid target");
4510 }
4511
4512 for (param = 0; param < 2; param++) {
4513 for (chan = 0; chan < num_src_deriv_channels; chan++)
4514 derivs[param * num_dst_deriv_channels + chan] =
4515 lp_build_emit_fetch(bld_base, inst, param+1, chan);
4516
4517 /* Fill in the rest with zeros. */
4518 for (chan = num_src_deriv_channels;
4519 chan < num_dst_deriv_channels; chan++)
4520 derivs[param * num_dst_deriv_channels + chan] =
4521 bld_base->base.zero;
4522 }
4523 }
4524
4525 if (target == TGSI_TEXTURE_CUBE ||
4526 target == TGSI_TEXTURE_CUBE_ARRAY ||
4527 target == TGSI_TEXTURE_SHADOWCUBE ||
4528 target == TGSI_TEXTURE_SHADOWCUBE_ARRAY)
4529 ac_prepare_cube_coords(&ctx->ac,
4530 opcode == TGSI_OPCODE_TXD,
4531 target == TGSI_TEXTURE_CUBE_ARRAY ||
4532 target == TGSI_TEXTURE_SHADOWCUBE_ARRAY,
4533 coords, derivs);
4534
4535 if (opcode == TGSI_OPCODE_TXD)
4536 for (int i = 0; i < num_deriv_channels * 2; i++)
4537 address[count++] = derivs[i];
4538
4539 /* Pack texture coordinates */
4540 address[count++] = coords[0];
4541 if (num_coords > 1)
4542 address[count++] = coords[1];
4543 if (num_coords > 2)
4544 address[count++] = coords[2];
4545
4546 /* 1D textures are allocated and used as 2D on GFX9. */
4547 if (ctx->screen->b.chip_class >= GFX9) {
4548 LLVMValueRef filler;
4549
4550 /* Use 0.5, so that we don't sample the border color. */
4551 if (opcode == TGSI_OPCODE_TXF)
4552 filler = bld_base->uint_bld.zero;
4553 else
4554 filler = LLVMConstReal(ctx->f32, 0.5);
4555
4556 if (target == TGSI_TEXTURE_1D ||
4557 target == TGSI_TEXTURE_SHADOW1D) {
4558 address[count++] = filler;
4559 } else if (target == TGSI_TEXTURE_1D_ARRAY ||
4560 target == TGSI_TEXTURE_SHADOW1D_ARRAY) {
4561 address[count] = address[count - 1];
4562 address[count - 1] = filler;
4563 count++;
4564 }
4565 }
4566
4567 /* Pack LOD or sample index */
4568 if (opcode == TGSI_OPCODE_TXL || opcode == TGSI_OPCODE_TXF)
4569 address[count++] = coords[3];
4570 else if (opcode == TGSI_OPCODE_TXL2)
4571 address[count++] = lp_build_emit_fetch(bld_base, inst, 1, TGSI_CHAN_X);
4572
4573 if (count > 16) {
4574 assert(!"Cannot handle more than 16 texture address parameters");
4575 count = 16;
4576 }
4577
4578 for (chan = 0; chan < count; chan++ ) {
4579 address[chan] = LLVMBuildBitCast(gallivm->builder,
4580 address[chan], ctx->i32, "");
4581 }
4582
4583 /* Adjust the sample index according to FMASK.
4584 *
4585 * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
4586 * which is the identity mapping. Each nibble says which physical sample
4587 * should be fetched to get that sample.
4588 *
4589 * For example, 0x11111100 means there are only 2 samples stored and
4590 * the second sample covers 3/4 of the pixel. When reading samples 0
4591 * and 1, return physical sample 0 (determined by the first two 0s
4592 * in FMASK), otherwise return physical sample 1.
4593 *
4594 * The sample index should be adjusted as follows:
4595 * sample_index = (fmask >> (sample_index * 4)) & 0xF;
4596 */
4597 if (target == TGSI_TEXTURE_2D_MSAA ||
4598 target == TGSI_TEXTURE_2D_ARRAY_MSAA) {
4599 struct lp_build_context *uint_bld = &bld_base->uint_bld;
4600 struct lp_build_emit_data txf_emit_data = *emit_data;
4601 LLVMValueRef txf_address[4];
4602 /* We only need .xy for non-arrays, and .xyz for arrays. */
4603 unsigned txf_count = target == TGSI_TEXTURE_2D_MSAA ? 2 : 3;
4604 struct tgsi_full_instruction inst = {};
4605
4606 memcpy(txf_address, address, sizeof(txf_address));
4607
4608 /* Read FMASK using TXF_LZ. */
4609 inst.Instruction.Opcode = TGSI_OPCODE_TXF_LZ;
4610 inst.Texture.Texture = target;
4611 txf_emit_data.inst = &inst;
4612 txf_emit_data.chan = 0;
4613 set_tex_fetch_args(ctx, &txf_emit_data,
4614 target, fmask_ptr, NULL,
4615 txf_address, txf_count, 0xf);
4616 build_tex_intrinsic(&tex_action, bld_base, &txf_emit_data);
4617
4618 /* Initialize some constants. */
4619 LLVMValueRef four = LLVMConstInt(ctx->i32, 4, 0);
4620 LLVMValueRef F = LLVMConstInt(ctx->i32, 0xF, 0);
4621
4622 /* Apply the formula. */
4623 LLVMValueRef fmask =
4624 LLVMBuildExtractElement(gallivm->builder,
4625 txf_emit_data.output[0],
4626 uint_bld->zero, "");
4627
4628 unsigned sample_chan = txf_count; /* the sample index is last */
4629
4630 LLVMValueRef sample_index4 =
4631 LLVMBuildMul(gallivm->builder, address[sample_chan], four, "");
4632
4633 LLVMValueRef shifted_fmask =
4634 LLVMBuildLShr(gallivm->builder, fmask, sample_index4, "");
4635
4636 LLVMValueRef final_sample =
4637 LLVMBuildAnd(gallivm->builder, shifted_fmask, F, "");
4638
4639 /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
4640 * resource descriptor is 0 (invalid),
4641 */
4642 LLVMValueRef fmask_desc =
4643 LLVMBuildBitCast(gallivm->builder, fmask_ptr,
4644 ctx->v8i32, "");
4645
4646 LLVMValueRef fmask_word1 =
4647 LLVMBuildExtractElement(gallivm->builder, fmask_desc,
4648 uint_bld->one, "");
4649
4650 LLVMValueRef word1_is_nonzero =
4651 LLVMBuildICmp(gallivm->builder, LLVMIntNE,
4652 fmask_word1, uint_bld->zero, "");
4653
4654 /* Replace the MSAA sample index. */
4655 address[sample_chan] =
4656 LLVMBuildSelect(gallivm->builder, word1_is_nonzero,
4657 final_sample, address[sample_chan], "");
4658 }
4659
4660 if (opcode == TGSI_OPCODE_TXF ||
4661 opcode == TGSI_OPCODE_TXF_LZ) {
4662 /* add tex offsets */
4663 if (inst->Texture.NumOffsets) {
4664 struct lp_build_context *uint_bld = &bld_base->uint_bld;
4665 const struct tgsi_texture_offset *off = inst->TexOffsets;
4666
4667 assert(inst->Texture.NumOffsets == 1);
4668
4669 switch (target) {
4670 case TGSI_TEXTURE_3D:
4671 address[2] = lp_build_add(uint_bld, address[2],
4672 ctx->imms[off->Index * TGSI_NUM_CHANNELS + off->SwizzleZ]);
4673 /* fall through */
4674 case TGSI_TEXTURE_2D:
4675 case TGSI_TEXTURE_SHADOW2D:
4676 case TGSI_TEXTURE_RECT:
4677 case TGSI_TEXTURE_SHADOWRECT:
4678 case TGSI_TEXTURE_2D_ARRAY:
4679 case TGSI_TEXTURE_SHADOW2D_ARRAY:
4680 address[1] =
4681 lp_build_add(uint_bld, address[1],
4682 ctx->imms[off->Index * TGSI_NUM_CHANNELS + off->SwizzleY]);
4683 /* fall through */
4684 case TGSI_TEXTURE_1D:
4685 case TGSI_TEXTURE_SHADOW1D:
4686 case TGSI_TEXTURE_1D_ARRAY:
4687 case TGSI_TEXTURE_SHADOW1D_ARRAY:
4688 address[0] =
4689 lp_build_add(uint_bld, address[0],
4690 ctx->imms[off->Index * TGSI_NUM_CHANNELS + off->SwizzleX]);
4691 break;
4692 /* texture offsets do not apply to other texture targets */
4693 }
4694 }
4695 }
4696
4697 if (opcode == TGSI_OPCODE_TG4) {
4698 unsigned gather_comp = 0;
4699
4700 /* DMASK was repurposed for GATHER4. 4 components are always
4701 * returned and DMASK works like a swizzle - it selects
4702 * the component to fetch. The only valid DMASK values are
4703 * 1=red, 2=green, 4=blue, 8=alpha. (e.g. 1 returns
4704 * (red,red,red,red) etc.) The ISA document doesn't mention
4705 * this.
4706 */
4707
4708 /* Get the component index from src1.x for Gather4. */
4709 if (!tgsi_is_shadow_target(target)) {
4710 LLVMValueRef comp_imm;
4711 struct tgsi_src_register src1 = inst->Src[1].Register;
4712
4713 assert(src1.File == TGSI_FILE_IMMEDIATE);
4714
4715 comp_imm = ctx->imms[src1.Index * TGSI_NUM_CHANNELS + src1.SwizzleX];
4716 gather_comp = LLVMConstIntGetZExtValue(comp_imm);
4717 gather_comp = CLAMP(gather_comp, 0, 3);
4718 }
4719
4720 dmask = 1 << gather_comp;
4721 }
4722
4723 set_tex_fetch_args(ctx, emit_data, target, res_ptr,
4724 samp_ptr, address, count, dmask);
4725 }
4726
4727 /* Gather4 should follow the same rules as bilinear filtering, but the hardware
4728 * incorrectly forces nearest filtering if the texture format is integer.
4729 * The only effect it has on Gather4, which always returns 4 texels for
4730 * bilinear filtering, is that the final coordinates are off by 0.5 of
4731 * the texel size.
4732 *
4733 * The workaround is to subtract 0.5 from the unnormalized coordinates,
4734 * or (0.5 / size) from the normalized coordinates.
4735 */
4736 static void si_lower_gather4_integer(struct si_shader_context *ctx,
4737 struct ac_image_args *args,
4738 unsigned target)
4739 {
4740 LLVMBuilderRef builder = ctx->gallivm.builder;
4741 LLVMValueRef coord = args->addr;
4742 LLVMValueRef half_texel[2];
4743 /* Texture coordinates start after:
4744 * {offset, bias, z-compare, derivatives}
4745 * Only the offset and z-compare can occur here.
4746 */
4747 unsigned coord_vgpr_index = (int)args->offset + (int)args->compare;
4748 int c;
4749
4750 if (target == TGSI_TEXTURE_RECT ||
4751 target == TGSI_TEXTURE_SHADOWRECT) {
4752 half_texel[0] = half_texel[1] = LLVMConstReal(ctx->f32, -0.5);
4753 } else {
4754 struct tgsi_full_instruction txq_inst = {};
4755 struct lp_build_emit_data txq_emit_data = {};
4756
4757 /* Query the texture size. */
4758 txq_inst.Texture.Texture = target;
4759 txq_emit_data.inst = &txq_inst;
4760 txq_emit_data.dst_type = ctx->v4i32;
4761 set_tex_fetch_args(ctx, &txq_emit_data, target,
4762 args->resource, NULL,
4763 &ctx->bld_base.uint_bld.zero,
4764 1, 0xf);
4765 txq_emit(NULL, &ctx->bld_base, &txq_emit_data);
4766
4767 /* Compute -0.5 / size. */
4768 for (c = 0; c < 2; c++) {
4769 half_texel[c] =
4770 LLVMBuildExtractElement(builder, txq_emit_data.output[0],
4771 LLVMConstInt(ctx->i32, c, 0), "");
4772 half_texel[c] = LLVMBuildUIToFP(builder, half_texel[c], ctx->f32, "");
4773 half_texel[c] =
4774 lp_build_emit_llvm_unary(&ctx->bld_base,
4775 TGSI_OPCODE_RCP, half_texel[c]);
4776 half_texel[c] = LLVMBuildFMul(builder, half_texel[c],
4777 LLVMConstReal(ctx->f32, -0.5), "");
4778 }
4779 }
4780
4781 for (c = 0; c < 2; c++) {
4782 LLVMValueRef tmp;
4783 LLVMValueRef index = LLVMConstInt(ctx->i32, coord_vgpr_index + c, 0);
4784
4785 tmp = LLVMBuildExtractElement(builder, coord, index, "");
4786 tmp = LLVMBuildBitCast(builder, tmp, ctx->f32, "");
4787 tmp = LLVMBuildFAdd(builder, tmp, half_texel[c], "");
4788 tmp = LLVMBuildBitCast(builder, tmp, ctx->i32, "");
4789 coord = LLVMBuildInsertElement(builder, coord, tmp, index, "");
4790 }
4791
4792 args->addr = coord;
4793 }
4794
4795 static void build_tex_intrinsic(const struct lp_build_tgsi_action *action,
4796 struct lp_build_tgsi_context *bld_base,
4797 struct lp_build_emit_data *emit_data)
4798 {
4799 struct si_shader_context *ctx = si_shader_context(bld_base);
4800 const struct tgsi_full_instruction *inst = emit_data->inst;
4801 struct ac_image_args args;
4802 unsigned opcode = inst->Instruction.Opcode;
4803 unsigned target = inst->Texture.Texture;
4804
4805 if (target == TGSI_TEXTURE_BUFFER) {
4806 emit_data->output[emit_data->chan] =
4807 ac_build_buffer_load_format(&ctx->ac,
4808 emit_data->args[0],
4809 emit_data->args[2],
4810 emit_data->args[1],
4811 true);
4812 return;
4813 }
4814
4815 memcpy(&args, emit_data->args, sizeof(args)); /* ugly */
4816
4817 args.opcode = ac_image_sample;
4818 args.compare = tgsi_is_shadow_target(target);
4819 args.offset = inst->Texture.NumOffsets > 0;
4820
4821 switch (opcode) {
4822 case TGSI_OPCODE_TXF:
4823 case TGSI_OPCODE_TXF_LZ:
4824 args.opcode = opcode == TGSI_OPCODE_TXF_LZ ||
4825 target == TGSI_TEXTURE_2D_MSAA ||
4826 target == TGSI_TEXTURE_2D_ARRAY_MSAA ?
4827 ac_image_load : ac_image_load_mip;
4828 args.compare = false;
4829 args.offset = false;
4830 break;
4831 case TGSI_OPCODE_LODQ:
4832 args.opcode = ac_image_get_lod;
4833 args.compare = false;
4834 args.offset = false;
4835 break;
4836 case TGSI_OPCODE_TEX:
4837 case TGSI_OPCODE_TEX2:
4838 case TGSI_OPCODE_TXP:
4839 if (ctx->type != PIPE_SHADER_FRAGMENT)
4840 args.level_zero = true;
4841 break;
4842 case TGSI_OPCODE_TEX_LZ:
4843 args.level_zero = true;
4844 break;
4845 case TGSI_OPCODE_TXB:
4846 case TGSI_OPCODE_TXB2:
4847 assert(ctx->type == PIPE_SHADER_FRAGMENT);
4848 args.bias = true;
4849 break;
4850 case TGSI_OPCODE_TXL:
4851 case TGSI_OPCODE_TXL2:
4852 args.lod = true;
4853 break;
4854 case TGSI_OPCODE_TXD:
4855 args.deriv = true;
4856 break;
4857 case TGSI_OPCODE_TG4:
4858 args.opcode = ac_image_gather4;
4859 args.level_zero = true;
4860 break;
4861 default:
4862 assert(0);
4863 return;
4864 }
4865
4866 /* The hardware needs special lowering for Gather4 with integer formats. */
4867 if (ctx->screen->b.chip_class <= VI &&
4868 opcode == TGSI_OPCODE_TG4) {
4869 struct tgsi_shader_info *info = &ctx->shader->selector->info;
4870 /* This will also work with non-constant indexing because of how
4871 * glsl_to_tgsi works and we intent to preserve that behavior.
4872 */
4873 const unsigned src_idx = 2;
4874 unsigned sampler = inst->Src[src_idx].Register.Index;
4875
4876 assert(inst->Src[src_idx].Register.File == TGSI_FILE_SAMPLER);
4877
4878 if (info->sampler_type[sampler] == TGSI_RETURN_TYPE_SINT ||
4879 info->sampler_type[sampler] == TGSI_RETURN_TYPE_UINT)
4880 si_lower_gather4_integer(ctx, &args, target);
4881 }
4882
4883 emit_data->output[emit_data->chan] =
4884 ac_build_image_opcode(&ctx->ac, &args);
4885 }
4886
4887 static void si_llvm_emit_txqs(
4888 const struct lp_build_tgsi_action *action,
4889 struct lp_build_tgsi_context *bld_base,
4890 struct lp_build_emit_data *emit_data)
4891 {
4892 struct si_shader_context *ctx = si_shader_context(bld_base);
4893 struct gallivm_state *gallivm = bld_base->base.gallivm;
4894 LLVMBuilderRef builder = gallivm->builder;
4895 LLVMValueRef res, samples;
4896 LLVMValueRef res_ptr, samp_ptr, fmask_ptr = NULL;
4897
4898 tex_fetch_ptrs(bld_base, emit_data, &res_ptr, &samp_ptr, &fmask_ptr);
4899
4900
4901 /* Read the samples from the descriptor directly. */
4902 res = LLVMBuildBitCast(builder, res_ptr, ctx->v8i32, "");
4903 samples = LLVMBuildExtractElement(
4904 builder, res,
4905 lp_build_const_int32(gallivm, 3), "");
4906 samples = LLVMBuildLShr(builder, samples,
4907 lp_build_const_int32(gallivm, 16), "");
4908 samples = LLVMBuildAnd(builder, samples,
4909 lp_build_const_int32(gallivm, 0xf), "");
4910 samples = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1),
4911 samples, "");
4912
4913 emit_data->output[emit_data->chan] = samples;
4914 }
4915
4916 static void si_llvm_emit_ddxy(
4917 const struct lp_build_tgsi_action *action,
4918 struct lp_build_tgsi_context *bld_base,
4919 struct lp_build_emit_data *emit_data)
4920 {
4921 struct si_shader_context *ctx = si_shader_context(bld_base);
4922 struct gallivm_state *gallivm = bld_base->base.gallivm;
4923 unsigned opcode = emit_data->info->opcode;
4924 LLVMValueRef val;
4925 int idx;
4926 unsigned mask;
4927
4928 if (opcode == TGSI_OPCODE_DDX_FINE)
4929 mask = AC_TID_MASK_LEFT;
4930 else if (opcode == TGSI_OPCODE_DDY_FINE)
4931 mask = AC_TID_MASK_TOP;
4932 else
4933 mask = AC_TID_MASK_TOP_LEFT;
4934
4935 /* for DDX we want to next X pixel, DDY next Y pixel. */
4936 idx = (opcode == TGSI_OPCODE_DDX || opcode == TGSI_OPCODE_DDX_FINE) ? 1 : 2;
4937
4938 val = LLVMBuildBitCast(gallivm->builder, emit_data->args[0], ctx->i32, "");
4939 val = ac_build_ddxy(&ctx->ac, ctx->screen->has_ds_bpermute,
4940 mask, idx, ctx->lds, val);
4941 emit_data->output[emit_data->chan] = val;
4942 }
4943
4944 /*
4945 * this takes an I,J coordinate pair,
4946 * and works out the X and Y derivatives.
4947 * it returns DDX(I), DDX(J), DDY(I), DDY(J).
4948 */
4949 static LLVMValueRef si_llvm_emit_ddxy_interp(
4950 struct lp_build_tgsi_context *bld_base,
4951 LLVMValueRef interp_ij)
4952 {
4953 struct si_shader_context *ctx = si_shader_context(bld_base);
4954 struct gallivm_state *gallivm = bld_base->base.gallivm;
4955 LLVMValueRef result[4], a;
4956 unsigned i;
4957
4958 for (i = 0; i < 2; i++) {
4959 a = LLVMBuildExtractElement(gallivm->builder, interp_ij,
4960 LLVMConstInt(ctx->i32, i, 0), "");
4961 result[i] = lp_build_emit_llvm_unary(bld_base, TGSI_OPCODE_DDX, a);
4962 result[2+i] = lp_build_emit_llvm_unary(bld_base, TGSI_OPCODE_DDY, a);
4963 }
4964
4965 return lp_build_gather_values(gallivm, result, 4);
4966 }
4967
4968 static void interp_fetch_args(
4969 struct lp_build_tgsi_context *bld_base,
4970 struct lp_build_emit_data *emit_data)
4971 {
4972 struct si_shader_context *ctx = si_shader_context(bld_base);
4973 struct gallivm_state *gallivm = bld_base->base.gallivm;
4974 const struct tgsi_full_instruction *inst = emit_data->inst;
4975
4976 if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET) {
4977 /* offset is in second src, first two channels */
4978 emit_data->args[0] = lp_build_emit_fetch(bld_base,
4979 emit_data->inst, 1,
4980 TGSI_CHAN_X);
4981 emit_data->args[1] = lp_build_emit_fetch(bld_base,
4982 emit_data->inst, 1,
4983 TGSI_CHAN_Y);
4984 emit_data->arg_count = 2;
4985 } else if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE) {
4986 LLVMValueRef sample_position;
4987 LLVMValueRef sample_id;
4988 LLVMValueRef halfval = lp_build_const_float(gallivm, 0.5f);
4989
4990 /* fetch sample ID, then fetch its sample position,
4991 * and place into first two channels.
4992 */
4993 sample_id = lp_build_emit_fetch(bld_base,
4994 emit_data->inst, 1, TGSI_CHAN_X);
4995 sample_id = LLVMBuildBitCast(gallivm->builder, sample_id,
4996 ctx->i32, "");
4997 sample_position = load_sample_position(ctx, sample_id);
4998
4999 emit_data->args[0] = LLVMBuildExtractElement(gallivm->builder,
5000 sample_position,
5001 lp_build_const_int32(gallivm, 0), "");
5002
5003 emit_data->args[0] = LLVMBuildFSub(gallivm->builder, emit_data->args[0], halfval, "");
5004 emit_data->args[1] = LLVMBuildExtractElement(gallivm->builder,
5005 sample_position,
5006 lp_build_const_int32(gallivm, 1), "");
5007 emit_data->args[1] = LLVMBuildFSub(gallivm->builder, emit_data->args[1], halfval, "");
5008 emit_data->arg_count = 2;
5009 }
5010 }
5011
5012 static void build_interp_intrinsic(const struct lp_build_tgsi_action *action,
5013 struct lp_build_tgsi_context *bld_base,
5014 struct lp_build_emit_data *emit_data)
5015 {
5016 struct si_shader_context *ctx = si_shader_context(bld_base);
5017 struct si_shader *shader = ctx->shader;
5018 struct gallivm_state *gallivm = bld_base->base.gallivm;
5019 struct lp_build_context *uint = &bld_base->uint_bld;
5020 LLVMValueRef interp_param;
5021 const struct tgsi_full_instruction *inst = emit_data->inst;
5022 int input_index = inst->Src[0].Register.Index;
5023 int chan;
5024 int i;
5025 LLVMValueRef attr_number;
5026 LLVMValueRef params = LLVMGetParam(ctx->main_fn, SI_PARAM_PRIM_MASK);
5027 int interp_param_idx;
5028 unsigned interp = shader->selector->info.input_interpolate[input_index];
5029 unsigned location;
5030
5031 assert(inst->Src[0].Register.File == TGSI_FILE_INPUT);
5032
5033 if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET ||
5034 inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE)
5035 location = TGSI_INTERPOLATE_LOC_CENTER;
5036 else
5037 location = TGSI_INTERPOLATE_LOC_CENTROID;
5038
5039 interp_param_idx = lookup_interp_param_index(interp, location);
5040 if (interp_param_idx == -1)
5041 return;
5042 else if (interp_param_idx)
5043 interp_param = LLVMGetParam(ctx->main_fn, interp_param_idx);
5044 else
5045 interp_param = NULL;
5046
5047 attr_number = lp_build_const_int32(gallivm, input_index);
5048
5049 if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET ||
5050 inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE) {
5051 LLVMValueRef ij_out[2];
5052 LLVMValueRef ddxy_out = si_llvm_emit_ddxy_interp(bld_base, interp_param);
5053
5054 /*
5055 * take the I then J parameters, and the DDX/Y for it, and
5056 * calculate the IJ inputs for the interpolator.
5057 * temp1 = ddx * offset/sample.x + I;
5058 * interp_param.I = ddy * offset/sample.y + temp1;
5059 * temp1 = ddx * offset/sample.x + J;
5060 * interp_param.J = ddy * offset/sample.y + temp1;
5061 */
5062 for (i = 0; i < 2; i++) {
5063 LLVMValueRef ix_ll = lp_build_const_int32(gallivm, i);
5064 LLVMValueRef iy_ll = lp_build_const_int32(gallivm, i + 2);
5065 LLVMValueRef ddx_el = LLVMBuildExtractElement(gallivm->builder,
5066 ddxy_out, ix_ll, "");
5067 LLVMValueRef ddy_el = LLVMBuildExtractElement(gallivm->builder,
5068 ddxy_out, iy_ll, "");
5069 LLVMValueRef interp_el = LLVMBuildExtractElement(gallivm->builder,
5070 interp_param, ix_ll, "");
5071 LLVMValueRef temp1, temp2;
5072
5073 interp_el = LLVMBuildBitCast(gallivm->builder, interp_el,
5074 ctx->f32, "");
5075
5076 temp1 = LLVMBuildFMul(gallivm->builder, ddx_el, emit_data->args[0], "");
5077
5078 temp1 = LLVMBuildFAdd(gallivm->builder, temp1, interp_el, "");
5079
5080 temp2 = LLVMBuildFMul(gallivm->builder, ddy_el, emit_data->args[1], "");
5081
5082 ij_out[i] = LLVMBuildFAdd(gallivm->builder, temp2, temp1, "");
5083 }
5084 interp_param = lp_build_gather_values(bld_base->base.gallivm, ij_out, 2);
5085 }
5086
5087 for (chan = 0; chan < 4; chan++) {
5088 LLVMValueRef llvm_chan;
5089 unsigned schan;
5090
5091 schan = tgsi_util_get_full_src_register_swizzle(&inst->Src[0], chan);
5092 llvm_chan = lp_build_const_int32(gallivm, schan);
5093
5094 if (interp_param) {
5095 interp_param = LLVMBuildBitCast(gallivm->builder,
5096 interp_param, LLVMVectorType(ctx->f32, 2), "");
5097 LLVMValueRef i = LLVMBuildExtractElement(
5098 gallivm->builder, interp_param, uint->zero, "");
5099 LLVMValueRef j = LLVMBuildExtractElement(
5100 gallivm->builder, interp_param, uint->one, "");
5101 emit_data->output[chan] = ac_build_fs_interp(&ctx->ac,
5102 llvm_chan, attr_number, params,
5103 i, j);
5104 } else {
5105 emit_data->output[chan] = ac_build_fs_interp_mov(&ctx->ac,
5106 lp_build_const_int32(gallivm, 2), /* P0 */
5107 llvm_chan, attr_number, params);
5108 }
5109 }
5110 }
5111
5112 static LLVMValueRef si_emit_ballot(struct si_shader_context *ctx,
5113 LLVMValueRef value)
5114 {
5115 struct gallivm_state *gallivm = &ctx->gallivm;
5116 LLVMValueRef args[3] = {
5117 value,
5118 ctx->i32_0,
5119 LLVMConstInt(ctx->i32, LLVMIntNE, 0)
5120 };
5121
5122 if (LLVMTypeOf(value) != ctx->i32)
5123 args[0] = LLVMBuildBitCast(gallivm->builder, value, ctx->i32, "");
5124
5125 return lp_build_intrinsic(gallivm->builder,
5126 "llvm.amdgcn.icmp.i32",
5127 ctx->i64, args, 3,
5128 LP_FUNC_ATTR_NOUNWIND |
5129 LP_FUNC_ATTR_READNONE |
5130 LP_FUNC_ATTR_CONVERGENT);
5131 }
5132
5133 static void vote_all_emit(
5134 const struct lp_build_tgsi_action *action,
5135 struct lp_build_tgsi_context *bld_base,
5136 struct lp_build_emit_data *emit_data)
5137 {
5138 struct si_shader_context *ctx = si_shader_context(bld_base);
5139 struct gallivm_state *gallivm = &ctx->gallivm;
5140 LLVMValueRef active_set, vote_set;
5141 LLVMValueRef tmp;
5142
5143 active_set = si_emit_ballot(ctx, ctx->i32_1);
5144 vote_set = si_emit_ballot(ctx, emit_data->args[0]);
5145
5146 tmp = LLVMBuildICmp(gallivm->builder, LLVMIntEQ, vote_set, active_set, "");
5147 emit_data->output[emit_data->chan] =
5148 LLVMBuildSExt(gallivm->builder, tmp, ctx->i32, "");
5149 }
5150
5151 static void vote_any_emit(
5152 const struct lp_build_tgsi_action *action,
5153 struct lp_build_tgsi_context *bld_base,
5154 struct lp_build_emit_data *emit_data)
5155 {
5156 struct si_shader_context *ctx = si_shader_context(bld_base);
5157 struct gallivm_state *gallivm = &ctx->gallivm;
5158 LLVMValueRef vote_set;
5159 LLVMValueRef tmp;
5160
5161 vote_set = si_emit_ballot(ctx, emit_data->args[0]);
5162
5163 tmp = LLVMBuildICmp(gallivm->builder, LLVMIntNE,
5164 vote_set, LLVMConstInt(ctx->i64, 0, 0), "");
5165 emit_data->output[emit_data->chan] =
5166 LLVMBuildSExt(gallivm->builder, tmp, ctx->i32, "");
5167 }
5168
5169 static void vote_eq_emit(
5170 const struct lp_build_tgsi_action *action,
5171 struct lp_build_tgsi_context *bld_base,
5172 struct lp_build_emit_data *emit_data)
5173 {
5174 struct si_shader_context *ctx = si_shader_context(bld_base);
5175 struct gallivm_state *gallivm = &ctx->gallivm;
5176 LLVMValueRef active_set, vote_set;
5177 LLVMValueRef all, none, tmp;
5178
5179 active_set = si_emit_ballot(ctx, ctx->i32_1);
5180 vote_set = si_emit_ballot(ctx, emit_data->args[0]);
5181
5182 all = LLVMBuildICmp(gallivm->builder, LLVMIntEQ, vote_set, active_set, "");
5183 none = LLVMBuildICmp(gallivm->builder, LLVMIntEQ,
5184 vote_set, LLVMConstInt(ctx->i64, 0, 0), "");
5185 tmp = LLVMBuildOr(gallivm->builder, all, none, "");
5186 emit_data->output[emit_data->chan] =
5187 LLVMBuildSExt(gallivm->builder, tmp, ctx->i32, "");
5188 }
5189
5190 static unsigned si_llvm_get_stream(struct lp_build_tgsi_context *bld_base,
5191 struct lp_build_emit_data *emit_data)
5192 {
5193 struct si_shader_context *ctx = si_shader_context(bld_base);
5194 struct tgsi_src_register src0 = emit_data->inst->Src[0].Register;
5195 LLVMValueRef imm;
5196 unsigned stream;
5197
5198 assert(src0.File == TGSI_FILE_IMMEDIATE);
5199
5200 imm = ctx->imms[src0.Index * TGSI_NUM_CHANNELS + src0.SwizzleX];
5201 stream = LLVMConstIntGetZExtValue(imm) & 0x3;
5202 return stream;
5203 }
5204
5205 /* Emit one vertex from the geometry shader */
5206 static void si_llvm_emit_vertex(
5207 const struct lp_build_tgsi_action *action,
5208 struct lp_build_tgsi_context *bld_base,
5209 struct lp_build_emit_data *emit_data)
5210 {
5211 struct si_shader_context *ctx = si_shader_context(bld_base);
5212 struct lp_build_context *uint = &bld_base->uint_bld;
5213 struct si_shader *shader = ctx->shader;
5214 struct tgsi_shader_info *info = &shader->selector->info;
5215 struct gallivm_state *gallivm = bld_base->base.gallivm;
5216 struct lp_build_if_state if_state;
5217 LLVMValueRef soffset = LLVMGetParam(ctx->main_fn,
5218 SI_PARAM_GS2VS_OFFSET);
5219 LLVMValueRef gs_next_vertex;
5220 LLVMValueRef can_emit, kill;
5221 unsigned chan, offset;
5222 int i;
5223 unsigned stream;
5224
5225 stream = si_llvm_get_stream(bld_base, emit_data);
5226
5227 /* Write vertex attribute values to GSVS ring */
5228 gs_next_vertex = LLVMBuildLoad(gallivm->builder,
5229 ctx->gs_next_vertex[stream],
5230 "");
5231
5232 /* If this thread has already emitted the declared maximum number of
5233 * vertices, skip the write: excessive vertex emissions are not
5234 * supposed to have any effect.
5235 *
5236 * If the shader has no writes to memory, kill it instead. This skips
5237 * further memory loads and may allow LLVM to skip to the end
5238 * altogether.
5239 */
5240 can_emit = LLVMBuildICmp(gallivm->builder, LLVMIntULT, gs_next_vertex,
5241 lp_build_const_int32(gallivm,
5242 shader->selector->gs_max_out_vertices), "");
5243
5244 bool use_kill = !info->writes_memory;
5245 if (use_kill) {
5246 kill = lp_build_select(&bld_base->base, can_emit,
5247 lp_build_const_float(gallivm, 1.0f),
5248 lp_build_const_float(gallivm, -1.0f));
5249
5250 ac_build_kill(&ctx->ac, kill);
5251 } else {
5252 lp_build_if(&if_state, gallivm, can_emit);
5253 }
5254
5255 offset = 0;
5256 for (i = 0; i < info->num_outputs; i++) {
5257 LLVMValueRef *out_ptr = ctx->outputs[i];
5258
5259 for (chan = 0; chan < 4; chan++) {
5260 if (!(info->output_usagemask[i] & (1 << chan)) ||
5261 ((info->output_streams[i] >> (2 * chan)) & 3) != stream)
5262 continue;
5263
5264 LLVMValueRef out_val = LLVMBuildLoad(gallivm->builder, out_ptr[chan], "");
5265 LLVMValueRef voffset =
5266 lp_build_const_int32(gallivm, offset *
5267 shader->selector->gs_max_out_vertices);
5268 offset++;
5269
5270 voffset = lp_build_add(uint, voffset, gs_next_vertex);
5271 voffset = lp_build_mul_imm(uint, voffset, 4);
5272
5273 out_val = LLVMBuildBitCast(gallivm->builder, out_val, ctx->i32, "");
5274
5275 ac_build_buffer_store_dword(&ctx->ac,
5276 ctx->gsvs_ring[stream],
5277 out_val, 1,
5278 voffset, soffset, 0,
5279 1, 1, true, true);
5280 }
5281 }
5282
5283 gs_next_vertex = lp_build_add(uint, gs_next_vertex,
5284 lp_build_const_int32(gallivm, 1));
5285
5286 LLVMBuildStore(gallivm->builder, gs_next_vertex, ctx->gs_next_vertex[stream]);
5287
5288 /* Signal vertex emission */
5289 ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_EMIT | AC_SENDMSG_GS | (stream << 8),
5290 LLVMGetParam(ctx->main_fn, SI_PARAM_GS_WAVE_ID));
5291 if (!use_kill)
5292 lp_build_endif(&if_state);
5293 }
5294
5295 /* Cut one primitive from the geometry shader */
5296 static void si_llvm_emit_primitive(
5297 const struct lp_build_tgsi_action *action,
5298 struct lp_build_tgsi_context *bld_base,
5299 struct lp_build_emit_data *emit_data)
5300 {
5301 struct si_shader_context *ctx = si_shader_context(bld_base);
5302 unsigned stream;
5303
5304 /* Signal primitive cut */
5305 stream = si_llvm_get_stream(bld_base, emit_data);
5306 ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_CUT | AC_SENDMSG_GS | (stream << 8),
5307 LLVMGetParam(ctx->main_fn, SI_PARAM_GS_WAVE_ID));
5308 }
5309
5310 static void si_llvm_emit_barrier(const struct lp_build_tgsi_action *action,
5311 struct lp_build_tgsi_context *bld_base,
5312 struct lp_build_emit_data *emit_data)
5313 {
5314 struct si_shader_context *ctx = si_shader_context(bld_base);
5315 struct gallivm_state *gallivm = bld_base->base.gallivm;
5316
5317 /* SI only (thanks to a hw bug workaround):
5318 * The real barrier instruction isn’t needed, because an entire patch
5319 * always fits into a single wave.
5320 */
5321 if (HAVE_LLVM >= 0x0309 &&
5322 ctx->screen->b.chip_class == SI &&
5323 ctx->type == PIPE_SHADER_TESS_CTRL) {
5324 emit_waitcnt(ctx, LGKM_CNT & VM_CNT);
5325 return;
5326 }
5327
5328 lp_build_intrinsic(gallivm->builder,
5329 HAVE_LLVM >= 0x0309 ? "llvm.amdgcn.s.barrier"
5330 : "llvm.AMDGPU.barrier.local",
5331 ctx->voidt, NULL, 0, LP_FUNC_ATTR_CONVERGENT);
5332 }
5333
5334 static const struct lp_build_tgsi_action tex_action = {
5335 .fetch_args = tex_fetch_args,
5336 .emit = build_tex_intrinsic,
5337 };
5338
5339 static const struct lp_build_tgsi_action interp_action = {
5340 .fetch_args = interp_fetch_args,
5341 .emit = build_interp_intrinsic,
5342 };
5343
5344 static void si_create_function(struct si_shader_context *ctx,
5345 const char *name,
5346 LLVMTypeRef *returns, unsigned num_returns,
5347 LLVMTypeRef *params, unsigned num_params,
5348 int last_sgpr)
5349 {
5350 int i;
5351
5352 si_llvm_create_func(ctx, name, returns, num_returns,
5353 params, num_params);
5354 si_llvm_shader_type(ctx->main_fn, ctx->type);
5355 ctx->return_value = LLVMGetUndef(ctx->return_type);
5356
5357 for (i = 0; i <= last_sgpr; ++i) {
5358 LLVMValueRef P = LLVMGetParam(ctx->main_fn, i);
5359
5360 /* The combination of:
5361 * - ByVal
5362 * - dereferenceable
5363 * - invariant.load
5364 * allows the optimization passes to move loads and reduces
5365 * SGPR spilling significantly.
5366 */
5367 if (LLVMGetTypeKind(LLVMTypeOf(P)) == LLVMPointerTypeKind) {
5368 lp_add_function_attr(ctx->main_fn, i + 1, LP_FUNC_ATTR_BYVAL);
5369 lp_add_function_attr(ctx->main_fn, i + 1, LP_FUNC_ATTR_NOALIAS);
5370 ac_add_attr_dereferenceable(P, UINT64_MAX);
5371 } else
5372 lp_add_function_attr(ctx->main_fn, i + 1, LP_FUNC_ATTR_INREG);
5373 }
5374
5375 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
5376 "no-signed-zeros-fp-math",
5377 "true");
5378
5379 if (ctx->screen->b.debug_flags & DBG_UNSAFE_MATH) {
5380 /* These were copied from some LLVM test. */
5381 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
5382 "less-precise-fpmad",
5383 "true");
5384 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
5385 "no-infs-fp-math",
5386 "true");
5387 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
5388 "no-nans-fp-math",
5389 "true");
5390 LLVMAddTargetDependentFunctionAttr(ctx->main_fn,
5391 "unsafe-fp-math",
5392 "true");
5393 }
5394 }
5395
5396 static void declare_streamout_params(struct si_shader_context *ctx,
5397 struct pipe_stream_output_info *so,
5398 LLVMTypeRef *params, LLVMTypeRef i32,
5399 unsigned *num_params)
5400 {
5401 int i;
5402
5403 /* Streamout SGPRs. */
5404 if (so->num_outputs) {
5405 if (ctx->type != PIPE_SHADER_TESS_EVAL)
5406 params[ctx->param_streamout_config = (*num_params)++] = i32;
5407 else
5408 ctx->param_streamout_config = *num_params - 1;
5409
5410 params[ctx->param_streamout_write_index = (*num_params)++] = i32;
5411 }
5412 /* A streamout buffer offset is loaded if the stride is non-zero. */
5413 for (i = 0; i < 4; i++) {
5414 if (!so->stride[i])
5415 continue;
5416
5417 params[ctx->param_streamout_offset[i] = (*num_params)++] = i32;
5418 }
5419 }
5420
5421 static unsigned llvm_get_type_size(LLVMTypeRef type)
5422 {
5423 LLVMTypeKind kind = LLVMGetTypeKind(type);
5424
5425 switch (kind) {
5426 case LLVMIntegerTypeKind:
5427 return LLVMGetIntTypeWidth(type) / 8;
5428 case LLVMFloatTypeKind:
5429 return 4;
5430 case LLVMPointerTypeKind:
5431 return 8;
5432 case LLVMVectorTypeKind:
5433 return LLVMGetVectorSize(type) *
5434 llvm_get_type_size(LLVMGetElementType(type));
5435 case LLVMArrayTypeKind:
5436 return LLVMGetArrayLength(type) *
5437 llvm_get_type_size(LLVMGetElementType(type));
5438 default:
5439 assert(0);
5440 return 0;
5441 }
5442 }
5443
5444 static void declare_tess_lds(struct si_shader_context *ctx)
5445 {
5446 struct gallivm_state *gallivm = &ctx->gallivm;
5447 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
5448 struct lp_build_context *uint = &bld_base->uint_bld;
5449
5450 unsigned lds_size = ctx->screen->b.chip_class >= CIK ? 65536 : 32768;
5451 ctx->lds = LLVMBuildIntToPtr(gallivm->builder, uint->zero,
5452 LLVMPointerType(LLVMArrayType(ctx->i32, lds_size / 4), LOCAL_ADDR_SPACE),
5453 "tess_lds");
5454 }
5455
5456 static unsigned si_get_max_workgroup_size(struct si_shader *shader)
5457 {
5458 const unsigned *properties = shader->selector->info.properties;
5459 unsigned max_work_group_size =
5460 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH] *
5461 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT] *
5462 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH];
5463
5464 if (!max_work_group_size) {
5465 /* This is a variable group size compute shader,
5466 * compile it for the maximum possible group size.
5467 */
5468 max_work_group_size = SI_MAX_VARIABLE_THREADS_PER_BLOCK;
5469 }
5470 return max_work_group_size;
5471 }
5472
5473 static void create_function(struct si_shader_context *ctx)
5474 {
5475 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
5476 struct gallivm_state *gallivm = bld_base->base.gallivm;
5477 struct si_shader *shader = ctx->shader;
5478 LLVMTypeRef params[SI_NUM_PARAMS + SI_MAX_ATTRIBS], v3i32;
5479 LLVMTypeRef returns[16+32*4];
5480 unsigned i, last_sgpr, num_params, num_return_sgprs;
5481 unsigned num_returns = 0;
5482 unsigned num_prolog_vgprs = 0;
5483
5484 v3i32 = LLVMVectorType(ctx->i32, 3);
5485
5486 params[SI_PARAM_RW_BUFFERS] = const_array(ctx->v16i8, SI_NUM_RW_BUFFERS);
5487 params[SI_PARAM_CONST_BUFFERS] = const_array(ctx->v16i8, SI_NUM_CONST_BUFFERS);
5488 params[SI_PARAM_SAMPLERS] = const_array(ctx->v8i32, SI_NUM_SAMPLERS);
5489 params[SI_PARAM_IMAGES] = const_array(ctx->v8i32, SI_NUM_IMAGES);
5490 params[SI_PARAM_SHADER_BUFFERS] = const_array(ctx->v4i32, SI_NUM_SHADER_BUFFERS);
5491
5492 switch (ctx->type) {
5493 case PIPE_SHADER_VERTEX:
5494 params[SI_PARAM_VERTEX_BUFFERS] = const_array(ctx->v16i8, SI_MAX_ATTRIBS);
5495 params[SI_PARAM_BASE_VERTEX] = ctx->i32;
5496 params[SI_PARAM_START_INSTANCE] = ctx->i32;
5497 params[SI_PARAM_DRAWID] = ctx->i32;
5498 num_params = SI_PARAM_DRAWID+1;
5499
5500 if (shader->key.as_es) {
5501 params[ctx->param_es2gs_offset = num_params++] = ctx->i32;
5502 } else if (shader->key.as_ls) {
5503 params[SI_PARAM_LS_OUT_LAYOUT] = ctx->i32;
5504 num_params = SI_PARAM_LS_OUT_LAYOUT+1;
5505 } else {
5506 if (shader->is_gs_copy_shader) {
5507 num_params = SI_PARAM_RW_BUFFERS+1;
5508 } else {
5509 params[SI_PARAM_VS_STATE_BITS] = ctx->i32;
5510 num_params = SI_PARAM_VS_STATE_BITS+1;
5511 }
5512
5513 /* The locations of the other parameters are assigned dynamically. */
5514 declare_streamout_params(ctx, &shader->selector->so,
5515 params, ctx->i32, &num_params);
5516 }
5517
5518 last_sgpr = num_params-1;
5519
5520 /* VGPRs */
5521 params[ctx->param_vertex_id = num_params++] = ctx->i32;
5522 params[ctx->param_rel_auto_id = num_params++] = ctx->i32;
5523 params[ctx->param_vs_prim_id = num_params++] = ctx->i32;
5524 params[ctx->param_instance_id = num_params++] = ctx->i32;
5525
5526 if (!shader->is_gs_copy_shader) {
5527 /* Vertex load indices. */
5528 ctx->param_vertex_index0 = num_params;
5529
5530 for (i = 0; i < shader->selector->info.num_inputs; i++)
5531 params[num_params++] = ctx->i32;
5532
5533 num_prolog_vgprs += shader->selector->info.num_inputs;
5534
5535 /* PrimitiveID output. */
5536 if (!shader->key.as_es && !shader->key.as_ls)
5537 for (i = 0; i <= VS_EPILOG_PRIMID_LOC; i++)
5538 returns[num_returns++] = ctx->f32;
5539 }
5540 break;
5541
5542 case PIPE_SHADER_TESS_CTRL:
5543 params[SI_PARAM_TCS_OFFCHIP_LAYOUT] = ctx->i32;
5544 params[SI_PARAM_TCS_OUT_OFFSETS] = ctx->i32;
5545 params[SI_PARAM_TCS_OUT_LAYOUT] = ctx->i32;
5546 params[SI_PARAM_TCS_IN_LAYOUT] = ctx->i32;
5547 params[ctx->param_oc_lds = SI_PARAM_TCS_OC_LDS] = ctx->i32;
5548 params[SI_PARAM_TESS_FACTOR_OFFSET] = ctx->i32;
5549 last_sgpr = SI_PARAM_TESS_FACTOR_OFFSET;
5550
5551 /* VGPRs */
5552 params[SI_PARAM_PATCH_ID] = ctx->i32;
5553 params[SI_PARAM_REL_IDS] = ctx->i32;
5554 num_params = SI_PARAM_REL_IDS+1;
5555
5556 /* SI_PARAM_TCS_OC_LDS and PARAM_TESS_FACTOR_OFFSET are
5557 * placed after the user SGPRs.
5558 */
5559 for (i = 0; i < SI_TCS_NUM_USER_SGPR + 2; i++)
5560 returns[num_returns++] = ctx->i32; /* SGPRs */
5561
5562 for (i = 0; i < 3; i++)
5563 returns[num_returns++] = ctx->f32; /* VGPRs */
5564 break;
5565
5566 case PIPE_SHADER_TESS_EVAL:
5567 params[SI_PARAM_TCS_OFFCHIP_LAYOUT] = ctx->i32;
5568 num_params = SI_PARAM_TCS_OFFCHIP_LAYOUT+1;
5569
5570 if (shader->key.as_es) {
5571 params[ctx->param_oc_lds = num_params++] = ctx->i32;
5572 params[num_params++] = ctx->i32;
5573 params[ctx->param_es2gs_offset = num_params++] = ctx->i32;
5574 } else {
5575 params[num_params++] = ctx->i32;
5576 declare_streamout_params(ctx, &shader->selector->so,
5577 params, ctx->i32, &num_params);
5578 params[ctx->param_oc_lds = num_params++] = ctx->i32;
5579 }
5580 last_sgpr = num_params - 1;
5581
5582 /* VGPRs */
5583 params[ctx->param_tes_u = num_params++] = ctx->f32;
5584 params[ctx->param_tes_v = num_params++] = ctx->f32;
5585 params[ctx->param_tes_rel_patch_id = num_params++] = ctx->i32;
5586 params[ctx->param_tes_patch_id = num_params++] = ctx->i32;
5587
5588 /* PrimitiveID output. */
5589 if (!shader->key.as_es)
5590 for (i = 0; i <= VS_EPILOG_PRIMID_LOC; i++)
5591 returns[num_returns++] = ctx->f32;
5592 break;
5593
5594 case PIPE_SHADER_GEOMETRY:
5595 params[SI_PARAM_GS2VS_OFFSET] = ctx->i32;
5596 params[SI_PARAM_GS_WAVE_ID] = ctx->i32;
5597 last_sgpr = SI_PARAM_GS_WAVE_ID;
5598
5599 /* VGPRs */
5600 params[SI_PARAM_VTX0_OFFSET] = ctx->i32;
5601 params[SI_PARAM_VTX1_OFFSET] = ctx->i32;
5602 params[SI_PARAM_PRIMITIVE_ID] = ctx->i32;
5603 params[SI_PARAM_VTX2_OFFSET] = ctx->i32;
5604 params[SI_PARAM_VTX3_OFFSET] = ctx->i32;
5605 params[SI_PARAM_VTX4_OFFSET] = ctx->i32;
5606 params[SI_PARAM_VTX5_OFFSET] = ctx->i32;
5607 params[SI_PARAM_GS_INSTANCE_ID] = ctx->i32;
5608 num_params = SI_PARAM_GS_INSTANCE_ID+1;
5609 break;
5610
5611 case PIPE_SHADER_FRAGMENT:
5612 params[SI_PARAM_ALPHA_REF] = ctx->f32;
5613 params[SI_PARAM_PRIM_MASK] = ctx->i32;
5614 last_sgpr = SI_PARAM_PRIM_MASK;
5615 params[SI_PARAM_PERSP_SAMPLE] = ctx->v2i32;
5616 params[SI_PARAM_PERSP_CENTER] = ctx->v2i32;
5617 params[SI_PARAM_PERSP_CENTROID] = ctx->v2i32;
5618 params[SI_PARAM_PERSP_PULL_MODEL] = v3i32;
5619 params[SI_PARAM_LINEAR_SAMPLE] = ctx->v2i32;
5620 params[SI_PARAM_LINEAR_CENTER] = ctx->v2i32;
5621 params[SI_PARAM_LINEAR_CENTROID] = ctx->v2i32;
5622 params[SI_PARAM_LINE_STIPPLE_TEX] = ctx->f32;
5623 params[SI_PARAM_POS_X_FLOAT] = ctx->f32;
5624 params[SI_PARAM_POS_Y_FLOAT] = ctx->f32;
5625 params[SI_PARAM_POS_Z_FLOAT] = ctx->f32;
5626 params[SI_PARAM_POS_W_FLOAT] = ctx->f32;
5627 params[SI_PARAM_FRONT_FACE] = ctx->i32;
5628 shader->info.face_vgpr_index = 20;
5629 params[SI_PARAM_ANCILLARY] = ctx->i32;
5630 params[SI_PARAM_SAMPLE_COVERAGE] = ctx->f32;
5631 params[SI_PARAM_POS_FIXED_PT] = ctx->i32;
5632 num_params = SI_PARAM_POS_FIXED_PT+1;
5633
5634 /* Color inputs from the prolog. */
5635 if (shader->selector->info.colors_read) {
5636 unsigned num_color_elements =
5637 util_bitcount(shader->selector->info.colors_read);
5638
5639 assert(num_params + num_color_elements <= ARRAY_SIZE(params));
5640 for (i = 0; i < num_color_elements; i++)
5641 params[num_params++] = ctx->f32;
5642
5643 num_prolog_vgprs += num_color_elements;
5644 }
5645
5646 /* Outputs for the epilog. */
5647 num_return_sgprs = SI_SGPR_ALPHA_REF + 1;
5648 num_returns =
5649 num_return_sgprs +
5650 util_bitcount(shader->selector->info.colors_written) * 4 +
5651 shader->selector->info.writes_z +
5652 shader->selector->info.writes_stencil +
5653 shader->selector->info.writes_samplemask +
5654 1 /* SampleMaskIn */;
5655
5656 num_returns = MAX2(num_returns,
5657 num_return_sgprs +
5658 PS_EPILOG_SAMPLEMASK_MIN_LOC + 1);
5659
5660 for (i = 0; i < num_return_sgprs; i++)
5661 returns[i] = ctx->i32;
5662 for (; i < num_returns; i++)
5663 returns[i] = ctx->f32;
5664 break;
5665
5666 case PIPE_SHADER_COMPUTE:
5667 params[SI_PARAM_GRID_SIZE] = v3i32;
5668 params[SI_PARAM_BLOCK_SIZE] = v3i32;
5669 params[SI_PARAM_BLOCK_ID] = v3i32;
5670 last_sgpr = SI_PARAM_BLOCK_ID;
5671
5672 params[SI_PARAM_THREAD_ID] = v3i32;
5673 num_params = SI_PARAM_THREAD_ID + 1;
5674 break;
5675 default:
5676 assert(0 && "unimplemented shader");
5677 return;
5678 }
5679
5680 assert(num_params <= ARRAY_SIZE(params));
5681
5682 si_create_function(ctx, "main", returns, num_returns, params,
5683 num_params, last_sgpr);
5684
5685 /* Reserve register locations for VGPR inputs the PS prolog may need. */
5686 if (ctx->type == PIPE_SHADER_FRAGMENT &&
5687 ctx->separate_prolog) {
5688 si_llvm_add_attribute(ctx->main_fn,
5689 "InitialPSInputAddr",
5690 S_0286D0_PERSP_SAMPLE_ENA(1) |
5691 S_0286D0_PERSP_CENTER_ENA(1) |
5692 S_0286D0_PERSP_CENTROID_ENA(1) |
5693 S_0286D0_LINEAR_SAMPLE_ENA(1) |
5694 S_0286D0_LINEAR_CENTER_ENA(1) |
5695 S_0286D0_LINEAR_CENTROID_ENA(1) |
5696 S_0286D0_FRONT_FACE_ENA(1) |
5697 S_0286D0_POS_FIXED_PT_ENA(1));
5698 } else if (ctx->type == PIPE_SHADER_COMPUTE) {
5699 si_llvm_add_attribute(ctx->main_fn,
5700 "amdgpu-max-work-group-size",
5701 si_get_max_workgroup_size(shader));
5702 }
5703
5704 shader->info.num_input_sgprs = 0;
5705 shader->info.num_input_vgprs = 0;
5706
5707 for (i = 0; i <= last_sgpr; ++i)
5708 shader->info.num_input_sgprs += llvm_get_type_size(params[i]) / 4;
5709
5710 for (; i < num_params; ++i)
5711 shader->info.num_input_vgprs += llvm_get_type_size(params[i]) / 4;
5712
5713 assert(shader->info.num_input_vgprs >= num_prolog_vgprs);
5714 shader->info.num_input_vgprs -= num_prolog_vgprs;
5715
5716 if (!ctx->screen->has_ds_bpermute &&
5717 bld_base->info &&
5718 (bld_base->info->opcode_count[TGSI_OPCODE_DDX] > 0 ||
5719 bld_base->info->opcode_count[TGSI_OPCODE_DDY] > 0 ||
5720 bld_base->info->opcode_count[TGSI_OPCODE_DDX_FINE] > 0 ||
5721 bld_base->info->opcode_count[TGSI_OPCODE_DDY_FINE] > 0 ||
5722 bld_base->info->opcode_count[TGSI_OPCODE_INTERP_OFFSET] > 0 ||
5723 bld_base->info->opcode_count[TGSI_OPCODE_INTERP_SAMPLE] > 0))
5724 ctx->lds =
5725 LLVMAddGlobalInAddressSpace(gallivm->module,
5726 LLVMArrayType(ctx->i32, 64),
5727 "ddxy_lds",
5728 LOCAL_ADDR_SPACE);
5729
5730 if ((ctx->type == PIPE_SHADER_VERTEX && shader->key.as_ls) ||
5731 ctx->type == PIPE_SHADER_TESS_CTRL)
5732 declare_tess_lds(ctx);
5733 }
5734
5735 /**
5736 * Load ESGS and GSVS ring buffer resource descriptors and save the variables
5737 * for later use.
5738 */
5739 static void preload_ring_buffers(struct si_shader_context *ctx)
5740 {
5741 struct gallivm_state *gallivm = ctx->bld_base.base.gallivm;
5742 LLVMBuilderRef builder = gallivm->builder;
5743
5744 LLVMValueRef buf_ptr = LLVMGetParam(ctx->main_fn,
5745 SI_PARAM_RW_BUFFERS);
5746
5747 if ((ctx->type == PIPE_SHADER_VERTEX &&
5748 ctx->shader->key.as_es) ||
5749 (ctx->type == PIPE_SHADER_TESS_EVAL &&
5750 ctx->shader->key.as_es) ||
5751 ctx->type == PIPE_SHADER_GEOMETRY) {
5752 unsigned ring =
5753 ctx->type == PIPE_SHADER_GEOMETRY ? SI_GS_RING_ESGS
5754 : SI_ES_RING_ESGS;
5755 LLVMValueRef offset = lp_build_const_int32(gallivm, ring);
5756
5757 ctx->esgs_ring =
5758 ac_build_indexed_load_const(&ctx->ac, buf_ptr, offset);
5759 }
5760
5761 if (ctx->shader->is_gs_copy_shader) {
5762 LLVMValueRef offset = lp_build_const_int32(gallivm, SI_RING_GSVS);
5763
5764 ctx->gsvs_ring[0] =
5765 ac_build_indexed_load_const(&ctx->ac, buf_ptr, offset);
5766 } else if (ctx->type == PIPE_SHADER_GEOMETRY) {
5767 const struct si_shader_selector *sel = ctx->shader->selector;
5768 struct lp_build_context *uint = &ctx->bld_base.uint_bld;
5769 LLVMValueRef offset = lp_build_const_int32(gallivm, SI_RING_GSVS);
5770 LLVMValueRef base_ring;
5771
5772 base_ring = ac_build_indexed_load_const(&ctx->ac, buf_ptr, offset);
5773
5774 /* The conceptual layout of the GSVS ring is
5775 * v0c0 .. vLv0 v0c1 .. vLc1 ..
5776 * but the real memory layout is swizzled across
5777 * threads:
5778 * t0v0c0 .. t15v0c0 t0v1c0 .. t15v1c0 ... t15vLcL
5779 * t16v0c0 ..
5780 * Override the buffer descriptor accordingly.
5781 */
5782 LLVMTypeRef v2i64 = LLVMVectorType(ctx->i64, 2);
5783 uint64_t stream_offset = 0;
5784
5785 for (unsigned stream = 0; stream < 4; ++stream) {
5786 unsigned num_components;
5787 unsigned stride;
5788 unsigned num_records;
5789 LLVMValueRef ring, tmp;
5790
5791 num_components = sel->info.num_stream_output_components[stream];
5792 if (!num_components)
5793 continue;
5794
5795 stride = 4 * num_components * sel->gs_max_out_vertices;
5796
5797 /* Limit on the stride field for <= CIK. */
5798 assert(stride < (1 << 14));
5799
5800 num_records = 64;
5801
5802 ring = LLVMBuildBitCast(builder, base_ring, v2i64, "");
5803 tmp = LLVMBuildExtractElement(builder, ring, uint->zero, "");
5804 tmp = LLVMBuildAdd(builder, tmp,
5805 LLVMConstInt(ctx->i64,
5806 stream_offset, 0), "");
5807 stream_offset += stride * 64;
5808
5809 ring = LLVMBuildInsertElement(builder, ring, tmp, uint->zero, "");
5810 ring = LLVMBuildBitCast(builder, ring, ctx->v4i32, "");
5811 tmp = LLVMBuildExtractElement(builder, ring, uint->one, "");
5812 tmp = LLVMBuildOr(builder, tmp,
5813 LLVMConstInt(ctx->i32,
5814 S_008F04_STRIDE(stride) |
5815 S_008F04_SWIZZLE_ENABLE(1), 0), "");
5816 ring = LLVMBuildInsertElement(builder, ring, tmp, uint->one, "");
5817 ring = LLVMBuildInsertElement(builder, ring,
5818 LLVMConstInt(ctx->i32, num_records, 0),
5819 LLVMConstInt(ctx->i32, 2, 0), "");
5820 ring = LLVMBuildInsertElement(builder, ring,
5821 LLVMConstInt(ctx->i32,
5822 S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
5823 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
5824 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
5825 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
5826 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5827 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) |
5828 S_008F0C_ELEMENT_SIZE(1) | /* element_size = 4 (bytes) */
5829 S_008F0C_INDEX_STRIDE(1) | /* index_stride = 16 (elements) */
5830 S_008F0C_ADD_TID_ENABLE(1),
5831 0),
5832 LLVMConstInt(ctx->i32, 3, 0), "");
5833 ring = LLVMBuildBitCast(builder, ring, ctx->v16i8, "");
5834
5835 ctx->gsvs_ring[stream] = ring;
5836 }
5837 }
5838 }
5839
5840 static void si_llvm_emit_polygon_stipple(struct si_shader_context *ctx,
5841 LLVMValueRef param_rw_buffers,
5842 unsigned param_pos_fixed_pt)
5843 {
5844 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
5845 struct gallivm_state *gallivm = bld_base->base.gallivm;
5846 LLVMBuilderRef builder = gallivm->builder;
5847 LLVMValueRef slot, desc, offset, row, bit, address[2];
5848
5849 /* Use the fixed-point gl_FragCoord input.
5850 * Since the stipple pattern is 32x32 and it repeats, just get 5 bits
5851 * per coordinate to get the repeating effect.
5852 */
5853 address[0] = unpack_param(ctx, param_pos_fixed_pt, 0, 5);
5854 address[1] = unpack_param(ctx, param_pos_fixed_pt, 16, 5);
5855
5856 /* Load the buffer descriptor. */
5857 slot = lp_build_const_int32(gallivm, SI_PS_CONST_POLY_STIPPLE);
5858 desc = ac_build_indexed_load_const(&ctx->ac, param_rw_buffers, slot);
5859
5860 /* The stipple pattern is 32x32, each row has 32 bits. */
5861 offset = LLVMBuildMul(builder, address[1],
5862 LLVMConstInt(ctx->i32, 4, 0), "");
5863 row = buffer_load_const(ctx, desc, offset);
5864 row = LLVMBuildBitCast(builder, row, ctx->i32, "");
5865 bit = LLVMBuildLShr(builder, row, address[0], "");
5866 bit = LLVMBuildTrunc(builder, bit, ctx->i1, "");
5867
5868 /* The intrinsic kills the thread if arg < 0. */
5869 bit = LLVMBuildSelect(builder, bit, LLVMConstReal(ctx->f32, 0),
5870 LLVMConstReal(ctx->f32, -1), "");
5871 ac_build_kill(&ctx->ac, bit);
5872 }
5873
5874 void si_shader_binary_read_config(struct ac_shader_binary *binary,
5875 struct si_shader_config *conf,
5876 unsigned symbol_offset)
5877 {
5878 unsigned i;
5879 const unsigned char *config =
5880 ac_shader_binary_config_start(binary, symbol_offset);
5881 bool really_needs_scratch = false;
5882
5883 /* LLVM adds SGPR spills to the scratch size.
5884 * Find out if we really need the scratch buffer.
5885 */
5886 for (i = 0; i < binary->reloc_count; i++) {
5887 const struct ac_shader_reloc *reloc = &binary->relocs[i];
5888
5889 if (!strcmp(scratch_rsrc_dword0_symbol, reloc->name) ||
5890 !strcmp(scratch_rsrc_dword1_symbol, reloc->name)) {
5891 really_needs_scratch = true;
5892 break;
5893 }
5894 }
5895
5896 /* XXX: We may be able to emit some of these values directly rather than
5897 * extracting fields to be emitted later.
5898 */
5899
5900 for (i = 0; i < binary->config_size_per_symbol; i+= 8) {
5901 unsigned reg = util_le32_to_cpu(*(uint32_t*)(config + i));
5902 unsigned value = util_le32_to_cpu(*(uint32_t*)(config + i + 4));
5903 switch (reg) {
5904 case R_00B028_SPI_SHADER_PGM_RSRC1_PS:
5905 case R_00B128_SPI_SHADER_PGM_RSRC1_VS:
5906 case R_00B228_SPI_SHADER_PGM_RSRC1_GS:
5907 case R_00B848_COMPUTE_PGM_RSRC1:
5908 conf->num_sgprs = MAX2(conf->num_sgprs, (G_00B028_SGPRS(value) + 1) * 8);
5909 conf->num_vgprs = MAX2(conf->num_vgprs, (G_00B028_VGPRS(value) + 1) * 4);
5910 conf->float_mode = G_00B028_FLOAT_MODE(value);
5911 conf->rsrc1 = value;
5912 break;
5913 case R_00B02C_SPI_SHADER_PGM_RSRC2_PS:
5914 conf->lds_size = MAX2(conf->lds_size, G_00B02C_EXTRA_LDS_SIZE(value));
5915 break;
5916 case R_00B84C_COMPUTE_PGM_RSRC2:
5917 conf->lds_size = MAX2(conf->lds_size, G_00B84C_LDS_SIZE(value));
5918 conf->rsrc2 = value;
5919 break;
5920 case R_0286CC_SPI_PS_INPUT_ENA:
5921 conf->spi_ps_input_ena = value;
5922 break;
5923 case R_0286D0_SPI_PS_INPUT_ADDR:
5924 conf->spi_ps_input_addr = value;
5925 break;
5926 case R_0286E8_SPI_TMPRING_SIZE:
5927 case R_00B860_COMPUTE_TMPRING_SIZE:
5928 /* WAVESIZE is in units of 256 dwords. */
5929 if (really_needs_scratch)
5930 conf->scratch_bytes_per_wave =
5931 G_00B860_WAVESIZE(value) * 256 * 4;
5932 break;
5933 case 0x4: /* SPILLED_SGPRS */
5934 conf->spilled_sgprs = value;
5935 break;
5936 case 0x8: /* SPILLED_VGPRS */
5937 conf->spilled_vgprs = value;
5938 break;
5939 default:
5940 {
5941 static bool printed;
5942
5943 if (!printed) {
5944 fprintf(stderr, "Warning: LLVM emitted unknown "
5945 "config register: 0x%x\n", reg);
5946 printed = true;
5947 }
5948 }
5949 break;
5950 }
5951 }
5952
5953 if (!conf->spi_ps_input_addr)
5954 conf->spi_ps_input_addr = conf->spi_ps_input_ena;
5955 }
5956
5957 void si_shader_apply_scratch_relocs(struct si_context *sctx,
5958 struct si_shader *shader,
5959 struct si_shader_config *config,
5960 uint64_t scratch_va)
5961 {
5962 unsigned i;
5963 uint32_t scratch_rsrc_dword0 = scratch_va;
5964 uint32_t scratch_rsrc_dword1 =
5965 S_008F04_BASE_ADDRESS_HI(scratch_va >> 32);
5966
5967 /* Enable scratch coalescing if LLVM sets ELEMENT_SIZE & INDEX_STRIDE
5968 * correctly.
5969 */
5970 if (HAVE_LLVM >= 0x0309)
5971 scratch_rsrc_dword1 |= S_008F04_SWIZZLE_ENABLE(1);
5972 else
5973 scratch_rsrc_dword1 |=
5974 S_008F04_STRIDE(config->scratch_bytes_per_wave / 64);
5975
5976 for (i = 0 ; i < shader->binary.reloc_count; i++) {
5977 const struct ac_shader_reloc *reloc =
5978 &shader->binary.relocs[i];
5979 if (!strcmp(scratch_rsrc_dword0_symbol, reloc->name)) {
5980 util_memcpy_cpu_to_le32(shader->binary.code + reloc->offset,
5981 &scratch_rsrc_dword0, 4);
5982 } else if (!strcmp(scratch_rsrc_dword1_symbol, reloc->name)) {
5983 util_memcpy_cpu_to_le32(shader->binary.code + reloc->offset,
5984 &scratch_rsrc_dword1, 4);
5985 }
5986 }
5987 }
5988
5989 static unsigned si_get_shader_binary_size(struct si_shader *shader)
5990 {
5991 unsigned size = shader->binary.code_size;
5992
5993 if (shader->prolog)
5994 size += shader->prolog->binary.code_size;
5995 if (shader->epilog)
5996 size += shader->epilog->binary.code_size;
5997 return size;
5998 }
5999
6000 int si_shader_binary_upload(struct si_screen *sscreen, struct si_shader *shader)
6001 {
6002 const struct ac_shader_binary *prolog =
6003 shader->prolog ? &shader->prolog->binary : NULL;
6004 const struct ac_shader_binary *epilog =
6005 shader->epilog ? &shader->epilog->binary : NULL;
6006 const struct ac_shader_binary *mainb = &shader->binary;
6007 unsigned bo_size = si_get_shader_binary_size(shader) +
6008 (!epilog ? mainb->rodata_size : 0);
6009 unsigned char *ptr;
6010
6011 assert(!prolog || !prolog->rodata_size);
6012 assert((!prolog && !epilog) || !mainb->rodata_size);
6013 assert(!epilog || !epilog->rodata_size);
6014
6015 /* GFX9 can fetch at most 128 bytes past the end of the shader.
6016 * Prevent VM faults.
6017 */
6018 if (sscreen->b.chip_class >= GFX9)
6019 bo_size += 128;
6020
6021 r600_resource_reference(&shader->bo, NULL);
6022 shader->bo = (struct r600_resource*)
6023 pipe_buffer_create(&sscreen->b.b, 0,
6024 PIPE_USAGE_IMMUTABLE,
6025 align(bo_size, SI_CPDMA_ALIGNMENT));
6026 if (!shader->bo)
6027 return -ENOMEM;
6028
6029 /* Upload. */
6030 ptr = sscreen->b.ws->buffer_map(shader->bo->buf, NULL,
6031 PIPE_TRANSFER_READ_WRITE);
6032
6033 if (prolog) {
6034 util_memcpy_cpu_to_le32(ptr, prolog->code, prolog->code_size);
6035 ptr += prolog->code_size;
6036 }
6037
6038 util_memcpy_cpu_to_le32(ptr, mainb->code, mainb->code_size);
6039 ptr += mainb->code_size;
6040
6041 if (epilog)
6042 util_memcpy_cpu_to_le32(ptr, epilog->code, epilog->code_size);
6043 else if (mainb->rodata_size > 0)
6044 util_memcpy_cpu_to_le32(ptr, mainb->rodata, mainb->rodata_size);
6045
6046 sscreen->b.ws->buffer_unmap(shader->bo->buf);
6047 return 0;
6048 }
6049
6050 static void si_shader_dump_disassembly(const struct ac_shader_binary *binary,
6051 struct pipe_debug_callback *debug,
6052 const char *name, FILE *file)
6053 {
6054 char *line, *p;
6055 unsigned i, count;
6056
6057 if (binary->disasm_string) {
6058 fprintf(file, "Shader %s disassembly:\n", name);
6059 fprintf(file, "%s", binary->disasm_string);
6060
6061 if (debug && debug->debug_message) {
6062 /* Very long debug messages are cut off, so send the
6063 * disassembly one line at a time. This causes more
6064 * overhead, but on the plus side it simplifies
6065 * parsing of resulting logs.
6066 */
6067 pipe_debug_message(debug, SHADER_INFO,
6068 "Shader Disassembly Begin");
6069
6070 line = binary->disasm_string;
6071 while (*line) {
6072 p = util_strchrnul(line, '\n');
6073 count = p - line;
6074
6075 if (count) {
6076 pipe_debug_message(debug, SHADER_INFO,
6077 "%.*s", count, line);
6078 }
6079
6080 if (!*p)
6081 break;
6082 line = p + 1;
6083 }
6084
6085 pipe_debug_message(debug, SHADER_INFO,
6086 "Shader Disassembly End");
6087 }
6088 } else {
6089 fprintf(file, "Shader %s binary:\n", name);
6090 for (i = 0; i < binary->code_size; i += 4) {
6091 fprintf(file, "@0x%x: %02x%02x%02x%02x\n", i,
6092 binary->code[i + 3], binary->code[i + 2],
6093 binary->code[i + 1], binary->code[i]);
6094 }
6095 }
6096 }
6097
6098 static void si_shader_dump_stats(struct si_screen *sscreen,
6099 struct si_shader *shader,
6100 struct pipe_debug_callback *debug,
6101 unsigned processor,
6102 FILE *file,
6103 bool check_debug_option)
6104 {
6105 struct si_shader_config *conf = &shader->config;
6106 unsigned num_inputs = shader->selector ? shader->selector->info.num_inputs : 0;
6107 unsigned code_size = si_get_shader_binary_size(shader);
6108 unsigned lds_increment = sscreen->b.chip_class >= CIK ? 512 : 256;
6109 unsigned lds_per_wave = 0;
6110 unsigned max_simd_waves = 10;
6111
6112 /* Compute LDS usage for PS. */
6113 switch (processor) {
6114 case PIPE_SHADER_FRAGMENT:
6115 /* The minimum usage per wave is (num_inputs * 48). The maximum
6116 * usage is (num_inputs * 48 * 16).
6117 * We can get anything in between and it varies between waves.
6118 *
6119 * The 48 bytes per input for a single primitive is equal to
6120 * 4 bytes/component * 4 components/input * 3 points.
6121 *
6122 * Other stages don't know the size at compile time or don't
6123 * allocate LDS per wave, but instead they do it per thread group.
6124 */
6125 lds_per_wave = conf->lds_size * lds_increment +
6126 align(num_inputs * 48, lds_increment);
6127 break;
6128 case PIPE_SHADER_COMPUTE:
6129 if (shader->selector) {
6130 unsigned max_workgroup_size =
6131 si_get_max_workgroup_size(shader);
6132 lds_per_wave = (conf->lds_size * lds_increment) /
6133 DIV_ROUND_UP(max_workgroup_size, 64);
6134 }
6135 break;
6136 }
6137
6138 /* Compute the per-SIMD wave counts. */
6139 if (conf->num_sgprs) {
6140 if (sscreen->b.chip_class >= VI)
6141 max_simd_waves = MIN2(max_simd_waves, 800 / conf->num_sgprs);
6142 else
6143 max_simd_waves = MIN2(max_simd_waves, 512 / conf->num_sgprs);
6144 }
6145
6146 if (conf->num_vgprs)
6147 max_simd_waves = MIN2(max_simd_waves, 256 / conf->num_vgprs);
6148
6149 /* LDS is 64KB per CU (4 SIMDs), which is 16KB per SIMD (usage above
6150 * 16KB makes some SIMDs unoccupied). */
6151 if (lds_per_wave)
6152 max_simd_waves = MIN2(max_simd_waves, 16384 / lds_per_wave);
6153
6154 if (!check_debug_option ||
6155 r600_can_dump_shader(&sscreen->b, processor)) {
6156 if (processor == PIPE_SHADER_FRAGMENT) {
6157 fprintf(file, "*** SHADER CONFIG ***\n"
6158 "SPI_PS_INPUT_ADDR = 0x%04x\n"
6159 "SPI_PS_INPUT_ENA = 0x%04x\n",
6160 conf->spi_ps_input_addr, conf->spi_ps_input_ena);
6161 }
6162
6163 fprintf(file, "*** SHADER STATS ***\n"
6164 "SGPRS: %d\n"
6165 "VGPRS: %d\n"
6166 "Spilled SGPRs: %d\n"
6167 "Spilled VGPRs: %d\n"
6168 "Private memory VGPRs: %d\n"
6169 "Code Size: %d bytes\n"
6170 "LDS: %d blocks\n"
6171 "Scratch: %d bytes per wave\n"
6172 "Max Waves: %d\n"
6173 "********************\n\n\n",
6174 conf->num_sgprs, conf->num_vgprs,
6175 conf->spilled_sgprs, conf->spilled_vgprs,
6176 conf->private_mem_vgprs, code_size,
6177 conf->lds_size, conf->scratch_bytes_per_wave,
6178 max_simd_waves);
6179 }
6180
6181 pipe_debug_message(debug, SHADER_INFO,
6182 "Shader Stats: SGPRS: %d VGPRS: %d Code Size: %d "
6183 "LDS: %d Scratch: %d Max Waves: %d Spilled SGPRs: %d "
6184 "Spilled VGPRs: %d PrivMem VGPRs: %d",
6185 conf->num_sgprs, conf->num_vgprs, code_size,
6186 conf->lds_size, conf->scratch_bytes_per_wave,
6187 max_simd_waves, conf->spilled_sgprs,
6188 conf->spilled_vgprs, conf->private_mem_vgprs);
6189 }
6190
6191 const char *si_get_shader_name(struct si_shader *shader, unsigned processor)
6192 {
6193 switch (processor) {
6194 case PIPE_SHADER_VERTEX:
6195 if (shader->key.as_es)
6196 return "Vertex Shader as ES";
6197 else if (shader->key.as_ls)
6198 return "Vertex Shader as LS";
6199 else
6200 return "Vertex Shader as VS";
6201 case PIPE_SHADER_TESS_CTRL:
6202 return "Tessellation Control Shader";
6203 case PIPE_SHADER_TESS_EVAL:
6204 if (shader->key.as_es)
6205 return "Tessellation Evaluation Shader as ES";
6206 else
6207 return "Tessellation Evaluation Shader as VS";
6208 case PIPE_SHADER_GEOMETRY:
6209 if (shader->is_gs_copy_shader)
6210 return "GS Copy Shader as VS";
6211 else
6212 return "Geometry Shader";
6213 case PIPE_SHADER_FRAGMENT:
6214 return "Pixel Shader";
6215 case PIPE_SHADER_COMPUTE:
6216 return "Compute Shader";
6217 default:
6218 return "Unknown Shader";
6219 }
6220 }
6221
6222 void si_shader_dump(struct si_screen *sscreen, struct si_shader *shader,
6223 struct pipe_debug_callback *debug, unsigned processor,
6224 FILE *file, bool check_debug_option)
6225 {
6226 if (!check_debug_option ||
6227 r600_can_dump_shader(&sscreen->b, processor))
6228 si_dump_shader_key(processor, &shader->key, file);
6229
6230 if (!check_debug_option && shader->binary.llvm_ir_string) {
6231 fprintf(file, "\n%s - main shader part - LLVM IR:\n\n",
6232 si_get_shader_name(shader, processor));
6233 fprintf(file, "%s\n", shader->binary.llvm_ir_string);
6234 }
6235
6236 if (!check_debug_option ||
6237 (r600_can_dump_shader(&sscreen->b, processor) &&
6238 !(sscreen->b.debug_flags & DBG_NO_ASM))) {
6239 fprintf(file, "\n%s:\n", si_get_shader_name(shader, processor));
6240
6241 if (shader->prolog)
6242 si_shader_dump_disassembly(&shader->prolog->binary,
6243 debug, "prolog", file);
6244
6245 si_shader_dump_disassembly(&shader->binary, debug, "main", file);
6246
6247 if (shader->epilog)
6248 si_shader_dump_disassembly(&shader->epilog->binary,
6249 debug, "epilog", file);
6250 fprintf(file, "\n");
6251 }
6252
6253 si_shader_dump_stats(sscreen, shader, debug, processor, file,
6254 check_debug_option);
6255 }
6256
6257 int si_compile_llvm(struct si_screen *sscreen,
6258 struct ac_shader_binary *binary,
6259 struct si_shader_config *conf,
6260 LLVMTargetMachineRef tm,
6261 LLVMModuleRef mod,
6262 struct pipe_debug_callback *debug,
6263 unsigned processor,
6264 const char *name)
6265 {
6266 int r = 0;
6267 unsigned count = p_atomic_inc_return(&sscreen->b.num_compilations);
6268
6269 if (r600_can_dump_shader(&sscreen->b, processor)) {
6270 fprintf(stderr, "radeonsi: Compiling shader %d\n", count);
6271
6272 if (!(sscreen->b.debug_flags & (DBG_NO_IR | DBG_PREOPT_IR))) {
6273 fprintf(stderr, "%s LLVM IR:\n\n", name);
6274 ac_dump_module(mod);
6275 fprintf(stderr, "\n");
6276 }
6277 }
6278
6279 if (sscreen->record_llvm_ir) {
6280 char *ir = LLVMPrintModuleToString(mod);
6281 binary->llvm_ir_string = strdup(ir);
6282 LLVMDisposeMessage(ir);
6283 }
6284
6285 if (!si_replace_shader(count, binary)) {
6286 r = si_llvm_compile(mod, binary, tm, debug);
6287 if (r)
6288 return r;
6289 }
6290
6291 si_shader_binary_read_config(binary, conf, 0);
6292
6293 /* Enable 64-bit and 16-bit denormals, because there is no performance
6294 * cost.
6295 *
6296 * If denormals are enabled, all floating-point output modifiers are
6297 * ignored.
6298 *
6299 * Don't enable denormals for 32-bit floats, because:
6300 * - Floating-point output modifiers would be ignored by the hw.
6301 * - Some opcodes don't support denormals, such as v_mad_f32. We would
6302 * have to stop using those.
6303 * - SI & CI would be very slow.
6304 */
6305 conf->float_mode |= V_00B028_FP_64_DENORMS;
6306
6307 FREE(binary->config);
6308 FREE(binary->global_symbol_offsets);
6309 binary->config = NULL;
6310 binary->global_symbol_offsets = NULL;
6311
6312 /* Some shaders can't have rodata because their binaries can be
6313 * concatenated.
6314 */
6315 if (binary->rodata_size &&
6316 (processor == PIPE_SHADER_VERTEX ||
6317 processor == PIPE_SHADER_TESS_CTRL ||
6318 processor == PIPE_SHADER_TESS_EVAL ||
6319 processor == PIPE_SHADER_FRAGMENT)) {
6320 fprintf(stderr, "radeonsi: The shader can't have rodata.");
6321 return -EINVAL;
6322 }
6323
6324 return r;
6325 }
6326
6327 static void si_llvm_build_ret(struct si_shader_context *ctx, LLVMValueRef ret)
6328 {
6329 if (LLVMGetTypeKind(LLVMTypeOf(ret)) == LLVMVoidTypeKind)
6330 LLVMBuildRetVoid(ctx->gallivm.builder);
6331 else
6332 LLVMBuildRet(ctx->gallivm.builder, ret);
6333 }
6334
6335 /* Generate code for the hardware VS shader stage to go with a geometry shader */
6336 struct si_shader *
6337 si_generate_gs_copy_shader(struct si_screen *sscreen,
6338 LLVMTargetMachineRef tm,
6339 struct si_shader_selector *gs_selector,
6340 struct pipe_debug_callback *debug)
6341 {
6342 struct si_shader_context ctx;
6343 struct si_shader *shader;
6344 struct gallivm_state *gallivm = &ctx.gallivm;
6345 LLVMBuilderRef builder;
6346 struct lp_build_tgsi_context *bld_base = &ctx.bld_base;
6347 struct lp_build_context *uint = &bld_base->uint_bld;
6348 struct si_shader_output_values *outputs;
6349 struct tgsi_shader_info *gsinfo = &gs_selector->info;
6350 int i, r;
6351
6352 outputs = MALLOC(gsinfo->num_outputs * sizeof(outputs[0]));
6353
6354 if (!outputs)
6355 return NULL;
6356
6357 shader = CALLOC_STRUCT(si_shader);
6358 if (!shader) {
6359 FREE(outputs);
6360 return NULL;
6361 }
6362
6363
6364 shader->selector = gs_selector;
6365 shader->is_gs_copy_shader = true;
6366
6367 si_init_shader_ctx(&ctx, sscreen, shader, tm);
6368 ctx.type = PIPE_SHADER_VERTEX;
6369
6370 builder = gallivm->builder;
6371
6372 create_function(&ctx);
6373 preload_ring_buffers(&ctx);
6374
6375 LLVMValueRef voffset =
6376 lp_build_mul_imm(uint, LLVMGetParam(ctx.main_fn,
6377 ctx.param_vertex_id), 4);
6378
6379 /* Fetch the vertex stream ID.*/
6380 LLVMValueRef stream_id;
6381
6382 if (gs_selector->so.num_outputs)
6383 stream_id = unpack_param(&ctx, ctx.param_streamout_config, 24, 2);
6384 else
6385 stream_id = uint->zero;
6386
6387 /* Fill in output information. */
6388 for (i = 0; i < gsinfo->num_outputs; ++i) {
6389 outputs[i].semantic_name = gsinfo->output_semantic_name[i];
6390 outputs[i].semantic_index = gsinfo->output_semantic_index[i];
6391
6392 for (int chan = 0; chan < 4; chan++) {
6393 outputs[i].vertex_stream[chan] =
6394 (gsinfo->output_streams[i] >> (2 * chan)) & 3;
6395 }
6396 }
6397
6398 LLVMBasicBlockRef end_bb;
6399 LLVMValueRef switch_inst;
6400
6401 end_bb = LLVMAppendBasicBlockInContext(gallivm->context, ctx.main_fn, "end");
6402 switch_inst = LLVMBuildSwitch(builder, stream_id, end_bb, 4);
6403
6404 for (int stream = 0; stream < 4; stream++) {
6405 LLVMBasicBlockRef bb;
6406 unsigned offset;
6407
6408 if (!gsinfo->num_stream_output_components[stream])
6409 continue;
6410
6411 if (stream > 0 && !gs_selector->so.num_outputs)
6412 continue;
6413
6414 bb = LLVMInsertBasicBlockInContext(gallivm->context, end_bb, "out");
6415 LLVMAddCase(switch_inst, lp_build_const_int32(gallivm, stream), bb);
6416 LLVMPositionBuilderAtEnd(builder, bb);
6417
6418 /* Fetch vertex data from GSVS ring */
6419 offset = 0;
6420 for (i = 0; i < gsinfo->num_outputs; ++i) {
6421 for (unsigned chan = 0; chan < 4; chan++) {
6422 if (!(gsinfo->output_usagemask[i] & (1 << chan)) ||
6423 outputs[i].vertex_stream[chan] != stream) {
6424 outputs[i].values[chan] = ctx.bld_base.base.undef;
6425 continue;
6426 }
6427
6428 LLVMValueRef soffset = LLVMConstInt(ctx.i32,
6429 offset * gs_selector->gs_max_out_vertices * 16 * 4, 0);
6430 offset++;
6431
6432 outputs[i].values[chan] =
6433 ac_build_buffer_load(&ctx.ac,
6434 ctx.gsvs_ring[0], 1,
6435 uint->zero, voffset,
6436 soffset, 0, 1, 1, true);
6437 }
6438 }
6439
6440 /* Streamout and exports. */
6441 if (gs_selector->so.num_outputs) {
6442 si_llvm_emit_streamout(&ctx, outputs,
6443 gsinfo->num_outputs,
6444 stream);
6445 }
6446
6447 if (stream == 0)
6448 si_llvm_export_vs(bld_base, outputs, gsinfo->num_outputs);
6449
6450 LLVMBuildBr(builder, end_bb);
6451 }
6452
6453 LLVMPositionBuilderAtEnd(builder, end_bb);
6454
6455 LLVMBuildRetVoid(gallivm->builder);
6456
6457 /* Dump LLVM IR before any optimization passes */
6458 if (sscreen->b.debug_flags & DBG_PREOPT_IR &&
6459 r600_can_dump_shader(&sscreen->b, PIPE_SHADER_GEOMETRY))
6460 ac_dump_module(bld_base->base.gallivm->module);
6461
6462 si_llvm_finalize_module(&ctx,
6463 r600_extra_shader_checks(&sscreen->b, PIPE_SHADER_GEOMETRY));
6464
6465 r = si_compile_llvm(sscreen, &ctx.shader->binary,
6466 &ctx.shader->config, ctx.tm,
6467 bld_base->base.gallivm->module,
6468 debug, PIPE_SHADER_GEOMETRY,
6469 "GS Copy Shader");
6470 if (!r) {
6471 if (r600_can_dump_shader(&sscreen->b, PIPE_SHADER_GEOMETRY))
6472 fprintf(stderr, "GS Copy Shader:\n");
6473 si_shader_dump(sscreen, ctx.shader, debug,
6474 PIPE_SHADER_GEOMETRY, stderr, true);
6475 r = si_shader_binary_upload(sscreen, ctx.shader);
6476 }
6477
6478 si_llvm_dispose(&ctx);
6479
6480 FREE(outputs);
6481
6482 if (r != 0) {
6483 FREE(shader);
6484 shader = NULL;
6485 }
6486 return shader;
6487 }
6488
6489 static void si_dump_shader_key(unsigned shader, struct si_shader_key *key,
6490 FILE *f)
6491 {
6492 int i;
6493
6494 fprintf(f, "SHADER KEY\n");
6495
6496 switch (shader) {
6497 case PIPE_SHADER_VERTEX:
6498 fprintf(f, " part.vs.prolog.instance_divisors = {");
6499 for (i = 0; i < ARRAY_SIZE(key->part.vs.prolog.instance_divisors); i++)
6500 fprintf(f, !i ? "%u" : ", %u",
6501 key->part.vs.prolog.instance_divisors[i]);
6502 fprintf(f, "}\n");
6503 fprintf(f, " part.vs.epilog.export_prim_id = %u\n", key->part.vs.epilog.export_prim_id);
6504 fprintf(f, " as_es = %u\n", key->as_es);
6505 fprintf(f, " as_ls = %u\n", key->as_ls);
6506
6507 fprintf(f, " mono.vs.fix_fetch = {");
6508 for (i = 0; i < SI_MAX_ATTRIBS; i++)
6509 fprintf(f, !i ? "%u" : ", %u", key->mono.vs.fix_fetch[i]);
6510 fprintf(f, "}\n");
6511 break;
6512
6513 case PIPE_SHADER_TESS_CTRL:
6514 fprintf(f, " part.tcs.epilog.prim_mode = %u\n", key->part.tcs.epilog.prim_mode);
6515 fprintf(f, " mono.tcs.inputs_to_copy = 0x%"PRIx64"\n", key->mono.tcs.inputs_to_copy);
6516 break;
6517
6518 case PIPE_SHADER_TESS_EVAL:
6519 fprintf(f, " part.tes.epilog.export_prim_id = %u\n", key->part.tes.epilog.export_prim_id);
6520 fprintf(f, " as_es = %u\n", key->as_es);
6521 break;
6522
6523 case PIPE_SHADER_GEOMETRY:
6524 fprintf(f, " part.gs.prolog.tri_strip_adj_fix = %u\n", key->part.gs.prolog.tri_strip_adj_fix);
6525 break;
6526
6527 case PIPE_SHADER_COMPUTE:
6528 break;
6529
6530 case PIPE_SHADER_FRAGMENT:
6531 fprintf(f, " part.ps.prolog.color_two_side = %u\n", key->part.ps.prolog.color_two_side);
6532 fprintf(f, " part.ps.prolog.flatshade_colors = %u\n", key->part.ps.prolog.flatshade_colors);
6533 fprintf(f, " part.ps.prolog.poly_stipple = %u\n", key->part.ps.prolog.poly_stipple);
6534 fprintf(f, " part.ps.prolog.force_persp_sample_interp = %u\n", key->part.ps.prolog.force_persp_sample_interp);
6535 fprintf(f, " part.ps.prolog.force_linear_sample_interp = %u\n", key->part.ps.prolog.force_linear_sample_interp);
6536 fprintf(f, " part.ps.prolog.force_persp_center_interp = %u\n", key->part.ps.prolog.force_persp_center_interp);
6537 fprintf(f, " part.ps.prolog.force_linear_center_interp = %u\n", key->part.ps.prolog.force_linear_center_interp);
6538 fprintf(f, " part.ps.prolog.bc_optimize_for_persp = %u\n", key->part.ps.prolog.bc_optimize_for_persp);
6539 fprintf(f, " part.ps.prolog.bc_optimize_for_linear = %u\n", key->part.ps.prolog.bc_optimize_for_linear);
6540 fprintf(f, " part.ps.epilog.spi_shader_col_format = 0x%x\n", key->part.ps.epilog.spi_shader_col_format);
6541 fprintf(f, " part.ps.epilog.color_is_int8 = 0x%X\n", key->part.ps.epilog.color_is_int8);
6542 fprintf(f, " part.ps.epilog.color_is_int10 = 0x%X\n", key->part.ps.epilog.color_is_int10);
6543 fprintf(f, " part.ps.epilog.last_cbuf = %u\n", key->part.ps.epilog.last_cbuf);
6544 fprintf(f, " part.ps.epilog.alpha_func = %u\n", key->part.ps.epilog.alpha_func);
6545 fprintf(f, " part.ps.epilog.alpha_to_one = %u\n", key->part.ps.epilog.alpha_to_one);
6546 fprintf(f, " part.ps.epilog.poly_line_smoothing = %u\n", key->part.ps.epilog.poly_line_smoothing);
6547 fprintf(f, " part.ps.epilog.clamp_color = %u\n", key->part.ps.epilog.clamp_color);
6548 break;
6549
6550 default:
6551 assert(0);
6552 }
6553
6554 if ((shader == PIPE_SHADER_GEOMETRY ||
6555 shader == PIPE_SHADER_TESS_EVAL ||
6556 shader == PIPE_SHADER_VERTEX) &&
6557 !key->as_es && !key->as_ls) {
6558 fprintf(f, " opt.hw_vs.kill_outputs = 0x%"PRIx64"\n", key->opt.hw_vs.kill_outputs);
6559 fprintf(f, " opt.hw_vs.kill_outputs2 = 0x%x\n", key->opt.hw_vs.kill_outputs2);
6560 fprintf(f, " opt.hw_vs.clip_disable = %u\n", key->opt.hw_vs.clip_disable);
6561 }
6562 }
6563
6564 static void si_init_shader_ctx(struct si_shader_context *ctx,
6565 struct si_screen *sscreen,
6566 struct si_shader *shader,
6567 LLVMTargetMachineRef tm)
6568 {
6569 struct lp_build_tgsi_context *bld_base;
6570 struct lp_build_tgsi_action tmpl = {};
6571
6572 si_llvm_context_init(ctx, sscreen, shader, tm,
6573 (shader && shader->selector) ? &shader->selector->info : NULL,
6574 (shader && shader->selector) ? shader->selector->tokens : NULL);
6575
6576 bld_base = &ctx->bld_base;
6577 bld_base->emit_fetch_funcs[TGSI_FILE_CONSTANT] = fetch_constant;
6578
6579 bld_base->op_actions[TGSI_OPCODE_INTERP_CENTROID] = interp_action;
6580 bld_base->op_actions[TGSI_OPCODE_INTERP_SAMPLE] = interp_action;
6581 bld_base->op_actions[TGSI_OPCODE_INTERP_OFFSET] = interp_action;
6582
6583 bld_base->op_actions[TGSI_OPCODE_TEX] = tex_action;
6584 bld_base->op_actions[TGSI_OPCODE_TEX_LZ] = tex_action;
6585 bld_base->op_actions[TGSI_OPCODE_TEX2] = tex_action;
6586 bld_base->op_actions[TGSI_OPCODE_TXB] = tex_action;
6587 bld_base->op_actions[TGSI_OPCODE_TXB2] = tex_action;
6588 bld_base->op_actions[TGSI_OPCODE_TXD] = tex_action;
6589 bld_base->op_actions[TGSI_OPCODE_TXF] = tex_action;
6590 bld_base->op_actions[TGSI_OPCODE_TXF_LZ] = tex_action;
6591 bld_base->op_actions[TGSI_OPCODE_TXL] = tex_action;
6592 bld_base->op_actions[TGSI_OPCODE_TXL2] = tex_action;
6593 bld_base->op_actions[TGSI_OPCODE_TXP] = tex_action;
6594 bld_base->op_actions[TGSI_OPCODE_TXQ].fetch_args = txq_fetch_args;
6595 bld_base->op_actions[TGSI_OPCODE_TXQ].emit = txq_emit;
6596 bld_base->op_actions[TGSI_OPCODE_TG4] = tex_action;
6597 bld_base->op_actions[TGSI_OPCODE_LODQ] = tex_action;
6598 bld_base->op_actions[TGSI_OPCODE_TXQS].emit = si_llvm_emit_txqs;
6599
6600 bld_base->op_actions[TGSI_OPCODE_LOAD].fetch_args = load_fetch_args;
6601 bld_base->op_actions[TGSI_OPCODE_LOAD].emit = load_emit;
6602 bld_base->op_actions[TGSI_OPCODE_STORE].fetch_args = store_fetch_args;
6603 bld_base->op_actions[TGSI_OPCODE_STORE].emit = store_emit;
6604 bld_base->op_actions[TGSI_OPCODE_RESQ].fetch_args = resq_fetch_args;
6605 bld_base->op_actions[TGSI_OPCODE_RESQ].emit = resq_emit;
6606
6607 tmpl.fetch_args = atomic_fetch_args;
6608 tmpl.emit = atomic_emit;
6609 bld_base->op_actions[TGSI_OPCODE_ATOMUADD] = tmpl;
6610 bld_base->op_actions[TGSI_OPCODE_ATOMUADD].intr_name = "add";
6611 bld_base->op_actions[TGSI_OPCODE_ATOMXCHG] = tmpl;
6612 bld_base->op_actions[TGSI_OPCODE_ATOMXCHG].intr_name = "swap";
6613 bld_base->op_actions[TGSI_OPCODE_ATOMCAS] = tmpl;
6614 bld_base->op_actions[TGSI_OPCODE_ATOMCAS].intr_name = "cmpswap";
6615 bld_base->op_actions[TGSI_OPCODE_ATOMAND] = tmpl;
6616 bld_base->op_actions[TGSI_OPCODE_ATOMAND].intr_name = "and";
6617 bld_base->op_actions[TGSI_OPCODE_ATOMOR] = tmpl;
6618 bld_base->op_actions[TGSI_OPCODE_ATOMOR].intr_name = "or";
6619 bld_base->op_actions[TGSI_OPCODE_ATOMXOR] = tmpl;
6620 bld_base->op_actions[TGSI_OPCODE_ATOMXOR].intr_name = "xor";
6621 bld_base->op_actions[TGSI_OPCODE_ATOMUMIN] = tmpl;
6622 bld_base->op_actions[TGSI_OPCODE_ATOMUMIN].intr_name = "umin";
6623 bld_base->op_actions[TGSI_OPCODE_ATOMUMAX] = tmpl;
6624 bld_base->op_actions[TGSI_OPCODE_ATOMUMAX].intr_name = "umax";
6625 bld_base->op_actions[TGSI_OPCODE_ATOMIMIN] = tmpl;
6626 bld_base->op_actions[TGSI_OPCODE_ATOMIMIN].intr_name = "smin";
6627 bld_base->op_actions[TGSI_OPCODE_ATOMIMAX] = tmpl;
6628 bld_base->op_actions[TGSI_OPCODE_ATOMIMAX].intr_name = "smax";
6629
6630 bld_base->op_actions[TGSI_OPCODE_MEMBAR].emit = membar_emit;
6631
6632 bld_base->op_actions[TGSI_OPCODE_CLOCK].emit = clock_emit;
6633
6634 bld_base->op_actions[TGSI_OPCODE_DDX].emit = si_llvm_emit_ddxy;
6635 bld_base->op_actions[TGSI_OPCODE_DDY].emit = si_llvm_emit_ddxy;
6636 bld_base->op_actions[TGSI_OPCODE_DDX_FINE].emit = si_llvm_emit_ddxy;
6637 bld_base->op_actions[TGSI_OPCODE_DDY_FINE].emit = si_llvm_emit_ddxy;
6638
6639 bld_base->op_actions[TGSI_OPCODE_VOTE_ALL].emit = vote_all_emit;
6640 bld_base->op_actions[TGSI_OPCODE_VOTE_ANY].emit = vote_any_emit;
6641 bld_base->op_actions[TGSI_OPCODE_VOTE_EQ].emit = vote_eq_emit;
6642
6643 bld_base->op_actions[TGSI_OPCODE_EMIT].emit = si_llvm_emit_vertex;
6644 bld_base->op_actions[TGSI_OPCODE_ENDPRIM].emit = si_llvm_emit_primitive;
6645 bld_base->op_actions[TGSI_OPCODE_BARRIER].emit = si_llvm_emit_barrier;
6646 }
6647
6648 #define EXP_TARGET (HAVE_LLVM >= 0x0500 ? 0 : 3)
6649 #define EXP_OUT0 (HAVE_LLVM >= 0x0500 ? 2 : 5)
6650
6651 /* Return true if the PARAM export has been eliminated. */
6652 static bool si_eliminate_const_output(struct si_shader_context *ctx,
6653 LLVMValueRef inst, unsigned offset)
6654 {
6655 struct si_shader *shader = ctx->shader;
6656 unsigned num_outputs = shader->selector->info.num_outputs;
6657 unsigned i, default_val; /* SPI_PS_INPUT_CNTL_i.DEFAULT_VAL */
6658 bool is_zero[4] = {}, is_one[4] = {};
6659
6660 for (i = 0; i < 4; i++) {
6661 LLVMBool loses_info;
6662 LLVMValueRef p = LLVMGetOperand(inst, EXP_OUT0 + i);
6663
6664 /* It's a constant expression. Undef outputs are eliminated too. */
6665 if (LLVMIsUndef(p)) {
6666 is_zero[i] = true;
6667 is_one[i] = true;
6668 } else if (LLVMIsAConstantFP(p)) {
6669 double a = LLVMConstRealGetDouble(p, &loses_info);
6670
6671 if (a == 0)
6672 is_zero[i] = true;
6673 else if (a == 1)
6674 is_one[i] = true;
6675 else
6676 return false; /* other constant */
6677 } else
6678 return false;
6679 }
6680
6681 /* Only certain combinations of 0 and 1 can be eliminated. */
6682 if (is_zero[0] && is_zero[1] && is_zero[2])
6683 default_val = is_zero[3] ? 0 : 1;
6684 else if (is_one[0] && is_one[1] && is_one[2])
6685 default_val = is_zero[3] ? 2 : 3;
6686 else
6687 return false;
6688
6689 /* The PARAM export can be represented as DEFAULT_VAL. Kill it. */
6690 LLVMInstructionEraseFromParent(inst);
6691
6692 /* Change OFFSET to DEFAULT_VAL. */
6693 for (i = 0; i < num_outputs; i++) {
6694 if (shader->info.vs_output_param_offset[i] == offset) {
6695 shader->info.vs_output_param_offset[i] =
6696 EXP_PARAM_DEFAULT_VAL_0000 + default_val;
6697 break;
6698 }
6699 }
6700 return true;
6701 }
6702
6703 struct si_vs_exports {
6704 unsigned num;
6705 unsigned offset[SI_MAX_VS_OUTPUTS];
6706 LLVMValueRef inst[SI_MAX_VS_OUTPUTS];
6707 };
6708
6709 static void si_eliminate_const_vs_outputs(struct si_shader_context *ctx)
6710 {
6711 struct si_shader *shader = ctx->shader;
6712 struct tgsi_shader_info *info = &shader->selector->info;
6713 LLVMBasicBlockRef bb;
6714 struct si_vs_exports exports;
6715 bool removed_any = false;
6716
6717 exports.num = 0;
6718
6719 if (ctx->type == PIPE_SHADER_FRAGMENT ||
6720 ctx->type == PIPE_SHADER_COMPUTE ||
6721 shader->key.as_es ||
6722 shader->key.as_ls)
6723 return;
6724
6725 /* Process all LLVM instructions. */
6726 bb = LLVMGetFirstBasicBlock(ctx->main_fn);
6727 while (bb) {
6728 LLVMValueRef inst = LLVMGetFirstInstruction(bb);
6729
6730 while (inst) {
6731 LLVMValueRef cur = inst;
6732 inst = LLVMGetNextInstruction(inst);
6733
6734 if (LLVMGetInstructionOpcode(cur) != LLVMCall)
6735 continue;
6736
6737 LLVMValueRef callee = lp_get_called_value(cur);
6738
6739 if (!lp_is_function(callee))
6740 continue;
6741
6742 const char *name = LLVMGetValueName(callee);
6743 unsigned num_args = LLVMCountParams(callee);
6744
6745 /* Check if this is an export instruction. */
6746 if ((num_args != 9 && num_args != 8) ||
6747 (strcmp(name, "llvm.SI.export") &&
6748 strcmp(name, "llvm.amdgcn.exp.f32")))
6749 continue;
6750
6751 LLVMValueRef arg = LLVMGetOperand(cur, EXP_TARGET);
6752 unsigned target = LLVMConstIntGetZExtValue(arg);
6753
6754 if (target < V_008DFC_SQ_EXP_PARAM)
6755 continue;
6756
6757 target -= V_008DFC_SQ_EXP_PARAM;
6758
6759 /* Eliminate constant value PARAM exports. */
6760 if (si_eliminate_const_output(ctx, cur, target)) {
6761 removed_any = true;
6762 } else {
6763 exports.offset[exports.num] = target;
6764 exports.inst[exports.num] = cur;
6765 exports.num++;
6766 }
6767 }
6768 bb = LLVMGetNextBasicBlock(bb);
6769 }
6770
6771 /* Remove holes in export memory due to removed PARAM exports.
6772 * This is done by renumbering all PARAM exports.
6773 */
6774 if (removed_any) {
6775 ubyte current_offset[SI_MAX_VS_OUTPUTS];
6776 unsigned new_count = 0;
6777 unsigned out, i;
6778
6779 /* Make a copy of the offsets. We need the old version while
6780 * we are modifying some of them. */
6781 assert(sizeof(current_offset) ==
6782 sizeof(shader->info.vs_output_param_offset));
6783 memcpy(current_offset, shader->info.vs_output_param_offset,
6784 sizeof(current_offset));
6785
6786 for (i = 0; i < exports.num; i++) {
6787 unsigned offset = exports.offset[i];
6788
6789 for (out = 0; out < info->num_outputs; out++) {
6790 if (current_offset[out] != offset)
6791 continue;
6792
6793 LLVMSetOperand(exports.inst[i], EXP_TARGET,
6794 LLVMConstInt(ctx->i32,
6795 V_008DFC_SQ_EXP_PARAM + new_count, 0));
6796 shader->info.vs_output_param_offset[out] = new_count;
6797 new_count++;
6798 break;
6799 }
6800 }
6801 shader->info.nr_param_exports = new_count;
6802 }
6803 }
6804
6805 static void si_count_scratch_private_memory(struct si_shader_context *ctx)
6806 {
6807 ctx->shader->config.private_mem_vgprs = 0;
6808
6809 /* Process all LLVM instructions. */
6810 LLVMBasicBlockRef bb = LLVMGetFirstBasicBlock(ctx->main_fn);
6811 while (bb) {
6812 LLVMValueRef next = LLVMGetFirstInstruction(bb);
6813
6814 while (next) {
6815 LLVMValueRef inst = next;
6816 next = LLVMGetNextInstruction(next);
6817
6818 if (LLVMGetInstructionOpcode(inst) != LLVMAlloca)
6819 continue;
6820
6821 LLVMTypeRef type = LLVMGetElementType(LLVMTypeOf(inst));
6822 /* No idea why LLVM aligns allocas to 4 elements. */
6823 unsigned alignment = LLVMGetAlignment(inst);
6824 unsigned dw_size = align(llvm_get_type_size(type) / 4, alignment);
6825 ctx->shader->config.private_mem_vgprs += dw_size;
6826 }
6827 bb = LLVMGetNextBasicBlock(bb);
6828 }
6829 }
6830
6831 static bool si_compile_tgsi_main(struct si_shader_context *ctx,
6832 struct si_shader *shader)
6833 {
6834 struct si_shader_selector *sel = shader->selector;
6835 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
6836
6837 switch (ctx->type) {
6838 case PIPE_SHADER_VERTEX:
6839 ctx->load_input = declare_input_vs;
6840 if (shader->key.as_ls)
6841 bld_base->emit_epilogue = si_llvm_emit_ls_epilogue;
6842 else if (shader->key.as_es)
6843 bld_base->emit_epilogue = si_llvm_emit_es_epilogue;
6844 else
6845 bld_base->emit_epilogue = si_llvm_emit_vs_epilogue;
6846 break;
6847 case PIPE_SHADER_TESS_CTRL:
6848 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_tcs;
6849 bld_base->emit_fetch_funcs[TGSI_FILE_OUTPUT] = fetch_output_tcs;
6850 bld_base->emit_store = store_output_tcs;
6851 bld_base->emit_epilogue = si_llvm_emit_tcs_epilogue;
6852 break;
6853 case PIPE_SHADER_TESS_EVAL:
6854 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_tes;
6855 if (shader->key.as_es)
6856 bld_base->emit_epilogue = si_llvm_emit_es_epilogue;
6857 else
6858 bld_base->emit_epilogue = si_llvm_emit_vs_epilogue;
6859 break;
6860 case PIPE_SHADER_GEOMETRY:
6861 bld_base->emit_fetch_funcs[TGSI_FILE_INPUT] = fetch_input_gs;
6862 bld_base->emit_epilogue = si_llvm_emit_gs_epilogue;
6863 break;
6864 case PIPE_SHADER_FRAGMENT:
6865 ctx->load_input = declare_input_fs;
6866 bld_base->emit_epilogue = si_llvm_return_fs_outputs;
6867 break;
6868 case PIPE_SHADER_COMPUTE:
6869 ctx->declare_memory_region = declare_compute_memory;
6870 break;
6871 default:
6872 assert(!"Unsupported shader type");
6873 return false;
6874 }
6875
6876 create_function(ctx);
6877 preload_ring_buffers(ctx);
6878
6879 if (ctx->type == PIPE_SHADER_GEOMETRY) {
6880 int i;
6881 for (i = 0; i < 4; i++) {
6882 ctx->gs_next_vertex[i] =
6883 lp_build_alloca(bld_base->base.gallivm,
6884 ctx->i32, "");
6885 }
6886 }
6887
6888 if (!lp_build_tgsi_llvm(bld_base, sel->tokens)) {
6889 fprintf(stderr, "Failed to translate shader from TGSI to LLVM\n");
6890 return false;
6891 }
6892
6893 si_llvm_build_ret(ctx, ctx->return_value);
6894 return true;
6895 }
6896
6897 /**
6898 * Compute the VS prolog key, which contains all the information needed to
6899 * build the VS prolog function, and set shader->info bits where needed.
6900 */
6901 static void si_get_vs_prolog_key(struct si_shader *shader,
6902 union si_shader_part_key *key)
6903 {
6904 struct tgsi_shader_info *info = &shader->selector->info;
6905
6906 memset(key, 0, sizeof(*key));
6907 key->vs_prolog.states = shader->key.part.vs.prolog;
6908 key->vs_prolog.num_input_sgprs = shader->info.num_input_sgprs;
6909 key->vs_prolog.last_input = MAX2(1, info->num_inputs) - 1;
6910
6911 /* Set the instanceID flag. */
6912 for (unsigned i = 0; i < info->num_inputs; i++)
6913 if (key->vs_prolog.states.instance_divisors[i])
6914 shader->info.uses_instanceid = true;
6915 }
6916
6917 /**
6918 * Compute the VS epilog key, which contains all the information needed to
6919 * build the VS epilog function, and set the PrimitiveID output offset.
6920 */
6921 static void si_get_vs_epilog_key(struct si_shader *shader,
6922 struct si_vs_epilog_bits *states,
6923 union si_shader_part_key *key)
6924 {
6925 memset(key, 0, sizeof(*key));
6926 key->vs_epilog.states = *states;
6927
6928 /* Set up the PrimitiveID output. */
6929 if (shader->key.part.vs.epilog.export_prim_id) {
6930 unsigned index = shader->selector->info.num_outputs;
6931 unsigned offset = shader->info.nr_param_exports++;
6932
6933 key->vs_epilog.prim_id_param_offset = offset;
6934 assert(index < ARRAY_SIZE(shader->info.vs_output_param_offset));
6935 shader->info.vs_output_param_offset[index] = offset;
6936 }
6937 }
6938
6939 /**
6940 * Compute the PS prolog key, which contains all the information needed to
6941 * build the PS prolog function, and set related bits in shader->config.
6942 */
6943 static void si_get_ps_prolog_key(struct si_shader *shader,
6944 union si_shader_part_key *key,
6945 bool separate_prolog)
6946 {
6947 struct tgsi_shader_info *info = &shader->selector->info;
6948
6949 memset(key, 0, sizeof(*key));
6950 key->ps_prolog.states = shader->key.part.ps.prolog;
6951 key->ps_prolog.colors_read = info->colors_read;
6952 key->ps_prolog.num_input_sgprs = shader->info.num_input_sgprs;
6953 key->ps_prolog.num_input_vgprs = shader->info.num_input_vgprs;
6954 key->ps_prolog.wqm = info->uses_derivatives &&
6955 (key->ps_prolog.colors_read ||
6956 key->ps_prolog.states.force_persp_sample_interp ||
6957 key->ps_prolog.states.force_linear_sample_interp ||
6958 key->ps_prolog.states.force_persp_center_interp ||
6959 key->ps_prolog.states.force_linear_center_interp ||
6960 key->ps_prolog.states.bc_optimize_for_persp ||
6961 key->ps_prolog.states.bc_optimize_for_linear);
6962
6963 if (info->colors_read) {
6964 unsigned *color = shader->selector->color_attr_index;
6965
6966 if (shader->key.part.ps.prolog.color_two_side) {
6967 /* BCOLORs are stored after the last input. */
6968 key->ps_prolog.num_interp_inputs = info->num_inputs;
6969 key->ps_prolog.face_vgpr_index = shader->info.face_vgpr_index;
6970 shader->config.spi_ps_input_ena |= S_0286CC_FRONT_FACE_ENA(1);
6971 }
6972
6973 for (unsigned i = 0; i < 2; i++) {
6974 unsigned interp = info->input_interpolate[color[i]];
6975 unsigned location = info->input_interpolate_loc[color[i]];
6976
6977 if (!(info->colors_read & (0xf << i*4)))
6978 continue;
6979
6980 key->ps_prolog.color_attr_index[i] = color[i];
6981
6982 if (shader->key.part.ps.prolog.flatshade_colors &&
6983 interp == TGSI_INTERPOLATE_COLOR)
6984 interp = TGSI_INTERPOLATE_CONSTANT;
6985
6986 switch (interp) {
6987 case TGSI_INTERPOLATE_CONSTANT:
6988 key->ps_prolog.color_interp_vgpr_index[i] = -1;
6989 break;
6990 case TGSI_INTERPOLATE_PERSPECTIVE:
6991 case TGSI_INTERPOLATE_COLOR:
6992 /* Force the interpolation location for colors here. */
6993 if (shader->key.part.ps.prolog.force_persp_sample_interp)
6994 location = TGSI_INTERPOLATE_LOC_SAMPLE;
6995 if (shader->key.part.ps.prolog.force_persp_center_interp)
6996 location = TGSI_INTERPOLATE_LOC_CENTER;
6997
6998 switch (location) {
6999 case TGSI_INTERPOLATE_LOC_SAMPLE:
7000 key->ps_prolog.color_interp_vgpr_index[i] = 0;
7001 shader->config.spi_ps_input_ena |=
7002 S_0286CC_PERSP_SAMPLE_ENA(1);
7003 break;
7004 case TGSI_INTERPOLATE_LOC_CENTER:
7005 key->ps_prolog.color_interp_vgpr_index[i] = 2;
7006 shader->config.spi_ps_input_ena |=
7007 S_0286CC_PERSP_CENTER_ENA(1);
7008 break;
7009 case TGSI_INTERPOLATE_LOC_CENTROID:
7010 key->ps_prolog.color_interp_vgpr_index[i] = 4;
7011 shader->config.spi_ps_input_ena |=
7012 S_0286CC_PERSP_CENTROID_ENA(1);
7013 break;
7014 default:
7015 assert(0);
7016 }
7017 break;
7018 case TGSI_INTERPOLATE_LINEAR:
7019 /* Force the interpolation location for colors here. */
7020 if (shader->key.part.ps.prolog.force_linear_sample_interp)
7021 location = TGSI_INTERPOLATE_LOC_SAMPLE;
7022 if (shader->key.part.ps.prolog.force_linear_center_interp)
7023 location = TGSI_INTERPOLATE_LOC_CENTER;
7024
7025 /* The VGPR assignment for non-monolithic shaders
7026 * works because InitialPSInputAddr is set on the
7027 * main shader and PERSP_PULL_MODEL is never used.
7028 */
7029 switch (location) {
7030 case TGSI_INTERPOLATE_LOC_SAMPLE:
7031 key->ps_prolog.color_interp_vgpr_index[i] =
7032 separate_prolog ? 6 : 9;
7033 shader->config.spi_ps_input_ena |=
7034 S_0286CC_LINEAR_SAMPLE_ENA(1);
7035 break;
7036 case TGSI_INTERPOLATE_LOC_CENTER:
7037 key->ps_prolog.color_interp_vgpr_index[i] =
7038 separate_prolog ? 8 : 11;
7039 shader->config.spi_ps_input_ena |=
7040 S_0286CC_LINEAR_CENTER_ENA(1);
7041 break;
7042 case TGSI_INTERPOLATE_LOC_CENTROID:
7043 key->ps_prolog.color_interp_vgpr_index[i] =
7044 separate_prolog ? 10 : 13;
7045 shader->config.spi_ps_input_ena |=
7046 S_0286CC_LINEAR_CENTROID_ENA(1);
7047 break;
7048 default:
7049 assert(0);
7050 }
7051 break;
7052 default:
7053 assert(0);
7054 }
7055 }
7056 }
7057 }
7058
7059 /**
7060 * Check whether a PS prolog is required based on the key.
7061 */
7062 static bool si_need_ps_prolog(const union si_shader_part_key *key)
7063 {
7064 return key->ps_prolog.colors_read ||
7065 key->ps_prolog.states.force_persp_sample_interp ||
7066 key->ps_prolog.states.force_linear_sample_interp ||
7067 key->ps_prolog.states.force_persp_center_interp ||
7068 key->ps_prolog.states.force_linear_center_interp ||
7069 key->ps_prolog.states.bc_optimize_for_persp ||
7070 key->ps_prolog.states.bc_optimize_for_linear ||
7071 key->ps_prolog.states.poly_stipple;
7072 }
7073
7074 /**
7075 * Compute the PS epilog key, which contains all the information needed to
7076 * build the PS epilog function.
7077 */
7078 static void si_get_ps_epilog_key(struct si_shader *shader,
7079 union si_shader_part_key *key)
7080 {
7081 struct tgsi_shader_info *info = &shader->selector->info;
7082 memset(key, 0, sizeof(*key));
7083 key->ps_epilog.colors_written = info->colors_written;
7084 key->ps_epilog.writes_z = info->writes_z;
7085 key->ps_epilog.writes_stencil = info->writes_stencil;
7086 key->ps_epilog.writes_samplemask = info->writes_samplemask;
7087 key->ps_epilog.states = shader->key.part.ps.epilog;
7088 }
7089
7090 /**
7091 * Build the GS prolog function. Rotate the input vertices for triangle strips
7092 * with adjacency.
7093 */
7094 static void si_build_gs_prolog_function(struct si_shader_context *ctx,
7095 union si_shader_part_key *key)
7096 {
7097 const unsigned num_sgprs = SI_GS_NUM_USER_SGPR + 2;
7098 const unsigned num_vgprs = 8;
7099 struct gallivm_state *gallivm = &ctx->gallivm;
7100 LLVMBuilderRef builder = gallivm->builder;
7101 LLVMTypeRef params[32];
7102 LLVMTypeRef returns[32];
7103 LLVMValueRef func, ret;
7104
7105 for (unsigned i = 0; i < num_sgprs; ++i) {
7106 params[i] = ctx->i32;
7107 returns[i] = ctx->i32;
7108 }
7109
7110 for (unsigned i = 0; i < num_vgprs; ++i) {
7111 params[num_sgprs + i] = ctx->i32;
7112 returns[num_sgprs + i] = ctx->f32;
7113 }
7114
7115 /* Create the function. */
7116 si_create_function(ctx, "gs_prolog", returns, num_sgprs + num_vgprs,
7117 params, num_sgprs + num_vgprs, num_sgprs - 1);
7118 func = ctx->main_fn;
7119
7120 /* Copy inputs to outputs. This should be no-op, as the registers match,
7121 * but it will prevent the compiler from overwriting them unintentionally.
7122 */
7123 ret = ctx->return_value;
7124 for (unsigned i = 0; i < num_sgprs; i++) {
7125 LLVMValueRef p = LLVMGetParam(func, i);
7126 ret = LLVMBuildInsertValue(builder, ret, p, i, "");
7127 }
7128 for (unsigned i = 0; i < num_vgprs; i++) {
7129 LLVMValueRef p = LLVMGetParam(func, num_sgprs + i);
7130 p = LLVMBuildBitCast(builder, p, ctx->f32, "");
7131 ret = LLVMBuildInsertValue(builder, ret, p, num_sgprs + i, "");
7132 }
7133
7134 if (key->gs_prolog.states.tri_strip_adj_fix) {
7135 /* Remap the input vertices for every other primitive. */
7136 const unsigned vtx_params[6] = {
7137 num_sgprs,
7138 num_sgprs + 1,
7139 num_sgprs + 3,
7140 num_sgprs + 4,
7141 num_sgprs + 5,
7142 num_sgprs + 6
7143 };
7144 LLVMValueRef prim_id, rotate;
7145
7146 prim_id = LLVMGetParam(func, num_sgprs + 2);
7147 rotate = LLVMBuildTrunc(builder, prim_id, ctx->i1, "");
7148
7149 for (unsigned i = 0; i < 6; ++i) {
7150 LLVMValueRef base, rotated, actual;
7151 base = LLVMGetParam(func, vtx_params[i]);
7152 rotated = LLVMGetParam(func, vtx_params[(i + 4) % 6]);
7153 actual = LLVMBuildSelect(builder, rotate, rotated, base, "");
7154 actual = LLVMBuildBitCast(builder, actual, ctx->f32, "");
7155 ret = LLVMBuildInsertValue(builder, ret, actual, vtx_params[i], "");
7156 }
7157 }
7158
7159 LLVMBuildRet(builder, ret);
7160 }
7161
7162 /**
7163 * Given a list of shader part functions, build a wrapper function that
7164 * runs them in sequence to form a monolithic shader.
7165 */
7166 static void si_build_wrapper_function(struct si_shader_context *ctx,
7167 LLVMValueRef *parts,
7168 unsigned num_parts,
7169 unsigned main_part)
7170 {
7171 struct gallivm_state *gallivm = &ctx->gallivm;
7172 LLVMBuilderRef builder = ctx->gallivm.builder;
7173 /* PS epilog has one arg per color component */
7174 LLVMTypeRef param_types[48];
7175 LLVMValueRef out[48];
7176 LLVMTypeRef function_type;
7177 unsigned num_params;
7178 unsigned num_out;
7179 MAYBE_UNUSED unsigned num_out_sgpr; /* used in debug checks */
7180 unsigned num_sgprs, num_vgprs;
7181 unsigned last_sgpr_param;
7182 unsigned gprs;
7183
7184 for (unsigned i = 0; i < num_parts; ++i) {
7185 lp_add_function_attr(parts[i], -1, LP_FUNC_ATTR_ALWAYSINLINE);
7186 LLVMSetLinkage(parts[i], LLVMPrivateLinkage);
7187 }
7188
7189 /* The parameters of the wrapper function correspond to those of the
7190 * first part in terms of SGPRs and VGPRs, but we use the types of the
7191 * main part to get the right types. This is relevant for the
7192 * dereferenceable attribute on descriptor table pointers.
7193 */
7194 num_sgprs = 0;
7195 num_vgprs = 0;
7196
7197 function_type = LLVMGetElementType(LLVMTypeOf(parts[0]));
7198 num_params = LLVMCountParamTypes(function_type);
7199
7200 for (unsigned i = 0; i < num_params; ++i) {
7201 LLVMValueRef param = LLVMGetParam(parts[0], i);
7202
7203 if (ac_is_sgpr_param(param)) {
7204 assert(num_vgprs == 0);
7205 num_sgprs += llvm_get_type_size(LLVMTypeOf(param)) / 4;
7206 } else {
7207 num_vgprs += llvm_get_type_size(LLVMTypeOf(param)) / 4;
7208 }
7209 }
7210 assert(num_vgprs + num_sgprs <= ARRAY_SIZE(param_types));
7211
7212 num_params = 0;
7213 last_sgpr_param = 0;
7214 gprs = 0;
7215 while (gprs < num_sgprs + num_vgprs) {
7216 LLVMValueRef param = LLVMGetParam(parts[main_part], num_params);
7217 unsigned size;
7218
7219 param_types[num_params] = LLVMTypeOf(param);
7220 if (gprs < num_sgprs)
7221 last_sgpr_param = num_params;
7222 size = llvm_get_type_size(param_types[num_params]) / 4;
7223 num_params++;
7224
7225 assert(ac_is_sgpr_param(param) == (gprs < num_sgprs));
7226 assert(gprs + size <= num_sgprs + num_vgprs &&
7227 (gprs >= num_sgprs || gprs + size <= num_sgprs));
7228
7229 gprs += size;
7230 }
7231
7232 si_create_function(ctx, "wrapper", NULL, 0, param_types, num_params, last_sgpr_param);
7233
7234 /* Record the arguments of the function as if they were an output of
7235 * a previous part.
7236 */
7237 num_out = 0;
7238 num_out_sgpr = 0;
7239
7240 for (unsigned i = 0; i < num_params; ++i) {
7241 LLVMValueRef param = LLVMGetParam(ctx->main_fn, i);
7242 LLVMTypeRef param_type = LLVMTypeOf(param);
7243 LLVMTypeRef out_type = i <= last_sgpr_param ? ctx->i32 : ctx->f32;
7244 unsigned size = llvm_get_type_size(param_type) / 4;
7245
7246 if (size == 1) {
7247 if (param_type != out_type)
7248 param = LLVMBuildBitCast(builder, param, out_type, "");
7249 out[num_out++] = param;
7250 } else {
7251 LLVMTypeRef vector_type = LLVMVectorType(out_type, size);
7252
7253 if (LLVMGetTypeKind(param_type) == LLVMPointerTypeKind) {
7254 param = LLVMBuildPtrToInt(builder, param, ctx->i64, "");
7255 param_type = ctx->i64;
7256 }
7257
7258 if (param_type != vector_type)
7259 param = LLVMBuildBitCast(builder, param, vector_type, "");
7260
7261 for (unsigned j = 0; j < size; ++j)
7262 out[num_out++] = LLVMBuildExtractElement(
7263 builder, param, LLVMConstInt(ctx->i32, j, 0), "");
7264 }
7265
7266 if (i <= last_sgpr_param)
7267 num_out_sgpr = num_out;
7268 }
7269
7270 /* Now chain the parts. */
7271 for (unsigned part = 0; part < num_parts; ++part) {
7272 LLVMValueRef in[48];
7273 LLVMValueRef ret;
7274 LLVMTypeRef ret_type;
7275 unsigned out_idx = 0;
7276
7277 num_params = LLVMCountParams(parts[part]);
7278 assert(num_params <= ARRAY_SIZE(param_types));
7279
7280 /* Derive arguments for the next part from outputs of the
7281 * previous one.
7282 */
7283 for (unsigned param_idx = 0; param_idx < num_params; ++param_idx) {
7284 LLVMValueRef param;
7285 LLVMTypeRef param_type;
7286 bool is_sgpr;
7287 unsigned param_size;
7288 LLVMValueRef arg = NULL;
7289
7290 param = LLVMGetParam(parts[part], param_idx);
7291 param_type = LLVMTypeOf(param);
7292 param_size = llvm_get_type_size(param_type) / 4;
7293 is_sgpr = ac_is_sgpr_param(param);
7294
7295 if (is_sgpr) {
7296 #if HAVE_LLVM < 0x0400
7297 LLVMRemoveAttribute(param, LLVMByValAttribute);
7298 #else
7299 unsigned kind_id = LLVMGetEnumAttributeKindForName("byval", 5);
7300 LLVMRemoveEnumAttributeAtIndex(parts[part], param_idx + 1, kind_id);
7301 #endif
7302 lp_add_function_attr(parts[part], param_idx + 1, LP_FUNC_ATTR_INREG);
7303 }
7304
7305 assert(out_idx + param_size <= (is_sgpr ? num_out_sgpr : num_out));
7306 assert(is_sgpr || out_idx >= num_out_sgpr);
7307
7308 if (param_size == 1)
7309 arg = out[out_idx];
7310 else
7311 arg = lp_build_gather_values(gallivm, &out[out_idx], param_size);
7312
7313 if (LLVMTypeOf(arg) != param_type) {
7314 if (LLVMGetTypeKind(param_type) == LLVMPointerTypeKind) {
7315 arg = LLVMBuildBitCast(builder, arg, ctx->i64, "");
7316 arg = LLVMBuildIntToPtr(builder, arg, param_type, "");
7317 } else {
7318 arg = LLVMBuildBitCast(builder, arg, param_type, "");
7319 }
7320 }
7321
7322 in[param_idx] = arg;
7323 out_idx += param_size;
7324 }
7325
7326 ret = LLVMBuildCall(builder, parts[part], in, num_params, "");
7327 ret_type = LLVMTypeOf(ret);
7328
7329 /* Extract the returned GPRs. */
7330 num_out = 0;
7331 num_out_sgpr = 0;
7332
7333 if (LLVMGetTypeKind(ret_type) != LLVMVoidTypeKind) {
7334 assert(LLVMGetTypeKind(ret_type) == LLVMStructTypeKind);
7335
7336 unsigned ret_size = LLVMCountStructElementTypes(ret_type);
7337
7338 for (unsigned i = 0; i < ret_size; ++i) {
7339 LLVMValueRef val =
7340 LLVMBuildExtractValue(builder, ret, i, "");
7341
7342 out[num_out++] = val;
7343
7344 if (LLVMTypeOf(val) == ctx->i32) {
7345 assert(num_out_sgpr + 1 == num_out);
7346 num_out_sgpr = num_out;
7347 }
7348 }
7349 }
7350 }
7351
7352 LLVMBuildRetVoid(builder);
7353 }
7354
7355 int si_compile_tgsi_shader(struct si_screen *sscreen,
7356 LLVMTargetMachineRef tm,
7357 struct si_shader *shader,
7358 bool is_monolithic,
7359 struct pipe_debug_callback *debug)
7360 {
7361 struct si_shader_selector *sel = shader->selector;
7362 struct si_shader_context ctx;
7363 struct lp_build_tgsi_context *bld_base;
7364 LLVMModuleRef mod;
7365 int r = -1;
7366
7367 /* Dump TGSI code before doing TGSI->LLVM conversion in case the
7368 * conversion fails. */
7369 if (r600_can_dump_shader(&sscreen->b, sel->info.processor) &&
7370 !(sscreen->b.debug_flags & DBG_NO_TGSI)) {
7371 tgsi_dump(sel->tokens, 0);
7372 si_dump_streamout(&sel->so);
7373 }
7374
7375 si_init_shader_ctx(&ctx, sscreen, shader, tm);
7376 ctx.separate_prolog = !is_monolithic;
7377
7378 memset(shader->info.vs_output_param_offset, EXP_PARAM_UNDEFINED,
7379 sizeof(shader->info.vs_output_param_offset));
7380
7381 shader->info.uses_instanceid = sel->info.uses_instanceid;
7382
7383 bld_base = &ctx.bld_base;
7384 ctx.load_system_value = declare_system_value;
7385
7386 if (!si_compile_tgsi_main(&ctx, shader)) {
7387 si_llvm_dispose(&ctx);
7388 return -1;
7389 }
7390
7391 if (is_monolithic && ctx.type == PIPE_SHADER_VERTEX) {
7392 LLVMValueRef parts[3];
7393 bool need_prolog;
7394 bool need_epilog;
7395
7396 need_prolog = sel->info.num_inputs;
7397 need_epilog = !shader->key.as_es && !shader->key.as_ls;
7398
7399 parts[need_prolog ? 1 : 0] = ctx.main_fn;
7400
7401 if (need_prolog) {
7402 union si_shader_part_key prolog_key;
7403 si_get_vs_prolog_key(shader, &prolog_key);
7404 si_build_vs_prolog_function(&ctx, &prolog_key);
7405 parts[0] = ctx.main_fn;
7406 }
7407
7408 if (need_epilog) {
7409 union si_shader_part_key epilog_key;
7410 si_get_vs_epilog_key(shader, &shader->key.part.vs.epilog, &epilog_key);
7411 si_build_vs_epilog_function(&ctx, &epilog_key);
7412 parts[need_prolog ? 2 : 1] = ctx.main_fn;
7413 }
7414
7415 si_build_wrapper_function(&ctx, parts, 1 + need_prolog + need_epilog,
7416 need_prolog ? 1 : 0);
7417 } else if (is_monolithic && ctx.type == PIPE_SHADER_TESS_CTRL) {
7418 LLVMValueRef parts[2];
7419 union si_shader_part_key epilog_key;
7420
7421 parts[0] = ctx.main_fn;
7422
7423 memset(&epilog_key, 0, sizeof(epilog_key));
7424 epilog_key.tcs_epilog.states = shader->key.part.tcs.epilog;
7425 si_build_tcs_epilog_function(&ctx, &epilog_key);
7426 parts[1] = ctx.main_fn;
7427
7428 si_build_wrapper_function(&ctx, parts, 2, 0);
7429 } else if (is_monolithic && ctx.type == PIPE_SHADER_TESS_EVAL &&
7430 !shader->key.as_es) {
7431 LLVMValueRef parts[2];
7432 union si_shader_part_key epilog_key;
7433
7434 parts[0] = ctx.main_fn;
7435
7436 si_get_vs_epilog_key(shader, &shader->key.part.tes.epilog, &epilog_key);
7437 si_build_vs_epilog_function(&ctx, &epilog_key);
7438 parts[1] = ctx.main_fn;
7439
7440 si_build_wrapper_function(&ctx, parts, 2, 0);
7441 } else if (is_monolithic && ctx.type == PIPE_SHADER_GEOMETRY) {
7442 LLVMValueRef parts[2];
7443 union si_shader_part_key prolog_key;
7444
7445 parts[1] = ctx.main_fn;
7446
7447 memset(&prolog_key, 0, sizeof(prolog_key));
7448 prolog_key.gs_prolog.states = shader->key.part.gs.prolog;
7449 si_build_gs_prolog_function(&ctx, &prolog_key);
7450 parts[0] = ctx.main_fn;
7451
7452 si_build_wrapper_function(&ctx, parts, 2, 1);
7453 } else if (is_monolithic && ctx.type == PIPE_SHADER_FRAGMENT) {
7454 LLVMValueRef parts[3];
7455 union si_shader_part_key prolog_key;
7456 union si_shader_part_key epilog_key;
7457 bool need_prolog;
7458
7459 si_get_ps_prolog_key(shader, &prolog_key, false);
7460 need_prolog = si_need_ps_prolog(&prolog_key);
7461
7462 parts[need_prolog ? 1 : 0] = ctx.main_fn;
7463
7464 if (need_prolog) {
7465 si_build_ps_prolog_function(&ctx, &prolog_key);
7466 parts[0] = ctx.main_fn;
7467 }
7468
7469 si_get_ps_epilog_key(shader, &epilog_key);
7470 si_build_ps_epilog_function(&ctx, &epilog_key);
7471 parts[need_prolog ? 2 : 1] = ctx.main_fn;
7472
7473 si_build_wrapper_function(&ctx, parts, need_prolog ? 3 : 2, need_prolog ? 1 : 0);
7474 }
7475
7476 mod = bld_base->base.gallivm->module;
7477
7478 /* Dump LLVM IR before any optimization passes */
7479 if (sscreen->b.debug_flags & DBG_PREOPT_IR &&
7480 r600_can_dump_shader(&sscreen->b, ctx.type))
7481 ac_dump_module(mod);
7482
7483 si_llvm_finalize_module(&ctx,
7484 r600_extra_shader_checks(&sscreen->b, ctx.type));
7485
7486 /* Post-optimization transformations and analysis. */
7487 si_eliminate_const_vs_outputs(&ctx);
7488
7489 if ((debug && debug->debug_message) ||
7490 r600_can_dump_shader(&sscreen->b, ctx.type))
7491 si_count_scratch_private_memory(&ctx);
7492
7493 /* Compile to bytecode. */
7494 r = si_compile_llvm(sscreen, &shader->binary, &shader->config, tm,
7495 mod, debug, ctx.type, "TGSI shader");
7496 si_llvm_dispose(&ctx);
7497 if (r) {
7498 fprintf(stderr, "LLVM failed to compile shader\n");
7499 return r;
7500 }
7501
7502 /* Validate SGPR and VGPR usage for compute to detect compiler bugs.
7503 * LLVM 3.9svn has this bug.
7504 */
7505 if (sel->type == PIPE_SHADER_COMPUTE) {
7506 unsigned wave_size = 64;
7507 unsigned max_vgprs = 256;
7508 unsigned max_sgprs = sscreen->b.chip_class >= VI ? 800 : 512;
7509 unsigned max_sgprs_per_wave = 128;
7510 unsigned max_block_threads = si_get_max_workgroup_size(shader);
7511 unsigned min_waves_per_cu = DIV_ROUND_UP(max_block_threads, wave_size);
7512 unsigned min_waves_per_simd = DIV_ROUND_UP(min_waves_per_cu, 4);
7513
7514 max_vgprs = max_vgprs / min_waves_per_simd;
7515 max_sgprs = MIN2(max_sgprs / min_waves_per_simd, max_sgprs_per_wave);
7516
7517 if (shader->config.num_sgprs > max_sgprs ||
7518 shader->config.num_vgprs > max_vgprs) {
7519 fprintf(stderr, "LLVM failed to compile a shader correctly: "
7520 "SGPR:VGPR usage is %u:%u, but the hw limit is %u:%u\n",
7521 shader->config.num_sgprs, shader->config.num_vgprs,
7522 max_sgprs, max_vgprs);
7523
7524 /* Just terminate the process, because dependent
7525 * shaders can hang due to bad input data, but use
7526 * the env var to allow shader-db to work.
7527 */
7528 if (!debug_get_bool_option("SI_PASS_BAD_SHADERS", false))
7529 abort();
7530 }
7531 }
7532
7533 /* Add the scratch offset to input SGPRs. */
7534 if (shader->config.scratch_bytes_per_wave)
7535 shader->info.num_input_sgprs += 1; /* scratch byte offset */
7536
7537 /* Calculate the number of fragment input VGPRs. */
7538 if (ctx.type == PIPE_SHADER_FRAGMENT) {
7539 shader->info.num_input_vgprs = 0;
7540 shader->info.face_vgpr_index = -1;
7541
7542 if (G_0286CC_PERSP_SAMPLE_ENA(shader->config.spi_ps_input_addr))
7543 shader->info.num_input_vgprs += 2;
7544 if (G_0286CC_PERSP_CENTER_ENA(shader->config.spi_ps_input_addr))
7545 shader->info.num_input_vgprs += 2;
7546 if (G_0286CC_PERSP_CENTROID_ENA(shader->config.spi_ps_input_addr))
7547 shader->info.num_input_vgprs += 2;
7548 if (G_0286CC_PERSP_PULL_MODEL_ENA(shader->config.spi_ps_input_addr))
7549 shader->info.num_input_vgprs += 3;
7550 if (G_0286CC_LINEAR_SAMPLE_ENA(shader->config.spi_ps_input_addr))
7551 shader->info.num_input_vgprs += 2;
7552 if (G_0286CC_LINEAR_CENTER_ENA(shader->config.spi_ps_input_addr))
7553 shader->info.num_input_vgprs += 2;
7554 if (G_0286CC_LINEAR_CENTROID_ENA(shader->config.spi_ps_input_addr))
7555 shader->info.num_input_vgprs += 2;
7556 if (G_0286CC_LINE_STIPPLE_TEX_ENA(shader->config.spi_ps_input_addr))
7557 shader->info.num_input_vgprs += 1;
7558 if (G_0286CC_POS_X_FLOAT_ENA(shader->config.spi_ps_input_addr))
7559 shader->info.num_input_vgprs += 1;
7560 if (G_0286CC_POS_Y_FLOAT_ENA(shader->config.spi_ps_input_addr))
7561 shader->info.num_input_vgprs += 1;
7562 if (G_0286CC_POS_Z_FLOAT_ENA(shader->config.spi_ps_input_addr))
7563 shader->info.num_input_vgprs += 1;
7564 if (G_0286CC_POS_W_FLOAT_ENA(shader->config.spi_ps_input_addr))
7565 shader->info.num_input_vgprs += 1;
7566 if (G_0286CC_FRONT_FACE_ENA(shader->config.spi_ps_input_addr)) {
7567 shader->info.face_vgpr_index = shader->info.num_input_vgprs;
7568 shader->info.num_input_vgprs += 1;
7569 }
7570 if (G_0286CC_ANCILLARY_ENA(shader->config.spi_ps_input_addr))
7571 shader->info.num_input_vgprs += 1;
7572 if (G_0286CC_SAMPLE_COVERAGE_ENA(shader->config.spi_ps_input_addr))
7573 shader->info.num_input_vgprs += 1;
7574 if (G_0286CC_POS_FIXED_PT_ENA(shader->config.spi_ps_input_addr))
7575 shader->info.num_input_vgprs += 1;
7576 }
7577
7578 return 0;
7579 }
7580
7581 /**
7582 * Create, compile and return a shader part (prolog or epilog).
7583 *
7584 * \param sscreen screen
7585 * \param list list of shader parts of the same category
7586 * \param type shader type
7587 * \param key shader part key
7588 * \param prolog whether the part being requested is a prolog
7589 * \param tm LLVM target machine
7590 * \param debug debug callback
7591 * \param build the callback responsible for building the main function
7592 * \return non-NULL on success
7593 */
7594 static struct si_shader_part *
7595 si_get_shader_part(struct si_screen *sscreen,
7596 struct si_shader_part **list,
7597 enum pipe_shader_type type,
7598 bool prolog,
7599 union si_shader_part_key *key,
7600 LLVMTargetMachineRef tm,
7601 struct pipe_debug_callback *debug,
7602 void (*build)(struct si_shader_context *,
7603 union si_shader_part_key *),
7604 const char *name)
7605 {
7606 struct si_shader_part *result;
7607
7608 mtx_lock(&sscreen->shader_parts_mutex);
7609
7610 /* Find existing. */
7611 for (result = *list; result; result = result->next) {
7612 if (memcmp(&result->key, key, sizeof(*key)) == 0) {
7613 mtx_unlock(&sscreen->shader_parts_mutex);
7614 return result;
7615 }
7616 }
7617
7618 /* Compile a new one. */
7619 result = CALLOC_STRUCT(si_shader_part);
7620 result->key = *key;
7621
7622 struct si_shader shader = {};
7623 struct si_shader_context ctx;
7624 struct gallivm_state *gallivm = &ctx.gallivm;
7625
7626 si_init_shader_ctx(&ctx, sscreen, &shader, tm);
7627 ctx.type = type;
7628
7629 switch (type) {
7630 case PIPE_SHADER_VERTEX:
7631 break;
7632 case PIPE_SHADER_TESS_CTRL:
7633 assert(!prolog);
7634 shader.key.part.tcs.epilog = key->tcs_epilog.states;
7635 break;
7636 case PIPE_SHADER_GEOMETRY:
7637 assert(prolog);
7638 break;
7639 case PIPE_SHADER_FRAGMENT:
7640 if (prolog)
7641 shader.key.part.ps.prolog = key->ps_prolog.states;
7642 else
7643 shader.key.part.ps.epilog = key->ps_epilog.states;
7644 break;
7645 default:
7646 unreachable("bad shader part");
7647 }
7648
7649 build(&ctx, key);
7650
7651 /* Compile. */
7652 si_llvm_finalize_module(&ctx,
7653 r600_extra_shader_checks(&sscreen->b, PIPE_SHADER_FRAGMENT));
7654
7655 if (si_compile_llvm(sscreen, &result->binary, &result->config, tm,
7656 gallivm->module, debug, ctx.type, name)) {
7657 FREE(result);
7658 result = NULL;
7659 goto out;
7660 }
7661
7662 result->next = *list;
7663 *list = result;
7664
7665 out:
7666 si_llvm_dispose(&ctx);
7667 mtx_unlock(&sscreen->shader_parts_mutex);
7668 return result;
7669 }
7670
7671 /**
7672 * Build the vertex shader prolog function.
7673 *
7674 * The inputs are the same as VS (a lot of SGPRs and 4 VGPR system values).
7675 * All inputs are returned unmodified. The vertex load indices are
7676 * stored after them, which will be used by the API VS for fetching inputs.
7677 *
7678 * For example, the expected outputs for instance_divisors[] = {0, 1, 2} are:
7679 * input_v0,
7680 * input_v1,
7681 * input_v2,
7682 * input_v3,
7683 * (VertexID + BaseVertex),
7684 * (InstanceID + StartInstance),
7685 * (InstanceID / 2 + StartInstance)
7686 */
7687 static void si_build_vs_prolog_function(struct si_shader_context *ctx,
7688 union si_shader_part_key *key)
7689 {
7690 struct gallivm_state *gallivm = &ctx->gallivm;
7691 LLVMTypeRef *params, *returns;
7692 LLVMValueRef ret, func;
7693 int last_sgpr, num_params, num_returns, i;
7694
7695 ctx->param_vertex_id = key->vs_prolog.num_input_sgprs;
7696 ctx->param_instance_id = key->vs_prolog.num_input_sgprs + 3;
7697
7698 /* 4 preloaded VGPRs + vertex load indices as prolog outputs */
7699 params = alloca((key->vs_prolog.num_input_sgprs + 4) *
7700 sizeof(LLVMTypeRef));
7701 returns = alloca((key->vs_prolog.num_input_sgprs + 4 +
7702 key->vs_prolog.last_input + 1) *
7703 sizeof(LLVMTypeRef));
7704 num_params = 0;
7705 num_returns = 0;
7706
7707 /* Declare input and output SGPRs. */
7708 num_params = 0;
7709 for (i = 0; i < key->vs_prolog.num_input_sgprs; i++) {
7710 params[num_params++] = ctx->i32;
7711 returns[num_returns++] = ctx->i32;
7712 }
7713 last_sgpr = num_params - 1;
7714
7715 /* 4 preloaded VGPRs (outputs must be floats) */
7716 for (i = 0; i < 4; i++) {
7717 params[num_params++] = ctx->i32;
7718 returns[num_returns++] = ctx->f32;
7719 }
7720
7721 /* Vertex load indices. */
7722 for (i = 0; i <= key->vs_prolog.last_input; i++)
7723 returns[num_returns++] = ctx->f32;
7724
7725 /* Create the function. */
7726 si_create_function(ctx, "vs_prolog", returns, num_returns, params,
7727 num_params, last_sgpr);
7728 func = ctx->main_fn;
7729
7730 /* Copy inputs to outputs. This should be no-op, as the registers match,
7731 * but it will prevent the compiler from overwriting them unintentionally.
7732 */
7733 ret = ctx->return_value;
7734 for (i = 0; i < key->vs_prolog.num_input_sgprs; i++) {
7735 LLVMValueRef p = LLVMGetParam(func, i);
7736 ret = LLVMBuildInsertValue(gallivm->builder, ret, p, i, "");
7737 }
7738 for (i = num_params - 4; i < num_params; i++) {
7739 LLVMValueRef p = LLVMGetParam(func, i);
7740 p = LLVMBuildBitCast(gallivm->builder, p, ctx->f32, "");
7741 ret = LLVMBuildInsertValue(gallivm->builder, ret, p, i, "");
7742 }
7743
7744 /* Compute vertex load indices from instance divisors. */
7745 for (i = 0; i <= key->vs_prolog.last_input; i++) {
7746 unsigned divisor = key->vs_prolog.states.instance_divisors[i];
7747 LLVMValueRef index;
7748
7749 if (divisor) {
7750 /* InstanceID / Divisor + StartInstance */
7751 index = get_instance_index_for_fetch(ctx,
7752 SI_SGPR_START_INSTANCE,
7753 divisor);
7754 } else {
7755 /* VertexID + BaseVertex */
7756 index = LLVMBuildAdd(gallivm->builder,
7757 LLVMGetParam(func, ctx->param_vertex_id),
7758 LLVMGetParam(func, SI_SGPR_BASE_VERTEX), "");
7759 }
7760
7761 index = LLVMBuildBitCast(gallivm->builder, index, ctx->f32, "");
7762 ret = LLVMBuildInsertValue(gallivm->builder, ret, index,
7763 num_params++, "");
7764 }
7765
7766 si_llvm_build_ret(ctx, ret);
7767 }
7768
7769 /**
7770 * Build the vertex shader epilog function. This is also used by the tessellation
7771 * evaluation shader compiled as VS.
7772 *
7773 * The input is PrimitiveID.
7774 *
7775 * If PrimitiveID is required by the pixel shader, export it.
7776 * Otherwise, do nothing.
7777 */
7778 static void si_build_vs_epilog_function(struct si_shader_context *ctx,
7779 union si_shader_part_key *key)
7780 {
7781 struct gallivm_state *gallivm = &ctx->gallivm;
7782 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
7783 LLVMTypeRef params[5];
7784 int num_params, i;
7785
7786 /* Declare input VGPRs. */
7787 num_params = key->vs_epilog.states.export_prim_id ?
7788 (VS_EPILOG_PRIMID_LOC + 1) : 0;
7789 assert(num_params <= ARRAY_SIZE(params));
7790
7791 for (i = 0; i < num_params; i++)
7792 params[i] = ctx->f32;
7793
7794 /* Create the function. */
7795 si_create_function(ctx, "vs_epilog", NULL, 0, params, num_params, -1);
7796
7797 /* Emit exports. */
7798 if (key->vs_epilog.states.export_prim_id) {
7799 struct lp_build_context *base = &bld_base->base;
7800 struct ac_export_args args;
7801
7802 args.enabled_channels = 0x1; /* enabled channels */
7803 args.valid_mask = 0; /* whether the EXEC mask is valid */
7804 args.done = 0; /* DONE bit */
7805 args.target = V_008DFC_SQ_EXP_PARAM +
7806 key->vs_epilog.prim_id_param_offset;
7807 args.compr = 0; /* COMPR flag (0 = 32-bit export) */
7808 args.out[0] = LLVMGetParam(ctx->main_fn,
7809 VS_EPILOG_PRIMID_LOC); /* X */
7810 args.out[1] = base->undef; /* Y */
7811 args.out[2] = base->undef; /* Z */
7812 args.out[3] = base->undef; /* W */
7813
7814 ac_build_export(&ctx->ac, &args);
7815 }
7816
7817 LLVMBuildRetVoid(gallivm->builder);
7818 }
7819
7820 /**
7821 * Create & compile a vertex shader epilog. This a helper used by VS and TES.
7822 */
7823 static bool si_get_vs_epilog(struct si_screen *sscreen,
7824 LLVMTargetMachineRef tm,
7825 struct si_shader *shader,
7826 struct pipe_debug_callback *debug,
7827 struct si_vs_epilog_bits *states)
7828 {
7829 union si_shader_part_key epilog_key;
7830
7831 si_get_vs_epilog_key(shader, states, &epilog_key);
7832
7833 shader->epilog = si_get_shader_part(sscreen, &sscreen->vs_epilogs,
7834 PIPE_SHADER_VERTEX, true,
7835 &epilog_key, tm, debug,
7836 si_build_vs_epilog_function,
7837 "Vertex Shader Epilog");
7838 return shader->epilog != NULL;
7839 }
7840
7841 /**
7842 * Select and compile (or reuse) vertex shader parts (prolog & epilog).
7843 */
7844 static bool si_shader_select_vs_parts(struct si_screen *sscreen,
7845 LLVMTargetMachineRef tm,
7846 struct si_shader *shader,
7847 struct pipe_debug_callback *debug)
7848 {
7849 struct tgsi_shader_info *info = &shader->selector->info;
7850 union si_shader_part_key prolog_key;
7851
7852 /* Get the prolog. */
7853 si_get_vs_prolog_key(shader, &prolog_key);
7854
7855 /* The prolog is a no-op if there are no inputs. */
7856 if (info->num_inputs) {
7857 shader->prolog =
7858 si_get_shader_part(sscreen, &sscreen->vs_prologs,
7859 PIPE_SHADER_VERTEX, true,
7860 &prolog_key, tm, debug,
7861 si_build_vs_prolog_function,
7862 "Vertex Shader Prolog");
7863 if (!shader->prolog)
7864 return false;
7865 }
7866
7867 /* Get the epilog. */
7868 if (!shader->key.as_es && !shader->key.as_ls &&
7869 !si_get_vs_epilog(sscreen, tm, shader, debug,
7870 &shader->key.part.vs.epilog))
7871 return false;
7872
7873 return true;
7874 }
7875
7876 /**
7877 * Select and compile (or reuse) TES parts (epilog).
7878 */
7879 static bool si_shader_select_tes_parts(struct si_screen *sscreen,
7880 LLVMTargetMachineRef tm,
7881 struct si_shader *shader,
7882 struct pipe_debug_callback *debug)
7883 {
7884 if (shader->key.as_es)
7885 return true;
7886
7887 /* TES compiled as VS. */
7888 return si_get_vs_epilog(sscreen, tm, shader, debug,
7889 &shader->key.part.tes.epilog);
7890 }
7891
7892 /**
7893 * Compile the TCS epilog function. This writes tesselation factors to memory
7894 * based on the output primitive type of the tesselator (determined by TES).
7895 */
7896 static void si_build_tcs_epilog_function(struct si_shader_context *ctx,
7897 union si_shader_part_key *key)
7898 {
7899 struct gallivm_state *gallivm = &ctx->gallivm;
7900 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
7901 LLVMTypeRef params[16];
7902 LLVMValueRef func;
7903 int last_sgpr, num_params;
7904
7905 /* Declare inputs. Only RW_BUFFERS and TESS_FACTOR_OFFSET are used. */
7906 params[SI_PARAM_RW_BUFFERS] = const_array(ctx->v16i8, SI_NUM_RW_BUFFERS);
7907 params[SI_PARAM_CONST_BUFFERS] = ctx->i64;
7908 params[SI_PARAM_SAMPLERS] = ctx->i64;
7909 params[SI_PARAM_IMAGES] = ctx->i64;
7910 params[SI_PARAM_SHADER_BUFFERS] = ctx->i64;
7911 params[SI_PARAM_TCS_OFFCHIP_LAYOUT] = ctx->i32;
7912 params[SI_PARAM_TCS_OUT_OFFSETS] = ctx->i32;
7913 params[SI_PARAM_TCS_OUT_LAYOUT] = ctx->i32;
7914 params[SI_PARAM_TCS_IN_LAYOUT] = ctx->i32;
7915 params[ctx->param_oc_lds = SI_PARAM_TCS_OC_LDS] = ctx->i32;
7916 params[SI_PARAM_TESS_FACTOR_OFFSET] = ctx->i32;
7917 last_sgpr = SI_PARAM_TESS_FACTOR_OFFSET;
7918 num_params = last_sgpr + 1;
7919
7920 params[num_params++] = ctx->i32; /* patch index within the wave (REL_PATCH_ID) */
7921 params[num_params++] = ctx->i32; /* invocation ID within the patch */
7922 params[num_params++] = ctx->i32; /* LDS offset where tess factors should be loaded from */
7923
7924 /* Create the function. */
7925 si_create_function(ctx, "tcs_epilog", NULL, 0, params, num_params, last_sgpr);
7926 declare_tess_lds(ctx);
7927 func = ctx->main_fn;
7928
7929 si_write_tess_factors(bld_base,
7930 LLVMGetParam(func, last_sgpr + 1),
7931 LLVMGetParam(func, last_sgpr + 2),
7932 LLVMGetParam(func, last_sgpr + 3));
7933
7934 LLVMBuildRetVoid(gallivm->builder);
7935 }
7936
7937 /**
7938 * Select and compile (or reuse) TCS parts (epilog).
7939 */
7940 static bool si_shader_select_tcs_parts(struct si_screen *sscreen,
7941 LLVMTargetMachineRef tm,
7942 struct si_shader *shader,
7943 struct pipe_debug_callback *debug)
7944 {
7945 union si_shader_part_key epilog_key;
7946
7947 /* Get the epilog. */
7948 memset(&epilog_key, 0, sizeof(epilog_key));
7949 epilog_key.tcs_epilog.states = shader->key.part.tcs.epilog;
7950
7951 shader->epilog = si_get_shader_part(sscreen, &sscreen->tcs_epilogs,
7952 PIPE_SHADER_TESS_CTRL, false,
7953 &epilog_key, tm, debug,
7954 si_build_tcs_epilog_function,
7955 "Tessellation Control Shader Epilog");
7956 return shader->epilog != NULL;
7957 }
7958
7959 /**
7960 * Select and compile (or reuse) GS parts (prolog).
7961 */
7962 static bool si_shader_select_gs_parts(struct si_screen *sscreen,
7963 LLVMTargetMachineRef tm,
7964 struct si_shader *shader,
7965 struct pipe_debug_callback *debug)
7966 {
7967 union si_shader_part_key prolog_key;
7968
7969 if (!shader->key.part.gs.prolog.tri_strip_adj_fix)
7970 return true;
7971
7972 memset(&prolog_key, 0, sizeof(prolog_key));
7973 prolog_key.gs_prolog.states = shader->key.part.gs.prolog;
7974
7975 shader->prolog = si_get_shader_part(sscreen, &sscreen->gs_prologs,
7976 PIPE_SHADER_GEOMETRY, true,
7977 &prolog_key, tm, debug,
7978 si_build_gs_prolog_function,
7979 "Geometry Shader Prolog");
7980 return shader->prolog != NULL;
7981 }
7982
7983 /**
7984 * Build the pixel shader prolog function. This handles:
7985 * - two-side color selection and interpolation
7986 * - overriding interpolation parameters for the API PS
7987 * - polygon stippling
7988 *
7989 * All preloaded SGPRs and VGPRs are passed through unmodified unless they are
7990 * overriden by other states. (e.g. per-sample interpolation)
7991 * Interpolated colors are stored after the preloaded VGPRs.
7992 */
7993 static void si_build_ps_prolog_function(struct si_shader_context *ctx,
7994 union si_shader_part_key *key)
7995 {
7996 struct gallivm_state *gallivm = &ctx->gallivm;
7997 LLVMTypeRef *params;
7998 LLVMValueRef ret, func;
7999 int last_sgpr, num_params, num_returns, i, num_color_channels;
8000
8001 assert(si_need_ps_prolog(key));
8002
8003 /* Number of inputs + 8 color elements. */
8004 params = alloca((key->ps_prolog.num_input_sgprs +
8005 key->ps_prolog.num_input_vgprs + 8) *
8006 sizeof(LLVMTypeRef));
8007
8008 /* Declare inputs. */
8009 num_params = 0;
8010 for (i = 0; i < key->ps_prolog.num_input_sgprs; i++)
8011 params[num_params++] = ctx->i32;
8012 last_sgpr = num_params - 1;
8013
8014 for (i = 0; i < key->ps_prolog.num_input_vgprs; i++)
8015 params[num_params++] = ctx->f32;
8016
8017 /* Declare outputs (same as inputs + add colors if needed) */
8018 num_returns = num_params;
8019 num_color_channels = util_bitcount(key->ps_prolog.colors_read);
8020 for (i = 0; i < num_color_channels; i++)
8021 params[num_returns++] = ctx->f32;
8022
8023 /* Create the function. */
8024 si_create_function(ctx, "ps_prolog", params, num_returns, params,
8025 num_params, last_sgpr);
8026 func = ctx->main_fn;
8027
8028 /* Copy inputs to outputs. This should be no-op, as the registers match,
8029 * but it will prevent the compiler from overwriting them unintentionally.
8030 */
8031 ret = ctx->return_value;
8032 for (i = 0; i < num_params; i++) {
8033 LLVMValueRef p = LLVMGetParam(func, i);
8034 ret = LLVMBuildInsertValue(gallivm->builder, ret, p, i, "");
8035 }
8036
8037 /* Polygon stippling. */
8038 if (key->ps_prolog.states.poly_stipple) {
8039 /* POS_FIXED_PT is always last. */
8040 unsigned pos = key->ps_prolog.num_input_sgprs +
8041 key->ps_prolog.num_input_vgprs - 1;
8042 LLVMValueRef ptr[2], list;
8043
8044 /* Get the pointer to rw buffers. */
8045 ptr[0] = LLVMGetParam(func, SI_SGPR_RW_BUFFERS);
8046 ptr[1] = LLVMGetParam(func, SI_SGPR_RW_BUFFERS_HI);
8047 list = lp_build_gather_values(gallivm, ptr, 2);
8048 list = LLVMBuildBitCast(gallivm->builder, list, ctx->i64, "");
8049 list = LLVMBuildIntToPtr(gallivm->builder, list,
8050 const_array(ctx->v16i8, SI_NUM_RW_BUFFERS), "");
8051
8052 si_llvm_emit_polygon_stipple(ctx, list, pos);
8053 }
8054
8055 if (key->ps_prolog.states.bc_optimize_for_persp ||
8056 key->ps_prolog.states.bc_optimize_for_linear) {
8057 unsigned i, base = key->ps_prolog.num_input_sgprs;
8058 LLVMValueRef center[2], centroid[2], tmp, bc_optimize;
8059
8060 /* The shader should do: if (PRIM_MASK[31]) CENTROID = CENTER;
8061 * The hw doesn't compute CENTROID if the whole wave only
8062 * contains fully-covered quads.
8063 *
8064 * PRIM_MASK is after user SGPRs.
8065 */
8066 bc_optimize = LLVMGetParam(func, SI_PS_NUM_USER_SGPR);
8067 bc_optimize = LLVMBuildLShr(gallivm->builder, bc_optimize,
8068 LLVMConstInt(ctx->i32, 31, 0), "");
8069 bc_optimize = LLVMBuildTrunc(gallivm->builder, bc_optimize,
8070 ctx->i1, "");
8071
8072 if (key->ps_prolog.states.bc_optimize_for_persp) {
8073 /* Read PERSP_CENTER. */
8074 for (i = 0; i < 2; i++)
8075 center[i] = LLVMGetParam(func, base + 2 + i);
8076 /* Read PERSP_CENTROID. */
8077 for (i = 0; i < 2; i++)
8078 centroid[i] = LLVMGetParam(func, base + 4 + i);
8079 /* Select PERSP_CENTROID. */
8080 for (i = 0; i < 2; i++) {
8081 tmp = LLVMBuildSelect(gallivm->builder, bc_optimize,
8082 center[i], centroid[i], "");
8083 ret = LLVMBuildInsertValue(gallivm->builder, ret,
8084 tmp, base + 4 + i, "");
8085 }
8086 }
8087 if (key->ps_prolog.states.bc_optimize_for_linear) {
8088 /* Read LINEAR_CENTER. */
8089 for (i = 0; i < 2; i++)
8090 center[i] = LLVMGetParam(func, base + 8 + i);
8091 /* Read LINEAR_CENTROID. */
8092 for (i = 0; i < 2; i++)
8093 centroid[i] = LLVMGetParam(func, base + 10 + i);
8094 /* Select LINEAR_CENTROID. */
8095 for (i = 0; i < 2; i++) {
8096 tmp = LLVMBuildSelect(gallivm->builder, bc_optimize,
8097 center[i], centroid[i], "");
8098 ret = LLVMBuildInsertValue(gallivm->builder, ret,
8099 tmp, base + 10 + i, "");
8100 }
8101 }
8102 }
8103
8104 /* Force per-sample interpolation. */
8105 if (key->ps_prolog.states.force_persp_sample_interp) {
8106 unsigned i, base = key->ps_prolog.num_input_sgprs;
8107 LLVMValueRef persp_sample[2];
8108
8109 /* Read PERSP_SAMPLE. */
8110 for (i = 0; i < 2; i++)
8111 persp_sample[i] = LLVMGetParam(func, base + i);
8112 /* Overwrite PERSP_CENTER. */
8113 for (i = 0; i < 2; i++)
8114 ret = LLVMBuildInsertValue(gallivm->builder, ret,
8115 persp_sample[i], base + 2 + i, "");
8116 /* Overwrite PERSP_CENTROID. */
8117 for (i = 0; i < 2; i++)
8118 ret = LLVMBuildInsertValue(gallivm->builder, ret,
8119 persp_sample[i], base + 4 + i, "");
8120 }
8121 if (key->ps_prolog.states.force_linear_sample_interp) {
8122 unsigned i, base = key->ps_prolog.num_input_sgprs;
8123 LLVMValueRef linear_sample[2];
8124
8125 /* Read LINEAR_SAMPLE. */
8126 for (i = 0; i < 2; i++)
8127 linear_sample[i] = LLVMGetParam(func, base + 6 + i);
8128 /* Overwrite LINEAR_CENTER. */
8129 for (i = 0; i < 2; i++)
8130 ret = LLVMBuildInsertValue(gallivm->builder, ret,
8131 linear_sample[i], base + 8 + i, "");
8132 /* Overwrite LINEAR_CENTROID. */
8133 for (i = 0; i < 2; i++)
8134 ret = LLVMBuildInsertValue(gallivm->builder, ret,
8135 linear_sample[i], base + 10 + i, "");
8136 }
8137
8138 /* Force center interpolation. */
8139 if (key->ps_prolog.states.force_persp_center_interp) {
8140 unsigned i, base = key->ps_prolog.num_input_sgprs;
8141 LLVMValueRef persp_center[2];
8142
8143 /* Read PERSP_CENTER. */
8144 for (i = 0; i < 2; i++)
8145 persp_center[i] = LLVMGetParam(func, base + 2 + i);
8146 /* Overwrite PERSP_SAMPLE. */
8147 for (i = 0; i < 2; i++)
8148 ret = LLVMBuildInsertValue(gallivm->builder, ret,
8149 persp_center[i], base + i, "");
8150 /* Overwrite PERSP_CENTROID. */
8151 for (i = 0; i < 2; i++)
8152 ret = LLVMBuildInsertValue(gallivm->builder, ret,
8153 persp_center[i], base + 4 + i, "");
8154 }
8155 if (key->ps_prolog.states.force_linear_center_interp) {
8156 unsigned i, base = key->ps_prolog.num_input_sgprs;
8157 LLVMValueRef linear_center[2];
8158
8159 /* Read LINEAR_CENTER. */
8160 for (i = 0; i < 2; i++)
8161 linear_center[i] = LLVMGetParam(func, base + 8 + i);
8162 /* Overwrite LINEAR_SAMPLE. */
8163 for (i = 0; i < 2; i++)
8164 ret = LLVMBuildInsertValue(gallivm->builder, ret,
8165 linear_center[i], base + 6 + i, "");
8166 /* Overwrite LINEAR_CENTROID. */
8167 for (i = 0; i < 2; i++)
8168 ret = LLVMBuildInsertValue(gallivm->builder, ret,
8169 linear_center[i], base + 10 + i, "");
8170 }
8171
8172 /* Interpolate colors. */
8173 for (i = 0; i < 2; i++) {
8174 unsigned writemask = (key->ps_prolog.colors_read >> (i * 4)) & 0xf;
8175 unsigned face_vgpr = key->ps_prolog.num_input_sgprs +
8176 key->ps_prolog.face_vgpr_index;
8177 LLVMValueRef interp[2], color[4];
8178 LLVMValueRef interp_ij = NULL, prim_mask = NULL, face = NULL;
8179
8180 if (!writemask)
8181 continue;
8182
8183 /* If the interpolation qualifier is not CONSTANT (-1). */
8184 if (key->ps_prolog.color_interp_vgpr_index[i] != -1) {
8185 unsigned interp_vgpr = key->ps_prolog.num_input_sgprs +
8186 key->ps_prolog.color_interp_vgpr_index[i];
8187
8188 /* Get the (i,j) updated by bc_optimize handling. */
8189 interp[0] = LLVMBuildExtractValue(gallivm->builder, ret,
8190 interp_vgpr, "");
8191 interp[1] = LLVMBuildExtractValue(gallivm->builder, ret,
8192 interp_vgpr + 1, "");
8193 interp_ij = lp_build_gather_values(gallivm, interp, 2);
8194 }
8195
8196 /* Use the absolute location of the input. */
8197 prim_mask = LLVMGetParam(func, SI_PS_NUM_USER_SGPR);
8198
8199 if (key->ps_prolog.states.color_two_side) {
8200 face = LLVMGetParam(func, face_vgpr);
8201 face = LLVMBuildBitCast(gallivm->builder, face, ctx->i32, "");
8202 }
8203
8204 interp_fs_input(ctx,
8205 key->ps_prolog.color_attr_index[i],
8206 TGSI_SEMANTIC_COLOR, i,
8207 key->ps_prolog.num_interp_inputs,
8208 key->ps_prolog.colors_read, interp_ij,
8209 prim_mask, face, color);
8210
8211 while (writemask) {
8212 unsigned chan = u_bit_scan(&writemask);
8213 ret = LLVMBuildInsertValue(gallivm->builder, ret, color[chan],
8214 num_params++, "");
8215 }
8216 }
8217
8218 /* Tell LLVM to insert WQM instruction sequence when needed. */
8219 if (key->ps_prolog.wqm) {
8220 LLVMAddTargetDependentFunctionAttr(func,
8221 "amdgpu-ps-wqm-outputs", "");
8222 }
8223
8224 si_llvm_build_ret(ctx, ret);
8225 }
8226
8227 /**
8228 * Build the pixel shader epilog function. This handles everything that must be
8229 * emulated for pixel shader exports. (alpha-test, format conversions, etc)
8230 */
8231 static void si_build_ps_epilog_function(struct si_shader_context *ctx,
8232 union si_shader_part_key *key)
8233 {
8234 struct gallivm_state *gallivm = &ctx->gallivm;
8235 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
8236 LLVMTypeRef params[16+8*4+3];
8237 LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
8238 int last_sgpr, num_params, i;
8239 struct si_ps_exports exp = {};
8240
8241 /* Declare input SGPRs. */
8242 params[SI_PARAM_RW_BUFFERS] = ctx->i64;
8243 params[SI_PARAM_CONST_BUFFERS] = ctx->i64;
8244 params[SI_PARAM_SAMPLERS] = ctx->i64;
8245 params[SI_PARAM_IMAGES] = ctx->i64;
8246 params[SI_PARAM_SHADER_BUFFERS] = ctx->i64;
8247 params[SI_PARAM_ALPHA_REF] = ctx->f32;
8248 last_sgpr = SI_PARAM_ALPHA_REF;
8249
8250 /* Declare input VGPRs. */
8251 num_params = (last_sgpr + 1) +
8252 util_bitcount(key->ps_epilog.colors_written) * 4 +
8253 key->ps_epilog.writes_z +
8254 key->ps_epilog.writes_stencil +
8255 key->ps_epilog.writes_samplemask;
8256
8257 num_params = MAX2(num_params,
8258 last_sgpr + 1 + PS_EPILOG_SAMPLEMASK_MIN_LOC + 1);
8259
8260 assert(num_params <= ARRAY_SIZE(params));
8261
8262 for (i = last_sgpr + 1; i < num_params; i++)
8263 params[i] = ctx->f32;
8264
8265 /* Create the function. */
8266 si_create_function(ctx, "ps_epilog", NULL, 0, params, num_params, last_sgpr);
8267 /* Disable elimination of unused inputs. */
8268 si_llvm_add_attribute(ctx->main_fn,
8269 "InitialPSInputAddr", 0xffffff);
8270
8271 /* Process colors. */
8272 unsigned vgpr = last_sgpr + 1;
8273 unsigned colors_written = key->ps_epilog.colors_written;
8274 int last_color_export = -1;
8275
8276 /* Find the last color export. */
8277 if (!key->ps_epilog.writes_z &&
8278 !key->ps_epilog.writes_stencil &&
8279 !key->ps_epilog.writes_samplemask) {
8280 unsigned spi_format = key->ps_epilog.states.spi_shader_col_format;
8281
8282 /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
8283 if (colors_written == 0x1 && key->ps_epilog.states.last_cbuf > 0) {
8284 /* Just set this if any of the colorbuffers are enabled. */
8285 if (spi_format &
8286 ((1llu << (4 * (key->ps_epilog.states.last_cbuf + 1))) - 1))
8287 last_color_export = 0;
8288 } else {
8289 for (i = 0; i < 8; i++)
8290 if (colors_written & (1 << i) &&
8291 (spi_format >> (i * 4)) & 0xf)
8292 last_color_export = i;
8293 }
8294 }
8295
8296 while (colors_written) {
8297 LLVMValueRef color[4];
8298 int mrt = u_bit_scan(&colors_written);
8299
8300 for (i = 0; i < 4; i++)
8301 color[i] = LLVMGetParam(ctx->main_fn, vgpr++);
8302
8303 si_export_mrt_color(bld_base, color, mrt,
8304 num_params - 1,
8305 mrt == last_color_export, &exp);
8306 }
8307
8308 /* Process depth, stencil, samplemask. */
8309 if (key->ps_epilog.writes_z)
8310 depth = LLVMGetParam(ctx->main_fn, vgpr++);
8311 if (key->ps_epilog.writes_stencil)
8312 stencil = LLVMGetParam(ctx->main_fn, vgpr++);
8313 if (key->ps_epilog.writes_samplemask)
8314 samplemask = LLVMGetParam(ctx->main_fn, vgpr++);
8315
8316 if (depth || stencil || samplemask)
8317 si_export_mrt_z(bld_base, depth, stencil, samplemask, &exp);
8318 else if (last_color_export == -1)
8319 si_export_null(bld_base);
8320
8321 if (exp.num)
8322 si_emit_ps_exports(ctx, &exp);
8323
8324 /* Compile. */
8325 LLVMBuildRetVoid(gallivm->builder);
8326 }
8327
8328 /**
8329 * Select and compile (or reuse) pixel shader parts (prolog & epilog).
8330 */
8331 static bool si_shader_select_ps_parts(struct si_screen *sscreen,
8332 LLVMTargetMachineRef tm,
8333 struct si_shader *shader,
8334 struct pipe_debug_callback *debug)
8335 {
8336 union si_shader_part_key prolog_key;
8337 union si_shader_part_key epilog_key;
8338
8339 /* Get the prolog. */
8340 si_get_ps_prolog_key(shader, &prolog_key, true);
8341
8342 /* The prolog is a no-op if these aren't set. */
8343 if (si_need_ps_prolog(&prolog_key)) {
8344 shader->prolog =
8345 si_get_shader_part(sscreen, &sscreen->ps_prologs,
8346 PIPE_SHADER_FRAGMENT, true,
8347 &prolog_key, tm, debug,
8348 si_build_ps_prolog_function,
8349 "Fragment Shader Prolog");
8350 if (!shader->prolog)
8351 return false;
8352 }
8353
8354 /* Get the epilog. */
8355 si_get_ps_epilog_key(shader, &epilog_key);
8356
8357 shader->epilog =
8358 si_get_shader_part(sscreen, &sscreen->ps_epilogs,
8359 PIPE_SHADER_FRAGMENT, false,
8360 &epilog_key, tm, debug,
8361 si_build_ps_epilog_function,
8362 "Fragment Shader Epilog");
8363 if (!shader->epilog)
8364 return false;
8365
8366 /* Enable POS_FIXED_PT if polygon stippling is enabled. */
8367 if (shader->key.part.ps.prolog.poly_stipple) {
8368 shader->config.spi_ps_input_ena |= S_0286CC_POS_FIXED_PT_ENA(1);
8369 assert(G_0286CC_POS_FIXED_PT_ENA(shader->config.spi_ps_input_addr));
8370 }
8371
8372 /* Set up the enable bits for per-sample shading if needed. */
8373 if (shader->key.part.ps.prolog.force_persp_sample_interp &&
8374 (G_0286CC_PERSP_CENTER_ENA(shader->config.spi_ps_input_ena) ||
8375 G_0286CC_PERSP_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
8376 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_CENTER_ENA;
8377 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_CENTROID_ENA;
8378 shader->config.spi_ps_input_ena |= S_0286CC_PERSP_SAMPLE_ENA(1);
8379 }
8380 if (shader->key.part.ps.prolog.force_linear_sample_interp &&
8381 (G_0286CC_LINEAR_CENTER_ENA(shader->config.spi_ps_input_ena) ||
8382 G_0286CC_LINEAR_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
8383 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_CENTER_ENA;
8384 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_CENTROID_ENA;
8385 shader->config.spi_ps_input_ena |= S_0286CC_LINEAR_SAMPLE_ENA(1);
8386 }
8387 if (shader->key.part.ps.prolog.force_persp_center_interp &&
8388 (G_0286CC_PERSP_SAMPLE_ENA(shader->config.spi_ps_input_ena) ||
8389 G_0286CC_PERSP_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
8390 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_SAMPLE_ENA;
8391 shader->config.spi_ps_input_ena &= C_0286CC_PERSP_CENTROID_ENA;
8392 shader->config.spi_ps_input_ena |= S_0286CC_PERSP_CENTER_ENA(1);
8393 }
8394 if (shader->key.part.ps.prolog.force_linear_center_interp &&
8395 (G_0286CC_LINEAR_SAMPLE_ENA(shader->config.spi_ps_input_ena) ||
8396 G_0286CC_LINEAR_CENTROID_ENA(shader->config.spi_ps_input_ena))) {
8397 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_SAMPLE_ENA;
8398 shader->config.spi_ps_input_ena &= C_0286CC_LINEAR_CENTROID_ENA;
8399 shader->config.spi_ps_input_ena |= S_0286CC_LINEAR_CENTER_ENA(1);
8400 }
8401
8402 /* POW_W_FLOAT requires that one of the perspective weights is enabled. */
8403 if (G_0286CC_POS_W_FLOAT_ENA(shader->config.spi_ps_input_ena) &&
8404 !(shader->config.spi_ps_input_ena & 0xf)) {
8405 shader->config.spi_ps_input_ena |= S_0286CC_PERSP_CENTER_ENA(1);
8406 assert(G_0286CC_PERSP_CENTER_ENA(shader->config.spi_ps_input_addr));
8407 }
8408
8409 /* At least one pair of interpolation weights must be enabled. */
8410 if (!(shader->config.spi_ps_input_ena & 0x7f)) {
8411 shader->config.spi_ps_input_ena |= S_0286CC_LINEAR_CENTER_ENA(1);
8412 assert(G_0286CC_LINEAR_CENTER_ENA(shader->config.spi_ps_input_addr));
8413 }
8414
8415 /* The sample mask input is always enabled, because the API shader always
8416 * passes it through to the epilog. Disable it here if it's unused.
8417 */
8418 if (!shader->key.part.ps.epilog.poly_line_smoothing &&
8419 !shader->selector->info.reads_samplemask)
8420 shader->config.spi_ps_input_ena &= C_0286CC_SAMPLE_COVERAGE_ENA;
8421
8422 return true;
8423 }
8424
8425 void si_multiwave_lds_size_workaround(struct si_screen *sscreen,
8426 unsigned *lds_size)
8427 {
8428 /* SPI barrier management bug:
8429 * Make sure we have at least 4k of LDS in use to avoid the bug.
8430 * It applies to workgroup sizes of more than one wavefront.
8431 */
8432 if (sscreen->b.family == CHIP_BONAIRE ||
8433 sscreen->b.family == CHIP_KABINI ||
8434 sscreen->b.family == CHIP_MULLINS)
8435 *lds_size = MAX2(*lds_size, 8);
8436 }
8437
8438 static void si_fix_resource_usage(struct si_screen *sscreen,
8439 struct si_shader *shader)
8440 {
8441 unsigned min_sgprs = shader->info.num_input_sgprs + 2; /* VCC */
8442
8443 shader->config.num_sgprs = MAX2(shader->config.num_sgprs, min_sgprs);
8444
8445 if (shader->selector->type == PIPE_SHADER_COMPUTE &&
8446 si_get_max_workgroup_size(shader) > 64) {
8447 si_multiwave_lds_size_workaround(sscreen,
8448 &shader->config.lds_size);
8449 }
8450 }
8451
8452 int si_shader_create(struct si_screen *sscreen, LLVMTargetMachineRef tm,
8453 struct si_shader *shader,
8454 struct pipe_debug_callback *debug)
8455 {
8456 struct si_shader_selector *sel = shader->selector;
8457 struct si_shader *mainp = *si_get_main_shader_part(sel, &shader->key);
8458 int r;
8459
8460 /* LS, ES, VS are compiled on demand if the main part hasn't been
8461 * compiled for that stage.
8462 *
8463 * Vertex shaders are compiled on demand when a vertex fetch
8464 * workaround must be applied.
8465 */
8466 if (shader->is_monolithic) {
8467 /* Monolithic shader (compiled as a whole, has many variants,
8468 * may take a long time to compile).
8469 */
8470 r = si_compile_tgsi_shader(sscreen, tm, shader, true, debug);
8471 if (r)
8472 return r;
8473 } else {
8474 /* The shader consists of 2-3 parts:
8475 *
8476 * - the middle part is the user shader, it has 1 variant only
8477 * and it was compiled during the creation of the shader
8478 * selector
8479 * - the prolog part is inserted at the beginning
8480 * - the epilog part is inserted at the end
8481 *
8482 * The prolog and epilog have many (but simple) variants.
8483 */
8484
8485 /* Copy the compiled TGSI shader data over. */
8486 shader->is_binary_shared = true;
8487 shader->binary = mainp->binary;
8488 shader->config = mainp->config;
8489 shader->info.num_input_sgprs = mainp->info.num_input_sgprs;
8490 shader->info.num_input_vgprs = mainp->info.num_input_vgprs;
8491 shader->info.face_vgpr_index = mainp->info.face_vgpr_index;
8492 memcpy(shader->info.vs_output_param_offset,
8493 mainp->info.vs_output_param_offset,
8494 sizeof(mainp->info.vs_output_param_offset));
8495 shader->info.uses_instanceid = mainp->info.uses_instanceid;
8496 shader->info.nr_pos_exports = mainp->info.nr_pos_exports;
8497 shader->info.nr_param_exports = mainp->info.nr_param_exports;
8498
8499 /* Select prologs and/or epilogs. */
8500 switch (sel->type) {
8501 case PIPE_SHADER_VERTEX:
8502 if (!si_shader_select_vs_parts(sscreen, tm, shader, debug))
8503 return -1;
8504 break;
8505 case PIPE_SHADER_TESS_CTRL:
8506 if (!si_shader_select_tcs_parts(sscreen, tm, shader, debug))
8507 return -1;
8508 break;
8509 case PIPE_SHADER_TESS_EVAL:
8510 if (!si_shader_select_tes_parts(sscreen, tm, shader, debug))
8511 return -1;
8512 break;
8513 case PIPE_SHADER_GEOMETRY:
8514 if (!si_shader_select_gs_parts(sscreen, tm, shader, debug))
8515 return -1;
8516 break;
8517 case PIPE_SHADER_FRAGMENT:
8518 if (!si_shader_select_ps_parts(sscreen, tm, shader, debug))
8519 return -1;
8520
8521 /* Make sure we have at least as many VGPRs as there
8522 * are allocated inputs.
8523 */
8524 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
8525 shader->info.num_input_vgprs);
8526 break;
8527 }
8528
8529 /* Update SGPR and VGPR counts. */
8530 if (shader->prolog) {
8531 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
8532 shader->prolog->config.num_sgprs);
8533 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
8534 shader->prolog->config.num_vgprs);
8535 }
8536 if (shader->epilog) {
8537 shader->config.num_sgprs = MAX2(shader->config.num_sgprs,
8538 shader->epilog->config.num_sgprs);
8539 shader->config.num_vgprs = MAX2(shader->config.num_vgprs,
8540 shader->epilog->config.num_vgprs);
8541 }
8542 }
8543
8544 si_fix_resource_usage(sscreen, shader);
8545 si_shader_dump(sscreen, shader, debug, sel->info.processor,
8546 stderr, true);
8547
8548 /* Upload. */
8549 r = si_shader_binary_upload(sscreen, shader);
8550 if (r) {
8551 fprintf(stderr, "LLVM failed to upload shader\n");
8552 return r;
8553 }
8554
8555 return 0;
8556 }
8557
8558 void si_shader_destroy(struct si_shader *shader)
8559 {
8560 if (shader->scratch_bo)
8561 r600_resource_reference(&shader->scratch_bo, NULL);
8562
8563 r600_resource_reference(&shader->bo, NULL);
8564
8565 if (!shader->is_binary_shared)
8566 radeon_shader_binary_clean(&shader->binary);
8567
8568 free(shader->shader_log);
8569 }