88b082de45186cbc61b554cece96c945fca111f8
[mesa.git] / src / gallium / drivers / radeonsi / si_shader.c
1 /*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * on the rights to use, copy, modify, merge, publish, distribute, sub
9 * license, and/or sell copies of the Software, and to permit persons to whom
10 * the Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
23 */
24
25 #include <llvm/Config/llvm-config.h>
26
27 #include "util/u_memory.h"
28 #include "util/u_string.h"
29 #include "tgsi/tgsi_build.h"
30 #include "tgsi/tgsi_strings.h"
31 #include "tgsi/tgsi_util.h"
32 #include "tgsi/tgsi_dump.h"
33 #include "tgsi/tgsi_from_mesa.h"
34
35 #include "ac_binary.h"
36 #include "ac_exp_param.h"
37 #include "ac_shader_util.h"
38 #include "ac_rtld.h"
39 #include "ac_llvm_util.h"
40 #include "si_shader_internal.h"
41 #include "si_pipe.h"
42 #include "sid.h"
43
44 #include "compiler/nir/nir.h"
45 #include "compiler/nir/nir_serialize.h"
46
47 static const char scratch_rsrc_dword0_symbol[] =
48 "SCRATCH_RSRC_DWORD0";
49
50 static const char scratch_rsrc_dword1_symbol[] =
51 "SCRATCH_RSRC_DWORD1";
52
53 static void si_init_shader_ctx(struct si_shader_context *ctx,
54 struct si_screen *sscreen,
55 struct ac_llvm_compiler *compiler,
56 unsigned wave_size,
57 bool nir);
58
59 static void si_llvm_emit_barrier(const struct lp_build_tgsi_action *action,
60 struct lp_build_tgsi_context *bld_base,
61 struct lp_build_emit_data *emit_data);
62
63 static void si_dump_shader_key(const struct si_shader *shader, FILE *f);
64
65 static void si_build_vs_prolog_function(struct si_shader_context *ctx,
66 union si_shader_part_key *key);
67 static void si_build_tcs_epilog_function(struct si_shader_context *ctx,
68 union si_shader_part_key *key);
69 static void si_build_ps_prolog_function(struct si_shader_context *ctx,
70 union si_shader_part_key *key);
71 static void si_build_ps_epilog_function(struct si_shader_context *ctx,
72 union si_shader_part_key *key);
73 static void si_fix_resource_usage(struct si_screen *sscreen,
74 struct si_shader *shader);
75
76 /* Ideally pass the sample mask input to the PS epilog as v14, which
77 * is its usual location, so that the shader doesn't have to add v_mov.
78 */
79 #define PS_EPILOG_SAMPLEMASK_MIN_LOC 14
80
81 static bool llvm_type_is_64bit(struct si_shader_context *ctx,
82 LLVMTypeRef type)
83 {
84 if (type == ctx->ac.i64 || type == ctx->ac.f64)
85 return true;
86
87 return false;
88 }
89
90 /** Whether the shader runs as a combination of multiple API shaders */
91 static bool is_multi_part_shader(struct si_shader_context *ctx)
92 {
93 if (ctx->screen->info.chip_class <= GFX8)
94 return false;
95
96 return ctx->shader->key.as_ls ||
97 ctx->shader->key.as_es ||
98 ctx->type == PIPE_SHADER_TESS_CTRL ||
99 ctx->type == PIPE_SHADER_GEOMETRY;
100 }
101
102 /** Whether the shader runs on a merged HW stage (LSHS or ESGS) */
103 static bool is_merged_shader(struct si_shader_context *ctx)
104 {
105 return ctx->shader->key.as_ngg || is_multi_part_shader(ctx);
106 }
107
108 /**
109 * Returns a unique index for a per-patch semantic name and index. The index
110 * must be less than 32, so that a 32-bit bitmask of used inputs or outputs
111 * can be calculated.
112 */
113 unsigned si_shader_io_get_unique_index_patch(unsigned semantic_name, unsigned index)
114 {
115 switch (semantic_name) {
116 case TGSI_SEMANTIC_TESSOUTER:
117 return 0;
118 case TGSI_SEMANTIC_TESSINNER:
119 return 1;
120 case TGSI_SEMANTIC_PATCH:
121 assert(index < 30);
122 return 2 + index;
123
124 default:
125 assert(!"invalid semantic name");
126 return 0;
127 }
128 }
129
130 /**
131 * Returns a unique index for a semantic name and index. The index must be
132 * less than 64, so that a 64-bit bitmask of used inputs or outputs can be
133 * calculated.
134 */
135 unsigned si_shader_io_get_unique_index(unsigned semantic_name, unsigned index,
136 unsigned is_varying)
137 {
138 switch (semantic_name) {
139 case TGSI_SEMANTIC_POSITION:
140 return 0;
141 case TGSI_SEMANTIC_GENERIC:
142 /* Since some shader stages use the the highest used IO index
143 * to determine the size to allocate for inputs/outputs
144 * (in LDS, tess and GS rings). GENERIC should be placed right
145 * after POSITION to make that size as small as possible.
146 */
147 if (index < SI_MAX_IO_GENERIC)
148 return 1 + index;
149
150 assert(!"invalid generic index");
151 return 0;
152 case TGSI_SEMANTIC_FOG:
153 return SI_MAX_IO_GENERIC + 1;
154 case TGSI_SEMANTIC_COLOR:
155 assert(index < 2);
156 return SI_MAX_IO_GENERIC + 2 + index;
157 case TGSI_SEMANTIC_BCOLOR:
158 assert(index < 2);
159 /* If it's a varying, COLOR and BCOLOR alias. */
160 if (is_varying)
161 return SI_MAX_IO_GENERIC + 2 + index;
162 else
163 return SI_MAX_IO_GENERIC + 4 + index;
164 case TGSI_SEMANTIC_TEXCOORD:
165 assert(index < 8);
166 return SI_MAX_IO_GENERIC + 6 + index;
167
168 /* These are rarely used between LS and HS or ES and GS. */
169 case TGSI_SEMANTIC_CLIPDIST:
170 assert(index < 2);
171 return SI_MAX_IO_GENERIC + 6 + 8 + index;
172 case TGSI_SEMANTIC_CLIPVERTEX:
173 return SI_MAX_IO_GENERIC + 6 + 8 + 2;
174 case TGSI_SEMANTIC_PSIZE:
175 return SI_MAX_IO_GENERIC + 6 + 8 + 3;
176
177 /* These can't be written by LS, HS, and ES. */
178 case TGSI_SEMANTIC_LAYER:
179 return SI_MAX_IO_GENERIC + 6 + 8 + 4;
180 case TGSI_SEMANTIC_VIEWPORT_INDEX:
181 return SI_MAX_IO_GENERIC + 6 + 8 + 5;
182 case TGSI_SEMANTIC_PRIMID:
183 STATIC_ASSERT(SI_MAX_IO_GENERIC + 6 + 8 + 6 <= 63);
184 return SI_MAX_IO_GENERIC + 6 + 8 + 6;
185 default:
186 fprintf(stderr, "invalid semantic name = %u\n", semantic_name);
187 assert(!"invalid semantic name");
188 return 0;
189 }
190 }
191
192 /**
193 * Get the value of a shader input parameter and extract a bitfield.
194 */
195 static LLVMValueRef unpack_llvm_param(struct si_shader_context *ctx,
196 LLVMValueRef value, unsigned rshift,
197 unsigned bitwidth)
198 {
199 if (LLVMGetTypeKind(LLVMTypeOf(value)) == LLVMFloatTypeKind)
200 value = ac_to_integer(&ctx->ac, value);
201
202 if (rshift)
203 value = LLVMBuildLShr(ctx->ac.builder, value,
204 LLVMConstInt(ctx->i32, rshift, 0), "");
205
206 if (rshift + bitwidth < 32) {
207 unsigned mask = (1 << bitwidth) - 1;
208 value = LLVMBuildAnd(ctx->ac.builder, value,
209 LLVMConstInt(ctx->i32, mask, 0), "");
210 }
211
212 return value;
213 }
214
215 LLVMValueRef si_unpack_param(struct si_shader_context *ctx,
216 struct ac_arg param, unsigned rshift,
217 unsigned bitwidth)
218 {
219 LLVMValueRef value = ac_get_arg(&ctx->ac, param);
220
221 return unpack_llvm_param(ctx, value, rshift, bitwidth);
222 }
223
224 static LLVMValueRef get_rel_patch_id(struct si_shader_context *ctx)
225 {
226 switch (ctx->type) {
227 case PIPE_SHADER_TESS_CTRL:
228 return si_unpack_param(ctx, ctx->args.tcs_rel_ids, 0, 8);
229
230 case PIPE_SHADER_TESS_EVAL:
231 return ac_get_arg(&ctx->ac, ctx->tes_rel_patch_id);
232
233 default:
234 assert(0);
235 return NULL;
236 }
237 }
238
239 /* Tessellation shaders pass outputs to the next shader using LDS.
240 *
241 * LS outputs = TCS inputs
242 * TCS outputs = TES inputs
243 *
244 * The LDS layout is:
245 * - TCS inputs for patch 0
246 * - TCS inputs for patch 1
247 * - TCS inputs for patch 2 = get_tcs_in_current_patch_offset (if RelPatchID==2)
248 * - ...
249 * - TCS outputs for patch 0 = get_tcs_out_patch0_offset
250 * - Per-patch TCS outputs for patch 0 = get_tcs_out_patch0_patch_data_offset
251 * - TCS outputs for patch 1
252 * - Per-patch TCS outputs for patch 1
253 * - TCS outputs for patch 2 = get_tcs_out_current_patch_offset (if RelPatchID==2)
254 * - Per-patch TCS outputs for patch 2 = get_tcs_out_current_patch_data_offset (if RelPatchID==2)
255 * - ...
256 *
257 * All three shaders VS(LS), TCS, TES share the same LDS space.
258 */
259
260 static LLVMValueRef
261 get_tcs_in_patch_stride(struct si_shader_context *ctx)
262 {
263 return si_unpack_param(ctx, ctx->vs_state_bits, 8, 13);
264 }
265
266 static unsigned get_tcs_out_vertex_dw_stride_constant(struct si_shader_context *ctx)
267 {
268 assert(ctx->type == PIPE_SHADER_TESS_CTRL);
269
270 if (ctx->shader->key.mono.u.ff_tcs_inputs_to_copy)
271 return util_last_bit64(ctx->shader->key.mono.u.ff_tcs_inputs_to_copy) * 4;
272
273 return util_last_bit64(ctx->shader->selector->outputs_written) * 4;
274 }
275
276 static LLVMValueRef get_tcs_out_vertex_dw_stride(struct si_shader_context *ctx)
277 {
278 unsigned stride = get_tcs_out_vertex_dw_stride_constant(ctx);
279
280 return LLVMConstInt(ctx->i32, stride, 0);
281 }
282
283 static LLVMValueRef get_tcs_out_patch_stride(struct si_shader_context *ctx)
284 {
285 if (ctx->shader->key.mono.u.ff_tcs_inputs_to_copy)
286 return si_unpack_param(ctx, ctx->tcs_out_lds_layout, 0, 13);
287
288 const struct tgsi_shader_info *info = &ctx->shader->selector->info;
289 unsigned tcs_out_vertices = info->properties[TGSI_PROPERTY_TCS_VERTICES_OUT];
290 unsigned vertex_dw_stride = get_tcs_out_vertex_dw_stride_constant(ctx);
291 unsigned num_patch_outputs = util_last_bit64(ctx->shader->selector->patch_outputs_written);
292 unsigned patch_dw_stride = tcs_out_vertices * vertex_dw_stride +
293 num_patch_outputs * 4;
294 return LLVMConstInt(ctx->i32, patch_dw_stride, 0);
295 }
296
297 static LLVMValueRef
298 get_tcs_out_patch0_offset(struct si_shader_context *ctx)
299 {
300 return LLVMBuildMul(ctx->ac.builder,
301 si_unpack_param(ctx, ctx->tcs_out_lds_offsets, 0, 16),
302 LLVMConstInt(ctx->i32, 4, 0), "");
303 }
304
305 static LLVMValueRef
306 get_tcs_out_patch0_patch_data_offset(struct si_shader_context *ctx)
307 {
308 return LLVMBuildMul(ctx->ac.builder,
309 si_unpack_param(ctx, ctx->tcs_out_lds_offsets, 16, 16),
310 LLVMConstInt(ctx->i32, 4, 0), "");
311 }
312
313 static LLVMValueRef
314 get_tcs_in_current_patch_offset(struct si_shader_context *ctx)
315 {
316 LLVMValueRef patch_stride = get_tcs_in_patch_stride(ctx);
317 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
318
319 return LLVMBuildMul(ctx->ac.builder, patch_stride, rel_patch_id, "");
320 }
321
322 static LLVMValueRef
323 get_tcs_out_current_patch_offset(struct si_shader_context *ctx)
324 {
325 LLVMValueRef patch0_offset = get_tcs_out_patch0_offset(ctx);
326 LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
327 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
328
329 return ac_build_imad(&ctx->ac, patch_stride, rel_patch_id, patch0_offset);
330 }
331
332 static LLVMValueRef
333 get_tcs_out_current_patch_data_offset(struct si_shader_context *ctx)
334 {
335 LLVMValueRef patch0_patch_data_offset =
336 get_tcs_out_patch0_patch_data_offset(ctx);
337 LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
338 LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
339
340 return ac_build_imad(&ctx->ac, patch_stride, rel_patch_id, patch0_patch_data_offset);
341 }
342
343 static LLVMValueRef get_num_tcs_out_vertices(struct si_shader_context *ctx)
344 {
345 unsigned tcs_out_vertices =
346 ctx->shader->selector ?
347 ctx->shader->selector->info.properties[TGSI_PROPERTY_TCS_VERTICES_OUT] : 0;
348
349 /* If !tcs_out_vertices, it's either the fixed-func TCS or the TCS epilog. */
350 if (ctx->type == PIPE_SHADER_TESS_CTRL && tcs_out_vertices)
351 return LLVMConstInt(ctx->i32, tcs_out_vertices, 0);
352
353 return si_unpack_param(ctx, ctx->tcs_offchip_layout, 6, 6);
354 }
355
356 static LLVMValueRef get_tcs_in_vertex_dw_stride(struct si_shader_context *ctx)
357 {
358 unsigned stride;
359
360 switch (ctx->type) {
361 case PIPE_SHADER_VERTEX:
362 stride = ctx->shader->selector->lshs_vertex_stride / 4;
363 return LLVMConstInt(ctx->i32, stride, 0);
364
365 case PIPE_SHADER_TESS_CTRL:
366 if (ctx->screen->info.chip_class >= GFX9 &&
367 ctx->shader->is_monolithic) {
368 stride = ctx->shader->key.part.tcs.ls->lshs_vertex_stride / 4;
369 return LLVMConstInt(ctx->i32, stride, 0);
370 }
371 return si_unpack_param(ctx, ctx->vs_state_bits, 24, 8);
372
373 default:
374 assert(0);
375 return NULL;
376 }
377 }
378
379 static LLVMValueRef unpack_sint16(struct si_shader_context *ctx,
380 LLVMValueRef i32, unsigned index)
381 {
382 assert(index <= 1);
383
384 if (index == 1)
385 return LLVMBuildAShr(ctx->ac.builder, i32,
386 LLVMConstInt(ctx->i32, 16, 0), "");
387
388 return LLVMBuildSExt(ctx->ac.builder,
389 LLVMBuildTrunc(ctx->ac.builder, i32,
390 ctx->ac.i16, ""),
391 ctx->i32, "");
392 }
393
394 void si_llvm_load_input_vs(
395 struct si_shader_context *ctx,
396 unsigned input_index,
397 LLVMValueRef out[4])
398 {
399 const struct tgsi_shader_info *info = &ctx->shader->selector->info;
400 unsigned vs_blit_property = info->properties[TGSI_PROPERTY_VS_BLIT_SGPRS_AMD];
401
402 if (vs_blit_property) {
403 LLVMValueRef vertex_id = ctx->abi.vertex_id;
404 LLVMValueRef sel_x1 = LLVMBuildICmp(ctx->ac.builder,
405 LLVMIntULE, vertex_id,
406 ctx->i32_1, "");
407 /* Use LLVMIntNE, because we have 3 vertices and only
408 * the middle one should use y2.
409 */
410 LLVMValueRef sel_y1 = LLVMBuildICmp(ctx->ac.builder,
411 LLVMIntNE, vertex_id,
412 ctx->i32_1, "");
413
414 unsigned param_vs_blit_inputs = ctx->vs_blit_inputs.arg_index;
415 if (input_index == 0) {
416 /* Position: */
417 LLVMValueRef x1y1 = LLVMGetParam(ctx->main_fn,
418 param_vs_blit_inputs);
419 LLVMValueRef x2y2 = LLVMGetParam(ctx->main_fn,
420 param_vs_blit_inputs + 1);
421
422 LLVMValueRef x1 = unpack_sint16(ctx, x1y1, 0);
423 LLVMValueRef y1 = unpack_sint16(ctx, x1y1, 1);
424 LLVMValueRef x2 = unpack_sint16(ctx, x2y2, 0);
425 LLVMValueRef y2 = unpack_sint16(ctx, x2y2, 1);
426
427 LLVMValueRef x = LLVMBuildSelect(ctx->ac.builder, sel_x1,
428 x1, x2, "");
429 LLVMValueRef y = LLVMBuildSelect(ctx->ac.builder, sel_y1,
430 y1, y2, "");
431
432 out[0] = LLVMBuildSIToFP(ctx->ac.builder, x, ctx->f32, "");
433 out[1] = LLVMBuildSIToFP(ctx->ac.builder, y, ctx->f32, "");
434 out[2] = LLVMGetParam(ctx->main_fn,
435 param_vs_blit_inputs + 2);
436 out[3] = ctx->ac.f32_1;
437 return;
438 }
439
440 /* Color or texture coordinates: */
441 assert(input_index == 1);
442
443 if (vs_blit_property == SI_VS_BLIT_SGPRS_POS_COLOR) {
444 for (int i = 0; i < 4; i++) {
445 out[i] = LLVMGetParam(ctx->main_fn,
446 param_vs_blit_inputs + 3 + i);
447 }
448 } else {
449 assert(vs_blit_property == SI_VS_BLIT_SGPRS_POS_TEXCOORD);
450 LLVMValueRef x1 = LLVMGetParam(ctx->main_fn,
451 param_vs_blit_inputs + 3);
452 LLVMValueRef y1 = LLVMGetParam(ctx->main_fn,
453 param_vs_blit_inputs + 4);
454 LLVMValueRef x2 = LLVMGetParam(ctx->main_fn,
455 param_vs_blit_inputs + 5);
456 LLVMValueRef y2 = LLVMGetParam(ctx->main_fn,
457 param_vs_blit_inputs + 6);
458
459 out[0] = LLVMBuildSelect(ctx->ac.builder, sel_x1,
460 x1, x2, "");
461 out[1] = LLVMBuildSelect(ctx->ac.builder, sel_y1,
462 y1, y2, "");
463 out[2] = LLVMGetParam(ctx->main_fn,
464 param_vs_blit_inputs + 7);
465 out[3] = LLVMGetParam(ctx->main_fn,
466 param_vs_blit_inputs + 8);
467 }
468 return;
469 }
470
471 union si_vs_fix_fetch fix_fetch;
472 LLVMValueRef t_list_ptr;
473 LLVMValueRef t_offset;
474 LLVMValueRef t_list;
475 LLVMValueRef vertex_index;
476 LLVMValueRef tmp;
477
478 /* Load the T list */
479 t_list_ptr = ac_get_arg(&ctx->ac, ctx->vertex_buffers);
480
481 t_offset = LLVMConstInt(ctx->i32, input_index, 0);
482
483 t_list = ac_build_load_to_sgpr(&ctx->ac, t_list_ptr, t_offset);
484
485 vertex_index = LLVMGetParam(ctx->main_fn,
486 ctx->vertex_index0.arg_index +
487 input_index);
488
489 /* Use the open-coded implementation for all loads of doubles and
490 * of dword-sized data that needs fixups. We need to insert conversion
491 * code anyway, and the amd/common code does it for us.
492 *
493 * Note: On LLVM <= 8, we can only open-code formats with
494 * channel size >= 4 bytes.
495 */
496 bool opencode = ctx->shader->key.mono.vs_fetch_opencode & (1 << input_index);
497 fix_fetch.bits = ctx->shader->key.mono.vs_fix_fetch[input_index].bits;
498 if (opencode ||
499 (fix_fetch.u.log_size == 3 && fix_fetch.u.format == AC_FETCH_FORMAT_FLOAT) ||
500 (fix_fetch.u.log_size == 2)) {
501 tmp = ac_build_opencoded_load_format(
502 &ctx->ac, fix_fetch.u.log_size, fix_fetch.u.num_channels_m1 + 1,
503 fix_fetch.u.format, fix_fetch.u.reverse, !opencode,
504 t_list, vertex_index, ctx->ac.i32_0, ctx->ac.i32_0, 0, true);
505 for (unsigned i = 0; i < 4; ++i)
506 out[i] = LLVMBuildExtractElement(ctx->ac.builder, tmp, LLVMConstInt(ctx->i32, i, false), "");
507 return;
508 }
509
510 /* Do multiple loads for special formats. */
511 unsigned required_channels = util_last_bit(info->input_usage_mask[input_index]);
512 LLVMValueRef fetches[4];
513 unsigned num_fetches;
514 unsigned fetch_stride;
515 unsigned channels_per_fetch;
516
517 if (fix_fetch.u.log_size <= 1 && fix_fetch.u.num_channels_m1 == 2) {
518 num_fetches = MIN2(required_channels, 3);
519 fetch_stride = 1 << fix_fetch.u.log_size;
520 channels_per_fetch = 1;
521 } else {
522 num_fetches = 1;
523 fetch_stride = 0;
524 channels_per_fetch = required_channels;
525 }
526
527 for (unsigned i = 0; i < num_fetches; ++i) {
528 LLVMValueRef voffset = LLVMConstInt(ctx->i32, fetch_stride * i, 0);
529 fetches[i] = ac_build_buffer_load_format(&ctx->ac, t_list, vertex_index, voffset,
530 channels_per_fetch, 0, true);
531 }
532
533 if (num_fetches == 1 && channels_per_fetch > 1) {
534 LLVMValueRef fetch = fetches[0];
535 for (unsigned i = 0; i < channels_per_fetch; ++i) {
536 tmp = LLVMConstInt(ctx->i32, i, false);
537 fetches[i] = LLVMBuildExtractElement(
538 ctx->ac.builder, fetch, tmp, "");
539 }
540 num_fetches = channels_per_fetch;
541 channels_per_fetch = 1;
542 }
543
544 for (unsigned i = num_fetches; i < 4; ++i)
545 fetches[i] = LLVMGetUndef(ctx->f32);
546
547 if (fix_fetch.u.log_size <= 1 && fix_fetch.u.num_channels_m1 == 2 &&
548 required_channels == 4) {
549 if (fix_fetch.u.format == AC_FETCH_FORMAT_UINT || fix_fetch.u.format == AC_FETCH_FORMAT_SINT)
550 fetches[3] = ctx->ac.i32_1;
551 else
552 fetches[3] = ctx->ac.f32_1;
553 } else if (fix_fetch.u.log_size == 3 &&
554 (fix_fetch.u.format == AC_FETCH_FORMAT_SNORM ||
555 fix_fetch.u.format == AC_FETCH_FORMAT_SSCALED ||
556 fix_fetch.u.format == AC_FETCH_FORMAT_SINT) &&
557 required_channels == 4) {
558 /* For 2_10_10_10, the hardware returns an unsigned value;
559 * convert it to a signed one.
560 */
561 LLVMValueRef tmp = fetches[3];
562 LLVMValueRef c30 = LLVMConstInt(ctx->i32, 30, 0);
563
564 /* First, recover the sign-extended signed integer value. */
565 if (fix_fetch.u.format == AC_FETCH_FORMAT_SSCALED)
566 tmp = LLVMBuildFPToUI(ctx->ac.builder, tmp, ctx->i32, "");
567 else
568 tmp = ac_to_integer(&ctx->ac, tmp);
569
570 /* For the integer-like cases, do a natural sign extension.
571 *
572 * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
573 * and happen to contain 0, 1, 2, 3 as the two LSBs of the
574 * exponent.
575 */
576 tmp = LLVMBuildShl(ctx->ac.builder, tmp,
577 fix_fetch.u.format == AC_FETCH_FORMAT_SNORM ?
578 LLVMConstInt(ctx->i32, 7, 0) : c30, "");
579 tmp = LLVMBuildAShr(ctx->ac.builder, tmp, c30, "");
580
581 /* Convert back to the right type. */
582 if (fix_fetch.u.format == AC_FETCH_FORMAT_SNORM) {
583 LLVMValueRef clamp;
584 LLVMValueRef neg_one = LLVMConstReal(ctx->f32, -1.0);
585 tmp = LLVMBuildSIToFP(ctx->ac.builder, tmp, ctx->f32, "");
586 clamp = LLVMBuildFCmp(ctx->ac.builder, LLVMRealULT, tmp, neg_one, "");
587 tmp = LLVMBuildSelect(ctx->ac.builder, clamp, neg_one, tmp, "");
588 } else if (fix_fetch.u.format == AC_FETCH_FORMAT_SSCALED) {
589 tmp = LLVMBuildSIToFP(ctx->ac.builder, tmp, ctx->f32, "");
590 }
591
592 fetches[3] = tmp;
593 }
594
595 for (unsigned i = 0; i < 4; ++i)
596 out[i] = ac_to_float(&ctx->ac, fetches[i]);
597 }
598
599 static void declare_input_vs(
600 struct si_shader_context *ctx,
601 unsigned input_index,
602 const struct tgsi_full_declaration *decl,
603 LLVMValueRef out[4])
604 {
605 si_llvm_load_input_vs(ctx, input_index, out);
606 }
607
608 LLVMValueRef si_get_primitive_id(struct si_shader_context *ctx,
609 unsigned swizzle)
610 {
611 if (swizzle > 0)
612 return ctx->i32_0;
613
614 switch (ctx->type) {
615 case PIPE_SHADER_VERTEX:
616 return ac_get_arg(&ctx->ac, ctx->vs_prim_id);
617 case PIPE_SHADER_TESS_CTRL:
618 return ac_get_arg(&ctx->ac, ctx->args.tcs_patch_id);
619 case PIPE_SHADER_TESS_EVAL:
620 return ac_get_arg(&ctx->ac, ctx->args.tes_patch_id);
621 case PIPE_SHADER_GEOMETRY:
622 return ac_get_arg(&ctx->ac, ctx->args.gs_prim_id);
623 default:
624 assert(0);
625 return ctx->i32_0;
626 }
627 }
628
629 /**
630 * Return the value of tgsi_ind_register for indexing.
631 * This is the indirect index with the constant offset added to it.
632 */
633 LLVMValueRef si_get_indirect_index(struct si_shader_context *ctx,
634 const struct tgsi_ind_register *ind,
635 unsigned addr_mul,
636 int rel_index)
637 {
638 LLVMValueRef result;
639
640 if (ind->File == TGSI_FILE_ADDRESS) {
641 result = ctx->addrs[ind->Index][ind->Swizzle];
642 result = LLVMBuildLoad(ctx->ac.builder, result, "");
643 } else {
644 struct tgsi_full_src_register src = {};
645
646 src.Register.File = ind->File;
647 src.Register.Index = ind->Index;
648
649 /* Set the second index to 0 for constants. */
650 if (ind->File == TGSI_FILE_CONSTANT)
651 src.Register.Dimension = 1;
652
653 result = ctx->bld_base.emit_fetch_funcs[ind->File](&ctx->bld_base, &src,
654 TGSI_TYPE_SIGNED,
655 ind->Swizzle);
656 result = ac_to_integer(&ctx->ac, result);
657 }
658
659 return ac_build_imad(&ctx->ac, result, LLVMConstInt(ctx->i32, addr_mul, 0),
660 LLVMConstInt(ctx->i32, rel_index, 0));
661 }
662
663 /**
664 * Like si_get_indirect_index, but restricts the return value to a (possibly
665 * undefined) value inside [0..num).
666 */
667 LLVMValueRef si_get_bounded_indirect_index(struct si_shader_context *ctx,
668 const struct tgsi_ind_register *ind,
669 int rel_index, unsigned num)
670 {
671 LLVMValueRef result = si_get_indirect_index(ctx, ind, 1, rel_index);
672
673 return si_llvm_bound_index(ctx, result, num);
674 }
675
676 static LLVMValueRef get_dw_address_from_generic_indices(struct si_shader_context *ctx,
677 LLVMValueRef vertex_dw_stride,
678 LLVMValueRef base_addr,
679 LLVMValueRef vertex_index,
680 LLVMValueRef param_index,
681 ubyte name, ubyte index)
682 {
683 if (vertex_dw_stride) {
684 base_addr = ac_build_imad(&ctx->ac, vertex_index,
685 vertex_dw_stride, base_addr);
686 }
687
688 if (param_index) {
689 base_addr = ac_build_imad(&ctx->ac, param_index,
690 LLVMConstInt(ctx->i32, 4, 0), base_addr);
691 }
692
693 int param = name == TGSI_SEMANTIC_PATCH ||
694 name == TGSI_SEMANTIC_TESSINNER ||
695 name == TGSI_SEMANTIC_TESSOUTER ?
696 si_shader_io_get_unique_index_patch(name, index) :
697 si_shader_io_get_unique_index(name, index, false);
698
699 /* Add the base address of the element. */
700 return LLVMBuildAdd(ctx->ac.builder, base_addr,
701 LLVMConstInt(ctx->i32, param * 4, 0), "");
702 }
703
704 /**
705 * Calculate a dword address given an input or output register and a stride.
706 */
707 static LLVMValueRef get_dw_address(struct si_shader_context *ctx,
708 const struct tgsi_full_dst_register *dst,
709 const struct tgsi_full_src_register *src,
710 LLVMValueRef vertex_dw_stride,
711 LLVMValueRef base_addr)
712 {
713 struct tgsi_shader_info *info = &ctx->shader->selector->info;
714 ubyte *name, *index, *array_first;
715 int input_index;
716 struct tgsi_full_dst_register reg;
717 LLVMValueRef vertex_index = NULL;
718 LLVMValueRef ind_index = NULL;
719
720 /* Set the register description. The address computation is the same
721 * for sources and destinations. */
722 if (src) {
723 reg.Register.File = src->Register.File;
724 reg.Register.Index = src->Register.Index;
725 reg.Register.Indirect = src->Register.Indirect;
726 reg.Register.Dimension = src->Register.Dimension;
727 reg.Indirect = src->Indirect;
728 reg.Dimension = src->Dimension;
729 reg.DimIndirect = src->DimIndirect;
730 } else
731 reg = *dst;
732
733 /* If the register is 2-dimensional (e.g. an array of vertices
734 * in a primitive), calculate the base address of the vertex. */
735 if (reg.Register.Dimension) {
736 if (reg.Dimension.Indirect)
737 vertex_index = si_get_indirect_index(ctx, &reg.DimIndirect,
738 1, reg.Dimension.Index);
739 else
740 vertex_index = LLVMConstInt(ctx->i32, reg.Dimension.Index, 0);
741 }
742
743 /* Get information about the register. */
744 if (reg.Register.File == TGSI_FILE_INPUT) {
745 name = info->input_semantic_name;
746 index = info->input_semantic_index;
747 array_first = info->input_array_first;
748 } else if (reg.Register.File == TGSI_FILE_OUTPUT) {
749 name = info->output_semantic_name;
750 index = info->output_semantic_index;
751 array_first = info->output_array_first;
752 } else {
753 assert(0);
754 return NULL;
755 }
756
757 if (reg.Register.Indirect) {
758 /* Add the relative address of the element. */
759 if (reg.Indirect.ArrayID)
760 input_index = array_first[reg.Indirect.ArrayID];
761 else
762 input_index = reg.Register.Index;
763
764 ind_index = si_get_indirect_index(ctx, &reg.Indirect,
765 1, reg.Register.Index - input_index);
766 } else {
767 input_index = reg.Register.Index;
768 }
769
770 return get_dw_address_from_generic_indices(ctx, vertex_dw_stride,
771 base_addr, vertex_index,
772 ind_index, name[input_index],
773 index[input_index]);
774 }
775
776 /* The offchip buffer layout for TCS->TES is
777 *
778 * - attribute 0 of patch 0 vertex 0
779 * - attribute 0 of patch 0 vertex 1
780 * - attribute 0 of patch 0 vertex 2
781 * ...
782 * - attribute 0 of patch 1 vertex 0
783 * - attribute 0 of patch 1 vertex 1
784 * ...
785 * - attribute 1 of patch 0 vertex 0
786 * - attribute 1 of patch 0 vertex 1
787 * ...
788 * - per patch attribute 0 of patch 0
789 * - per patch attribute 0 of patch 1
790 * ...
791 *
792 * Note that every attribute has 4 components.
793 */
794 static LLVMValueRef get_tcs_tes_buffer_address(struct si_shader_context *ctx,
795 LLVMValueRef rel_patch_id,
796 LLVMValueRef vertex_index,
797 LLVMValueRef param_index)
798 {
799 LLVMValueRef base_addr, vertices_per_patch, num_patches, total_vertices;
800 LLVMValueRef param_stride, constant16;
801
802 vertices_per_patch = get_num_tcs_out_vertices(ctx);
803 num_patches = si_unpack_param(ctx, ctx->tcs_offchip_layout, 0, 6);
804 total_vertices = LLVMBuildMul(ctx->ac.builder, vertices_per_patch,
805 num_patches, "");
806
807 constant16 = LLVMConstInt(ctx->i32, 16, 0);
808 if (vertex_index) {
809 base_addr = ac_build_imad(&ctx->ac, rel_patch_id,
810 vertices_per_patch, vertex_index);
811 param_stride = total_vertices;
812 } else {
813 base_addr = rel_patch_id;
814 param_stride = num_patches;
815 }
816
817 base_addr = ac_build_imad(&ctx->ac, param_index, param_stride, base_addr);
818 base_addr = LLVMBuildMul(ctx->ac.builder, base_addr, constant16, "");
819
820 if (!vertex_index) {
821 LLVMValueRef patch_data_offset =
822 si_unpack_param(ctx, ctx->tcs_offchip_layout, 12, 20);
823
824 base_addr = LLVMBuildAdd(ctx->ac.builder, base_addr,
825 patch_data_offset, "");
826 }
827 return base_addr;
828 }
829
830 /* This is a generic helper that can be shared by the NIR and TGSI backends */
831 static LLVMValueRef get_tcs_tes_buffer_address_from_generic_indices(
832 struct si_shader_context *ctx,
833 LLVMValueRef vertex_index,
834 LLVMValueRef param_index,
835 ubyte name, ubyte index)
836 {
837 unsigned param_index_base;
838
839 param_index_base = name == TGSI_SEMANTIC_PATCH ||
840 name == TGSI_SEMANTIC_TESSINNER ||
841 name == TGSI_SEMANTIC_TESSOUTER ?
842 si_shader_io_get_unique_index_patch(name, index) :
843 si_shader_io_get_unique_index(name, index, false);
844
845 if (param_index) {
846 param_index = LLVMBuildAdd(ctx->ac.builder, param_index,
847 LLVMConstInt(ctx->i32, param_index_base, 0),
848 "");
849 } else {
850 param_index = LLVMConstInt(ctx->i32, param_index_base, 0);
851 }
852
853 return get_tcs_tes_buffer_address(ctx, get_rel_patch_id(ctx),
854 vertex_index, param_index);
855 }
856
857 static LLVMValueRef get_tcs_tes_buffer_address_from_reg(
858 struct si_shader_context *ctx,
859 const struct tgsi_full_dst_register *dst,
860 const struct tgsi_full_src_register *src)
861 {
862 struct tgsi_shader_info *info = &ctx->shader->selector->info;
863 ubyte *name, *index, *array_first;
864 struct tgsi_full_src_register reg;
865 LLVMValueRef vertex_index = NULL;
866 LLVMValueRef param_index = NULL;
867 unsigned param_base;
868
869 reg = src ? *src : tgsi_full_src_register_from_dst(dst);
870
871 if (reg.Register.Dimension) {
872 if (reg.Dimension.Indirect)
873 vertex_index = si_get_indirect_index(ctx, &reg.DimIndirect,
874 1, reg.Dimension.Index);
875 else
876 vertex_index = LLVMConstInt(ctx->i32, reg.Dimension.Index, 0);
877 }
878
879 /* Get information about the register. */
880 if (reg.Register.File == TGSI_FILE_INPUT) {
881 name = info->input_semantic_name;
882 index = info->input_semantic_index;
883 array_first = info->input_array_first;
884 } else if (reg.Register.File == TGSI_FILE_OUTPUT) {
885 name = info->output_semantic_name;
886 index = info->output_semantic_index;
887 array_first = info->output_array_first;
888 } else {
889 assert(0);
890 return NULL;
891 }
892
893 if (reg.Register.Indirect) {
894 if (reg.Indirect.ArrayID)
895 param_base = array_first[reg.Indirect.ArrayID];
896 else
897 param_base = reg.Register.Index;
898
899 param_index = si_get_indirect_index(ctx, &reg.Indirect,
900 1, reg.Register.Index - param_base);
901 } else {
902 param_base = reg.Register.Index;
903 }
904
905 return get_tcs_tes_buffer_address_from_generic_indices(ctx, vertex_index,
906 param_index, name[param_base],
907 index[param_base]);
908 }
909
910 static LLVMValueRef buffer_load(struct lp_build_tgsi_context *bld_base,
911 LLVMTypeRef type, unsigned swizzle,
912 LLVMValueRef buffer, LLVMValueRef offset,
913 LLVMValueRef base, bool can_speculate)
914 {
915 struct si_shader_context *ctx = si_shader_context(bld_base);
916 LLVMValueRef value, value2;
917 LLVMTypeRef vec_type = LLVMVectorType(type, 4);
918
919 if (swizzle == ~0) {
920 value = ac_build_buffer_load(&ctx->ac, buffer, 4, NULL, base, offset,
921 0, ac_glc, can_speculate, false);
922
923 return LLVMBuildBitCast(ctx->ac.builder, value, vec_type, "");
924 }
925
926 if (!llvm_type_is_64bit(ctx, type)) {
927 value = ac_build_buffer_load(&ctx->ac, buffer, 4, NULL, base, offset,
928 0, ac_glc, can_speculate, false);
929
930 value = LLVMBuildBitCast(ctx->ac.builder, value, vec_type, "");
931 return LLVMBuildExtractElement(ctx->ac.builder, value,
932 LLVMConstInt(ctx->i32, swizzle, 0), "");
933 }
934
935 value = ac_build_buffer_load(&ctx->ac, buffer, 1, NULL, base, offset,
936 swizzle * 4, ac_glc, can_speculate, false);
937
938 value2 = ac_build_buffer_load(&ctx->ac, buffer, 1, NULL, base, offset,
939 swizzle * 4 + 4, ac_glc, can_speculate, false);
940
941 return si_llvm_emit_fetch_64bit(bld_base, type, value, value2);
942 }
943
944 /**
945 * Load from LSHS LDS storage.
946 *
947 * \param type output value type
948 * \param swizzle offset (typically 0..3); it can be ~0, which loads a vec4
949 * \param dw_addr address in dwords
950 */
951 static LLVMValueRef lshs_lds_load(struct lp_build_tgsi_context *bld_base,
952 LLVMTypeRef type, unsigned swizzle,
953 LLVMValueRef dw_addr)
954 {
955 struct si_shader_context *ctx = si_shader_context(bld_base);
956 LLVMValueRef value;
957
958 if (swizzle == ~0) {
959 LLVMValueRef values[TGSI_NUM_CHANNELS];
960
961 for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; chan++)
962 values[chan] = lshs_lds_load(bld_base, type, chan, dw_addr);
963
964 return ac_build_gather_values(&ctx->ac, values,
965 TGSI_NUM_CHANNELS);
966 }
967
968 /* Split 64-bit loads. */
969 if (llvm_type_is_64bit(ctx, type)) {
970 LLVMValueRef lo, hi;
971
972 lo = lshs_lds_load(bld_base, ctx->i32, swizzle, dw_addr);
973 hi = lshs_lds_load(bld_base, ctx->i32, swizzle + 1, dw_addr);
974 return si_llvm_emit_fetch_64bit(bld_base, type, lo, hi);
975 }
976
977 dw_addr = LLVMBuildAdd(ctx->ac.builder, dw_addr,
978 LLVMConstInt(ctx->i32, swizzle, 0), "");
979
980 value = ac_lds_load(&ctx->ac, dw_addr);
981
982 return LLVMBuildBitCast(ctx->ac.builder, value, type, "");
983 }
984
985 /**
986 * Store to LSHS LDS storage.
987 *
988 * \param swizzle offset (typically 0..3)
989 * \param dw_addr address in dwords
990 * \param value value to store
991 */
992 static void lshs_lds_store(struct si_shader_context *ctx,
993 unsigned dw_offset_imm, LLVMValueRef dw_addr,
994 LLVMValueRef value)
995 {
996 dw_addr = LLVMBuildAdd(ctx->ac.builder, dw_addr,
997 LLVMConstInt(ctx->i32, dw_offset_imm, 0), "");
998
999 ac_lds_store(&ctx->ac, dw_addr, value);
1000 }
1001
1002 enum si_tess_ring {
1003 TCS_FACTOR_RING,
1004 TESS_OFFCHIP_RING_TCS,
1005 TESS_OFFCHIP_RING_TES,
1006 };
1007
1008 static LLVMValueRef get_tess_ring_descriptor(struct si_shader_context *ctx,
1009 enum si_tess_ring ring)
1010 {
1011 LLVMBuilderRef builder = ctx->ac.builder;
1012 LLVMValueRef addr = ac_get_arg(&ctx->ac,
1013 ring == TESS_OFFCHIP_RING_TES ?
1014 ctx->tes_offchip_addr :
1015 ctx->tcs_out_lds_layout);
1016
1017 /* TCS only receives high 13 bits of the address. */
1018 if (ring == TESS_OFFCHIP_RING_TCS || ring == TCS_FACTOR_RING) {
1019 addr = LLVMBuildAnd(builder, addr,
1020 LLVMConstInt(ctx->i32, 0xfff80000, 0), "");
1021 }
1022
1023 if (ring == TCS_FACTOR_RING) {
1024 unsigned tf_offset = ctx->screen->tess_offchip_ring_size;
1025 addr = LLVMBuildAdd(builder, addr,
1026 LLVMConstInt(ctx->i32, tf_offset, 0), "");
1027 }
1028
1029 uint32_t rsrc3 = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1030 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1031 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1032 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
1033
1034 if (ctx->screen->info.chip_class >= GFX10)
1035 rsrc3 |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
1036 S_008F0C_OOB_SELECT(3) |
1037 S_008F0C_RESOURCE_LEVEL(1);
1038 else
1039 rsrc3 |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1040 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
1041
1042 LLVMValueRef desc[4];
1043 desc[0] = addr;
1044 desc[1] = LLVMConstInt(ctx->i32,
1045 S_008F04_BASE_ADDRESS_HI(ctx->screen->info.address32_hi), 0);
1046 desc[2] = LLVMConstInt(ctx->i32, 0xffffffff, 0);
1047 desc[3] = LLVMConstInt(ctx->i32, rsrc3, false);
1048
1049 return ac_build_gather_values(&ctx->ac, desc, 4);
1050 }
1051
1052 static LLVMValueRef fetch_input_tcs(
1053 struct lp_build_tgsi_context *bld_base,
1054 const struct tgsi_full_src_register *reg,
1055 enum tgsi_opcode_type type, unsigned swizzle_in)
1056 {
1057 struct si_shader_context *ctx = si_shader_context(bld_base);
1058 LLVMValueRef dw_addr, stride;
1059 unsigned swizzle = swizzle_in & 0xffff;
1060 stride = get_tcs_in_vertex_dw_stride(ctx);
1061 dw_addr = get_tcs_in_current_patch_offset(ctx);
1062 dw_addr = get_dw_address(ctx, NULL, reg, stride, dw_addr);
1063
1064 return lshs_lds_load(bld_base, tgsi2llvmtype(bld_base, type), swizzle, dw_addr);
1065 }
1066
1067 static LLVMValueRef si_nir_load_tcs_varyings(struct ac_shader_abi *abi,
1068 LLVMTypeRef type,
1069 LLVMValueRef vertex_index,
1070 LLVMValueRef param_index,
1071 unsigned const_index,
1072 unsigned location,
1073 unsigned driver_location,
1074 unsigned component,
1075 unsigned num_components,
1076 bool is_patch,
1077 bool is_compact,
1078 bool load_input)
1079 {
1080 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1081 struct tgsi_shader_info *info = &ctx->shader->selector->info;
1082 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
1083 LLVMValueRef dw_addr, stride;
1084 ubyte name, index;
1085
1086 driver_location = driver_location / 4;
1087
1088 if (load_input) {
1089 name = info->input_semantic_name[driver_location];
1090 index = info->input_semantic_index[driver_location];
1091 } else {
1092 name = info->output_semantic_name[driver_location];
1093 index = info->output_semantic_index[driver_location];
1094 }
1095
1096 assert((name == TGSI_SEMANTIC_PATCH ||
1097 name == TGSI_SEMANTIC_TESSINNER ||
1098 name == TGSI_SEMANTIC_TESSOUTER) == is_patch);
1099
1100 if (load_input) {
1101 stride = get_tcs_in_vertex_dw_stride(ctx);
1102 dw_addr = get_tcs_in_current_patch_offset(ctx);
1103 } else {
1104 if (is_patch) {
1105 stride = NULL;
1106 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1107 } else {
1108 stride = get_tcs_out_vertex_dw_stride(ctx);
1109 dw_addr = get_tcs_out_current_patch_offset(ctx);
1110 }
1111 }
1112
1113 if (!param_index) {
1114 param_index = LLVMConstInt(ctx->i32, const_index, 0);
1115 }
1116
1117 dw_addr = get_dw_address_from_generic_indices(ctx, stride, dw_addr,
1118 vertex_index, param_index,
1119 name, index);
1120
1121 LLVMValueRef value[4];
1122 for (unsigned i = 0; i < num_components; i++) {
1123 unsigned offset = i;
1124 if (llvm_type_is_64bit(ctx, type))
1125 offset *= 2;
1126
1127 offset += component;
1128 value[i + component] = lshs_lds_load(bld_base, type, offset, dw_addr);
1129 }
1130
1131 return ac_build_varying_gather_values(&ctx->ac, value, num_components, component);
1132 }
1133
1134 static LLVMValueRef fetch_output_tcs(
1135 struct lp_build_tgsi_context *bld_base,
1136 const struct tgsi_full_src_register *reg,
1137 enum tgsi_opcode_type type, unsigned swizzle_in)
1138 {
1139 struct si_shader_context *ctx = si_shader_context(bld_base);
1140 LLVMValueRef dw_addr, stride;
1141 unsigned swizzle = (swizzle_in & 0xffff);
1142
1143 if (reg->Register.Dimension) {
1144 stride = get_tcs_out_vertex_dw_stride(ctx);
1145 dw_addr = get_tcs_out_current_patch_offset(ctx);
1146 dw_addr = get_dw_address(ctx, NULL, reg, stride, dw_addr);
1147 } else {
1148 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1149 dw_addr = get_dw_address(ctx, NULL, reg, NULL, dw_addr);
1150 }
1151
1152 return lshs_lds_load(bld_base, tgsi2llvmtype(bld_base, type), swizzle, dw_addr);
1153 }
1154
1155 static LLVMValueRef fetch_input_tes(
1156 struct lp_build_tgsi_context *bld_base,
1157 const struct tgsi_full_src_register *reg,
1158 enum tgsi_opcode_type type, unsigned swizzle_in)
1159 {
1160 struct si_shader_context *ctx = si_shader_context(bld_base);
1161 LLVMValueRef base, addr;
1162 unsigned swizzle = (swizzle_in & 0xffff);
1163
1164 base = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
1165 addr = get_tcs_tes_buffer_address_from_reg(ctx, NULL, reg);
1166
1167 return buffer_load(bld_base, tgsi2llvmtype(bld_base, type), swizzle,
1168 ctx->tess_offchip_ring, base, addr, true);
1169 }
1170
1171 LLVMValueRef si_nir_load_input_tes(struct ac_shader_abi *abi,
1172 LLVMTypeRef type,
1173 LLVMValueRef vertex_index,
1174 LLVMValueRef param_index,
1175 unsigned const_index,
1176 unsigned location,
1177 unsigned driver_location,
1178 unsigned component,
1179 unsigned num_components,
1180 bool is_patch,
1181 bool is_compact,
1182 bool load_input)
1183 {
1184 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1185 struct tgsi_shader_info *info = &ctx->shader->selector->info;
1186 LLVMValueRef base, addr;
1187
1188 driver_location = driver_location / 4;
1189 ubyte name = info->input_semantic_name[driver_location];
1190 ubyte index = info->input_semantic_index[driver_location];
1191
1192 assert((name == TGSI_SEMANTIC_PATCH ||
1193 name == TGSI_SEMANTIC_TESSINNER ||
1194 name == TGSI_SEMANTIC_TESSOUTER) == is_patch);
1195
1196 base = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
1197
1198 if (!param_index) {
1199 param_index = LLVMConstInt(ctx->i32, const_index, 0);
1200 }
1201
1202 addr = get_tcs_tes_buffer_address_from_generic_indices(ctx, vertex_index,
1203 param_index,
1204 name, index);
1205
1206 /* TODO: This will generate rather ordinary llvm code, although it
1207 * should be easy for the optimiser to fix up. In future we might want
1208 * to refactor buffer_load(), but for now this maximises code sharing
1209 * between the NIR and TGSI backends.
1210 */
1211 LLVMValueRef value[4];
1212 for (unsigned i = 0; i < num_components; i++) {
1213 unsigned offset = i;
1214 if (llvm_type_is_64bit(ctx, type)) {
1215 offset *= 2;
1216 if (offset == 4) {
1217 ubyte name = info->input_semantic_name[driver_location + 1];
1218 ubyte index = info->input_semantic_index[driver_location + 1];
1219 addr = get_tcs_tes_buffer_address_from_generic_indices(ctx,
1220 vertex_index,
1221 param_index,
1222 name, index);
1223 }
1224
1225 offset = offset % 4;
1226 }
1227
1228 offset += component;
1229 value[i + component] = buffer_load(&ctx->bld_base, type, offset,
1230 ctx->tess_offchip_ring, base, addr, true);
1231 }
1232
1233 return ac_build_varying_gather_values(&ctx->ac, value, num_components, component);
1234 }
1235
1236 static void store_output_tcs(struct lp_build_tgsi_context *bld_base,
1237 const struct tgsi_full_instruction *inst,
1238 const struct tgsi_opcode_info *info,
1239 unsigned index,
1240 LLVMValueRef dst[4])
1241 {
1242 struct si_shader_context *ctx = si_shader_context(bld_base);
1243 const struct tgsi_full_dst_register *reg = &inst->Dst[index];
1244 const struct tgsi_shader_info *sh_info = &ctx->shader->selector->info;
1245 unsigned chan_index;
1246 LLVMValueRef dw_addr, stride;
1247 LLVMValueRef buffer, base, buf_addr;
1248 LLVMValueRef values[4];
1249 bool skip_lds_store;
1250 bool is_tess_factor = false, is_tess_inner = false;
1251
1252 /* Only handle per-patch and per-vertex outputs here.
1253 * Vectors will be lowered to scalars and this function will be called again.
1254 */
1255 if (reg->Register.File != TGSI_FILE_OUTPUT ||
1256 (dst[0] && LLVMGetTypeKind(LLVMTypeOf(dst[0])) == LLVMVectorTypeKind)) {
1257 si_llvm_emit_store(bld_base, inst, info, index, dst);
1258 return;
1259 }
1260
1261 if (reg->Register.Dimension) {
1262 stride = get_tcs_out_vertex_dw_stride(ctx);
1263 dw_addr = get_tcs_out_current_patch_offset(ctx);
1264 dw_addr = get_dw_address(ctx, reg, NULL, stride, dw_addr);
1265 skip_lds_store = !sh_info->reads_pervertex_outputs;
1266 } else {
1267 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1268 dw_addr = get_dw_address(ctx, reg, NULL, NULL, dw_addr);
1269 skip_lds_store = !sh_info->reads_perpatch_outputs;
1270
1271 if (!reg->Register.Indirect) {
1272 int name = sh_info->output_semantic_name[reg->Register.Index];
1273
1274 /* Always write tess factors into LDS for the TCS epilog. */
1275 if (name == TGSI_SEMANTIC_TESSINNER ||
1276 name == TGSI_SEMANTIC_TESSOUTER) {
1277 /* The epilog doesn't read LDS if invocation 0 defines tess factors. */
1278 skip_lds_store = !sh_info->reads_tessfactor_outputs &&
1279 ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs;
1280 is_tess_factor = true;
1281 is_tess_inner = name == TGSI_SEMANTIC_TESSINNER;
1282 }
1283 }
1284 }
1285
1286 buffer = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TCS);
1287
1288 base = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
1289 buf_addr = get_tcs_tes_buffer_address_from_reg(ctx, reg, NULL);
1290
1291 uint32_t writemask = reg->Register.WriteMask;
1292 while (writemask) {
1293 chan_index = u_bit_scan(&writemask);
1294 LLVMValueRef value = dst[chan_index];
1295
1296 if (inst->Instruction.Saturate)
1297 value = ac_build_clamp(&ctx->ac, value);
1298
1299 /* Skip LDS stores if there is no LDS read of this output. */
1300 if (!skip_lds_store)
1301 lshs_lds_store(ctx, chan_index, dw_addr, value);
1302
1303 value = ac_to_integer(&ctx->ac, value);
1304 values[chan_index] = value;
1305
1306 if (reg->Register.WriteMask != 0xF && !is_tess_factor) {
1307 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 1,
1308 buf_addr, base,
1309 4 * chan_index, ac_glc);
1310 }
1311
1312 /* Write tess factors into VGPRs for the epilog. */
1313 if (is_tess_factor &&
1314 ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs) {
1315 if (!is_tess_inner) {
1316 LLVMBuildStore(ctx->ac.builder, value, /* outer */
1317 ctx->invoc0_tess_factors[chan_index]);
1318 } else if (chan_index < 2) {
1319 LLVMBuildStore(ctx->ac.builder, value, /* inner */
1320 ctx->invoc0_tess_factors[4 + chan_index]);
1321 }
1322 }
1323 }
1324
1325 if (reg->Register.WriteMask == 0xF && !is_tess_factor) {
1326 LLVMValueRef value = ac_build_gather_values(&ctx->ac,
1327 values, 4);
1328 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, buf_addr,
1329 base, 0, ac_glc);
1330 }
1331 }
1332
1333 static void si_nir_store_output_tcs(struct ac_shader_abi *abi,
1334 const struct nir_variable *var,
1335 LLVMValueRef vertex_index,
1336 LLVMValueRef param_index,
1337 unsigned const_index,
1338 LLVMValueRef src,
1339 unsigned writemask)
1340 {
1341 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1342 struct tgsi_shader_info *info = &ctx->shader->selector->info;
1343 const unsigned component = var->data.location_frac;
1344 unsigned driver_location = var->data.driver_location;
1345 LLVMValueRef dw_addr, stride;
1346 LLVMValueRef buffer, base, addr;
1347 LLVMValueRef values[8];
1348 bool skip_lds_store;
1349 bool is_tess_factor = false, is_tess_inner = false;
1350
1351 driver_location = driver_location / 4;
1352 ubyte name = info->output_semantic_name[driver_location];
1353 ubyte index = info->output_semantic_index[driver_location];
1354
1355 bool is_const = !param_index;
1356 if (!param_index)
1357 param_index = LLVMConstInt(ctx->i32, const_index, 0);
1358
1359 const bool is_patch = var->data.patch ||
1360 var->data.location == VARYING_SLOT_TESS_LEVEL_INNER ||
1361 var->data.location == VARYING_SLOT_TESS_LEVEL_OUTER;
1362
1363 assert((name == TGSI_SEMANTIC_PATCH ||
1364 name == TGSI_SEMANTIC_TESSINNER ||
1365 name == TGSI_SEMANTIC_TESSOUTER) == is_patch);
1366
1367 if (!is_patch) {
1368 stride = get_tcs_out_vertex_dw_stride(ctx);
1369 dw_addr = get_tcs_out_current_patch_offset(ctx);
1370 dw_addr = get_dw_address_from_generic_indices(ctx, stride, dw_addr,
1371 vertex_index, param_index,
1372 name, index);
1373
1374 skip_lds_store = !info->reads_pervertex_outputs;
1375 } else {
1376 dw_addr = get_tcs_out_current_patch_data_offset(ctx);
1377 dw_addr = get_dw_address_from_generic_indices(ctx, NULL, dw_addr,
1378 vertex_index, param_index,
1379 name, index);
1380
1381 skip_lds_store = !info->reads_perpatch_outputs;
1382
1383 if (is_const && const_index == 0) {
1384 int name = info->output_semantic_name[driver_location];
1385
1386 /* Always write tess factors into LDS for the TCS epilog. */
1387 if (name == TGSI_SEMANTIC_TESSINNER ||
1388 name == TGSI_SEMANTIC_TESSOUTER) {
1389 /* The epilog doesn't read LDS if invocation 0 defines tess factors. */
1390 skip_lds_store = !info->reads_tessfactor_outputs &&
1391 ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs;
1392 is_tess_factor = true;
1393 is_tess_inner = name == TGSI_SEMANTIC_TESSINNER;
1394 }
1395 }
1396 }
1397
1398 buffer = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TCS);
1399
1400 base = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
1401
1402 addr = get_tcs_tes_buffer_address_from_generic_indices(ctx, vertex_index,
1403 param_index, name, index);
1404
1405 for (unsigned chan = component; chan < 8; chan++) {
1406 if (!(writemask & (1 << chan)))
1407 continue;
1408 LLVMValueRef value = ac_llvm_extract_elem(&ctx->ac, src, chan - component);
1409
1410 unsigned buffer_store_offset = chan % 4;
1411 if (chan == 4) {
1412 ubyte name = info->output_semantic_name[driver_location + 1];
1413 ubyte index = info->output_semantic_index[driver_location + 1];
1414 addr = get_tcs_tes_buffer_address_from_generic_indices(ctx,
1415 vertex_index,
1416 param_index,
1417 name, index);
1418 }
1419
1420 /* Skip LDS stores if there is no LDS read of this output. */
1421 if (!skip_lds_store)
1422 lshs_lds_store(ctx, chan, dw_addr, value);
1423
1424 value = ac_to_integer(&ctx->ac, value);
1425 values[chan] = value;
1426
1427 if (writemask != 0xF && !is_tess_factor) {
1428 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 1,
1429 addr, base,
1430 4 * buffer_store_offset,
1431 ac_glc);
1432 }
1433
1434 /* Write tess factors into VGPRs for the epilog. */
1435 if (is_tess_factor &&
1436 ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs) {
1437 if (!is_tess_inner) {
1438 LLVMBuildStore(ctx->ac.builder, value, /* outer */
1439 ctx->invoc0_tess_factors[chan]);
1440 } else if (chan < 2) {
1441 LLVMBuildStore(ctx->ac.builder, value, /* inner */
1442 ctx->invoc0_tess_factors[4 + chan]);
1443 }
1444 }
1445 }
1446
1447 if (writemask == 0xF && !is_tess_factor) {
1448 LLVMValueRef value = ac_build_gather_values(&ctx->ac,
1449 values, 4);
1450 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, addr,
1451 base, 0, ac_glc);
1452 }
1453 }
1454
1455 LLVMValueRef si_llvm_load_input_gs(struct ac_shader_abi *abi,
1456 unsigned input_index,
1457 unsigned vtx_offset_param,
1458 LLVMTypeRef type,
1459 unsigned swizzle)
1460 {
1461 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1462 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
1463 struct si_shader *shader = ctx->shader;
1464 LLVMValueRef vtx_offset, soffset;
1465 struct tgsi_shader_info *info = &shader->selector->info;
1466 unsigned semantic_name = info->input_semantic_name[input_index];
1467 unsigned semantic_index = info->input_semantic_index[input_index];
1468 unsigned param;
1469 LLVMValueRef value;
1470
1471 param = si_shader_io_get_unique_index(semantic_name, semantic_index, false);
1472
1473 /* GFX9 has the ESGS ring in LDS. */
1474 if (ctx->screen->info.chip_class >= GFX9) {
1475 unsigned index = vtx_offset_param;
1476
1477 switch (index / 2) {
1478 case 0:
1479 vtx_offset = si_unpack_param(ctx, ctx->gs_vtx01_offset,
1480 index % 2 ? 16 : 0, 16);
1481 break;
1482 case 1:
1483 vtx_offset = si_unpack_param(ctx, ctx->gs_vtx23_offset,
1484 index % 2 ? 16 : 0, 16);
1485 break;
1486 case 2:
1487 vtx_offset = si_unpack_param(ctx, ctx->gs_vtx45_offset,
1488 index % 2 ? 16 : 0, 16);
1489 break;
1490 default:
1491 assert(0);
1492 return NULL;
1493 }
1494
1495 unsigned offset = param * 4 + swizzle;
1496 vtx_offset = LLVMBuildAdd(ctx->ac.builder, vtx_offset,
1497 LLVMConstInt(ctx->i32, offset, false), "");
1498
1499 LLVMValueRef ptr = ac_build_gep0(&ctx->ac, ctx->esgs_ring, vtx_offset);
1500 LLVMValueRef value = LLVMBuildLoad(ctx->ac.builder, ptr, "");
1501 if (llvm_type_is_64bit(ctx, type)) {
1502 ptr = LLVMBuildGEP(ctx->ac.builder, ptr,
1503 &ctx->ac.i32_1, 1, "");
1504 LLVMValueRef values[2] = {
1505 value,
1506 LLVMBuildLoad(ctx->ac.builder, ptr, "")
1507 };
1508 value = ac_build_gather_values(&ctx->ac, values, 2);
1509 }
1510 return LLVMBuildBitCast(ctx->ac.builder, value, type, "");
1511 }
1512
1513 /* GFX6: input load from the ESGS ring in memory. */
1514 if (swizzle == ~0) {
1515 LLVMValueRef values[TGSI_NUM_CHANNELS];
1516 unsigned chan;
1517 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1518 values[chan] = si_llvm_load_input_gs(abi, input_index, vtx_offset_param,
1519 type, chan);
1520 }
1521 return ac_build_gather_values(&ctx->ac, values,
1522 TGSI_NUM_CHANNELS);
1523 }
1524
1525 /* Get the vertex offset parameter on GFX6. */
1526 LLVMValueRef gs_vtx_offset = ac_get_arg(&ctx->ac,
1527 ctx->gs_vtx_offset[vtx_offset_param]);
1528
1529 vtx_offset = LLVMBuildMul(ctx->ac.builder, gs_vtx_offset,
1530 LLVMConstInt(ctx->i32, 4, 0), "");
1531
1532 soffset = LLVMConstInt(ctx->i32, (param * 4 + swizzle) * 256, 0);
1533
1534 value = ac_build_buffer_load(&ctx->ac, ctx->esgs_ring, 1, ctx->i32_0,
1535 vtx_offset, soffset, 0, ac_glc, true, false);
1536 if (llvm_type_is_64bit(ctx, type)) {
1537 LLVMValueRef value2;
1538 soffset = LLVMConstInt(ctx->i32, (param * 4 + swizzle + 1) * 256, 0);
1539
1540 value2 = ac_build_buffer_load(&ctx->ac, ctx->esgs_ring, 1,
1541 ctx->i32_0, vtx_offset, soffset,
1542 0, ac_glc, true, false);
1543 return si_llvm_emit_fetch_64bit(bld_base, type, value, value2);
1544 }
1545 return LLVMBuildBitCast(ctx->ac.builder, value, type, "");
1546 }
1547
1548 static LLVMValueRef si_nir_load_input_gs(struct ac_shader_abi *abi,
1549 unsigned location,
1550 unsigned driver_location,
1551 unsigned component,
1552 unsigned num_components,
1553 unsigned vertex_index,
1554 unsigned const_index,
1555 LLVMTypeRef type)
1556 {
1557 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1558
1559 LLVMValueRef value[4];
1560 for (unsigned i = 0; i < num_components; i++) {
1561 unsigned offset = i;
1562 if (llvm_type_is_64bit(ctx, type))
1563 offset *= 2;
1564
1565 offset += component;
1566 value[i + component] = si_llvm_load_input_gs(&ctx->abi, driver_location / 4 + const_index,
1567 vertex_index, type, offset);
1568 }
1569
1570 return ac_build_varying_gather_values(&ctx->ac, value, num_components, component);
1571 }
1572
1573 static LLVMValueRef fetch_input_gs(
1574 struct lp_build_tgsi_context *bld_base,
1575 const struct tgsi_full_src_register *reg,
1576 enum tgsi_opcode_type type,
1577 unsigned swizzle_in)
1578 {
1579 struct si_shader_context *ctx = si_shader_context(bld_base);
1580 struct tgsi_shader_info *info = &ctx->shader->selector->info;
1581 unsigned swizzle = swizzle_in & 0xffff;
1582
1583 unsigned semantic_name = info->input_semantic_name[reg->Register.Index];
1584 if (swizzle != ~0 && semantic_name == TGSI_SEMANTIC_PRIMID)
1585 return si_get_primitive_id(ctx, swizzle);
1586
1587 if (!reg->Register.Dimension)
1588 return NULL;
1589
1590 return si_llvm_load_input_gs(&ctx->abi, reg->Register.Index,
1591 reg->Dimension.Index,
1592 tgsi2llvmtype(bld_base, type),
1593 swizzle);
1594 }
1595
1596 static int lookup_interp_param_index(unsigned interpolate, unsigned location)
1597 {
1598 switch (interpolate) {
1599 case TGSI_INTERPOLATE_CONSTANT:
1600 return 0;
1601
1602 case TGSI_INTERPOLATE_LINEAR:
1603 if (location == TGSI_INTERPOLATE_LOC_SAMPLE)
1604 return SI_PARAM_LINEAR_SAMPLE;
1605 else if (location == TGSI_INTERPOLATE_LOC_CENTROID)
1606 return SI_PARAM_LINEAR_CENTROID;
1607 else
1608 return SI_PARAM_LINEAR_CENTER;
1609 break;
1610 case TGSI_INTERPOLATE_COLOR:
1611 case TGSI_INTERPOLATE_PERSPECTIVE:
1612 if (location == TGSI_INTERPOLATE_LOC_SAMPLE)
1613 return SI_PARAM_PERSP_SAMPLE;
1614 else if (location == TGSI_INTERPOLATE_LOC_CENTROID)
1615 return SI_PARAM_PERSP_CENTROID;
1616 else
1617 return SI_PARAM_PERSP_CENTER;
1618 break;
1619 default:
1620 fprintf(stderr, "Warning: Unhandled interpolation mode.\n");
1621 return -1;
1622 }
1623 }
1624
1625 static LLVMValueRef si_build_fs_interp(struct si_shader_context *ctx,
1626 unsigned attr_index, unsigned chan,
1627 LLVMValueRef prim_mask,
1628 LLVMValueRef i, LLVMValueRef j)
1629 {
1630 if (i || j) {
1631 return ac_build_fs_interp(&ctx->ac,
1632 LLVMConstInt(ctx->i32, chan, 0),
1633 LLVMConstInt(ctx->i32, attr_index, 0),
1634 prim_mask, i, j);
1635 }
1636 return ac_build_fs_interp_mov(&ctx->ac,
1637 LLVMConstInt(ctx->i32, 2, 0), /* P0 */
1638 LLVMConstInt(ctx->i32, chan, 0),
1639 LLVMConstInt(ctx->i32, attr_index, 0),
1640 prim_mask);
1641 }
1642
1643 /**
1644 * Interpolate a fragment shader input.
1645 *
1646 * @param ctx context
1647 * @param input_index index of the input in hardware
1648 * @param semantic_name TGSI_SEMANTIC_*
1649 * @param semantic_index semantic index
1650 * @param num_interp_inputs number of all interpolated inputs (= BCOLOR offset)
1651 * @param colors_read_mask color components read (4 bits for each color, 8 bits in total)
1652 * @param interp_param interpolation weights (i,j)
1653 * @param prim_mask SI_PARAM_PRIM_MASK
1654 * @param face SI_PARAM_FRONT_FACE
1655 * @param result the return value (4 components)
1656 */
1657 static void interp_fs_input(struct si_shader_context *ctx,
1658 unsigned input_index,
1659 unsigned semantic_name,
1660 unsigned semantic_index,
1661 unsigned num_interp_inputs,
1662 unsigned colors_read_mask,
1663 LLVMValueRef interp_param,
1664 LLVMValueRef prim_mask,
1665 LLVMValueRef face,
1666 LLVMValueRef result[4])
1667 {
1668 LLVMValueRef i = NULL, j = NULL;
1669 unsigned chan;
1670
1671 /* fs.constant returns the param from the middle vertex, so it's not
1672 * really useful for flat shading. It's meant to be used for custom
1673 * interpolation (but the intrinsic can't fetch from the other two
1674 * vertices).
1675 *
1676 * Luckily, it doesn't matter, because we rely on the FLAT_SHADE state
1677 * to do the right thing. The only reason we use fs.constant is that
1678 * fs.interp cannot be used on integers, because they can be equal
1679 * to NaN.
1680 *
1681 * When interp is false we will use fs.constant or for newer llvm,
1682 * amdgcn.interp.mov.
1683 */
1684 bool interp = interp_param != NULL;
1685
1686 if (interp) {
1687 interp_param = LLVMBuildBitCast(ctx->ac.builder, interp_param,
1688 LLVMVectorType(ctx->f32, 2), "");
1689
1690 i = LLVMBuildExtractElement(ctx->ac.builder, interp_param,
1691 ctx->i32_0, "");
1692 j = LLVMBuildExtractElement(ctx->ac.builder, interp_param,
1693 ctx->i32_1, "");
1694 }
1695
1696 if (semantic_name == TGSI_SEMANTIC_COLOR &&
1697 ctx->shader->key.part.ps.prolog.color_two_side) {
1698 LLVMValueRef is_face_positive;
1699
1700 /* If BCOLOR0 is used, BCOLOR1 is at offset "num_inputs + 1",
1701 * otherwise it's at offset "num_inputs".
1702 */
1703 unsigned back_attr_offset = num_interp_inputs;
1704 if (semantic_index == 1 && colors_read_mask & 0xf)
1705 back_attr_offset += 1;
1706
1707 is_face_positive = LLVMBuildICmp(ctx->ac.builder, LLVMIntNE,
1708 face, ctx->i32_0, "");
1709
1710 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1711 LLVMValueRef front, back;
1712
1713 front = si_build_fs_interp(ctx,
1714 input_index, chan,
1715 prim_mask, i, j);
1716 back = si_build_fs_interp(ctx,
1717 back_attr_offset, chan,
1718 prim_mask, i, j);
1719
1720 result[chan] = LLVMBuildSelect(ctx->ac.builder,
1721 is_face_positive,
1722 front,
1723 back,
1724 "");
1725 }
1726 } else if (semantic_name == TGSI_SEMANTIC_FOG) {
1727 result[0] = si_build_fs_interp(ctx, input_index,
1728 0, prim_mask, i, j);
1729 result[1] =
1730 result[2] = LLVMConstReal(ctx->f32, 0.0f);
1731 result[3] = LLVMConstReal(ctx->f32, 1.0f);
1732 } else {
1733 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
1734 result[chan] = si_build_fs_interp(ctx,
1735 input_index, chan,
1736 prim_mask, i, j);
1737 }
1738 }
1739 }
1740
1741 void si_llvm_load_input_fs(
1742 struct si_shader_context *ctx,
1743 unsigned input_index,
1744 LLVMValueRef out[4])
1745 {
1746 struct si_shader *shader = ctx->shader;
1747 struct tgsi_shader_info *info = &shader->selector->info;
1748 LLVMValueRef main_fn = ctx->main_fn;
1749 LLVMValueRef interp_param = NULL;
1750 int interp_param_idx;
1751 enum tgsi_semantic semantic_name = info->input_semantic_name[input_index];
1752 unsigned semantic_index = info->input_semantic_index[input_index];
1753 enum tgsi_interpolate_mode interp_mode = info->input_interpolate[input_index];
1754 enum tgsi_interpolate_loc interp_loc = info->input_interpolate_loc[input_index];
1755
1756 /* Get colors from input VGPRs (set by the prolog). */
1757 if (semantic_name == TGSI_SEMANTIC_COLOR) {
1758 unsigned colors_read = shader->selector->info.colors_read;
1759 unsigned mask = colors_read >> (semantic_index * 4);
1760 unsigned offset = SI_PARAM_POS_FIXED_PT + 1 +
1761 (semantic_index ? util_bitcount(colors_read & 0xf) : 0);
1762 LLVMValueRef undef = LLVMGetUndef(ctx->f32);
1763
1764 out[0] = mask & 0x1 ? LLVMGetParam(main_fn, offset++) : undef;
1765 out[1] = mask & 0x2 ? LLVMGetParam(main_fn, offset++) : undef;
1766 out[2] = mask & 0x4 ? LLVMGetParam(main_fn, offset++) : undef;
1767 out[3] = mask & 0x8 ? LLVMGetParam(main_fn, offset++) : undef;
1768 return;
1769 }
1770
1771 interp_param_idx = lookup_interp_param_index(interp_mode, interp_loc);
1772 if (interp_param_idx == -1)
1773 return;
1774 else if (interp_param_idx) {
1775 interp_param = LLVMGetParam(ctx->main_fn, interp_param_idx);
1776 }
1777
1778 interp_fs_input(ctx, input_index, semantic_name,
1779 semantic_index, 0, /* this param is unused */
1780 shader->selector->info.colors_read, interp_param,
1781 ac_get_arg(&ctx->ac, ctx->args.prim_mask),
1782 LLVMGetParam(main_fn, SI_PARAM_FRONT_FACE),
1783 &out[0]);
1784 }
1785
1786 static void declare_input_fs(
1787 struct si_shader_context *ctx,
1788 unsigned input_index,
1789 const struct tgsi_full_declaration *decl,
1790 LLVMValueRef out[4])
1791 {
1792 si_llvm_load_input_fs(ctx, input_index, out);
1793 }
1794
1795 LLVMValueRef si_get_sample_id(struct si_shader_context *ctx)
1796 {
1797 return si_unpack_param(ctx, ctx->args.ancillary, 8, 4);
1798 }
1799
1800 static LLVMValueRef get_base_vertex(struct ac_shader_abi *abi)
1801 {
1802 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1803
1804 /* For non-indexed draws, the base vertex set by the driver
1805 * (for direct draws) or the CP (for indirect draws) is the
1806 * first vertex ID, but GLSL expects 0 to be returned.
1807 */
1808 LLVMValueRef vs_state = ac_get_arg(&ctx->ac,
1809 ctx->vs_state_bits);
1810 LLVMValueRef indexed;
1811
1812 indexed = LLVMBuildLShr(ctx->ac.builder, vs_state, ctx->i32_1, "");
1813 indexed = LLVMBuildTrunc(ctx->ac.builder, indexed, ctx->i1, "");
1814
1815 return LLVMBuildSelect(ctx->ac.builder, indexed,
1816 ac_get_arg(&ctx->ac, ctx->args.base_vertex),
1817 ctx->i32_0, "");
1818 }
1819
1820 static LLVMValueRef get_block_size(struct ac_shader_abi *abi)
1821 {
1822 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1823
1824 LLVMValueRef values[3];
1825 LLVMValueRef result;
1826 unsigned i;
1827 unsigned *properties = ctx->shader->selector->info.properties;
1828
1829 if (properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH] != 0) {
1830 unsigned sizes[3] = {
1831 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH],
1832 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT],
1833 properties[TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH]
1834 };
1835
1836 for (i = 0; i < 3; ++i)
1837 values[i] = LLVMConstInt(ctx->i32, sizes[i], 0);
1838
1839 result = ac_build_gather_values(&ctx->ac, values, 3);
1840 } else {
1841 result = ac_get_arg(&ctx->ac, ctx->block_size);
1842 }
1843
1844 return result;
1845 }
1846
1847 /**
1848 * Load a dword from a constant buffer.
1849 */
1850 static LLVMValueRef buffer_load_const(struct si_shader_context *ctx,
1851 LLVMValueRef resource,
1852 LLVMValueRef offset)
1853 {
1854 return ac_build_buffer_load(&ctx->ac, resource, 1, NULL, offset, NULL,
1855 0, 0, true, true);
1856 }
1857
1858 static LLVMValueRef load_sample_position(struct ac_shader_abi *abi, LLVMValueRef sample_id)
1859 {
1860 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1861 LLVMValueRef desc = ac_get_arg(&ctx->ac, ctx->rw_buffers);
1862 LLVMValueRef buf_index = LLVMConstInt(ctx->i32, SI_PS_CONST_SAMPLE_POSITIONS, 0);
1863 LLVMValueRef resource = ac_build_load_to_sgpr(&ctx->ac, desc, buf_index);
1864
1865 /* offset = sample_id * 8 (8 = 2 floats containing samplepos.xy) */
1866 LLVMValueRef offset0 = LLVMBuildMul(ctx->ac.builder, sample_id, LLVMConstInt(ctx->i32, 8, 0), "");
1867 LLVMValueRef offset1 = LLVMBuildAdd(ctx->ac.builder, offset0, LLVMConstInt(ctx->i32, 4, 0), "");
1868
1869 LLVMValueRef pos[4] = {
1870 buffer_load_const(ctx, resource, offset0),
1871 buffer_load_const(ctx, resource, offset1),
1872 LLVMConstReal(ctx->f32, 0),
1873 LLVMConstReal(ctx->f32, 0)
1874 };
1875
1876 return ac_build_gather_values(&ctx->ac, pos, 4);
1877 }
1878
1879 static LLVMValueRef load_sample_mask_in(struct ac_shader_abi *abi)
1880 {
1881 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1882 return ac_to_integer(&ctx->ac, ac_get_arg(&ctx->ac, ctx->args.sample_coverage));
1883 }
1884
1885 static LLVMValueRef si_load_tess_coord(struct ac_shader_abi *abi)
1886 {
1887 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1888 LLVMValueRef coord[4] = {
1889 ac_get_arg(&ctx->ac, ctx->tes_u),
1890 ac_get_arg(&ctx->ac, ctx->tes_v),
1891 ctx->ac.f32_0,
1892 ctx->ac.f32_0
1893 };
1894
1895 /* For triangles, the vector should be (u, v, 1-u-v). */
1896 if (ctx->shader->selector->info.properties[TGSI_PROPERTY_TES_PRIM_MODE] ==
1897 PIPE_PRIM_TRIANGLES) {
1898 coord[2] = LLVMBuildFSub(ctx->ac.builder, ctx->ac.f32_1,
1899 LLVMBuildFAdd(ctx->ac.builder,
1900 coord[0], coord[1], ""), "");
1901 }
1902 return ac_build_gather_values(&ctx->ac, coord, 4);
1903 }
1904
1905 static LLVMValueRef load_tess_level(struct si_shader_context *ctx,
1906 unsigned semantic_name)
1907 {
1908 LLVMValueRef base, addr;
1909
1910 int param = si_shader_io_get_unique_index_patch(semantic_name, 0);
1911
1912 base = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
1913 addr = get_tcs_tes_buffer_address(ctx, get_rel_patch_id(ctx), NULL,
1914 LLVMConstInt(ctx->i32, param, 0));
1915
1916 return buffer_load(&ctx->bld_base, ctx->f32,
1917 ~0, ctx->tess_offchip_ring, base, addr, true);
1918
1919 }
1920
1921 static LLVMValueRef load_tess_level_default(struct si_shader_context *ctx,
1922 unsigned semantic_name)
1923 {
1924 LLVMValueRef buf, slot, val[4];
1925 int i, offset;
1926
1927 slot = LLVMConstInt(ctx->i32, SI_HS_CONST_DEFAULT_TESS_LEVELS, 0);
1928 buf = ac_get_arg(&ctx->ac, ctx->rw_buffers);
1929 buf = ac_build_load_to_sgpr(&ctx->ac, buf, slot);
1930 offset = semantic_name == TGSI_SEMANTIC_TESS_DEFAULT_INNER_LEVEL ? 4 : 0;
1931
1932 for (i = 0; i < 4; i++)
1933 val[i] = buffer_load_const(ctx, buf,
1934 LLVMConstInt(ctx->i32, (offset + i) * 4, 0));
1935 return ac_build_gather_values(&ctx->ac, val, 4);
1936 }
1937
1938 static LLVMValueRef si_load_tess_level(struct ac_shader_abi *abi,
1939 unsigned varying_id,
1940 bool load_default_state)
1941 {
1942 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1943 unsigned semantic_name;
1944
1945 if (load_default_state) {
1946 switch (varying_id) {
1947 case VARYING_SLOT_TESS_LEVEL_INNER:
1948 semantic_name = TGSI_SEMANTIC_TESS_DEFAULT_INNER_LEVEL;
1949 break;
1950 case VARYING_SLOT_TESS_LEVEL_OUTER:
1951 semantic_name = TGSI_SEMANTIC_TESS_DEFAULT_OUTER_LEVEL;
1952 break;
1953 default:
1954 unreachable("unknown tess level");
1955 }
1956 return load_tess_level_default(ctx, semantic_name);
1957 }
1958
1959 switch (varying_id) {
1960 case VARYING_SLOT_TESS_LEVEL_INNER:
1961 semantic_name = TGSI_SEMANTIC_TESSINNER;
1962 break;
1963 case VARYING_SLOT_TESS_LEVEL_OUTER:
1964 semantic_name = TGSI_SEMANTIC_TESSOUTER;
1965 break;
1966 default:
1967 unreachable("unknown tess level");
1968 }
1969
1970 return load_tess_level(ctx, semantic_name);
1971
1972 }
1973
1974 static LLVMValueRef si_load_patch_vertices_in(struct ac_shader_abi *abi)
1975 {
1976 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
1977 if (ctx->type == PIPE_SHADER_TESS_CTRL)
1978 return si_unpack_param(ctx, ctx->tcs_out_lds_layout, 13, 6);
1979 else if (ctx->type == PIPE_SHADER_TESS_EVAL)
1980 return get_num_tcs_out_vertices(ctx);
1981 else
1982 unreachable("invalid shader stage for TGSI_SEMANTIC_VERTICESIN");
1983 }
1984
1985 void si_load_system_value(struct si_shader_context *ctx,
1986 unsigned index,
1987 const struct tgsi_full_declaration *decl)
1988 {
1989 LLVMValueRef value = 0;
1990
1991 assert(index < RADEON_LLVM_MAX_SYSTEM_VALUES);
1992
1993 switch (decl->Semantic.Name) {
1994 case TGSI_SEMANTIC_INSTANCEID:
1995 value = ctx->abi.instance_id;
1996 break;
1997
1998 case TGSI_SEMANTIC_VERTEXID:
1999 value = LLVMBuildAdd(ctx->ac.builder,
2000 ctx->abi.vertex_id,
2001 ac_get_arg(&ctx->ac, ctx->args.base_vertex), "");
2002 break;
2003
2004 case TGSI_SEMANTIC_VERTEXID_NOBASE:
2005 /* Unused. Clarify the meaning in indexed vs. non-indexed
2006 * draws if this is ever used again. */
2007 assert(false);
2008 break;
2009
2010 case TGSI_SEMANTIC_BASEVERTEX:
2011 value = get_base_vertex(&ctx->abi);
2012 break;
2013
2014 case TGSI_SEMANTIC_BASEINSTANCE:
2015 value = ac_get_arg(&ctx->ac, ctx->args.start_instance);
2016 break;
2017
2018 case TGSI_SEMANTIC_DRAWID:
2019 value = ac_get_arg(&ctx->ac, ctx->args.draw_id);
2020 break;
2021
2022 case TGSI_SEMANTIC_INVOCATIONID:
2023 if (ctx->type == PIPE_SHADER_TESS_CTRL) {
2024 value = si_unpack_param(ctx, ctx->args.tcs_rel_ids, 8, 5);
2025 } else if (ctx->type == PIPE_SHADER_GEOMETRY) {
2026 if (ctx->screen->info.chip_class >= GFX10) {
2027 value = LLVMBuildAnd(ctx->ac.builder,
2028 ac_get_arg(&ctx->ac, ctx->args.gs_invocation_id),
2029 LLVMConstInt(ctx->i32, 127, 0), "");
2030 } else {
2031 value = ac_get_arg(&ctx->ac, ctx->args.gs_invocation_id);
2032 }
2033 } else {
2034 assert(!"INVOCATIONID not implemented");
2035 }
2036 break;
2037
2038 case TGSI_SEMANTIC_POSITION:
2039 {
2040 LLVMValueRef pos[4] = {
2041 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_X_FLOAT),
2042 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Y_FLOAT),
2043 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Z_FLOAT),
2044 ac_build_fdiv(&ctx->ac, ctx->ac.f32_1,
2045 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_W_FLOAT)),
2046 };
2047 value = ac_build_gather_values(&ctx->ac, pos, 4);
2048 break;
2049 }
2050
2051 case TGSI_SEMANTIC_FACE:
2052 value = ac_get_arg(&ctx->ac, ctx->args.front_face);
2053 break;
2054
2055 case TGSI_SEMANTIC_SAMPLEID:
2056 value = si_get_sample_id(ctx);
2057 break;
2058
2059 case TGSI_SEMANTIC_SAMPLEPOS: {
2060 LLVMValueRef pos[4] = {
2061 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_X_FLOAT),
2062 LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Y_FLOAT),
2063 LLVMConstReal(ctx->f32, 0),
2064 LLVMConstReal(ctx->f32, 0)
2065 };
2066 pos[0] = ac_build_fract(&ctx->ac, pos[0], 32);
2067 pos[1] = ac_build_fract(&ctx->ac, pos[1], 32);
2068 value = ac_build_gather_values(&ctx->ac, pos, 4);
2069 break;
2070 }
2071
2072 case TGSI_SEMANTIC_SAMPLEMASK:
2073 /* This can only occur with the OpenGL Core profile, which
2074 * doesn't support smoothing.
2075 */
2076 value = LLVMGetParam(ctx->main_fn, SI_PARAM_SAMPLE_COVERAGE);
2077 break;
2078
2079 case TGSI_SEMANTIC_TESSCOORD:
2080 value = si_load_tess_coord(&ctx->abi);
2081 break;
2082
2083 case TGSI_SEMANTIC_VERTICESIN:
2084 value = si_load_patch_vertices_in(&ctx->abi);
2085 break;
2086
2087 case TGSI_SEMANTIC_TESSINNER:
2088 case TGSI_SEMANTIC_TESSOUTER:
2089 value = load_tess_level(ctx, decl->Semantic.Name);
2090 break;
2091
2092 case TGSI_SEMANTIC_TESS_DEFAULT_OUTER_LEVEL:
2093 case TGSI_SEMANTIC_TESS_DEFAULT_INNER_LEVEL:
2094 value = load_tess_level_default(ctx, decl->Semantic.Name);
2095 break;
2096
2097 case TGSI_SEMANTIC_PRIMID:
2098 value = si_get_primitive_id(ctx, 0);
2099 break;
2100
2101 case TGSI_SEMANTIC_GRID_SIZE:
2102 value = ac_get_arg(&ctx->ac, ctx->args.num_work_groups);
2103 break;
2104
2105 case TGSI_SEMANTIC_BLOCK_SIZE:
2106 value = get_block_size(&ctx->abi);
2107 break;
2108
2109 case TGSI_SEMANTIC_BLOCK_ID:
2110 {
2111 LLVMValueRef values[3];
2112
2113 for (int i = 0; i < 3; i++) {
2114 values[i] = ctx->i32_0;
2115 if (ctx->args.workgroup_ids[i].used) {
2116 values[i] = ac_get_arg(&ctx->ac, ctx->args.workgroup_ids[i]);
2117 }
2118 }
2119 value = ac_build_gather_values(&ctx->ac, values, 3);
2120 break;
2121 }
2122
2123 case TGSI_SEMANTIC_THREAD_ID:
2124 value = ac_get_arg(&ctx->ac, ctx->args.local_invocation_ids);
2125 break;
2126
2127 case TGSI_SEMANTIC_HELPER_INVOCATION:
2128 value = ac_build_load_helper_invocation(&ctx->ac);
2129 break;
2130
2131 case TGSI_SEMANTIC_SUBGROUP_SIZE:
2132 value = LLVMConstInt(ctx->i32, ctx->ac.wave_size, 0);
2133 break;
2134
2135 case TGSI_SEMANTIC_SUBGROUP_INVOCATION:
2136 value = ac_get_thread_id(&ctx->ac);
2137 break;
2138
2139 case TGSI_SEMANTIC_SUBGROUP_EQ_MASK:
2140 {
2141 LLVMValueRef id = ac_get_thread_id(&ctx->ac);
2142 if (ctx->ac.wave_size == 64)
2143 id = LLVMBuildZExt(ctx->ac.builder, id, ctx->i64, "");
2144 value = LLVMBuildShl(ctx->ac.builder,
2145 LLVMConstInt(ctx->ac.iN_wavemask, 1, 0), id, "");
2146 if (ctx->ac.wave_size == 32)
2147 value = LLVMBuildZExt(ctx->ac.builder, value, ctx->i64, "");
2148 value = LLVMBuildBitCast(ctx->ac.builder, value, ctx->v2i32, "");
2149 break;
2150 }
2151
2152 case TGSI_SEMANTIC_SUBGROUP_GE_MASK:
2153 case TGSI_SEMANTIC_SUBGROUP_GT_MASK:
2154 case TGSI_SEMANTIC_SUBGROUP_LE_MASK:
2155 case TGSI_SEMANTIC_SUBGROUP_LT_MASK:
2156 {
2157 LLVMValueRef id = ac_get_thread_id(&ctx->ac);
2158 if (decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_GT_MASK ||
2159 decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_LE_MASK) {
2160 /* All bits set except LSB */
2161 value = LLVMConstInt(ctx->ac.iN_wavemask, -2, 0);
2162 } else {
2163 /* All bits set */
2164 value = LLVMConstInt(ctx->ac.iN_wavemask, -1, 0);
2165 }
2166 if (ctx->ac.wave_size == 64)
2167 id = LLVMBuildZExt(ctx->ac.builder, id, ctx->i64, "");
2168 value = LLVMBuildShl(ctx->ac.builder, value, id, "");
2169 if (decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_LE_MASK ||
2170 decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_LT_MASK)
2171 value = LLVMBuildNot(ctx->ac.builder, value, "");
2172 if (ctx->ac.wave_size == 32)
2173 value = LLVMBuildZExt(ctx->ac.builder, value, ctx->i64, "");
2174 value = LLVMBuildBitCast(ctx->ac.builder, value, ctx->v2i32, "");
2175 break;
2176 }
2177
2178 case TGSI_SEMANTIC_CS_USER_DATA_AMD:
2179 value = ac_get_arg(&ctx->ac, ctx->cs_user_data);
2180 break;
2181
2182 default:
2183 assert(!"unknown system value");
2184 return;
2185 }
2186
2187 ctx->system_values[index] = value;
2188 }
2189
2190 void si_declare_compute_memory(struct si_shader_context *ctx)
2191 {
2192 struct si_shader_selector *sel = ctx->shader->selector;
2193 unsigned lds_size = sel->info.properties[TGSI_PROPERTY_CS_LOCAL_SIZE];
2194
2195 LLVMTypeRef i8p = LLVMPointerType(ctx->i8, AC_ADDR_SPACE_LDS);
2196 LLVMValueRef var;
2197
2198 assert(!ctx->ac.lds);
2199
2200 var = LLVMAddGlobalInAddressSpace(ctx->ac.module,
2201 LLVMArrayType(ctx->i8, lds_size),
2202 "compute_lds",
2203 AC_ADDR_SPACE_LDS);
2204 LLVMSetAlignment(var, 64 * 1024);
2205
2206 ctx->ac.lds = LLVMBuildBitCast(ctx->ac.builder, var, i8p, "");
2207 }
2208
2209 void si_tgsi_declare_compute_memory(struct si_shader_context *ctx,
2210 const struct tgsi_full_declaration *decl)
2211 {
2212 assert(decl->Declaration.MemType == TGSI_MEMORY_TYPE_SHARED);
2213 assert(decl->Range.First == decl->Range.Last);
2214
2215 si_declare_compute_memory(ctx);
2216 }
2217
2218 static LLVMValueRef load_const_buffer_desc_fast_path(struct si_shader_context *ctx)
2219 {
2220 LLVMValueRef ptr =
2221 ac_get_arg(&ctx->ac, ctx->const_and_shader_buffers);
2222 struct si_shader_selector *sel = ctx->shader->selector;
2223
2224 /* Do the bounds checking with a descriptor, because
2225 * doing computation and manual bounds checking of 64-bit
2226 * addresses generates horrible VALU code with very high
2227 * VGPR usage and very low SIMD occupancy.
2228 */
2229 ptr = LLVMBuildPtrToInt(ctx->ac.builder, ptr, ctx->ac.intptr, "");
2230
2231 LLVMValueRef desc0, desc1;
2232 desc0 = ptr;
2233 desc1 = LLVMConstInt(ctx->i32,
2234 S_008F04_BASE_ADDRESS_HI(ctx->screen->info.address32_hi), 0);
2235
2236 uint32_t rsrc3 = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
2237 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
2238 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
2239 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
2240
2241 if (ctx->screen->info.chip_class >= GFX10)
2242 rsrc3 |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
2243 S_008F0C_OOB_SELECT(3) |
2244 S_008F0C_RESOURCE_LEVEL(1);
2245 else
2246 rsrc3 |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
2247 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
2248
2249 LLVMValueRef desc_elems[] = {
2250 desc0,
2251 desc1,
2252 LLVMConstInt(ctx->i32, (sel->info.const_file_max[0] + 1) * 16, 0),
2253 LLVMConstInt(ctx->i32, rsrc3, false)
2254 };
2255
2256 return ac_build_gather_values(&ctx->ac, desc_elems, 4);
2257 }
2258
2259 static LLVMValueRef load_const_buffer_desc(struct si_shader_context *ctx, int i)
2260 {
2261 LLVMValueRef list_ptr = ac_get_arg(&ctx->ac,
2262 ctx->const_and_shader_buffers);
2263
2264 return ac_build_load_to_sgpr(&ctx->ac, list_ptr,
2265 LLVMConstInt(ctx->i32, si_get_constbuf_slot(i), 0));
2266 }
2267
2268 static LLVMValueRef load_ubo(struct ac_shader_abi *abi, LLVMValueRef index)
2269 {
2270 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
2271 struct si_shader_selector *sel = ctx->shader->selector;
2272
2273 LLVMValueRef ptr = ac_get_arg(&ctx->ac, ctx->const_and_shader_buffers);
2274
2275 if (sel->info.const_buffers_declared == 1 &&
2276 sel->info.shader_buffers_declared == 0) {
2277 return load_const_buffer_desc_fast_path(ctx);
2278 }
2279
2280 index = si_llvm_bound_index(ctx, index, ctx->num_const_buffers);
2281 index = LLVMBuildAdd(ctx->ac.builder, index,
2282 LLVMConstInt(ctx->i32, SI_NUM_SHADER_BUFFERS, 0), "");
2283
2284 return ac_build_load_to_sgpr(&ctx->ac, ptr, index);
2285 }
2286
2287 static LLVMValueRef
2288 load_ssbo(struct ac_shader_abi *abi, LLVMValueRef index, bool write)
2289 {
2290 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
2291 LLVMValueRef rsrc_ptr = ac_get_arg(&ctx->ac,
2292 ctx->const_and_shader_buffers);
2293
2294 index = si_llvm_bound_index(ctx, index, ctx->num_shader_buffers);
2295 index = LLVMBuildSub(ctx->ac.builder,
2296 LLVMConstInt(ctx->i32, SI_NUM_SHADER_BUFFERS - 1, 0),
2297 index, "");
2298
2299 return ac_build_load_to_sgpr(&ctx->ac, rsrc_ptr, index);
2300 }
2301
2302 static LLVMValueRef fetch_constant(
2303 struct lp_build_tgsi_context *bld_base,
2304 const struct tgsi_full_src_register *reg,
2305 enum tgsi_opcode_type type,
2306 unsigned swizzle_in)
2307 {
2308 struct si_shader_context *ctx = si_shader_context(bld_base);
2309 struct si_shader_selector *sel = ctx->shader->selector;
2310 const struct tgsi_ind_register *ireg = &reg->Indirect;
2311 unsigned buf, idx;
2312 unsigned swizzle = swizzle_in & 0xffff;
2313
2314 LLVMValueRef addr, bufp;
2315
2316 if (swizzle_in == LP_CHAN_ALL) {
2317 unsigned chan;
2318 LLVMValueRef values[4];
2319 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan)
2320 values[chan] = fetch_constant(bld_base, reg, type, chan);
2321
2322 return ac_build_gather_values(&ctx->ac, values, 4);
2323 }
2324
2325 /* Split 64-bit loads. */
2326 if (tgsi_type_is_64bit(type)) {
2327 LLVMValueRef lo, hi;
2328
2329 lo = fetch_constant(bld_base, reg, TGSI_TYPE_UNSIGNED, swizzle);
2330 hi = fetch_constant(bld_base, reg, TGSI_TYPE_UNSIGNED, (swizzle_in >> 16));
2331 return si_llvm_emit_fetch_64bit(bld_base, tgsi2llvmtype(bld_base, type),
2332 lo, hi);
2333 }
2334
2335 idx = reg->Register.Index * 4 + swizzle;
2336 if (reg->Register.Indirect) {
2337 addr = si_get_indirect_index(ctx, ireg, 16, idx * 4);
2338 } else {
2339 addr = LLVMConstInt(ctx->i32, idx * 4, 0);
2340 }
2341
2342 /* Fast path when user data SGPRs point to constant buffer 0 directly. */
2343 if (sel->info.const_buffers_declared == 1 &&
2344 sel->info.shader_buffers_declared == 0) {
2345 LLVMValueRef desc = load_const_buffer_desc_fast_path(ctx);
2346 LLVMValueRef result = buffer_load_const(ctx, desc, addr);
2347 return bitcast(bld_base, type, result);
2348 }
2349
2350 assert(reg->Register.Dimension);
2351 buf = reg->Dimension.Index;
2352
2353 if (reg->Dimension.Indirect) {
2354 LLVMValueRef ptr = ac_get_arg(&ctx->ac, ctx->const_and_shader_buffers);
2355 LLVMValueRef index;
2356 index = si_get_bounded_indirect_index(ctx, &reg->DimIndirect,
2357 reg->Dimension.Index,
2358 ctx->num_const_buffers);
2359 index = LLVMBuildAdd(ctx->ac.builder, index,
2360 LLVMConstInt(ctx->i32, SI_NUM_SHADER_BUFFERS, 0), "");
2361 bufp = ac_build_load_to_sgpr(&ctx->ac, ptr, index);
2362 } else
2363 bufp = load_const_buffer_desc(ctx, buf);
2364
2365 return bitcast(bld_base, type, buffer_load_const(ctx, bufp, addr));
2366 }
2367
2368 /* Initialize arguments for the shader export intrinsic */
2369 static void si_llvm_init_export_args(struct si_shader_context *ctx,
2370 LLVMValueRef *values,
2371 unsigned target,
2372 struct ac_export_args *args)
2373 {
2374 LLVMValueRef f32undef = LLVMGetUndef(ctx->ac.f32);
2375 unsigned spi_shader_col_format = V_028714_SPI_SHADER_32_ABGR;
2376 unsigned chan;
2377 bool is_int8, is_int10;
2378
2379 /* Default is 0xf. Adjusted below depending on the format. */
2380 args->enabled_channels = 0xf; /* writemask */
2381
2382 /* Specify whether the EXEC mask represents the valid mask */
2383 args->valid_mask = 0;
2384
2385 /* Specify whether this is the last export */
2386 args->done = 0;
2387
2388 /* Specify the target we are exporting */
2389 args->target = target;
2390
2391 if (ctx->type == PIPE_SHADER_FRAGMENT) {
2392 const struct si_shader_key *key = &ctx->shader->key;
2393 unsigned col_formats = key->part.ps.epilog.spi_shader_col_format;
2394 int cbuf = target - V_008DFC_SQ_EXP_MRT;
2395
2396 assert(cbuf >= 0 && cbuf < 8);
2397 spi_shader_col_format = (col_formats >> (cbuf * 4)) & 0xf;
2398 is_int8 = (key->part.ps.epilog.color_is_int8 >> cbuf) & 0x1;
2399 is_int10 = (key->part.ps.epilog.color_is_int10 >> cbuf) & 0x1;
2400 }
2401
2402 args->compr = false;
2403 args->out[0] = f32undef;
2404 args->out[1] = f32undef;
2405 args->out[2] = f32undef;
2406 args->out[3] = f32undef;
2407
2408 LLVMValueRef (*packf)(struct ac_llvm_context *ctx, LLVMValueRef args[2]) = NULL;
2409 LLVMValueRef (*packi)(struct ac_llvm_context *ctx, LLVMValueRef args[2],
2410 unsigned bits, bool hi) = NULL;
2411
2412 switch (spi_shader_col_format) {
2413 case V_028714_SPI_SHADER_ZERO:
2414 args->enabled_channels = 0; /* writemask */
2415 args->target = V_008DFC_SQ_EXP_NULL;
2416 break;
2417
2418 case V_028714_SPI_SHADER_32_R:
2419 args->enabled_channels = 1; /* writemask */
2420 args->out[0] = values[0];
2421 break;
2422
2423 case V_028714_SPI_SHADER_32_GR:
2424 args->enabled_channels = 0x3; /* writemask */
2425 args->out[0] = values[0];
2426 args->out[1] = values[1];
2427 break;
2428
2429 case V_028714_SPI_SHADER_32_AR:
2430 if (ctx->screen->info.chip_class >= GFX10) {
2431 args->enabled_channels = 0x3; /* writemask */
2432 args->out[0] = values[0];
2433 args->out[1] = values[3];
2434 } else {
2435 args->enabled_channels = 0x9; /* writemask */
2436 args->out[0] = values[0];
2437 args->out[3] = values[3];
2438 }
2439 break;
2440
2441 case V_028714_SPI_SHADER_FP16_ABGR:
2442 packf = ac_build_cvt_pkrtz_f16;
2443 break;
2444
2445 case V_028714_SPI_SHADER_UNORM16_ABGR:
2446 packf = ac_build_cvt_pknorm_u16;
2447 break;
2448
2449 case V_028714_SPI_SHADER_SNORM16_ABGR:
2450 packf = ac_build_cvt_pknorm_i16;
2451 break;
2452
2453 case V_028714_SPI_SHADER_UINT16_ABGR:
2454 packi = ac_build_cvt_pk_u16;
2455 break;
2456
2457 case V_028714_SPI_SHADER_SINT16_ABGR:
2458 packi = ac_build_cvt_pk_i16;
2459 break;
2460
2461 case V_028714_SPI_SHADER_32_ABGR:
2462 memcpy(&args->out[0], values, sizeof(values[0]) * 4);
2463 break;
2464 }
2465
2466 /* Pack f16 or norm_i16/u16. */
2467 if (packf) {
2468 for (chan = 0; chan < 2; chan++) {
2469 LLVMValueRef pack_args[2] = {
2470 values[2 * chan],
2471 values[2 * chan + 1]
2472 };
2473 LLVMValueRef packed;
2474
2475 packed = packf(&ctx->ac, pack_args);
2476 args->out[chan] = ac_to_float(&ctx->ac, packed);
2477 }
2478 args->compr = 1; /* COMPR flag */
2479 }
2480 /* Pack i16/u16. */
2481 if (packi) {
2482 for (chan = 0; chan < 2; chan++) {
2483 LLVMValueRef pack_args[2] = {
2484 ac_to_integer(&ctx->ac, values[2 * chan]),
2485 ac_to_integer(&ctx->ac, values[2 * chan + 1])
2486 };
2487 LLVMValueRef packed;
2488
2489 packed = packi(&ctx->ac, pack_args,
2490 is_int8 ? 8 : is_int10 ? 10 : 16,
2491 chan == 1);
2492 args->out[chan] = ac_to_float(&ctx->ac, packed);
2493 }
2494 args->compr = 1; /* COMPR flag */
2495 }
2496 }
2497
2498 static void si_alpha_test(struct lp_build_tgsi_context *bld_base,
2499 LLVMValueRef alpha)
2500 {
2501 struct si_shader_context *ctx = si_shader_context(bld_base);
2502
2503 if (ctx->shader->key.part.ps.epilog.alpha_func != PIPE_FUNC_NEVER) {
2504 static LLVMRealPredicate cond_map[PIPE_FUNC_ALWAYS + 1] = {
2505 [PIPE_FUNC_LESS] = LLVMRealOLT,
2506 [PIPE_FUNC_EQUAL] = LLVMRealOEQ,
2507 [PIPE_FUNC_LEQUAL] = LLVMRealOLE,
2508 [PIPE_FUNC_GREATER] = LLVMRealOGT,
2509 [PIPE_FUNC_NOTEQUAL] = LLVMRealONE,
2510 [PIPE_FUNC_GEQUAL] = LLVMRealOGE,
2511 };
2512 LLVMRealPredicate cond = cond_map[ctx->shader->key.part.ps.epilog.alpha_func];
2513 assert(cond);
2514
2515 LLVMValueRef alpha_ref = LLVMGetParam(ctx->main_fn,
2516 SI_PARAM_ALPHA_REF);
2517 LLVMValueRef alpha_pass =
2518 LLVMBuildFCmp(ctx->ac.builder, cond, alpha, alpha_ref, "");
2519 ac_build_kill_if_false(&ctx->ac, alpha_pass);
2520 } else {
2521 ac_build_kill_if_false(&ctx->ac, ctx->i1false);
2522 }
2523 }
2524
2525 static LLVMValueRef si_scale_alpha_by_sample_mask(struct lp_build_tgsi_context *bld_base,
2526 LLVMValueRef alpha,
2527 unsigned samplemask_param)
2528 {
2529 struct si_shader_context *ctx = si_shader_context(bld_base);
2530 LLVMValueRef coverage;
2531
2532 /* alpha = alpha * popcount(coverage) / SI_NUM_SMOOTH_AA_SAMPLES */
2533 coverage = LLVMGetParam(ctx->main_fn,
2534 samplemask_param);
2535 coverage = ac_to_integer(&ctx->ac, coverage);
2536
2537 coverage = ac_build_intrinsic(&ctx->ac, "llvm.ctpop.i32",
2538 ctx->i32,
2539 &coverage, 1, AC_FUNC_ATTR_READNONE);
2540
2541 coverage = LLVMBuildUIToFP(ctx->ac.builder, coverage,
2542 ctx->f32, "");
2543
2544 coverage = LLVMBuildFMul(ctx->ac.builder, coverage,
2545 LLVMConstReal(ctx->f32,
2546 1.0 / SI_NUM_SMOOTH_AA_SAMPLES), "");
2547
2548 return LLVMBuildFMul(ctx->ac.builder, alpha, coverage, "");
2549 }
2550
2551 static void si_llvm_emit_clipvertex(struct si_shader_context *ctx,
2552 struct ac_export_args *pos, LLVMValueRef *out_elts)
2553 {
2554 unsigned reg_index;
2555 unsigned chan;
2556 unsigned const_chan;
2557 LLVMValueRef base_elt;
2558 LLVMValueRef ptr = ac_get_arg(&ctx->ac, ctx->rw_buffers);
2559 LLVMValueRef constbuf_index = LLVMConstInt(ctx->i32,
2560 SI_VS_CONST_CLIP_PLANES, 0);
2561 LLVMValueRef const_resource = ac_build_load_to_sgpr(&ctx->ac, ptr, constbuf_index);
2562
2563 for (reg_index = 0; reg_index < 2; reg_index ++) {
2564 struct ac_export_args *args = &pos[2 + reg_index];
2565
2566 args->out[0] =
2567 args->out[1] =
2568 args->out[2] =
2569 args->out[3] = LLVMConstReal(ctx->f32, 0.0f);
2570
2571 /* Compute dot products of position and user clip plane vectors */
2572 for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
2573 for (const_chan = 0; const_chan < TGSI_NUM_CHANNELS; const_chan++) {
2574 LLVMValueRef addr =
2575 LLVMConstInt(ctx->i32, ((reg_index * 4 + chan) * 4 +
2576 const_chan) * 4, 0);
2577 base_elt = buffer_load_const(ctx, const_resource,
2578 addr);
2579 args->out[chan] = ac_build_fmad(&ctx->ac, base_elt,
2580 out_elts[const_chan], args->out[chan]);
2581 }
2582 }
2583
2584 args->enabled_channels = 0xf;
2585 args->valid_mask = 0;
2586 args->done = 0;
2587 args->target = V_008DFC_SQ_EXP_POS + 2 + reg_index;
2588 args->compr = 0;
2589 }
2590 }
2591
2592 static void si_dump_streamout(struct pipe_stream_output_info *so)
2593 {
2594 unsigned i;
2595
2596 if (so->num_outputs)
2597 fprintf(stderr, "STREAMOUT\n");
2598
2599 for (i = 0; i < so->num_outputs; i++) {
2600 unsigned mask = ((1 << so->output[i].num_components) - 1) <<
2601 so->output[i].start_component;
2602 fprintf(stderr, " %i: BUF%i[%i..%i] <- OUT[%i].%s%s%s%s\n",
2603 i, so->output[i].output_buffer,
2604 so->output[i].dst_offset, so->output[i].dst_offset + so->output[i].num_components - 1,
2605 so->output[i].register_index,
2606 mask & 1 ? "x" : "",
2607 mask & 2 ? "y" : "",
2608 mask & 4 ? "z" : "",
2609 mask & 8 ? "w" : "");
2610 }
2611 }
2612
2613 void si_emit_streamout_output(struct si_shader_context *ctx,
2614 LLVMValueRef const *so_buffers,
2615 LLVMValueRef const *so_write_offsets,
2616 struct pipe_stream_output *stream_out,
2617 struct si_shader_output_values *shader_out)
2618 {
2619 unsigned buf_idx = stream_out->output_buffer;
2620 unsigned start = stream_out->start_component;
2621 unsigned num_comps = stream_out->num_components;
2622 LLVMValueRef out[4];
2623
2624 assert(num_comps && num_comps <= 4);
2625 if (!num_comps || num_comps > 4)
2626 return;
2627
2628 /* Load the output as int. */
2629 for (int j = 0; j < num_comps; j++) {
2630 assert(stream_out->stream == shader_out->vertex_stream[start + j]);
2631
2632 out[j] = ac_to_integer(&ctx->ac, shader_out->values[start + j]);
2633 }
2634
2635 /* Pack the output. */
2636 LLVMValueRef vdata = NULL;
2637
2638 switch (num_comps) {
2639 case 1: /* as i32 */
2640 vdata = out[0];
2641 break;
2642 case 2: /* as v2i32 */
2643 case 3: /* as v3i32 */
2644 if (ac_has_vec3_support(ctx->screen->info.chip_class, false)) {
2645 vdata = ac_build_gather_values(&ctx->ac, out, num_comps);
2646 break;
2647 }
2648 /* as v4i32 (aligned to 4) */
2649 out[3] = LLVMGetUndef(ctx->i32);
2650 /* fall through */
2651 case 4: /* as v4i32 */
2652 vdata = ac_build_gather_values(&ctx->ac, out, util_next_power_of_two(num_comps));
2653 break;
2654 }
2655
2656 ac_build_buffer_store_dword(&ctx->ac, so_buffers[buf_idx],
2657 vdata, num_comps,
2658 so_write_offsets[buf_idx],
2659 ctx->i32_0,
2660 stream_out->dst_offset * 4, ac_glc | ac_slc);
2661 }
2662
2663 /**
2664 * Write streamout data to buffers for vertex stream @p stream (different
2665 * vertex streams can occur for GS copy shaders).
2666 */
2667 static void si_llvm_emit_streamout(struct si_shader_context *ctx,
2668 struct si_shader_output_values *outputs,
2669 unsigned noutput, unsigned stream)
2670 {
2671 struct si_shader_selector *sel = ctx->shader->selector;
2672 struct pipe_stream_output_info *so = &sel->so;
2673 LLVMBuilderRef builder = ctx->ac.builder;
2674 int i;
2675
2676 /* Get bits [22:16], i.e. (so_param >> 16) & 127; */
2677 LLVMValueRef so_vtx_count =
2678 si_unpack_param(ctx, ctx->streamout_config, 16, 7);
2679
2680 LLVMValueRef tid = ac_get_thread_id(&ctx->ac);
2681
2682 /* can_emit = tid < so_vtx_count; */
2683 LLVMValueRef can_emit =
2684 LLVMBuildICmp(builder, LLVMIntULT, tid, so_vtx_count, "");
2685
2686 /* Emit the streamout code conditionally. This actually avoids
2687 * out-of-bounds buffer access. The hw tells us via the SGPR
2688 * (so_vtx_count) which threads are allowed to emit streamout data. */
2689 ac_build_ifcc(&ctx->ac, can_emit, 6501);
2690 {
2691 /* The buffer offset is computed as follows:
2692 * ByteOffset = streamout_offset[buffer_id]*4 +
2693 * (streamout_write_index + thread_id)*stride[buffer_id] +
2694 * attrib_offset
2695 */
2696
2697 LLVMValueRef so_write_index =
2698 ac_get_arg(&ctx->ac,
2699 ctx->streamout_write_index);
2700
2701 /* Compute (streamout_write_index + thread_id). */
2702 so_write_index = LLVMBuildAdd(builder, so_write_index, tid, "");
2703
2704 /* Load the descriptor and compute the write offset for each
2705 * enabled buffer. */
2706 LLVMValueRef so_write_offset[4] = {};
2707 LLVMValueRef so_buffers[4];
2708 LLVMValueRef buf_ptr = ac_get_arg(&ctx->ac,
2709 ctx->rw_buffers);
2710
2711 for (i = 0; i < 4; i++) {
2712 if (!so->stride[i])
2713 continue;
2714
2715 LLVMValueRef offset = LLVMConstInt(ctx->i32,
2716 SI_VS_STREAMOUT_BUF0 + i, 0);
2717
2718 so_buffers[i] = ac_build_load_to_sgpr(&ctx->ac, buf_ptr, offset);
2719
2720 LLVMValueRef so_offset = ac_get_arg(&ctx->ac,
2721 ctx->streamout_offset[i]);
2722 so_offset = LLVMBuildMul(builder, so_offset, LLVMConstInt(ctx->i32, 4, 0), "");
2723
2724 so_write_offset[i] = ac_build_imad(&ctx->ac, so_write_index,
2725 LLVMConstInt(ctx->i32, so->stride[i]*4, 0),
2726 so_offset);
2727 }
2728
2729 /* Write streamout data. */
2730 for (i = 0; i < so->num_outputs; i++) {
2731 unsigned reg = so->output[i].register_index;
2732
2733 if (reg >= noutput)
2734 continue;
2735
2736 if (stream != so->output[i].stream)
2737 continue;
2738
2739 si_emit_streamout_output(ctx, so_buffers, so_write_offset,
2740 &so->output[i], &outputs[reg]);
2741 }
2742 }
2743 ac_build_endif(&ctx->ac, 6501);
2744 }
2745
2746 static void si_export_param(struct si_shader_context *ctx, unsigned index,
2747 LLVMValueRef *values)
2748 {
2749 struct ac_export_args args;
2750
2751 si_llvm_init_export_args(ctx, values,
2752 V_008DFC_SQ_EXP_PARAM + index, &args);
2753 ac_build_export(&ctx->ac, &args);
2754 }
2755
2756 static void si_build_param_exports(struct si_shader_context *ctx,
2757 struct si_shader_output_values *outputs,
2758 unsigned noutput)
2759 {
2760 struct si_shader *shader = ctx->shader;
2761 unsigned param_count = 0;
2762
2763 for (unsigned i = 0; i < noutput; i++) {
2764 unsigned semantic_name = outputs[i].semantic_name;
2765 unsigned semantic_index = outputs[i].semantic_index;
2766
2767 if (outputs[i].vertex_stream[0] != 0 &&
2768 outputs[i].vertex_stream[1] != 0 &&
2769 outputs[i].vertex_stream[2] != 0 &&
2770 outputs[i].vertex_stream[3] != 0)
2771 continue;
2772
2773 switch (semantic_name) {
2774 case TGSI_SEMANTIC_LAYER:
2775 case TGSI_SEMANTIC_VIEWPORT_INDEX:
2776 case TGSI_SEMANTIC_CLIPDIST:
2777 case TGSI_SEMANTIC_COLOR:
2778 case TGSI_SEMANTIC_BCOLOR:
2779 case TGSI_SEMANTIC_PRIMID:
2780 case TGSI_SEMANTIC_FOG:
2781 case TGSI_SEMANTIC_TEXCOORD:
2782 case TGSI_SEMANTIC_GENERIC:
2783 break;
2784 default:
2785 continue;
2786 }
2787
2788 if ((semantic_name != TGSI_SEMANTIC_GENERIC ||
2789 semantic_index < SI_MAX_IO_GENERIC) &&
2790 shader->key.opt.kill_outputs &
2791 (1ull << si_shader_io_get_unique_index(semantic_name,
2792 semantic_index, true)))
2793 continue;
2794
2795 si_export_param(ctx, param_count, outputs[i].values);
2796
2797 assert(i < ARRAY_SIZE(shader->info.vs_output_param_offset));
2798 shader->info.vs_output_param_offset[i] = param_count++;
2799 }
2800
2801 shader->info.nr_param_exports = param_count;
2802 }
2803
2804 /**
2805 * Vertex color clamping.
2806 *
2807 * This uses a state constant loaded in a user data SGPR and
2808 * an IF statement is added that clamps all colors if the constant
2809 * is true.
2810 */
2811 static void si_vertex_color_clamping(struct si_shader_context *ctx,
2812 struct si_shader_output_values *outputs,
2813 unsigned noutput)
2814 {
2815 LLVMValueRef addr[SI_MAX_VS_OUTPUTS][4];
2816 bool has_colors = false;
2817
2818 /* Store original colors to alloca variables. */
2819 for (unsigned i = 0; i < noutput; i++) {
2820 if (outputs[i].semantic_name != TGSI_SEMANTIC_COLOR &&
2821 outputs[i].semantic_name != TGSI_SEMANTIC_BCOLOR)
2822 continue;
2823
2824 for (unsigned j = 0; j < 4; j++) {
2825 addr[i][j] = ac_build_alloca_undef(&ctx->ac, ctx->f32, "");
2826 LLVMBuildStore(ctx->ac.builder, outputs[i].values[j], addr[i][j]);
2827 }
2828 has_colors = true;
2829 }
2830
2831 if (!has_colors)
2832 return;
2833
2834 /* The state is in the first bit of the user SGPR. */
2835 LLVMValueRef cond = ac_get_arg(&ctx->ac, ctx->vs_state_bits);
2836 cond = LLVMBuildTrunc(ctx->ac.builder, cond, ctx->i1, "");
2837
2838 ac_build_ifcc(&ctx->ac, cond, 6502);
2839
2840 /* Store clamped colors to alloca variables within the conditional block. */
2841 for (unsigned i = 0; i < noutput; i++) {
2842 if (outputs[i].semantic_name != TGSI_SEMANTIC_COLOR &&
2843 outputs[i].semantic_name != TGSI_SEMANTIC_BCOLOR)
2844 continue;
2845
2846 for (unsigned j = 0; j < 4; j++) {
2847 LLVMBuildStore(ctx->ac.builder,
2848 ac_build_clamp(&ctx->ac, outputs[i].values[j]),
2849 addr[i][j]);
2850 }
2851 }
2852 ac_build_endif(&ctx->ac, 6502);
2853
2854 /* Load clamped colors */
2855 for (unsigned i = 0; i < noutput; i++) {
2856 if (outputs[i].semantic_name != TGSI_SEMANTIC_COLOR &&
2857 outputs[i].semantic_name != TGSI_SEMANTIC_BCOLOR)
2858 continue;
2859
2860 for (unsigned j = 0; j < 4; j++) {
2861 outputs[i].values[j] =
2862 LLVMBuildLoad(ctx->ac.builder, addr[i][j], "");
2863 }
2864 }
2865 }
2866
2867 /* Generate export instructions for hardware VS shader stage or NGG GS stage
2868 * (position and parameter data only).
2869 */
2870 void si_llvm_export_vs(struct si_shader_context *ctx,
2871 struct si_shader_output_values *outputs,
2872 unsigned noutput)
2873 {
2874 struct si_shader *shader = ctx->shader;
2875 struct ac_export_args pos_args[4] = {};
2876 LLVMValueRef psize_value = NULL, edgeflag_value = NULL, layer_value = NULL, viewport_index_value = NULL;
2877 unsigned pos_idx;
2878 int i;
2879
2880 si_vertex_color_clamping(ctx, outputs, noutput);
2881
2882 /* Build position exports. */
2883 for (i = 0; i < noutput; i++) {
2884 switch (outputs[i].semantic_name) {
2885 case TGSI_SEMANTIC_POSITION:
2886 si_llvm_init_export_args(ctx, outputs[i].values,
2887 V_008DFC_SQ_EXP_POS, &pos_args[0]);
2888 break;
2889 case TGSI_SEMANTIC_PSIZE:
2890 psize_value = outputs[i].values[0];
2891 break;
2892 case TGSI_SEMANTIC_LAYER:
2893 layer_value = outputs[i].values[0];
2894 break;
2895 case TGSI_SEMANTIC_VIEWPORT_INDEX:
2896 viewport_index_value = outputs[i].values[0];
2897 break;
2898 case TGSI_SEMANTIC_EDGEFLAG:
2899 edgeflag_value = outputs[i].values[0];
2900 break;
2901 case TGSI_SEMANTIC_CLIPDIST:
2902 if (!shader->key.opt.clip_disable) {
2903 unsigned index = 2 + outputs[i].semantic_index;
2904 si_llvm_init_export_args(ctx, outputs[i].values,
2905 V_008DFC_SQ_EXP_POS + index,
2906 &pos_args[index]);
2907 }
2908 break;
2909 case TGSI_SEMANTIC_CLIPVERTEX:
2910 if (!shader->key.opt.clip_disable) {
2911 si_llvm_emit_clipvertex(ctx, pos_args,
2912 outputs[i].values);
2913 }
2914 break;
2915 }
2916 }
2917
2918 /* We need to add the position output manually if it's missing. */
2919 if (!pos_args[0].out[0]) {
2920 pos_args[0].enabled_channels = 0xf; /* writemask */
2921 pos_args[0].valid_mask = 0; /* EXEC mask */
2922 pos_args[0].done = 0; /* last export? */
2923 pos_args[0].target = V_008DFC_SQ_EXP_POS;
2924 pos_args[0].compr = 0; /* COMPR flag */
2925 pos_args[0].out[0] = ctx->ac.f32_0; /* X */
2926 pos_args[0].out[1] = ctx->ac.f32_0; /* Y */
2927 pos_args[0].out[2] = ctx->ac.f32_0; /* Z */
2928 pos_args[0].out[3] = ctx->ac.f32_1; /* W */
2929 }
2930
2931 bool pos_writes_edgeflag = shader->selector->info.writes_edgeflag &&
2932 !shader->key.as_ngg;
2933
2934 /* Write the misc vector (point size, edgeflag, layer, viewport). */
2935 if (shader->selector->info.writes_psize ||
2936 pos_writes_edgeflag ||
2937 shader->selector->info.writes_viewport_index ||
2938 shader->selector->info.writes_layer) {
2939 pos_args[1].enabled_channels = shader->selector->info.writes_psize |
2940 (pos_writes_edgeflag << 1) |
2941 (shader->selector->info.writes_layer << 2);
2942
2943 pos_args[1].valid_mask = 0; /* EXEC mask */
2944 pos_args[1].done = 0; /* last export? */
2945 pos_args[1].target = V_008DFC_SQ_EXP_POS + 1;
2946 pos_args[1].compr = 0; /* COMPR flag */
2947 pos_args[1].out[0] = ctx->ac.f32_0; /* X */
2948 pos_args[1].out[1] = ctx->ac.f32_0; /* Y */
2949 pos_args[1].out[2] = ctx->ac.f32_0; /* Z */
2950 pos_args[1].out[3] = ctx->ac.f32_0; /* W */
2951
2952 if (shader->selector->info.writes_psize)
2953 pos_args[1].out[0] = psize_value;
2954
2955 if (pos_writes_edgeflag) {
2956 /* The output is a float, but the hw expects an integer
2957 * with the first bit containing the edge flag. */
2958 edgeflag_value = LLVMBuildFPToUI(ctx->ac.builder,
2959 edgeflag_value,
2960 ctx->i32, "");
2961 edgeflag_value = ac_build_umin(&ctx->ac,
2962 edgeflag_value,
2963 ctx->i32_1);
2964
2965 /* The LLVM intrinsic expects a float. */
2966 pos_args[1].out[1] = ac_to_float(&ctx->ac, edgeflag_value);
2967 }
2968
2969 if (ctx->screen->info.chip_class >= GFX9) {
2970 /* GFX9 has the layer in out.z[10:0] and the viewport
2971 * index in out.z[19:16].
2972 */
2973 if (shader->selector->info.writes_layer)
2974 pos_args[1].out[2] = layer_value;
2975
2976 if (shader->selector->info.writes_viewport_index) {
2977 LLVMValueRef v = viewport_index_value;
2978
2979 v = ac_to_integer(&ctx->ac, v);
2980 v = LLVMBuildShl(ctx->ac.builder, v,
2981 LLVMConstInt(ctx->i32, 16, 0), "");
2982 v = LLVMBuildOr(ctx->ac.builder, v,
2983 ac_to_integer(&ctx->ac, pos_args[1].out[2]), "");
2984 pos_args[1].out[2] = ac_to_float(&ctx->ac, v);
2985 pos_args[1].enabled_channels |= 1 << 2;
2986 }
2987 } else {
2988 if (shader->selector->info.writes_layer)
2989 pos_args[1].out[2] = layer_value;
2990
2991 if (shader->selector->info.writes_viewport_index) {
2992 pos_args[1].out[3] = viewport_index_value;
2993 pos_args[1].enabled_channels |= 1 << 3;
2994 }
2995 }
2996 }
2997
2998 for (i = 0; i < 4; i++)
2999 if (pos_args[i].out[0])
3000 shader->info.nr_pos_exports++;
3001
3002 /* Navi10-14 skip POS0 exports if EXEC=0 and DONE=0, causing a hang.
3003 * Setting valid_mask=1 prevents it and has no other effect.
3004 */
3005 if (ctx->screen->info.family == CHIP_NAVI10 ||
3006 ctx->screen->info.family == CHIP_NAVI12 ||
3007 ctx->screen->info.family == CHIP_NAVI14)
3008 pos_args[0].valid_mask = 1;
3009
3010 pos_idx = 0;
3011 for (i = 0; i < 4; i++) {
3012 if (!pos_args[i].out[0])
3013 continue;
3014
3015 /* Specify the target we are exporting */
3016 pos_args[i].target = V_008DFC_SQ_EXP_POS + pos_idx++;
3017
3018 if (pos_idx == shader->info.nr_pos_exports)
3019 /* Specify that this is the last export */
3020 pos_args[i].done = 1;
3021
3022 ac_build_export(&ctx->ac, &pos_args[i]);
3023 }
3024
3025 /* Build parameter exports. */
3026 si_build_param_exports(ctx, outputs, noutput);
3027 }
3028
3029 /**
3030 * Forward all outputs from the vertex shader to the TES. This is only used
3031 * for the fixed function TCS.
3032 */
3033 static void si_copy_tcs_inputs(struct lp_build_tgsi_context *bld_base)
3034 {
3035 struct si_shader_context *ctx = si_shader_context(bld_base);
3036 LLVMValueRef invocation_id, buffer, buffer_offset;
3037 LLVMValueRef lds_vertex_stride, lds_base;
3038 uint64_t inputs;
3039
3040 invocation_id = si_unpack_param(ctx, ctx->args.tcs_rel_ids, 8, 5);
3041 buffer = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TCS);
3042 buffer_offset = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
3043
3044 lds_vertex_stride = get_tcs_in_vertex_dw_stride(ctx);
3045 lds_base = get_tcs_in_current_patch_offset(ctx);
3046 lds_base = ac_build_imad(&ctx->ac, invocation_id, lds_vertex_stride,
3047 lds_base);
3048
3049 inputs = ctx->shader->key.mono.u.ff_tcs_inputs_to_copy;
3050 while (inputs) {
3051 unsigned i = u_bit_scan64(&inputs);
3052
3053 LLVMValueRef lds_ptr = LLVMBuildAdd(ctx->ac.builder, lds_base,
3054 LLVMConstInt(ctx->i32, 4 * i, 0),
3055 "");
3056
3057 LLVMValueRef buffer_addr = get_tcs_tes_buffer_address(ctx,
3058 get_rel_patch_id(ctx),
3059 invocation_id,
3060 LLVMConstInt(ctx->i32, i, 0));
3061
3062 LLVMValueRef value = lshs_lds_load(bld_base, ctx->ac.i32, ~0, lds_ptr);
3063
3064 ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, buffer_addr,
3065 buffer_offset, 0, ac_glc);
3066 }
3067 }
3068
3069 static void si_write_tess_factors(struct lp_build_tgsi_context *bld_base,
3070 LLVMValueRef rel_patch_id,
3071 LLVMValueRef invocation_id,
3072 LLVMValueRef tcs_out_current_patch_data_offset,
3073 LLVMValueRef invoc0_tf_outer[4],
3074 LLVMValueRef invoc0_tf_inner[2])
3075 {
3076 struct si_shader_context *ctx = si_shader_context(bld_base);
3077 struct si_shader *shader = ctx->shader;
3078 unsigned tess_inner_index, tess_outer_index;
3079 LLVMValueRef lds_base, lds_inner, lds_outer, byteoffset, buffer;
3080 LLVMValueRef out[6], vec0, vec1, tf_base, inner[4], outer[4];
3081 unsigned stride, outer_comps, inner_comps, i, offset;
3082
3083 /* Add a barrier before loading tess factors from LDS. */
3084 if (!shader->key.part.tcs.epilog.invoc0_tess_factors_are_def)
3085 si_llvm_emit_barrier(NULL, bld_base, NULL);
3086
3087 /* Do this only for invocation 0, because the tess levels are per-patch,
3088 * not per-vertex.
3089 *
3090 * This can't jump, because invocation 0 executes this. It should
3091 * at least mask out the loads and stores for other invocations.
3092 */
3093 ac_build_ifcc(&ctx->ac,
3094 LLVMBuildICmp(ctx->ac.builder, LLVMIntEQ,
3095 invocation_id, ctx->i32_0, ""), 6503);
3096
3097 /* Determine the layout of one tess factor element in the buffer. */
3098 switch (shader->key.part.tcs.epilog.prim_mode) {
3099 case PIPE_PRIM_LINES:
3100 stride = 2; /* 2 dwords, 1 vec2 store */
3101 outer_comps = 2;
3102 inner_comps = 0;
3103 break;
3104 case PIPE_PRIM_TRIANGLES:
3105 stride = 4; /* 4 dwords, 1 vec4 store */
3106 outer_comps = 3;
3107 inner_comps = 1;
3108 break;
3109 case PIPE_PRIM_QUADS:
3110 stride = 6; /* 6 dwords, 2 stores (vec4 + vec2) */
3111 outer_comps = 4;
3112 inner_comps = 2;
3113 break;
3114 default:
3115 assert(0);
3116 return;
3117 }
3118
3119 for (i = 0; i < 4; i++) {
3120 inner[i] = LLVMGetUndef(ctx->i32);
3121 outer[i] = LLVMGetUndef(ctx->i32);
3122 }
3123
3124 if (shader->key.part.tcs.epilog.invoc0_tess_factors_are_def) {
3125 /* Tess factors are in VGPRs. */
3126 for (i = 0; i < outer_comps; i++)
3127 outer[i] = out[i] = invoc0_tf_outer[i];
3128 for (i = 0; i < inner_comps; i++)
3129 inner[i] = out[outer_comps+i] = invoc0_tf_inner[i];
3130 } else {
3131 /* Load tess_inner and tess_outer from LDS.
3132 * Any invocation can write them, so we can't get them from a temporary.
3133 */
3134 tess_inner_index = si_shader_io_get_unique_index_patch(TGSI_SEMANTIC_TESSINNER, 0);
3135 tess_outer_index = si_shader_io_get_unique_index_patch(TGSI_SEMANTIC_TESSOUTER, 0);
3136
3137 lds_base = tcs_out_current_patch_data_offset;
3138 lds_inner = LLVMBuildAdd(ctx->ac.builder, lds_base,
3139 LLVMConstInt(ctx->i32,
3140 tess_inner_index * 4, 0), "");
3141 lds_outer = LLVMBuildAdd(ctx->ac.builder, lds_base,
3142 LLVMConstInt(ctx->i32,
3143 tess_outer_index * 4, 0), "");
3144
3145 for (i = 0; i < outer_comps; i++) {
3146 outer[i] = out[i] =
3147 lshs_lds_load(bld_base, ctx->ac.i32, i, lds_outer);
3148 }
3149 for (i = 0; i < inner_comps; i++) {
3150 inner[i] = out[outer_comps+i] =
3151 lshs_lds_load(bld_base, ctx->ac.i32, i, lds_inner);
3152 }
3153 }
3154
3155 if (shader->key.part.tcs.epilog.prim_mode == PIPE_PRIM_LINES) {
3156 /* For isolines, the hardware expects tess factors in the
3157 * reverse order from what GLSL / TGSI specify.
3158 */
3159 LLVMValueRef tmp = out[0];
3160 out[0] = out[1];
3161 out[1] = tmp;
3162 }
3163
3164 /* Convert the outputs to vectors for stores. */
3165 vec0 = ac_build_gather_values(&ctx->ac, out, MIN2(stride, 4));
3166 vec1 = NULL;
3167
3168 if (stride > 4)
3169 vec1 = ac_build_gather_values(&ctx->ac, out+4, stride - 4);
3170
3171 /* Get the buffer. */
3172 buffer = get_tess_ring_descriptor(ctx, TCS_FACTOR_RING);
3173
3174 /* Get the offset. */
3175 tf_base = ac_get_arg(&ctx->ac,
3176 ctx->tcs_factor_offset);
3177 byteoffset = LLVMBuildMul(ctx->ac.builder, rel_patch_id,
3178 LLVMConstInt(ctx->i32, 4 * stride, 0), "");
3179
3180 ac_build_ifcc(&ctx->ac,
3181 LLVMBuildICmp(ctx->ac.builder, LLVMIntEQ,
3182 rel_patch_id, ctx->i32_0, ""), 6504);
3183
3184 /* Store the dynamic HS control word. */
3185 offset = 0;
3186 if (ctx->screen->info.chip_class <= GFX8) {
3187 ac_build_buffer_store_dword(&ctx->ac, buffer,
3188 LLVMConstInt(ctx->i32, 0x80000000, 0),
3189 1, ctx->i32_0, tf_base,
3190 offset, ac_glc);
3191 offset += 4;
3192 }
3193
3194 ac_build_endif(&ctx->ac, 6504);
3195
3196 /* Store the tessellation factors. */
3197 ac_build_buffer_store_dword(&ctx->ac, buffer, vec0,
3198 MIN2(stride, 4), byteoffset, tf_base,
3199 offset, ac_glc);
3200 offset += 16;
3201 if (vec1)
3202 ac_build_buffer_store_dword(&ctx->ac, buffer, vec1,
3203 stride - 4, byteoffset, tf_base,
3204 offset, ac_glc);
3205
3206 /* Store the tess factors into the offchip buffer if TES reads them. */
3207 if (shader->key.part.tcs.epilog.tes_reads_tess_factors) {
3208 LLVMValueRef buf, base, inner_vec, outer_vec, tf_outer_offset;
3209 LLVMValueRef tf_inner_offset;
3210 unsigned param_outer, param_inner;
3211
3212 buf = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TCS);
3213 base = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
3214
3215 param_outer = si_shader_io_get_unique_index_patch(
3216 TGSI_SEMANTIC_TESSOUTER, 0);
3217 tf_outer_offset = get_tcs_tes_buffer_address(ctx, rel_patch_id, NULL,
3218 LLVMConstInt(ctx->i32, param_outer, 0));
3219
3220 unsigned outer_vec_size =
3221 ac_has_vec3_support(ctx->screen->info.chip_class, false) ?
3222 outer_comps : util_next_power_of_two(outer_comps);
3223 outer_vec = ac_build_gather_values(&ctx->ac, outer, outer_vec_size);
3224
3225 ac_build_buffer_store_dword(&ctx->ac, buf, outer_vec,
3226 outer_comps, tf_outer_offset,
3227 base, 0, ac_glc);
3228 if (inner_comps) {
3229 param_inner = si_shader_io_get_unique_index_patch(
3230 TGSI_SEMANTIC_TESSINNER, 0);
3231 tf_inner_offset = get_tcs_tes_buffer_address(ctx, rel_patch_id, NULL,
3232 LLVMConstInt(ctx->i32, param_inner, 0));
3233
3234 inner_vec = inner_comps == 1 ? inner[0] :
3235 ac_build_gather_values(&ctx->ac, inner, inner_comps);
3236 ac_build_buffer_store_dword(&ctx->ac, buf, inner_vec,
3237 inner_comps, tf_inner_offset,
3238 base, 0, ac_glc);
3239 }
3240 }
3241
3242 ac_build_endif(&ctx->ac, 6503);
3243 }
3244
3245 static LLVMValueRef
3246 si_insert_input_ret(struct si_shader_context *ctx, LLVMValueRef ret,
3247 struct ac_arg param, unsigned return_index)
3248 {
3249 return LLVMBuildInsertValue(ctx->ac.builder, ret,
3250 ac_get_arg(&ctx->ac, param),
3251 return_index, "");
3252 }
3253
3254 static LLVMValueRef
3255 si_insert_input_ret_float(struct si_shader_context *ctx, LLVMValueRef ret,
3256 struct ac_arg param, unsigned return_index)
3257 {
3258 LLVMBuilderRef builder = ctx->ac.builder;
3259 LLVMValueRef p = ac_get_arg(&ctx->ac, param);
3260
3261 return LLVMBuildInsertValue(builder, ret,
3262 ac_to_float(&ctx->ac, p),
3263 return_index, "");
3264 }
3265
3266 static LLVMValueRef
3267 si_insert_input_ptr(struct si_shader_context *ctx, LLVMValueRef ret,
3268 struct ac_arg param, unsigned return_index)
3269 {
3270 LLVMBuilderRef builder = ctx->ac.builder;
3271 LLVMValueRef ptr = ac_get_arg(&ctx->ac, param);
3272 ptr = LLVMBuildPtrToInt(builder, ptr, ctx->i32, "");
3273 return LLVMBuildInsertValue(builder, ret, ptr, return_index, "");
3274 }
3275
3276 /* This only writes the tessellation factor levels. */
3277 static void si_llvm_emit_tcs_epilogue(struct ac_shader_abi *abi,
3278 unsigned max_outputs,
3279 LLVMValueRef *addrs)
3280 {
3281 struct si_shader_context *ctx = si_shader_context_from_abi(abi);
3282 struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
3283 LLVMBuilderRef builder = ctx->ac.builder;
3284 LLVMValueRef rel_patch_id, invocation_id, tf_lds_offset;
3285
3286 si_copy_tcs_inputs(bld_base);
3287
3288 rel_patch_id = get_rel_patch_id(ctx);
3289 invocation_id = si_unpack_param(ctx, ctx->args.tcs_rel_ids, 8, 5);
3290 tf_lds_offset = get_tcs_out_current_patch_data_offset(ctx);
3291
3292 if (ctx->screen->info.chip_class >= GFX9) {
3293 LLVMBasicBlockRef blocks[2] = {
3294 LLVMGetInsertBlock(builder),
3295 ctx->merged_wrap_if_entry_block
3296 };
3297 LLVMValueRef values[2